Package org.voltdb.catalog

Examples of org.voltdb.catalog.Statement


        // Update EstimatorState.prefetch any time we transition to a MarkovVertex where the
        // underlying Statement catalog object was marked as prefetchable
        // Do we want to put this traversal above?
        if (hstore_conf.site.exec_prefetch_queries) {
            for (MarkovVertex vertex : initialEst.getMarkovPath()) {
                Statement statement = (Statement) vertex.getCatalogItem();
                if (statement.getPrefetchable()) {
                    if (debug.val)
                        LOG.debug(String.format("%s - Checking whether we can prefetch %s on partitions %s",
                                 TransactionUtil.formatTxnName(catalog_proc, txn_id),
                                 statement.fullName(), vertex.getPartitions()));
                    if (vertex.getPartitions().isEmpty() == false && vertex.getPartitions().get() != base_partition) {
                        state.addPrefetchableStatement(vertex.getCountedStatement());
                    }
                }
            } // FOR
View Full Code Here


           
        MarkovVertex current = state.getCurrent();
        PartitionSet touchedPartitions = state.getTouchedPartitions();
        MarkovVertex next_v = null;
        MarkovEdge next_e = null;
        Statement last_stmt = null;
        int stmt_idxs[] = null;

        // We can cache what the path is based on the first and last query in the batch
        // We only want to do this for batches that are large enough.
        if (hstore_conf.site.markov_endpoint_caching &&
View Full Code Here

        for (Entry<Integer, List<QueryTrace>> e : txn_trace.getBatches().entrySet()) {
            int batch_size = e.getValue().size();
            if (trace.val) LOG.trace(String.format("Batch #%d: %d traces", e.getKey(), batch_size));
           
            // Generate the data structures we will need to give to the TransactionEstimator
            Statement catalog_stmts[] = new Statement[batch_size];
            PartitionSet partitions[] = new PartitionSet[batch_size];
            this.populateQueryBatch(e.getValue(), s.getBasePartition(), catalog_stmts, partitions);
       
            synchronized (s.getMarkovGraph()) {
                this.executeQueries(s, catalog_stmts, partitions);
View Full Code Here

    public List<CountedStatement> getQueryEstimate(int partition) {
        if (this.countedStmts[partition] == null) {
            this.countedStmts[partition] = new ArrayList<CountedStatement>();
            QueryEstimate est = this.query_estimates[partition];
            for (int i = 0, cnt = est.getStmtIdsCount(); i < cnt; i++) {
                Statement catalog_stmt = this.catalogContext.getStatementById(est.getStmtIds(i));
                assert(catalog_stmt != null) : "Invalid Statement id '" + est.getStmtIds(i) + "'";
                this.countedStmts[partition].add(new CountedStatement(catalog_stmt, est.getStmtCounters(i)));
            } // FOR
        }
        return (this.countedStmts[partition]);
View Full Code Here

        this.candidate_edges.clear();
        this.next_statements.clear();
        this.past_partitions.addAll(element.getPartitions());
       
        if (trace.val) LOG.trace("Current Vertex: " + element);
        Statement cur_catalog_stmt = element.getCatalogItem();
        int cur_catalog_stmt_index = element.getQueryCounter();
        MarkovGraph markov = (MarkovGraph)this.getGraph();
       
        // At our current vertex we need to gather all of our neighbors
        // and get unique Statements that we could be executing next
        Collection<MarkovVertex> next_vertices = markov.getSuccessors(element);
        if (next_vertices == null || next_vertices.isEmpty()) {
            if (debug.val) LOG.debug("No succesors were found for " + element + ". Halting traversal");
            return;
        }
        if (trace.val) LOG.trace("Successors: " + next_vertices);
       
        // Step #1
        // Get all of the unique Statement+StatementInstanceIndex pairs for the vertices
        // that are adjacent to our current vertex
        for (MarkovVertex next : next_vertices) {
            Statement next_catalog_stmt = next.getCatalogItem();
            int next_catalog_stmt_index = next.getQueryCounter();
           
            // Sanity Check: If this vertex is the same Statement as the current vertex,
            // then its instance counter must be greater than the current vertex's counter
            if (next_catalog_stmt.equals(cur_catalog_stmt)) {
                if (next_catalog_stmt_index <= cur_catalog_stmt_index) {
                    LOG.error("CURRENT: " + element + " [commit=" + element.isCommitVertex() + "]");
                    LOG.error("NEXT: " + next + " [commit=" + next.isCommitVertex() + "]");
                }
                assert(next_catalog_stmt_index > cur_catalog_stmt_index) :
                    String.format("%s[#%d] > %s[#%d]",
                                  next_catalog_stmt.fullName(), next_catalog_stmt_index,
                                  cur_catalog_stmt.fullName(), cur_catalog_stmt_index);
            }
           
            // Check whether it's COMMIT/ABORT
            if (next.isCommitVertex() || next.isAbortVertex()) {
                MarkovEdge candidate = markov.findEdge(element, next);
                assert(candidate != null);
                this.candidate_edges.add(candidate);
            } else {
                this.next_statements.add(next.getCountedStatement());
            }
        } // FOR

        // Now for the unique set of Statement+StatementIndex pairs, figure out which partitions
        // the queries will go to.
        MarkovEdge candidate_edge;
        for (CountedStatement cstmt : this.next_statements) {
            Statement catalog_stmt = cstmt.statement;
            Integer catalog_stmt_index = cstmt.counter;
            if (debug.val) LOG.debug("Examining " + cstmt);
           
            // Get the mapping objects (if any) for next
            // This is the only way we can predict what partitions we will touch
            Map<StmtParameter, SortedSet<ParameterMapping>> stmtMappings = this.allMappings.get(catalog_stmt, catalog_stmt_index);
            if (stmtMappings == null) {
                if (debug.val) {
                    LOG.warn("No parameter mappings for " + catalog_stmt);
                    if (trace.val) LOG.trace(this.allMappings.debug(catalog_stmt));
                }
                continue;
            }
           
            // Go through the StmtParameters and map values from ProcParameters
            StmtParameter stmt_params[] = catalog_stmt.getParameters().values();
            Object stmt_args[] = new Object[stmt_params.length]; // this.getStatementParamsArray(catalog_stmt);
            boolean stmt_args_set = false;
           
            // XXX: This method may return null because it's being used for other
            // purposes in the BatchPlanner.
            int stmt_args_offsets[] = this.p_estimator.getStatementEstimationParameters(catalog_stmt);
            if (stmt_args_offsets == null) {
                stmt_args_offsets = new int[stmt_args.length];
                for (int i = 0; i < stmt_args.length; i++)
                    stmt_args_offsets[i] = i;
            }
            assert(stmt_args_offsets != null) :
                "Unexpected null StmtParameter offsets for " + catalog_stmt.fullName();
            for (int offset : stmt_args_offsets) {
                StmtParameter catalog_stmt_param = stmt_params[offset];
                assert(catalog_stmt_param != null);
                if (trace.val)
                    LOG.trace("Retrieving ParameterMappings for " + catalog_stmt_param.fullName());
               
                Collection<ParameterMapping> mappings = stmtMappings.get(catalog_stmt_param);
                if (mappings == null || mappings.isEmpty()) {
                    if (trace.val)
                        LOG.trace("No parameter mappings exists for " + catalog_stmt_param.fullName());
                    continue;
                }
                if (debug.val)
                    LOG.debug("Found " + mappings.size() + " mapping(s) for " + catalog_stmt_param.fullName());
       
                // Special Case:
                // If the number of possible Statements we could execute next is greater than one,
                // then we need to prune our list by removing those Statements who have a StmtParameter
                // that are correlated to a ProcParameter that doesn't exist (such as referencing an
                // array element that is greater than the size of that current array)
                // TODO: For now we are just going always pick the first mapping
                // that comes back. Is there any choice that we would need to make in order
                // to have a better prediction about what the transaction might do?
                if (debug.val && mappings.size() > 1) {
                    LOG.warn("Multiple parameter mappings for " + catalog_stmt_param.fullName());
                    if (trace.val) {
                        int ctr = 0;
                        for (ParameterMapping m : mappings) {
                            LOG.trace("[" + (ctr++) + "] Mapping: " + m);
                        } // FOR
                    }
                }
                for (ParameterMapping m : mappings) {
                    if (trace.val) LOG.trace("Mapping: " + m);
                    ProcParameter catalog_proc_param = m.getProcParameter();
                    if (catalog_proc_param.getIsarray()) {
                        Object proc_inner_args[] = (Object[])procParams[m.getProcParameter().getIndex()];
                        if (trace.val)
                            LOG.trace(CatalogUtil.getDisplayName(m.getProcParameter(), true) + " is an array: " +
                                      Arrays.toString(proc_inner_args));
                       
                        // TODO: If this Mapping references an array element that is not available for this
                        // current transaction, should we just skip this mapping or skip the entire query?
                        if (proc_inner_args.length <= m.getProcParameterIndex()) {
                            if (trace.val)
                                LOG.trace("Unable to map parameters: " +
                                          "proc_inner_args.length[" + proc_inner_args.length + "] <= " +
                                          "c.getProcParameterIndex[" + m.getProcParameterIndex() + "]");
                            continue;
                        }
                        stmt_args[offset] = proc_inner_args[m.getProcParameterIndex()];
                        stmt_args_set = true;
                        if (trace.val)
                            LOG.trace("Mapped " + CatalogUtil.getDisplayName(m.getProcParameter()) + "[" + m.getProcParameterIndex() + "] to " +
                                      CatalogUtil.getDisplayName(catalog_stmt_param) + " [value=" + stmt_args[offset] + "]");
                    } else {
                        stmt_args[offset] = procParams[m.getProcParameter().getIndex()];
                        stmt_args_set = true;
                        if (trace.val)
                            LOG.trace("Mapped " + CatalogUtil.getDisplayName(m.getProcParameter()) + " to " +
                                      CatalogUtil.getDisplayName(catalog_stmt_param) + " [value=" + stmt_args[offset] + "]");
                    }
                    break;
                } // FOR (Mapping)
            } // FOR (StmtParameter)
               
            // If we set any of the stmt_args in the previous step, then we can throw it
            // to our good old friend the PartitionEstimator and see whether we can figure
            // things out for this Statement
            if (stmt_args_set) {
                if (trace.val)
                    LOG.trace("Mapped StmtParameters: " + Arrays.toString(stmt_args));
                this.stmt_partitions.clear();
                try {
                    this.p_estimator.getAllPartitions(this.stmt_partitions, catalog_stmt, stmt_args, this.base_partition);
                } catch (Exception ex) {
                    String msg = "Failed to calculate partitions for " + catalog_stmt + " using parameters " + Arrays.toString(stmt_args);
                    LOG.error(msg, ex);
                    this.stop();
                    return;
                }
                if (trace.val)
                    LOG.trace("Estimated Partitions for " + catalog_stmt + ": " + this.stmt_partitions);
               
                // Now for this given list of partitions, find a Vertex in our next set
                // that has the same partitions
                if (this.stmt_partitions.isEmpty() == false) {
                    candidate_edge = null;
                    if (trace.val)
                        LOG.trace("Partitions:" + this.stmt_partitions + " / Past:" + this.past_partitions);
                    for (MarkovVertex next_v : next_vertices) {
                        if (trace.val) LOG.trace("Checking whether " + next_v + " is the correct transition");
                        if (next_v.isEqual(catalog_stmt, this.stmt_partitions, this.past_partitions, catalog_stmt_index, true)) {
                            // BINGO!!!
                            assert(candidate_edge == null);
                            try {
                                candidate_edge = markov.findEdge(element, next_v);
                            } catch (NullPointerException ex) {
                                continue;
                            }
                            assert(candidate_edge != null);
                            this.candidate_edges.add(candidate_edge);
                            if (trace.val)
                                LOG.trace("Found candidate edge to " + next_v + " [" + candidate_edge + "]");
                            break;
                        } else if (trace.val) {
                            Map<String, Object> m = new LinkedHashMap<String, Object>();
                            m.put("stmt", next_v.getCatalogItem().equals(catalog_stmt));
                            m.put("stmtCtr", next_v.getQueryCounter() == catalog_stmt_index);
                            m.put("partitions", next_v.getPartitions().equals(this.stmt_partitions));
                            m.put("past", next_v.getPastPartitions().equals(this.past_partitions));
                            LOG.trace("Invalid candidate transition:\n" + StringUtil.formatMaps(m));
                        }
                    } // FOR (Vertex
                    if (trace.val && candidate_edge == null)
                        LOG.trace(String.format("Failed to find candidate edge from %s to %s [partitions=%s]",
                                  element, catalog_stmt.fullName(), this.stmt_partitions));
                }
            }
            // Without any stmt_args, there's nothing we can do here...
            else if (trace.val) {
                LOG.trace("No stmt_args for " + catalog_stmt + ". Skipping...");
View Full Code Here

        if (hstore_conf.site.exec_prefetch_queries) {
            Procedure catalog_proc = catalogContext.procedures.getIgnoreCase("neworder");
            String prefetchables[] = { "getStockInfo" };
            this.neworder_prefetchables = new Statement[prefetchables.length];
            for (int i = 0; i < this.neworder_prefetchables.length; i++) {
                Statement catalog_stmt = catalog_proc.getStatements().getIgnoreCase(prefetchables[i]);
                assert(catalog_stmt != null) :
                    String.format("Invalid prefetchable Statement %s.%s",
                                  catalog_proc.getName(), prefetchables[i]);
                this.neworder_prefetchables[i] = catalog_stmt;
            } // FOR
View Full Code Here

                      "[touchedPartitions=%s, confidence=%.03f, hashCode=%d]%s",
                      estimate.getClass().getSimpleName(), vertex.getClass().getSimpleName(),
                      estimate.touched_partitions, estimate.confidence, estimate.hashCode(),
                      (trace.val ? "\n"+vertex.debug() : "")));
       
        Statement catalog_stmt = vertex.getCatalogItem();
        PartitionSet partitions = vertex.getPartitions();
        boolean readQuery = (catalog_stmt.getQuerytype() == QueryType.SELECT.getValue());
        for (int partition : partitions.values()) {
            if (estimate.isDoneProbabilitySet(partition) == false) {
                estimate.setDoneProbability(partition, vertex.getDoneProbability(partition));
            }
            if (estimate.isWriteProbabilitySet(partition) == false) {
View Full Code Here

     * testReplicatedInsert
     */
    public void testReplicatedInsert() throws Exception {
        Object params[] = new Object[]{ 9999, "FUCK", "YO", "COUCH" };
        Procedure proc = this.getProcedure("InsertCountry");
        Statement stmt = CollectionUtil.first(proc.getStatements());
        this.batch = new SQLStmt[]{ new SQLStmt(stmt) };
        this.args = new ParameterSet[]{ new ParameterSet(params) };
       
        BatchPlanner planner = new BatchPlanner(batch, catalog_proc, p_estimator);
        this.touched_partitions.clear();
View Full Code Here

     */
    public void testFragmentOrder() throws Exception {
        Procedure catalog_proc = this.getProcedure(BatchPlannerConflictProc.class);
       
        // Create a big batch and make sure that the fragments are in the correct order
        Statement stmts[] = new Statement[]{
            catalog_proc.getStatements().getIgnoreCase("ReplicatedInsert"),
            catalog_proc.getStatements().getIgnoreCase("ReplicatedSelect")
        };
        SQLStmt batch[] = new SQLStmt[stmts.length];
        ParameterSet params[] = new ParameterSet[stmts.length];
        int stmtCounters[] = new int[stmts.length];
        for (int i = 0; i < stmts.length; i++) {
            batch[i] = new SQLStmt(stmts[i]);
            params[i] = new ParameterSet(this.randomStatementParameters(stmts[i]));
            stmtCounters[i] = i;
        } // FOR
       
        BatchPlanner planner = new BatchPlanner(batch, catalog_proc, p_estimator);
        this.touched_partitions.clear();
        BatchPlan plan = planner.plan(TXN_ID,
                                      BASE_PARTITION,
                                      catalogContext.getAllPartitionIds(),
                                      this.touched_partitions,
                                      params);
        assertNotNull(plan);
        assertFalse(plan.hasMisprediction());
       
        List<WorkFragment.Builder> builders = new ArrayList<WorkFragment.Builder>();
        plan.getWorkFragmentsBuilders(TXN_ID, stmtCounters, builders);
        assertFalse(builders.isEmpty());

        List<Statement> batchStmtOrder = new ArrayList<Statement>();
        boolean first = true;
        Statement last = null;
        for (WorkFragment.Builder builder : builders) {
            assertNotNull(builder);
            for (int frag_id : builder.getFragmentIdList()) {
                PlanFragment catalog_frag = CatalogUtil.getPlanFragment(catalog_proc, frag_id);
                assertNotNull(catalog_frag);
                Statement current = catalog_frag.getParent();
                if (last == null || last.equals(current) == false) {
                    batchStmtOrder.add(current);
                }
                last = current;
               
View Full Code Here

            // we should make sure that it only has distributed queries...
            if (builder.getPartitionId() != BASE_PARTITION) {
                for (int frag_id : builder.getFragmentIdList()) {
                    PlanFragment catalog_frag = CatalogUtil.getPlanFragment(catalogContext.catalog, frag_id);
                    assertNotNull(catalog_frag);
                    Statement catalog_stmt = catalog_frag.getParent();
                    assertNotNull(catalog_stmt);
                    assert(catalog_stmt.getMs_fragments().contains(catalog_frag));
                } // FOR
            }
           
            // The InputDepId for all WorkFragments should always be the same
            Set<Integer> all_ids = new HashSet<Integer>(builder.getInputDepIdList());
View Full Code Here

TOP

Related Classes of org.voltdb.catalog.Statement

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.