Examples of org.voltdb.catalog.Statement

org.voltdb.catalog.Statement
A parameterized SQL statement embedded in a stored procedure

        // MAIN LOGIC LOOP
        // This is where we go through each SQLStmt in the batch and figure out
        // what partitions it needs to touch.
        // ----------------------------------------------------------------------------
        for (int stmt_index = 0; stmt_index < this.batchSize; stmt_index++) {
            final Statement catalog_stmt = this.catalog_stmts[stmt_index];
            assert (catalog_stmt != null) :
                String.format("The Statement at index %d is null for %s",
                              stmt_index, this.catalog_proc);
            final Object params[] = batchArgs[stmt_index].toArray();
            if (trace.val)
                LOG.trace(String.format("[#%d-%02d] Calculating touched partitions plans for %s",
                          txn_id, stmt_index, catalog_stmt.fullName()));


            Map<PlanFragment, PartitionSet> frag_partitions = plan.frag_partitions[stmt_index];
            PartitionSet stmt_all_partitions = plan.stmt_partitions[stmt_index];


            boolean has_singlepartition_plan = catalog_stmt.getHas_singlesited();
            boolean is_replicated_only = this.stmt_is_replicatedonly[stmt_index];
            boolean is_read_only = this.stmt_is_readonly[stmt_index];
            boolean is_singlePartition = has_singlepartition_plan;
            boolean is_local = true;
            CatalogMap<PlanFragment> fragments = null;


            // OPTIMIZATION: Fast partition look-up caching
            // OPTIMIZATION: Read-only queries on replicated tables always just
            // go to the local partition
            // OPTIMIZATION: If we're force to be single-partitioned, pretend
            // that the table is replicated
            if ((this.cache_isSinglePartition != null && this.cache_isSinglePartition[stmt_index]) ||
                (is_replicated_only && is_read_only) ||
                (this.force_singlePartition)) {
                if (trace.val) {
                    if (this.cache_isSinglePartition[stmt_index]) {
                        LOG.trace(String.format("[#%d-%02d] Using fast-lookup for %s. " +
                              "Skipping PartitionEstimator",
                                  txn_id, stmt_index, catalog_stmt.fullName()));
                    } else {
                        LOG.trace(String.format("[#%d-%02d] %s is read-only and replicate-only." +
                              "Skipping PartitionEstimator",
                              txn_id, stmt_index, catalog_stmt.fullName()));
                    }
                }
                assert (has_singlepartition_plan);


                if (this.cache_singlePartitionFragmentPartitions == null) {
                    this.cache_singlePartitionFragmentPartitions = CACHED_FRAGMENT_PARTITION_MAPS[base_partition];
                }
                Map<PlanFragment, PartitionSet> cached_frag_partitions = this.cache_singlePartitionFragmentPartitions.get(catalog_stmt);
                if (cached_frag_partitions == null) {
                    cached_frag_partitions = new HashMap<PlanFragment, PartitionSet>();
                    PartitionSet p = this.catalogContext.getPartitionSetSingleton(base_partition);
                    for (PlanFragment catalog_frag : catalog_stmt.getFragments().values()) {
                        cached_frag_partitions.put(catalog_frag, p);
                    } // FOR
                    this.cache_singlePartitionFragmentPartitions.put(catalog_stmt, cached_frag_partitions);
                }
                if (plan.stmt_partitions_swap[stmt_index] == null) {
                    plan.stmt_partitions_swap[stmt_index] = plan.stmt_partitions[stmt_index];
                    plan.frag_partitions_swap[stmt_index] = plan.frag_partitions[stmt_index];
                }
                stmt_all_partitions = plan.stmt_partitions[stmt_index] = this.catalogContext.getPartitionSetSingleton(base_partition);
                frag_partitions = plan.frag_partitions[stmt_index] = cached_frag_partitions;
            }


            // Otherwise figure out whether the query can execute as
            // single-partitioned or not
            else {
                if (debug.val)
                    LOG.debug(String.format("[#%d-%02d] Computing touched partitions %s in txn #%d", txn_id,
                              stmt_index, catalog_stmt.fullName(), txn_id));


                if (plan.stmt_partitions_swap[stmt_index] != null) {
                    stmt_all_partitions = plan.stmt_partitions[stmt_index] = plan.stmt_partitions_swap[stmt_index];
                    plan.stmt_partitions_swap[stmt_index] = null;
                    stmt_all_partitions.clear();


                    frag_partitions = plan.frag_partitions[stmt_index] = plan.frag_partitions_swap[stmt_index];
                    plan.frag_partitions_swap[stmt_index] = null;
                }


                try {
                    // OPTIMIZATION: If we were told that the transaction is suppose to be
                    // single-partitioned, then we will throw the single-partitioned PlanFragments
                    // at the PartitionEstimator to get back what partitions each PlanFragment
                    // will need to go to. If we get multiple partitions, then we know that we
                    // mispredicted and we should throw a MispredictionException
                    // If we originally didn't predict that it was single-partitioned, then we
                    // actually still need to check whether the query should be single-partitioned or not.
                    // This is because a query may actually just want to execute on just one
                    // partition (note that it could be a local partition or the remote partition).
                    // We'll assume that it's single-partition <<--- Can we cache that??
                    while (true) {
                        if (is_singlePartition == false) stmt_all_partitions.clear();
                        fragments = (is_singlePartition ? catalog_stmt.getFragments() : catalog_stmt.getMs_fragments());
                        if (debug.val)
                            LOG.debug(String.format("[#%d-%02d] Estimating for %d %s-partition fragments",
                                      txn_id, stmt_index, fragments.size(),
                                      (is_singlePartition ? "single" : "multi")));


                        // PARTITION ESTIMATOR
                        if (hstore_conf.site.planner_profiling && profiler != null)
                            ProfileMeasurementUtil.swap(profiler.plan_time, profiler.partest_time);
                        this.p_estimator.getAllFragmentPartitions(frag_partitions,
                                                                  stmt_all_partitions,
                                                                  fragments.values(),
                                                                  params,
                                                                  base_partition);
                        if (hstore_conf.site.planner_profiling && profiler != null)
                            ProfileMeasurementUtil.swap(profiler.partest_time, profiler.plan_time);


                        int stmt_all_partitions_size = stmt_all_partitions.size();
                        if (is_singlePartition && stmt_all_partitions_size > 1) {
                            // If this was suppose to be multi-partitioned, then
                            // we want to stop right here!!
                            if (predict_singlePartitioned) {
                                if (trace.val)
                                    LOG.trace(String.format("Mispredicted txn #%d - Multiple Partitions %s",
                                              txn_id, stmt_all_partitions));
                                mispredict = true;
                                break;
                            }
                            // Otherwise we can let it wrap back around and construct the fragment
                            // mapping for the multi-partition PlanFragments
                            is_singlePartition = false;
                            continue;
                        }
                        is_local = (stmt_all_partitions_size == 1 && stmt_all_partitions.contains(base_partition));
                        if (is_local == false && predict_singlePartitioned) {
                            // Again, this is not what was suppose to happen!
                            if (trace.val)
                                LOG.trace(String.format("Mispredicted txn #%d - Remote Partitions %s",
                                          txn_id, stmt_all_partitions));
                            mispredict = true;
                            break;
                        } else if (predict_partitions.containsAll(stmt_all_partitions) == false) {
                            // Again, this is not what was suppose to happen!
                            if (trace.val)
                                LOG.trace(String.format("Mispredicted txn #%d - Unallocated Partitions %s / %s",
                                          txn_id, stmt_all_partitions, predict_partitions));
                            mispredict = true;
                            break;
                        }
                        // Score! We have a plan that works!
                        break;
                    } // WHILE
                    // Bad Mojo!
                } catch (Exception ex) {
                    String msg = "";
                    for (int i = 0; i < this.batchSize; i++) {
                        msg += String.format("[#%d-%02d] %s %s\n%5s\n", txn_id, i, catalog_stmt.fullName(),
                                catalog_stmt.getSqltext(), Arrays.toString(batchArgs[i].toArray()));
                    } // FOR
                    LOG.fatal("\n" + msg);
                    throw new RuntimeException("Unexpected error when planning " + catalog_stmt.fullName(), ex);
                }
            }
            if (debug.val)
                LOG.debug(String.format("[#%d-%02d] is_singlepartition=%s, partitions=%s",
                          txn_id, stmt_index, is_singlePartition, stmt_all_partitions));


            // Get a sorted list of the PlanFragments that we need to execute
            // for this query
            if (is_singlePartition) {
                if (this.sorted_singlep_fragments[stmt_index] == null) {
                    this.sorted_singlep_fragments[stmt_index] = PlanNodeUtil.getSortedPlanFragments(catalog_stmt, true);
                }
                plan.frag_list[stmt_index] = this.sorted_singlep_fragments[stmt_index];


                // Only mark that we touched these partitions if the Statement
                // is not on a replicated table or it's not read-only
                if (is_replicated_only == false || is_read_only == false) {
                    touched_partitions.put(stmt_all_partitions.get());
                }
            }
            // Distributed Query
            else {
                if (this.sorted_multip_fragments[stmt_index] == null) {
                    this.sorted_multip_fragments[stmt_index] = PlanNodeUtil.getSortedPlanFragments(catalog_stmt, false);
                }
                plan.frag_list[stmt_index] = this.sorted_multip_fragments[stmt_index];


                // Always mark that we are touching these partitions
                touched_partitions.put(stmt_all_partitions.values());


                // Note that will want to update is_singlePartitioned here for non-readonly replicated
                // querys when we have a one partition cluster because those queries don't have
                // single-partition query plans
                // if (this.num_partitions == 1 && is_replicated_only && is_read_only == false) {
                // is_singlePartition = true;
                // }
            }


            plan.readonly = plan.readonly && catalog_stmt.getReadonly();
            plan.all_singlepartitioned = plan.all_singlepartitioned && is_singlePartition;
            plan.all_local = plan.all_local && is_local;


            // Keep track of whether the current query in the batch was
            // single-partitioned or not
            plan.singlepartition_bitmap[stmt_index] = is_singlePartition;


            // Misprediction!!
            if (mispredict) {
                // If this is the first Statement in the batch that hits the mispredict,
                // then we need to create the histogram and populate it with the
                // partitions from the previous queries
                int start_idx = stmt_index;
                if (mispredict_h == null) {
                    mispredict_h = new FastIntHistogram();
                    start_idx = 0;
                }
                for (int i = start_idx; i <= stmt_index; i++) {
                    if (debug.val)
                        LOG.debug(String.format("Pending mispredict for txn #%d. " +
                                  "Checking whether to add partitions for batch statement %02d",
                                   txn_id, i));


                    // Make sure that we don't count the local partition if it
                    // was reading a replicated table.
                    if (this.stmt_is_replicatedonly[i] == false || 
                        (this.stmt_is_replicatedonly[i] && this.stmt_is_readonly[i] == false)) {
                        if (trace.val)
                            LOG.trace(String.format("%s touches non-replicated table. " +
                                      "Including %d partitions in mispredict histogram for txn #%d",
                                      this.catalog_stmts[i].fullName(), plan.stmt_partitions[i].size(), txn_id));
                        mispredict_h.put(plan.stmt_partitions[i]);
                    }
                } // FOR
                continue;
            }


            // ----------------------
            // DEBUG DUMP
            // ----------------------
            if (debug.val) {
                List<PlanFragment> _fragments = null;
                if (is_singlePartition && this.sorted_singlep_fragments[stmt_index] != null) {
                    _fragments = this.sorted_singlep_fragments[stmt_index];
                } else {
                    _fragments = this.sorted_multip_fragments[stmt_index];
                }


                Map<?, ?> maps[] = new Map[_fragments.size() + 1];
                int ii = 0;
                for (PlanFragment catalog_frag : _fragments) {
                    Map<String, Object> m = new LinkedHashMap<String, Object>();
                    PartitionSet p = plan.frag_partitions[stmt_index].get(catalog_frag);
                    boolean frag_local = (p.size() == 1 && p.contains(base_partition));
                    m.put(String.format("[%02d] Fragment", ii), catalog_frag.fullName());
                    m.put(String.format("     Partitions"), p);
                    m.put(String.format("     IsLocal"), frag_local);
                    ii++;
                    maps[ii] = m;
                } // FOR


                Map<String, Object> header = new LinkedHashMap<String, Object>();
                header.put("Batch Statement", String.format("#%d / %d", stmt_index, this.batchSize));
                header.put("Catalog Statement", catalog_stmt.fullName());
                header.put("Statement SQL", catalog_stmt.getSqltext());
                header.put("All Partitions", plan.stmt_partitions[stmt_index]);
                header.put("Local Partition", base_partition);
                header.put("IsSingledPartitioned", is_singlePartition);
                header.put("IsStmtLocal", is_local);
                header.put("IsReplicatedOnly", is_replicated_only);

View Full Code Here

     * testPlanner
     */
    @Test
    public void testPlanner() throws Exception {
        Procedure catalog_proc = this.getProcedure(MockMapReduce.class);
        Statement catalog_stmt = this.getStatement(catalog_proc, catalog_proc.getMapinputquery());
        
        AbstractPlanNode root = PlanNodeUtil.getRootPlanNodeForStatement(catalog_stmt, false);
        assertNotNull(root);
        System.err.println(PlanNodeUtil.debug(root));

View Full Code Here

     * Compile a statement and return the final plan graph.
     * @param sql
     * @param paramCount
     */
    public AbstractPlanNode compile(String sql, int paramCount) {
        Statement catalogStmt = proc.getStatements().add("stmt-" + String.valueOf(compileCounter++));
        catalogStmt.setSqltext(sql);
        catalogStmt.setSinglepartition(false);
        catalogStmt.setBatched(false);
        catalogStmt.setParamnum(paramCount);


        // determine the type of the query
        QueryType qtype = QueryType.SELECT;
        catalogStmt.setReadonly(true);
        if (sql.toLowerCase().startsWith("insert")) {
            qtype = QueryType.INSERT;
            catalogStmt.setReadonly(false);
        }
        if (sql.toLowerCase().startsWith("update")) {
            qtype = QueryType.UPDATE;
            catalogStmt.setReadonly(false);
        }
        if (sql.toLowerCase().startsWith("delete")) {
            qtype = QueryType.DELETE;
            catalogStmt.setReadonly(false);
        }
        catalogStmt.setQuerytype(qtype.getValue());
        // name will look like "basename-stmt-#"
        String name = catalogStmt.getParent().getTypeName() + "-" + catalogStmt.getTypeName();


        DatabaseEstimates estimates = new DatabaseEstimates();
        TrivialCostModel costModel = new TrivialCostModel();
        QueryPlanner planner =
            new QueryPlanner(catalog.getClusters().get("cluster"), db, hsql,
                             estimates, true, false);


        CompiledPlan plan = null;
        plan = planner.compilePlan(costModel, catalogStmt.getSqltext(), catalogStmt.getTypeName(),
                                   catalogStmt.getParent().getTypeName(), catalogStmt.getSinglepartition(), null);


        if (plan == null)
        {
            String msg = "planner.compilePlan returned null plan";
            String plannerMsg = planner.getErrorMessage();
            if (plannerMsg != null)
            {
                msg += " with error: \"" + plannerMsg + "\"";
            }
            throw new NullPointerException(msg);
        }


        // Input Parameters
        // We will need to update the system catalogs with this new information
        // If this is an adhoc query then there won't be any parameters
        for (ParameterInfo param : plan.parameters) {
            StmtParameter catalogParam = catalogStmt.getParameters().add(String.valueOf(param.index));
            catalogParam.setJavatype(param.type.getValue());
            catalogParam.setIndex(param.index);
        }


        // Output Columns
        int index = 0;
        for (Integer plancol : plan.columns) {
            PlanColumn planColumn = planner.getPlannerContext().get(plancol);
            Column column = catalogStmt.getOutput_columns().add(planColumn.getDisplayName()); // String.valueOf(index));
            column.setNullable(false);
            column.setIndex(index);
//            column.setName(planColumn.displayName());
            column.setType(planColumn.type().getValue());
            column.setSize(planColumn.width());

View Full Code Here

     * testCreateExplicitEdges
     */
    public void testCreateExplicitEdges() throws Exception {
        this.generator.initialize(agraph);
        
        Statement catalog_stmt = this.catalog_proc.getStatements().get(TARGET_STATEMENT);
        assertNotNull(catalog_stmt);
        this.generator.createExplicitEdges(agraph, catalog_stmt);
        
        assert(this.agraph.getVertexCount() > 0);
        assert(this.agraph.getEdgeCount() > 0);

View Full Code Here

     * testCreateSharedParamEdges
     */
    public void testCreateSharedParamEdges() throws Exception {
        this.generator.initialize(agraph);
        
        Statement catalog_stmts[] = this.catalog_proc.getStatements().values();
        Statement catalog_stmt0 = catalog_stmts[0];
        assertNotNull(catalog_stmt0);
        this.generator.createExplicitEdges(agraph, catalog_stmt0);
        
        Statement catalog_stmt1 = catalog_stmts[1];
        assertNotNull(catalog_stmt1);
        this.generator.createExplicitEdges(agraph, catalog_stmt1);
        
        List<ProcParameter> catalog_stmt0_params = new ArrayList<ProcParameter>();
        for (StmtParameter param : catalog_stmt0.getParameters()) {

View Full Code Here

     */
    public void testInvalidQuery() throws Exception {
        // Check to make sure that we don't try to update a query when we don't have
        // any vertical partitions defined
        Procedure catalog_proc = this.getProcedure("TestInvalid");
        Statement catalog_stmt = CollectionUtil.first(catalog_proc.getStatements());
        boolean ret = vp_planner.process(catalog_stmt);
        assertFalse(ret);
    }

View Full Code Here

        m.put(catalog_tbl, view_tbl);
        
        for (int i = 0; i < REWRITE_SQLS.length; i++) {
            String procName = String.format("TestRewrite%d", i);
            Procedure catalog_proc = this.getProcedure(procName);
            Statement catalog_stmt = CollectionUtil.first(catalog_proc.getStatements());
        
            String orig_sql = catalog_stmt.getSqltext();
            String new_sql = vp_planner.rewriteSQL(catalog_stmt, m);
            assertNotNull(new_sql);
            assertFalse(new_sql.isEmpty());
            System.err.println(String.format("BEFORE: %s\n" +
                                             "AFTER:  %s\n\n",

View Full Code Here

    /**
     * testSingleColumnQuery
     */
    public void testSingleColumnQuery() throws Exception {
        Procedure catalog_proc = this.getProcedure("TestSingleTable");
        Statement catalog_stmt = CollectionUtil.first(catalog_proc.getStatements());
        Object params[] = new Object[]{ "ABC" };
        
        Integer base_partition = new Random().nextInt(NUM_PARTITIONS);
        
        // Double check that this is always a distributed query

View Full Code Here

        MaterializedViewInfo view = CatalogUtil.getVerticalPartition(catalog_tbl);
        assertNotNull(view);
        assertEquals(CatalogUtil.debug(view.getGroupbycols()), vp_cols.length, view.getGroupbycols().size());
        
        Procedure catalog_proc = this.getProcedure("TestOrderBy");
        Statement catalog_stmt = CollectionUtil.first(catalog_proc.getStatements());
        assert(catalog_stmt != null) : catalog_proc.getStatements();
        Object params[] = new Object[]{ "ABC" };
        
        Integer base_partition = new Random().nextInt(NUM_PARTITIONS);

View Full Code Here

     */
    public void testGetPartitionsTransactionTrace() throws Exception {
        Procedure catalog_proc = this.getProcedure(GetNewDestination.class);
        Object txn_params[] = new Object[] { 1l, 4l, 0l, 5l };
        
        Statement catalog_stmt = this.getStatement(catalog_proc, "GetData");
        Object query_params[] = new Object[] { 1l, 1l, 4l, 0l, 5l };
        
        TransactionTrace txn_trace = new TransactionTrace(1001l, catalog_proc, txn_params);
        QueryTrace query_trace = new QueryTrace(catalog_stmt, query_params, 0);
        txn_trace.addQuery(query_trace);

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.voltdb.catalog.Statement

edu.brown.BaseTestCase

edu.brown.catalog.CatalogUtil

edu.brown.catalog.conflicts.ConflictGraphExport

edu.brown.catalog.conflicts.ConflictSetTableDumper

edu.brown.catalog.conflicts.ConflictSetTableDumper$DumpGraph

edu.brown.catalog.PatchCatalog

edu.brown.catalog.special.VerticalPartitionColumn

edu.brown.catalog.TestCatalogUtil

edu.brown.catalog.TestCatalogUtil2

edu.brown.costmodel.MarkovCostModel

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.