Examples of edu.brown.designer.partitioners.plan.PartitionPlan

Package edu.brown.designer.partitioners.plan

Examples of edu.brown.designer.partitioners.plan.PartitionPlan

edu.brown.designer.partitioners.plan.PartitionPlan
@author pavlo


        // ----------------------------------------------------
        // (2) Combine all of the PartitionTrees from each Procedure into single
        // partition tree that we can use to generate the PartitionMapping table
        // ----------------------------------------------------
        PartitionPlan pplan = null;
        if (!partition_trees.isEmpty()) {
            LOG.debug("Generating final partition plan using " + partition_trees.size() + " PartitionTrees");
            pplan = this.generatePlan(partition_trees);
        }
        return (pplan);

View Full Code Here

        //
        // Iterate through all the ptrees and generate a PartitionMapping table
        //
        Map<PartitionTree, PartitionPlan> partition_plans = new HashMap<PartitionTree, PartitionPlan>();
        for (final PartitionTree ptree : ptrees) {
            final PartitionPlan partition_plan = new PartitionPlan();
            assert (ptree != null);


            //
            // A PartitionTree may have multiple roots because some vertices
            // could be isolated
            // for replication or failed to be important enough to be linked in
            // the "main" tree
            //
            for (DesignerVertex root : ptree.getRoots()) {
                // System.out.println("ROOT: " + root);
                new VertexTreeWalker<DesignerVertex, DesignerEdge>(ptree) {
                    @Override
                    protected void populate_children(VertexTreeWalker.Children<DesignerVertex> children, DesignerVertex element) {
                        for (DesignerVertex v : this.getGraph().getSuccessors(element)) {
                            if (!this.hasVisited(v)) {
                                children.addAfter(v);
                            }
                        } // FOR
                        return;
                    }


                    @Override
                    protected void callback(DesignerVertex element) {
                        LOG.debug("SimpleCountingMapper.CALLBACK -> " + element.getCatalogItem());
                        DesignerVertex parent = this.getPrevious();


                        PartitionMethodType method = (PartitionMethodType) (element.getAttribute(ptree, PartitionTree.VertexAttributes.METHOD.name()));
                        Column attribute = null;
                        Table parent_table = null;
                        Column parent_attribute = null;


                        if (method == PartitionMethodType.REPLICATION) {
                            // Anything???
                        } else {
                            attribute = (Column) element.getAttribute(ptree, PartitionTree.VertexAttributes.ATTRIBUTE.name());
                            //
                            // If this vertex is a dependent on a parent, then
                            // we also need to get the
                            // mapping of columns
                            //
                            if (parent != null) {
                                DesignerEdge edge = ptree.findEdge(parent, element);
                                if (edge == null) {
                                    LOG.fatal("Failed to find edge between parent '" + parent + "' and child '" + element + "'");
                                    this.stop();
                                    return;
                                }
                                parent_attribute = (Column) element.getAttribute(ptree, PartitionTree.VertexAttributes.PARENT_ATTRIBUTE.name());
                                parent_table = parent.getCatalogItem();
                                method = PartitionMethodType.MAP; // Why do we
                                                                  // have to set
                                                                  // this here?
                                // if (parent_attribute == null) {
                                // PartitionEntry entry = new
                                // PartitionEntry(element.getTable(), method,
                                // attribute, parent_table, parent_attribute);
                                // System.out.println(element.getAttributeValues(ptree));
                                // System.out.println(entry.toString());
                                // System.exit(1);
                                // }
                            }
                        }


                        TableEntry entry = new TableEntry(method, attribute, parent_table, parent_attribute);
                        partition_plan.getTableEntries().put((Table) element.getCatalogItem(), entry);
                        return;
                    }
                }.traverse(root);
            } // FOR roots
            LOG.info(partition_plan);
            partition_plans.put(ptree, partition_plan);
        } // FOR trees


        //
        // Now for each relation, make a tally for the different ways that it
        // could be partitioned
        // This will then be used to generate the final PartitionMapping
        //
        PartitionPlan pplan = new PartitionPlan();
        for (Table catalog_tbl : info.catalogContext.database.getTables()) {
            //
            // For each table, look at the PartitionPlan entries that we created
            // above and see
            // whether it references our table. If it does, then we need to
            // increase our count
            // by one.
            //
            Map<TableEntry, Double> counts = new HashMap<TableEntry, Double>();
            LOG.debug("Counting PartitionPlan entries for " + catalog_tbl);
            for (PartitionTree ptree : partition_plans.keySet()) {
                PartitionPlan partition_plan = partition_plans.get(ptree);
                // System.out.println("Mapping Tables: " + mapping.keySet());
                //
                // We found a partition plan that references our table, so then
                // we need to grab
                // the entry and include it in our count list. Note that the
                // PartitionPlan.Entry
                // object knows how to properly tell whether it has the same
                // attributes as
                // other entry objects, so the count should be properly updated.
                //
                if (partition_plan.getTableEntries().containsKey(catalog_tbl)) {
                    //
                    // Exclude HASH entries without attributes...
                    //
                    TableEntry entry = partition_plan.getTableEntries().get(catalog_tbl);
                    if (entry.getMethod() == PartitionMethodType.HASH && entry.getAttribute() == null) {
                        LOG.warn("Skipping entry for " + catalog_tbl + " because it does not have any partitioning attributes");
                    } else {
                        LOG.debug("Match: " + partition_plan);
                        //

View Full Code Here


        // Target PartitionPlan
        if (this.target_plan_path != null && this.target_plan_path != null) {
            if (debug.val)
                LOG.debug("Loading in target PartitionPlan from '" + this.target_plan_path + "'");
            this.target_plan = new PartitionPlan();
            try {
                this.target_plan.load(this.target_plan_path, catalog_db);
            } catch (IOException ex) {
                throw new RuntimeException("Failed to load target PartitionPlan '" + this.target_plan_path + "'", ex);
            }

View Full Code Here

        // Enable compact output
        final boolean table_output = (args.getOptParams().contains("table"));


        // If given a PartitionPlan, then update the catalog
        File pplan_path = new File(args.getParam(ArgumentsParser.PARAM_PARTITION_PLAN));
        PartitionPlan pplan = new PartitionPlan();
        pplan.load(pplan_path, args.catalogContext.database);
        if (args.getBooleanParam(ArgumentsParser.PARAM_PARTITION_PLAN_REMOVE_PROCS, false)) {
            for (Procedure catalog_proc : pplan.proc_entries.keySet()) {
                pplan.setNullProcParameter(catalog_proc);
            } // FOR
        }
        if (args.getBooleanParam(ArgumentsParser.PARAM_PARTITION_PLAN_RANDOM_PROCS, false)) {
            for (Procedure catalog_proc : pplan.proc_entries.keySet()) {
                pplan.setRandomProcParameter(catalog_proc);
            } // FOR
        }
        pplan.apply(args.catalogContext.database);


        System.out.println("Applied PartitionPlan '" + pplan_path + "' to catalog\n" + pplan);
        System.out.print(StringUtil.DOUBLE_LINE);
        // if (!table_output) {
        //
        // }
        // } else if (!table_output) {
        // System.err.println("PartitionPlan file '" + pplan_path +
        // "' does not exist. Ignoring...");
        // }
        if (args.hasParam(ArgumentsParser.PARAM_PARTITION_PLAN_OUTPUT)) {
            String output = args.getParam(ArgumentsParser.PARAM_PARTITION_PLAN_OUTPUT);
            if (output.equals("-"))
                output = pplan_path.getAbsolutePath();
            pplan.save(new File(output));
            System.out.println("Saved PartitionPlan to '" + output + "'");
        }


        System.out.flush();

View Full Code Here

     * edu.brown.designer.partitioners.AbstractPartitioner#generate(edu.brown
     * .designer.DesignerHints)
     */
    @Override
    public PartitionPlan generate(DesignerHints hints) throws Exception {
        final PartitionPlan pplan = new PartitionPlan();


        // Generate an AccessGraph and select the column with the greatest
        // weight for each table
        final AccessGraph agraph = this.generateAccessGraph();
        final boolean calculate_memory = (hints.force_replication_size_limit != null && hints.max_memory_per_partition != 0);


        double total_partitionRatio = 0.0;
        long total_partitionSize = 0l;


        for (DesignerVertex v : agraph.getVertices()) {
            Table catalog_tbl = v.getCatalogItem();
            String table_key = CatalogKey.createKey(catalog_tbl);


            Collection<Column> forced_columns = hints.getForcedTablePartitionCandidates(catalog_tbl);
            TableStatistics ts = info.stats.getTableStatistics(catalog_tbl);
            assert (ts != null) : "Null TableStatistics for " + catalog_tbl;
            double partition_size = (calculate_memory ? (ts.tuple_size_total / (double) info.getNumPartitions()) : 0);
            double partition_ratio = (calculate_memory ? (ts.tuple_size_total / (double) hints.max_memory_per_partition) : 0);
            TableEntry pentry = null;


            if (debug.val) {
                Map<String, Object> m = new ListOrderedMap<String, Object>();
                m.put("Read Only", ts.readonly);
                m.put("Table Size", StringUtil.formatSize(ts.tuple_size_total));
                m.put("Table Partition Size", StringUtil.formatSize((long)partition_size));
                m.put("Table Partition Ratio", String.format("%.02f", partition_ratio));
                m.put("Total Partition Size", String.format("%s / %s", StringUtil.formatSize(total_partitionSize), StringUtil.formatSize(hints.max_memory_per_partition)));
                m.put("Total Partition Ratio", String.format("%.02f", total_partitionRatio));
                LOG.debug(String.format("%s\n%s", catalog_tbl.getName(), StringUtil.formatMaps(m)));
            }


            // -------------------------------
            // Replication
            // -------------------------------
            if (hints.force_replication.contains(table_key) || (calculate_memory && ts.readonly && hints.enable_replication_readonly && partition_ratio <= hints.force_replication_size_limit)) {
                total_partitionRatio += partition_ratio;
                total_partitionSize += ts.tuple_size_total;
                Column catalog_col = ReplicatedColumn.get(catalog_tbl);
                pentry = new TableEntry(PartitionMethodType.REPLICATION, catalog_col);
                if (debug.val)
                    LOG.debug(String.format("Replicating %s at all partitions [%s]", catalog_tbl.getName(), catalog_col.fullName()));


                // -------------------------------
                // Forced Selection
                // -------------------------------
            } else if (forced_columns.isEmpty() == false) {
                // Assume there is only one candidate
                assert (forced_columns.size() == 1) : "Unexpected number of forced columns: " + forced_columns;
                Column catalog_col = CollectionUtil.first(forced_columns);
                pentry = new TableEntry(PartitionMethodType.HASH, catalog_col);
                total_partitionRatio += partition_size / (double) hints.max_memory_per_partition;
                total_partitionSize += partition_size;
                if (debug.val)
                    LOG.debug(String.format("Forcing %s to be partitioned by specific column [%s]", catalog_tbl.getName(), catalog_col.fullName()));


                // -------------------------------
                // Select Most Popular
                // -------------------------------
            } else {
                // If there are no edges, then we'll just randomly pick a column
                // since it doesn't matter
                final Collection<DesignerEdge> edges = agraph.getIncidentEdges(v);
                if (edges.isEmpty())
                    continue;
                if (trace.val)
                    LOG.trace(catalog_tbl + " has " + edges.size() + " edges in AccessGraph");


                ObjectHistogram<Column> column_histogram = null;
                ObjectHistogram<Column> join_column_histogram = new ObjectHistogram<Column>();
                ObjectHistogram<Column> self_column_histogram = new ObjectHistogram<Column>();
                // Map<Column, Double> unsorted = new HashMap<Column, Double>();
                for (DesignerEdge e : edges) {
                    Collection<DesignerVertex> vertices = agraph.getIncidentVertices(e);
                    DesignerVertex v0 = CollectionUtil.get(vertices, 0);
                    DesignerVertex v1 = CollectionUtil.get(vertices, 1);
                    boolean self = (v0.equals(v) && v1.equals(v));
                    column_histogram = (self ? self_column_histogram : join_column_histogram);


                    double edge_weight = e.getTotalWeight();
                    PredicatePairs cset = e.getAttribute(AccessGraph.EdgeAttributes.COLUMNSET);
                    if (trace.val)
                        LOG.trace("Examining ColumnSet for " + e.toString(true));


                    Histogram<Column> cset_histogram = cset.buildHistogramForType(Column.class);
                    Collection<Column> columns = cset_histogram.values();
                    if (trace.val)
                        LOG.trace("Constructed Histogram for " + catalog_tbl + " from ColumnSet:\n"
                                + cset_histogram.setDebugLabels(CatalogUtil.getHistogramLabels(cset_histogram.values())).toString(100, 50));
                    for (Column catalog_col : columns) {
                        if (!catalog_col.getParent().equals(catalog_tbl))
                            continue;
                        if (catalog_col.getNullable())
                            continue;
                        long cnt = cset_histogram.get(catalog_col);
                        if (trace.val)
                            LOG.trace("Found Match: " + catalog_col.fullName() + " [cnt=" + cnt + "]");
                        column_histogram.put(catalog_col, Math.round(cnt * edge_weight));
                    } // FOR
                    // System.err.println(cset.debug());
                    // LOG.info("[" + e.getTotalWeight() + "]: " + cset);
                } // FOR


                // If there were no join columns, then use the self-reference
                // histogram
                column_histogram = (join_column_histogram.isEmpty() ? self_column_histogram : join_column_histogram);
                if (column_histogram.isEmpty()) {
                    EventObserver<DesignerVertex> observer = new EventObserver<DesignerVertex>() {
                        @Override
                        public void update(EventObservable<DesignerVertex> o, DesignerVertex v) {
                            for (DesignerEdge e : agraph.getIncidentEdges(v)) {
                                LOG.info(e.getAttribute(AccessGraph.EdgeAttributes.COLUMNSET));
                            }
                            LOG.info(StringUtil.repeat("-", 100));
                        }
                    };
                    LOG.info("Edges: " + edges);
                    GraphVisualizationPanel.createFrame(agraph, observer).setVisible(true);
                    // ThreadUtil.sleep(10000);
                }


                // We might not find anything if we are calculating the lower
                // bounds using only one transaction
                // if (column_histogram.isEmpty()) {
                // if (trace.val)
                // LOG.trace("Failed to find any ColumnSets for " +
                // catalog_tbl);
                // continue;
                // }
                assert (!column_histogram.isEmpty()) : "Failed to find any ColumnSets for " + catalog_tbl;
                if (trace.val)
                    LOG.trace("Column Histogram:\n" + column_histogram);


                Column catalog_col = CollectionUtil.first(column_histogram.getMaxCountValues());
                pentry = new TableEntry(PartitionMethodType.HASH, catalog_col, null, null);
                total_partitionRatio += partition_size / (double) hints.max_memory_per_partition;
                total_partitionSize += partition_size;


                if (debug.val)
                    LOG.debug(String.format("Selected %s's most popular column for partitioning [%s]", catalog_tbl.getName(), catalog_col.fullName()));
            }
            pplan.table_entries.put(catalog_tbl, pentry);


            if (debug.val)
                LOG.debug(String.format("Current Partition Size: %s", StringUtil.formatSize(total_partitionSize), StringUtil.formatSize(hints.max_memory_per_partition)));
            assert (total_partitionRatio <= 1) : String.format("Too much memory per partition: %s / %s", StringUtil.formatSize(total_partitionSize),
                    StringUtil.formatSize(hints.max_memory_per_partition));
        } // FOR


        for (Table catalog_tbl : info.catalogContext.database.getTables()) {
            if (pplan.getTableEntry(catalog_tbl) == null) {
                Column catalog_col = CollectionUtil.random(catalog_tbl.getColumns());
                assert (catalog_col != null) : "Failed to randomly pick column for " + catalog_tbl;
                pplan.table_entries.put(catalog_tbl, new TableEntry(PartitionMethodType.HASH, catalog_col, null, null));
                if (debug.val)
                    LOG.debug(String.format("No partitioning column selected for %s. Choosing a random attribute [%s]", catalog_tbl, catalog_col.fullName()));
            }
        } // FOR


        if (hints.enable_procparameter_search) {
            if (debug.val)
                LOG.debug("Selecting partitioning ProcParameter for " + this.info.catalogContext.database.getProcedures().size() + " Procedures");
            pplan.apply(info.catalogContext.database);


            // Temporarily disable multi-attribute parameters
            boolean multiproc_orig = hints.enable_multi_partitioning;
            hints.enable_multi_partitioning = false;


            for (Procedure catalog_proc : this.info.catalogContext.database.getProcedures()) {
                if (PartitionerUtil.shouldIgnoreProcedure(hints, catalog_proc))
                    continue;


                Set<String> param_order = PartitionerUtil.generateProcParameterOrder(info, info.catalogContext.database, catalog_proc, hints);
                if (param_order.isEmpty() == false) {
                    ProcParameter catalog_proc_param = CatalogKey.getFromKey(info.catalogContext.database, CollectionUtil.first(param_order), ProcParameter.class);
                    if (debug.val)
                        LOG.debug(String.format("PARTITION %-25s%s", catalog_proc.getName(), CatalogUtil.getDisplayName(catalog_proc_param)));


                    // Create a new PartitionEntry for this procedure and set it
                    // to be always single-partitioned
                    // We will check down below whether that's always true or
                    // not
                    ProcedureEntry pentry = new ProcedureEntry(PartitionMethodType.HASH, catalog_proc_param, true);
                    pplan.getProcedureEntries().put(catalog_proc, pentry);
                }
            } // FOR


            hints.enable_multi_partitioning = multiproc_orig;
        }

View Full Code Here

0 1 2

TOP

Related Classes of edu.brown.designer.partitioners.plan.PartitionPlan

edu.brown.api.BenchmarkComponent

edu.brown.catalog.special.VerticalPartitionColumn

edu.brown.costmodel.SingleSitedCostModel

edu.brown.costmodel.TimeIntervalCostModel

edu.brown.designer.Designer

edu.brown.designer.DesignerHints

edu.brown.designer.LowerBoundsCalculator

edu.brown.designer.partitioners.BranchAndBoundPartitioner

edu.brown.designer.partitioners.BranchAndBoundPartitioner$TraverseThread

edu.brown.designer.partitioners.HeuristicPartitioner

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.