Package edu.brown.utils

Examples of edu.brown.utils.PartitionEstimator


       
        // Get the hasher we will use for this HStoreSite
        this.hasher = ClassUtil.newInstance(hstore_conf.global.hasher_class,
                                             new Object[]{ this.catalogContext, num_partitions },
                                             new Class<?>[]{ CatalogContext.class, int.class });
        this.p_estimator = new PartitionEstimator(this.catalogContext, this.hasher);
        this.remoteTxnEstimator = new RemoteEstimator(this.p_estimator);
       
        // ARIES
        if(hstore_conf.site.aries){
            // Don't use both recovery modes
View Full Code Here


        singleton = new HStoreSite(site_id, catalogContext, hstore_conf);
       
        // For every partition in our local site, we want to setup a new ExecutionSite
        // Thankfully I had enough sense to have PartitionEstimator take in the local partition
        // as a parameter, so we can share a single instance across all ExecutionSites
        PartitionEstimator p_estimator = singleton.getPartitionEstimator();
       
        // ----------------------------------------------------------------------------
        // MarkovGraphs
        // ----------------------------------------------------------------------------
        Map<Integer, MarkovGraphsContainer> markovs = null;
View Full Code Here

     */
    public void testPartitionEstimator() throws Exception {
        Integer base_partition = 1;
        Database clone_db = CatalogCloner.cloneDatabase(catalog_db);
        CatalogContext clone_catalogContext = new CatalogContext(clone_db.getCatalog());
        PartitionEstimator p_estimator = new PartitionEstimator(clone_catalogContext);
        info = this.generateInfo(clone_catalogContext);
       
        Table catalog_tbl = this.getTable(clone_db, TM1Constants.TABLENAME_SUBSCRIBER);
        Column target_col = this.getColumn(catalog_tbl, "S_ID");
        Collection<VerticalPartitionColumn> candidates = VerticalPartitionerUtil.generateCandidates(target_col, info.stats);
        assertNotNull(candidates);
        assertFalse(candidates.isEmpty());
        VerticalPartitionColumn vpc = CollectionUtil.first(candidates);
        assertNotNull(vpc);
        assertFalse(vpc.isUpdateApplied());
       
        // Get the original partitions for the queries before we apply the optimizations
        Map<Statement, Object[]> stmt_params = new HashMap<Statement, Object[]>();
        for (Statement catalog_stmt : vpc.getOptimizedQueries()) {
            // We first need to generate random input parameters
            Object params[] = new Object[catalog_stmt.getParameters().size()];
            for (int i = 0; i < params.length; i++) {
                StmtParameter catalog_param = catalog_stmt.getParameters().get(i);
                VoltType vtype = VoltType.get(catalog_param.getJavatype());
                params[i] = VoltTypeUtil.getRandomValue(vtype);
            } // FOR
            stmt_params.put(catalog_stmt, params);
           
            // Then get the list of partitions that it will access
            // This should always be *all* partitions
            PartitionSet partitions = new PartitionSet();
            p_estimator.getAllPartitions(partitions, catalog_stmt, params, base_partition);
            assertNotNull(partitions);
            assertEquals(CatalogUtil.getNumberOfPartitions(clone_db), partitions.size());
        } // FOR
       
        // Now apply the optimized queries
        // The number of partitions that our Statements touch should be reduced to one
        vpc.applyUpdate();
        assert(vpc.isUpdateApplied());
        for (Statement catalog_stmt : vpc.getOptimizedQueries()) {
            Object params[] = stmt_params.get(catalog_stmt);
            assertNotNull(params);
            PartitionSet partitions = new PartitionSet();
            p_estimator.getAllPartitions(partitions, catalog_stmt, params, base_partition);
            assertNotNull(partitions);
            assertEquals(1, partitions.size());
        } // FOR

    }
View Full Code Here

     * @param args
     * @return
     */
    public static MarkovEstimate predictPath(MarkovGraph markov, MarkovEstimator t_estimator, Object args[]) {
        CatalogContext catalogContext = t_estimator.getCatalogContext();
        PartitionEstimator p_estimator = t_estimator.getPartitionEstimator();
       
        int base_partition = HStoreConstants.NULL_PARTITION_ID;
        try {
            base_partition = p_estimator.getBasePartition(markov.getProcedure(), args);
        } catch (Exception ex) {
            String msg = String.format("Failed to calculate base partition for <%s, %s>",
                                       markov.getProcedure().getName(), Arrays.toString(args));
            LOG.fatal(msg, ex);
            throw new RuntimeException(msg, ex);
View Full Code Here

   
    @Override
    protected void setUp() throws Exception {
        super.setUp(ProjectType.MAPREDUCE);
        this.addPartitions(NUM_PARTITIONS);
        p_estimator = new PartitionEstimator(catalogContext, new DefaultHasher(catalogContext, NUM_PARTITIONS));
    }
View Full Code Here

            ArgumentsParser.PARAM_MAPPINGS,
            ArgumentsParser.PARAM_MARKOV
        );
       
        // Word up
        PartitionEstimator p_estimator = new PartitionEstimator(args.catalogContext);
       
        // Create MarkovGraphsContainer
        File input_path = args.getFileParam(ArgumentsParser.PARAM_MARKOV);
        Map<Integer, MarkovGraphsContainer> m = MarkovUtil.load(args.catalogContext, input_path);
       
        // Blah blah blah...
        Map<Integer, MarkovEstimator> t_estimators = new HashMap<Integer, MarkovEstimator>();
        for (Integer id : m.keySet()) {
            t_estimators.put(id, new MarkovEstimator(args.catalogContext, p_estimator, m.get(id)));
        } // FOR
       
        final Set<String> skip = new HashSet<String>();
       
        Map<Procedure, AtomicInteger> totals = new TreeMap<Procedure, AtomicInteger>();
        Map<Procedure, AtomicInteger> correct_partitions_txns = new HashMap<Procedure, AtomicInteger>();
        Map<Procedure, AtomicInteger> correct_path_txns = new HashMap<Procedure, AtomicInteger>();
        Map<Procedure, AtomicInteger> multip_txns = new HashMap<Procedure, AtomicInteger>();
        for (Procedure catalog_proc : args.catalog_db.getProcedures()) {
            if (!catalog_proc.getSystemproc()) {
                totals.put(catalog_proc, new AtomicInteger(0));
                correct_partitions_txns.put(catalog_proc, new AtomicInteger(0));
                correct_path_txns.put(catalog_proc, new AtomicInteger(0));
                multip_txns.put(catalog_proc, new AtomicInteger(0));
            }
        } // FOR
       
        // Loop through each of the procedures and measure how accurate we are in our predictions
        for (TransactionTrace xact : args.workload.getTransactions()) {
            LOG.info(xact.debug(args.catalog_db));
           
            Procedure catalog_proc = xact.getCatalogItem(args.catalog_db);
            if (skip.contains(catalog_proc.getName())) continue;
           
            int partition = HStoreConstants.NULL_PARTITION_ID;
            try {
                partition = p_estimator.getBasePartition(catalog_proc, xact.getParams(), true);
            } catch (Exception ex) {
                ex.printStackTrace();
                assert(false);
            }
            assert(partition >= 0);
View Full Code Here

        LOG.info(String.format("Compressing workload based on %d partitions%s",
                 args.catalogContext.numberOfPartitions,
                 (intervals != null ? " over " + intervals + " intervals" : "")));
        LOG.info("BEFORE:\n" + args.workload.getProcedureHistogram());
       
        PartitionEstimator p_estimator = new PartitionEstimator(args.catalogContext);
        WorkloadSummarizer ws = new WorkloadSummarizer(args.catalog_db, p_estimator, args.param_mappings);
        if (intervals != null) ws.setIntervals(intervals);
        Workload new_workload = ws.process(args.workload);
        assert(new_workload != null);
        LOG.info("AFTER:\n" + new_workload.getProcedureHistogram());
View Full Code Here

    /**
     * Constructor
     */
    @SuppressWarnings("unchecked")
    public TimeIntervalCostModel(CatalogContext catalogContext, Class<? extends T> inner_class, int num_intervals) {
        super(TimeIntervalCostModel.class, catalogContext, new PartitionEstimator(catalogContext));
        this.num_intervals = num_intervals;
        this.cost_models = (T[]) (new AbstractCostModel[num_intervals]);

        try {
            Constructor<?> constructor = ClassUtil.getConstructor(inner_class, Database.class, PartitionEstimator.class);
View Full Code Here

    public FeatureClusterer(CatalogContext catalogContext, Procedure catalog_proc, Workload workload, PartitionSet all_partitions, int num_threads) {
        this.catalogContext = catalogContext;
        this.catalog_proc = catalog_proc;
        this.workload = workload;
        this.thresholds = new EstimationThresholds(); // FIXME
        this.p_estimator = new PartitionEstimator(catalogContext);
        this.all_partitions = all_partitions;
        this.total_num_partitions = catalogContext.numberOfPartitions;
        this.calculate_threadPool = Executors.newFixedThreadPool(num_threads);
       
        for (SplitType type : SplitType.values()) {
View Full Code Here

        assert(data != null);
        assert(args.workload.getTransactionCount() == data.numInstances());
       
        PartitionSet partitions = null;
        if (args.hasParam(ArgumentsParser.PARAM_WORKLOAD_RANDOM_PARTITIONS)) {
            PartitionEstimator p_estimator = new PartitionEstimator(args.catalogContext);
            final ObjectHistogram<Integer> h = new ObjectHistogram<Integer>();
            for (TransactionTrace txn_trace : args.workload.getTransactions()) {
                int base_partition = p_estimator.getBasePartition(txn_trace);
                h.put(base_partition);
            } // FOR
//            System.err.println("# OF PARTITIONS: " + h.getValueCount());
//            h.setKeepZeroEntries(true);
//            for (Integer p : CatalogUtil.getAllPartitionIds(args.catalog_db)) {
View Full Code Here

TOP

Related Classes of edu.brown.utils.PartitionEstimator

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.