Package edu.brown.hstore.estimators.markov

Examples of edu.brown.hstore.estimators.markov.MarkovEstimate


        this.addPartitions(NUM_PARTITIONS);
       
        this.catalog_proc = this.getProcedure(TARGET_PROCEDURE);
        this.markov = new MarkovGraph(this.catalog_proc).initialize();
       
        this.est = new MarkovEstimate(catalogContext);
        assertFalse(this.est.isValid());
    }
View Full Code Here


           
        // Even though it won't correctly identify the exact path that we're going to
        // take (i.e., what partitions the getStockInfo queries will need), the path
        // should be complete (i.e., we should see each Statement in NewOrder executed
        // at least once)
        MarkovEstimate initialEst = state.getInitialEstimate();
//        System.err.println("FIRST ESTIMATE:\n" + initialEst);
        List<MarkovVertex> initialPath = initialEst.getMarkovPath();
        assertFalse(initialPath.isEmpty());
        Set<Statement> seenStmts = new HashSet<Statement>();
        for (MarkovVertex v : initialPath) {
            Statement stmt = v.getCatalogItem();
            if ((stmt instanceof StatementWrapper) == false) {
View Full Code Here

     */
    @Test
    public void testMultipleStartTransaction() throws Exception {
        Set<MarkovEstimatorState> all_states = new HashSet<MarkovEstimatorState>();
        int expected = 20;
        MarkovEstimate initialEst = null;
        for (int i = 0; i < expected; i++) {
            MarkovEstimatorState state = t_estimator.startTransaction(XACT_ID.getAndIncrement(),
                                                                      this.catalog_proc,
                                                                      multip_trace.getParams());
            assertNotNull(state);
            assertFalse(all_states.contains(state));
            all_states.add(state);
           
            if (initialEst == null) {
                initialEst = state.getInitialEstimate();
                // System.err.println("FIRST ESTIMATE:\n" + initialEst);
            } else {
                MarkovEstimate est = state.getInitialEstimate();
                // System.err.printf("ESTIMATE #%02d:\n%s", i, est);
                assertEquals(initialEst.getTouchedPartitions(thresholds), est.getTouchedPartitions(thresholds));
            }
           
        } // FOR
        assertEquals(expected, all_states.size());
    }
View Full Code Here

        long txn_id = XACT_ID.getAndIncrement();
        MarkovEstimatorState state = t_estimator.startTransaction(txn_id, this.catalog_proc, singlep_trace.getParams());
        assertNotNull(state);
        assertNotNull(state.getLastEstimate());
       
        MarkovEstimate est = state.getInitialEstimate();
        System.err.println(est);
        assertNotNull(est);
        assertTrue(est.toString(), est.isInitialized());
//        assertTrue(est.toString(), est.isSinglePartitionProbabilitySet());
        assertTrue(est.toString(), est.isAbortProbabilitySet());
        assertTrue(est.toString(), est.isConfidenceCoefficientSet());
        // assertTrue(est.toString(), est.getConfidenceCoefficient() >= 0f);
//        assertEquals(est.toString(), 1.0f, est.getSinglePartitionProbability());
        assertEquals(est.toString(), 1.0f, est.getConfidenceCoefficient());

        //        System.err.println(est.toString());
       
        MarkovGraph markov = state.getMarkovGraph();
        List<MarkovVertex> initial_path = est.getMarkovPath();
        assertNotNull(initial_path);
        assertFalse(initial_path.isEmpty());
       
        System.err.println("# of Vertices: " + markov.getVertexCount());
        System.err.println("# of Edges:    " + markov.getEdgeCount());
        System.err.println("Confidence:    " + String.format("%.4f", est.getConfidenceCoefficient()));
        System.err.println("\nINITIAL PATH:\n" + StringUtil.join("\n", initial_path));
//        System.err.println(multip_trace.debug(catalog_db));

        PartitionSet partitions = new PartitionSet();
        p_estimator.getAllPartitions(partitions, singlep_trace);
        assertNotNull(partitions);
//        assert(partitions.size() > 1) : partitions;
        System.err.println("partitions: " + partitions);
       
//        GraphvizExport<Vertex, Edge> gv = MarkovUtil.exportGraphviz(markov, false, null);
//        gv.highlightPath(markov.getPath(initial_path), "blue");
//        gv.writeToTempFile(this.catalog_proc, 0);
//
//        MarkovUtil.exportGraphviz(markov, false, markov.getPath(multip_path)).writeToTempFile(this.catalog_proc, 1);
       
        Collection<Integer> est_partitions = est.getTouchedPartitions(thresholds);
        assertNotNull(est_partitions);
        assertEquals(partitions.size(), est_partitions.size());
        assertEquals(partitions, est_partitions);
       
        assert(est.isSinglePartitioned(this.thresholds));
        assertTrue(est.isAbortable(this.thresholds));
       
        for (int partition : catalogContext.getAllPartitionIds()) {
            if (partitions.contains(partition)) { //  == BASE_PARTITION) {
                assertFalse("isFinishedPartition(" + partition + ")", est.isDonePartition(thresholds, partition));
                assertTrue("isWritePartition(" + partition + ")", est.isWritePartition(thresholds, partition));
                assertTrue("isTargetPartition(" + partition + ")", est.isTargetPartition(thresholds, partition));
            } else {
                assertTrue("isFinishedPartition(" + partition + ")", est.isDonePartition(thresholds, partition));
                assertFalse("isWritePartition(" + partition + ")", est.isWritePartition(thresholds, partition));
                assertFalse("isTargetPartition(" + partition + ")", est.isTargetPartition(thresholds, partition));
            }
        } // FOR
    }
View Full Code Here

        long txn_id = XACT_ID.getAndIncrement();
        MarkovEstimatorState state = t_estimator.startTransaction(txn_id, this.catalog_proc, txn_trace.getParams());
        assertNotNull(state);
        assertNotNull(state.getLastEstimate());
       
        MarkovEstimate initialEst = state.getInitialEstimate();
        assertNotNull(initialEst);
        assertTrue(initialEst.toString(), initialEst.isInitialized());
//        assertTrue(initialEst.toString(), initialEst.isSinglePartitionProbabilitySet());
        assertTrue(initialEst.toString(), initialEst.isAbortProbabilitySet());
//        assertTrue(initialEst.toString(), initialEst.getSinglePartitionProbability() < 1.0f);
        assertTrue(initialEst.toString(), initialEst.isConfidenceCoefficientSet());
        assertTrue(initialEst.toString(), initialEst.getConfidenceCoefficient() >= 0f);
        assertTrue(initialEst.toString(), initialEst.getConfidenceCoefficient() <= 1f);
        assertFalse(initialEst.toString(), initialEst.isSinglePartitioned(this.thresholds));
        assertTrue(initialEst.toString(), initialEst.isAbortable(this.thresholds));
       
        MarkovGraph markov = state.getMarkovGraph();
        List<MarkovVertex> initial_path = initialEst.getMarkovPath();
        assertNotNull(initial_path);
        assertFalse(initial_path.isEmpty());
       
        System.err.println("# of Vertices: " + markov.getVertexCount());
        System.err.println("# of Edges:    " + markov.getEdgeCount());
        System.err.println("Confidence:    " + String.format("%.4f", initialEst.getConfidenceCoefficient()));
        System.err.println("\nINITIAL PATH:\n" + StringUtil.join("\n", initial_path));

        PartitionSet partitions = new PartitionSet();
        p_estimator.getAllPartitions(partitions, txn_trace);
        assertNotNull(partitions);
        assert(partitions.size() > 1) : partitions;
       
        Collection<Integer> est_partitions = initialEst.getTouchedPartitions(thresholds);
        assertNotNull(est_partitions);
        assertEquals(partitions.size(), est_partitions.size());
        assertEquals(partitions, est_partitions);
    }
View Full Code Here

        long txn_id = XACT_ID.getAndIncrement();
        MarkovEstimatorState state = t_estimator.startTransaction(txn_id, this.catalog_proc, txn_trace.getParams());
        assertNotNull(state);
        assertNotNull(state.getLastEstimate());
       
        MarkovEstimate initialEst = state.getInitialEstimate();
        assertNotNull(initialEst);
        assertTrue(initialEst.toString(), initialEst.isInitialized());
//        assertTrue(initialEst.toString(), initialEst.isSinglePartitionProbabilitySet());
        assertTrue(initialEst.toString(), initialEst.isAbortProbabilitySet());
//        assertTrue(initialEst.toString(), initialEst.getSinglePartitionProbability() < 1.0f);
        assertTrue(initialEst.toString(), initialEst.isConfidenceCoefficientSet());
        assertTrue(initialEst.toString(), initialEst.getConfidenceCoefficient() >= 0f);
        assertTrue(initialEst.toString(), initialEst.getConfidenceCoefficient() <= 1f);

        // Get the list of partitions that we're going to touch in the beginning
        // We should never mark these as finished in subsequent estimates
        PartitionSet touched = initialEst.getTouchedPartitions(thresholds);
        assertFalse(touched.isEmpty());
        assertFalse(touched.contains(HStoreConstants.NULL_PARTITION_ID));
        System.err.println("TOUCHED: " + touched);
        assertFalse(touched.toString(), touched.size() == 1);
       
        // Execute a bunch of batches
        // All should say that the txn is not finished with the partitions until we
        // get to the one that contains the updateStock queries, which should be the last.
        Statement lastBatchStmt = this.getStatement(this.catalog_proc, "updateStock");
        for (int i = 0, cnt = txn_trace.getBatchCount(); i < cnt; i++) {
            List<QueryTrace> queries = txn_trace.getBatchQueries(i);
            assertFalse(queries.isEmpty());
            boolean is_last = (i+1 == cnt);
            Statement stmts[] = new Statement[queries.size()];
            PartitionSet partitions[] = new PartitionSet[queries.size()];
           
            boolean found = false;
            int idx = 0;
            for (QueryTrace q : queries) {
                stmts[idx] = q.getCatalogItem(catalogContext.database);
                assertNotNull(stmts[idx]);
                partitions[idx] = new PartitionSet();
                p_estimator.getAllPartitions(partitions[idx], q, state.getBasePartition());
                assertFalse(partitions[idx].isEmpty());
                assertFalse(partitions[idx].contains(HStoreConstants.NULL_PARTITION_ID));
               
                found = found || stmts[idx].equals(lastBatchStmt);
                idx++;
            } // FOR
            if (is_last) assertTrue(StringUtil.join("\n", queries), found);
               
            MarkovEstimate est = t_estimator.executeQueries(state, stmts, partitions);
            assertNotNull(est);
           
            for (int partition : catalogContext.getAllPartitionIds()) {
                String debug = String.format("Batch %02d / Partition %02d / isLast=%s\n%s",
                                             est.getBatchId(), partition, is_last, est.toString());
                assertTrue(debug, est.isDoneProbabilitySet(partition));
                assertTrue(debug, est.isWriteProbabilitySet(partition));
//                assertTrue(debug, est.isReadOnlyProbabilitySet(partition));
               
                if (touched.contains(partition) && is_last == false) { //  || partition == state.getBasePartition())) {
                    assertFalse(debug, est.isDonePartition(thresholds, partition));
                    assertTrue(debug, est.isWritePartition(thresholds, partition));
                } else {
                    assertTrue(debug, est.isDonePartition(thresholds, partition));
                }
            } // FOR
        } // FOR
    }
View Full Code Here

            p_estimator.getAllPartitions(partitions[idx], q, base_partition);
            idx++;
        } // FOR
       
        MarkovEstimatorState states[] = new MarkovEstimatorState[2];
        MarkovEstimate ests[] = new MarkovEstimate[states.length];
        for (int i = 0; i < states.length; i++) {
            HStoreConf.singleton().site.markov_fast_path = (i != 0);
            states[i] = t_estimator.startTransaction(XACT_ID.getAndIncrement(), this.catalog_proc, txn_trace.getParams());
            assertNotNull(states[i]);
            assertNotNull(states[i].getLastEstimate());
View Full Code Here

        TransactionTrace txn_trace = singlep_trace;
        assertNotNull(txn_trace);
        MarkovEstimatorState s = this.t_estimator.processTransactionTrace(txn_trace);
        assertNotNull(s);
       
        MarkovEstimate initialEst = s.getInitialEstimate();
        assertNotNull(initialEst);
        assertTrue(initialEst.toString(), initialEst.isInitialized());
//        assertTrue(initialEst.toString(), initialEst.isSinglePartitionProbabilitySet());
        assertTrue(initialEst.toString(), initialEst.isAbortProbabilitySet());
//        assertTrue(initialEst.toString(), initialEst.getSinglePartitionProbability() < 1.0f);
        assertTrue(initialEst.toString(), initialEst.isConfidenceCoefficientSet());
        assertTrue(initialEst.toString(), initialEst.getConfidenceCoefficient() >= 0f);
        assertTrue(initialEst.toString(), initialEst.getConfidenceCoefficient() <= 1f);
        assertTrue(initialEst.toString(), initialEst.getMarkovPath().isEmpty() == false);
       
        // We should have an MarkovEstimate for each batch
        assertEquals(txn_trace.getBatchCount(), s.getEstimateCount());
        List<Estimate> estimates = s.getEstimates();
        for (int i = 0, cnt = txn_trace.getBatchCount(); i < cnt; i++) {
            List<QueryTrace> queries = txn_trace.getBatchQueries(i);
            assertFalse(queries.isEmpty());
           
            MarkovEstimate est = (MarkovEstimate)estimates.get(i);
            assertNotSame(initialEst, est);
            assertNotNull(est);
//            assertTrue(est.toString(), est.isSinglePartitionProbabilitySet());
            assertTrue(est.toString(), est.isAbortProbabilitySet());
            assertTrue(est.toString(), est.isSinglePartitioned(thresholds));
            assertTrue(est.toString(), est.isConfidenceCoefficientSet());
            assertTrue(est.toString(), est.getConfidenceCoefficient() >= 0f);
            assertTrue(est.toString(), est.getConfidenceCoefficient() <= 1f);
            assertTrue(est.toString(), est.getMarkovPath().isEmpty() == false);
           
            // The last vertex in each MarkovEstimate should correspond to the last query in each batch
            MarkovVertex last_v = est.getVertex();
            assertNotNull(last_v);
            System.err.println("LAST VERTEX: " + last_v);
            assertEquals(CollectionUtil.last(queries).getCatalogItem(catalogContext.database), last_v.getCatalogItem());
        } // FOR
    }
View Full Code Here

        // ESTIMATOR STATE
        sb.append("\nESTIMATOR STATE:\n");
        EstimatorState s = ts.getEstimatorState();
        if (s instanceof MarkovEstimatorState) {
            MarkovGraph markov = ((MarkovEstimatorState)s).getMarkovGraph();
            MarkovEstimate initialEst = s.getInitialEstimate();
            List<MarkovEdge> initialPath = markov.getPath(initialEst.getMarkovPath());
            List<MarkovVertex> actualPath = ((MarkovEstimatorState)s).getActualPath();
           
            sb.append(s.toString());
            try {
                GraphvizExport<MarkovVertex, MarkovEdge> gv = MarkovUtil.exportGraphviz(markov, true, initialPath);
View Full Code Here

        double cost = 0.0d;

        this.e_read_partitions.clear();
        this.e_write_partitions.clear();
        MarkovEstimate initialEst = s.getInitialEstimate();
        assert(initialEst != null);
        assert(initialEst.isInitialized());
        for (Integer partition : initialEst.getTouchedPartitions(this.thresholds)) {
            if (initialEst.isDonePartition(this.thresholds, partition.intValue()) == false) {
                for (MarkovVertex v : initialEst.getMarkovPath()) {
                    if (v.getPartitions().contains(partition) == false)
                        continue;
                    if (((Statement) v.getCatalogItem()).getReadonly()) {
                        this.e_read_partitions.add(partition);
                    } else {
                        this.e_write_partitions.add(partition);
                    }
                } // FOR
            }
        } // FOR

        List<MarkovVertex> initialPath = initialEst.getMarkovPath();
        List<MarkovVertex> actualPath = s.getActualPath();
        this.a_read_partitions.clear();
        this.a_write_partitions.clear();
        MarkovUtil.getReadWritePartitions(actualPath, this.a_read_partitions, this.a_write_partitions);

        // Try fast version
        try {
            if (this.force_full_comparison || !this.comparePathsFast(CollectionUtil.last(initialPath), actualPath)) {
                // Otherwise we have to do the full path comparison to figure
                // out just how wrong we are
                cost = this.comparePathsFull(s);
                this.full_path_counter.put(catalog_proc);
            } else {
                this.fast_path_counter.put(catalog_proc);
            }
        } catch (Throwable ex) {
            System.err.println(txn_trace.debug(catalogContext.database));
            System.err.println("COST = " + cost);
            System.err.println("BASE PARTITION = " + s.getBasePartition());
            System.err.println("PENALTIES = " + this.penalties);
            System.err.println("ESTIMATED PARTITIONS: " + this.e_all_partitions);
            System.err.println("ACTUAL PARTITIONS: " + this.a_all_partitions);
            System.err.println("MARKOV GRAPH: " + MarkovUtil.exportGraphviz(s.getMarkovGraph(), true, null).writeToTempFile(catalog_proc));
            System.err.println();

            String e_path = "ESTIMATED PATH:\n" + StringUtil.join("\n", initialEst.getMarkovPath());
            String a_path = "ACTUAL PATH:\n" + StringUtil.join("\n", s.getActualPath());
            System.err.println(StringUtil.columns(e_path, a_path));
            System.err.println("MARKOV ESTIMATE:\n" + s.getInitialEstimate());

            throw new RuntimeException(ex);
View Full Code Here

TOP

Related Classes of edu.brown.hstore.estimators.markov.MarkovEstimate

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.