Package edu.brown.markov.containers

Examples of edu.brown.markov.containers.MarkovGraphsContainer

166
167
168
169
170
171
172
173
174
175
176
        // ----------------------------------------------------------------------------
        // PartitionExecutor Initialization
        // ----------------------------------------------------------------------------
        boolean first = true;
        for (int local_partition : singleton.getLocalPartitionIds().values()) {
            MarkovGraphsContainer local_markovs = null;
            if (markovs != null) {
                if (markovs.containsKey(MarkovUtil.GLOBAL_MARKOV_CONTAINER_ID)) {
                    local_markovs = markovs.get(MarkovUtil.GLOBAL_MARKOV_CONTAINER_ID);
                } else {
                    local_markovs = markovs.get(local_partition);
View Full Code Here
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
            // because we want divide the MarkovGraphContainers by the base partition
            int base_partition = this.p_estimator.getBasePartition(txn_trace);
            partition_h.put(base_partition);

            // Build up the MarkovGraph for this specific cluster
            MarkovGraphsContainer markovs = state.markovs_per_partition[base_partition];
            MarkovGraph markov = markovs.get(c, this.catalog_proc);
            if (markov == null) {
                markov = markovs.getOrCreate(c, this.catalog_proc).initialize();
                markovs.put(c, markov);
            }
            markov.processTransaction(txn_trace, this.p_estimator);
           
            state.clusters_per_partition[base_partition].put(c);
        } // FOR
View Full Code Here
881
882
883
884
885
886
887
888
889
890
891
892
893
        // cost model estimation.
        final CountDownLatch costmodel_latch = new CountDownLatch(this.all_partitions.size());
        if (trace.val) LOG.trace(String.format("Generating MarkovGraphs for %d partitions", costmodel_latch.getCount()));
       
        for (final int partition : this.all_partitions) {
            final MarkovGraphsContainer markovs = state.markovs_per_partition[partition];
            if (trace.val) LOG.trace(String.format("Calculating Partition #%d probabilities for %d clusters", partition, markovs.size()));
            for (Entry<Integer, Map<Procedure, MarkovGraph>> e : markovs.entrySet()) {
                // if (debug.val) LOG.debug(String.format("Partition %d - Cluster %d", partition, i++));
               
                // Calculate the probabilities for each graph
                for (MarkovGraph markov : e.getValue().values()) {
                    markov.calculateProbabilities(catalogContext.getAllPartitionIds());
View Full Code Here
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
        assert(procedures.size() > 0) : "No procedures";
       
        Map<Integer, MarkovGraphsContainer> m = MarkovGraphsContainerUtil.loadProcedures(args.catalogContext,
                                                                                         input_path, procedures);
        for (Procedure catalog_proc : procedures) {
            MarkovGraphsContainer markovs = null;
           
            if (m.containsKey(MarkovUtil.GLOBAL_MARKOV_CONTAINER_ID)) {
                markovs = m.get(MarkovUtil.GLOBAL_MARKOV_CONTAINER_ID);
            } else {
                assert(args.getOptParamCount() > 1) : "Missing partition argument";
                int partition = args.getIntOptParam(1);
                markovs = m.get(partition);
            }
           
            Map<Integer, MarkovGraph> markov_set = markovs.getAll(catalog_proc);
            if (markov_set == null || markov_set.isEmpty()) {
                System.err.println("Skipping " + catalog_proc + " because there is no Markov graph");
                continue;
            }
           
View Full Code Here
916
917
918
919
920
921
922
923
924
925
926
                    ProfileMeasurement profiler = profilers[thread_id];
                    assert (profiler != null);

                    MarkovCostModel costmodels[] = thread_costmodels[thread_id];
                    for (int p = 0; p < num_partitions; p++) {
                        MarkovGraphsContainer markovs = (global ? thread_markovs[thread_id].get(MarkovUtil.GLOBAL_MARKOV_CONTAINER_ID) : thread_markovs[thread_id].get(p));
                        MarkovEstimator t_estimator = new MarkovEstimator(args.catalogContext, p_estimator, markovs);
                        costmodels[p] = new MarkovCostModel(args.catalogContext, p_estimator, t_estimator, args.thresholds);
                        if (force_fullpath)
                            thread_costmodels[thread_id][p].forceFullPathComparison();
                        if (force_regenerate)
View Full Code Here
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
        Procedure catalog_proc = this.getProcedure(GetNewDestination.class);
       
        // Make a bunch of MarkovGraphsContainers
        Map<Integer, MarkovGraphsContainer> markovs = new HashMap<Integer, MarkovGraphsContainer>();
        for (int i = 1000; i < 1010; i++) {
            MarkovGraphsContainer m = new MarkovGraphsContainer();
            for (int p : catalogContext.getAllPartitionIds()) {
                m.getOrCreate(p, catalog_proc).initialize();
            } // FOR
            markovs.put(i, m);
        } // FOR
       
        // Serialize them out to a file. This will also make a nice little index in the file
        File temp = FileUtil.getTempFile("markovs", true);
        assertNotNull(temp);
        MarkovGraphsContainerUtil.save(markovs, temp);
//        System.err.println("MARKOV FILE: " + temp);
       
        // Now read it back in make sure everything is there
        Map<Integer, MarkovGraphsContainer> clone = MarkovUtil.load(catalogContext, temp);
        assertNotNull(clone);
        assertEquals(markovs.size(), clone.size());
        assert(markovs.keySet().containsAll(clone.keySet()));
        for (Integer id : markovs.keySet()) {
            MarkovGraphsContainer clone_m = clone.get(id);
            assertNotNull(clone_m);
        } // FOR
    }
View Full Code Here
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
        }
    }
   
   
    public void testSerialization() throws Exception {
        MarkovGraphsContainer markovs = new MarkovGraphsContainer();
        for (Integer p : CatalogUtil.getAllPartitionIds(catalog_db)) {
            markovs.getOrCreate(p, catalog_proc, true);
        } // FOR
   
        // Serialize them out to a file. This will also make a nice little index in the file
        tempFile = FileUtil.getTempFile("markovs", false);
        assertNotNull(tempFile);
        markovs.save(tempFile);
        System.err.println("MARKOV FILE: " + tempFile);
   
        // Now read it back in make sure everything is there
        MarkovGraphsContainer clone = new MarkovGraphsContainer();
        clone.load(tempFile, catalog_db);
        assertNotNull(clone);
        assertEquals(markovs.size(), clone.size());
        assert(markovs.keySet().containsAll(clone.keySet()));
        for (Integer id : markovs.keySet()) {
            MarkovGraph clone_m = clone.get(id, catalog_proc);
            assertNotNull(clone_m);
        } // FOR
    }
View Full Code Here
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
                throw new VoltAbortException(msg);
            }
               
            MarkovEstimator t_estimator = (MarkovEstimator)this.executor.getTransactionEstimator();
            assert(t_estimator != null);
            MarkovGraphsContainer markovs = t_estimator.getMarkovGraphsContainer();
           
            if (t_estimator.getMarkovGraphsContainer() != null) {
                boolean is_global = t_estimator.getMarkovGraphsContainer().isGlobal();

                // We will only write out our file if we are the first partition in the list at this site
                if (is_global == false || (is_global == true && isFirstLocalPartition())) {
                   
                    if (debug.val) LOG.debug(String.format("Recalculating MarkovGraph probabilities at partition %d [save=%s, global=%s]",
                                                       this.partitionId, save_to_file, is_global));
                    int ctr = 0;
                    for (MarkovGraph m : markovs.getAll()) {
                        try {
                             m.calculateProbabilities(this.catalogContext.getAllPartitionIds());
                        } catch (Throwable ex) {
                            LOG.fatal(String.format("Failed to recalculate probabilities for %s MarkovGraph #%d: %s", m.getProcedure().getName(), m.getGraphId(), ex.getMessage()));
                            File output = MarkovUtil.exportGraphviz(m, true, false, true, null).writeToTempFile();
                            LOG.fatal("Wrote out invalid MarkovGraph: " + output.getAbsolutePath());
                            this.executor.crash(ex);
                            assert(false) : "I shouldn't have gotten here!";
                        }
                        ctr++;
                    } // FOR
                    if (debug.val) LOG.debug(String.format("Recalculated %d MarkovGraph probabilities at partition %d", ctr, this.partitionId));
                   
                    File f = null;
                    if (save_to_file) {
                        File dir = new File(hstore_conf.global.temp_dir);
                        f = FileUtil.getTempFile(dir, "markovs-" + this.partitionId, true);
                        LOG.info(String.format("Saving updated MarkovGraphs to '" + f + "'"));
                        try {
                            markovs.save(f);
                        } catch (Throwable ex) {
                            throw new RuntimeException("Failed to save MarkovGraphContainer for site " + HStoreThreadManager.formatSiteName(this.executor.getSiteId()), ex);
                        }
                    }
                    result[0].addRow(
View Full Code Here
TOP

Related Classes of edu.brown.markov.containers.MarkovGraphsContainer

  • edu.brown.costmodel.MarkovCostModel
  • edu.brown.hstore.HStore
  • edu.brown.markov.FeatureClusterer
  • edu.brown.markov.MarkovGraph
  • edu.brown.markov.MarkovGraphvizExport
  • edu.brown.markov.TestMarkovGraphsContainer
  • edu.brown.markov.TestMarkovUtil
  • edu.brown.utils.ArgumentsParser
  • org.json.JSONObject
  • org.voltdb.catalog.Procedure
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.