Package edu.brown.statistics

Examples of edu.brown.statistics.TableStatistics


                }
                LOG.fatal(msg);
                return;
            }
            final String table_key = CatalogKey.createKey(catalog_tbl);
            TableStatistics table_stats = this.stats.getTableStatistics(table_key);
           
            // Temporary Loader Procedure
            TransactionTrace loader_xact = new TransactionTrace(xact_id, catalog_proc, new Object[0]) {
                /**
                 * We need this wrapper so that when CatalogUtil tries to figure out what
                 * the index of partitioning parameter we can just use the column index of partitioning
                 * attribute of the table that we are inserting into
                 */
                private final Procedure proc = new Procedure() {
                    @Override
                    public int getPartitionparameter() {
                        return (catalog_tbl.getPartitioncolumn().getIndex());
                    }
                    @Override
                    public Catalog getCatalog() {
                        return (catalog);
                    }
                };
                @Override
                public Procedure getCatalogItem(Database catalog_db) {
                    return (proc);
                }
            }; // Nasty...
           
            // Use a temporary query trace to wrap the "insert" of each tuple
            Statement catalog_stmt = this.stats_load_stmts.get(table_key);
            if (catalog_stmt == null) {
                catalog_stmt = catalog_proc.getStatements().add("INSERT_" + catalog_tbl.getName());
                catalog_stmt.setQuerytype(QueryType.INSERT.getValue());
                this.stats_load_stmts.put(table_key, catalog_stmt);

                // TERRIBLE HACK!
                // 2011-01-25 :: Why do I need to do this??
//                String stmt_key = CatalogKey.createKey(catalog_stmt);
//                CatalogUtil.CACHE_STATEMENT_COLUMNS_KEYS.put(stmt_key, new java.util.HashSet<String>());
//                CatalogUtil.CACHE_STATEMENT_COLUMNS_KEYS.get(stmt_key).add(table_key);
            }
            QueryTrace loader_query = new QueryTrace(catalog_stmt, new Object[0], 0);
            loader_xact.addQuery(loader_query);
           
            // Gather column types
            int num_columns = voltTable.getColumnCount();
            VoltType col_types[] = new VoltType[num_columns];
            for (int i = 0; i < num_columns; i++) {
                Column catalog_col = catalog_tbl.getColumns().get(i);
                col_types[i] = VoltType.get((byte)catalog_col.getType());
            } // FOR

            loader_query.params = new Object[num_columns];
            int num_tuples = voltTable.getRowCount();
            try {
                LOG.info("Processing " + num_tuples + " (sample=10) tuples for statistics on " + catalog_tbl.getName());
                boolean show_progress = (num_tuples > 25000);
                for (int i = 0; i < num_tuples; i += 10) {
                    if (i >= num_tuples) break;
                    VoltTableRow tuple = voltTable.fetchRow(i);
                    for (int j = 0; j < num_columns; j++) {
                        loader_query.params[j] = tuple.get(j, col_types[j]);
                    } // FOR
                    table_stats.process(stats_catalog_db, loader_xact);
                    if (show_progress && i > 0 && i % 10000 == 0) LOG.info(i + " tuples");
//                        if (i > 25000) break;
                } // FOR
                LOG.info("Processing finished for " + catalog_tbl.getName());
            } catch (Exception ex) {
View Full Code Here


            Collection<Column> columns = PartitionerUtil.generateColumnOrder(info, this.agraph, catalog_tbl, hints, false, true);
            assert (!columns.isEmpty()) : "No potential partitioning columns selected for " + catalog_tbl;
            this.orig_table_attributes.put(catalog_tbl, (ListOrderedSet<Column>) CollectionUtil.addAll(new ListOrderedSet<Column>(), columns));

            // Table Size (when the table is and is not replicated)
            TableStatistics ts = info.stats.getTableStatistics(catalog_tbl);
            this.table_nonreplicated_size.put(catalog_tbl, Math.round(ts.tuple_size_total / (double) this.num_partitions));
            this.table_replicated_size.put(catalog_tbl, ts.tuple_size_total);

            if (trace.val)
                LOG.trace(catalog_tbl.getName() + ": " + columns);
View Full Code Here

        // everything with replicating them first
        if (hints.force_replication_size_limit != null) {
            final Map<Table, Double> replication_weights = new HashMap<Table, Double>();
            final TreeSet<Table> temp_list = new TreeSet<Table>(new PartitionerUtil.CatalogWeightComparator<Table>(replication_weights));
            for (Table catalog_tbl : info.catalogContext.database.getTables()) {
                TableStatistics ts = info.stats.getTableStatistics(catalog_tbl);
                assert (ts != null);
                double size_ratio = ts.tuple_size_total / (double) hints.max_memory_per_partition;
                if (ts.readonly && size_ratio <= hints.force_replication_size_limit) {
                    if (debug.val)
                        LOG.debug(CatalogUtil.getDisplayName(catalog_tbl) + " is read-only and only " + String.format("%.02f", (size_ratio * 100)) + "% of total memory. Forcing replication...");
View Full Code Here

     * @param partition_col
     * @param partition
     * @return
     */
    public long estimate(Table catalog_tbl, Column partition_col, int partition) {
        TableStatistics table_stats = this.stats.getTableStatistics(catalog_tbl);
        ColumnStatistics col_stats = table_stats.getColumnStatistics(partition_col);
        String col_key = CatalogKey.createKey(partition_col);

        ObjectHistogram<Integer> h = this.cache_table_partition.get(col_key);
        if (h == null) {
            h = new ObjectHistogram<Integer>();
View Full Code Here

        long estimate = 0;

        // For now we'll just estimate the table to be based on the maximum
        // number of
        // tuples for all possible partitions
        TableStatistics table_stats = this.stats.getTableStatistics(catalog_tbl);
        assert (table_stats != null) : "Missing statistics for " + catalog_tbl;
        if (debug.val && table_stats.tuple_size_total == 0) {
            LOG.warn(this.stats.debug(CatalogUtil.getDatabase(catalog_tbl)));
        }
        // assert(table_stats.tuple_size_total != 0) : "Size estimate for " +
View Full Code Here

        //
        // TOP: Stats Viewers
        //
        TableStatsPanel firstPanel = null;
        for (String table_key : this.args.stats.table_stats.keySet()) {
            TableStatistics stats = this.args.stats.getTableStatistics(table_key);
            TableStatsPanel statsPanel = new TableStatsPanel(stats);
            this.tableStatsPanels.put(table_key, statsPanel);
            if (firstPanel == null) firstPanel = statsPanel;
        } // FOR
       
View Full Code Here

            assert(catalogContext != null);
            final Table catalog_tbl = catalogContext.getTableByName(tableName);
            assert(catalog_tbl != null) : "Invalid table name '" + tableName + "'";
           
            synchronized (m_tableStatsData) {
                TableStatistics stats = m_tableStatsData.get(catalog_tbl);
                if (stats == null) {
                    stats = new TableStatistics(catalog_tbl);
                    stats.preprocess(catalogContext.database);
                    m_tableStatsData.put(catalog_tbl, stats);
                }
                vt.resetRowPosition();
                while (vt.advanceRow()) {
                    VoltTableRow row = vt.getRow();
                    stats.process(catalogContext.database, row);
                } // WHILE
            } // SYNCH
        }
        return (cr);
    }
View Full Code Here

                        if (this.cost_model.isCachingEnabled()) {
                            if (trace.val)
                                LOG.trace("Invalidating VerticalPartition Statements in cost model: " + vp_col.getOptimizedQueries());
                            this.cost_model.invalidateCache(vp_col.getOptimizedQueries());
                        }
                        TableStatistics tstats = VerticalPartitionerUtil.computeTableStatistics(vp_col, info.stats);
                        assert (tstats != null);
                        // Add the vp's sys table to the list of tables that we
                        // need to estimate the memory
                        assert (catalog_view.getDest() != null) : "Missing parent table for " + catalog_view.fullName();
                        assert (this.current_vertical_partitions.contains(catalog_view.getDest()) == false) : vp_col;
View Full Code Here

     */
    public static TableStatistics computeTableStatistics(VerticalPartitionColumn vp_col, WorkloadStatistics stats) {
        MaterializedViewInfo catalog_view = vp_col.createMaterializedView();
        Table view_tbl = catalog_view.getDest();
        assert (view_tbl != null) : "Destination table for " + catalog_view + " is null?";
        TableStatistics tbl_stats = stats.getTableStatistics(view_tbl);
        if (tbl_stats == null) {
            tbl_stats = new TableStatistics(view_tbl);
            stats.addTableStatistics(view_tbl, tbl_stats);

            Table orig_tbl = catalog_view.getParent();
            TableStatistics orig_tbl_stats = stats.getTableStatistics(orig_tbl);
            assert (orig_tbl_stats != null) : "Missing TableStatistics " + orig_tbl;

            tbl_stats.readonly = true;
            tbl_stats.tuple_count_total = orig_tbl_stats.tuple_count_total;

View Full Code Here

            String table_key = CatalogKey.createKey(catalog_tbl);
            TableEntry pentry = null;
            Column col = null;
            Collection<Column> pkey_columns = CatalogUtil.getPrimaryKeyColumns(catalog_tbl);

            TableStatistics ts = info.stats.getTableStatistics(catalog_tbl);
            assert (ts != null) : "Null TableStatistics for " + catalog_tbl;
            double size_ratio = (calculate_memory ? (ts.tuple_size_total / (double) hints.max_memory_per_partition) : 0);

            // Replication
            if (hints.force_replication.contains(table_key) || (calculate_memory && ts.readonly && size_ratio <= hints.force_replication_size_limit) || pkey_columns.isEmpty()) {
View Full Code Here

TOP

Related Classes of edu.brown.statistics.TableStatistics

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.