Package org.apache.tajo.catalog.statistics

Examples of org.apache.tajo.catalog.statistics.TableStat


      }
    }
  }

  public TableStat getTableStat() {
    TableStat stat = new TableStat();

    ColumnStat columnStat;
    for (int i = 0; i < schema.getColumnNum(); i++) {
      columnStat = new ColumnStat(schema.getColumn(i));
      columnStat.setNumNulls(numNulls[i]);
      columnStat.setMinValue(minValues.get(i));
      columnStat.setMaxValue(maxValues.get(i));
      stat.addColumnStat(columnStat);
    }

    stat.setNumRows(this.numRows);
    stat.setNumBytes(this.numBytes);

    return stat;
  }
View Full Code Here


      totalSize = sm.calculateSize(path);
    } catch (IOException e) {
      LOG.error("Cannot calculate the size of the relation", e);
    }

    TableStat stat = new TableStat();
    stat.setNumBytes(totalSize);
    meta.setStat(stat);

    TableDesc desc = CatalogUtil.newTableDesc(tableName, meta, path);
    catalog.addTable(desc);
View Full Code Here

    builder.setId(context.getTaskId().getProto());

    if (context.hasResultStats()) {
      builder.setResultStats(context.getResultStats().getProto());
    } else {
      builder.setResultStats(new TableStat().getProto());
    }

    Iterator<Entry<Integer,String>> it = context.getRepartitions();
    if (it.hasNext()) {
      do {
View Full Code Here

                return;
              }

              ResultSetMetaData rsmd = res.getMetaData();
              TableDesc desc = client.getResultDesc(queryId);
              TableStat stat = desc.getMeta().getStat();
              String volume = FileUtil.humanReadableByteCount(stat.getNumBytes(), false);
              long resultRows = stat.getNumRows();
              sout.println("result: " + desc.getPath() + ", " + resultRows + " rows (" + volume + ")");

              int numOfColumns = rsmd.getColumnCount();
              for (int i = 1; i <= numOfColumns; i++) {
                if (i > 1) sout.print(",  ");
View Full Code Here

    String tableName = null;
    Path path = null;
    StoreType storeType = null;
    Options options;
    TableStat stat = null;

    try {
      String sql =
          "SELECT " + C_TABLE_ID + ", path, store_type from " + TB_TABLES
              + " WHERE " + C_TABLE_ID + "='" + name + "'";
      if (LOG.isDebugEnabled()) {
        LOG.debug(sql);
      }
      stmt = getConnection().createStatement();
      res = stmt.executeQuery(sql);
      if (!res.next()) { // there is no table of the given name.
        return null;
      }
      tableName = res.getString(C_TABLE_ID).trim();
      path = new Path(res.getString("path").trim());
      storeType = CatalogUtil.getStoreType(res.getString("store_type").trim());
    } catch (SQLException se) {
      throw new IOException(se);
    } finally {
      CatalogUtil.closeSQLWrapper(res, stmt);
    }

    Schema schema = null;
    try {
      String sql = "SELECT column_name, data_type, type_length from " + TB_COLUMNS
          + " WHERE " + C_TABLE_ID + "='" + name + "' ORDER by column_id asc";

      stmt = getConnection().createStatement();
      if (LOG.isDebugEnabled()) {
        LOG.debug(sql);
      }
      res = stmt.executeQuery(sql);

      schema = new Schema();
      while (res.next()) {
        String columnName = tableName + "."
            + res.getString("column_name").trim();
        Type dataType = getDataType(res.getString("data_type")
            .trim());
        int typeLength = res.getInt("type_length");
        if (typeLength > 0) {
          schema.addColumn(columnName, dataType, typeLength);
        } else {
          schema.addColumn(columnName, dataType);
        }
      }
    } catch (SQLException se) {
      throw new IOException(se);
    } finally {
      CatalogUtil.closeSQLWrapper(res, stmt);
    }

    options = Options.create();
    try {
      String sql = "SELECT key_, value_ from " + TB_OPTIONS
          + " WHERE " + C_TABLE_ID + "='" + name + "'";
      stmt = getConnection().createStatement();
      if (LOG.isDebugEnabled()) {
        LOG.debug(sql);
      }
      res = stmt.executeQuery(sql);

      while (res.next()) {
        options.put(
            res.getString("key_"),
            res.getString("value_"));
      }
    } catch (SQLException se) {
      throw new IOException(se);
    } finally {
      CatalogUtil.closeSQLWrapper(res, stmt);
    }

    try {
      String sql = "SELECT num_rows, num_bytes from " + TB_STATISTICS
          + " WHERE " + C_TABLE_ID + "='" + name + "'";
      if (LOG.isDebugEnabled()) {
        LOG.debug(sql);
      }
      stmt = getConnection().createStatement();
      res = stmt.executeQuery(sql);

      if (res.next()) {
        stat = new TableStat();
        stat.setNumRows(res.getLong("num_rows"));
        stat.setNumBytes(res.getLong("num_bytes"));
      }
    } catch (SQLException se) {
      throw new IOException(se);
    } finally {
      CatalogUtil.closeSQLWrapper(res, stmt);
View Full Code Here

    String tableName = null;
    Path path = null;
    StoreType storeType = null;
    Options options;
    TableStat stat = null;

    try {
      rlock.lock();
      stmt = getConnection().createStatement();

      try {
        String sql =
            "SELECT " + C_TABLE_ID + ", path, store_type from " + TB_TABLES
            + " WHERE " + C_TABLE_ID + "='" + name + "'";
        if (LOG.isDebugEnabled()) {
          LOG.debug(sql);
        }

        res = stmt.executeQuery(sql);
        if (!res.next()) { // there is no table of the given name.
          return null;
        }
        tableName = res.getString(C_TABLE_ID).trim();
        path = new Path(res.getString("path").trim());
        storeType = CatalogUtil.getStoreType(res.getString("store_type").trim());
      } catch (SQLException se) {
        throw new IOException(se);
      } finally {
        CatalogUtil.closeSQLWrapper(res);
      }
     
      Schema schema = null;
      try {
        String sql = "SELECT column_name, data_type, type_length from " + TB_COLUMNS
            + " WHERE " + C_TABLE_ID + "='" + name + "' ORDER by column_id asc";

        if (LOG.isDebugEnabled()) {
          LOG.debug(sql);
        }
        res = stmt.executeQuery(sql);

        schema = new Schema();
        while (res.next()) {
          String columnName = tableName + "."
              + res.getString("column_name").trim();
          Type dataType = getDataType(res.getString("data_type")
              .trim());
          int typeLength = res.getInt("type_length");
          if (typeLength > 0) {
            schema.addColumn(columnName, dataType, typeLength);
          } else {
            schema.addColumn(columnName, dataType);
          }
        }
      } catch (SQLException se) {
        throw new IOException(se);
      } finally {
        CatalogUtil.closeSQLWrapper(res);
      }
     
      options = Options.create();
      try {
        String sql = "SELECT key_, value_ from " + TB_OPTIONS
            + " WHERE " + C_TABLE_ID + "='" + name + "'";

        if (LOG.isDebugEnabled()) {
          LOG.debug(sql);
        }
        res = stmt.executeQuery(sql);       
       
        while (res.next()) {
          options.put(
              res.getString("key_"),
              res.getString("value_"));         
        }
      } catch (SQLException se) {
        throw new IOException(se);
      } finally {
        CatalogUtil.closeSQLWrapper(res);
      }

      try {
        String sql = "SELECT num_rows, num_bytes from " + TB_STATISTICS
            + " WHERE " + C_TABLE_ID + "='" + name + "'";
        if (LOG.isDebugEnabled()) {
          LOG.debug(sql);
        }

        res = stmt.executeQuery(sql);

        if (res.next()) {
          stat = new TableStat();
          stat.setNumRows(res.getLong("num_rows"));
          stat.setNumBytes(res.getLong("num_bytes"));
        }
      } catch (SQLException se) {
        throw new IOException(se);
      } finally {
        CatalogUtil.closeSQLWrapper(res);
View Full Code Here

  public static QueryUnit [] createRangePartitionedTasks(SubQuery subQuery,
                                                         SubQuery childSubQuery, DataChannel channel, int maxNum)
      throws InternalException {
    ExecutionBlock execBlock = subQuery.getBlock();
    TableStat stat = childSubQuery.getTableStat();
    if (stat.getNumRows() == 0) {
      return new QueryUnit[0];
    }

    ScanNode scan = execBlock.getScanNodes()[0];
    Path tablePath;
    tablePath = subQuery.getContext().getStorageManager().getTablePath(scan.getTableName());

    SortNode sortNode = PlannerUtil.findTopNode(childSubQuery.getBlock().getPlan(), NodeType.SORT);
    SortSpec [] sortSpecs = sortNode.getSortKeys();
    Schema sortSchema = new Schema(channel.getPartitionKey());

    // calculate the number of maximum query ranges
    TupleRange mergedRange = TupleUtil.columnStatToRange(channel.getSchema(), sortSchema, stat.getColumnStats());
    RangePartitionAlgorithm partitioner = new UniformRangePartition(sortSchema, mergedRange);
    BigDecimal card = partitioner.getTotalCardinality();

    // if the number of the range cardinality is less than the desired number of tasks,
    // we set the the number of tasks to the number of range cardinality.
View Full Code Here

    List<ExecutionBlock> childBlocks = masterPlan.getChilds(subQuery.getId());
    for (ExecutionBlock childBlock : childBlocks) {
      SubQuery childExecSM = subQuery.getContext().getSubQuery(childBlock.getId());
      tableStats.add(childExecSM.getTableStat());
    }
    TableStat totalStat = StatisticsUtil.computeStatFromUnionBlock(tableStats);

    if (totalStat.getNumRows() == 0) {
      return new QueryUnit[0];
    }

    ScanNode scan = execBlock.getScanNodes()[0];
    Path tablePath;
View Full Code Here

    String tableName = "gettable";
    Options opts = new Options();
    opts.put("file.delimiter", ",");
    TableMeta meta = CatalogUtil.newTableMeta(schema, StoreType.CSV, opts);

    TableStat stat = new TableStat();
    stat.setNumRows(957685);
    stat.setNumBytes(1023234);
    meta.setStat(stat);

    TableDesc desc = new TableDescImpl(tableName, meta, new Path(CommonTestingUtil.getTestDir(), "gettable"));

    store.addTable(desc);
View Full Code Here

        context.addRepartition(partNum, getDataFile(partNum).getName());
      }
    }
   
    // Collect and aggregated statistics data
    TableStat aggregated = StatisticsUtil.aggregateTableStat(statSet);
    context.setResultStats(aggregated);
   
    return null;
  }
View Full Code Here

TOP

Related Classes of org.apache.tajo.catalog.statistics.TableStat

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.