Package org.apache.hadoop.hive.metastore.api

Examples of org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc


      String dbName = null;
      String tableName = null;
      String partName = null;
      String colName = null;

      ColumnStatisticsDesc statsDesc = colStats.getStatsDesc();
      dbName = statsDesc.getDbName().toLowerCase();
      tableName = statsDesc.getTableName().toLowerCase();
      partName = lowerCaseConvertPartName(statsDesc.getPartName());

      statsDesc.setDbName(dbName);
      statsDesc.setTableName(tableName);
      statsDesc.setPartName(partName);

      long time = System.currentTimeMillis() / 1000;
      statsDesc.setLastAnalyzed(time);

      List<ColumnStatisticsObj> statsObjs =  colStats.getStatsObj();

      for (ColumnStatisticsObj statsObj:statsObjs) {
        colName = statsObj.getColName().toLowerCase();
View Full Code Here


    boolean committed = false;

    try {
    openTransaction();
    List<ColumnStatisticsObj> statsObjs = colStats.getStatsObj();
    ColumnStatisticsDesc statsDesc = colStats.getStatsDesc();
    Table table = ensureGetTable(statsDesc.getDbName(), statsDesc.getTableName());
    Partition partition = convertToPart(getMPartition(
        statsDesc.getDbName(), statsDesc.getTableName(), partVals));
    for (ColumnStatisticsObj statsObj:statsObjs) {
      // We have to get partition again because DataNucleus
      MPartition mPartition = getMPartition(
          statsDesc.getDbName(), statsDesc.getTableName(), partVals);
      if (partition == null) {
        throw new NoSuchObjectException("Partition for which stats is gathered doesn't exist.");
      }
      MPartitionColumnStatistics mStatsObj =
          StatObjectConverter.convertToMPartitionColumnStatistics(mPartition, statsDesc, statsObj);
View Full Code Here

          GetHelper<ColumnStatistics> ctx) throws MetaException, NoSuchObjectException {
        List<MTableColumnStatistics> mStats = getMTableColumnStatistics(getTable(), colNames);
        if (mStats.isEmpty()) return null;
        // LastAnalyzed is stored per column, but thrift object has it per multiple columns.
        // Luckily, nobody actually uses it, so we will set to lowest value of all columns for now.
        ColumnStatisticsDesc desc = StatObjectConverter.getTableColumnStatisticsDesc(mStats.get(0));
        List<ColumnStatisticsObj> statObjs = new ArrayList<ColumnStatisticsObj>(mStats.size());
        for (MTableColumnStatistics mStat : mStats) {
          if (desc.getLastAnalyzed() > mStat.getLastAnalyzed()) {
            desc.setLastAnalyzed(mStat.getLastAnalyzed());
          }
          statObjs.add(StatObjectConverter.getTableColumnStatisticsObj(mStat));
        }
        return new ColumnStatistics(desc, statObjs);
      }
View Full Code Here

            getMPartitionColumnStatistics(getTable(), partNames, colNames);
        List<ColumnStatistics> result = new ArrayList<ColumnStatistics>(
            Math.min(mStats.size(), partNames.size()));
        String lastPartName = null;
        List<ColumnStatisticsObj> curList = null;
        ColumnStatisticsDesc csd = null;
        for (int i = 0; i <= mStats.size(); ++i) {
          boolean isLast = i == mStats.size();
          MPartitionColumnStatistics mStatsObj = isLast ? null : mStats.get(i);
          String partName = isLast ? null : (String)mStatsObj.getPartitionName();
          if (isLast || !partName.equals(lastPartName)) {
View Full Code Here

    boolean committed = false;

    openTransaction();
    try {
      List<ColumnStatisticsObj> statsObjs = colStats.getStatsObj();
      ColumnStatisticsDesc statsDesc = colStats.getStatsDesc();

      // DataNucleus objects get detached all over the place for no (real) reason.
      // So let's not use them anywhere unless absolutely necessary.
      Table table = ensureGetTable(statsDesc.getDbName(), statsDesc.getTableName());
      for (ColumnStatisticsObj statsObj:statsObjs) {
        // We have to get mtable again because DataNucleus.
        MTableColumnStatistics mStatsObj = StatObjectConverter.convertToMTableColumnStatistics(
            ensureGetMTable(statsDesc.getDbName(), statsDesc.getTableName()), statsDesc, statsObj);
        writeMTableColumnStatistics(table, mStatsObj);
      }
      committed = commitTransaction();
      return committed;
    } finally {
View Full Code Here

      InvalidInputException
    {
      String dbName = null;
      String tableName = null;
      String colName = null;
      ColumnStatisticsDesc statsDesc = colStats.getStatsDesc();
      dbName = statsDesc.getDbName().toLowerCase();
      tableName = statsDesc.getTableName().toLowerCase();

      statsDesc.setDbName(dbName);
      statsDesc.setTableName(tableName);
      long time = System.currentTimeMillis() / 1000;
      statsDesc.setLastAnalyzed(time);

      List<ColumnStatisticsObj> statsObjs =  colStats.getStatsObj();

      for (ColumnStatisticsObj statsObj:statsObjs) {
        colName = statsObj.getColName().toLowerCase();
View Full Code Here

      String dbName = null;
      String tableName = null;
      String partName = null;
      String colName = null;

      ColumnStatisticsDesc statsDesc = colStats.getStatsDesc();
      dbName = statsDesc.getDbName().toLowerCase();
      tableName = statsDesc.getTableName().toLowerCase();
      partName = lowerCaseConvertPartName(statsDesc.getPartName());

      statsDesc.setDbName(dbName);
      statsDesc.setTableName(tableName);
      statsDesc.setPartName(partName);

      long time = System.currentTimeMillis() / 1000;
      statsDesc.setLastAnalyzed(time);

      List<ColumnStatisticsObj> statsObjs =  colStats.getStatsObj();

      for (ColumnStatisticsObj statsObj:statsObjs) {
        colName = statsObj.getColName().toLowerCase();
View Full Code Here

      query.closeAll();
      return null;
    }
    List<Object[]> list = ensureList(qResult);
    if (list.isEmpty()) return null;
    ColumnStatisticsDesc csd = new ColumnStatisticsDesc(true, dbName, tableName);
    ColumnStatistics result = makeColumnStats(list, csd, 0);
    timingTrace(doTrace, queryText, start, queryTime);
    query.closeAll();
    return result;
  }
View Full Code Here

      boolean isLast = i == list.size();
      String partName = isLast ? null : (String)list.get(i)[0];
      if (!isLast && partName.equals(lastPartName)) {
        continue;
      } else if (from != i) {
        ColumnStatisticsDesc csd = new ColumnStatisticsDesc(false, dbName, tableName);
        csd.setPartName(lastPartName);
        result.add(makeColumnStats(list.subList(from, i), csd, 1));
      }
      lastPartName = partName;
      from = i;
    }
View Full Code Here

      String[] colType = new String[] {"double", "string"};
      boolean isTblLevel = true;
      String partName = null;
      List<ColumnStatisticsObj> statsObjs = new ArrayList<ColumnStatisticsObj>();

      ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc();
      statsDesc.setDbName(dbName);
      statsDesc.setTableName(tblName);
      statsDesc.setIsTblLevel(isTblLevel);
      statsDesc.setPartName(partName);

      ColumnStatisticsObj statsObj = new ColumnStatisticsObj();
      statsObj.setColName(colName[0]);
      statsObj.setColType(colType[0]);

      ColumnStatisticsData statsData = new ColumnStatisticsData();
      DoubleColumnStatsData numericStats = new DoubleColumnStatsData();
      statsData.setDoubleStats(numericStats);

      statsData.getDoubleStats().setHighValue(highValue);
      statsData.getDoubleStats().setLowValue(lowValue);
      statsData.getDoubleStats().setNumDVs(numDVs);
      statsData.getDoubleStats().setNumNulls(numNulls);

      statsObj.setStatsData(statsData);
      statsObjs.add(statsObj);

      statsObj = new ColumnStatisticsObj();
      statsObj.setColName(colName[1]);
      statsObj.setColType(colType[1]);

      statsData = new ColumnStatisticsData();
      StringColumnStatsData stringStats = new StringColumnStatsData();
      statsData.setStringStats(stringStats);
      statsData.getStringStats().setAvgColLen(avgColLen);
      statsData.getStringStats().setMaxColLen(maxColLen);
      statsData.getStringStats().setNumDVs(numDVs);
      statsData.getStringStats().setNumNulls(numNulls);

      statsObj.setStatsData(statsData);
      statsObjs.add(statsObj);

      ColumnStatistics colStats = new ColumnStatistics();
      colStats.setStatsDesc(statsDesc);
      colStats.setStatsObj(statsObjs);

      // write stats objs persistently
      client.updateTableColumnStatistics(colStats);

      // retrieve the stats obj that was just written
      ColumnStatistics colStats2 = client.getTableColumnStatistics(dbName, tblName, colName[0]);

     // compare stats obj to ensure what we get is what we wrote
      assertNotNull(colStats2);
      assertEquals(colStats2.getStatsDesc().getDbName(), dbName);
      assertEquals(colStats2.getStatsDesc().getTableName(), tblName);
      assertEquals(colStats2.getStatsObj().get(0).getColName(), colName[0]);
      assertEquals(colStats2.getStatsObj().get(0).getStatsData().getDoubleStats().getLowValue(),
        lowValue);
      assertEquals(colStats2.getStatsObj().get(0).getStatsData().getDoubleStats().getHighValue(),
        highValue);
      assertEquals(colStats2.getStatsObj().get(0).getStatsData().getDoubleStats().getNumNulls(),
        numNulls);
      assertEquals(colStats2.getStatsObj().get(0).getStatsData().getDoubleStats().getNumDVs(),
        numDVs);
      assertEquals(colStats2.getStatsDesc().isIsTblLevel(), isTblLevel);

      // test delete column stats; if no col name is passed all column stats associated with the
      // table is deleted
      boolean status = client.deleteTableColumnStatistics(dbName, tblName, null);
      assertTrue(status);
      // try to query stats for a column for which stats doesn't exist
      try {
        colStats2 = client.getTableColumnStatistics(dbName, tblName, colName[1]);
        assertTrue(true);
      } catch (NoSuchObjectException e) {
        System.out.println("Statistics for column=" + colName[1] + " not found");
      }

      colStats.setStatsDesc(statsDesc);
      colStats.setStatsObj(statsObjs);

      // update table level column stats
      client.updateTableColumnStatistics(colStats);

      // query column stats for column whose stats were updated in the previous call
      colStats2 = client.getTableColumnStatistics(dbName, tblName, colName[0]);

      // partition level column statistics test
      // create a table with multiple partitions
      cleanUp(dbName, tblName, typeName);

      List<List<String>> values = new ArrayList<List<String>>();
      values.add(makeVals("2008-07-01 14:13:12", "14"));
      values.add(makeVals("2008-07-01 14:13:12", "15"));
      values.add(makeVals("2008-07-02 14:13:12", "15"));
      values.add(makeVals("2008-07-03 14:13:12", "151"));

      createMultiPartitionTableSchema(dbName, tblName, typeName, values);

      List<String> partitions = client.listPartitionNames(dbName, tblName, (short)-1);

      partName = partitions.get(0);
      isTblLevel = false;

      // create a new columnstatistics desc to represent partition level column stats
      statsDesc = new ColumnStatisticsDesc();
      statsDesc.setDbName(dbName);
      statsDesc.setTableName(tblName);
      statsDesc.setPartName(partName);
      statsDesc.setIsTblLevel(isTblLevel);

      colStats = new ColumnStatistics();
      colStats.setStatsDesc(statsDesc);
      colStats.setStatsObj(statsObjs);
View Full Code Here

TOP

Related Classes of org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.