Examples of org.apache.hadoop.hive.ql.stats.StatsAggregator.cleanUp()

Class org.apache.hadoop.hive.ql.stats.StatsAggregator

Examples of org.apache.hadoop.hive.ql.stats.StatsAggregator.cleanUp()

org.apache.hadoop.hive.ql.stats.StatsAggregator.cleanUp()
This method is called after all statistics have been aggregated. Since we support multiple statistics, we do not perform automatic cleanup after aggregation. After this method is called, closeConnection must be called as well. This method is also used to clear the temporary statistics that have been published without being aggregated. Typically this happens when a job fails, or is forcibly stopped after publishing some statistics. @param keyPrefix a prefix of the keys used in StatsPublisher to publish stats. It is the same as the first parameter in aggregateStats(). @return true if cleanup is successful, false otherwise.

  @Override
  protected void tearDown() {
    StatsAggregator sa = StatsFactory.getStatsAggregator();
    assertNotNull(sa);
    assertTrue(sa.connect(conf));
    assertTrue(sa.cleanUp("file_0"));
    assertTrue(sa.closeConnection());
  }


  private void fillStatMap(String numRows, String rawDataSize) {
    stats.clear();

View Full Code Here

      assertEquals("0", rowsX);
      String usizeX = statsAggregator.aggregateStats("file_00002",
          StatsSetupConst.RAW_DATA_SIZE);
      assertEquals("0", usizeX);


      assertTrue(statsAggregator.cleanUp("file_0000"));


      // close connections
      assertTrue(statsPublisher.closeConnection());
      assertTrue(statsAggregator.closeConnection());

View Full Code Here

      assertEquals("1100", rows1);
      String usize1 = statsAggregator.aggregateStats("file_00001",
          StatsSetupConst.RAW_DATA_SIZE);
      assertEquals("11000", usize1);


      assertTrue(statsAggregator.cleanUp("file_0000"));


      // close connections
      assertTrue(statsPublisher.closeConnection());
      assertTrue(statsAggregator.closeConnection());

View Full Code Here

      assertEquals("1000", rows0);
      usize0 = statsAggregator.aggregateStats("file_00000",
          StatsSetupConst.RAW_DATA_SIZE);
      assertEquals("0", usize0);


      assertTrue(statsAggregator.cleanUp("file_0000"));


      // close connections
      assertTrue(statsPublisher.closeConnection());
      assertTrue(statsAggregator.closeConnection());

View Full Code Here

      assertTrue(statsPublisher.publishStat("file_00001_a", stats));
      fillStatMap("500", "4000");
      assertTrue(statsPublisher.publishStat("file_00001_b", stats));


      // cleanUp
      assertTrue(statsAggregator.cleanUp("file_00000"));


      // now clean-up just for one key
      String rows0 = statsAggregator.aggregateStats("file_00000", StatsSetupConst.ROW_COUNT);
      assertEquals("0", rows0);
      String usize0 = statsAggregator.aggregateStats("file_00000",

View Full Code Here

      assertEquals("900", rows1);
      String usize1 = statsAggregator.aggregateStats("file_00001",
          StatsSetupConst.RAW_DATA_SIZE);
      assertEquals("7000", usize1);


      assertTrue(statsAggregator.cleanUp("file_0000"));


      // close connections
      assertTrue(statsPublisher.closeConnection());
      assertTrue(statsAggregator.closeConnection());

View Full Code Here


        // In case of a non-partitioned table, the key for stats temporary store is "rootDir"
        if (statsAggregator != null) {
          updateStats(collectableStats, tblStats, statsAggregator, parameters,
              work.getAggKey(), atomic);
          statsAggregator.cleanUp(work.getAggKey());
        }
      } else {
        // Partitioned table:
        // Need to get the old stats of the partition
        // and update the table stats based on the old and new stats.

View Full Code Here

          tPart.setParameters(parameters);
          String tableFullName = table.getDbName() + "." + table.getTableName();
          db.alterPartition(tableFullName, new Partition(table, tPart));


          if (statsAggregator != null) {
            statsAggregator.cleanUp(partitionID);
          }


          console.printInfo("Partition " + tableFullName + partn.getSpec() +
              " stats: [" + newPartStats.toString() + ']');
        }

View Full Code Here

    StatsAggregator statsAggregator;
    String statsImplementationClass = HiveConf.getVar(config, HiveConf.ConfVars.HIVESTATSDBCLASS);
    StatsFactory.setImplementation(statsImplementationClass, config);
    statsAggregator = StatsFactory.getStatsAggregator();
    if (statsAggregator.connect(config)) {
      statsAggregator.cleanUp(jobID + Path.SEPARATOR); // Adding the path separator to avoid an Id
                                                       // being a prefix of another ID
      statsAggregator.closeConnection();
    }
  }
}

View Full Code Here

    StatsFactory.setImplementation(statsImplementationClass, conf);


    stats = new HashMap<String, String>();
    StatsAggregator sa = StatsFactory.getStatsAggregator();
    sa.connect(conf);
    sa.cleanUp("file_0");
    sa.closeConnection();
  }




  private void fillStatMap(String numRows, String rawDataSize) {

View Full Code Here

0 1 2 3

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.