Examples of org.apache.hadoop.mapred.Counters.findCounter()

org.apache.hadoop.mapred.Counters.findCounter()
Find the counter for the given enum. The same enum will always return the same counter. @param key the counter key @return the matching counter object

    fs.delete(outputPath, true);


    RunningJob job = JobClient.runJob(conf);


    Counters counters = job.getCounters();
    int blocks = (int) counters.findCounter(Blocks.Total).getCounter();


    LOG.info("number of blocks: " + blocks);


    LOG.info("Writing index file...");
    LineReader reader = new LineReader(fs.open(new Path(outputPath + "/part-00000")));

View Full Code Here

    FileSystem.get(conf).delete(new Path(outputPath), true);


    RunningJob job = JobClient.runJob(conf);


    Counters counters = job.getCounters();
    int numDocs = (int) counters.findCounter(Count.DOCS).getCounter();


    String inputFile = outputPath + "/" + "part-00000";


    sLogger.info("Writing " + numDocs + " doc offseta to " + indexFile);
    LineReader reader = new LineReader(fs.open(new Path(inputFile)));

View Full Code Here

    for (Map.Entry<String, ? extends Number> entry : counterNameToValue.entrySet()) {


      String[] cNames = entry.getKey().split("::");
      String groupName = cNames[0];
      String counterName = cNames[1];
      Counter counter = counters.findCounter(groupName, counterName);
      counter.setValue(entry.getValue().longValue());
    }
    return CounterGroup.counterGroupInfoMap(counters);
  }

View Full Code Here


      // find out CPU msecs
      // In the case that we can't find out this number, we just skip the step to print
      // it out.
      if (ctrs != null) {
        Counter counterCpuMsec = ctrs.findCounter("org.apache.hadoop.mapred.Task$Counter",
            "CPU_MILLISECONDS");
        if (counterCpuMsec != null) {
          long newCpuMSec = counterCpuMsec.getValue();
          if (newCpuMSec > 0) {
            cpuMsec = newCpuMSec;

View Full Code Here

        success = rj.isSuccessful();
      }
    }


    if (ctrs != null) {
      Counter counterCpuMsec = ctrs.findCounter("org.apache.hadoop.mapred.Task$Counter",
          "CPU_MILLISECONDS");
      if (counterCpuMsec != null) {
        long newCpuMSec = counterCpuMsec.getValue();
        if (newCpuMSec > cpuMsec) {
          cpuMsec = newCpuMSec;

View Full Code Here

      for (String counterName : stats.getCountersFor(groupName)) { //retreiving counters in that group
        Long counterValue = stats.getCounterValue(groupName, counterName);
        counterNameToValue.put(groupName+"::"+counterName, counterValue);


        //creating counter
        Counter counter = counters.findCounter(groupName, counterName);
        counter.setValue(counterValue);
      }
    }
    setCounterGroupMap(CounterGroup.counterGroupInfoMap(counters));
    return counterNameToValue;

View Full Code Here

      for (int i = 0; i < runTasks.length; ++i) {
        final TaskInfo specInfo;
        final Counters counters = runTasks[i].getCounters();
        switch (type) {
          case MAP:
            runInputBytes[i] = counters.findCounter("FileSystemCounters",
                    "HDFS_BYTES_READ").getValue()
                    - counters.findCounter(TaskCounter.SPLIT_RAW_BYTES).getValue();
            runInputRecords[i] = (int) counters.findCounter(
                    TaskCounter.MAP_INPUT_RECORDS).getValue();
            runOutputBytes[i] = counters

View Full Code Here

        final Counters counters = runTasks[i].getCounters();
        switch (type) {
          case MAP:
            runInputBytes[i] = counters.findCounter("FileSystemCounters",
                    "HDFS_BYTES_READ").getValue()
                    - counters.findCounter(TaskCounter.SPLIT_RAW_BYTES).getValue();
            runInputRecords[i] = (int) counters.findCounter(
                    TaskCounter.MAP_INPUT_RECORDS).getValue();
            runOutputBytes[i] = counters
                    .findCounter(TaskCounter.MAP_OUTPUT_BYTES).getValue();
            runOutputRecords[i] = (int) counters.findCounter(

View Full Code Here

        switch (type) {
          case MAP:
            runInputBytes[i] = counters.findCounter("FileSystemCounters",
                    "HDFS_BYTES_READ").getValue()
                    - counters.findCounter(TaskCounter.SPLIT_RAW_BYTES).getValue();
            runInputRecords[i] = (int) counters.findCounter(
                    TaskCounter.MAP_INPUT_RECORDS).getValue();
            runOutputBytes[i] = counters
                    .findCounter(TaskCounter.MAP_OUTPUT_BYTES).getValue();
            runOutputRecords[i] = (int) counters.findCounter(
                    TaskCounter.MAP_OUTPUT_RECORDS).getValue();

View Full Code Here

            runInputBytes[i] = counters.findCounter("FileSystemCounters",
                    "HDFS_BYTES_READ").getValue()
                    - counters.findCounter(TaskCounter.SPLIT_RAW_BYTES).getValue();
            runInputRecords[i] = (int) counters.findCounter(
                    TaskCounter.MAP_INPUT_RECORDS).getValue();
            runOutputBytes[i] = counters
                    .findCounter(TaskCounter.MAP_OUTPUT_BYTES).getValue();
            runOutputRecords[i] = (int) counters.findCounter(
                    TaskCounter.MAP_OUTPUT_RECORDS).getValue();


            specInfo = spec.getTaskInfo(TaskType.MAP, i);

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.