Package org.apache.hadoop.mapred.Counters

Examples of org.apache.hadoop.mapred.Counters.Group

298
299
300
301
302
303
304
305
306
307
308
309
310
    @SuppressWarnings("deprecation")
    private static void parseAndAddJobCounters(Map<String, String> job, String counters) {
        try {
            Counters counterGroups = Counters.fromEscapedCompactString(counters);
            for (Group otherGroup : counterGroups) {
                Group group = counterGroups.getGroup(otherGroup.getName());
                for (Counter otherCounter : otherGroup) {
                    Counter counter = group.getCounterForName(otherCounter.getName());
                    job.put(otherCounter.getName(), String.valueOf(counter.getValue()));
                }
            }
        } catch (ParseException e) {
           LOG.warn("Failed to parse job counters", e);
View Full Code Here
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
        RunningJob job = getJob(JobID.forName(jobid));
        if (job == null) {
          System.out.println("Could not find job " + jobid);
        } else {
          Counters counters = job.getCounters();
          Group group = counters.getGroup(counterGroupName);
          Counter counter = group.getCounterForName(counterName);
          System.out.println(counter.getCounter());
          exitCode = 0;
        }
      } else if (killJob) {
        RunningJob job = getJob(JobID.forName(jobid));
View Full Code Here
197
198
199
200
201
202
203
204
205
206
  @SuppressWarnings("deprecation")
  @Test
  public void testGroupIteratorConcurrency() {
    Counters counters = new Counters();
    counters.incrCounter("group1", "counter1", 1);
    Group group = counters.getGroup("group1");
    Iterator<Counter> iterator = group.iterator();
    counters.incrCounter("group1", "counter2", 1);
    iterator.next();
  }
View Full Code Here
212
213
214
215
216
217
218
219
220
221
222
223
    counters.findCounter("fs1", FileSystemCounter.BYTES_READ).increment(1);
    counters.findCounter("fs2", FileSystemCounter.BYTES_READ).increment(1);
   
    // Iterate over the counters in this group while updating counters in
    // the group
    Group group = counters.getGroup(FileSystemCounter.class.getName());
    Iterator<Counter> iterator = group.iterator();
    counters.findCounter("fs3", FileSystemCounter.BYTES_READ).increment(1);
    assertTrue(iterator.hasNext());
    iterator.next();
    counters.findCounter("fs3", FileSystemCounter.BYTES_READ).increment(1);
    assertTrue(iterator.hasNext());
View Full Code Here
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233

  private long getCounterValue(Counters counters, String key)
     throws ParseException {
    for (String groupName : counters.getGroupNames()) {
       Group totalGroup = counters.getGroup(groupName);
       Iterator<Counter> itrCounter = totalGroup.iterator();
       while (itrCounter.hasNext()) {
         Counter counter = itrCounter.next();
         if (counter.getName().equals(key)) {
           return counter.getValue();
         }
View Full Code Here
58
59
60
61
62
63
64
65
66
67
68
69
70
      outFile.delete();
      assertEquals(outputExpect, output);
     
      Counters counters = job.running_.getCounters();
      assertNotNull("Counters", counters);
      Group group = counters.getGroup("UserCounters");
      assertNotNull("Group", group);
      Counter counter = group.getCounterForName("InputLines");
      assertNotNull("Counter", counter);
      assertEquals(3, counter.getCounter());
    } finally {
      try {
        INPUT_FILE.delete();
View Full Code Here
196
197
198
199
200
201
202
203
204
205
  @SuppressWarnings("deprecation")
  @Test
  public void testGroupIteratorConcurrency() {
    Counters counters = new Counters();
    counters.incrCounter("group1", "counter1", 1);
    Group group = counters.getGroup("group1");
    Iterator<Counter> iterator = group.iterator();
    counters.incrCounter("group1", "counter2", 1);
    iterator.next();
  }
View Full Code Here
99
100
101
102
103
104
105
106
107
108
        } catch (IOException e) {
            log.error("Error getting job client, continuing", e);
            return 1;
        }

        Group fsGroup = counters.getGroup("FileSystemCounters");
        long hdfsBytes = fsGroup.getCounter("HDFS_BYTES_WRITTEN");
        long s3Bytes = fsGroup.getCounter("S3N_BYTES_WRITTEN");
        return hdfsBytes + s3Bytes;
    }
View Full Code Here
201
202
203
204
205
206
207
208
209
210
211
212
213
        } catch (IOException e) {
            log.error("Error getting job client, continuing", e);
            return true;
        }

        Group fsGroup = counters.getGroup("FileSystemCounters");
        long hdfsBytes = fsGroup.getCounter("HDFS_BYTES_WRITTEN");
        long s3Bytes = fsGroup.getCounter("S3N_BYTES_WRITTEN");
        log.info(String.format("Total of %s bytes were written by this m/r job", (hdfsBytes + s3Bytes)));
        if ((0 == s3Bytes) && (HDFS_DIRECTORY_SIZE == hdfsBytes)) {
            log.info("No s3 output and empty HDFS directory created");
            return false;
        } else {
View Full Code Here
300
301
302
303
304
305
306
307
308
309
310
311
312
    @SuppressWarnings("deprecation")
    private static void parseAndAddJobCounters(Map<String, String> job, String counters) {
        try {
            Counters counterGroups = Counters.fromEscapedCompactString(counters);
            for (Group otherGroup : counterGroups) {
                Group group = counterGroups.getGroup(otherGroup.getName());
                for (Counter otherCounter : otherGroup) {
                    Counter counter = group.getCounterForName(otherCounter.getName());
                    job.put(otherCounter.getName(), String.valueOf(counter.getValue()));
                }
            }
        } catch (ParseException e) {
           LOG.warn("Failed to parse job counters", e);
View Full Code Here
TOP

Related Classes of org.apache.hadoop.mapred.Counters.Group

  • com.atlantbh.jmeter.plugins.hadooputilities.jobstatistics.JobLayer
  • com.atlantbh.jmeter.plugins.hadooputilities.jobstatistics.TaskLayer
  • com.netflix.lipstick.warnings.JobWarnings
  • org.apache.hadoop.mapred.gridmix.test.system.GridmixJobVerification
  • org.apache.hadoop.mapred.GridMixRunner
  • org.apache.hadoop.mapred.HistoryViewer
  • org.apache.hadoop.mapred.JobClient
  • org.apache.hadoop.mapred.pipes.TestPipeApplication$TestTaskReporter
  • org.apache.hadoop.mapred.TestCounters
  • org.apache.hadoop.mrunit.internal.counters.CounterWrapper
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.