Package org.apache.hadoop.mapred.Counters

Examples of org.apache.hadoop.mapred.Counters.Group

349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
     * these values are passed via configuration file to PORank, by using the unique
     * operation identifier
     */
    private void saveCounters(Job job, String operationID) {
        Counters counters;
        Group groupCounters;

        Long previousValue = 0L;
        Long previousSum = 0L;
        ArrayList<Pair<String,Long>> counterPairs;

        try {
            counters = HadoopShims.getCounters(job);
            groupCounters = counters.getGroup(getGroupName(counters.getGroupNames()));

            Iterator<Counter> it = groupCounters.iterator();
            HashMap<Integer,Long> counterList = new HashMap<Integer, Long>();

            while(it.hasNext()) {
                try{
                    Counter c = it.next();
View Full Code Here
604
605
606
607
608
609
610
611
612
613
614
    public Counters.Counter getCounter(String group, String name) {
      Counters.Counter counter = null;
      if (counters != null) {
        counter = counters.findCounter(group, name);
        if (counter == null) {
          Group grp = counters.addGroup(group, group);
          counter = grp.addCounter(name, name, 10);
        }
      }
      return counter;
    }
View Full Code Here
198
199
200
201
202
203
204
205
206
207
  @SuppressWarnings("deprecation")
  @Test
  public void testGroupIteratorConcurrency() {
    Counters counters = new Counters();
    counters.incrCounter("group1", "counter1", 1);
    Group group = counters.getGroup("group1");
    Iterator<Counter> iterator = group.iterator();
    counters.incrCounter("group1", "counter2", 1);
    iterator.next();
  }
View Full Code Here
213
214
215
216
217
218
219
220
221
222
223
224
    counters.findCounter("fs1", FileSystemCounter.BYTES_READ).increment(1);
    counters.findCounter("fs2", FileSystemCounter.BYTES_READ).increment(1);
   
    // Iterate over the counters in this group while updating counters in
    // the group
    Group group = counters.getGroup(FileSystemCounter.class.getName());
    Iterator<Counter> iterator = group.iterator();
    counters.findCounter("fs3", FileSystemCounter.BYTES_READ).increment(1);
    assertTrue(iterator.hasNext());
    iterator.next();
    counters.findCounter("fs3", FileSystemCounter.BYTES_READ).increment(1);
    assertTrue(iterator.hasNext());
View Full Code Here
494
495
496
497
498
499
500
501
502
503
504
505
506
    try {
      RunningJob running = jc.getJob(JobID.forName(mapreduceID));
      Counters jobCounters = running.getCounters();
      Iterator<Group> groups = jobCounters.iterator();
      while (groups.hasNext()) {
        Group g = groups.next();
        String gn = g.getName();
        Iterator<Counters.Counter> cs = g.iterator();
        while (cs.hasNext()) {
          Counters.Counter c = cs.next();
          String n = c.getName();
          long v = c.getCounter();
          retv.put(mapreduceID + "." + jobName + "." + gn + "." + n, "" + v);
View Full Code Here
54
55
56
57
58
59
60
61
62
63
64
65
66
      outFile.delete();
      assertEquals(outputExpect, output);
     
      Counters counters = job.running_.getCounters();
      assertNotNull("Counters", counters);
      Group group = counters.getGroup("UserCounters");
      assertNotNull("Group", group);
      Counter counter = group.getCounterForName("InputLines");
      assertNotNull("Counter", counter);
      assertEquals(3, counter.getCounter());
    } finally {
      File outFileCRC = new File(OUTPUT_DIR, ".part-00000.crc").getAbsoluteFile();
      INPUT_FILE.delete();
View Full Code Here
207
208
209
210
211
212
213
214
215
216
  @SuppressWarnings("deprecation")
  @Test
  public void testGroupIteratorConcurrency() {
    Counters counters = new Counters();
    counters.incrCounter("group1", "counter1", 1);
    Group group = counters.getGroup("group1");
    Iterator<Counter> iterator = group.iterator();
    counters.incrCounter("group1", "counter2", 1);
    iterator.next();
  }
View Full Code Here
222
223
224
225
226
227
228
229
230
231
232
233
    counters.findCounter("fs1", FileSystemCounter.BYTES_READ).increment(1);
    counters.findCounter("fs2", FileSystemCounter.BYTES_READ).increment(1);
   
    // Iterate over the counters in this group while updating counters in
    // the group
    Group group = counters.getGroup(FileSystemCounter.class.getName());
    Iterator<Counter> iterator = group.iterator();
    counters.findCounter("fs3", FileSystemCounter.BYTES_READ).increment(1);
    assertTrue(iterator.hasNext());
    iterator.next();
    counters.findCounter("fs3", FileSystemCounter.BYTES_READ).increment(1);
    assertTrue(iterator.hasNext());
View Full Code Here
42
43
44
45
46
47
48
49
50
51
52
  }
 
  private void validateCounters() throws IOException {
    Counters counters = job.running_.getCounters();
    assertNotNull("Counters", counters);
    Group group = counters.getGroup("UserCounters");
    assertNotNull("Group", group);
    Counter counter = group.getCounterForName("InputLines");
    assertNotNull("Counter", counter);
    assertEquals(3, counter.getCounter());
  }
View Full Code Here
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
          if (counters == null) {
            System.out.println("Counters not available for retired job " +
                jobid);
            exitCode = -1;
          } else {
            Group group = counters.getGroup(counterGroupName);
            Counter counter = group.getCounterForName(counterName);
            System.out.println(counter.getCounter());
            exitCode = 0;
          }
        }
      } else if (killJob) {
View Full Code Here
TOP

Related Classes of org.apache.hadoop.mapred.Counters.Group

  • com.atlantbh.jmeter.plugins.hadooputilities.jobstatistics.JobLayer
  • com.atlantbh.jmeter.plugins.hadooputilities.jobstatistics.TaskLayer
  • com.netflix.lipstick.warnings.JobWarnings
  • org.apache.hadoop.mapred.gridmix.test.system.GridmixJobVerification
  • org.apache.hadoop.mapred.GridMixRunner
  • org.apache.hadoop.mapred.HistoryViewer
  • org.apache.hadoop.mapred.JobClient
  • org.apache.hadoop.mapred.pipes.TestPipeApplication$TestTaskReporter
  • org.apache.hadoop.mapred.TestCounters
  • org.apache.hadoop.mrunit.internal.counters.CounterWrapper
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.