Package org.apache.hadoop.mapred.Counters

Examples of org.apache.hadoop.mapred.Counters.Group

222
223
224
225
226
227
228
229
230
231
232
233
    counters.findCounter("fs1", FileSystemCounter.BYTES_READ).increment(1);
    counters.findCounter("fs2", FileSystemCounter.BYTES_READ).increment(1);
   
    // Iterate over the counters in this group while updating counters in
    // the group
    Group group = counters.getGroup(FileSystemCounter.class.getName());
    Iterator<Counter> iterator = group.iterator();
    counters.findCounter("fs3", FileSystemCounter.BYTES_READ).increment(1);
    assertTrue(iterator.hasNext());
    iterator.next();
    counters.findCounter("fs3", FileSystemCounter.BYTES_READ).increment(1);
    assertTrue(iterator.hasNext());
View Full Code Here
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
              System.out.println("Counters not available for retired job "
                  + jobid);
            }
            exitCode = -1;
          } else {
            Group group = counters.getGroup(counterGroupName);
            Counter counter = group.getCounterForName(counterName);
            System.out.println(counter.getCounter());
            exitCode = 0;
          }
        }
      } else if (killJob) {
View Full Code Here
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
              System.out.println("Counters not available for retired job "
                  + jobid);
            }
            exitCode = -1;
          } else {
            Group group = counters.getGroup(counterGroupName);
            Counter counter = group.getCounterForName(counterName);
            System.out.println(counter.getCounter());
            exitCode = 0;
          }
        }
      } else if (killJob) {
View Full Code Here
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
  public Collection<Pair<String, String>> findCounterValues() {
    final Collection<Pair<String, String>> counters = new LinkedList<Pair<String, String>>();
    final Iterable<String> groupNames = getGroupNames();
    if (mapred != null) {
      for (String groupName : groupNames) {
        final Group group = mapred.getGroup(groupName);
        collectCounters(counters, groupName, group.iterator());
      }
    } else {
      for (String groupName : groupNames) {
        final CounterGroup group = mapreduce.getGroup(groupName);
        collectCounters(counters, groupName, group.iterator());
      }
    }
    return counters;
  }
View Full Code Here
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
     * these values are passed via configuration file to PORank, by using the unique
     * operation identifier
     */
    private void saveCounters(Job job, String operationID) {
        Counters counters;
        Group groupCounters;

        Long previousValue = 0L;
        Long previousSum = 0L;
        ArrayList<Pair<String,Long>> counterPairs;

        try {
            counters = HadoopShims.getCounters(job);
            groupCounters = counters.getGroup(getGroupName(counters.getGroupNames()));

            Iterator<Counter> it = groupCounters.iterator();
            HashMap<Integer,Long> counterList = new HashMap<Integer, Long>();

            while(it.hasNext()) {
                try{
                    Counter c = it.next();
View Full Code Here
299
300
301
302
303
304
305
306
307
308
309
310
311
    @SuppressWarnings("deprecation")
    private static void parseAndAddJobCounters(Map<String, String> job, String counters) {
        try {
            Counters counterGroups = Counters.fromEscapedCompactString(counters);
            for (Group otherGroup : counterGroups) {
                Group group = counterGroups.getGroup(otherGroup.getName());
                for (Counter otherCounter : otherGroup) {
                    Counter counter = group.getCounterForName(otherCounter.getName());
                    job.put(otherCounter.getName(), String.valueOf(counter.getValue()));
                }
            }
        } catch (ParseException e) {
           LOG.warn("Failed to parse job counters", e);
View Full Code Here
238
239
240
241
242
243
244
245
246
247
248
249
    }

    private Counters covertToHadoopCounters(TezCounters tezCounters) {
        Counters counters = new Counters();
        for (CounterGroup tezGrp : tezCounters) {
            Group grp = counters.addGroup(tezGrp.getName(), tezGrp.getDisplayName());
            for (TezCounter counter : tezGrp) {
                grp.addCounter(counter.getName(), counter.getDisplayName(), counter.getValue());
            }
        }
        return counters;
    }
View Full Code Here
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
              System.out.println("Counters not available for retired job "
                  + jobid);
            }
            exitCode = -1;
          } else {
            Group group = counters.getGroup(counterGroupName);
            Counter counter = group.getCounterForName(counterName);
            System.out.println(counter.getCounter());
            exitCode = 0;
          }
        }
      } else if (killJob) {
View Full Code Here
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
        RunningJob job = getJob(JobID.forName(jobid));
        if (job == null) {
          System.out.println("Could not find job " + jobid);
        } else {
          Counters counters = job.getCounters();
          Group group = counters.getGroup(counterGroupName);
          Counter counter = group.getCounterForName(counterName);
          System.out.println(counter.getCounter());
          exitCode = 0;
        }
      } else if (killJob) {
        RunningJob job = getJob(JobID.forName(jobid));
View Full Code Here
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
     * these values are passed via configuration file to PORank, by using the unique
     * operation identifier
     */
    private void saveCounters(Job job, String operationID) {
        Counters counters;
        Group groupCounters;

        Long previousValue = 0L;
        Long previousSum = 0L;
        ArrayList<Pair<String,Long>> counterPairs;

        try {
            counters = HadoopShims.getCounters(job);

            String groupName = getGroupName(counters.getGroupNames());
            // In case that the counter group was not find, we need to find
            // out why. Only acceptable state is that the relation has been
            // empty.
            if (groupName == null) {
                Counter outputRecords =
                    counters.getGroup(MRPigStatsUtil.TASK_COUNTER_GROUP)
                    .getCounterForName(MRPigStatsUtil.MAP_OUTPUT_RECORDS);

                if(outputRecords.getCounter() == 0) {
                    globalCounters.put(operationID, new ArrayList<Pair<String, Long>>());
                    return;
                } else {
                  throw new RuntimeException("Did not found RANK counter group for operationId: " + operationID);
                }
            }
            groupCounters = counters.getGroup(groupName);

            Iterator<Counter> it = groupCounters.iterator();
            HashMap<Integer,Long> counterList = new HashMap<Integer, Long>();

            while(it.hasNext()) {
                try{
                    Counter c = it.next();
View Full Code Here
TOP

Related Classes of org.apache.hadoop.mapred.Counters.Group

  • com.atlantbh.jmeter.plugins.hadooputilities.jobstatistics.JobLayer
  • com.atlantbh.jmeter.plugins.hadooputilities.jobstatistics.TaskLayer
  • com.netflix.lipstick.warnings.JobWarnings
  • org.apache.hadoop.mapred.gridmix.test.system.GridmixJobVerification
  • org.apache.hadoop.mapred.GridMixRunner
  • org.apache.hadoop.mapred.HistoryViewer
  • org.apache.hadoop.mapred.JobClient
  • org.apache.hadoop.mapred.pipes.TestPipeApplication$TestTaskReporter
  • org.apache.hadoop.mapred.TestCounters
  • org.apache.hadoop.mrunit.internal.counters.CounterWrapper
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.