Package org.apache.hadoop.mapred

Examples of org.apache.hadoop.mapred.Counters.findCounter()

218
219
220
221
222
223
224
225
226
227
228
                    - counters.findCounter(TaskCounter.SPLIT_RAW_BYTES).getValue();
            runInputRecords[i] = (int) counters.findCounter(
                    TaskCounter.MAP_INPUT_RECORDS).getValue();
            runOutputBytes[i] = counters
                    .findCounter(TaskCounter.MAP_OUTPUT_BYTES).getValue();
            runOutputRecords[i] = (int) counters.findCounter(
                    TaskCounter.MAP_OUTPUT_RECORDS).getValue();

            specInfo = spec.getTaskInfo(TaskType.MAP, i);
            specInputRecords[i] = specInfo.getInputRecords();
            specInputBytes[i] = specInfo.getInputBytes();
View Full Code Here
236
237
238
239
240
241
242
243
244
245
246
                    runInputBytes[i], runOutputBytes[i], runInputRecords[i],
                    runOutputRecords[i]));
            break;
          case REDUCE:
            runInputBytes[i] = 0;
            runInputRecords[i] = (int) counters.findCounter(
                    TaskCounter.REDUCE_INPUT_RECORDS).getValue();
            runOutputBytes[i] = counters.findCounter("FileSystemCounters",
                    "HDFS_BYTES_WRITTEN").getValue();
            runOutputRecords[i] = (int) counters.findCounter(
                    TaskCounter.REDUCE_OUTPUT_RECORDS).getValue();
View Full Code Here
238
239
240
241
242
243
244
245
246
247
248
            break;
          case REDUCE:
            runInputBytes[i] = 0;
            runInputRecords[i] = (int) counters.findCounter(
                    TaskCounter.REDUCE_INPUT_RECORDS).getValue();
            runOutputBytes[i] = counters.findCounter("FileSystemCounters",
                    "HDFS_BYTES_WRITTEN").getValue();
            runOutputRecords[i] = (int) counters.findCounter(
                    TaskCounter.REDUCE_OUTPUT_RECORDS).getValue();

            specInfo = spec.getTaskInfo(TaskType.REDUCE, i);
View Full Code Here
240
241
242
243
244
245
246
247
248
249
250
            runInputBytes[i] = 0;
            runInputRecords[i] = (int) counters.findCounter(
                    TaskCounter.REDUCE_INPUT_RECORDS).getValue();
            runOutputBytes[i] = counters.findCounter("FileSystemCounters",
                    "HDFS_BYTES_WRITTEN").getValue();
            runOutputRecords[i] = (int) counters.findCounter(
                    TaskCounter.REDUCE_OUTPUT_RECORDS).getValue();

            specInfo = spec.getTaskInfo(TaskType.REDUCE, i);
            // There is no reliable counter for reduce input bytes. The
            // variable-length encoding of intermediate records and other noise
View Full Code Here
257
258
259
260
261
262
263
264
265
266
267
            specOutputBytes[i] = specInfo.getOutputBytes();
            LOG.info(String.format(type + " SPEC: (%9d) -> %9d :: %5d -> %5d\n",
                    specInfo.getInputBytes(), specOutputBytes[i],
                    specInputRecords[i], specOutputRecords[i]));
            LOG.info(String
                    .format(type + " RUN:  (%9d) -> %9d :: %5d -> %5d\n", counters
                            .findCounter(TaskCounter.REDUCE_SHUFFLE_BYTES).getValue(),
                            runOutputBytes[i], runInputRecords[i], runOutputRecords[i]));
            break;
          default:
            fail("Unexpected type: " + type);
View Full Code Here
185
186
187
188
189
190
191
192
193
194
195
      for (int i = 0; i < runTasks.length; ++i) {
        final TaskInfo specInfo;
        final Counters counters = runTasks[i].getCounters();
        switch (type) {
          case MAP:
             runInputBytes[i] = counters.findCounter("FileSystemCounters",
                 "HDFS_BYTES_READ").getValue() -
                 counters.findCounter(Task.Counter.SPLIT_RAW_BYTES).getValue();
             runInputRecords[i] =
               (int)counters.findCounter(MAP_INPUT_RECORDS).getValue();
             runOutputBytes[i] =
View Full Code Here
187
188
189
190
191
192
193
194
195
196
197
        final Counters counters = runTasks[i].getCounters();
        switch (type) {
          case MAP:
             runInputBytes[i] = counters.findCounter("FileSystemCounters",
                 "HDFS_BYTES_READ").getValue() -
                 counters.findCounter(Task.Counter.SPLIT_RAW_BYTES).getValue();
             runInputRecords[i] =
               (int)counters.findCounter(MAP_INPUT_RECORDS).getValue();
             runOutputBytes[i] =
               counters.findCounter(MAP_OUTPUT_BYTES).getValue();
             runOutputRecords[i] =
View Full Code Here
189
190
191
192
193
194
195
196
197
198
199
          case MAP:
             runInputBytes[i] = counters.findCounter("FileSystemCounters",
                 "HDFS_BYTES_READ").getValue() -
                 counters.findCounter(Task.Counter.SPLIT_RAW_BYTES).getValue();
             runInputRecords[i] =
               (int)counters.findCounter(MAP_INPUT_RECORDS).getValue();
             runOutputBytes[i] =
               counters.findCounter(MAP_OUTPUT_BYTES).getValue();
             runOutputRecords[i] =
               (int)counters.findCounter(MAP_OUTPUT_RECORDS).getValue();
View Full Code Here
191
192
193
194
195
196
197
198
199
200
201
                 "HDFS_BYTES_READ").getValue() -
                 counters.findCounter(Task.Counter.SPLIT_RAW_BYTES).getValue();
             runInputRecords[i] =
               (int)counters.findCounter(MAP_INPUT_RECORDS).getValue();
             runOutputBytes[i] =
               counters.findCounter(MAP_OUTPUT_BYTES).getValue();
             runOutputRecords[i] =
               (int)counters.findCounter(MAP_OUTPUT_RECORDS).getValue();

            specInfo = spec.getTaskInfo(TaskType.MAP, i);
            specInputRecords[i] = specInfo.getInputRecords();
View Full Code Here
193
194
195
196
197
198
199
200
201
202
203
             runInputRecords[i] =
               (int)counters.findCounter(MAP_INPUT_RECORDS).getValue();
             runOutputBytes[i] =
               counters.findCounter(MAP_OUTPUT_BYTES).getValue();
             runOutputRecords[i] =
               (int)counters.findCounter(MAP_OUTPUT_RECORDS).getValue();

            specInfo = spec.getTaskInfo(TaskType.MAP, i);
            specInputRecords[i] = specInfo.getInputRecords();
            specInputBytes[i] = specInfo.getInputBytes();
            specOutputRecords[i] = specInfo.getOutputRecords();
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.