Package org.apache.hadoop.mapreduce.task.reduce

Examples of org.apache.hadoop.mapreduce.task.reduce.Shuffle

348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
      Class combinerClass = conf.getCombinerClass();
      CombineOutputCollector combineCollector =
        (null != combinerClass) ?
            new CombineOutputCollector(reduceCombineOutputCounter) : null;

      Shuffle shuffle =
        new Shuffle(getTaskID(), job, FileSystem.getLocal(job), umbilical,
                    super.lDirAlloc, reporter, codec,
                    combinerClass, combineCollector,
                    spilledRecordsCounter, reduceCombineInputCounter,
                    shuffledMapsCounter,
                    reduceShuffleBytes, failedShuffleCounter,
                    mergedMapOutputsCounter,
                    taskStatus, copyPhase, sortPhase, this);
      rIter = shuffle.run();
    } else {
      final FileSystem rfs = FileSystem.getLocal(job).getRaw();
      rIter = Merger.merge(job, rfs, job.getMapOutputKeyClass(),
                           job.getMapOutputValueClass(), codec,
                           getMapFiles(rfs, true),
View Full Code Here
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
      Class combinerClass = conf.getCombinerClass();
      CombineOutputCollector combineCollector =
        (null != combinerClass) ?
        new CombineOutputCollector(reduceCombineOutputCounter, reporter, conf) : null;

      Shuffle shuffle =
        new Shuffle(getTaskID(), job, FileSystem.getLocal(job), umbilical,
                    super.lDirAlloc, reporter, codec,
                    combinerClass, combineCollector,
                    spilledRecordsCounter, reduceCombineInputCounter,
                    shuffledMapsCounter,
                    reduceShuffleBytes, failedShuffleCounter,
                    mergedMapOutputsCounter,
                    taskStatus, copyPhase, sortPhase, this,
                    mapOutputFile);
      rIter = shuffle.run();
    } else {
      // local job runner doesn't have a copy phase
      copyPhase.complete();
      final FileSystem rfs = FileSystem.getLocal(job).getRaw();
      rIter = Merger.merge(job, rfs, job.getMapOutputKeyClass(),
View Full Code Here
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
      Class combinerClass = conf.getCombinerClass();
      CombineOutputCollector combineCollector =
        (null != combinerClass) ?
        new CombineOutputCollector(reduceCombineOutputCounter, reporter, conf) : null;

      Shuffle shuffle =
        new Shuffle(getTaskID(), job, FileSystem.getLocal(job), umbilical,
                    super.lDirAlloc, reporter, codec,
                    combinerClass, combineCollector,
                    spilledRecordsCounter, reduceCombineInputCounter,
                    shuffledMapsCounter,
                    reduceShuffleBytes, failedShuffleCounter,
                    mergedMapOutputsCounter,
                    taskStatus, copyPhase, sortPhase, this,
                    mapOutputFile);
      rIter = shuffle.run();
    } else {
      // local job runner doesn't have a copy phase
      copyPhase.complete();
      final FileSystem rfs = FileSystem.getLocal(job).getRaw();
      rIter = Merger.merge(job, rfs, job.getMapOutputKeyClass(),
View Full Code Here
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
      Class combinerClass = conf.getCombinerClass();
      CombineOutputCollector combineCollector =
        (null != combinerClass) ?
            new CombineOutputCollector(reduceCombineOutputCounter) : null;

      Shuffle shuffle =
        new Shuffle(getTaskID(), job, FileSystem.getLocal(job), umbilical,
                    super.lDirAlloc, reporter, codec,
                    combinerClass, combineCollector,
                    spilledRecordsCounter, reduceCombineInputCounter,
                    shuffledMapsCounter,
                    reduceShuffleBytes, failedShuffleCounter,
                    mergedMapOutputsCounter,
                    taskStatus, copyPhase, sortPhase, this);
      rIter = shuffle.run();
    } else {
      final FileSystem rfs = FileSystem.getLocal(job).getRaw();
      rIter = Merger.merge(job, rfs, job.getMapOutputKeyClass(),
                           job.getMapOutputValueClass(), codec,
                           getMapFiles(rfs, true),
View Full Code Here
TOP

Related Classes of org.apache.hadoop.mapreduce.task.reduce.Shuffle

  • org.apache.hadoop.mapred.RawKeyValueIterator
  • org.apache.hadoop.mapred.ReduceTask
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.