Package org.apache.hadoop.io

Examples of org.apache.hadoop.io.RawComparator

801
802
803
804
805
806
807
808
809
810
811
    // merge config params
    Class keyClass = (Class)ConfigUtils.getIntermediateInputKeyClass(job);
    Class valueClass = (Class)ConfigUtils.getIntermediateInputValueClass(job);
    final Path tmpDir = new Path(inputContext.getUniqueIdentifier());
    final RawComparator comparator =
      (RawComparator)ConfigUtils.getIntermediateInputKeyComparator(job);

    // segments required to vacate memory
    List<Segment> memDiskSegments = new ArrayList<Segment>();
    long inMemToDiskBytes = 0;
View Full Code Here
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
      JobConf job = (JobConf)getConf();
      Reducer combiner =
        (Reducer)ReflectionUtils.newInstance(combinerClass, job);
      Class keyClass = job.getMapOutputKeyClass();
      Class valClass = job.getMapOutputValueClass();
      RawComparator comparator = job.getOutputKeyComparator();
      try {
        CombineValuesIterator values = new CombineValuesIterator(
            kvIter, comparator, keyClass, valClass, job, Reporter.NULL,
            inCounter);
        while (values.more()) {
View Full Code Here
72
73
74
75
76
77
78
79
80
81
82
      options.configure(job);
      if (Pair.class.equals(keyType.getTypeClass()) && options.getGroupingComparatorClass() != null) {
        PType<?> pairKey = keyType.getSubTypes().get(0);
        return new SecondarySortShuffler(getMapForKeyType(pairKey));
      } else if (options.getSortComparatorClass() != null) {
        RawComparator rc = ReflectionUtils.newInstance(
            options.getSortComparatorClass(),
            job.getConfiguration());
        map = new TreeMap<Object, Collection<T>>(rc);
        return new MapShuffler<S, T>(map, keyType);
      }
View Full Code Here
397
398
399
400
401
402
403
404
405
406
407
    sortPhase.complete();                         // sort is complete
    setPhase(TaskStatus.Phase.REDUCE);
    statusUpdate(umbilical);
    Class keyClass = job.getMapOutputKeyClass();
    Class valueClass = job.getMapOutputValueClass();
    RawComparator comparator = job.getOutputValueGroupingComparator();

    if (useNewApi) {
      runNewReducer(job, umbilical, reporter, rIter, comparator,
                    keyClass, valueClass);
    } else {
View Full Code Here
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
        Counters.Counter inCounter) throws IOException {
      JobConf job = (JobConf)getConf();
      Reducer combiner = ReflectionUtils.newInstance(combinerClass, job);
      Class keyClass = job.getMapOutputKeyClass();
      Class valClass = job.getMapOutputValueClass();
      RawComparator comparator = job.getOutputKeyComparator();
      try {
        CombineValuesIterator values = new CombineValuesIterator(
            kvIter, comparator, keyClass, valClass, job, Reporter.NULL,
            inCounter);
        while (values.more()) {
View Full Code Here
410
411
412
413
414
415
416
417
418
419
420
    sortPhase.complete();                         // sort is complete
    setPhase(TaskStatus.Phase.REDUCE);
    statusUpdate(umbilical);
    Class keyClass = job.getMapOutputKeyClass();
    Class valueClass = job.getMapOutputValueClass();
    RawComparator comparator = job.getOutputValueGroupingComparator();

    if (useNewApi) {
      runNewReducer(job, umbilical, reporter, rIter, comparator,
                    keyClass, valueClass);
    } else {
View Full Code Here
751
752
753
754
755
756
757
758
759
760
761
    // merge config params
    Class keyClass = (Class)ConfigUtils.getIntermediateInputKeyClass(job);
    Class valueClass = (Class)ConfigUtils.getIntermediateInputValueClass(job);
    final Path tmpDir = new Path(inputContext.getUniqueIdentifier());
    final RawComparator comparator =
      (RawComparator)ConfigUtils.getIntermediateInputKeyComparator(job);

    // segments required to vacate memory
    List<Segment> memDiskSegments = new ArrayList<Segment>();
    long inMemToDiskBytes = 0;
View Full Code Here
122
123
124
125
126
127
128
129
130
131
132
    Class keyClass = ConfigUtils.getIntermediateInputKeyClass(jobConf);
    Class valueClass = ConfigUtils.getIntermediateInputValueClass(jobConf);
    LOG.info("Using keyClass: " + keyClass);
    LOG.info("Using valueClass: " + valueClass);
    RawComparator comparator =
        ConfigUtils.getInputKeySecondaryGroupingComparator(jobConf);
    LOG.info("Using comparator: " + comparator);

    reduceInputKeyCounter =
        mrReporter.getCounter(TaskCounter.REDUCE_INPUT_GROUPS);
View Full Code Here
114
115
116
117
118
119
120
121
122
123
124
    Class keyClass = ConfigUtils.getIntermediateInputKeyClass(jobConf);
    Class valueClass = ConfigUtils.getIntermediateInputValueClass(jobConf);
    LOG.info("Using keyClass: " + keyClass);
    LOG.info("Using valueClass: " + valueClass);
    RawComparator comparator =
        ConfigUtils.getInputKeySecondaryGroupingComparator(jobConf);
    LOG.info("Using comparator: " + comparator);

    reduceInputKeyCounter =
        mrReporter.getCounter(TaskCounter.REDUCE_INPUT_GROUPS);
View Full Code Here
377
378
379
380
381
382
383
384
385
386
387
    sortPhase.complete();                         // sort is complete
    setPhase(TaskStatus.Phase.REDUCE);
    statusUpdate(umbilical);
    Class keyClass = job.getMapOutputKeyClass();
    Class valueClass = job.getMapOutputValueClass();
    RawComparator comparator = job.getOutputValueGroupingComparator();

    if (useNewApi) {
      runNewReducer(job, umbilical, reporter, rIter, comparator,
                    keyClass, valueClass);
    } else {
View Full Code Here
TOP

Related Classes of org.apache.hadoop.io.RawComparator

  • com.datasalt.pangool.tuplemr.mapred.SortComparator
  • com.ebay.erl.mobius.core.mapred.ConfigurableJob
  • org.apache.crunch.impl.mem.collect.Shuffler
  • org.apache.hadoop.mapred.ReduceTask
  • org.apache.hadoop.mapred.ReduceTask$ReduceCopier
  • org.apache.hadoop.mrunit.mapreduce.TestMapReduceDriver
  • org.apache.hadoop.mrunit.TestMapReduceDriver
  • org.apache.tez.mapreduce.processor.reduce.ReduceProcessor
  • org.apache.tez.runtime.library.common.shuffle.impl.MergeManager
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.