Package org.apache.hadoop.io

Examples of org.apache.hadoop.io.RawComparator

404
405
406
407
408
409
410
411
412
413
414
    sortPhase.complete();                         // sort is complete
    setPhase(TaskStatus.Phase.REDUCE);
    statusUpdate(umbilical);
    Class keyClass = job.getMapOutputKeyClass();
    Class valueClass = job.getMapOutputValueClass();
    RawComparator comparator = job.getOutputValueGroupingComparator();

    if (useNewApi) {
      runNewReducer(job, umbilical, reporter, rIter, comparator,
                    keyClass, valueClass);
    } else {
View Full Code Here
400
401
402
403
404
405
406
407
408
409
410
    sortPhase.complete();                         // sort is complete
    setPhase(TaskStatus.Phase.REDUCE);
    statusUpdate(umbilical);
    Class keyClass = job.getMapOutputKeyClass();
    Class valueClass = job.getMapOutputValueClass();
    RawComparator comparator = job.getOutputValueGroupingComparator();

    if (useNewApi) {
      runNewReducer(job, umbilical, reporter, rIter, comparator,
                    keyClass, valueClass);
    } else {
View Full Code Here
426
427
428
429
430
431
432
433
434
435
436
    sortPhase.complete();                         // sort is complete
    setPhase(TaskStatus.Phase.REDUCE);
    statusUpdate(umbilical);
    Class keyClass = job.getMapOutputKeyClass();
    Class valueClass = job.getMapOutputValueClass();
    RawComparator comparator = job.getOutputValueGroupingComparator();

    if (useNewApi) {
      runNewReducer(job, umbilical, reporter, rIter, comparator,
                    keyClass, valueClass);
    } else {
View Full Code Here
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
        Counters.Counter inCounter) throws IOException {
      JobConf job = (JobConf)getConf();
      Reducer combiner = ReflectionUtils.newInstance(combinerClass, job);
      Class keyClass = job.getMapOutputKeyClass();
      Class valClass = job.getMapOutputValueClass();
      RawComparator comparator = job.getOutputKeyComparator();
      try {
        CombineValuesIterator values = new CombineValuesIterator(
            kvIter, comparator, keyClass, valClass, job, Reporter.NULL,
            inCounter);
        while (values.more()) {
View Full Code Here
400
401
402
403
404
405
406
407
408
409
410
    sortPhase.complete();                         // sort is complete
    setPhase(TaskStatus.Phase.REDUCE);
    statusUpdate(umbilical);
    Class keyClass = job.getMapOutputKeyClass();
    Class valueClass = job.getMapOutputValueClass();
    RawComparator comparator = job.getOutputValueGroupingComparator();

    if (useNewApi) {
      runNewReducer(job, umbilical, reporter, rIter, comparator,
                    keyClass, valueClass);
    } else {
View Full Code Here
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
      JobConf job = (JobConf)getConf();
      Reducer combiner =
        (Reducer)ReflectionUtils.newInstance(combinerClass, job);
      Class keyClass = job.getMapOutputKeyClass();
      Class valClass = job.getMapOutputValueClass();
      RawComparator comparator = job.getOutputKeyComparator();
      try {
        CombineValuesIterator values = new CombineValuesIterator(
            kvIter, comparator, keyClass, valClass, job, Reporter.NULL,
            inCounter);
        while (values.more()) {
View Full Code Here
409
410
411
412
413
414
415
416
417
418
419
    sortPhase.complete();                         // sort is complete
    setPhase(TaskStatus.Phase.REDUCE);
    statusUpdate(umbilical);
    Class keyClass = job.getMapOutputKeyClass();
    Class valueClass = job.getMapOutputValueClass();
    RawComparator comparator = job.getOutputValueGroupingComparator();

    if (useNewApi) {
      runNewReducer(job, umbilical, reporter, rIter, comparator,
                    keyClass, valueClass);
    } else {
View Full Code Here
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
    for(int depth = 0; depth < criteria.getElements().size(); depth++) {
      Field field = schema.getField(depth);
      Field.Type type = field.getType();
      SortElement sortElement = criteria.getElements().get(depth);
      Order sort = sortElement.getOrder();
      RawComparator comparator = sortElement.getCustomComparator();

      if(comparator != null) {
        //custom comparator for OBJECT
        int length1 = WritableComparator.readVInt(b1, o.offset1);
        int length2 = WritableComparator.readVInt(b2, o.offset2);
        o.offset1 += WritableUtils.decodeVIntSize(b1[o.offset1]);
        o.offset2 += WritableUtils.decodeVIntSize(b2[o.offset2]);
        int comparison = comparator.compare(b1, o.offset1, length1, b2,
            o.offset2, length2);
        o.offset1 += length1;
        o.offset2 += length2;
        if(comparison != 0) {
          return (sort == Order.ASC) ? comparison : -comparison;
View Full Code Here
402
403
404
405
406
407
408
409
410
411
412
    sortPhase.complete();                         // sort is complete
    setPhase(TaskStatus.Phase.REDUCE);
    statusUpdate(umbilical);
    Class keyClass = job.getMapOutputKeyClass();
    Class valueClass = job.getMapOutputValueClass();
    RawComparator comparator = job.getOutputValueGroupingComparator();

    if (useNewApi) {
      runNewReducer(job, umbilical, reporter, rIter, comparator,
                    keyClass, valueClass);
    } else {
View Full Code Here
111
112
113
114
115
116
117
118
119
120
121
    Class keyClass = ConfigUtils.getIntermediateInputKeyClass(jobConf);
    Class valueClass = ConfigUtils.getIntermediateInputValueClass(jobConf);
    LOG.info("Using keyClass: " + keyClass);
    LOG.info("Using valueClass: " + valueClass);
    RawComparator comparator =
        ConfigUtils.getInputKeySecondaryGroupingComparator(jobConf);
    LOG.info("Using comparator: " + comparator);

    reduceInputKeyCounter =
        mrReporter.getCounter(TaskCounter.REDUCE_INPUT_GROUPS);
View Full Code Here
TOP

Related Classes of org.apache.hadoop.io.RawComparator

  • com.datasalt.pangool.tuplemr.mapred.SortComparator
  • com.ebay.erl.mobius.core.mapred.ConfigurableJob
  • org.apache.crunch.impl.mem.collect.Shuffler
  • org.apache.hadoop.mapred.ReduceTask
  • org.apache.hadoop.mapred.ReduceTask$ReduceCopier
  • org.apache.hadoop.mrunit.mapreduce.TestMapReduceDriver
  • org.apache.hadoop.mrunit.TestMapReduceDriver
  • org.apache.tez.mapreduce.processor.reduce.ReduceProcessor
  • org.apache.tez.runtime.library.common.shuffle.impl.MergeManager
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.