Package org.apache.hadoop.io.SequenceFile.Sorter

Examples of org.apache.hadoop.io.SequenceFile.Sorter.RawKeyValueIterator

348
349
350
351
352
353
354
355
356
357
358
        mapOutputFile.getSpillFile(getTaskId(), numSpills).getName());
         
        //invoke the sort
        for (int i = 0; i < partitions; i++) {
          sortImpl[i].setInputBuffer(keyValBuffer);
          RawKeyValueIterator rIter = sortImpl[i].sort();
         
          startPartition(i);
          if (rIter != null) {
            //invoke the combiner if one is defined
            if (job.getCombinerClass() != null) {
View Full Code Here
684
685
686
687
688
689
690
691
692
693
694
        //in the merge method)
        SequenceFile.Writer writer = sorter.cloneFileAttributes(
            inMemFileSys.makeQualified(inMemClosedFiles[0]),
            localFileSys.makeQualified(inMemClosedFiles[0]), null);
       
        RawKeyValueIterator rIter = null;
        try {
          rIter = sorter.merge(inMemClosedFiles, true, inMemClosedFiles.length,
                       new Path(reduceTask.getTaskId()));
        } catch (Exception e) {
          //make sure that we delete the ondisk file that we created earlier
View Full Code Here
822
823
824
825
826
827
828
829
830
831
832
          //is called (we delete empty sequence files as soon as we see them
          //in the merge method)
          SequenceFile.Writer writer = sorter.cloneFileAttributes(
              inMemFileSys.makeQualified(inMemClosedFiles[0]),
              localFileSys.makeQualified(inMemClosedFiles[0]), null);
          RawKeyValueIterator rIter;
          try {
            rIter = sorter.merge(inMemClosedFiles, true,
              inMemClosedFiles.length, new Path(reduceTask.getTaskId()));
          } catch (Exception e) {
            //make sure that we delete the ondisk file that we created earlier
View Full Code Here
382
383
384
385
386
387
388
389
390
391
392
         
        //invoke the sort
        for (int i = 0; i < partitions; i++) {
          sortImpl[i].setInputBuffer(keyValBuffer);
          sortImpl[i].setProgressable(reporter);
          RawKeyValueIterator rIter = sortImpl[i].sort();
         
          startPartition(i);
          if (rIter != null) {
            //invoke the combiner if one is defined
            if (job.getCombinerClass() != null) {
View Full Code Here
537
538
539
540
541
542
543
544
545
546
547
            s.preserveInput(true);
            s.doSync();
            segmentList.add(i, s);
          }
          segmentStart = finalOut.getPos();
          RawKeyValueIterator kvIter = sorter.merge(segmentList, new Path(getTaskId()));
          SequenceFile.Writer writer = SequenceFile.createWriter(job, finalOut,
                                                                 job.getMapOutputKeyClass(), job.getMapOutputValueClass(),
                                                                 compressionType, codec);
          sorter.writeFile(kvIter, writer);
          //close the file - required esp. for block compression to ensure
View Full Code Here
598
599
600
601
602
603
604
605
606
607
608
        if (inMemClosedFiles.length == 0) {
          LOG.info(reduceTask.getTaskId() + "Nothing to merge from " +
              inMemFileSys.getUri());
          return numCopied == numOutputs;
        }
        RawKeyValueIterator rIter =
          sorter.merge(inMemClosedFiles, true, inMemClosedFiles.length,
                       new Path(reduceTask.getTaskId()));
        //name this output file same as the name of the first file that is
        //there in the current list of inmem files (this is guaranteed to be
        //absent on the disk currently. So we don't overwrite a prev.
View Full Code Here
720
721
722
723
724
725
726
727
728
729
730
        //Note that the above Path[] could be of length 0 if all copies are
        //in flight. So we make sure that we have some 'closed' map
        //output files to merge to get the benefit of in-memory merge
        if (inMemClosedFiles.length >=
          (int)(MAX_INMEM_FILESYS_USE/MAX_INMEM_FILESIZE_FRACTION)) {
          RawKeyValueIterator rIter = sorter.merge(inMemClosedFiles, true,
              inMemClosedFiles.length, new Path(reduceTask.getTaskId()));
          //name this output file same as the name of the first file that is
          //there in the current list of inmem files (this is guaranteed to be
          //absent on the disk currently. So we don't overwrite a prev.
          //created spill)
View Full Code Here
382
383
384
385
386
387
388
389
390
391
392
         
        //invoke the sort
        for (int i = 0; i < partitions; i++) {
          sortImpl[i].setInputBuffer(keyValBuffer);
          sortImpl[i].setProgressable(reporter);
          RawKeyValueIterator rIter = sortImpl[i].sort();
         
          startPartition(i);
          if (rIter != null) {
            //invoke the combiner if one is defined
            if (job.getCombinerClass() != null) {
View Full Code Here
537
538
539
540
541
542
543
544
545
546
547
            s.preserveInput(true);
            s.doSync();
            segmentList.add(i, s);
          }
          segmentStart = finalOut.getPos();
          RawKeyValueIterator kvIter = sorter.merge(segmentList, new Path(getTaskId()));
          SequenceFile.Writer writer = SequenceFile.createWriter(job, finalOut,
                                                                 job.getMapOutputKeyClass(), job.getMapOutputValueClass(),
                                                                 compressionType, codec);
          sorter.writeFile(kvIter, writer);
          //close the file - required esp. for block compression to ensure
View Full Code Here
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
            s.preserveInput(true);
            s.doSync();
            segmentList.add(i, s);
          }
          segmentStart = finalOut.getPos();
          RawKeyValueIterator kvIter = sorter.merge(segmentList, new Path(getTaskId()));
          SequenceFile.Writer writer = SequenceFile.createWriter(job, finalOut,
                                                                 job.getMapOutputKeyClass(), job.getMapOutputValueClass(),
                                                                 compressionType, codec);
          sorter.writeFile(kvIter, writer);
          //close the file - required esp. for block compression to ensure
View Full Code Here
TOP

Related Classes of org.apache.hadoop.io.SequenceFile.Sorter.RawKeyValueIterator

  • org.apache.hadoop.io.TestSequenceFileMergeProgress
  • org.apache.hadoop.mapred.MapTask$MapOutputBuffer
  • org.apache.hadoop.mapred.ReduceTaskRunner
  • org.apache.hadoop.mapred.ReduceTaskRunner$InMemFSMergeThread
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.