Package org.apache.hadoop.io

Examples of org.apache.hadoop.io.FileChunk

855
856
857
858
859
860
861
862
863
864
865
          }
        }

        final FileStatus fStatus = localFS.getFileStatus(outputPath);
        // add to list of final disk outputs.
        onDiskMapOutputs.add(new FileChunk(outputPath, 0, fStatus.getLen(), false));

        LOG.info("Merged " + numMemDiskSegments + " segments, " +
                 inMemToDiskBytes + " bytes to disk to satisfy " +
                 "reduce memory limit");
        inMemToDiskBytes = 0;
View Full Code Here
627
628
629
630
631
632
633
          writer.close();
        }
      }

      // Note the output of the merge
      closeOnDiskFile(new FileChunk(outputPath, 0, outFileLen, false));
    }
View Full Code Here
710
711
712
713
714
715
716
717
718
719
720
        localFS.delete(outputPath, true);
        throw e;
      }

      final long outputLen = localFS.getFileStatus(outputPath).getLen();
      closeOnDiskFile(new FileChunk(outputPath, 0, outputLen, false));

      LOG.info(inputContext.getUniqueIdentifier() +
          " Finished merging " + inputs.size() +
          " map output files on disk of total-size " +
          approxOutputSize + "." +
View Full Code Here
88
89
90
91
92
93
94
95
96
97
98
    this.tmpOutputPath = tmpOutputPath;
    this.disk = null;

    if (type == Type.DISK || type == Type.DISK_DIRECT) {
      boolean preserve = (type == Type.DISK_DIRECT); // type disk are temp files.
      this.outputPath = new FileChunk(outputPath, offset, size, preserve);
    } else {
      this.outputPath = null;
    }

  }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.io.FileChunk

  • org.apache.tez.runtime.library.common.shuffle.impl.MapOutput
  • org.apache.tez.runtime.library.common.shuffle.impl.MergeManager
  • org.apache.tez.runtime.library.common.shuffle.impl.MergeManager$InMemoryMerger
  • org.apache.tez.runtime.library.common.shuffle.impl.MergeManager$OnDiskMerger
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.