Examples of org.apache.tez.runtime.library.common.task.local.output.TezTaskOutputFiles

Package org.apache.tez.runtime.library.common.task.local.output

Examples of org.apache.tez.runtime.library.common.task.local.output.TezTaskOutputFiles

org.apache.tez.runtime.library.common.task.local.output.TezTaskOutputFiles
Manipulate the working area for the transient store for maps and reduces. This class is used by map and reduce tasks to identify the directories that they need to write to/read from for intermediate files. The callers of these methods are from child space and see mapreduce.cluster.local.dir as taskTracker/jobCache/jobId/attemptId This class should not be used from TaskTracker space.

    this.combiner = combiner;


    this.reduceCombineInputCounter = reduceCombineInputCounter;
    this.spilledRecordsCounter = spilledRecordsCounter;
    this.mergedMapOutputsCounter = mergedMapOutputsCounter;
    this.mapOutputFile = new TezTaskOutputFiles(conf, inputContext.getUniqueIdentifier());
    
    this.localFS = localFS;
    this.rfs = ((LocalFileSystem)localFS).getRaw();
    
    this.numDiskToDiskMerges = inputContext.getCounters().findCounter(TaskCounter.NUM_DISK_TO_DISK_MERGES);

View Full Code Here

      long maxTaskAvailableMemory, long memoryAvailable) {
    this.conf = conf;    
    this.maxAvailableTaskMemory = maxTaskAvailableMemory;
    this.initialMemoryAvailable = memoryAvailable;
    
    this.fileNameAllocator = new TezTaskOutputFiles(conf,
        uniqueIdentifier);
    this.localDirAllocator = new LocalDirAllocator(TezRuntimeFrameworkConfigs.LOCAL_DIRS);
    
    // Setup configuration
    final float maxInMemCopyUse = conf.getFloat(

View Full Code Here

    assertEquals(SHUFFLE_PORT, eventProto.getPort());
    assertEquals(uniqueId, eventProto.getPathComponent());


    // Verify the data
    // Verify the actual data
    TezTaskOutput taskOutput = new TezTaskOutputFiles(conf, uniqueId);
    Path outputFilePath = null;
    Path spillFilePath = null;
    try {
      outputFilePath = taskOutput.getOutputFile();
    } catch (DiskErrorException e) {
      if (numRecordsWritten > 0) {
        fail();
      } else {
        // Record checking not required.
        return;
      }
    }
    try {
      spillFilePath = taskOutput.getOutputIndexFile();
    } catch (DiskErrorException e) {
      if (numRecordsWritten > 0) {
        fail();
      } else {
        // Record checking not required.

View Full Code Here

      assertFalse(eventProto.hasPort());
      assertFalse(eventProto.hasPathComponent());
    }


    // Verify the actual data
    TezTaskOutput taskOutput = new TezTaskOutputFiles(conf, uniqueId);
    Path outputFilePath = null;
    Path spillFilePath = null;
    try {
      outputFilePath = taskOutput.getOutputFile();
    } catch (DiskErrorException e) {
      if (numRecordsWritten > 0) {
        fail();
      } else {
        // Record checking not required.
        return;
      }
    }
    try {
      spillFilePath = taskOutput.getOutputIndexFile();
    } catch (DiskErrorException e) {
      if (numRecordsWritten > 0) {
        fail();
      } else {
        // Record checking not required.

View Full Code Here

    this.combiner = combiner;


    this.reduceCombineInputCounter = reduceCombineInputCounter;
    this.spilledRecordsCounter = spilledRecordsCounter;
    this.mergedMapOutputsCounter = mergedMapOutputsCounter;
    this.mapOutputFile = new TezTaskOutputFiles(conf, inputContext.getUniqueIdentifier());
    
    this.localFS = localFS;
    this.rfs = ((LocalFileSystem)localFS).getRaw();


  }

View Full Code Here


  @Private
  void configureAndStart() {
    Preconditions.checkState(initialMemoryAvailable != -1,
        "Initial memory must be configured before starting");
    this.fileNameAllocator = new TezTaskOutputFiles(conf,
        uniqueIdentifier);
    this.localDirAllocator = new LocalDirAllocator(TezJobConfig.LOCAL_DIRS);


    // Setup configuration
    final float maxInMemCopyUse = conf.getFloat(

View Full Code Here

  private volatile long usedMemory = 0;


  public BroadcastInputManager(String uniqueIdentifier, Configuration conf) {
    this.conf = conf;


    this.fileNameAllocator = new TezTaskOutputFiles(conf,
        uniqueIdentifier);
    this.localDirAllocator = new LocalDirAllocator(TezJobConfig.LOCAL_DIRS);


    // Setup configuration
    final float maxInMemCopyUse = conf.getFloat(

View Full Code Here

    this.combiner = combiner;


    this.reduceCombineInputCounter = reduceCombineInputCounter;
    this.spilledRecordsCounter = spilledRecordsCounter;
    this.mergedMapOutputsCounter = mergedMapOutputsCounter;
    this.mapOutputFile = new TezTaskOutputFiles(conf, inputContext.getUniqueIdentifier());
    
    this.localFS = localFS;
    this.rfs = ((LocalFileSystem)localFS).getRaw();


    if (ConfigUtils.isIntermediateInputCompressed(conf)) {

View Full Code Here

    this.combiner = combiner;


    this.reduceCombineInputCounter = reduceCombineInputCounter;
    this.spilledRecordsCounter = spilledRecordsCounter;
    this.mergedMapOutputsCounter = mergedMapOutputsCounter;
    this.mapOutputFile = new TezTaskOutputFiles(conf, inputContext.getUniqueIdentifier());
    
    this.localFS = localFS;
    this.rfs = ((LocalFileSystem)localFS).getRaw();
    
    this.numDiskToDiskMerges = inputContext.getCounters().findCounter(TaskCounter.NUM_DISK_TO_DISK_MERGES);

View Full Code Here

      long maxTaskAvailableMemory, long memoryAvailable) {
    this.conf = conf;    
    this.maxAvailableTaskMemory = maxTaskAvailableMemory;
    this.initialMemoryAvailable = memoryAvailable;
    
    this.fileNameAllocator = new TezTaskOutputFiles(conf,
        uniqueIdentifier);
    this.localDirAllocator = new LocalDirAllocator(TezJobConfig.LOCAL_DIRS);
    
    // Setup configuration
    final float maxInMemCopyUse = conf.getFloat(

View Full Code Here

TOP

Related Classes of org.apache.tez.runtime.library.common.task.local.output.TezTaskOutputFiles

org.apache.hadoop.fs.Path

org.apache.tez.runtime.library.broadcast.input.BroadcastInputManager

org.apache.tez.runtime.library.common.shuffle.impl.MergeManager

org.apache.tez.runtime.library.common.writers.TestUnorderedPartitionedKVWriter

org.apache.tez.runtime.library.shuffle.common.impl.SimpleFetchedInputAllocator

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.