Examples of InputAttemptIdentifier

org.apache.tez.runtime.library.common.InputAttemptIdentifier
Container for a task number and an attempt number for the task.

Examples of org.apache.tez.runtime.library.common.InputAttemptIdentifier

      // with the first map, typically lost map. So, penalize only that map
      // and add the rest
      if (isShutDown.get()) {
        LOG.info("Not reporting fetch failure, since an Exception was caught after shutdown");
      } else {
        InputAttemptIdentifier firstAttempt = srcAttempts.get(0);
        LOG.warn("Fetch Failure from host while connecting: " + host + ", attempt: " + firstAttempt
            + " Informing ShuffleManager: ", e);
        return new HostFetchResult(new FetchResult(host, port, partition, remaining),
            new InputAttemptIdentifier[] { firstAttempt }, false);
      }

View Full Code Here

Examples of org.apache.tez.runtime.library.common.InputAttemptIdentifier

  @VisibleForTesting
  protected HostFetchResult setupLocalDiskFetch() {


    Iterator<InputAttemptIdentifier> iterator = remaining.iterator();
    while (iterator.hasNext()) {
      InputAttemptIdentifier srcAttemptId = iterator.next();
      //TODO: check for shutdown? - See TEZ-1480
      long startTime = System.currentTimeMillis();


      FetchedInput fetchedInput = null;
      try {
        TezIndexRecord idxRecord;
        idxRecord = getTezIndexRecord(srcAttemptId);


        fetchedInput = new LocalDiskFetchedInput(idxRecord.getStartOffset(),
            idxRecord.getRawLength(), idxRecord.getPartLength(), srcAttemptId,
            getShuffleInputFileName(srcAttemptId.getPathComponent(), null), conf,
            new FetchedInputCallback() {
              @Override
              public void fetchComplete(FetchedInput fetchedInput) {}


              @Override

View Full Code Here

Examples of org.apache.tez.runtime.library.common.InputAttemptIdentifier

    }
  }


  private InputAttemptIdentifier[] fetchInputs(DataInputStream input) {
    FetchedInput fetchedInput = null;
    InputAttemptIdentifier srcAttemptId = null;
    long decompressedLength = -1;
    long compressedLength = -1;


    try {
      long startTime = System.currentTimeMillis();

View Full Code Here

Examples of org.apache.tez.runtime.library.common.InputAttemptIdentifier

    } catch (IOException e) {
      // ioErrs.increment(1);
      // If we got a read error at this stage, it implies there was a problem
      // with the first map, typically lost map. So, penalize only that map
      // and add the rest
      InputAttemptIdentifier firstAttempt = srcAttempts.get(0);
      LOG.warn("Fetch Failure from host while connecting: " + host
          + ", attempt: " + firstAttempt + " Informing ShuffleManager: ", e);
      fetcherCallback.fetchFailed(host, firstAttempt, false);
      return new FetchResult(host, port, partition, remaining);
    }

View Full Code Here

Examples of org.apache.tez.runtime.library.common.InputAttemptIdentifier


  }


  private InputAttemptIdentifier[] fetchInputs(DataInputStream input) {
    FetchedInput fetchedInput = null;
    InputAttemptIdentifier srcAttemptId = null;
    long decompressedLength = -1;
    long compressedLength = -1;


    try {
      long startTime = System.currentTimeMillis();

View Full Code Here

Examples of org.apache.tez.runtime.library.common.InputAttemptIdentifier

        }
      } else {
        // If we got a read error at this stage, it implies there was a problem
        // with the first map, typically lost map. So, penalize only that map
        // and add the rest
        InputAttemptIdentifier firstMap = srcAttempts.get(0);
        scheduler.copyFailed(firstMap, host, connectSucceeded);
      }
      
      // Add back all the remaining maps, WITHOUT marking them as failed
      for(InputAttemptIdentifier left: remaining) {

View Full Code Here

Examples of org.apache.tez.runtime.library.common.InputAttemptIdentifier

  private static InputAttemptIdentifier[] EMPTY_ATTEMPT_ID_ARRAY = new InputAttemptIdentifier[0];
  
  private InputAttemptIdentifier[] copyMapOutput(MapHost host,
                                DataInputStream input) {
    MapOutput mapOutput = null;
    InputAttemptIdentifier srcAttemptId = null;
    long decompressedLength = -1;
    long compressedLength = -1;
    
    try {
      long startTime = System.currentTimeMillis();

View Full Code Here

Examples of org.apache.tez.runtime.library.common.InputAttemptIdentifier

    } catch (InvalidProtocolBufferException e) {
      throw new TezUncheckedException("Unable to parse DataMovementEvent payload", e);
    } 
    int partitionId = dmEvent.getSourceIndex();
    URI baseUri = getBaseURI(shufflePayload.getHost(), shufflePayload.getPort(), partitionId);
    InputAttemptIdentifier srcAttemptIdentifier = new InputAttemptIdentifier(dmEvent.getTargetIndex(), dmEvent.getVersion(), shufflePayload.getPathComponent());
    LOG.info("DataMovementEvent baseUri:" + baseUri + ", src: " + srcAttemptIdentifier);
    scheduler.addKnownMapOutput(shufflePayload.getHost(), partitionId, baseUri.toString(), srcAttemptIdentifier);
    
    // TODO NEWTEZ See if this duration hack can be removed.
    int duration = shufflePayload.getRunDuration();

View Full Code Here

Examples of org.apache.tez.runtime.library.common.InputAttemptIdentifier

      scheduler.informMaxMapRunTime(maxMapRuntime);
    }
  }
  
  private void processTaskFailedEvent(InputFailedEvent ifEvent) {
    InputAttemptIdentifier taIdentifier = new InputAttemptIdentifier(ifEvent.getTargetIndex(), ifEvent.getVersion());
    scheduler.obsoleteMapOutput(taIdentifier);
    LOG.info("Obsoleting output of src-task: " + taIdentifier);
  }

View Full Code Here

Examples of org.apache.tez.runtime.library.common.InputAttemptIdentifier

    public void merge(List<MapOutput> inputs) throws IOException {
      if (inputs == null || inputs.size() == 0) {
        return;
      }


      InputAttemptIdentifier dummyMapId = inputs.get(0).getAttemptIdentifier(); 
      List<Segment> inMemorySegments = new ArrayList<Segment>();
      long mergeOutputSize = 
        createInMemorySegments(inputs, inMemorySegments, 0);
      int noInMemorySegments = inMemorySegments.size();

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.