Package org.apache.tez.runtime.library.common

Examples of org.apache.tez.runtime.library.common.InputAttemptIdentifier

102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
    if (shufflePayload.hasEmptyPartitions()) {
      byte[] emptyPartitions = TezUtils.decompressByteStringToByteArray(shufflePayload
          .getEmptyPartitions());
      BitSet emptyPartionsBitSet = TezUtils.fromByteArray(emptyPartitions);
      if (emptyPartionsBitSet.get(srcIndex)) {
        InputAttemptIdentifier srcAttemptIdentifier = new InputAttemptIdentifier(dme.getTargetIndex(),
            dme.getVersion());
        LOG.info("Source partition: " + srcIndex + " did not generate any data. Not fetching.");
        shuffleManager.addCompletedInputWithNoData(srcAttemptIdentifier);
        return;
      }
    } else {
      InputAttemptIdentifier srcAttemptIdentifier = new InputAttemptIdentifier(dme.getTargetIndex(),
          dme.getVersion(), shufflePayload.getPathComponent());
      if (shufflePayload.hasData()) {
        DataProto dataProto = shufflePayload.getData();
        FetchedInput fetchedInput = inputAllocator.allocate(dataProto.getRawLength(),
            dataProto.getCompressedLength(), srcAttemptIdentifier);
View Full Code Here
144
145
146
147
148
149
150
151
          + fetchedInput.getType());
    }
  }
 
  private void processInputFailedEvent(InputFailedEvent ife) {
    InputAttemptIdentifier srcAttemptIdentifier = new InputAttemptIdentifier(ife.getTargetIndex(), ife.getVersion());
    shuffleManager.obsoleteKnownInput(srcAttemptIdentifier);
  }
View Full Code Here
310
311
312
313
314
315
316
317
318
319
320
321
322
    // remove from the obsolete list.
    List<InputAttemptIdentifier> pendingInputsForHost = inputHost
        .clearAndGetPendingInputs();
    for (Iterator<InputAttemptIdentifier> inputIter = pendingInputsForHost
        .iterator(); inputIter.hasNext();) {
      InputAttemptIdentifier input = inputIter.next();
      // Avoid adding attempts which have already completed.
      if (completedInputSet.contains(input.getInputIdentifier())) {
        inputIter.remove();
      }
      // Avoid adding attempts which have been marked as OBSOLETE
      if (obsoletedInputs.contains(input)) {
        inputIter.remove();
View Full Code Here
TOP

Related Classes of org.apache.tez.runtime.library.common.InputAttemptIdentifier

  • org.apache.tez.mapreduce.processor.map.TestMapProcessor
  • org.apache.tez.runtime.library.broadcast.input.BroadcastShuffleInputEventHandler
  • org.apache.tez.runtime.library.broadcast.input.BroadcastShuffleManager
  • org.apache.tez.runtime.library.broadcast.input.TestBroadcastInputManager
  • org.apache.tez.runtime.library.common.shuffle.impl.Fetcher
  • org.apache.tez.runtime.library.common.shuffle.impl.MergeManager$InMemoryMerger
  • org.apache.tez.runtime.library.common.shuffle.impl.MergeManager$IntermediateMemoryToMemoryMerger
  • org.apache.tez.runtime.library.common.shuffle.impl.ShuffleInputEventHandler
  • org.apache.tez.runtime.library.common.shuffle.impl.ShuffleScheduler
  • org.apache.tez.runtime.library.common.shuffle.impl.TestFetcher
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.