Package org.apache.tez.runtime.library.common.shuffle.impl

Examples of org.apache.tez.runtime.library.common.shuffle.impl.ShuffleHeader

360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
      long startTime = System.currentTimeMillis();
      int responsePartition = -1;
      // Read the shuffle header
      String pathComponent = null;
      try {
        ShuffleHeader header = new ShuffleHeader();
        header.readFields(input);
        pathComponent = header.getMapId();

        srcAttemptId = pathToAttemptMap.get(pathComponent);
        compressedLength = header.getCompressedLength();
        decompressedLength = header.getUncompressedLength();
        responsePartition = header.getPartition();
      } catch (IllegalArgumentException e) {
        // badIdErrs.increment(1);
        LOG.warn("Invalid src id ", e);
        // Don't know which one was bad, so consider all of them as bad
        return remaining.toArray(new InputAttemptIdentifier[remaining.size()]);
View Full Code Here
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
      long startTime = System.currentTimeMillis();
      int responsePartition = -1;
      // Read the shuffle header
      String pathComponent = null;
      try {
        ShuffleHeader header = new ShuffleHeader();
        header.readFields(input);
        pathComponent = header.getMapId();

        srcAttemptId = pathToAttemptMap.get(pathComponent);
        compressedLength = header.getCompressedLength();
        decompressedLength = header.getUncompressedLength();
        responsePartition = header.getPartition();
      } catch (IllegalArgumentException e) {
        // badIdErrs.increment(1);
        LOG.warn("Invalid src id ", e);
        // Don't know which one was bad, so consider all of them as bad
        return remaining.toArray(new InputAttemptIdentifier[remaining.size()]);
View Full Code Here
88
89
90
91
92
93
94
95
96
97
98
        ++spindex;
      }
      length += IFILE_EOF_LENGTH;
     
      shuffleHeaders.add(
          new ShuffleHeader(
              outputContext.getUniqueIdentifier(), // TODO Verify that this is correct.
              length + IFILE_CHECKSUM_LENGTH, length, i)
          );
      LOG.info("shuffleHeader[" + i + "]:" +
          " rawLen=" + length + " partLen=" + (length + IFILE_CHECKSUM_LENGTH) +
View Full Code Here
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
    }

    protected ChannelFuture sendMapOutput(ChannelHandlerContext ctx, Channel ch,
        String user, String jobId, String mapId, int reduce)
        throws IOException {
      final ShuffleHeader header = sorter.getShuffleHeader(reduce);
      final DataOutputBuffer dob = new DataOutputBuffer();
      header.write(dob);
      ch.write(wrappedBuffer(dob.getData(), 0, dob.getLength()));

      ChannelFuture writeFuture =
          ch.write(
              new ChunkedStream(
                  sorter.getSortedStream(reduce), sslFileBufferSize
                  )
              );
      metrics.shuffleConnections.incr();
      metrics.shuffleOutputBytes.incr(header.getCompressedLength()); // optimistic
      return writeFuture;
    }
View Full Code Here
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
      long startTime = System.currentTimeMillis();
      int responsePartition = -1;
      // Read the shuffle header
      String pathComponent = null;
      try {
        ShuffleHeader header = new ShuffleHeader();
        header.readFields(input);
        pathComponent = header.getMapId();

        srcAttemptId = pathToAttemptMap.get(pathComponent);
        compressedLength = header.getCompressedLength();
        decompressedLength = header.getUncompressedLength();
        responsePartition = header.getPartition();
      } catch (IllegalArgumentException e) {
        // badIdErrs.increment(1);
        LOG.warn("Invalid src id ", e);
        // Don't know which one was bad, so consider all of them as bad
        return remaining.toArray(new InputAttemptIdentifier[remaining.size()]);
View Full Code Here
TOP

Related Classes of org.apache.tez.runtime.library.common.shuffle.impl.ShuffleHeader

  • org.apache.tez.runtime.library.common.shuffle.server.ShuffleHandler$Shuffle
  • org.apache.tez.runtime.library.common.sort.impl.dflt.InMemoryShuffleSorter
  • org.apache.tez.runtime.library.shuffle.common.Fetcher
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.