Package org.apache.hadoop.io.ReadaheadPool

Examples of org.apache.hadoop.io.ReadaheadPool.ReadaheadRequest


        String filePath = mapOutputFileName.toUri().getPath();
        mapOutputIn = SecureIOUtils.openForRead(
            new File(filePath), runAsUserName);
            //new File(mapOutputFileName.toUri().getPath()), runAsUserName);

        ReadaheadRequest curReadahead = null;
       
        //seek to the correct offset for the reduce
        mapOutputIn.skip(info.startOffset);
        long rem = info.partLength;
        long offset = info.startOffset;
        while (rem > 0) {
          if (tracker.manageOsCacheInShuffle && tracker.readaheadPool != null) {
            curReadahead = tracker.readaheadPool.readaheadStream(filePath,
                mapOutputIn.getFD(), offset, tracker.readaheadLength,
                info.startOffset + info.partLength, curReadahead);
          }
          int len = mapOutputIn.read(buffer, 0,
              (int) Math.min(rem, MAX_BYTES_TO_READ));
          if (len < 0) {
            break;
          }
          rem -= len;
          offset += len;
          try {
            shuffleMetrics.outputBytes(len);
            outStream.write(buffer, 0, len);
            outStream.flush();
          } catch (IOException ie) {
            isInputException = false;
            throw ie;
          }
          totalRead += len;
        }
       
        if (curReadahead != null) {
          curReadahead.cancel();
        }

        // drop cache if possible
        if (tracker.manageOsCacheInShuffle && info.partLength > 0) {
          NativeIO.posixFadviseIfPossible(mapOutputIn.getFD(),
View Full Code Here


        String filePath = mapOutputFileName.toUri().getPath();
        mapOutputIn = SecureIOUtils.openForRead(
            new File(filePath), runAsUserName);
            //new File(mapOutputFileName.toUri().getPath()), runAsUserName);

        ReadaheadRequest curReadahead = null;
       
        //seek to the correct offset for the reduce
        mapOutputIn.skip(info.startOffset);
        long rem = info.partLength;
        long offset = info.startOffset;
        while (rem > 0) {
          if (tracker.manageOsCacheInShuffle && tracker.readaheadPool != null) {
            curReadahead = tracker.readaheadPool.readaheadStream(filePath,
                mapOutputIn.getFD(), offset, tracker.readaheadLength,
                info.startOffset + info.partLength, curReadahead);
          }
          int len = mapOutputIn.read(buffer, 0,
              (int) Math.min(rem, MAX_BYTES_TO_READ));
          if (len < 0) {
            break;
          }
          rem -= len;
          offset += len;
          try {
            shuffleMetrics.outputBytes(len);
            outStream.write(buffer, 0, len);
            outStream.flush();
          } catch (IOException ie) {
            isInputException = false;
            throw ie;
          }
          totalRead += len;
        }
       
        if (curReadahead != null) {
          curReadahead.cancel();
        }

        // drop cache if possible
        if (tracker.manageOsCacheInShuffle && info.partLength > 0) {
          NativeIO.posixFadviseIfPossible(mapOutputIn.getFD(),
View Full Code Here

        String filePath = mapOutputFileName.toUri().getPath();
        mapOutputIn = SecureIOUtils.openForRead(
            new File(filePath), runAsUserName);
            //new File(mapOutputFileName.toUri().getPath()), runAsUserName);

        ReadaheadRequest curReadahead = null;
       
        //seek to the correct offset for the reduce
        mapOutputIn.skip(info.startOffset);
        long rem = info.partLength;
        long offset = info.startOffset;
        while (rem > 0) {
          if (tracker.manageOsCacheInShuffle && tracker.readaheadPool != null) {
            curReadahead = tracker.readaheadPool.readaheadStream(filePath,
                mapOutputIn.getFD(), offset, tracker.readaheadLength,
                info.startOffset + info.partLength, curReadahead);
          }
          int len = mapOutputIn.read(buffer, 0,
              (int) Math.min(rem, MAX_BYTES_TO_READ));
          if (len < 0) {
            break;
          }
          rem -= len;
          offset += len;
          try {
            shuffleMetrics.outputBytes(len);
            outStream.write(buffer, 0, len);
            outStream.flush();
          } catch (IOException ie) {
            isInputException = false;
            throw ie;
          }
          totalRead += len;
        }
       
        if (curReadahead != null) {
          curReadahead.cancel();
        }

        // drop cache if possible
        if (tracker.manageOsCacheInShuffle && info.partLength > 0) {
          NativeIO.posixFadviseIfPossible(mapOutputIn.getFD(),
View Full Code Here

        String filePath = mapOutputFileName.toUri().getPath();
        mapOutputIn = SecureIOUtils.openForRead(
            new File(filePath), runAsUserName);
            //new File(mapOutputFileName.toUri().getPath()), runAsUserName);

        ReadaheadRequest curReadahead = null;
       
        //seek to the correct offset for the reduce
        mapOutputIn.skip(info.startOffset);
        long rem = info.partLength;
        long offset = info.startOffset;
        while (rem > 0) {
          if (tracker.manageOsCacheInShuffle && tracker.readaheadPool != null) {
            curReadahead = tracker.readaheadPool.readaheadStream(filePath,
                mapOutputIn.getFD(), offset, tracker.readaheadLength,
                info.startOffset + info.partLength, curReadahead);
          }
          int len = mapOutputIn.read(buffer, 0,
              (int) Math.min(rem, MAX_BYTES_TO_READ));
          if (len < 0) {
            break;
          }
          rem -= len;
          offset += len;
          try {
            shuffleMetrics.outputBytes(len);
            outStream.write(buffer, 0, len);
            outStream.flush();
          } catch (IOException ie) {
            isInputException = false;
            throw ie;
          }
          totalRead += len;
        }
       
        if (curReadahead != null) {
          curReadahead.cancel();
        }

        // drop cache if possible
        if (tracker.manageOsCacheInShuffle && info.partLength > 0) {
          NativeIO.posixFadviseIfPossible(mapOutputIn.getFD(),
View Full Code Here

TOP

Related Classes of org.apache.hadoop.io.ReadaheadPool.ReadaheadRequest

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.