Package org.apache.hadoop.raid.StripeReader

Examples of org.apache.hadoop.raid.StripeReader.LocationPair


          throws IOException {
    final int stripeSize = raidInfo.codec.stripeLength;
    final FileStatus fileStatus = dfs.getFileStatus(filePath);
    final BlockLocation[] fileBlocks =
      dfs.getFileBlockLocations(fileStatus, 0, fileStatus.getLen());
    LocationPair lp = StripeReader.getBlockLocation(raidInfo.codec, dfs,
        filePath, 0, conf);
    int startBlockIdx = lp.getStripeIdx() * stripeSize +
        lp.getBlockIdxInStripe();
    int startStripeIdx = lp.getStripeIdx();
    int endStripeIdx = (startBlockIdx + fileBlocks.length) / stripeSize;
    long blockSize = DirectoryStripeReader.getParityBlockSize(conf,
        lp.getListFileStatus());
    long numBlocks = DirectoryStripeReader.getBlockNum(lp.getListFileStatus());
    HashMap<Integer, Integer> allCorruptBlocksPerStripe =
        new HashMap<Integer, Integer>();
    checkParityBlocks(filePath, allCorruptBlocksPerStripe, blockSize,
        startStripeIdx, endStripeIdx,
        RaidNode.numStripes(numBlocks, stripeSize), raidInfo);
    DirectoryStripeReader sReader = (DirectoryStripeReader)
        StripeReader.getStripeReader(raidInfo.codec,
        conf, blockSize, dfs, lp.getStripeIdx(), fileStatus);
    // Get the corrupt block information for all stripes related to the file
    while (sReader.getCurStripeIdx() <= endStripeIdx) {
      int stripe = (int)sReader.getCurStripeIdx();
      BlockLocation[] bls = sReader.getNextStripeBlockLocations();
      for (BlockLocation bl : bls) {
View Full Code Here


    long startTime = System.currentTimeMillis();
    if (crc != null) {
      crc.reset();
    }
    int blockIdx = (int) (errorOffset/blockSize);
    LocationPair lp = null;
    int erasedLocationToFix;
    if (fixSource) {
      lp = StripeReader.getBlockLocation(codec, srcFs,
          srcFile, blockIdx, conf);
      erasedLocationToFix = codec.parityLength + lp.getBlockIdxInStripe();
    } else {
      lp = StripeReader.getParityBlockLocation(codec, blockIdx);
      erasedLocationToFix = lp.getBlockIdxInStripe();
    }

    FileStatus srcStat = srcFs.getFileStatus(srcFile);
    FileStatus parityStat = parityFs.getFileStatus(parityFile);

    InputStream[] inputs = null;
    List<Integer> erasedLocations = new ArrayList<Integer>();
    // Start off with one erased location.
    erasedLocations.add(erasedLocationToFix);
    List<Integer> locationsToRead = new ArrayList<Integer>(
        codec.parityLength + codec.stripeLength);

    int boundedBufferCapacity = 2;
    ParallelStreamReader parallelReader = null;
    LOG.info("Need to write " + limit +
             " bytes for erased location index " + erasedLocationToFix);
   
    long startOffsetInBlock = 0;
    if (partial) {
      startOffsetInBlock = errorOffset % blockSize;
    }

    // will be resized later
    int[] erasedLocationsArray = new int[0];
    int[] locationsToReadArray = new int[0];
    int[] locationsNotToReadArray = new int[0];

    try {
      numReadBytes = 0;
      long written;
      // Loop while the number of written bytes is less than the max.
      for (written = 0; written < limit; ) {
        try {
          if (parallelReader == null) {
            long offsetInBlock = written + startOffsetInBlock;
            StripeReader sReader = StripeReader.getStripeReader(codec,
                conf, blockSize, srcFs, lp.getStripeIdx(), srcStat);
            inputs = sReader.buildInputs(srcFs, srcFile,
                srcStat, parityFs, parityFile, parityStat,
                lp.getStripeIdx(), offsetInBlock, erasedLocations,
                locationsToRead, code);
           
            /*
             * locationsToRead have now been populated and erasedLocations
             * might have been updated with more erased locations.
View Full Code Here

    long decodingTime = 0;
    if (crc != null) {
      crc.reset();
    }
    int blockIdx = (int) (errorOffset/blockSize);
    LocationPair lp = null;
    int erasedLocationToFix;
   
    if (recoverFromStripeStore) {
      erasedLocationToFix = si.getBlockIdxInStripe(lostBlock);
    } else if (fixSource) {
      lp = StripeReader.getBlockLocation(codec, srcFs,
          srcFile, blockIdx, conf);
      erasedLocationToFix = codec.parityLength + lp.getBlockIdxInStripe();
    } else {
      lp = StripeReader.getParityBlockLocation(codec, blockIdx);
      erasedLocationToFix = lp.getBlockIdxInStripe();
    }

    FileStatus srcStat = srcFs.getFileStatus(srcFile);
    FileStatus parityStat = null;
    if (!recoverFromStripeStore) {
      parityStat = parityFs.getFileStatus(parityFile);
    }

    InputStream[] inputs = null;
    List<Integer> erasedLocations = new ArrayList<Integer>();
    // Start off with one erased location.
    erasedLocations.add(erasedLocationToFix);
    Set<Integer> locationsToNotRead = new HashSet<Integer>();

    int boundedBufferCapacity = 2;
    ParallelStreamReader parallelReader = null;
    LOG.info("Need to write " + limit +
             " bytes for erased location index " + erasedLocationToFix);
   
    long startOffsetInBlock = 0;
    if (partial) {
      startOffsetInBlock = errorOffset % blockSize;
    }
   
    try {
      int[] locationsToFix = new int[codec.parityLength];
      numReadBytes = 0;
      numReadBytesRemoteRack = 0;
      remoteRackFlag = new boolean[codec.parityLength + codec.stripeLength];
      for (int id = 0 ; id < codec.parityLength + codec.stripeLength ; id++) {
        remoteRackFlag[id] = false;
      }
      boolean stripeVerified = (si == null);
      long written;
      // Loop while the number of written bytes is less than the max.
      for (written = 0; written < limit; ) {
        try {
          if (parallelReader == null) {
            long offsetInBlock = written + startOffsetInBlock;
            if (recoverFromStripeStore) {
              inputs = StripeReader.buildInputsFromStripeInfo((DistributedFileSystem)srcFs,
                  srcStat, codec, si, offsetInBlock,
                  limit, erasedLocations, locationsToNotRead, code);
            } else {
              StripeReader sReader = StripeReader.getStripeReader(codec,
                conf, blockSize, srcFs, lp.getStripeIdx(), srcStat);
              inputs = sReader.buildInputs(srcFs, srcFile,
                srcStat, parityFs, parityFile, parityStat,
                lp.getStripeIdx(), offsetInBlock, erasedLocations,
                locationsToNotRead, code);
            }
            int i = 0;
            for (int location : locationsToNotRead) {
              locationsToFix[i] = location;
View Full Code Here

          throws IOException {
    final int stripeSize = raidInfo.codec.stripeLength;
    final Path filePath = fileStatus.getPath();
    final BlockLocation[] fileBlocks =
      dfs.getFileBlockLocations(fileStatus, 0, fileStatus.getLen());
    LocationPair lp = StripeReader.getBlockLocation(raidInfo.codec, dfs,
        filePath, 0, conf, raidInfo.parityPair.getListFileStatus());
    int startBlockIdx = lp.getStripeIdx() * stripeSize +
        lp.getBlockIdxInStripe();
    int startStripeIdx = lp.getStripeIdx();
    long endStripeIdx = RaidNode.numStripes((long)(startBlockIdx + fileBlocks.length),
        stripeSize);
    long blockSize = DirectoryStripeReader.getParityBlockSize(conf,
        lp.getListFileStatus());
    long numBlocks = DirectoryStripeReader.getBlockNum(lp.getListFileStatus());
    HashMap<Integer, Integer> allCorruptBlocksPerStripe =
        new HashMap<Integer, Integer>();
    checkParityBlocks(filePath, allCorruptBlocksPerStripe, blockSize,
        startStripeIdx, endStripeIdx,
        RaidNode.numStripes(numBlocks, stripeSize), raidInfo);
    DirectoryStripeReader sReader = new DirectoryStripeReader(conf,
        raidInfo.codec, dfs, lp.getStripeIdx(), -1L, filePath.getParent(),
        lp.getListFileStatus());
    // Get the corrupt block information for all stripes related to the file
    while (sReader.getCurrentStripeIdx() < endStripeIdx) {
      int stripe = (int)sReader.getCurrentStripeIdx();
      BlockLocation[] bls = sReader.getNextStripeBlockLocations();
      for (BlockLocation bl : bls) {
View Full Code Here

TOP

Related Classes of org.apache.hadoop.raid.StripeReader.LocationPair

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.