Examples of org.apache.hadoop.raid.ParityFilePair$FileStatusCache

org.apache.hadoop.raid.ParityFilePair
Caches the listStatus result in a thread local cache. This greatly increases the speed of repeatedly calling getParityFile() from the same thread.

        // we will rename the parity paths as well.
        srcParityList.add(parityPath);
        destParityList.add(new Path(codec.parityDirectory, makeRelative(dst)));
      } else {
        // check the HAR for file
        ParityFilePair parityInHar = 
            ParityFilePair.getParityFile(codec, src, conf);
        if (null != parityInHar) {
          // this means we have the parity file in HAR
          // will throw an exception.
          throw new IOException("We can not rename the file whose parity file" +

View Full Code Here

        this.innerConf.setBoolean("fs.hdfs.impl.disable.cache", true);
        
        // load the parity files
        this.parityFilePairs = new ArrayList<ParityFilePair>();
        for (Codec codec : Codec.getCodecs()) {
          ParityFilePair ppair = ParityFilePair.getParityFile(codec, 
              this.path, this.innerConf);
          this.parityFilePairs.add(ppair);
        }
        
        // Open a stream to the first block.

View Full Code Here

        srcParityList.add(parityPath);
        destParityList.add(new Path(codec.parityDirectory, makeRelative(dst)));
      } catch (FileNotFoundException ex) {
        // parity file does not exist
        // check the HAR for file
        ParityFilePair parityInHar = 
            ParityFilePair.getParityFile(codec, srcStat, conf);
        if (null != parityInHar) {
          // this means we have the parity file in HAR
          // will throw an exception.
          throw new IOException("We can not rename the file whose parity file" +

View Full Code Here

      // generate the parity files.
      doRaid(srcPath, codec);
      FileStatus stat = fileSys.getFileStatus(srcPath);
      
      // verify the GOOD_FILE
      ParityFilePair pfPair = ParityFilePair.getParityFile(codec, stat, conf);
      assertNotNull(pfPair);
      assertTrue(FastFileCheck.checkFile(conf, (DistributedFileSystem)fileSys, 
          fileSys, srcPath, pfPair.getPath(), codec, 
          RaidUtils.NULL_PROGRESSABLE, false));
      
      // verify the BAD_FILE
      fileSys.delete(srcPath);
      TestRaidDfs.createTestFilePartialLastBlock(fileSys, srcPath, 
          1, 8, 8192L);
      fileSys.setTimes(pfPair.getPath(), 
          fileSys.getFileStatus(srcPath).getModificationTime(), -1);
      stat = fileSys.getFileStatus(srcPath);
      pfPair = ParityFilePair.getParityFile(codec, stat, conf);
      assertNotNull(pfPair);
      assertFalse(FastFileCheck.checkFile(conf, (DistributedFileSystem)fileSys, 
          fileSys, srcPath, pfPair.getPath(), codec, 
          RaidUtils.NULL_PROGRESSABLE, false));
    } finally {
      stopCluster();
    }
  }

View Full Code Here

        return State.NOT_FOUND.toString();
      }
      boolean result = false;
      boolean raided = false;
      for (Codec codec : Codec.getCodecs()) {
        ParityFilePair pfPair = ParityFilePair.getParityFile(codec, srcStat, conf);
        if (pfPair != null) {
          raided = true;
          Path parityPath = pfPair.getPath();
          try {
            result = checkFile(conf, fs, fs, p, parityPath, codec, reporter
                , false);
          } catch (IOException ex) {
            LOG.warn("Encounter exception when checking the file: " + p, ex);

View Full Code Here

TOP

Related Classes of org.apache.hadoop.raid.ParityFilePair$FileStatusCache

org.apache.hadoop.fs.FileStatus

org.apache.hadoop.fs.FileSystem

org.apache.hadoop.fs.HarFileSystem

org.apache.hadoop.fs.Path

org.apache.hadoop.hdfs.DistributedRaidFileSystem

org.apache.hadoop.hdfs.DistributedRaidFileSystem$ExtFSDataInputStream$ExtFsInputStream

org.apache.hadoop.raid.tools.FastFileCheck$FileCheckMapper

org.apache.hadoop.raid.tools.TestFastFileCheck

java.net.URI

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.