Examples of org.apache.hadoop.raid.Codec

org.apache.hadoop.raid.Codec
A class with the information of a raid codec. A raid codec has the information of 1. Which ErasureCode used 2. Stripe and parity length 3. Parity directory location 4. Codec priority

   *              null if it is the block which is currently being written to
   * @return the block locations of companion blocks
   */
  List<LocatedBlock> getCompanionBlocks(String path, FileInfo info, Block block)
      throws IOException {
    Codec codec = info.codec;
    switch (info.type) {
      case NOT_RAID:
        return Collections.emptyList();
      case HAR_TEMP_PARITY:
        return getCompanionBlocksForHarParityBlock(

View Full Code Here

        
        while (nextLocation < Codec.getCodecs().size()) {
          
          try {
            int idx = nextLocation++;
            Codec codec = Codec.getCodecs().get(idx);
            
            DecoderInputStream recoveryStream = 
                RaidNode.unRaidCorruptInputStream(innerConf, path, 
                codec, parityFilePairs.get(idx), blockSize, offset, limit);

View Full Code Here

  public void testGetCompanionBLocks() throws IOException {


    try {
      setupCluster();
      String[] files = new String[] {"/dir/file1", "/dir/file2", "/dir/file3"};
      Codec codec = Codec.getCodec("rs");
      for (String file : files) {
        TestRaidDfs.createTestFile(fs, new Path(file), 3, 2, 8192L);
      }
      FileStatus stat = fs.getFileStatus(new Path("/dir"));
      RaidNode.doDirRaid(conf, stat, new Path(codec.parityDirectory), codec,

View Full Code Here

   * @param inode the inode of the path file 
   * @return the block locations of companion blocks
   */
  List<LocatedBlock> getCompanionBlocks(String path, FileInfo info, Block block, FSInodeInfo inode)
      throws IOException {
    Codec codec = info.codec;
    switch (info.type) {
      case NOT_RAID:
        return Collections.emptyList();
      case HAR_TEMP_PARITY:
        return getCompanionBlocksForHarParityBlock(

View Full Code Here

      String uniqueSubtringId = c.parityDirectory + "/";
      if (fileName.contains(uniqueSubtringId)) {
        return c;
      }
    }
    Codec c = this.acceptedCodecs.get(0);
    FSNamesystem.LOG.error("F4: Could not find any valid codec for the file: "
     + fileName + ", hence returning the first one: " + c.id);
    return c;
  }

View Full Code Here

     + fileName + ", hence returning the first one: " + c.id);
    return c;
  }


  private String getParityStagingDir(String parityFileName) {
    Codec c  = getCodec(parityFileName);
    return c.parityDirectory + this.stagingDir;
  }

View Full Code Here

        if (isStaging(fileName)) {
          nameWithINode = getSourceFile(fileName,
                                        getParityStagingDir(fileName));
        }
        if (nameWithINode == null) {
          Codec c = getCodec(fileName);
          nameWithINode = getSourceFile(fileName, c.parityDirectory);
        }
        return ((nameWithINode ==  null) ? null : nameWithINode.name);
      case TEMP_PARITY:
        Codec c = getCodec(fileName);
        nameWithINode = getSourceFile(fileName, c.tmpParityDirectory);
        return ((nameWithINode ==  null) ? null : nameWithINode.name);
      default:
        FSNamesystem.LOG.error("file type bad");
        return null;

View Full Code Here

          if (srcFileName == null ||
              this.namesystem.getHdfsFileInfo(srcFileName) == null) {
            srcFileName = null;
            FSNamesystem.LOG.error("F4: " + srcFileName + " does not exist");
          }
          Codec c = getCodec(fileName);
          parityLength = c.parityLength;
          stripeLength = c.stripeLength;
          stripeIndex = blockIndex / parityLength;
          break;
        default:

View Full Code Here

        int oldNextLocation = nextLocation;
        // First search for parity without stripe store
        while (nextLocation < Codec.getCodecs().size()) {
          try {
            int idx = nextLocation++;
            Codec codec = Codec.getCodecs().get(idx);
            if (!parityFilePairs.containsKey(codec.id)) {
              parityFilePairs.put(codec.id, ParityFilePair.getParityFile(codec, 
                  srcStat, this.innerConf));
            }
            
            DecoderInputStream recoveryStream = 
                RaidNode.unRaidCorruptInputStream(innerConf, path, 
                codec, parityFilePairs.get(codec.id), lostBlock.getBlock(),
                blockSize, offset, limit, false);
            
            if (null != recoveryStream) {
              return recoveryStream;
            }
            
          } catch (Exception e) {
            LOG.info("Ignoring error in using alternate path " + path, e);
          }
        }
        // Second look up in the stripe store for dir-raid
        nextLocation = oldNextLocation;
        while (nextLocation < Codec.getCodecs().size()) {
          try {
            int idx = nextLocation++;
            Codec codec = Codec.getCodecs().get(idx);
            if (!codec.isDirRaid) 
              continue;
            DecoderInputStream recoveryStream = 
                RaidNode.unRaidCorruptInputStream(innerConf, path, 
                codec, parityFilePairs.get(idx), lostBlock.getBlock(),

View Full Code Here

      return false;
    }


    // delete the parity file
    for (int i = 0; i < parityPathList.size(); i++) {
      Codec codec = usedCodec.get(i);
      if (codec.isDirRaid && !stat.isDir()) {
        // will not delete the parity for the whole directory if we 
        // only want to delete one file in the directory.
        continue;
      }

View Full Code Here

0 1

TOP

Related Classes of org.apache.hadoop.raid.Codec

org.apache.hadoop.hdfs.DistributedRaidFileSystem

org.apache.hadoop.hdfs.DistributedRaidFileSystem$ExtFSDataInputStream$ExtFsInputStream

org.apache.hadoop.hdfs.server.namenode.BlockPlacementPolicyRaid

org.apache.hadoop.hdfs.server.namenode.FaultTolerantBlockPlacementPolicy

org.apache.hadoop.hdfs.server.namenode.TestDirectoryRaidBlockPlacement

org.apache.hadoop.raid.tools.FastFileCheck$FileCheckMapper

org.apache.hadoop.raid.tools.RSBenchmark

org.apache.hadoop.raid.tools.TestFastFileCheck

org.json.JSONArray

java.lang.IllegalArgumentException

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.