Package org.apache.hadoop.raid

Examples of org.apache.hadoop.raid.Codec

400
401
402
403
404
405
406
407
408
409
410
      return false;
    }
   
    // rename the parity file
    for (int i=0; i<srcParityList.size(); i++) {
      Codec codec = usedCodec.get(i);
      if (codec.isDirRaid && !srcStat.isDir()) {
        // if we just rename one file in a dir-raided directory,
        // and we also have other files in the same directory,
        // we will keep the parity file.
        Path parent = src.getParent();
View Full Code Here
211
212
213
214
215
216
217
218
219
220
221
        TestRaidDfs.createTestFile(fs, filePath, 1, 1, blockSize);
        printLocatedBlocks(filePath);
      }
      LOG.info("Created " + (numFiles - 1) + " files");
      // create fake parity file
      Codec code = Codec.getCodec("xor");
      long numStripes = RaidNode.numStripes(numFiles, code.stripeLength);
      Path parityPath = new Path(code.parityDirectory, "dir");
      TestRaidDfs.createTestFile(fs, parityPath, 1,
          (int)numStripes * code.parityLength, blockSize);
      LOG.info("Create parity file: " + parityPath);
View Full Code Here
270
271
272
273
274
275
276
277
278
279
280
      for (int i = 0; i < numFiles; i++) {
        String file = parentDir + "file" + i;
        TestRaidDfs.createTestFile(fs, new Path(file), 3, 1, blockSize);
      }
      LOG.info("Created " + numFiles + " files");
      Codec code = Codec.getCodec("xor");
      FSNamesystem fsNameSys = cluster.getNameNode().namesystem;
      for (DatanodeDescriptor dd: fsNameSys.datanodeMap.values()) {
        LOG.info(dd);
      }
      // create fake parity file
View Full Code Here
302
303
304
305
306
307
308
309
310
311
312
  public void testGetCompanionBLocks() throws IOException {

    try {
      setupCluster(false, 1L, racks1, hosts1);
      String[] files = new String[] {"/dir/file1", "/dir/file2", "/dir/file3"};
      Codec codec = Codec.getCodec("rs");
      for (String file : files) {
        TestRaidDfs.createTestFile(fs, new Path(file), 3, 2, 8192L);
      }
      FileStatus stat = fs.getFileStatus(new Path("/dir"));
      RaidNode.doRaid(conf, stat, new Path(codec.parityDirectory), codec,
View Full Code Here
114
115
116
117
118
119
120
121
122
123
124
      Path srcPath = new Path("/user/dikang/raidtest/file0");
      int numBlocks = 8;
      TestRaidDfs.createTestFilePartialLastBlock(fileSys, srcPath,
          1, numBlocks, 8192L);
      assertTrue(fileSys.exists(srcPath));
      Codec codec = Codec.getCodec("rs");
      FileStatus stat = fileSys.getFileStatus(srcPath);
     
      // verify good file
      assertTrue(FastFileCheck.checkFile(conf, (DistributedFileSystem)fileSys,
          fileSys, srcPath, null, codec,
View Full Code Here
151
152
153
154
155
156
157
158
159
160
161
      Path srcPath = new Path("/user/dikang/raidtest/file0");
      TestRaidDfs.createTestFilePartialLastBlock(fileSys, srcPath,
          1, 8, 8192L);
     
      assertTrue(fileSys.exists(srcPath));
      Codec codec = Codec.getCodec("rs");

      // generate the parity files.
      doRaid(srcPath, codec);
      FileStatus stat = fileSys.getFileStatus(srcPath);
     
View Full Code Here
225
226
227
228
229
230
231
232
233
234
235
      FileSystem fs = p.getFileSystem(conf);
      if (!fs.exists(p)) {
        return State.NOT_FOUND.toString();
      }
     
      Codec codec = Codec.getCodec("rs");
      boolean result = false;
      try {
        result = checkFile(conf, fs, fs, p, null, codec, reporter, true);
      } catch (IOException ex) {
        LOG.warn("Encounter exception when checking file: " + p +
View Full Code Here
70
71
72
73
74
75
76
77
78
79
80
    if (encodeMethod.equals("jerasure")) {
      rsCode = new JerasureCode();
    } else {
      rsCode = new ReedSolomonCode();
    }
    Codec rsCodec = Codec.getCodec(encodeMethod);
    rsCode.init(rsCodec.stripeLength, rsCodec.parityLength);
    sourceVecTmp = new byte[rsCode.stripeSize()][];
    sourceVec = new byte[rsCode.stripeSize()][];
    parityVec = new byte[rsCode.paritySize()][];
    totalVec = new byte[rsCode.stripeSize() + rsCode.paritySize()][];
View Full Code Here
TOP

Related Classes of org.apache.hadoop.raid.Codec

  • org.apache.hadoop.hdfs.DistributedRaidFileSystem
  • org.apache.hadoop.hdfs.DistributedRaidFileSystem$ExtFSDataInputStream$ExtFsInputStream
  • org.apache.hadoop.hdfs.server.namenode.BlockPlacementPolicyRaid
  • org.apache.hadoop.hdfs.server.namenode.FaultTolerantBlockPlacementPolicy
  • org.apache.hadoop.hdfs.server.namenode.TestDirectoryRaidBlockPlacement
  • org.apache.hadoop.raid.tools.FastFileCheck$FileCheckMapper
  • org.apache.hadoop.raid.tools.RSBenchmark
  • org.apache.hadoop.raid.tools.TestFastFileCheck
  • org.json.JSONArray
  • java.lang.IllegalArgumentException
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.