Examples of org.apache.hadoop.fs.ChecksumFileSystem

org.apache.hadoop.fs.ChecksumFileSystem
Abstract Checksumed FileSystem. It provide a basice implementation of a Checksumed FileSystem, which creates a checksum file for each raw file. It generates & verifies checksums at the client side.

      if (copyCrc) {
        if (!(srcFS instanceof ChecksumFileSystem)) {
          throw new IOException("Source file system does not have crc files");
        }


        ChecksumFileSystem csfs = (ChecksumFileSystem) srcFS;
        File dstcs = FileSystem.getLocal(srcFS.getConf()).pathToFile(
            csfs.getChecksumFile(new Path(dst.getCanonicalPath())));
        copyToLocal(csfs.getRawFileSystem(), csfs.getChecksumFile(src), dstcs, false);
      }
    }
    else {
      // once FileUtil.copy() supports tmp file, we don't need to mkdirs().
      dst.mkdirs();

View Full Code Here

    Random rand = new Random(seed);
    rand.nextBytes(expected);


    // test DFS
    MiniDFSCluster cluster = new MiniDFSCluster(conf, 1, true, null);
    ChecksumFileSystem fileSys = (ChecksumFileSystem)cluster.getFileSystem();
    try {
      testChecker(fileSys, true);
      testChecker(fileSys, false);
    } finally {
      fileSys.close();
      cluster.shutdown();
    }
    
    
    // test Local FS
    fileSys = FileSystem.getLocal(conf);
    try {
      testChecker(fileSys, true);
      testChecker(fileSys, false);
    }finally {
      fileSys.close();
    }
  }

View Full Code Here

    }
    dir = new LocalDirAllocator("mapred.temp.dir");


    System.setProperty("test.build.data", localDir);
    System.out.println("Local = " + localDir);
    ChecksumFileSystem checkedLocal = FileSystem.getLocal(conf);
    FileSystem rawLocal = checkedLocal.getRawFileSystem();
    for(int i=0; i < reps; ++i) {
      writeAndReadLocalFile("local", conf, SIZE);
      writeAndReadFile(rawLocal, "raw", conf, SIZE);
      writeAndReadFile(checkedLocal, "checked", conf, SIZE);
    }

View Full Code Here

             "org.apache.hadoop.dfs.ChecksumDistributedFileSystem");
    rand.nextBytes(expected);


    // test DFS
    MiniDFSCluster cluster = new MiniDFSCluster(conf, 1, true, null);
    ChecksumFileSystem fileSys = (ChecksumFileSystem)cluster.getFileSystem();
    try {
      testChecker(fileSys, true);
      testChecker(fileSys, false);
      testSeekAndRead(fileSys);
    } finally {
      fileSys.close();
      cluster.shutdown();
    }
    
    
    // test Local FS
    fileSys = FileSystem.getLocal(conf);
    try {
      testChecker(fileSys, true);
      testChecker(fileSys, false);
      testFileCorruption((LocalFileSystem)fileSys);
      testSeekAndRead(fileSys);
    }finally {
      fileSys.close();
    }
  }

View Full Code Here

    String localDir = conf.get("mapred.temp.dir");
    dir = new LocalDirAllocator("mapred.temp.dir");


    System.setProperty("test.build.data", localDir);
    System.out.println("Local = " + localDir);
    ChecksumFileSystem checkedLocal = FileSystem.getLocal(conf);
    FileSystem rawLocal = checkedLocal.getRawFileSystem();
    for(int i=0; i < reps; ++i) {
      writeAndReadLocalFile("local", conf, SIZE);
      writeAndReadFile(rawLocal, "raw", conf, SIZE);
      writeAndReadFile(checkedLocal, "checked", conf, SIZE);
    }

View Full Code Here

    String localDir = conf.get("mapred.temp.dir");
    dir = new LocalDirAllocator("mapred.temp.dir");


    System.setProperty("test.build.data", localDir);
    System.out.println("Local = " + localDir);
    ChecksumFileSystem checkedLocal = FileSystem.getLocal(conf);
    FileSystem rawLocal = checkedLocal.getRawFileSystem();
    for(int i=0; i < reps; ++i) {
      writeAndReadLocalFile("local", conf, SIZE);
      writeAndReadFile(rawLocal, "raw", conf, SIZE);
      writeAndReadFile(checkedLocal, "checked", conf, SIZE);
    }

View Full Code Here

    String localDir = conf.get("mapred.temp.dir");
    dir = new LocalDirAllocator("mapred.temp.dir");


    System.setProperty("test.build.data", localDir);
    System.out.println("Local = " + localDir);
    ChecksumFileSystem checkedLocal = FileSystem.getLocal(conf);
    FileSystem rawLocal = checkedLocal.getRawFileSystem();
    for(int i=0; i < reps; ++i) {
      writeAndReadLocalFile("local", conf, SIZE);
      writeAndReadFile(rawLocal, "raw", conf, SIZE);
      writeAndReadFile(checkedLocal, "checked", conf, SIZE);
    }

View Full Code Here

    }
    dir = new LocalDirAllocator("mapred.temp.dir");


    System.setProperty("test.build.data", localDir);
    System.out.println("Local = " + localDir);
    ChecksumFileSystem checkedLocal = FileSystem.getLocal(conf);
    FileSystem rawLocal = checkedLocal.getRawFileSystem();
    for(int i=0; i < reps; ++i) {
      writeAndReadLocalFile("local", conf, SIZE);
      writeAndReadFile(rawLocal, "raw", conf, SIZE);
      writeAndReadFile(checkedLocal, "checked", conf, SIZE);
    }

View Full Code Here

   * @return PathData of the crc file
   * @throws IOException is anything goes wrong
   */
  public PathData getChecksumFile() throws IOException {
    checkIfExists(FileTypeRequirement.SHOULD_NOT_BE_DIRECTORY);
    ChecksumFileSystem srcFs = (ChecksumFileSystem)fs;
    Path srcPath = srcFs.getChecksumFile(path);
    return new PathData(srcFs.getRawFileSystem(), srcPath.toString());
  }

View Full Code Here

             "org.apache.hadoop.hdfs.ChecksumDistributedFileSystem");
    rand.nextBytes(expected);


    // test DFS
    MiniDFSCluster cluster = new MiniDFSCluster(conf, 1, true, null);
    ChecksumFileSystem fileSys = (ChecksumFileSystem)cluster.getFileSystem();
    try {
      testChecker(fileSys, true);
      testChecker(fileSys, false);
      testSeekAndRead(fileSys);
    } finally {
      fileSys.close();
      cluster.shutdown();
    }
    
    
    // test Local FS
    fileSys = FileSystem.getLocal(conf);
    try {
      testChecker(fileSys, true);
      testChecker(fileSys, false);
      testFileCorruption((LocalFileSystem)fileSys);
      testSeekAndRead(fileSys);
    }finally {
      fileSys.close();
    }
  }

View Full Code Here

0 1

TOP

Related Classes of org.apache.hadoop.fs.ChecksumFileSystem

org.apache.hadoop.conf.Configuration

org.apache.hadoop.dfs.BenchmarkThroughput

org.apache.hadoop.dfs.TestFSInputChecker

org.apache.hadoop.fs.shell.PathData

org.apache.hadoop.hdfs.BenchmarkThroughput

org.apache.hadoop.hdfs.TestFSInputChecker

org.apache.hadoop.util.PureJavaCrc32

org.springframework.data.hadoop.fs.FsShell

java.io.InputStream

java.io.FileNotFoundException

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.