Examples of org.apache.hadoop.fs.ChecksumFileSystem

org.apache.hadoop.fs.ChecksumFileSystem
Abstract Checksumed FileSystem. It provide a basice implementation of a Checksumed FileSystem, which creates a checksum file for each raw file. It generates & verifies checksums at the client side.

    String localDir = conf.get("mapred.temp.dir");
    dir = new LocalDirAllocator("mapred.temp.dir");


    System.setProperty("test.build.data", localDir);
    System.out.println("Local = " + localDir);
    ChecksumFileSystem checkedLocal = FileSystem.getLocal(conf);
    FileSystem rawLocal = checkedLocal.getRawFileSystem();
    for(int i=0; i < reps; ++i) {
      writeAndReadLocalFile("local", conf, SIZE);
      writeAndReadFile(rawLocal, "raw", conf, SIZE);
      writeAndReadFile(checkedLocal, "checked", conf, SIZE);
    }

View Full Code Here

    Random rand = new Random(seed);
    rand.nextBytes(expected);


    // test DFS
    MiniDFSCluster cluster = new MiniDFSCluster(conf, 1, true, null);
    ChecksumFileSystem fileSys = (ChecksumFileSystem)cluster.getFileSystem();
    try {
      testChecker(fileSys, true);
      testChecker(fileSys, false);
    } finally {
      fileSys.close();
      cluster.shutdown();
    }
    
    
    // test Local FS
    fileSys = FileSystem.getLocal(conf);
    try {
      testChecker(fileSys, true);
      testChecker(fileSys, false);
    }finally {
      fileSys.close();
    }
  }

View Full Code Here

    String localDir = conf.get("mapred.temp.dir");
    dir = new LocalDirAllocator("mapred.temp.dir");


    System.setProperty("test.build.data", localDir);
    System.out.println("Local = " + localDir);
    ChecksumFileSystem checkedLocal = FileSystem.getLocal(conf);
    FileSystem rawLocal = checkedLocal.getRawFileSystem();
    for(int i=0; i < reps; ++i) {
      writeAndReadLocalFile("local", conf, SIZE);
      writeAndReadFile(rawLocal, "raw", conf, SIZE);
      writeAndReadFile(checkedLocal, "checked", conf, SIZE);
    }

View Full Code Here

    }
    dir = new LocalDirAllocator("mapred.temp.dir");


    System.setProperty("test.build.data", localDir);
    System.out.println("Local = " + localDir);
    ChecksumFileSystem checkedLocal = FileSystem.getLocal(conf);
    FileSystem rawLocal = checkedLocal.getRawFileSystem();
    for(int i=0; i < reps; ++i) {
      writeAndReadLocalFile("local", conf, SIZE);
      writeAndReadFile(rawLocal, "raw", conf, SIZE);
      writeAndReadFile(checkedLocal, "checked", conf, SIZE);
    }

View Full Code Here

0 1

TOP

Related Classes of org.apache.hadoop.fs.ChecksumFileSystem

org.apache.hadoop.conf.Configuration

org.apache.hadoop.dfs.BenchmarkThroughput

org.apache.hadoop.dfs.TestFSInputChecker

org.apache.hadoop.fs.shell.PathData

org.apache.hadoop.hdfs.BenchmarkThroughput

org.apache.hadoop.hdfs.TestFSInputChecker

org.apache.hadoop.util.PureJavaCrc32

org.springframework.data.hadoop.fs.FsShell

java.io.InputStream

java.io.FileNotFoundException

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.