Package org.apache.hadoop.fs

Examples of org.apache.hadoop.fs.ChecksumFileSystem$FSDataBoundedInputStream

328
329
330
331
332
333
334
335
336
337
338
339
340
341
      if (copyCrc) {
        if (!(srcFS instanceof ChecksumFileSystem)) {
          throw new IOException("Source file system does not have crc files");
        }

        ChecksumFileSystem csfs = (ChecksumFileSystem) srcFS;
        File dstcs = FileSystem.getLocal(srcFS.getConf()).pathToFile(
            csfs.getChecksumFile(new Path(dst.getCanonicalPath())));
        copyToLocal(csfs.getRawFileSystem(), csfs.getChecksumFile(src), dstcs, false);
      }
    }
    else {
      // once FileUtil.copy() supports tmp file, we don't need to mkdirs().
      dst.mkdirs();
View Full Code Here
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
    Random rand = new Random(seed);
    rand.nextBytes(expected);

    // test DFS
    MiniDFSCluster cluster = new MiniDFSCluster(conf, 1, true, null);
    ChecksumFileSystem fileSys = (ChecksumFileSystem)cluster.getFileSystem();
    try {
      testChecker(fileSys, true);
      testChecker(fileSys, false);
    } finally {
      fileSys.close();
      cluster.shutdown();
    }
   
   
    // test Local FS
    fileSys = FileSystem.getLocal(conf);
    try {
      testChecker(fileSys, true);
      testChecker(fileSys, false);
    }finally {
      fileSys.close();
    }
  }
View Full Code Here
200
201
202
203
204
205
206
207
208
209
210
211
    }
    dir = new LocalDirAllocator("mapred.temp.dir");

    System.setProperty("test.build.data", localDir);
    System.out.println("Local = " + localDir);
    ChecksumFileSystem checkedLocal = FileSystem.getLocal(conf);
    FileSystem rawLocal = checkedLocal.getRawFileSystem();
    for(int i=0; i < reps; ++i) {
      writeAndReadLocalFile("local", conf, SIZE);
      writeAndReadFile(rawLocal, "raw", conf, SIZE);
      writeAndReadFile(checkedLocal, "checked", conf, SIZE);
    }
View Full Code Here
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
             "org.apache.hadoop.dfs.ChecksumDistributedFileSystem");
    rand.nextBytes(expected);

    // test DFS
    MiniDFSCluster cluster = new MiniDFSCluster(conf, 1, true, null);
    ChecksumFileSystem fileSys = (ChecksumFileSystem)cluster.getFileSystem();
    try {
      testChecker(fileSys, true);
      testChecker(fileSys, false);
      testSeekAndRead(fileSys);
    } finally {
      fileSys.close();
      cluster.shutdown();
    }
   
   
    // test Local FS
    fileSys = FileSystem.getLocal(conf);
    try {
      testChecker(fileSys, true);
      testChecker(fileSys, false);
      testFileCorruption((LocalFileSystem)fileSys);
      testSeekAndRead(fileSys);
    }finally {
      fileSys.close();
    }
  }
View Full Code Here
195
196
197
198
199
200
201
202
203
204
205
206
    String localDir = conf.get("mapred.temp.dir");
    dir = new LocalDirAllocator("mapred.temp.dir");

    System.setProperty("test.build.data", localDir);
    System.out.println("Local = " + localDir);
    ChecksumFileSystem checkedLocal = FileSystem.getLocal(conf);
    FileSystem rawLocal = checkedLocal.getRawFileSystem();
    for(int i=0; i < reps; ++i) {
      writeAndReadLocalFile("local", conf, SIZE);
      writeAndReadFile(rawLocal, "raw", conf, SIZE);
      writeAndReadFile(checkedLocal, "checked", conf, SIZE);
    }
View Full Code Here
195
196
197
198
199
200
201
202
203
204
205
206
    String localDir = conf.get("mapred.temp.dir");
    dir = new LocalDirAllocator("mapred.temp.dir");

    System.setProperty("test.build.data", localDir);
    System.out.println("Local = " + localDir);
    ChecksumFileSystem checkedLocal = FileSystem.getLocal(conf);
    FileSystem rawLocal = checkedLocal.getRawFileSystem();
    for(int i=0; i < reps; ++i) {
      writeAndReadLocalFile("local", conf, SIZE);
      writeAndReadFile(rawLocal, "raw", conf, SIZE);
      writeAndReadFile(checkedLocal, "checked", conf, SIZE);
    }
View Full Code Here
195
196
197
198
199
200
201
202
203
204
205
206
    String localDir = conf.get("mapred.temp.dir");
    dir = new LocalDirAllocator("mapred.temp.dir");

    System.setProperty("test.build.data", localDir);
    System.out.println("Local = " + localDir);
    ChecksumFileSystem checkedLocal = FileSystem.getLocal(conf);
    FileSystem rawLocal = checkedLocal.getRawFileSystem();
    for(int i=0; i < reps; ++i) {
      writeAndReadLocalFile("local", conf, SIZE);
      writeAndReadFile(rawLocal, "raw", conf, SIZE);
      writeAndReadFile(checkedLocal, "checked", conf, SIZE);
    }
View Full Code Here
199
200
201
202
203
204
205
206
207
208
209
210
    }
    dir = new LocalDirAllocator("mapred.temp.dir");

    System.setProperty("test.build.data", localDir);
    System.out.println("Local = " + localDir);
    ChecksumFileSystem checkedLocal = FileSystem.getLocal(conf);
    FileSystem rawLocal = checkedLocal.getRawFileSystem();
    for(int i=0; i < reps; ++i) {
      writeAndReadLocalFile("local", conf, SIZE);
      writeAndReadFile(rawLocal, "raw", conf, SIZE);
      writeAndReadFile(checkedLocal, "checked", conf, SIZE);
    }
View Full Code Here
175
176
177
178
179
180
181
182
183
   * @return PathData of the crc file
   * @throws IOException is anything goes wrong
   */
  public PathData getChecksumFile() throws IOException {
    checkIfExists(FileTypeRequirement.SHOULD_NOT_BE_DIRECTORY);
    ChecksumFileSystem srcFs = (ChecksumFileSystem)fs;
    Path srcPath = srcFs.getChecksumFile(path);
    return new PathData(srcFs.getRawFileSystem(), srcPath.toString());
  }
View Full Code Here
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
             "org.apache.hadoop.hdfs.ChecksumDistributedFileSystem");
    rand.nextBytes(expected);

    // test DFS
    MiniDFSCluster cluster = new MiniDFSCluster(conf, 1, true, null);
    ChecksumFileSystem fileSys = (ChecksumFileSystem)cluster.getFileSystem();
    try {
      testChecker(fileSys, true);
      testChecker(fileSys, false);
      testSeekAndRead(fileSys);
    } finally {
      fileSys.close();
      cluster.shutdown();
    }
   
   
    // test Local FS
    fileSys = FileSystem.getLocal(conf);
    try {
      testChecker(fileSys, true);
      testChecker(fileSys, false);
      testFileCorruption((LocalFileSystem)fileSys);
      testSeekAndRead(fileSys);
    }finally {
      fileSys.close();
    }
  }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.fs.ChecksumFileSystem$FSDataBoundedInputStream

  • org.apache.hadoop.conf.Configuration
  • org.apache.hadoop.dfs.BenchmarkThroughput
  • org.apache.hadoop.dfs.TestFSInputChecker
  • org.apache.hadoop.fs.shell.PathData
  • org.apache.hadoop.hdfs.BenchmarkThroughput
  • org.apache.hadoop.hdfs.TestFSInputChecker
  • org.apache.hadoop.util.PureJavaCrc32
  • org.springframework.data.hadoop.fs.FsShell
  • java.io.InputStream
  • java.io.FileNotFoundException
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.