Package org.apache.hadoop.fs

Examples of org.apache.hadoop.fs.HarFileSystem$HarFSDataInputStream$HarFsInputStream


    String harDirName = srcParent.getName() + RaidNode.HAR_SUFFIX;
    Path HarPath = new Path(outDir,harDirName);
    if (fsDest.exists(HarPath)) {
      URI HarPathUri = HarPath.toUri();
      Path inHarPath = new Path("har://",HarPathUri.getPath()+"/"+outPath.toUri().getPath());
      FileSystem fsHar = new HarFileSystem(fsDest);
      fsHar.initialize(inHarPath.toUri(), conf);
      FileStatus inHar = FileStatusCache.get(fsHar, inHarPath);
      if (inHar != null) {
        if (verifyParity(srcStatus, inHar, codec, conf)) {
          return new ParityFilePair(inHarPath, inHar, fsHar);
        }
View Full Code Here


    String harDirName = srcParent.getName() + HAR_SUFFIX;
    Path HarPath = new Path(outDir,harDirName);
    if (fsDest.exists(HarPath)) { 
      URI HarPathUri = HarPath.toUri();
      Path inHarPath = new Path("har://",HarPathUri.getPath()+"/"+outPath.toUri().getPath());
      FileSystem fsHar = new HarFileSystem(fsDest);
      fsHar.initialize(inHarPath.toUri(), conf);
      if (fsHar.exists(inHarPath)) {
        FileStatus inHar = fsHar.getFileStatus(inHarPath);
        if (inHar.getModificationTime() == srcStatus.getModificationTime()) {
          return new ParityFilePair(inHarPath,fsHar);
        }
      }
    }
View Full Code Here

  protected static int usefulHar(
    ErasureCodeType code,
    FileSystem srcFs, FileSystem destFs,
    Path harPath, String destPrefix, Configuration conf) throws IOException {

    FileSystem fsHar = new HarFileSystem(destFs);
    String harURIPath = harPath.toUri().getPath();
    Path qualifiedPath = new Path("har://", harURIPath +
      Path.SEPARATOR + harPath.getParent().toUri().getPath());
    fsHar.initialize(qualifiedPath.toUri(), conf);
    FileStatus[] filesInHar = fsHar.listStatus(qualifiedPath);
    if (filesInHar.length == 0) {
      return 0;
    }
    int numUseless = 0;
    for (FileStatus one: filesInHar) {
View Full Code Here

    String harDirName = srcParent.getName() + HAR_SUFFIX;
    Path HarPath = new Path(outDir,harDirName);
    if (fsDest.exists(HarPath)) { 
      URI HarPathUri = HarPath.toUri();
      Path inHarPath = new Path("har://",HarPathUri.getPath()+"/"+outPath.toUri().getPath());
      FileSystem fsHar = new HarFileSystem(fsDest);
      fsHar.initialize(inHarPath.toUri(), conf);
      if (fsHar.exists(inHarPath)) {
        FileStatus inHar = fsHar.getFileStatus(inHarPath);
        if (inHar.getModificationTime() == srcStatus.getModificationTime()) {
          return new ParityFilePair(inHarPath,fsHar);
        }
      }
    }
View Full Code Here

  protected static int usefulHar(
    ErasureCodeType code,
    FileSystem srcFs, FileSystem destFs,
    Path harPath, String destPrefix, Configuration conf) throws IOException {

    FileSystem fsHar = new HarFileSystem(destFs);
    String harURIPath = harPath.toUri().getPath();
    Path qualifiedPath = new Path("har://", harURIPath +
      Path.SEPARATOR + harPath.getParent().toUri().getPath());
    fsHar.initialize(qualifiedPath.toUri(), conf);
    FileStatus[] filesInHar = fsHar.listStatus(qualifiedPath);
    if (filesInHar.length == 0) {
      return 0;
    }
    int numUseless = 0;
    for (FileStatus one: filesInHar) {
View Full Code Here

      return new ParityFilePair(outPath,fsDest);
    }

    URI HarPathUri = HarPath.toUri();
    Path inHarPath = new Path("har://",HarPathUri.getPath()+"/"+outPath.toUri().getPath());
    FileSystem fsHar = new HarFileSystem(fsDest);
    fsHar.initialize(inHarPath.toUri(), conf);

    if (!fsHar.exists(inHarPath)) { // case 2: no file inside HAR
      return new ParityFilePair(outPath,fsDest);
    }

    if (! fsDest.exists(outPath)) { // case 3: only inside HAR
      return new ParityFilePair(inHarPath,fsHar);
    }

    // both inside and outside HAR. Should return most recent
    FileStatus inHar = fsHar.getFileStatus(inHarPath);
    FileStatus outHar = fsDest.getFileStatus(outPath);

    if (inHar.getModificationTime() >= outHar.getModificationTime()) {
      return new ParityFilePair(inHarPath,fsHar);
    }
View Full Code Here

    }   
  }

  @Override
  protected void openForRead() throws IOException {
      _harfs = new HarFileSystem(FileSystem.get(_conf));
     
      //Path qualifiedPath = new Path("har://", input_file.toUri() +
      //        Path.SEPARATOR + input_file.getParent().toUri().getPath());
      Path qualifiedPath = new Path("har://", _file_path.toUri().getPath());
              //Path.SEPARATOR + input_file.getParent().toUri().getPath());
View Full Code Here

    fileList.add(createFile(inputPath, fs, new byte[0], sub2.getName(), "zero-length"));

    final String fullHarPathStr = makeArchive();

    // Create fresh HarFs:
    final HarFileSystem harFileSystem = new HarFileSystem(fs);
    try {
      final URI harUri = new URI(fullHarPathStr);
      harFileSystem.initialize(harUri, fs.getConf());
      // now read the file content and compare it against the expected:
      int readFileCount = 0;
      for (final String pathStr0 : fileList) {
        final Path path = new Path(fullHarPathStr + Path.SEPARATOR + pathStr0);
        final String baseName = path.getName();
        final FileStatus status = harFileSystem.getFileStatus(path);
        if (status.isFile()) {
          // read the file:
          final byte[] actualContentSimple = readAllSimple(
              harFileSystem.open(path), true);
         
          final byte[] actualContentBuffer = readAllWithBuffer(
              harFileSystem.open(path), true);
          assertArrayEquals(actualContentSimple, actualContentBuffer);
         
          final byte[] actualContentFully = readAllWithReadFully(
              actualContentSimple.length,
              harFileSystem.open(path), true);
          assertArrayEquals(actualContentSimple, actualContentFully);
         
          final byte[] actualContentSeek = readAllWithSeek(
              actualContentSimple.length,
              harFileSystem.open(path), true);
          assertArrayEquals(actualContentSimple, actualContentSeek);
         
          final byte[] actualContentRead4
          = readAllWithRead4(harFileSystem.open(path), true);
          assertArrayEquals(actualContentSimple, actualContentRead4);
         
          final byte[] actualContentSkip = readAllWithSkip(
              actualContentSimple.length,
              harFileSystem.open(path),
              harFileSystem.open(path),
              true);
          assertArrayEquals(actualContentSimple, actualContentSkip);
         
          if ("bin".equals(baseName)) {
            assertArrayEquals(binContent, actualContentSimple);
          } else if ("zero-length".equals(baseName)) {
            assertEquals(0, actualContentSimple.length);
          } else {
            String actual = new String(actualContentSimple, "UTF-8");
            assertEquals(baseName, actual);
          }
          readFileCount++;
        }
      }
      assertEquals(fileList.size(), readFileCount);
    } finally {
      harFileSystem.close();
    }
  }
View Full Code Here

    localFs.delete(tmpPath, true);
    localFs.mkdirs(tmpPath);
    assertTrue(localFs.exists(tmpPath));
   
    // Create fresh HarFs:
    final HarFileSystem harFileSystem = new HarFileSystem(fs);
    try {
      final URI harUri = new URI(fullHarPathStr);
      harFileSystem.initialize(harUri, fs.getConf());
     
      final Path sourcePath = new Path(fullHarPathStr + Path.SEPARATOR + "a");
      final Path targetPath = new Path(tmpPath, "straus");
      // copy the Har file to a local file system:
      harFileSystem.copyToLocalFile(false, sourcePath, targetPath);
      FileStatus straus = localFs.getFileStatus(targetPath);
      // the file should contain just 1 character:
      assertEquals(1, straus.getLen());
    } finally {
      harFileSystem.close();
      localFs.delete(tmpPath, true);     
    }
  }
View Full Code Here

      String harDirName = srcParent.getName() + RaidNode.HAR_SUFFIX;
      Path HarPath = new Path(outDir, harDirName);
      if (fsDest.exists(HarPath)) {
        URI HarPathUri = HarPath.toUri();
        Path inHarPath = new Path("har://",HarPathUri.getPath()+"/"+outPath.toUri().getPath());
        FileSystem fsHar = new HarFileSystem(fsDest);
        fsHar.initialize(inHarPath.toUri(), conf);
        FileStatus inHar = FileStatusCache.get(fsHar, inHarPath);
        if (inHar != null) {
          ParityFilePair pfp = verifyParity(srcStat, inHar, codec, conf, fsHar);
          if (pfp != null) {
            return pfp;
View Full Code Here

TOP

Related Classes of org.apache.hadoop.fs.HarFileSystem$HarFSDataInputStream$HarFsInputStream

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.