Package org.apache.hadoop.fs

Examples of org.apache.hadoop.fs.HarReader


    Set<Path> allowedPaths = new HashSet<Path>(relativePaths);
   
    Map<String, HashSet<String>> tree = new HashMap<String, HashSet<String>>();
    HashSet<String> toTake = new HashSet<String>();
   
    HarReader harReader = new HarReader(harSrc, conf);
    List<HarStatus> allStatuses = new ArrayList<HarStatus>();
    try {
      while (harReader.hasNext()) {
        allStatuses.add(harReader.getNext());
      }
    } finally {
      if (harReader != null) {
        harReader.close();
      }
    }

    Path root = new Path(Path.SEPARATOR);
    // decide which of the har files we need to process
View Full Code Here


      }
    }
  }
 
  private void copyToLocal(Path archivePath, Path local) throws IOException {
    HarReader harReader = new HarReader(archivePath, conf);
    FileSystem localFS = FileSystem.getLocal(conf);
    FileSystem fs = archivePath.getFileSystem(conf);
    if (!localFS.getFileStatus(local).isDir()) {
      throw new IOException("Path " + local + " is not a directory");
    }
    try {
      while (harReader.hasNext()) {
        HarStatus harStatus = harReader.getNext();
        String harPath = harStatus.getName();
       
        // skip top level dir
        if (harPath.equals(Path.SEPARATOR)) {
          continue;
        }
        String relativePath = harPath.substring(1);
        Path output = new Path(local, relativePath);
        if (harStatus.isDir()) {
          localFS.mkdirs(output);
        } else {
          OutputStream outputStream = null;
          FSDataInputStream inputStream = null;
         
          try {
            outputStream = localFS.create(output);
            Path partFile = new Path(archivePath, harStatus.getPartName());
            inputStream = new HarFSDataInputStream(fs, partFile,
                harStatus.getStartIndex(), harStatus.getLength(), conf.getInt("io.file.buffer.size", 4096));
            IOUtils.copyBytes(inputStream, outputStream, conf);
          } finally {
            if (outputStream != null) {
              outputStream.close();
            }
          }
        }
      }
    } finally {
      if (harReader != null) {
        harReader.close();
      }
    }
  }
View Full Code Here

TOP

Related Classes of org.apache.hadoop.fs.HarReader

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.