Examples of org.apache.hadoop.fs.FileSystem.listFiles()

org.apache.hadoop.fs.FileSystem.listFiles()
List the statuses and block locations of the files in the given path. If the path is a directory, if recursive is false, returns files in the directory; if recursive is true, return files in the subtree rooted at the path. If the path is a file, return the file's status and block locations. @param f is the path @param recursive if the subdirectories need to be traversed recursively @return an iterator that traverses statuses of the files @throws FileNotFoundException when the path does not exist;IOException see specific implementation


  /** Open the output generated by this format. */
  public static SequenceFile.Reader[] getReaders(Configuration conf, File dir)
    throws IOException {
    FileSystem fs = FileSystem.get(conf);
    File[] names = fs.listFiles(dir);
    
    // sort names, so that hash partitioning works
    Arrays.sort(names);
    
    SequenceFile.Reader[] parts = new SequenceFile.Reader[names.length];

View Full Code Here

      }
      Path p = new Path(uri);
      FileSystem pathfs = p.getFileSystem(conf);
      p = pathfs.makeQualified(p);
      tezJarPaths.add(p);
      RemoteIterator<LocatedFileStatus> iter = pathfs.listFiles(p, false);
      while (iter.hasNext()) {
        LocatedFileStatus fStatus = iter.next();
        String rsrcName = fStatus.getPath().getName();
        // FIXME currently not checking for duplicates due to quirks
        // in assembly generation

View Full Code Here

        LOG.error(message);
        throw new TezUncheckedException(message);
      }
      Path p = new Path(uri);
      FileSystem pathfs = p.getFileSystem(conf);
      RemoteIterator<LocatedFileStatus> iter = pathfs.listFiles(p, false);
      while (iter.hasNext()) {
        LocatedFileStatus fStatus = iter.next();
        String rsrcName = fStatus.getPath().getName();
        // FIXME currently not checking for duplicates due to quirks
        // in assembly generation

View Full Code Here

          // Find list of files in this path(recursively if -recursive option
          // is specified).
          List<FileStatus> historyLogs = new ArrayList<FileStatus>();


          RemoteIterator<LocatedFileStatus> iter =
            fs.listFiles(thisPath, isRecursive);
          while (iter.hasNext()) {
            LocatedFileStatus child = iter.next();
            String fileName = child.getPath().getName();


            if (!(fileName.endsWith(".crc") || fileName.startsWith("."))) {

View Full Code Here

    outputPath = filePath;
    if (cl.hasOption("--outputpath")) {
      outputPath = (String) cl.getValue("--outputpath");
    }
    FileSystem hdfs = FileSystem.get(new Configuration());
    RemoteIterator<LocatedFileStatus> itr = hdfs.listFiles(new Path(filePath), true);
    while (itr.hasNext()) {
      LocatedFileStatus status = itr.next();
      String fileName = status.getPath().getName();
      if (status.getLen() > 0) {
        String outputFileName = outputPath + File.separator + fileName + ".json";

View Full Code Here

    outputFileName = Boolean.parseBoolean((String) cl.getValue("--outputfilename"));
    if (cl.hasOption("--outputpath")) {
      outputPath = (String) cl.getValue("--outputpath");
    }
    FileSystem hdfs = FileSystem.get(new Configuration());
    RemoteIterator<LocatedFileStatus> itr = hdfs.listFiles(new Path(filePath), true);
    while (itr.hasNext()) {
      LocatedFileStatus status = itr.next();
      String fileName = status.getPath().getName();
      if (fileName.substring(fileName.lastIndexOf(".") + 1).contains("json")) {
        createSequenceFileFromJSON(hdfs.open(status.getPath()), fileName);

View Full Code Here

    outputPath = filePath;
    if (cl.hasOption("--outputpath")) {
      outputPath = (String) cl.getValue("--outputpath");
    }
    FileSystem hdfs = FileSystem.get(new Configuration());
    RemoteIterator<LocatedFileStatus> itr = hdfs.listFiles(new Path(filePath), true);
    while (itr.hasNext()) {
      LocatedFileStatus status = itr.next();
      String fileName = status.getPath().getName();
      Path filePath = status.getPath();
      if (status.getLen() > 0) {

View Full Code Here

      }
      Path p = new Path(uri);
      FileSystem pathfs = p.getFileSystem(conf);
      p = pathfs.makeQualified(p);
      tezJarPaths.add(p);
      RemoteIterator<LocatedFileStatus> iter = pathfs.listFiles(p, false);
      while (iter.hasNext()) {
        LocatedFileStatus fStatus = iter.next();
        String rsrcName = fStatus.getPath().getName();
        // FIXME currently not checking for duplicates due to quirks
        // in assembly generation

View Full Code Here

          // Find list of files in this path(recursively if -recursive option
          // is specified).
          List<FileStatus> historyLogs = new ArrayList<FileStatus>();


          RemoteIterator<LocatedFileStatus> iter =
            fs.listFiles(thisPath, isRecursive);
          while (iter.hasNext()) {
            LocatedFileStatus child = iter.next();
            String fileName = child.getPath().getName();


            if (!(fileName.endsWith(".crc") || fileName.startsWith("."))) {

View Full Code Here

          // Find list of files in this path(recursively if -recursive option
          // is specified).
          List<FileStatus> historyLogs = new ArrayList<FileStatus>();


          RemoteIterator<LocatedFileStatus> iter =
            fs.listFiles(thisPath, isRecursive);
          while (iter.hasNext()) {
            LocatedFileStatus child = iter.next();
            String fileName = child.getPath().getName();


            if (!(fileName.endsWith(".crc") || fileName.startsWith("."))) {

View Full Code Here

0 1

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.