Package org.apache.hadoop.fs

Examples of org.apache.hadoop.fs.LocatedFileStatus


    private static void addFilesFromHdfs(FileSystem fs, Path source, File destDir) throws IOException, NoSuchAlgorithmException {
        LOGGER.debug("Adding files from HDFS %s -> %s", source.toString(), destDir.getAbsolutePath());
        RemoteIterator<LocatedFileStatus> hdfsFiles = fs.listFiles(source, true);
        while (hdfsFiles.hasNext()) {
            LocatedFileStatus hdfsFile = hdfsFiles.next();
            if (hdfsFile.isDirectory()) {
                continue;
            }

            File locallyCachedFile = getLocalCacheFileName(hdfsFile, destDir);
            if (locallyCachedFile.exists()) {
                LOGGER.info("HDFS file %s already cached at %s. Skipping sync.", hdfsFile.getPath().toString(), locallyCachedFile.getPath());
            } else {
                fs.copyToLocalFile(hdfsFile.getPath(), new Path(locallyCachedFile.getAbsolutePath()));
                locallyCachedFile.setLastModified(hdfsFile.getModificationTime());
                LOGGER.info("Caching HDFS file %s -> %s", hdfsFile.getPath().toString(), locallyCachedFile.getPath());
            }

            addLibFile(locallyCachedFile);
        }
    }
View Full Code Here


            getUri(), getWorkingDirectory())); // fully-qualify path
  }

  private LocatedFileStatus makeQualifiedLocated(
      HdfsLocatedFileStatus f, Path parent) {
    return new LocatedFileStatus(f.getLen(), f.isDir(), f.getReplication(),
        f.getBlockSize(), f.getModificationTime(),
        f.getAccessTime(),
        f.getPermission(), f.getOwner(), f.getGroup(),
        null,
        (f.getFullPath(parent)).makeQualified(
View Full Code Here

      }

      @Override
      public boolean hasNext() throws IOException {
        while (curStat == null && hasNextNoFilter()) {
          LocatedFileStatus next = makeQualifiedLocated(
              (HdfsLocatedFileStatus)thisListing.getPartialListing()[i++], p);
          if (filter.accept(next.getPath())) {
            curStat = next;
          }
        }
        return curStat != null;
      }
     
      /** Check if there is a next item before applying the given filter */
      private boolean hasNextNoFilter() throws IOException {
        if (thisListing == null) {
          return false;
        }
        if (i>=thisListing.getPartialListing().length
            && thisListing.hasMore()) {
          // current listing is exhausted & fetch a new listing
          thisListing = dfs.listPaths(src, thisListing.getLastName(), true);
          statistics.incrementReadOps(1);
          if (thisListing == null) {
            return false;
          }
          i = 0;
        }
        return (i<thisListing.getPartialListing().length);
      }

      @Override
      public LocatedFileStatus next() throws IOException {
        if (hasNext()) {
          LocatedFileStatus tmp = curStat;
          curStat = null;
          return tmp;
        }
        throw new java.util.NoSuchElementException("No more entry in " + p);
      }
View Full Code Here

    long minFileSize = conf.getLong(MINIMUM_RAIDABLE_FILESIZE_KEY,
        MINIMUM_RAIDABLE_FILESIZE);
    List<LocatedFileStatus> lfs = new ArrayList<LocatedFileStatus>();
    RemoteIterator<LocatedFileStatus> iter = srcFs.listLocatedStatus(p);
    while (iter.hasNext()) {
      LocatedFileStatus stat = iter.next();
      if (stat.isDir()) {
        return null;
      }
      // We don't raid too small files
      if (stat.getLen() < minFileSize) {
        continue;
      }
      lfs.add(stat);
    }
    if (lfs.size() == 0)
View Full Code Here

  LocatedFileStatus getLocatedFileStatus(
      FileSystem fs, Path p) throws IOException {
    HashMap<String, LocatedFileStatus> cache =
        locatedFileStatusCache.get();
    LocatedFileStatus result = cache.get(p.toUri().getPath());
    if (result != null) {
      return result;
    }
    Path parent = p.getParent();
    String parentPath = parent.toUri().getPath();
    //If we already did listlocatedStatus on parent path,
    //it means path p doesn't exist, we don't need to list again
    if (cache.containsKey(parentPath) &&
        cache.get(parentPath) == null) {
      return null;
    }
   
    RemoteIterator<LocatedFileStatus> iter = fs.listLocatedStatus(parent);
    while (iter.hasNext()) {
      LocatedFileStatus stat = iter.next();
      cache.put(stat.getPath().toUri().getPath(), stat);
    }
    // trick: add parent path to the cache with value = null
    cache.put(parentPath, null);
    result = cache.get(p.toUri().getPath());
    // This may still return null
View Full Code Here

  }

  List<BlockInfo> getBlockInfos(
    FileSystem fs, Path path, long start, long length)
      throws IOException {
    LocatedFileStatus stat = getLocatedFileStatus(fs, path);
    List<BlockInfo> result = new ArrayList<BlockInfo>();
    long end = start + length;
    if (stat != null) {
      for (BlockLocation loc : stat.getBlockLocations()) {
        if (loc.getOffset() >= start && loc.getOffset() < end) {
          result.add(new BlockInfo(loc, path));
        }
      }
    }
View Full Code Here

  private static LocatedFileStatus toLocatedFileStatus(
      HdfsFileStatus stat, LocatedBlocks locs, String src) {
    if (stat == null) {
      return null;
    }
    return new LocatedFileStatus(stat.getLen(),
        stat.isDir(), stat.getReplication(),
        stat.getBlockSize(), stat.getModificationTime(),
        stat.getAccessTime(),
        stat.getPermission(), stat.getOwner(), stat.getGroup(),
        stat.getFullPath(new Path(src)), // full path
View Full Code Here

        }
        FileStatus result = stats[i++];
        BlockLocation[] locs = result.isDir() ? null :
            getBlockLocations(
                result.getPath().toUri().getPath(), 0, result.getLen());
        return new LocatedFileStatus(result, locs);
      }
    };
  }
View Full Code Here

        for (FileStatus globStat: matches) {
          if (globStat.isDirectory()) {
            RemoteIterator<LocatedFileStatus> iter =
                fs.listLocatedStatus(globStat.getPath());
            while (iter.hasNext()) {
              LocatedFileStatus stat = iter.next();
              if (inputFilter.accept(stat.getPath())) {
                if (recursive && stat.isDirectory()) {
                  addInputPathRecursively(result, fs, stat.getPath(),
                      inputFilter);
                } else {
                  result.add(stat);
                }
              }
View Full Code Here

  protected void addInputPathRecursively(List<FileStatus> result,
      FileSystem fs, Path path, PathFilter inputFilter)
      throws IOException {
    RemoteIterator<LocatedFileStatus> iter = fs.listLocatedStatus(path);
    while (iter.hasNext()) {
      LocatedFileStatus stat = iter.next();
      if (inputFilter.accept(stat.getPath())) {
        if (stat.isDirectory()) {
          addInputPathRecursively(result, fs, stat.getPath(), inputFilter);
        } else {
          result.add(stat);
        }
      }
    }
View Full Code Here

TOP

Related Classes of org.apache.hadoop.fs.LocatedFileStatus

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.