Package org.apache.hadoop.fs

Examples of org.apache.hadoop.fs.LocatedFileStatus


      }

      @Override
      public boolean hasNext() throws IOException {
        while (curStat == null && hasNextNoFilter()) {
          LocatedFileStatus next = makeQualifiedLocated(
              (HdfsLocatedFileStatus)thisListing.getPartialListing()[i++], p);
          if (filter.accept(next.getPath())) {
            curStat = next;
          }
        }
        return curStat != null;
      }
     
      /** Check if there is a next item before applying the given filter */
      private boolean hasNextNoFilter() throws IOException {
        if (thisListing == null) {
          return false;
        }
        if (i>=thisListing.getPartialListing().length
            && thisListing.hasMore()) {
          // current listing is exhausted & fetch a new listing
          thisListing = dfs.listPaths(src, thisListing.getLastName(), true);
          statistics.incrementReadOps(1);
          if (thisListing == null) {
            return false;
          }
          i = 0;
        }
        return (i<thisListing.getPartialListing().length);
      }

      @Override
      public LocatedFileStatus next() throws IOException {
        if (hasNext()) {
          LocatedFileStatus tmp = curStat;
          curStat = null;
          return tmp;
        }
        throw new java.util.NoSuchElementException("No more entry in " + p);
      }
View Full Code Here


    RemoteIterator<LocatedFileStatus> iter =
        GridmixTestUtils.dfs.listFiles(distCacheDir, false);
    int numFiles = filesSizesExpected.size();
    for (int i = 0; i < numFiles; i++) {
      assertTrue("Missing distributed cache files.", iter.hasNext());
      LocatedFileStatus stat = iter.next();
      assertTrue("File size of distributed cache file "
          + stat.getPath().toUri().getPath() + " is wrong.",
          filesSizesExpected.remove(stat.getLen()));

      FsPermission perm = stat.getPermission();
      assertEquals("Wrong permissions for distributed cache file "
          + stat.getPath().toUri().getPath(),
          new FsPermission((short)0644), perm);
    }
    assertFalse("Number of files under distributed cache dir is wrong.",
        iter.hasNext());
  }
View Full Code Here

            getUri(), getWorkingDirectory())); // fully-qualify path
  }

  private LocatedFileStatus makeQualifiedLocated(
      HdfsLocatedFileStatus f, Path parent) {
    return new LocatedFileStatus(f.getLen(), f.isDir(), f.getReplication(),
        f.getBlockSize(), f.getModificationTime(),
        f.getAccessTime(),
        f.getPermission(), f.getOwner(), f.getGroup(),
        null,
        (f.getFullPath(parent)).makeQualified(
View Full Code Here

      }

      @Override
      public boolean hasNext() throws IOException {
        while (curStat == null && hasNextNoFilter()) {
          LocatedFileStatus next = makeQualifiedLocated(
              (HdfsLocatedFileStatus)thisListing.getPartialListing()[i++], p);
          if (filter.accept(next.getPath())) {
            curStat = next;
          }
        }
        return curStat != null;
      }
     
      /** Check if there is a next item before applying the given filter */
      private boolean hasNextNoFilter() throws IOException {
        if (thisListing == null) {
          return false;
        }
        if (i>=thisListing.getPartialListing().length
            && thisListing.hasMore()) {
          // current listing is exhausted & fetch a new listing
          thisListing = dfs.listPaths(src, thisListing.getLastName(), true);
          statistics.incrementReadOps(1);
          if (thisListing == null) {
            return false;
          }
          i = 0;
        }
        return (i<thisListing.getPartialListing().length);
      }

      @Override
      public LocatedFileStatus next() throws IOException {
        if (hasNext()) {
          LocatedFileStatus tmp = curStat;
          curStat = null;
          return tmp;
        }
        throw new java.util.NoSuchElementException("No more entry in " + p);
      }
View Full Code Here

    private void doWalk(Path path, FileStatusCallback callback, AtomicLong taskCount, SettableFuture<Void> future)
    {
        try {
            RemoteIterator<LocatedFileStatus> iterator = listLocatedStatus(fileSystem, path);
            while (iterator.hasNext()) {
                LocatedFileStatus status = iterator.next();
                if (isDirectory(status)) {
                    recursiveWalk(status.getPath(), callback, taskCount, future);
                }
                else {
                    callback.process(status, status.getBlockLocations());
                }
                if (future.isDone()) {
                    return;
                }
            }
View Full Code Here

          return stats.hasNext();
        }

        @Override
        public LocatedFileStatus next() throws IOException {
          LocatedFileStatus result = stats.next();
          return new LocatedFileStatus(
              ProxyFileSystem23.super.swizzleFileStatus(result, false),
              result.getBlockLocations());
        }
      };
    }
View Full Code Here

          return stats.hasNext();
        }

        @Override
        public LocatedFileStatus next() throws IOException {
          LocatedFileStatus result = stats.next();
          return new LocatedFileStatus(
              ProxyFileSystem23.super.swizzleFileStatus(result, false),
              result.getBlockLocations());
        }
      };
    }
View Full Code Here

    private LocatedFileStatus createLocatedFileStatus(FileStatus status)
    {
        try {
            BlockLocation[] fakeLocation = getFileBlockLocations(status, 0, status.getLen());
            return new LocatedFileStatus(status, fakeLocation);
        }
        catch (IOException e) {
            throw propagate(e);
        }
    }
View Full Code Here

    {
        try (SetThreadName ignored = new SetThreadName("HiveHdfsWalker")) {
            RemoteIterator<LocatedFileStatus> iterator = getLocatedFileStatusRemoteIterator(path);

            while (iterator.hasNext()) {
                LocatedFileStatus status = getLocatedFileStatus(iterator);

                // ignore hidden files. Hive ignores files starting with _ and . as well.
                String fileName = status.getPath().getName();
                if (fileName.startsWith("_") || fileName.startsWith(".")) {
                    continue;
                }
                if (!isDirectory(status)) {
                    callback.process(status, status.getBlockLocations());
                }
                else if (recursive) {
                    recursiveWalk(status.getPath(), callback, taskCount, future);
                }
                if (future.isDone()) {
                    return;
                }
            }
View Full Code Here

            public RemoteIterator<LocatedFileStatus> listLocatedStatus(Path f)
                    throws IOException
            {
                ImmutableList.Builder<LocatedFileStatus> list = ImmutableList.builder();
                for (FileStatus status : paths.get(f.toString())) {
                    list.add(new LocatedFileStatus(status, new BlockLocation[0]));
                }
                return remoteIterator(list.build().iterator());
            }

            @Override
View Full Code Here

TOP

Related Classes of org.apache.hadoop.fs.LocatedFileStatus

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.