Examples of org.apache.hadoop.fs.FileSystem.listStatus()

org.apache.hadoop.fs.FileSystem.listStatus()
List the statuses of the files/directories in the given path if the path is a directory. @param f given path @return the statuses of the files/directories in the given patch @throws IOException

        }


        Stack<FileStatus> pathstack = new Stack<FileStatus>();
        for(pathstack.push(srcfilestat); !pathstack.empty(); ) {
          FileStatus cur = pathstack.pop();
          FileStatus[] children = srcfs.listStatus(cur.getPath());
          for(int i = 0; i < children.length; i++) {
            boolean skipfile = false;
            final FileStatus child = children[i]; 
            final String dst = makeRelative(root, child.getPath());
            ++srcCount;

View Full Code Here

    }


    System.err.println("The final cursor is " + firstCursor);


    System.err.println("That directory ends up with "
                       + localfs.listStatus(firstCursor).length
                       + " subdirectories");


    Path cachesBase = firstCursor;


    assertFalse

View Full Code Here


    assertFalse
      ("DistributedCache did not delete the gensym'ed distcache "
           + "directory names when it deleted the files they contained "
           + "because they collectively exceeded the size limit.",
       localfs.listStatus(cachesBase).length > 1);
    
    
    // Now we test the number of sub directories limit
    // Create the temporary cache files to be used in the tests.
    Path thirdCacheFile = new Path(TEST_ROOT_DIR, "thirdcachefile");

View Full Code Here


    assertFalse
      ("DistributedCache did not delete the gensym'ed distcache "
           + "directory names when it deleted the files they contained "
           + "because there were too many.",
       localfs.listStatus(cachesBase).length > LOCAL_CACHE_SUBDIR_LIMIT);


    // Clean up the files created in this test
    new File(thirdCacheFile.toString()).delete();
    new File(fourthCacheFile.toString()).delete();
  }

View Full Code Here

        }
        Path dir = getTransactionInfoDir(conf);
        FileSystem fs = dir.getFileSystem(conf);
        FileStatus[] statusArray;
        try {
            statusArray = fs.listStatus(dir);
        } catch (FileNotFoundException e) {
            statusArray = null;
            if (LOG.isDebugEnabled()) {
                LOG.debug(MessageFormat.format("Target file is not found: {0}", dir), e);
            }

View Full Code Here

    LOG.info("Commiting Job [{0}]", jobContext.getJobID());
    Configuration configuration = jobContext.getConfiguration();
    Path tableOutput = BlurOutputFormat.getOutputPath(configuration);
    makeSureNoEmptyShards(configuration, tableOutput);
    FileSystem fileSystem = tableOutput.getFileSystem(configuration);
    for (FileStatus fileStatus : fileSystem.listStatus(tableOutput)) {
      if (isShard(fileStatus)) {
        commitOrAbortJob(jobContext, fileStatus.getPath(), true);
      }
    }

View Full Code Here

    }
  }


  private void commitOrAbortJob(JobContext jobContext, Path shardPath, boolean commit) throws IOException {
    FileSystem fileSystem = shardPath.getFileSystem(jobContext.getConfiguration());
    FileStatus[] listStatus = fileSystem.listStatus(shardPath);
    for (FileStatus fileStatus : listStatus) {
      Path path = fileStatus.getPath();
      String name = path.getName();
      boolean taskComplete = name.endsWith(".task_complete");
      if (fileStatus.isDir()) {

View Full Code Here

  public void run() {
    Path path = _shardContext.getHdfsDirPath();
    Configuration configuration = _shardContext.getTableContext().getConfiguration();
    try {
      FileSystem fileSystem = path.getFileSystem(configuration);
      SortedSet<FileStatus> listStatus = sort(fileSystem.listStatus(path));
      List<HdfsDirectory> indexesToImport = new ArrayList<HdfsDirectory>();
      for (FileStatus fileStatus : listStatus) {
        Path file = fileStatus.getPath();
        if (fileStatus.isDir() && file.getName().endsWith(".commit")) {
          HdfsDirectory hdfsDirectory = new HdfsDirectory(configuration, file);

View Full Code Here

      FileSystem fileSystem = FileSystem.get(tablePath.toUri(), _configuration);
      if (!fileSystem.exists(tablePath)) {
        LOG.error("Table [{0}] is missing, defined location [{1}]", table, tablePath.toUri());
        throw new RuntimeException("Table [" + table + "] is missing, defined location [" + tablePath.toUri() + "]");
      }
      FileStatus[] listStatus = fileSystem.listStatus(tablePath);
      for (FileStatus status : listStatus) {
        if (status.isDir()) {
          String name = status.getPath().getName();
          if (name.startsWith(SHARD_PREFIX)) {
            result.add(name);

View Full Code Here

    LOG.info("Abort Job [{0}]", jobContext.getJobID());
    Configuration configuration = jobContext.getConfiguration();
    Path tableOutput = BlurOutputFormat.getOutputPath(configuration);
    makeSureNoEmptyShards(configuration, tableOutput);
    FileSystem fileSystem = tableOutput.getFileSystem(configuration);
    for (FileStatus fileStatus : fileSystem.listStatus(tableOutput)) {
      if (isShard(fileStatus)) {
        commitOrAbortJob(jobContext, fileStatus.getPath(), false);
      }
    }
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.