List of listPaths() Examples

Examples of listPaths()

org.apache.hadoop.fs.FileSystem.listPaths()
List files in a directory.
org.apache.hadoop.hdfs.DFSClient.listPaths()
Get a partial listing of the indicated directory Recommend to use HdfsFileStatus.EMPTY_NAME as startAfter if the application wants to fetch a listing starting from the first entry in the directory @param src the directory name @param startAfter the name to start listing after @return a partial listing starting after startAfter
org.apache.pig.PigServer.listPaths()
List the contents of a directory. @param dir name of directory to list @return array of strings, one for each file name @throws IOException

Examples of org.apache.hadoop.fs.FileSystem.listPaths()

    }
    List<Path> result = new ArrayList<Path>(); 
    for (Path p: dirs) {
      FileSystem fs = p.getFileSystem(job); 
      Path[] matches =
        fs.listPaths(fs.globPaths(p, hiddenFileFilter), hiddenFileFilter);
      for (Path match: matches) {
        result.add(fs.makeQualified(match));
      }
    }

View Full Code Here

Examples of org.apache.hadoop.fs.FileSystem.listPaths()

    for (Path p: inputDirs) {
      FileSystem fs = p.getFileSystem(job);
      if (fs.exists(p)) {
        // make sure all paths are files to avoid exception
        // while generating splits
        for (Path subPath : fs.listPaths(p, hiddenFileFilter)) {
          FileSystem subFS = subPath.getFileSystem(job); 
          if (!subFS.exists(subPath)) {
            result.add(new IOException(
                                       "Input path does not exist: " + subPath)); 
          } else {

View Full Code Here

Examples of org.apache.hadoop.fs.FileSystem.listPaths()

  public static String readOutput(Path outDir, 
                                  JobConf conf) throws IOException {
    FileSystem fs = outDir.getFileSystem(conf);
    StringBuffer result = new StringBuffer();
    {
      Path[] fileList = fs.listPaths(outDir);
      for(int i=0; i < fileList.length; ++i) {
        BufferedReader file = 
          new BufferedReader(new InputStreamReader(fs.open(fileList[i])));
        String line = file.readLine();
        while (line != null) {

View Full Code Here

Examples of org.apache.hadoop.fs.FileSystem.listPaths()

        job = new StreamJob(argv, mayExit);      
        job.go();


        fileSys = dfs.getFileSystem();
        String line = null;
        Path[] fileList = fileSys.listPaths(new Path(OUTPUT_DIR));
        for (int i = 0; i < fileList.length; i++){
          System.out.println(fileList[i].toString());
          BufferedReader bread =
            new BufferedReader(new InputStreamReader(fileSys.open(fileList[i])));
          line = bread.readLine();

View Full Code Here

Examples of org.apache.hadoop.fs.FileSystem.listPaths()

        job.go();


  fileSys = dfs.getFileSystem();
        String line = null;
        String line2 = null;
        Path[] fileList = fileSys.listPaths(new Path(OUTPUT_DIR));
        for (int i = 0; i < fileList.length; i++){
          System.out.println(fileList[i].toString());
          BufferedReader bread =
            new BufferedReader(new InputStreamReader(fileSys.open(fileList[i])));
          line = bread.readLine();

View Full Code Here

Examples of org.apache.hadoop.fs.FileSystem.listPaths()


  /** Open the output generated by this format. */
  public static SequenceFile.Reader[] getReaders(Configuration conf, Path dir)
    throws IOException {
    FileSystem fs = dir.getFileSystem(conf);
    Path[] names = fs.listPaths(dir);
    
    // sort names, so that hash partitioning works
    Arrays.sort(names);
    
    SequenceFile.Reader[] parts = new SequenceFile.Reader[names.length];

View Full Code Here

Examples of org.apache.hadoop.fs.FileSystem.listPaths()

        ArrayList dirs = new ArrayList();
        for (int i = 1; i < args.length; i++) {
          if (args[i] == null) continue;
          if (args[i].equals("-dir")) {
            Path dir = new Path(args[++i]);
            Path[] files = fs.listPaths(dir, new PathFilter() {
              public boolean accept(Path pathname) {
                try {
                  if (fs.isDirectory(pathname)) return true;
                } catch (IOException e) {};
                return false;

View Full Code Here

Examples of org.apache.hadoop.fs.FileSystem.listPaths()

    ArrayList segs = new ArrayList();
    long sliceSize = 0;
    boolean filter = false;
    for (int i = 1; i < args.length; i++) {
      if (args[i].equals("-dir")) {
        Path[] files = fs.listPaths(new Path(args[++i]), new PathFilter() {
          public boolean accept(Path f) {
            try {
              if (fs.isDirectory(f)) return true;
            } catch (IOException e) {}
            ;

View Full Code Here

Examples of org.apache.hadoop.fs.FileSystem.listPaths()

    output.collect(key, result);
  }


  public void invert(Path linkDb, final Path segmentsDir, boolean normalize, boolean filter, boolean force) throws IOException {
    final FileSystem fs = FileSystem.get(getConf());
    Path[] files = fs.listPaths(segmentsDir, new PathFilter() {
      public boolean accept(Path f) {
        try {
          if (fs.isDirectory(f)) return true;
        } catch (IOException ioe) {};
        return false;

View Full Code Here

Examples of org.apache.hadoop.fs.FileSystem.listPaths()

    boolean normalize = true;
    boolean force = false;
    for (int i = 1; i < args.length; i++) {
      if (args[i].equals("-dir")) {
        segDir = new Path(args[++i]);
        Path[] files = fs.listPaths(segDir, new PathFilter() {
          public boolean accept(Path f) {
            try {
              if (fs.isDirectory(f)) return true;
            } catch (IOException ioe) {};
            return false;

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.