Package org.apache.hadoop.fs

Examples of org.apache.hadoop.fs.FileSystem.listPaths()

183
184
185
186
187
188
189
190
191
192
193
    output.collect(key, result);
  }

  public void invert(Path linkDb, final Path segmentsDir) throws IOException {
    final FileSystem fs = FileSystem.get(getConf());
    Path[] files = fs.listPaths(segmentsDir, new PathFilter() {
      public boolean accept(Path f) {
        try {
          if (fs.isDirectory(f)) return true;
        } catch (IOException ioe) {};
        return false;
View Full Code Here
299
300
301
302
303
304
305
306
307
308
309
    Path db = new Path(args[0]);
    ArrayList segs = new ArrayList();
    for (int i = 1; i < args.length; i++) {
      if (args[i].equals("-dir")) {
        segDir = new Path(args[++i]);
        Path[] files = fs.listPaths(segDir, new PathFilter() {
          public boolean accept(Path f) {
            try {
              if (fs.isDirectory(f)) return true;
            } catch (IOException ioe) {};
            return false;
View Full Code Here
62
63
64
65
66
67
68
69
70
71
72
  /** Open the output generated by this format. */
  public static SequenceFile.Reader[] getReaders(Configuration conf, Path dir)
    throws IOException {
    FileSystem fs = FileSystem.get(conf);
    Path[] names = fs.listPaths(dir);
   
    // sort names, so that hash partitioning works
    Arrays.sort(names);
   
    SequenceFile.Reader[] parts = new SequenceFile.Reader[names.length];
View Full Code Here
81
82
83
84
85
86
87
88
89
90
91
  /** Open the output generated by this format. */
  public static SequenceFile.Reader[] getReaders(Configuration conf, Path dir)
    throws IOException {
    FileSystem fs = FileSystem.get(conf);
    Path[] names = fs.listPaths(dir);
   
    // sort names, so that hash partitioning works
    Arrays.sort(names);
   
    SequenceFile.Reader[] parts = new SequenceFile.Reader[names.length];
View Full Code Here
70
71
72
73
74
75
76
77
78
79
80
    conf.setNumMapTasks(numMaps);
    conf.setNumReduceTasks(numReduces);
    JobClient.runJob(conf);
    StringBuffer result = new StringBuffer();
    {
      Path[] fileList = fs.listPaths(outDir);
      for(int i=0; i < fileList.length; ++i) {
        BufferedReader file =
          new BufferedReader(new InputStreamReader(fs.open(fileList[i])));
        String line = file.readLine();
        while (line != null) {
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.