Package org.apache.hadoop.fs

Examples of org.apache.hadoop.fs.PathFilter

111
112
113
114
115
116
117
118
119
120
121
     */
    public static FileStatus[] getDataChunkFiles(FileSystem fs,
                                                 Path path,
                                                 final int partitionId,
                                                 final int replicaType) throws IOException {
        return fs.listStatus(path, new PathFilter() {

            public boolean accept(Path input) {
                if(input.getName().matches("^" + Integer.toString(partitionId) + "_"
                                           + Integer.toString(replicaType) + "_[\\d]+\\.data")) {
                    return true;
View Full Code Here
143
144
145
146
147
148
149
150
151
152
153
    public static FileStatus[] getDataChunkFiles(FileSystem fs,
                                                 Path path,
                                                 final int partitionId,
                                                 final int replicaType,
                                                 final int chunkId) throws IOException {
        return fs.listStatus(path, new PathFilter() {

            public boolean accept(Path input) {
                if(input.getName().matches("^" + Integer.toString(partitionId) + "_"
                                           + Integer.toString(replicaType) + "_"
                                           + Integer.toString(chunkId) + "\\.data")) {
View Full Code Here
476
477
478
479
480
481
482
483
484
485
486
                    logger.info("Setting permission to 755 for " + nodePath);
                }

                if(checkSumType != CheckSumType.NONE) {

                    FileStatus[] storeFiles = outputFs.listStatus(nodePath, new PathFilter() {

                        public boolean accept(Path arg0) {
                            if(arg0.getName().endsWith("checksum")
                               && !arg0.getName().startsWith(".")) {
                                return true;
View Full Code Here
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
   
    // creates a MultiPathFilter with the hiddenFileFilter and the
    // user provided one (if any).
    List<PathFilter> filters = new ArrayList<PathFilter>();
    filters.add(hiddenFileFilter);
    PathFilter jobFilter = getInputPathFilter(job);
    if (jobFilter != null) {
      filters.add(jobFilter);
    }
    PathFilter inputFilter = new MultiPathFilter(filters);
   
    for (int i=0; i < dirs.length; ++i) {
      Path p = dirs[i];
      FileSystem fs = p.getFileSystem(job.getConfiguration());
      FileStatus[] matches = fs.globStatus(p, inputFilter);
View Full Code Here
938
939
940
941
942
943
944
945
946
947
948
      // a path filter that matches 4 parts of the filenames namely
      //  - jt-hostname
      //  - job-id
      //  - username
      //  - jobname
      PathFilter filter = new PathFilter() {
        public boolean accept(Path path) {
          String fileName = path.getName();
          try {
            fileName = decodeJobHistoryFileName(fileName);
          } catch (IOException ioe) {
View Full Code Here
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
   
    // creates a MultiPathFilter with the hiddenFileFilter and the
    // user provided one (if any).
    List<PathFilter> filters = new ArrayList<PathFilter>();
    filters.add(hiddenFileFilter);
    PathFilter jobFilter = getInputPathFilter(job);
    if (jobFilter != null) {
      filters.add(jobFilter);
    }
    PathFilter inputFilter = new MultiPathFilter(filters);

    for (Path p: dirs) {
      FileSystem fs = p.getFileSystem(job);
      FileStatus[] matches = fs.globStatus(p, inputFilter);
      if (matches == null) {
View Full Code Here
324
325
326
327
328
329
330
331
332
333
334
      Path p = Store.getStoreHomedir(tabledir, split.getEncodedName(),
        family.getName());
      if (!fs.exists(p)) continue;
      // Look for reference files.  Call listStatus with anonymous instance of PathFilter.
      FileStatus [] ps = FSUtils.listStatus(fs, p,
          new PathFilter () {
            public boolean accept(Path path) {
              return StoreFile.isReference(path);
            }
          }
      );
View Full Code Here
254
255
256
257
258
259
260
261
262
263
264
   * @throws IOException
   */
  private static FileStatus getTableInfoPath(final FileSystem fs,
      final Path tabledir)
  throws IOException {
    FileStatus [] status = FSUtils.listStatus(fs, tabledir, new PathFilter() {
      @Override
      public boolean accept(Path p) {
        // Accept any file that starts with TABLEINFO_NAME
        return p.getName().startsWith(TABLEINFO_NAME);
      }
View Full Code Here
390
391
392
393
394
395
396
397
398
399
400
  private static boolean isConverged(String filePath, Configuration conf, FileSystem fs) throws IOException {
   
    Path clusterPath = new Path(filePath + "/*");
    List<Path> result = new ArrayList<Path>();
   
    PathFilter clusterFileFilter = new PathFilter() {
      @Override
      public boolean accept(Path path) {
        return path.getName().startsWith("part");
      }
    };
View Full Code Here
44
45
46
47
48
49
50
51
52
53
54
    Configuration job = new Configuration();
    Path clusterPath = new Path(clusterPathStr + "/*");
    List<Path> result = new ArrayList<Path>();
    //log.info("I am here");
    // filter out the files
    PathFilter clusterFileFilter = new PathFilter() {
      @Override
      public boolean accept(Path path) {
        return path.getName().startsWith("part");
      }
    };
View Full Code Here
TOP

Related Classes of org.apache.hadoop.fs.PathFilter

  • com.aamend.hadoop.clustering.job.CanopyDriver
  • com.alexholmes.hadooputils.test.TextIOJobBuilder
  • com.bah.geterdun.WriteAheadLogManager
  • com.linkedin.camus.etl.kafka.CamusJob
  • com.m6d.filecrush.crush.Crush
  • com.twitter.elephantbird.mapreduce.output.LuceneIndexOutputFormat
  • com.twitter.elephantbird.util.HdfsUtils
  • com.twitter.elephantbird.util.TestPathFilters
  • org.apache.accumulo.server.upgrade.UpgradeMetadataTable
  • org.apache.blur.manager.writer.IndexImporter
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.