Package org.apache.hadoop.fs

Examples of org.apache.hadoop.fs.PathFilter

7945
7946
7947
7948
7949
7950
7951
7952
7953
7954
7955
        !conf.getBoolVar(HiveConf.ConfVars.LOCALMODEAUTO)) {
      return;
    }

    final Context lCtx = ctx;
    PathFilter p = new PathFilter () {
        public boolean accept(Path file) {
          return !lCtx.isMRTmpFileURI(file.toUri().getPath());
        }
      };
    List<ExecDriver> mrtasks = Utilities.getMRTasks(rootTasks);
View Full Code Here
287
288
289
290
291
292
293
294
295
296
297
  private void checkAndResumeUpdateOperation(Path path) throws Exception {
    // Before loading the state information, check whether .new file exists.
    // If it does, the prior updateFile is failed on half way. We need to
    // complete replacing the old file first.
    FileStatus[] newChildNodes =
        fs.listStatus(path, new PathFilter() {
      @Override
      public boolean accept(Path path) {
        return path.getName().endsWith(".new");
      }
    });
View Full Code Here
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
    // creates a MultiPathFilter with the hiddenFileFilter and the
    // user provided one (if any).
    List<PathFilter> filters = new ArrayList<PathFilter>();
    filters.add(hiddenFileFilter);
    PathFilter jobFilter = getInputPathFilter(job);
    if (jobFilter != null) {
      filters.add(jobFilter);
    }
    PathFilter inputFilter = new MultiPathFilter(filters);

    FileStatus[] result;
    int numThreads = job
        .getInt(
            org.apache.hadoop.mapreduce.lib.input.FileInputFormat.LIST_STATUS_NUM_THREADS,
View Full Code Here
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
    // creates a MultiPathFilter with the hiddenFileFilter and the
    // user provided one (if any).
    List<PathFilter> filters = new ArrayList<PathFilter>();
    filters.add(hiddenFileFilter);
    PathFilter jobFilter = getInputPathFilter(job);
    if (jobFilter != null) {
      filters.add(jobFilter);
    }
    PathFilter inputFilter = new MultiPathFilter(filters);

    for (Path p : dirs) {
      FileSystem fs = p.getFileSystem(job.getConf());
      FileStatus[] matches = fs.globStatus(p, inputFilter);
      if (matches == null) {
View Full Code Here
664
665
666
667
668
669
670
671
672
673
674
   * @throws IOException When scanning the files fails.
   */
  static List<Path> getStoreFiles(FileSystem fs, Path regionDir)
      throws IOException {
    List<Path> res = new ArrayList<Path>();
    PathFilter dirFilter = new FSUtils.DirFilter(fs);
    FileStatus[] familyDirs = fs.listStatus(regionDir, dirFilter);
    for(FileStatus dir : familyDirs) {
      FileStatus[] files = fs.listStatus(dir.getPath());
      for (FileStatus file : files) {
        if (!file.isDir()) {
View Full Code Here
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
   
    // creates a MultiPathFilter with the hiddenFileFilter and the
    // user provided one (if any).
    List<PathFilter> filters = new ArrayList<PathFilter>();
    filters.add(hiddenFileFilter);
    PathFilter jobFilter = getInputPathFilter(job);
    if (jobFilter != null) {
      filters.add(jobFilter);
    }
    PathFilter inputFilter = new MultiPathFilter(filters);
   
    for (int i=0; i < dirs.length; ++i) {
      Path p = dirs[i];
      FileSystem fs = p.getFileSystem(job.getConfiguration());
      FileStatus[] matches = fs.globStatus(p, inputFilter);
View Full Code Here
758
759
760
761
762
763
764
765
766
767
768
      tableDescriptor.getColumnFamilies()[0].getDataBlockEncoding());
  }

  private static FileStatus getTableInfoPath(final FileSystem fs, final Path tabledir)
      throws IOException {
    FileStatus[] status = FSUtils.listStatus(fs, tabledir, new PathFilter() {
      @Override
      public boolean accept(Path p) {
        // Accept any file that starts with TABLEINFO_NAME
        return p.getName().startsWith(FSTableDescriptors.TABLEINFO_NAME);
      }
View Full Code Here
318
319
320
321
322
323
324
325
326
327
328
      Path p = Store.getStoreHomedir(tabledir, split.getEncodedName(),
        family.getName());
      // Look for reference files.  Call listStatus with an anonymous
      // instance of PathFilter.
      FileStatus [] ps = this.master.fs.listStatus(p,
          new PathFilter () {
            public boolean accept(Path path) {
              return StoreFile.isReference(path);
            }
          }
      );
View Full Code Here
290
291
292
293
294
295
296
297
298
299
300
   * @param hbaseRootDir
   * @throws IOException
   */
  private void cleanOldLogFiles(final Path hbaseRootDir)
  throws IOException {
    FileStatus [] oldlogfiles = fs.listStatus(hbaseRootDir, new PathFilter () {
      public boolean accept(Path p) {
        return p.getName().startsWith("log_");
      }
    });
    // Return if nothing to do.
View Full Code Here
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
   * @throws IOException When scanning the files fails.
   */
  static List<Path> getStoreFiles(FileSystem fs, Path regionDir)
  throws IOException {
    List<Path> res = new ArrayList<Path>();
    PathFilter dirFilter = new FSUtils.DirFilter(fs);
    FileStatus[] familyDirs = fs.listStatus(regionDir, dirFilter);
    for(FileStatus dir : familyDirs) {
      FileStatus[] files = fs.listStatus(dir.getPath());
      for (FileStatus file : files) {
        if (!file.isDir()) {
View Full Code Here
TOP

Related Classes of org.apache.hadoop.fs.PathFilter

  • com.aamend.hadoop.clustering.job.CanopyDriver
  • com.alexholmes.hadooputils.test.TextIOJobBuilder
  • com.bah.geterdun.WriteAheadLogManager
  • com.linkedin.camus.etl.kafka.CamusJob
  • com.m6d.filecrush.crush.Crush
  • com.twitter.elephantbird.mapreduce.output.LuceneIndexOutputFormat
  • com.twitter.elephantbird.util.HdfsUtils
  • com.twitter.elephantbird.util.TestPathFilters
  • org.apache.accumulo.server.upgrade.UpgradeMetadataTable
  • org.apache.blur.manager.writer.IndexImporter
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.