List of globStatus() Examples

Examples of globStatus()

org.apache.hadoop.fs.FileSystem.globStatus()

Return all the files that match filePattern and are not checksum files. Results are sorted by their names.
A filename pattern is composed of regular characters and special pattern matching characters, which are:

?
Matches any single character.

*
Matches zero or more characters.

[abc]
Matches a single character from character set {a,b,c}.

[a-b]
Matches a single character from the character range {a...b}. Note that character a must be lexicographically less than or equal to character b.

[^a]
Matches a single character that is not from character set or range {a}. Note that the ^ character must occur immediately to the right of the opening bracket.

\c
Removes (escapes) any special meaning of character c.

{ab,cd}
Matches a string from the string set {ab, cd}

{ab,c{de,fh}}
Matches a string from the string set {ab, cde, cfh}

@param pathPattern a regular expression specifying a pth pattern @return an array of paths that match the path pattern @throws IOException

Examples of org.apache.hadoop.fs.FileSystem.globStatus()

            LOG.debug(MessageFormat.format(
                    "Listing temporary input: {0} (fs={1})",
                    pathPattern,
                    fs.getUri()));
        }
        FileStatus[] statusList = fs.globStatus(pathPattern);
        if (statusList == null || statusList.length == 0) {
            return Collections.emptyList();
        }
        List<Path> results = new ArrayList<Path>();
        for (FileStatus status : statusList) {

View Full Code Here

Examples of org.apache.hadoop.fs.FileSystem.globStatus()

    private List<InputSplit> getSplits(Configuration configuration, List<Path> paths) throws IOException {
        long splitSize = configuration.getLong(KEY_DEFAULT_SPLIT_SIZE, DEFAULT_SPLIT_SIZE);
        List<InputSplit> results = new ArrayList<InputSplit>();
        for (Path path : paths) {
            FileSystem fs = path.getFileSystem(configuration);
            FileStatus[] statuses = fs.globStatus(path);
            if (statuses == null) {
                continue;
            }
            for (FileStatus status : statuses) {
                BlockMap blockMap = BlockMap.create(

View Full Code Here

Examples of org.apache.hadoop.fs.FileSystem.globStatus()

            context.setError();
            return false;
        }
        List<FileStatus> files;
        try {
            files = asList(fs.globStatus(path));
        } catch (Exception e) {
            LOG.error(MessageFormat.format(
                    "[OT-CLEAN-E01002] Failed to glob path pattern: {0}",
                    path), e);
            context.setError();

View Full Code Here

Examples of org.apache.hadoop.fs.FileSystem.globStatus()

            TsvIoFactory<T> factory = new TsvIoFactory<T>(targetTableModel);
            Configuration conf = new Configuration();
            fs = FileSystem.get(new URI(filePath), conf);


            // ディレクトリ内のファイルを取得し、ファイルを出力する
            FileStatus[] status = fs.globStatus(new Path(filePath));
            Path[] listedPaths = FileUtil.stat2Paths(status);
            if (listedPaths == null) {
                LOG.info("TG-COLLECTOR-02006",
                        tableName, filePath);
                return -1;

View Full Code Here

Examples of org.apache.hadoop.fs.FileSystem.globStatus()

    }


    private void loadResult(String tablename, String excelFileName) throws IOException {
        Configuration conf = ConfigurationFactory.getDefault().newInstance();
        FileSystem fs = FileSystem.get(conf);
        FileStatus[] status = fs.globStatus(new Path(computeOutputPath(fs, excelFileName)));
        Path[] listedPaths = FileUtil.stat2Paths(status);
        for (Path path : listedPaths) {
            if (isSystemFile(path)) {
                continue;
            }

View Full Code Here

Examples of org.apache.hadoop.fs.FileSystem.globStatus()

    }


    private List<TestDataModel> collect(CacheStorage storage, Path contents) throws IOException {
        List<TestDataModel> results = new ArrayList<TestDataModel>();
        FileSystem fs = storage.getFileSystem();
        for (FileStatus status : fs.globStatus(contents)) {
            results.addAll(collectContent(fs, status));
        }
        Collections.sort(results);
        return results;
    }

View Full Code Here

Examples of org.apache.hadoop.fs.FileSystem.globStatus()

   */
  public static PathData[] expandAsGlob(String pattern, Configuration conf)
  throws IOException {
    Path globPath = new Path(pattern);
    FileSystem fs = globPath.getFileSystem(conf);    
    FileStatus[] stats = fs.globStatus(globPath);
    PathData[] items = null;
    
    if (stats == null) {
      // remove any quoting in the glob pattern
      pattern = pattern.replaceAll("\\\\(.)", "$1");

View Full Code Here

Examples of org.apache.hadoop.fs.FileSystem.globStatus()

    LongWritable value = new LongWritable();
    int minSupport = Integer.valueOf(params.get("minSupport", "3"));
    Configuration conf = new Configuration();
    
    FileSystem fs = FileSystem.get(new Path(params.get("output") + "/parallelcounting").toUri(), conf);
    FileStatus[] outputFiles = fs.globStatus(new Path(params.get("output") + "/parallelcounting/part-*"));
    
    PriorityQueue<Pair<String,Long>> queue = new PriorityQueue<Pair<String,Long>>(11,
        new Comparator<Pair<String,Long>>() {
          
          @Override

View Full Code Here

Examples of org.apache.hadoop.fs.FileSystem.globStatus()

  public static List<Pair<String,TopKStringPatterns>> readFrequentPattern(Parameters params) throws IOException {
    
    Configuration conf = new Configuration();
    
    FileSystem fs = FileSystem.get(new Path(params.get("output") + "/frequentPatterns").toUri(), conf);
    FileStatus[] outputFiles = fs.globStatus(new Path(params.get("output") + "/frequentPatterns/part-*"));
    
    List<Pair<String,TopKStringPatterns>> ret = new ArrayList<Pair<String,TopKStringPatterns>>();
    for (FileStatus fileStatus : outputFiles) {
      Path path = fileStatus.getPath();
      ret.addAll(FPGrowth.readFrequentPattern(fs, conf, path));

View Full Code Here

Examples of org.apache.hadoop.fs.FileSystem.globStatus()

    
    Writable key = new Text();
    Configuration conf = new Configuration();
    
    FileSystem fs = FileSystem.get(wordCountPath.toUri(), conf);
    FileStatus[] outputFiles = fs.globStatus(new Path(wordCountPath.toString()
                                                      + OUTPUT_FILES_PATTERN));
    
    long chunkSizeLimit = chunkSizeInMegabytes * 1024 * 1024;
    int chunkIndex = 0;
    Path chunkPath = getPath(dictionaryPathBase + DICTIONARY_FILE,

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.