Examples of org.apache.hadoop.fs.FileSystem.globStatus()

org.apache.hadoop.fs.FileSystem.globStatus()

Return all the files that match filePattern and are not checksum files. Results are sorted by their names.
A filename pattern is composed of regular characters and special pattern matching characters, which are:

?
Matches any single character.

*
Matches zero or more characters.

[abc]
Matches a single character from character set {a,b,c}.

[a-b]
Matches a single character from the character range {a...b}. Note that character a must be lexicographically less than or equal to character b.

[^a]
Matches a single character that is not from character set or range {a}. Note that the ^ character must occur immediately to the right of the opening bracket.

\c
Removes (escapes) any special meaning of character c.

{ab,cd}
Matches a string from the string set {ab, cd}

{ab,c{de,fh}}
Matches a string from the string set {ab, cde, cfh}

@param pathPattern a regular expression specifying a pth pattern @return an array of paths that match the path pattern @throws IOException

    reader.close();
    Assert.assertEquals(3, counts.get("apple").intValue());
    Assert.assertEquals(2, counts.get("banana").intValue());
    Assert.assertEquals(1, counts.get("carrot").intValue());
    
    outputFiles = fileSystem.globStatus(outputPath.suffix("/testwritenonschema-r-00000.avro"));
    Assert.assertEquals(1, outputFiles.length);
    reader = new DataFileReader<GenericData.Record>(
        new FsInput(outputFiles[0].getPath(), job.getConfiguration()),
        new GenericDatumReader<GenericData.Record>(STATS_SCHEMA));
    counts = new HashMap<String, Integer>();

View Full Code Here

    outputPath.getFileSystem(job.getConfiguration()).delete(outputPath); 
    FileOutputFormat.setOutputPath(job, outputPath);


    Assert.assertTrue(job.waitForCompletion(true));
    FileSystem fileSystem = FileSystem.get(job.getConfiguration());
    FileStatus[] outputFiles = fileSystem.globStatus(outputPath.suffix("/myavro3-*"));
    Assert.assertEquals(1, outputFiles.length);
    DataFileReader<TextStats> reader = new DataFileReader<TextStats>(
        new FsInput(outputFiles[0].getPath(), job.getConfiguration()),
        new SpecificDatumReader<TextStats>());
    Map<String, Integer> counts = new HashMap<String, Integer>();

View Full Code Here


    Assert.assertTrue(job.waitForCompletion(true));


    // Check that the results from the MapReduce were as expected.
    FileSystem fileSystem = FileSystem.get(job.getConfiguration());
    FileStatus[] outputFiles = fileSystem.globStatus(outputPath.suffix("/myavro3-*"));
    Assert.assertEquals(1, outputFiles.length);
    DataFileReader<TextStats> reader = new DataFileReader<TextStats>(
        new FsInput(outputFiles[0].getPath(), job.getConfiguration()),
        new SpecificDatumReader<TextStats>());
    Map<String, Integer> counts = new HashMap<String, Integer>();

View Full Code Here


    Assert.assertTrue(job.waitForCompletion(true));


    // Check that the results from the MapReduce were as expected.
    FileSystem fileSystem = FileSystem.get(job.getConfiguration());
    FileStatus[] outputFiles = fileSystem.globStatus(outputPath.suffix("/part-*"));
    Assert.assertEquals(1, outputFiles.length);
    DataFileReader<TextStats> reader = new DataFileReader<TextStats>(
        new FsInput(outputFiles[0].getPath(), job.getConfiguration()),
        new SpecificDatumReader<TextStats>());
    Map<String, Integer> counts = new HashMap<String, Integer>();

View Full Code Here


    Assert.assertTrue(job.waitForCompletion(true));


    // Check that the results from the MapReduce were as expected.
    FileSystem fileSystem = FileSystem.get(job.getConfiguration());
    FileStatus[] outputFiles = fileSystem.globStatus(outputPath.suffix("/part-*"));
    Assert.assertEquals(1, outputFiles.length);
    DataFileReader<ReflectStats> reader = new DataFileReader<ReflectStats>(
        new FsInput(outputFiles[0].getPath(), job.getConfiguration()),
        new ReflectDatumReader<ReflectStats>());
    Map<String, Integer> counts = new HashMap<String, Integer>();

View Full Code Here


    Assert.assertTrue(job.waitForCompletion(true));


    // Check that the results from the MapReduce were as expected.
    FileSystem fileSystem = FileSystem.get(job.getConfiguration());
    FileStatus[] outputFiles = fileSystem.globStatus(outputPath.suffix("/part-*"));
    Assert.assertEquals(1, outputFiles.length);
    DataFileReader<TextStats> reader = new DataFileReader<TextStats>(
        new FsInput(outputFiles[0].getPath(), job.getConfiguration()),
        new SpecificDatumReader<TextStats>());
    Map<String, Integer> counts = new HashMap<String, Integer>();

View Full Code Here


    Assert.assertTrue(job.waitForCompletion(true));


    // Check that the results from the MapReduce were as expected.
    FileSystem fileSystem = FileSystem.get(job.getConfiguration());
    FileStatus[] outputFiles = fileSystem.globStatus(outputPath.suffix("/part-*"));
    Assert.assertEquals(1, outputFiles.length);
    Path filePath = outputFiles[0].getPath();
    InputStream inputStream = filePath.getFileSystem(job.getConfiguration()).open(filePath);
    Assert.assertNotNull(inputStream);
    BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream));

View Full Code Here

    private static Path getStagingPath(org.apache.falcon.entity.v0.cluster.Cluster cluster, Path path)
        throws FalconException {
        try {
            FileSystem fs = HadoopClientFactory.get().createFileSystem(ClusterHelper.getConfiguration(cluster));
            FileStatus latest = null;
            FileStatus[] files = fs.globStatus(path, new PathFilter() {
                @Override
                public boolean accept(Path path) {
                    if (path.getName().equals("logs")) {
                        return false;
                    }

View Full Code Here

                equals(sourcePath.toString()) : "Source path is not a subset of include path";


        String relativePath = includePath.toString().substring(sourcePath.toString().length());
        String fixedPath = getFixedPath(relativePath);


        FileStatus[] files = fs.globStatus(new Path(targetPath.toString() + "/" + fixedPath));
        if (files != null) {
            for (FileStatus file : files) {
                fs.create(new Path(file.getPath(), EntityUtil.SUCCEEDED_FILE_NAME)).close();
                LOG.info("Created " + new Path(file.getPath(), EntityUtil.SUCCEEDED_FILE_NAME));
            }

View Full Code Here

        return usage;
    }


    private long usage(Path inPath, Configuration conf) throws IOException, FalconException {
        FileSystem fs = HadoopClientFactory.get().createFileSystem(inPath.toUri(), conf);
        FileStatus[] fileStatuses = fs.globStatus(inPath);
        if (fileStatuses == null || fileStatuses.length == 0) {
            return 0;
        }
        long totalSize = 0;
        for (FileStatus fileStatus : fileStatuses) {

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.