Package com.scaleunlimited.cascading

Examples of com.scaleunlimited.cascading.BasePath.mkdirs()

    public void testFindAllSubdirs() throws Exception {
        // Make a loop dir with a subdir
        String subdirName = "bogus";
        BasePath path0 = CrawlDirUtils.makeLoopDir(_platform, _outputPath, 0);
        BasePath subdirPath0 = _platform.makePath(path0, subdirName);
        // And another one without the subdir
        BasePath path1 = CrawlDirUtils.makeLoopDir(_platform, _outputPath, 1);

        BasePath[] allSubdirPathsArr = CrawlDirUtils.findAllSubdirs(_platform, _outputPath, subdirName);
View Full Code Here

        BasePath[] allSubdirPathsArr = CrawlDirUtils.findAllSubdirs(_platform, _outputPath, subdirName);
        Assert.assertEquals(1, allSubdirPathsArr.length);

        // Now add a subdir to path1 as well
        BasePath subdirPath1 = _platform.makePath(path1, subdirName);
        BasePath[] strictSubdirPathsArr = CrawlDirUtils.findAllSubdirs(_platform, _outputPath, subdirName);
        Assert.assertEquals(2, strictSubdirPathsArr.length);
View Full Code Here

            // See if the user isn't starting from scratch then set up the
            // output directory and create an initial urls subdir.
            if (!outputPath.exists()) {

                // Create a "0-<timestamp>" sub-directory with just a /crawldb subdir
                // In the /crawldb dir the input file will have a single URL for the target domain.

                BasePath curLoopDir = CrawlDirUtils.makeLoopDir(platform, outputPath, 0);
View Full Code Here

  public static BasePath makeLoopDir(BasePlatform platform, BasePath outputDir, int loopNumber)
      throws Exception {
    String timestamp = new SimpleDateFormat("yyyyMMdd'T'HHmmss")
        .format(new Date());
    BasePath loopDir = platform.makePath(outputDir, "" + loopNumber + "-" + timestamp);
    return loopDir;

View Full Code Here

Copyright © 2018 All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact