Package org.apache.hadoop.mapred.lib

Examples of org.apache.hadoop.mapred.lib.CombineFileInputFormat$OneBlockInfoSizeComparator


    JobConf job = new JobConf(defaultConf);
    localFs.delete(workDir, true);
    writeFile(localFs, new Path(workDir, "test.txt"),
              "the quick\nbrown\nfox jumped\nover\n the lazy\n dog\n");
    FileInputFormat.setInputPaths(job, workDir);
    CombineFileInputFormat format = new CombineFileInputFormat() {
      @Override
      public RecordReader getRecordReader(InputSplit split, JobConf job, Reporter reporter) throws IOException {
        return new CombineFileRecordReader(job, (CombineFileSplit)split, reporter, CombineFileRecordReader.class);
      }
    };
    final int SIZE_SPLITS = 1;
    LOG.info("Trying to getSplits with splits = " + SIZE_SPLITS);
    InputSplit[] splits = format.getSplits(job, SIZE_SPLITS);
    LOG.info("Got getSplits = " + splits.length);
    assertEquals("splits == " + SIZE_SPLITS, SIZE_SPLITS, splits.length);
  }
View Full Code Here


    JobConf job = new JobConf(defaultConf);
    localFs.delete(workDir, true);
    writeFile(localFs, new Path(workDir, "test.txt"),
              "the quick\nbrown\nfox jumped\nover\n the lazy\n dog\n");
    FileInputFormat.setInputPaths(job, workDir);
    CombineFileInputFormat format = new CombineFileInputFormat() {
      @Override
      public RecordReader getRecordReader(InputSplit split, JobConf job, Reporter reporter) throws IOException {
        return new CombineFileRecordReader(job, (CombineFileSplit)split, reporter, CombineFileRecordReader.class);
      }
    };
    final int SIZE_SPLITS = 1;
    LOG.info("Trying to getSplits with splits = " + SIZE_SPLITS);
    InputSplit[] splits = format.getSplits(job, SIZE_SPLITS);
    LOG.info("Got getSplits = " + splits.length);
    assertEquals("splits == " + SIZE_SPLITS, SIZE_SPLITS, splits.length);
  }
View Full Code Here

    JobConf job = new JobConf(defaultConf);
    localFs.delete(workDir, true);
    writeFile(localFs, new Path(workDir, "test.txt"),
              "the quick\nbrown\nfox jumped\nover\n the lazy\n dog\n");
    FileInputFormat.setInputPaths(job, workDir);
    CombineFileInputFormat format = new CombineFileInputFormat() {
      @Override
      public RecordReader getRecordReader(InputSplit split, JobConf job, Reporter reporter) throws IOException {
        return new CombineFileRecordReader(job, (CombineFileSplit)split, reporter, CombineFileRecordReader.class);
      }
    };
    final int SIZE_SPLITS = 1;
    LOG.info("Trying to getSplits with splits = " + SIZE_SPLITS);
    InputSplit[] splits = format.getSplits(job, SIZE_SPLITS);
    LOG.info("Got getSplits = " + splits.length);
    assertEquals("splits == " + SIZE_SPLITS, SIZE_SPLITS, splits.length);
  }
View Full Code Here

TOP

Related Classes of org.apache.hadoop.mapred.lib.CombineFileInputFormat$OneBlockInfoSizeComparator

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.