Examples of org.apache.hadoop.mapred.lib.CombineFileSplit

org.apache.hadoop.mapred.lib.CombineFileSplit
A sub-collection of input files. Unlike {@link org.apache.hadoop.mapred.FileSplit}, CombineFileSplit * class does not represent a split of a file, but a split of input files into smaller sets. A split may contain blocks from different file but all the blocks in the same split are probably local to some rack
CombineFileSplit can be used to implement {@link org.apache.hadoop.mapred.RecordReader}'s, with reading one record per file. @see org.apache.hadoop.mapred.FileSplit @see CombineFileInputFormat


    public RecordReader getRecordReader(JobConf job, HadoopShims.InputSplitShim split,
        Reporter reporter,
        Class<RecordReader<K, V>> rrClass)
        throws IOException {
      CombineFileSplit cfSplit = (CombineFileSplit) split;
      return new CombineFileRecordReader(job, cfSplit, reporter, rrClass);
    }

View Full Code Here


    public RecordReader getRecordReader(JobConf job, HadoopShims.InputSplitShim split,
        Reporter reporter,
        Class<RecordReader<K, V>> rrClass)
        throws IOException {
      CombineFileSplit cfSplit = (CombineFileSplit) split;
      return new CombineFileRecordReader(job, cfSplit, reporter, rrClass);
    }

View Full Code Here


    public RecordReader getRecordReader(JobConf job, HadoopShims.InputSplitShim split,
        Reporter reporter,
        Class<RecordReader<K, V>> rrClass)
        throws IOException {
      CombineFileSplit cfSplit = (CombineFileSplit) split;
      return new CombineFileRecordReader(job, cfSplit, reporter, rrClass);
    }

View Full Code Here


    public RecordReader getRecordReader(JobConf job, HadoopShims.InputSplitShim split,
        Reporter reporter,
        Class<RecordReader<K, V>> rrClass)
        throws IOException {
      CombineFileSplit cfSplit = (CombineFileSplit) split;
      return new CombineFileRecordReader(job, cfSplit, reporter, rrClass);
    }

View Full Code Here


    public RecordReader getRecordReader(JobConf job, HadoopShims.InputSplitShim split,
        Reporter reporter,
        Class<RecordReader<K, V>> rrClass)
        throws IOException {
      CombineFileSplit cfSplit = (CombineFileSplit) split;
      return new CombineFileRecordReader(job, cfSplit, reporter, rrClass);
    }

View Full Code Here


    public RecordReader getRecordReader(JobConf job, HadoopShims.InputSplitShim split,
        Reporter reporter,
        Class<RecordReader<K, V>> rrClass)
        throws IOException {
      CombineFileSplit cfSplit = (CombineFileSplit) split;
      return new CombineFileRecordReader(job, cfSplit, reporter, rrClass);
    }

View Full Code Here


    public static Map<String, List<CombineFileSplitAdapter>> organizeSplitsByLocation(final InputSplit[] splits) throws IOException {
        Map<String, List<CombineFileSplitAdapter>> locationSplits = new HashMap<String, List<CombineFileSplitAdapter>>();


        for (InputSplit split : splits) {
            CombineFileSplit csplit = (CombineFileSplit) split;


            String location = extractLocation(csplit);


            List<CombineFileSplitAdapter> splitsForLocation = locationSplits.get(location);
            if (splitsForLocation == null) {

View Full Code Here


    public RecordReader getRecordReader(JobConf job, HadoopShims.InputSplitShim split,
        Reporter reporter,
        Class<RecordReader<K, V>> rrClass)
        throws IOException {
      CombineFileSplit cfSplit = (CombineFileSplit) split;
      return new CombineFileRecordReader(job, cfSplit, reporter, rrClass);
    }

View Full Code Here


    public RecordReader getRecordReader(JobConf job, HadoopShims.InputSplitShim split,
        Reporter reporter,
        Class<RecordReader<K, V>> rrClass)
        throws IOException {
      CombineFileSplit cfSplit = (CombineFileSplit) split;
      return new CombineFileRecordReader(job, cfSplit, reporter, rrClass);
    }

View Full Code Here


    public RecordReader getRecordReader(JobConf job, HadoopShims.InputSplitShim split,
        Reporter reporter,
        Class<RecordReader<K, V>> rrClass)
        throws IOException {
      CombineFileSplit cfSplit = (CombineFileSplit) split;
      return new CombineFileRecordReader(job, cfSplit, reporter, rrClass);
    }

View Full Code Here

0 1 2 3

TOP

Related Classes of org.apache.hadoop.mapred.lib.CombineFileSplit

com.alexholmes.hadooputils.combine.common.mapred.SplitMetricsCombineInputFormat

org.apache.hadoop.fs.Path

org.apache.hadoop.hive.shims.Hadoop20Shims$CombineFileInputFormatShim

org.apache.hadoop.hive.shims.Hadoop20SShims$CombineFileInputFormatShim

org.apache.hadoop.hive.shims.Hadoop23Shims$CombineFileInputFormatShim

org.apache.hadoop.hive.shims.HadoopShimsSecure$CombineFileInputFormatShim

org.terrier.structures.indexing.singlepass.hadoop.MultiFileCollectionInputFormat

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.