Examples of org.apache.hadoop.mapreduce.InputSplit

org.apache.hadoop.mapreduce.InputSplit
InputSplit represents the data to be processed by an individual {@link Mapper}.
Typically, it presents a byte-oriented view on the input and is the responsibility of {@link RecordReader} of the job to process this and presenta record-oriented view. @see InputFormat @see RecordReader

    }
    return isParent(possibleParent, child.getParent());
  }


  protected Path getCurrentFile(Context context) throws IOException {
    InputSplit split = context.getInputSplit();
    if (split != null && split instanceof FileSplit) {
      FileSplit inputSplit = (FileSplit) split;
      Path path = inputSplit.getPath();
      return path.makeQualified(path.getFileSystem(context.getConfiguration()));
    }

View Full Code Here

          Bytes.compareTo(keys.getFirst()[i], startRow) >= 0 ? 
            keys.getFirst()[i] : startRow;
        byte[] splitStop = stopRow.length == 0 || 
          Bytes.compareTo(keys.getSecond()[i], stopRow) <= 0 ? 
            keys.getSecond()[i] : stopRow;
        InputSplit split = new TableSplit(table.getTableName(),
          splitStart, splitStop, regionLocation);
        splits.add(split);
        if (LOG.isDebugEnabled()) 
          LOG.debug("getSplits: split -> " + (count++) + " -> " + split);
      }

View Full Code Here

    CrunchInputSplit crunchSplit = (CrunchInputSplit) inputSplit;
    Configuration conf = crunchSplit.getConf();
    if (conf == null) {
      conf = context.getConfiguration();
    }
    InputSplit delegateSplit = crunchSplit.getInputSplit();
    delegate.initialize(delegateSplit,
        TaskAttemptContextFactory.create(conf, context.getTaskAttemptID()));
  }

View Full Code Here

    }
    return isParent(possibleParent, child.getParent());
  }


  protected Path getCurrentFile(Context context) throws IOException {
    InputSplit split = context.getInputSplit();
    if (split != null && split instanceof FileSplit) {
      FileSplit inputSplit = (FileSplit) split;
      Path path = inputSplit.getPath();
      FileSystem fileSystem = path.getFileSystem(context.getConfiguration());
      return path.makeQualified(fileSystem.getUri(), fileSystem.getWorkingDirectory());

View Full Code Here

    Assert.assertNotNull(url);
    File data = new File(url.toURI());
    Path tmpFile = new Path(data.getAbsolutePath());
    
    // Setup the Mapper
    InputSplit split = new FileSplit(tmpFile, 0, fs.pathToFile(tmpFile).length(), null);
    AggregatingRecordReader rr = new AggregatingRecordReader();
    Path ocPath = new Path(tmpFile, "oc");
    OutputCommitter oc = new FileOutputCommitter(ocPath, context);
    fs.deleteOnExit(ocPath);
    StandaloneStatusReporter sr = new StandaloneStatusReporter();

View Full Code Here

    
    // to compute firstIds, process the splits in file order
    long slowest = 0; // duration of slowest map
    int firstId = 0;
    for (int p = 0; p < nbSplits; p++) {
      InputSplit split = splits.get(p);
      int hp = ArrayUtils.indexOf(sorted, split); // hadoop's partition
      
      RecordReader<LongWritable, Text> reader = input.createRecordReader(split, task);
      reader.initialize(split, task);

View Full Code Here

    secondOutput = new MockContext(new Step2Mapper(), conf, task.getTaskAttemptID(), numTrees);
    long slowest = 0; // duration of slowest map


    for (int partition = 0; partition < nbSplits; partition++) {
      
      InputSplit split = sorted[partition];
      RecordReader<LongWritable, Text> reader = input.createRecordReader(split, task);


      // load the output of the 1st step
      int nbConcerned = Step2Mapper.nbConcerned(nbSplits, numTrees, partition);
      TreeID[] fsKeys = new TreeID[nbConcerned];

View Full Code Here


    Step0Context context = new Step0Context(new Step0Mapper(), job.getConfiguration(),
                                            new TaskAttemptID(), NUM_MAPS);


    for (int p = 0; p < NUM_MAPS; p++) {
      InputSplit split = sorted[p];


      RecordReader<LongWritable, Text> reader = input.createRecordReader(split,
                                                                         context);
      reader.initialize(split, context);

View Full Code Here


    TaskAttemptContext context = new TaskAttemptContext(job.getConfiguration(),
        new TaskAttemptID());


    for (int p = 0; p < NUM_MAPS; p++) {
      InputSplit split = sorted[p];
      RecordReader<LongWritable, Text> reader = input.createRecordReader(split,
          context);
      reader.initialize(split, context);


      Long firstKey = null;

View Full Code Here


        String[] inpSlitsToRead = args[1].split(",");
        List<InputSplit> splits = cntxt.getSplits();


        for (int i = 0; i < inpSlitsToRead.length; i++) {
            InputSplit split = splits.get(Integer.parseInt(inpSlitsToRead[i]));
            HCatReader reader = DataTransferFactory.getHCatReader(split, cntxt.getConf());
            Iterator<HCatRecord> itr = reader.read();
            File f = new File(args[2] + "-" + i);
            f.delete();
            BufferedWriter outFile = new BufferedWriter(new FileWriter(f));

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.mapreduce.InputSplit

bulkimport.BulkImportJobExample$VerboseInputSampler$VerboseRandomSampler

co.gridport.kafka.hadoop.KafkaInputFormat

co.nubetech.hiho.merge.MergeKeyMapper

co.nubetech.hiho.merge.MergeValueMapper

com.alimama.mdrill.index.IndexMapper

com.asakusafw.runtime.stage.input.DefaultSplitCombinerTest

com.asakusafw.runtime.stage.input.StageInputSplit

com.asakusafw.testdriver.file.FileInputFormatDriver

com.facebook.giraph.hive.impl.input.benchmark.InputBenchmark

com.facebook.giraph.hive.input.benchmark.InputBenchmark

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.