Examples of org.apache.hadoop.mapreduce.InputSplit

org.apache.hadoop.mapreduce.InputSplit
InputSplit represents the data to be processed by an individual {@link Mapper}.
Typically, it presents a byte-oriented view on the input and is the responsibility of {@link RecordReader} of the job to process this and presenta record-oriented view. @see InputFormat @see RecordReader

  }


  private static void readSplits(Context context)
  {
    while (context.hasMoreSplitsToRead()) {
      InputSplit split = context.splitsQueue.poll();
      try {
        readSplit(split, context);
      } catch (Exception e) {
        LOG.error("Failed to read split " + split, e);
      }

View Full Code Here

            int numPaths = pigSplit.getNumPaths();
            processError("\nPigSplit contains " + numPaths + " wrappedSplits.");


            StringBuilder sb = new StringBuilder();
            for(int i = 0; i < numPaths; i++) {
              InputSplit wrappedSplit = pigSplit.getWrappedSplit(i);
              if (wrappedSplit instanceof FileSplit) {
                  FileSplit mapInputFileSplit = (FileSplit)wrappedSplit;
                  sb.append("\nInput-split: file=");
                  sb.append(mapInputFileSplit.getPath());
                  sb.append(" start-offset=");

View Full Code Here

            return false;
        }
        if(reader != null){
            reader.close();
        }
        InputSplit curSplit = inpSplits.get(curSplitIndex);
        TaskAttemptContext tAContext = HadoopShims.createTaskAttemptContext(conf, 
                new TaskAttemptID());
        reader = inputFormat.createRecordReader(curSplit, tAContext);
        reader.initialize(curSplit, tAContext);
        // create a dummy pigsplit - other than the actual split, the other

View Full Code Here

    checkAgainstLegacy(splits, legacySplits);


    int doubleCheckedTotalSize = 0;
    int previousSplitSize = -1;
    for (int i=0; i<splits.size(); ++i) {
      InputSplit split = splits.get(i);
      int currentSplitSize = 0;
      RecordReader<Text, FileStatus> recordReader = uniformSizeInputFormat.createRecordReader(
              split, null);
      StubContext stubContext = new StubContext(jobContext.getConfiguration(),
                                                recordReader, 0);

View Full Code Here

  public void testGetSplits() throws IOException, InterruptedException {
    List<InputSplit> splits = getInputSplits();


    Assert.assertTrue(splits.size() > 0);


    InputSplit split = splits.get(0);
    PartitionQuery query = ((GoraInputSplit)split).getQuery();
    Assert.assertTrue(Arrays.equals(Employee._ALL_FIELDS, query.getFields()));
  }

View Full Code Here

    JobContext jobContext = mock(JobContext.class);
    when(jobContext.getConfiguration()).thenReturn(conf);
    List<InputSplit> splits = fif.getSplits(jobContext);
    assertEquals(8, splits.size());
    for (int i = 0 ; i < splits.size() ; i++) {
      InputSplit split = splits.get(i);
      assertEquals(("host" + i), split.getLocations()[0]);
    }
  }

View Full Code Here

    JobContext jobContext = mock(JobContext.class);
    when(jobContext.getConfiguration()).thenReturn(conf);
    List<InputSplit> splits = fif.getSplits(jobContext);
    assertEquals(8, splits.size());
    for (int i = 0; i < splits.size(); i++) {
      InputSplit split = splits.get(i);
      assertEquals(("host" + i), split.getLocations()[0]);
    }
  }

View Full Code Here

    JobContext jobContext = mock(JobContext.class);
    when(jobContext.getConfiguration()).thenReturn(conf);
    List<InputSplit> splits = fif.getSplits(jobContext);
    assertEquals(1, splits.size());
    for (int i = 0; i < splits.size(); i++) {
      InputSplit split = splits.get(i);
      assertEquals(("host" + i), split.getLocations()[0]);
    }
  }

View Full Code Here

        GenerateDistCacheData.GRIDMIX_DISTCACHE_FILE_LIST));
    FileStatus stat = fs.getFileStatus(filesListFile);
    assertEquals("Wrong permissions of dist Cache files list file "
        + filesListFile, new FsPermission((short)0644), stat.getPermission());


    InputSplit split =
        new FileSplit(filesListFile, 0, stat.getLen(), (String[])null);
    TaskAttemptContext taskContext =
        MapReduceTestUtil.createDummyMapTaskAttemptContext(jobConf);
    RecordReader<LongWritable, BytesWritable> reader =
      new GenerateDistCacheData.GenDCDataFormat().createRecordReader(

View Full Code Here

    }
  }
  
  InputSplit readWrappedSplit(DataInput in, Class<? extends InputSplit> clazz) {
    try {
      InputSplit split = clazz.newInstance();
      if (split instanceof Writable) {
        ((Writable) split).readFields(in);
        return split;
      } else {
        throw new TezUncheckedException(
            split.getClass().getName() + " is not a Writable");          
      }
    } catch (Exception e) {
      throw new TezUncheckedException(e);
    }
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.mapreduce.InputSplit

bulkimport.BulkImportJobExample$VerboseInputSampler$VerboseRandomSampler

co.gridport.kafka.hadoop.KafkaInputFormat

co.nubetech.hiho.merge.MergeKeyMapper

co.nubetech.hiho.merge.MergeValueMapper

com.alimama.mdrill.index.IndexMapper

com.asakusafw.runtime.stage.input.DefaultSplitCombinerTest

com.asakusafw.runtime.stage.input.StageInputSplit

com.asakusafw.testdriver.file.FileInputFormatDriver

com.facebook.giraph.hive.impl.input.benchmark.InputBenchmark

com.facebook.giraph.hive.input.benchmark.InputBenchmark

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.