Examples of org.apache.hadoop.mapred.InputSplit

org.apache.hadoop.mapred.InputSplit
The description of the data for a single map task. @author Owen O'Malley

  public RecordReader getRecordReader(InputSplit split, JobConf job,
      Reporter reporter) throws IOException {


    HiveInputSplit hsplit = (HiveInputSplit) split;


    InputSplit inputSplit = hsplit.getInputSplit();
    String inputFormatClassName = null;
    Class inputFormatClass = null;
    try {
      inputFormatClassName = hsplit.inputFormatClassName();
      inputFormatClass = job.getClassByName(inputFormatClassName);

View Full Code Here

      } catch (HiveException e) {
        throw new RuntimeException(
            "Unable to get metadata for input table split" + split.getPath());
      }
    }
    InputSplit retA[] = newSplits.toArray((new FileSplit[newSplits.size()]));
    l4j.info("Number of input splits: " + splits.length + " new input splits: "
        + retA.length + ", sum of split lengths: " + sumSplitLengths);
    return retA;
  }

View Full Code Here

    return serialized;
  }


  private FileSplit getFileSplitFromEvent(RootInputDataInformationEvent event)
      throws IOException {
    InputSplit inputSplit = null;
    if (event.getDeserializedUserPayload() != null) {
      inputSplit = (InputSplit) event.getDeserializedUserPayload();
    } else {
      MRSplitProto splitProto = MRSplitProto.parseFrom(event.getUserPayload());
      SerializationFactory serializationFactory = new SerializationFactory(

View Full Code Here

    return serialized;
  }


  private FileSplit getFileSplitFromEvent(RootInputDataInformationEvent event)
      throws IOException {
    InputSplit inputSplit = null;
    if (event.getDeserializedUserPayload() != null) {
      inputSplit = (InputSplit) event.getDeserializedUserPayload();
    } else {
      MRSplitProto splitProto = MRSplitProto.parseFrom(event.getUserPayload());
      SerializationFactory serializationFactory = new SerializationFactory(

View Full Code Here

          boolean statsAvailable = false;
          for(FileStatus file: fileList) {
            if (!file.isDir()) {
              InputFormat<?, ?> inputFormat = (InputFormat<?, ?>) ReflectionUtils.newInstance(
                  table.getInputFormatClass(), jc);
              InputSplit dummySplit = new FileSplit(file.getPath(), 0, 0, new String[] { table
                  .getDataLocation().toString() });
              org.apache.hadoop.mapred.RecordReader<?, ?> recordReader = (org.apache.hadoop.mapred.RecordReader<?, ?>) inputFormat
                  .getRecordReader(dummySplit, jc, Reporter.NULL);
              StatsProvidingRecordReader statsRR;
              if (recordReader instanceof StatsProvidingRecordReader) {

View Full Code Here

        boolean statsAvailable = false;
        for(FileStatus file: fileList) {
          if (!file.isDir()) {
            InputFormat<?, ?> inputFormat = (InputFormat<?, ?>) ReflectionUtils.newInstance(
                partn.getInputFormatClass(), jc);
            InputSplit dummySplit = new FileSplit(file.getPath(), 0, 0,
                new String[] { partn.getLocation() });
            Object recordReader = inputFormat.getRecordReader(dummySplit, jc, Reporter.NULL);
            StatsProvidingRecordReader statsRR;
            if (recordReader instanceof StatsProvidingRecordReader) {
              statsRR = (StatsProvidingRecordReader) recordReader;

View Full Code Here

  public RecordReader getRecordReader(InputSplit split, JobConf job,
      Reporter reporter) throws IOException {


    HiveInputSplit hsplit = (HiveInputSplit) split;


    InputSplit inputSplit = hsplit.getInputSplit();
    String inputFormatClassName = null;
    Class inputFormatClass = null;
    try {
      inputFormatClassName = hsplit.inputFormatClassName();
      inputFormatClass = job.getClassByName(inputFormatClassName);

View Full Code Here


public class TestMockReporter extends TestCase {


  @Test
  public void testGetInputSplitForMapper() {
    InputSplit split = new MockReporter(MockReporter.ReporterType.Mapper, null).getInputSplit();
    assertTrue(null != split);
  }

View Full Code Here

    Constructor<?> constructor = Class.forName(className).getDeclaredConstructor();
    if (constructor == null) {
      throw new ReflectiveOperationException("Class " + className + " does not implement a default constructor.");
    }
    constructor.setAccessible(true);
    InputSplit split = (InputSplit) constructor.newInstance();
    ByteArrayDataInput byteArrayDataInput = ByteStreams.newDataInput(Base64.decodeBase64(base64));
    split.readFields(byteArrayDataInput);
    return split;
  }

View Full Code Here

  public RecordReader getRecordReader(InputSplit split, JobConf job,
      Reporter reporter) throws IOException {


    HiveInputSplit hsplit = (HiveInputSplit) split;


    InputSplit inputSplit = hsplit.getInputSplit();
    String inputFormatClassName = null;
    Class inputFormatClass = null;
    try {
      inputFormatClassName = hsplit.inputFormatClassName();
      inputFormatClass = job.getClassByName(inputFormatClassName);

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.mapred.InputSplit

com.ebay.erl.mobius.core.mapred.MobiusInputSampler

com.sap.hadoop.windowing.io.TableWindowingInput

com.taobao.zeus.jobs.sub.tool.DataPreviewJob

org.apache.accumulo.core.client.mapred.AccumuloInputFormatTest

org.apache.drill.exec.store.hive.HiveSubScan

org.apache.hadoop.contrib.mongoreduce.MongoStreamInputFormat

org.apache.hadoop.hbase.mapred.TestTableSnapshotInputFormat

org.apache.hadoop.hive.accumulo.mr.TestHiveAccumuloTableInputFormat

org.apache.hadoop.hive.ql.exec.StatsNoJobTask

org.apache.hadoop.hive.ql.exec.StatsNoJobTask$StatsCollection

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.