Examples of org.apache.hcatalog.mapreduce.HCatSplit

org.apache.hcatalog.mapreduce.HCatSplit
The HCatSplit wrapper around the InputSplit returned by the underlying InputFormat


  public SqoopHCatInputSplit(List<InputSplit> splits) {
    hCatSplits = new ArrayList<HCatSplit>();
    Set<String> locations = new HashSet<String>();
    for (int i = 0; i < splits.size(); ++i) {
      HCatSplit hsSplit = (HCatSplit) splits.get(i);
      hCatSplits.add(hsSplit);
      this.inputLength += hsSplit.getLength();
      locations.addAll(Arrays.asList(hsSplit.getLocations()));
    }
    this.hCatLocations = locations.toArray(new String[0]);
  }

View Full Code Here

  public void readFields(DataInput in) throws IOException {
    this.inputLength = in.readLong();
    int size = in.readInt();
    this.hCatSplits = new ArrayList<HCatSplit>(size);
    for (int i = 0; i < size; ++i) {
      HCatSplit hs = new HCatSplit();
      hs.readFields(in);
      hCatSplits.add(hs);
    }
  }

View Full Code Here

    public void readExternal(ObjectInput in) throws IOException,
        ClassNotFoundException {
        conf.readFields(in);
        int numOfSplits = in.readInt();
        for (int i = 0; i < numOfSplits; i++) {
            HCatSplit split = new HCatSplit();
            split.readFields(in);
            splits.add(split);
        }
    }

View Full Code Here

          conf.getInt(HCatConstants.HCAT_DESIRED_PARTITION_NUM_SPLITS, 0);
      org.apache.hadoop.mapred.InputSplit[] baseSplits =
          inputFormat.getSplits(jobConf, desiredNumSplits);


      for (org.apache.hadoop.mapred.InputSplit split : baseSplits) {
        splits.add(new HCatSplit(partitionInfo, split, allCols));
      }
    }


    return splits;
  }

View Full Code Here

  private RecordReader<WritableComparable, HCatRecord>
  createRecordReader(InputSplit split,
                     HCatSchema schema,
                     TaskAttemptContext taskContext)
    throws IOException, InterruptedException {
    HCatSplit hcatSplit = HCatUtils.castToHCatSplit(split);
    PartInfo partitionInfo = hcatSplit.getPartitionInfo();
    JobContext jobContext = taskContext;
    Configuration conf = jobContext.getConfiguration();


    HCatStorageHandler storageHandler = HCatUtil.getStorageHandler(
        conf, partitionInfo);

View Full Code Here

  public void readExternal(ObjectInput in) throws IOException,
      ClassNotFoundException {
    conf.readFields(in);
    int numOfSplits = in.readInt();
    for (int i=0 ; i < numOfSplits; i++) {
      HCatSplit split = new HCatSplit();
      split.readFields(in);
      splits.add(split);
    }
  }

View Full Code Here

  public void readExternal(ObjectInput in) throws IOException,
    ClassNotFoundException {
    conf.readFields(in);
    int numOfSplits = in.readInt();
    for (int i = 0; i < numOfSplits; i++) {
      HCatSplit split = new HCatSplit();
      split.readFields(in);
      splits.add(split);
    }
  }

View Full Code Here

TOP

Related Classes of org.apache.hcatalog.mapreduce.HCatSplit

org.apache.giraph.io.hcatalog.GiraphHCatInputFormat

org.apache.hadoop.io.Writable

org.apache.hadoop.mapreduce.InputSplit

org.apache.hcatalog.data.transfer.ReaderContext

org.apache.sqoop.mapreduce.hcat.SqoopHCatInputSplit

java.io.IOException

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.