Examples of org.apache.hadoop.zebra.io.KeyDistribution

org.apache.hadoop.zebra.io.KeyDistribution
Class used to convey the information of how on-disk data are distributed among key-partitioned buckets. This class is used by the MapReduce layer to calculate intelligent splits.

  void keySplitCG(int numSplits, int totalRows, String strProjection, Path path)
      throws IOException, ParseException {
    ColumnGroup.Reader reader = new ColumnGroup.Reader(path, conf);
    reader.setProjection(strProjection);
    long totalBytes = reader.getStatus().getSize();
    KeyDistribution keyDistri = reader.getKeyDistribution(numSplits * 10);
    Assert.assertEquals(totalBytes, keyDistri.length());
    reader.close();
    BytesWritable[] keys = null;
    if (keyDistri.size() >= numSplits) {
      keyDistri.resize(numSplits);
      Assert.assertEquals(totalBytes, keyDistri.length());
      RawComparable[] rawComparables = keyDistri.getKeys();
      keys = new BytesWritable[rawComparables.length];
      for (int i = 0; i < keys.length; ++i) {
        keys[i] = new BytesWritable();
        keys[i].setSize(rawComparables[i].size());
        System.arraycopy(rawComparables[i].buffer(),

View Full Code Here

    }
    
    // TODO: Does it make sense to interleave keys for all leaf tables if
    // numSplits <= 0 ?
    int nLeaves = readers.size();
    KeyDistribution keyDistri = null;
    for (int i = 0; i < nLeaves; ++i) {
      KeyDistribution btKeyDistri =
          readers.get(i).getKeyDistribution(
              (numSplits <= 0) ? -1 :
              Math.max(numSplits * 5 / nLeaves, numSplits));
      keyDistri = KeyDistribution.sum(keyDistri, btKeyDistri);
    }

View Full Code Here

      Path path) throws IOException, ParseException {
    BasicTable.Reader reader = new BasicTable.Reader(path, conf);
    reader.setProjection(strProjection);
    long totalBytes = reader.getStatus().getSize();
    BlockDistribution lastBd = new BlockDistribution();
    KeyDistribution keyDistri = reader.getKeyDistribution(numSplits * 10, 1, lastBd);
    Assert.assertEquals(totalBytes, keyDistri.length()+lastBd.getLength());
    reader.close();
    BytesWritable[] keys = null;
    if (keyDistri.size() >= numSplits) {
      keyDistri.resize(lastBd);
      Assert.assertEquals(totalBytes, keyDistri.length()+lastBd.getLength());
      RawComparable[] rawComparables = keyDistri.getKeys();
      keys = new BytesWritable[rawComparables.length];
      for (int i = 0; i < keys.length; ++i) {
        keys[i] = new BytesWritable();
        keys[i].setSize(rawComparables[i].size());
        System.arraycopy(rawComparables[i].buffer(),

View Full Code Here

      throws IOException, ParseException {
    ColumnGroup.Reader reader = new ColumnGroup.Reader(path, conf);
    reader.setProjection(strProjection);
    long totalBytes = reader.getStatus().getSize();
    BlockDistribution lastBd = new BlockDistribution();
    KeyDistribution keyDistri = reader.getKeyDistribution(numSplits * 10, 1, lastBd);
    Assert.assertEquals(totalBytes, keyDistri.length()+lastBd.getLength());
    reader.close();
    BytesWritable[] keys = null;
    if (keyDistri.size() >= numSplits) {
      keyDistri.resize(lastBd);
      Assert.assertEquals(totalBytes, keyDistri.length()+lastBd.getLength());
      RawComparable[] rawComparables = keyDistri.getKeys();
      keys = new BytesWritable[rawComparables.length];
      for (int i = 0; i < keys.length; ++i) {
        keys[i] = new BytesWritable();
        keys[i].setSize(rawComparables[i].size());
        System.arraycopy(rawComparables[i].buffer(),

View Full Code Here

      throws IOException, ParseException {
    ColumnGroup.Reader reader = new ColumnGroup.Reader(path, conf);
    reader.setProjection(strProjection);
    long totalBytes = reader.getStatus().getSize();
    BlockDistribution lastBd = new BlockDistribution();
    KeyDistribution keyDistri = reader.getKeyDistribution(numSplits * 10, 1, lastBd);
    Assert.assertEquals(totalBytes, keyDistri.length()+lastBd.getLength());
    reader.close();
    BytesWritable[] keys = null;
    if (keyDistri.size() >= numSplits) {
      keyDistri.resize(lastBd);
      Assert.assertEquals(totalBytes, keyDistri.length()+lastBd.getLength());
      RawComparable[] rawComparables = keyDistri.getKeys();
      keys = new BytesWritable[rawComparables.length];
      for (int i = 0; i < keys.length; ++i) {
        keys[i] = new BytesWritable();
        keys[i].setSize(rawComparables[i].size());
        System.arraycopy(rawComparables[i].buffer(),

View Full Code Here

    // numSplits <= 0 ?
    int nLeaves = readers.size();
    BlockDistribution lastBd = new BlockDistribution();
    ArrayList<KeyDistribution> btKeyDistributions = new ArrayList<KeyDistribution>();
    for (int i = 0; i < nLeaves; ++i) {
      KeyDistribution btKeyDistri =
          readers.get(i).getKeyDistribution(
              (numSplits <= 0) ? -1 :
              Math.max(numSplits * 5 / nLeaves, numSplits), nLeaves, lastBd);
      btKeyDistributions.add(btKeyDistri);
    }
    int btSize = btKeyDistributions.size();
    KeyDistribution[] btKds = new KeyDistribution[btSize];
    Object[] btArray = btKeyDistributions.toArray();
    for (int i = 0; i < btSize; i++)
      btKds[i] = (KeyDistribution) btArray[i];
    
    KeyDistribution keyDistri = KeyDistribution.merge(btKds);


    if (keyDistri == null) {
      // should never happen.
      SortedTableSplit split = new SortedTableSplit(null, null, null, conf);
      return new InputSplit[] { split };
    }
    
    keyDistri.resize(lastBd);
    
    RawComparable[] keys = keyDistri.getKeys();
    for (int i = 0; i <= keys.length; ++i) {
      RawComparable begin = (i == 0) ? null : keys[i - 1];
      RawComparable end = (i == keys.length) ? null : keys[i];
      BlockDistribution bd;
      if (i < keys.length)
        bd = keyDistri.getBlockDistribution(keys[i]);
      else
        bd = lastBd;
      BytesWritable beginB = null, endB = null;
      if (begin != null)
        beginB = new BytesWritable(begin.buffer());

View Full Code Here

      Path path) throws IOException, ParseException {
    BasicTable.Reader reader = new BasicTable.Reader(path, conf);
    reader.setProjection(strProjection);
    long totalBytes = reader.getStatus().getSize();
    BlockDistribution lastBd = new BlockDistribution();
    KeyDistribution keyDistri = reader.getKeyDistribution(numSplits * 10, 1, lastBd);
    Assert.assertEquals(totalBytes, keyDistri.length()+lastBd.getLength());
    reader.close();
    BytesWritable[] keys = null;
    if (keyDistri.size() >= numSplits) {
      keyDistri.resize(lastBd);
      Assert.assertEquals(totalBytes, keyDistri.length()+lastBd.getLength());
      RawComparable[] rawComparables = keyDistri.getKeys();
      keys = new BytesWritable[rawComparables.length];
      for (int i = 0; i < keys.length; ++i) {
        keys[i] = new BytesWritable();
        keys[i].setSize(rawComparables[i].size());
        System.arraycopy(rawComparables[i].buffer(),

View Full Code Here

      throws IOException, ParseException {
    ColumnGroup.Reader reader = new ColumnGroup.Reader(path, conf);
    reader.setProjection(strProjection);
    long totalBytes = reader.getStatus().getSize();
    BlockDistribution lastBd = new BlockDistribution();
    KeyDistribution keyDistri = reader.getKeyDistribution(numSplits * 10, 1, lastBd);
    Assert.assertEquals(totalBytes, keyDistri.length()+lastBd.getLength());
    reader.close();
    BytesWritable[] keys = null;
    if (keyDistri.size() >= numSplits) {
      keyDistri.resize(lastBd);
      Assert.assertEquals(totalBytes, keyDistri.length()+lastBd.getLength());
      RawComparable[] rawComparables = keyDistri.getKeys();
      keys = new BytesWritable[rawComparables.length];
      for (int i = 0; i < keys.length; ++i) {
        keys[i] = new BytesWritable();
        keys[i].setSize(rawComparables[i].size());
        System.arraycopy(rawComparables[i].buffer(),

View Full Code Here

    // numSplits <= 0 ?
    int nLeaves = readers.size();
    BlockDistribution lastBd = new BlockDistribution();
    ArrayList<KeyDistribution> btKeyDistributions = new ArrayList<KeyDistribution>();
    for (int i = 0; i < nLeaves; ++i) {
      KeyDistribution btKeyDistri =
          readers.get(i).getKeyDistribution(
              (numSplits <= 0) ? -1 :
              Math.max(numSplits * 5 / nLeaves, numSplits), nLeaves, lastBd);
      btKeyDistributions.add(btKeyDistri);
    }
    int btSize = btKeyDistributions.size();
    KeyDistribution[] btKds = new KeyDistribution[btSize];
    Object[] btArray = btKeyDistributions.toArray();
    for (int i = 0; i < btSize; i++)
      btKds[i] = (KeyDistribution) btArray[i];
    
    KeyDistribution keyDistri = KeyDistribution.merge(btKds);


    if (keyDistri == null) {
      // should never happen.
      SortedTableSplit split = new SortedTableSplit(null, null, null, conf);
      return new InputSplit[] { split };
    }
    
    keyDistri.resize(lastBd);
    
    RawComparable[] keys = keyDistri.getKeys();
    for (int i = 0; i <= keys.length; ++i) {
      RawComparable begin = (i == 0) ? null : keys[i - 1];
      RawComparable end = (i == keys.length) ? null : keys[i];
      BlockDistribution bd;
      if (i < keys.length)
        bd = keyDistri.getBlockDistribution(keys[i]);
      else
        bd = lastBd;
      BytesWritable beginB = null, endB = null;
      if (begin != null)
        beginB = new BytesWritable(begin.buffer());

View Full Code Here

    // numSplits <= 0 ?
    int nLeaves = readers.size();
    BlockDistribution lastBd = new BlockDistribution();
    ArrayList<KeyDistribution> btKeyDistributions = new ArrayList<KeyDistribution>();
    for (int i = 0; i < nLeaves; ++i) {
      KeyDistribution btKeyDistri =
          readers.get(i).getKeyDistribution(
              (numSplits <= 0) ? -1 :
              Math.max(numSplits * 5 / nLeaves, numSplits), nLeaves, lastBd);
      btKeyDistributions.add(btKeyDistri);
    }
    
    int btSize = btKeyDistributions.size();
    KeyDistribution[] btKds = new KeyDistribution[btSize];
    Object[] btArray = btKeyDistributions.toArray();
    for (int i = 0; i < btSize; i++)
      btKds[i] = (KeyDistribution) btArray[i];


    KeyDistribution keyDistri = KeyDistribution.merge(btKds);


    if (keyDistri == null) {
      // should never happen.
       return splits;
    }


    keyDistri.resize(lastBd);


    RawComparable[] keys = keyDistri.getKeys();
    for (int i = 0; i <= keys.length; ++i) {
      RawComparable begin = (i == 0) ? null : keys[i - 1];
      RawComparable end = (i == keys.length) ? null : keys[i];
      BlockDistribution bd;
      if (i < keys.length)
        bd = keyDistri.getBlockDistribution(keys[i]);
      else
        bd = lastBd;
      BytesWritable beginB = null, endB = null;
      if (begin != null)
        beginB = new BytesWritable(begin.buffer());

View Full Code Here

0 1

TOP

Related Classes of org.apache.hadoop.zebra.io.KeyDistribution

org.apache.hadoop.io.file.tfile.RawComparable

org.apache.hadoop.zebra.io.TestBasicTable

org.apache.hadoop.zebra.io.TestColumnGroup

org.apache.hadoop.zebra.io.TestColumnGroupWithWorkPath

org.apache.hadoop.zebra.mapred.TableInputFormat

org.apache.hadoop.zebra.mapreduce.TableInputFormat

org.apache.hadoop.zebra.tfile.ByteArray

org.apache.hadoop.zebra.tfile.RawComparable

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.