Examples of org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector

Package org.apache.hadoop.hive.ql.exec.vector

Examples of org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector

org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector
This class supports string and binary data by value reference -- i.e. each field is explicitly present, as opposed to provided by a dictionary reference. In some cases, all the values will be in the same byte array to begin with, but this need not be the case. If each value is in a separate byte array to start with, or not all of the values are in the same original byte array, you can still assign data by reference into this column vector. This gives flexibility to use this in multiple situations.
When setting data by reference, the caller is responsible for allocating the byte arrays used to hold the data. You can also set data by value, as long as you call the initBuffer() method first. You can mix "by value" and "by reference" in the same column vector, though that use is probably not typical.

    }


    VectorExpression udf = new CastStringToDate(0, 1);
    udf.setInputTypes(VectorExpression.Type.STRING);
    VectorizedRowBatch batch = new VectorizedRowBatch(2, 1);
    batch.cols[0] = new BytesColumnVector(1);
    batch.cols[1] = new LongColumnVector(1);
    BytesColumnVector bcv = (BytesColumnVector) batch.cols[0];
    byte[] bytes = new byte[0];
    try {
      bytes = "error".getBytes("UTF-8");
    } catch (UnsupportedEncodingException e) {
    }

View Full Code Here

  @Test
  public void testFilterStringBetween() {
    int seed = 17;
    VectorizedRowBatch vrb = VectorizedRowGroupGenUtil.getVectorizedRowBatch(
        3, 2, seed);
    vrb.cols[0] = new BytesColumnVector();
    BytesColumnVector bcv = (BytesColumnVector) vrb.cols[0];


    bcv.initBuffer();
    bcv.setVal(0, a, 0, 1);
    bcv.setVal(1, b, 0, 1);
    bcv.setVal(2, c, 0, 1);


    VectorExpression expr = new FilterStringColumnBetween(0, b, c);


    // basic test
    expr.evaluate(vrb);


    assertEquals(2, vrb.size);
    assertTrue(vrb.selectedInUse);
    assertEquals(1, vrb.selected[0]);
    assertEquals(2, vrb.selected[1]);


    // nulls
    vrb.selectedInUse = false;
    vrb.size = 3;
    bcv.noNulls = false;
    bcv.isNull[2] = true;
    expr.evaluate(vrb);
    assertEquals(1, vrb.size);
    assertEquals(1, vrb.selected[0]);
    assertTrue(vrb.selectedInUse);


    // repeating
    vrb.selectedInUse = false;
    vrb.size = 3;
    bcv.noNulls = true;
    bcv.isRepeating = true;
    expr.evaluate(vrb);
    assertEquals(0, vrb.size);


    // nulls and repeating
    vrb.selectedInUse = false;
    vrb.size = 3;
    bcv.noNulls = false;
    bcv.isRepeating = true;
    bcv.isNull[0] = true;
    bcv.setVal(0, b, 0, 1);
    expr.evaluate(vrb);
    assertEquals(0, vrb.size);
  }

View Full Code Here


  @Test
  // Load a BytesColumnVector by copying in large data, enough to force
  // the buffer to expand.
  public void testLoadBytesColumnVectorByValueLargeData()  {
    BytesColumnVector bcv = new BytesColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
    bcv.initBuffer(10); // initialize with estimated element size 10
    String s = "0123456789";
    while (s.length() < 500) {
      s += s;
    }
    byte[] b = null;
    try {
      b = s.getBytes("UTF-8");
    } catch (UnsupportedEncodingException e) {
      e.printStackTrace();
    }
    for (int i = 0; i != VectorizedRowBatch.DEFAULT_SIZE; i++) {
      bcv.setVal(i, b, 0, b.length);
    }
    Assert.assertTrue(bcv.bufferSize() >= b.length * VectorizedRowBatch.DEFAULT_SIZE);
  }

View Full Code Here

  }


  @Test
  // set values by reference, copy the data out, and verify equality
  public void testLoadBytesColumnVectorByRef() {
    BytesColumnVector bcv = new BytesColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
    String s = "red";
    byte[] b = null;
    try {
      b = s.getBytes("UTF-8");
    } catch (UnsupportedEncodingException e) {
      e.printStackTrace();
    }
    for (int i = 0; i != VectorizedRowBatch.DEFAULT_SIZE; i++) {
      bcv.setRef(i, b, 0, b.length);
    }
    // verify
    byte[] v = new byte[b.length];
    for (int i = 0; i != VectorizedRowBatch.DEFAULT_SIZE; i++) {
      Assert.assertTrue(bcv.length[i] == b.length);

View Full Code Here

  }


  VectorizedRowBatch makeStringBatch() {
    // create a batch with one string ("Bytes") column
    VectorizedRowBatch batch = new VectorizedRowBatch(3);
    BytesColumnVector v = new BytesColumnVector();
    batch.cols[0] = v;
    batch.cols[1] = new BytesColumnVector();          // to hold output if needed
    batch.cols[2] = new LongColumnVector(batch.size); // to hold boolean output
    /*
     * Add these 3 values:
     *
     * red
     * green
     * NULL
     */
    v.setRef(0, red, 0, red.length);
    v.isNull[0] = false;
    v.setRef(1, green, 0, green.length);
    v.isNull[1] = false;
    v.setRef(2,  emptyString,  0,  emptyString.length);
    v.isNull[2] = true;


    v.noNulls = false;


    batch.size = 3;

View Full Code Here

  }


  VectorizedRowBatch makeStringBatchMixedCase() {
    // create a batch with two string ("Bytes") columns
    VectorizedRowBatch batch = new VectorizedRowBatch(2, VectorizedRowBatch.DEFAULT_SIZE);
    BytesColumnVector v = new BytesColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
    batch.cols[0] = v;
    BytesColumnVector outV = new BytesColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
    outV.initBuffer();
    batch.cols[1] = outV;
    /*
     * Add these 3 values:
     *
     * mixedUp

View Full Code Here


  VectorizedRowBatch makeStringBatchMixedCharSize() {


    // create a new batch with one char column (for input) and one long column (for output)
    VectorizedRowBatch batch = new VectorizedRowBatch(2, VectorizedRowBatch.DEFAULT_SIZE);
    BytesColumnVector v = new BytesColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
    batch.cols[0] = v;
    LongColumnVector outV = new LongColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
    batch.cols[1] = outV;


    /*
     * Add these 3 values:
     *
     * mixedUp
     * green
     * NULL
     * <4 char string with mult-byte chars>
     */
    v.setRef(0, mixedUp, 0, mixedUp.length);
    v.isNull[0] = false;
    v.setRef(1, green, 0, green.length);
    v.isNull[1] = false;
    v.setRef(2,  emptyString,  0,  emptyString.length);
    v.isNull[2] = true;
    v.noNulls = false;
    v.setRef(3, multiByte, 0, 10);
    v.isNull[3] = false;


    batch.size = 4;
    return batch;
  }

View Full Code Here

  public void testColLower() {
    // has nulls, not repeating
    VectorizedRowBatch batch = makeStringBatchMixedCase();
    StringLower expr = new StringLower(0, 1);
    expr.evaluate(batch);
    BytesColumnVector outCol = (BytesColumnVector) batch.cols[1];
    int cmp = StringExpr.compare(mixedUpLower, 0, mixedUpLower.length, outCol.vector[0],
        outCol.start[0], outCol.length[0]);
    Assert.assertEquals(0, cmp);
    Assert.assertTrue(outCol.isNull[2]);
    int cmp2 = StringExpr.compare(green, 0, green.length, outCol.vector[1],

View Full Code Here

     */
    VectorizedRowBatch batch = makeStringBatchMixedCase();
    StringUpper expr = new StringUpper(0, 1);
    batch.cols[0].noNulls = true;
    expr.evaluate(batch);
    BytesColumnVector outCol = (BytesColumnVector) batch.cols[1];
    int cmp = StringExpr.compare(mixedUpUpper, 0, mixedUpUpper.length, outCol.vector[0],
        outCol.start[0], outCol.length[0]);
    Assert.assertEquals(0, cmp);
    Assert.assertTrue(outCol.noNulls);
  }

View Full Code Here

    Assert.assertTrue(outCol.noNulls);
  }


  private VectorizedRowBatch makeStringBatch2In1Out() {
    VectorizedRowBatch batch = new VectorizedRowBatch(3);
    BytesColumnVector v = new BytesColumnVector();
    batch.cols[0] = v;
    BytesColumnVector v2 = new BytesColumnVector();
    batch.cols[1] = v2;
    batch.cols[2] = new BytesColumnVector();


    v.setRef(0, red, 0, red.length);
    v.isNull[0] = false;
    v.setRef(1, green, 0, green.length);
    v.isNull[1] = false;
    v.setRef(2,  emptyString,  0,  emptyString.length);
    v.isNull[2] = true;
    v.noNulls = false;


    v2.setRef(0, red, 0, red.length);
    v2.isNull[0] = false;
    v2.setRef(1, green, 0, green.length);
    v2.isNull[1] = false;
    v2.setRef(2,  emptyString,  0,  emptyString.length);
    v2.isNull[2] = true;
    v2.noNulls = false;


    batch.size = 3;
    return batch;

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector

org.apache.hadoop.hive.ql.exec.vector.expressions.AbstractFilterStringColLikeStringScalar

org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.gen.VectorUDAFMaxString

org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.gen.VectorUDAFMinString

org.apache.hadoop.hive.ql.exec.vector.expressions.CastStringToDate

org.apache.hadoop.hive.ql.exec.vector.expressions.CastStringToDecimal

org.apache.hadoop.hive.ql.exec.vector.expressions.DecimalToStringUnaryUDF

org.apache.hadoop.hive.ql.exec.vector.expressions.FilterStringColumnInList

org.apache.hadoop.hive.ql.exec.vector.expressions.FuncLongToString

org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterStringColEqualStringColumn

org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterStringColEqualStringScalar

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.