Examples of org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector

Package org.apache.hadoop.hive.ql.exec.vector

Examples of org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector

org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector
This class supports string and binary data by value reference -- i.e. each field is explicitly present, as opposed to provided by a dictionary reference. In some cases, all the values will be in the same byte array to begin with, but this need not be the case. If each value is in a separate byte array to start with, or not all of the values are in the same original byte array, you can still assign data by reference into this column vector. This gives flexibility to use this in multiple situations.
When setting data by reference, the caller is responsible for allocating the byte arrays used to hold the data. You can also set data by value, as long as you call the initBuffer() method first. You can mix "by value" and "by reference" in the same column vector, though that use is probably not typical.

    return batch;
  }


  private VectorizedRowBatch makeStringBatchForColColCompare() {
    VectorizedRowBatch batch = new VectorizedRowBatch(4);
    BytesColumnVector v = new BytesColumnVector();
    batch.cols[0] = v;
    BytesColumnVector v2 = new BytesColumnVector();
    batch.cols[1] = v2;
    batch.cols[2] = new BytesColumnVector();
    batch.cols[3] = new LongColumnVector();


    v.setRef(0, blue, 0, blue.length);
    v.isNull[0] = false;
    v.setRef(1, green, 0, green.length);
    v.isNull[1] = false;
    v.setRef(2,  red,  0,  red.length);
    v.isNull[2] = false;
    v.setRef(3, emptyString, 0, emptyString.length);
    v.isNull[3] = true;
    v.noNulls = false;


    v2.setRef(0, red, 0, red.length);
    v2.isNull[0] = false;
    v2.setRef(1, green, 0, green.length);
    v2.isNull[1] = false;
    v2.setRef(2,  blue,  0,  blue.length);
    v2.isNull[2] = false;
    v2.setRef(3, red, 0, red.length);
    v2.isNull[3] = false;
    v2.noNulls = false;


    batch.size = 4;
    return batch;

View Full Code Here


    // has nulls, not repeating
    VectorizedRowBatch batch = makeStringBatch();
    StringConcatColScalar expr = new StringConcatColScalar(0, red, 1);
    expr.evaluate(batch);
    BytesColumnVector outCol = (BytesColumnVector) batch.cols[1];


    int cmp = StringExpr.compare(redred, 0, redred.length, outCol.vector[0],
        outCol.start[0], outCol.length[0]);
    Assert.assertEquals(0, cmp);
    Assert.assertTrue(outCol.isNull[2]);

View Full Code Here


    // has nulls, not repeating
    VectorizedRowBatch batch = makeStringBatch();
    StringConcatScalarCol expr = new StringConcatScalarCol(red, 0, 1);
    expr.evaluate(batch);
    BytesColumnVector outCol = (BytesColumnVector) batch.cols[1];


    int cmp = StringExpr.compare(redred, 0, redred.length, outCol.vector[0],
        outCol.start[0], outCol.length[0]);
    Assert.assertEquals(0, cmp);
    Assert.assertTrue(outCol.isNull[2]);

View Full Code Here


    // has nulls, not repeating
    VectorizedRowBatch batch = makeStringBatch2In1Out();
    StringConcatColCol expr = new StringConcatColCol(0, 1, 2);
    expr.evaluate(batch);
    BytesColumnVector outCol = (BytesColumnVector) batch.cols[2];


    int cmp = StringExpr.compare(redred, 0, redred.length, outCol.vector[0],
        outCol.start[0], outCol.length[0]);
    Assert.assertEquals(0, cmp);
    Assert.assertTrue(outCol.isNull[2]);

View Full Code Here


  @Test
  public void testSubstrStart() throws UnsupportedEncodingException {
    // Testing no nulls and no repeating
    VectorizedRowBatch batch = new VectorizedRowBatch(2);
    BytesColumnVector v = new BytesColumnVector();
    batch.cols[0] = v;
    BytesColumnVector outV = new BytesColumnVector();
    batch.cols[1] = outV;
    byte[] data1 = "abcd string".getBytes("UTF-8");
    byte[] data2 = "efgh string".getBytes("UTF-8");
    byte[] data3 = "efgh".getBytes("UTF-8");
    batch.size = 3;
    v.noNulls = true;
    v.setRef(0, data1, 0, data1.length);
    v.isNull[0] = false;
    v.setRef(1, data2, 0, data2.length);
    v.isNull[1] = false;
    v.setRef(2, data3, 0, data3.length);
    v.isNull[2] = false;


    StringSubstrColStart expr = new StringSubstrColStart(0, 6, 1);
    expr.evaluate(batch);
    BytesColumnVector outCol = (BytesColumnVector) batch.cols[1];
    Assert.assertEquals(3, batch.size);
    Assert.assertTrue(outCol.noNulls);
    Assert.assertFalse(outCol.isRepeating);
    byte[] expected = "string".getBytes("UTF-8");
    Assert.assertEquals(0,
    StringExpr.compare(
            expected, 0, expected.length, outCol.vector[0], outCol.start[0], outCol.length[0]
        )
    );


    Assert.assertEquals(0,
    StringExpr.compare(
            expected, 0, expected.length, outCol.vector[1], outCol.start[1], outCol.length[1]
        )
    );


    // This yields empty because starting idx is out of bounds.
    Assert.assertEquals(0,
    StringExpr.compare(
            emptyString, 0, emptyString.length, outCol.vector[2], outCol.start[2], outCol.length[2]
        )
    );


    outCol.noNulls = false;
    outCol.isRepeating = true;


    // Testing negative substring index.
    // Start index -6 should yield the last 6 characters of the string


    expr = new StringSubstrColStart(0, -6, 1);
    expr.evaluate(batch);
    outCol = (BytesColumnVector) batch.cols[1];
    Assert.assertEquals(3, batch.size);
    Assert.assertTrue(outCol.noNulls);
    Assert.assertFalse(outCol.isRepeating);
    Assert.assertEquals(0,
    StringExpr.compare(
            expected, 0, expected.length, outCol.vector[0], outCol.start[0], outCol.length[0]
        )
    );


    Assert.assertEquals(0,
    StringExpr.compare(
            expected, 0, expected.length, outCol.vector[1], outCol.start[1], outCol.length[1]
        )
    );


    Assert.assertEquals(0,
    StringExpr.compare(
            emptyString, 0, emptyString.length, outCol.vector[2], outCol.start[2], outCol.length[2]
        )
    );


    outCol.noNulls = false;
    outCol.isRepeating = true;


    // Testing substring starting from index 1


    expr = new StringSubstrColStart(0, 1, 1);
    expr.evaluate(batch);
    Assert.assertEquals(3, batch.size);
    Assert.assertTrue(outCol.noNulls);
    Assert.assertFalse(outCol.isRepeating);


    Assert.assertEquals(0,
    StringExpr.compare(
            data1, 0, data1.length, outCol.vector[0], outCol.start[0], outCol.length[0]
        )
    );


    Assert.assertEquals(0,
    StringExpr.compare(
            data2, 0, data2.length, outCol.vector[1], outCol.start[1], outCol.length[1]
        )
    );


    Assert.assertEquals(0,
    StringExpr.compare(
            data3, 0, data3.length, outCol.vector[2], outCol.start[2], outCol.length[2]
        )
    );


    outV.noNulls = false;
    outV.isRepeating = true;


    // Testing with nulls


    expr = new StringSubstrColStart(0, 6, 1);
    v.noNulls = false;
    v.isNull[0] = true;
    expr.evaluate(batch);
    Assert.assertEquals(3, batch.size);
    Assert.assertFalse(outV.noNulls);
    Assert.assertTrue(outV.isNull[0]);


    Assert.assertEquals(0,
    StringExpr.compare(
            expected, 0, expected.length, outCol.vector[1], outCol.start[1], outCol.length[1]
        )
    );


    Assert.assertEquals(0,
    StringExpr.compare(
            emptyString, 0, emptyString.length, outCol.vector[2], outCol.start[2], outCol.length[2]
        )
    );


    outCol.noNulls = false;
    outCol.isRepeating = false;


    // Testing with repeating and no nulls


    outV = new BytesColumnVector();
    v = new BytesColumnVector();
    v.isRepeating = true;
    v.noNulls = true;
    v.setRef(0, data1, 0, data1.length);
    batch = new VectorizedRowBatch(2);
    batch.cols[0] = v;
    batch.cols[1] = outV;
    expr.evaluate(batch);
    outCol = (BytesColumnVector) batch.cols[1];
    expected = "string".getBytes("UTF-8");
    Assert.assertTrue(outV.isRepeating);
    Assert.assertTrue(outV.noNulls);
    Assert.assertEquals(0,
    StringExpr.compare(
            expected, 0, expected.length, outCol.vector[0], outCol.start[0], outCol.length[0]
        )
    );


    // Testing multiByte string substring


    v = new BytesColumnVector();
    v.isRepeating = false;
    v.noNulls = true;
    v.setRef(0, multiByte, 0, 10);
    batch.cols[0] = v;
    batch.cols[1] = outV;
    outV.isRepeating = true;
    outV.noNulls = false;
    expr = new StringSubstrColStart(0, 3, 1);
    batch.size = 1;
    expr.evaluate(batch);
    outCol = (BytesColumnVector) batch.cols[1];
    Assert.assertFalse(outV.isRepeating);
    Assert.assertTrue(outV.noNulls);
    Assert.assertEquals(0,
    StringExpr.compare(
            // 3nd char starts from index 3 and total length should be 7 bytes as max is 10
            multiByte, 3, 10 - 3, outCol.vector[0], outCol.start[0], outCol.length[0]
        )
    );




    // Testing multiByte string with reference starting mid array


    v = new BytesColumnVector();
    v.isRepeating = false;
    v.noNulls = true;


    // string is 2 chars long (a 3 byte and a 4 byte char)
    v.setRef(0, multiByte, 3, 7);

View Full Code Here

  @Test
  public void testSubstrStartLen() throws UnsupportedEncodingException {
    // Testing no nulls and no repeating


    VectorizedRowBatch batch = new VectorizedRowBatch(2);
    BytesColumnVector v = new BytesColumnVector();
    batch.cols[0] = v;
    BytesColumnVector outV = new BytesColumnVector();
    batch.cols[1] = outV;
    byte[] data1 = "abcd string".getBytes("UTF-8");
    byte[] data2 = "efgh string".getBytes("UTF-8");
    byte[] data3 = "efgh".getBytes("UTF-8");
    batch.size = 3;
    v.noNulls = true;
    v.setRef(0, data1, 0, data1.length);
    v.isNull[0] = false;
    v.setRef(1, data2, 0, data2.length);
    v.isNull[1] = false;
    v.setRef(2, data3, 0, data3.length);
    v.isNull[2] = false;


    outV.isRepeating = true;
    outV.noNulls = false;


    StringSubstrColStartLen expr = new StringSubstrColStartLen(0, 6, 6, 1);
    expr.evaluate(batch);
    BytesColumnVector outCol = (BytesColumnVector) batch.cols[1];
    Assert.assertEquals(3, batch.size);
    Assert.assertTrue(outCol.noNulls);
    Assert.assertFalse(outCol.isRepeating);
    byte[] expected = "string".getBytes("UTF-8");
    Assert.assertEquals(0,
    StringExpr.compare(
            expected, 0, expected.length, outCol.vector[0], outCol.start[0], outCol.length[0]
        )
    );


    Assert.assertEquals(0,
    StringExpr.compare(
            expected, 0, expected.length, outCol.vector[1], outCol.start[1], outCol.length[1]
        )
    );


    Assert.assertEquals(0,
    StringExpr.compare(
            emptyString, 0, emptyString.length, outCol.vector[2], outCol.start[2], outCol.length[2]
        )
    );


    // Testing negative substring index
    outV.isRepeating = true;
    outV.noNulls = false;


    expr = new StringSubstrColStartLen(0, -6, 6, 1);
    expr.evaluate(batch);
    outCol = (BytesColumnVector) batch.cols[1];
    Assert.assertTrue(outCol.noNulls);
    Assert.assertFalse(outCol.isRepeating);
    Assert.assertEquals(3, batch.size);


    Assert.assertEquals(0,
    StringExpr.compare(
            expected, 0, expected.length, outCol.vector[0], outCol.start[0], outCol.length[0]
        )
    );


    Assert.assertEquals(0,
        StringExpr.compare(
            expected, 0, expected.length, outCol.vector[1], outCol.start[1], outCol.length[1]
        )
    );


    // This yields empty because starting index is out of bounds
    Assert.assertEquals(0,
    StringExpr.compare(
            emptyString, 0, emptyString.length, outCol.vector[2], outCol.start[2], outCol.length[2]
        )
    );


    //Testing substring index starting with 1 and zero length


    outV.isRepeating = true;
    outV.noNulls = false;


    expr = new StringSubstrColStartLen(0, 1, 0, 1);
    outCol = (BytesColumnVector) batch.cols[1];
    expr.evaluate(batch);
    Assert.assertEquals(3, batch.size);
    Assert.assertTrue(outCol.noNulls);
    Assert.assertFalse(outCol.isRepeating);
    Assert.assertEquals(0,
        StringExpr.compare(
            data1, 1, 0, outCol.vector[0], outCol.start[0], outCol.length[0]
        )
    );


    Assert.assertEquals(0,
        StringExpr.compare(
            data2, 1, 0, outCol.vector[1], outCol.start[1], outCol.length[1]
        )
    );


    Assert.assertEquals(0,
        StringExpr.compare(
            data3, 1, 0, outCol.vector[2], outCol.start[2], outCol.length[2]
        )
    );




    //Testing substring index starting with 0 and length equal to array length


    outV.isRepeating = true;
    outV.noNulls = false;


    expr = new StringSubstrColStartLen(0, 0, 11, 1);
    outCol = (BytesColumnVector) batch.cols[1];
    expr.evaluate(batch);
    Assert.assertEquals(3, batch.size);
    Assert.assertTrue(outCol.noNulls);
    Assert.assertFalse(outCol.isRepeating);
    Assert.assertEquals(0,
    StringExpr.compare(
            data1, 0, data1.length, outCol.vector[0], outCol.start[0], outCol.length[0]
        )
    );


    Assert.assertEquals(0,
    StringExpr.compare(
            data2, 0, data2.length, outCol.vector[1], outCol.start[1], outCol.length[1]
        )
    );


    Assert.assertEquals(0,
    StringExpr.compare(
            data3, 0, data3.length, outCol.vector[2], outCol.start[2], outCol.length[2]
        )
    );




    // Testing setting length larger than array length, which should cap to the length itself


    outV.isRepeating = true;
    outV.noNulls = false;


    expr = new StringSubstrColStartLen(0, 6, 10, 1);
    expr.evaluate(batch);
    outCol = (BytesColumnVector) batch.cols[1];
    Assert.assertEquals(3, batch.size);
    Assert.assertTrue(outCol.noNulls);
    Assert.assertFalse(outCol.isRepeating);
    Assert.assertEquals(0,
    StringExpr.compare(
            expected, 0, expected.length, outCol.vector[0], outCol.start[0], outCol.length[0]
        )
    );


    Assert.assertEquals(0,
    StringExpr.compare(
            expected, 0, expected.length, outCol.vector[1], outCol.start[1], outCol.length[1]
        )
    );


    Assert.assertEquals(0,
    StringExpr.compare(
            emptyString, 0, emptyString.length, outCol.vector[2], outCol.start[2], outCol.length[2]
        )
    );


    outV.isRepeating = true;
    outV.noNulls = true;


    // Testing with nulls


    v.noNulls = false;
    v.isNull[0] = true;
    expr.evaluate(batch);
    Assert.assertEquals(3, batch.size);
    Assert.assertFalse(outV.noNulls);
    Assert.assertTrue(outV.isNull[0]);
    Assert.assertFalse(outCol.isRepeating);
    Assert.assertEquals(0,
    StringExpr.compare(
            expected, 0, expected.length, outCol.vector[1], outCol.start[1], outCol.length[1]
        )
    );


    Assert.assertEquals(0,
        StringExpr.compare(
            emptyString, 0, emptyString.length, outCol.vector[2], outCol.start[2], outCol.length[2]
        )
    );




    // Testing with repeating and no nulls
    outV = new BytesColumnVector();
    v = new BytesColumnVector();
    outV.isRepeating = false;
    outV.noNulls = true;
    v.isRepeating = true;
    v.noNulls = false;
    v.setRef(0, data1, 0, data1.length);
    batch = new VectorizedRowBatch(2);
    batch.cols[0] = v;
    batch.cols[1] = outV;
    expr.evaluate(batch);
    outCol = (BytesColumnVector) batch.cols[1];
    Assert.assertTrue(outCol.noNulls);
    Assert.assertTrue(outCol.isRepeating);


    Assert.assertEquals(0,
    StringExpr.compare(
            expected, 0, expected.length, outCol.vector[0], outCol.start[0], outCol.length[0]
        )
    );


    // Testing with multiByte String
    v = new BytesColumnVector();
    v.isRepeating = false;
    v.noNulls = true;
    batch.size = 1;
    v.setRef(0, multiByte, 0, 10);
    batch.cols[0] = v;
    batch.cols[1] = outV;
    outV.isRepeating = true;
    outV.noNulls = false;
    expr = new StringSubstrColStartLen(0, 3, 2, 1);
    expr.evaluate(batch);
    Assert.assertEquals(1, batch.size);
    Assert.assertFalse(outV.isRepeating);
    Assert.assertTrue(outV.noNulls);
    Assert.assertEquals(0,
    StringExpr.compare(
            // 3rd char starts at index 3, and with length 2 it is covering the rest of the array.
            multiByte, 3, 10 - 3, outCol.vector[0], outCol.start[0], outCol.length[0]
        )
    );


    // Testing multiByte string with reference set to mid array
    v = new BytesColumnVector();
    v.isRepeating = false;
    v.noNulls = true;
    outV = new BytesColumnVector();
    batch.size = 1;
    v.setRef(0, multiByte, 3, 7);
    batch.cols[0] = v;
    batch.cols[1] = outV;
    outV.isRepeating = true;

View Full Code Here

  @Test
  public void testVectorLTrim() {
    VectorizedRowBatch b = makeTrimBatch();
    VectorExpression expr = new StringLTrim(0, 1);
    expr.evaluate(b);
    BytesColumnVector outV = (BytesColumnVector) b.cols[1];
    Assert.assertEquals(0,
        StringExpr.compare(emptyString, 0, 0, outV.vector[0], 0, 0));
    Assert.assertEquals(0,
        StringExpr.compare(blanksLeft, 2, 3, outV.vector[1], outV.start[1], outV.length[1]));
    Assert.assertEquals(0,

View Full Code Here

  @Test
  public void testVectorRTrim() {
    VectorizedRowBatch b = makeTrimBatch();
    VectorExpression expr = new StringRTrim(0, 1);
    expr.evaluate(b);
    BytesColumnVector outV = (BytesColumnVector) b.cols[1];
    Assert.assertEquals(0,
        StringExpr.compare(emptyString, 0, 0, outV.vector[0], 0, 0));
    Assert.assertEquals(0,
        StringExpr.compare(blanksLeft, 0, 5, outV.vector[1], outV.start[1], outV.length[1]));
    Assert.assertEquals(0,

View Full Code Here

  @Test
  public void testVectorTrim() {
    VectorizedRowBatch b = makeTrimBatch();
    VectorExpression expr = new StringTrim(0, 1);
    expr.evaluate(b);
    BytesColumnVector outV = (BytesColumnVector) b.cols[1];
    Assert.assertEquals(0,
        StringExpr.compare(emptyString, 0, 0, outV.vector[0], 0, 0));
    Assert.assertEquals(0,
        StringExpr.compare(blanksLeft, 2, 3, outV.vector[1], outV.start[1], outV.length[1]));
    Assert.assertEquals(0,

View Full Code Here

  }


  // Make a batch to test the trim functions.
  private VectorizedRowBatch makeTrimBatch() {
    VectorizedRowBatch b = new VectorizedRowBatch(2);
    BytesColumnVector inV = new BytesColumnVector();
    BytesColumnVector outV = new BytesColumnVector();
    b.cols[0] = inV;
    b.cols[1] = outV;
    inV.setRef(0, emptyString, 0, 0);
    inV.setRef(1, blanksLeft, 0, blanksLeft.length);
    inV.setRef(2, blanksRight, 0, blanksRight.length);

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector

org.apache.hadoop.hive.ql.exec.vector.expressions.AbstractFilterStringColLikeStringScalar

org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.gen.VectorUDAFMaxString

org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.gen.VectorUDAFMinString

org.apache.hadoop.hive.ql.exec.vector.expressions.CastStringToDate

org.apache.hadoop.hive.ql.exec.vector.expressions.CastStringToDecimal

org.apache.hadoop.hive.ql.exec.vector.expressions.DecimalToStringUnaryUDF

org.apache.hadoop.hive.ql.exec.vector.expressions.FilterStringColumnInList

org.apache.hadoop.hive.ql.exec.vector.expressions.FuncLongToString

org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterStringColEqualStringColumn

org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterStringColEqualStringScalar

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.