/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hive.ql.exec.vector.expressions;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.Random;
import junit.framework.Assert;
import org.apache.hadoop.hive.common.type.HiveVarchar;
import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.TimestampUtils;
import org.apache.hadoop.hive.ql.exec.vector.util.VectorizedRowGroupGenUtil;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.serde2.io.ByteWritable;
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable;
import org.apache.hadoop.hive.serde2.io.ShortWritable;
import org.apache.hadoop.hive.serde2.io.TimestampWritable;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.StandardStructObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.io.BooleanWritable;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.FloatWritable;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.junit.Test;
/**
* Unit tests for vector expression writers.
*/
public class TestVectorExpressionWriters {
private final int vectorSize = 5;
private VectorExpressionWriter getWriter(TypeInfo colTypeInfo) throws HiveException {
ExprNodeDesc columnDesc = new ExprNodeColumnDesc();
columnDesc.setTypeInfo(colTypeInfo);
VectorExpressionWriter vew = VectorExpressionWriterFactory
.genVectorExpressionWritable(columnDesc);
return vew;
}
private Writable getWritableValue(TypeInfo ti, double value) {
if (ti.equals(TypeInfoFactory.floatTypeInfo)) {
return new FloatWritable((float) value);
} else if (ti.equals(TypeInfoFactory.doubleTypeInfo)) {
return new DoubleWritable(value);
}
return null;
}
private Writable getWritableValue(TypeInfo ti, byte[] value) {
if (ti.equals(TypeInfoFactory.stringTypeInfo)) {
return new Text(value);
} else if (ti.equals(TypeInfoFactory.varcharTypeInfo)) {
return new HiveVarcharWritable(
new HiveVarchar(new Text(value).toString(), -1));
} else if (ti.equals(TypeInfoFactory.binaryTypeInfo)) {
return new BytesWritable(value);
}
return null;
}
private Writable getWritableValue(TypeInfo ti, long value) {
if (ti.equals(TypeInfoFactory.byteTypeInfo)) {
return new ByteWritable((byte) value);
} else if (ti.equals(TypeInfoFactory.shortTypeInfo)) {
return new ShortWritable((short) value);
} else if (ti.equals(TypeInfoFactory.intTypeInfo)) {
return new IntWritable( (int) value);
} else if (ti.equals(TypeInfoFactory.longTypeInfo)) {
return new LongWritable( (long) value);
} else if (ti.equals(TypeInfoFactory.booleanTypeInfo)) {
return new BooleanWritable( value == 0 ? false : true);
} else if (ti.equals(TypeInfoFactory.timestampTypeInfo)) {
Timestamp ts = new Timestamp(value);
TimestampUtils.assignTimeInNanoSec(value, ts);
TimestampWritable tw = new TimestampWritable(ts);
return tw;
}
return null;
}
private void testWriterDouble(TypeInfo type) throws HiveException {
DoubleColumnVector dcv = VectorizedRowGroupGenUtil.generateDoubleColumnVector(true, false,
this.vectorSize, new Random(10));
dcv.isNull[2] = true;
VectorExpressionWriter vew = getWriter(type);
for (int i = 0; i < vectorSize; i++) {
Writable w = (Writable) vew.writeValue(dcv, i);
if (w != null) {
Writable expected = getWritableValue(type, dcv.vector[i]);
Assert.assertEquals(expected, w);
} else {
Assert.assertTrue(dcv.isNull[i]);
}
}
}
private void testSetterDouble(TypeInfo type) throws HiveException {
DoubleColumnVector dcv = VectorizedRowGroupGenUtil.generateDoubleColumnVector(true, false,
this.vectorSize, new Random(10));
dcv.isNull[2] = true;
Object[] values = new Object[this.vectorSize];
VectorExpressionWriter vew = getWriter(type);
for (int i = 0; i < vectorSize; i++) {
values[i] = vew.initValue(null);
values[i] = vew.setValue(values[i], dcv, i);
if (values[i] != null) {
Writable expected = getWritableValue(type, dcv.vector[i]);
Assert.assertEquals(expected, values[i]);
} else {
Assert.assertTrue(dcv.isNull[i]);
}
}
}
private void testWriterLong(TypeInfo type) throws HiveException {
LongColumnVector lcv = VectorizedRowGroupGenUtil.generateLongColumnVector(true, false,
vectorSize, new Random(10));
lcv.isNull[3] = true;
VectorExpressionWriter vew = getWriter(type);
for (int i = 0; i < vectorSize; i++) {
Writable w = (Writable) vew.writeValue(lcv, i);
if (w != null) {
Writable expected = getWritableValue(type, lcv.vector[i]);
if (expected instanceof TimestampWritable) {
TimestampWritable t1 = (TimestampWritable) expected;
TimestampWritable t2 = (TimestampWritable) w;
Assert.assertTrue(t1.getNanos() == t2.getNanos());
Assert.assertTrue(t1.getSeconds() == t2.getSeconds());
continue;
}
Assert.assertEquals(expected, w);
} else {
Assert.assertTrue(lcv.isNull[i]);
}
}
}
private void testSetterLong(TypeInfo type) throws HiveException {
LongColumnVector lcv = VectorizedRowGroupGenUtil.generateLongColumnVector(true, false,
vectorSize, new Random(10));
lcv.isNull[3] = true;
Object[] values = new Object[this.vectorSize];
VectorExpressionWriter vew = getWriter(type);
for (int i = 0; i < vectorSize; i++) {
values[i] = vew.initValue(null);
values[i] = vew.setValue(values[i], lcv, i);
if (values[i] != null) {
Writable expected = getWritableValue(type, lcv.vector[i]);
if (expected instanceof TimestampWritable) {
TimestampWritable t1 = (TimestampWritable) expected;
TimestampWritable t2 = (TimestampWritable) values[i];
Assert.assertTrue(t1.getNanos() == t2.getNanos());
Assert.assertTrue(t1.getSeconds() == t2.getSeconds());
continue;
}
Assert.assertEquals(expected, values[i]);
} else {
Assert.assertTrue(lcv.isNull[i]);
}
}
}
private StructObjectInspector genStructOI() {
ArrayList<String> fieldNames1 = new ArrayList<String>();
fieldNames1.add("theInt");
fieldNames1.add("theBool");
ArrayList<ObjectInspector> fieldObjectInspectors1 = new ArrayList<ObjectInspector>();
fieldObjectInspectors1
.add(PrimitiveObjectInspectorFactory.writableIntObjectInspector);
fieldObjectInspectors1
.add(PrimitiveObjectInspectorFactory.writableBooleanObjectInspector);
return ObjectInspectorFactory
.getStandardStructObjectInspector(fieldNames1, fieldObjectInspectors1);
}
private void testStructLong(TypeInfo type) throws HiveException {
LongColumnVector icv = VectorizedRowGroupGenUtil.generateLongColumnVector(true, false,
vectorSize, new Random(10));
icv.isNull[3] = true;
LongColumnVector bcv = VectorizedRowGroupGenUtil.generateLongColumnVector(true, false,
vectorSize, new Random(10));
bcv.isNull[2] = true;
ArrayList<Object>[] values = (ArrayList<Object>[]) new ArrayList[this.vectorSize];
StructObjectInspector soi = genStructOI();
VectorExpressionWriter[] vew = VectorExpressionWriterFactory.getExpressionWriters(soi);
for (int i = 0; i < vectorSize; i++) {
values[i] = new ArrayList<Object>(2);
values[i].add(null);
values[i].add(null);
vew[0].setValue(values[i], icv, i);
vew[1].setValue(values[i], bcv, i);
Object theInt = values[i].get(0);
if (theInt == null) {
Assert.assertTrue(icv.isNull[i]);
} else {
IntWritable w = (IntWritable) theInt;
Assert.assertEquals((int) icv.vector[i], w.get());
}
Object theBool = values[i].get(1);
if (theBool == null) {
Assert.assertTrue(bcv.isNull[i]);
} else {
BooleanWritable w = (BooleanWritable) theBool;
Assert.assertEquals(bcv.vector[i] == 0 ? false : true, w.get());
}
}
}
private void testWriterText(TypeInfo type) throws HiveException {
Text t1 = new Text("alpha");
Text t2 = new Text("beta");
BytesColumnVector bcv = new BytesColumnVector(vectorSize);
bcv.noNulls = false;
bcv.initBuffer();
bcv.setVal(0, t1.getBytes(), 0, t1.getLength());
bcv.isNull[1] = true;
bcv.setVal(2, t2.getBytes(), 0, t2.getLength());
bcv.isNull[3] = true;
bcv.setVal(4, t1.getBytes(), 0, t1.getLength());
VectorExpressionWriter vew = getWriter(type);
for (int i = 0; i < vectorSize; i++) {
Writable w = (Writable) vew.writeValue(bcv, i);
if (w != null) {
byte [] val = new byte[bcv.length[i]];
System.arraycopy(bcv.vector[i], bcv.start[i], val, 0, bcv.length[i]);
Writable expected = getWritableValue(type, val);
Assert.assertEquals(expected, w);
} else {
Assert.assertTrue(bcv.isNull[i]);
}
}
}
private void testSetterText(TypeInfo type) throws HiveException {
Text t1 = new Text("alpha");
Text t2 = new Text("beta");
BytesColumnVector bcv = new BytesColumnVector(vectorSize);
bcv.noNulls = false;
bcv.initBuffer();
bcv.setVal(0, t1.getBytes(), 0, t1.getLength());
bcv.isNull[1] = true;
bcv.setVal(2, t2.getBytes(), 0, t2.getLength());
bcv.isNull[3] = true;
bcv.setVal(4, t1.getBytes(), 0, t1.getLength());
Object[] values = new Object[this.vectorSize];
VectorExpressionWriter vew = getWriter(type);
for (int i = 0; i < vectorSize; i++) {
values[i] = vew.initValue(null);
Writable w = (Writable) vew.setValue(values[i], bcv, i);
if (w != null) {
byte [] val = new byte[bcv.length[i]];
System.arraycopy(bcv.vector[i], bcv.start[i], val, 0, bcv.length[i]);
Writable expected = getWritableValue(type, val);
Assert.assertEquals(expected, w);
} else {
Assert.assertTrue(bcv.isNull[i]);
}
}
}
@Test
public void testVectorExpressionWriterDouble() throws HiveException {
testWriterDouble(TypeInfoFactory.doubleTypeInfo);
}
@Test
public void testVectorExpressionSetterDouble() throws HiveException {
testSetterDouble(TypeInfoFactory.doubleTypeInfo);
}
@Test
public void testVectorExpressionWriterFloat() throws HiveException {
testWriterDouble(TypeInfoFactory.floatTypeInfo);
}
@Test
public void testVectorExpressionSetterFloat() throws HiveException {
testSetterDouble(TypeInfoFactory.floatTypeInfo);
}
@Test
public void testVectorExpressionWriterLong() throws HiveException {
testWriterLong(TypeInfoFactory.longTypeInfo);
}
@Test
public void testVectorExpressionSetterLong() throws HiveException {
testSetterLong(TypeInfoFactory.longTypeInfo);
}
@Test
public void testVectorExpressionStructLong() throws HiveException {
testStructLong(TypeInfoFactory.longTypeInfo);
}
@Test
public void testVectorExpressionWriterInt() throws HiveException {
testWriterLong(TypeInfoFactory.intTypeInfo);
}
@Test
public void testVectorExpressionSetterInt() throws HiveException {
testSetterLong(TypeInfoFactory.intTypeInfo);
}
@Test
public void testVectorExpressionWriterShort() throws HiveException {
testWriterLong(TypeInfoFactory.shortTypeInfo);
}
@Test
public void testVectorExpressionSetterShort() throws HiveException {
testSetterLong(TypeInfoFactory.shortTypeInfo);
}
@Test
public void testVectorExpressionWriterBoolean() throws HiveException {
testWriterLong(TypeInfoFactory.booleanTypeInfo);
}
@Test
public void testVectorExpressionSetterBoolean() throws HiveException {
testSetterLong(TypeInfoFactory.booleanTypeInfo);
}
@Test
public void testVectorExpressionWriterTimestamp() throws HiveException {
testWriterLong(TypeInfoFactory.timestampTypeInfo);
}
@Test
public void testVectorExpressionSetterTimestamp() throws HiveException {
testSetterLong(TypeInfoFactory.timestampTypeInfo);
}
@Test
public void testVectorExpressionWriterByte() throws HiveException {
testWriterLong(TypeInfoFactory.byteTypeInfo);
}
@Test
public void testVectorExpressionSetterByte() throws HiveException {
testSetterLong(TypeInfoFactory.byteTypeInfo);
}
@Test
public void testVectorExpressionWriterString() throws HiveException {
testWriterText(TypeInfoFactory.stringTypeInfo);
}
@Test
public void testVectorExpressionSetterString() throws HiveException {
testSetterText(TypeInfoFactory.stringTypeInfo);
}
@Test
public void testVectorExpressionWriterVarchar() throws HiveException {
testWriterText(TypeInfoFactory.varcharTypeInfo);
}
@Test
public void testVectorExpressionSetterVarchar() throws HiveException {
testSetterText(TypeInfoFactory.varcharTypeInfo);
}
@Test
public void testVectorExpressionWriterBinary() throws HiveException {
testWriterText(TypeInfoFactory.binaryTypeInfo);
}
@Test
public void testVectorExpressionSetterBinary() throws HiveException {
testSetterText(TypeInfoFactory.binaryTypeInfo);
}
@Test
public void testTimeStampUtils(){
Timestamp ts = new Timestamp(0);
// Convert positive nanoseconds to timestamp object.
TimestampUtils.assignTimeInNanoSec(1234567891, ts);
Assert.assertEquals(234567891, ts.getNanos());
Assert.assertEquals(1234567891, TimestampUtils.getTimeNanoSec(ts));
// Test negative nanoseconds
TimestampUtils.assignTimeInNanoSec(-1234567891, ts);
Assert.assertEquals((1000000000-234567891), ts.getNanos());
Assert.assertEquals(-1234567891, TimestampUtils.getTimeNanoSec(ts));
// Test positive value smaller than a second.
TimestampUtils.assignTimeInNanoSec(234567891, ts);
Assert.assertEquals(234567891, ts.getNanos());
Assert.assertEquals(234567891, TimestampUtils.getTimeNanoSec(ts));
// Test negative value smaller than a second.
TimestampUtils.assignTimeInNanoSec(-234567891, ts);
Assert.assertEquals((1000000000-234567891), ts.getNanos());
Assert.assertEquals(-234567891, TimestampUtils.getTimeNanoSec(ts));
// Test a positive long timestamp
long big = 152414813551296L;
TimestampUtils.assignTimeInNanoSec(big, ts);
Assert.assertEquals(big % 1000000000, ts.getNanos());
Assert.assertEquals(big, TimestampUtils.getTimeNanoSec(ts));
// Test a negative long timestamp
big = -152414813551296L;
TimestampUtils.assignTimeInNanoSec(big, ts);
Assert.assertEquals((1000000000 + (big % 1000000000)), ts.getNanos());
Assert.assertEquals(big, TimestampUtils.getTimeNanoSec(ts));
// big/1000000 will yield zero nanoseconds
big = -1794750230000828416L;
ts = new Timestamp(0);
TimestampUtils.assignTimeInNanoSec(big, ts);
Assert.assertEquals((1000000000 + big % 1000000000), ts.getNanos());
Assert.assertEquals(big, TimestampUtils.getTimeNanoSec(ts));
// Very small nanosecond part
big = 1700000000000000016L;
ts = new Timestamp(0);
TimestampUtils.assignTimeInNanoSec(big, ts);
Assert.assertEquals(big % 1000000000, ts.getNanos());
Assert.assertEquals(big, TimestampUtils.getTimeNanoSec(ts));
// Very small nanosecond part
big = -1700000000000000016L;
ts = new Timestamp(0);
TimestampUtils.assignTimeInNanoSec(big, ts);
Assert.assertEquals((1000000000 + big % 1000000000), ts.getNanos());
Assert.assertEquals(big, TimestampUtils.getTimeNanoSec(ts));
}
}