Examples of org.apache.hadoop.hive.serde2.io.TimestampWritable

Package org.apache.hadoop.hive.serde2.io

Examples of org.apache.hadoop.hive.serde2.io.TimestampWritable

org.apache.hadoop.hive.serde2.io.TimestampWritable
TimestampWritable Writable equivalent of java.sq.Timestamp Timestamps are of the format YYYY-MM-DD HH:MM:SS.[fff...] We encode Unix timestamp in seconds in 4 bytes, using the MSB to signify whether the timestamp has a fractional portion. The fractional portion is reversed, and encoded as a VInt so timestamps with less precision use fewer bytes. 0.1 -> 1 0.01 -> 10 0.001 -> 100

        d.writeToByteStream(byteStream);
        return;
      }
      case TIMESTAMP: {
        TimestampObjectInspector toi = (TimestampObjectInspector) poi;
        TimestampWritable t = toi.getPrimitiveWritableObject(obj);
        t.writeToByteStream(byteStream);
        return;
      }


      case DECIMAL: {
        HiveDecimalObjectInspector bdoi = (HiveDecimalObjectInspector) poi;
        HiveDecimalWritable t = bdoi.getPrimitiveWritableObject(obj);
        if (t == null) {
          return;
        }
        t.writeToByteStream(byteStream);
        return;
      }


      default: {
        throw new RuntimeException("Unrecognized type: "

View Full Code Here

        d.set(deserializeInt(buffer, invert));
        return d;
      }


      case TIMESTAMP:
        TimestampWritable t = (reuse == null ? new TimestampWritable() :
            (TimestampWritable) reuse);
        byte[] bytes = new byte[TimestampWritable.BINARY_SORTABLE_LENGTH];


        for (int i = 0; i < bytes.length; i++) {
          bytes[i] = buffer.read(invert);
        }
        t.setBinarySortable(bytes, 0);
        return t;


      case DECIMAL: {
        // See serialization of decimal for explanation (below)

View Full Code Here

        serializeInt(buffer, v, invert);
        return;
      }
      case TIMESTAMP: {
        TimestampObjectInspector toi = (TimestampObjectInspector) poi;
        TimestampWritable t = toi.getPrimitiveWritableObject(o);
        byte[] data = t.getBinarySortable();
        for (int i = 0; i < data.length; i++) {
          writeByte(buffer, data[i], invert);
        }
        return;
      }

View Full Code Here


    // tiemestamp
    baos.reset();
    Timestamp timestampValue = new Timestamp(now.getTime());
    ByteStream.Output output = new ByteStream.Output();
    TimestampWritable timestampWritable = new TimestampWritable(new Timestamp(now.getTime()));
    timestampWritable.write(output);
    output.close();
    m.put(cfBytes, "timestamp".getBytes(), output.toByteArray());


    // char
    baos.reset();
    HiveChar charValue = new HiveChar("char", 4);
    JavaHiveCharObjectInspector charOI = (JavaHiveCharObjectInspector) PrimitiveObjectInspectorFactory
        .getPrimitiveJavaObjectInspector(new CharTypeInfo(4));
    LazyUtils.writePrimitiveUTF8(baos, charOI.create(charValue), charOI, false, (byte) 0, null);
    m.put(cfBytes, "char".getBytes(), baos.toByteArray());


    baos.reset();
    HiveVarchar varcharValue = new HiveVarchar("varchar", 7);
    JavaHiveVarcharObjectInspector varcharOI = (JavaHiveVarcharObjectInspector) PrimitiveObjectInspectorFactory
        .getPrimitiveJavaObjectInspector(new VarcharTypeInfo(7));
    LazyUtils.writePrimitiveUTF8(baos, varcharOI.create(varcharValue), varcharOI, false, (byte) 0,
        null);
    m.put(cfBytes, "varchar".getBytes(), baos.toByteArray());


    writer.addMutation(m);


    writer.close();


    for (Entry<Key,Value> e : conn.createScanner(tableName, new Authorizations())) {
      System.out.println(e);
    }


    // Create the RecordReader
    FileInputFormat.addInputPath(conf, new Path("unused"));
    InputSplit[] splits = inputformat.getSplits(conf, 0);
    assertEquals(splits.length, 1);
    RecordReader<Text,AccumuloHiveRow> reader = inputformat.getRecordReader(splits[0], conf, null);


    Text key = reader.createKey();
    AccumuloHiveRow value = reader.createValue();


    reader.next(key, value);


    Assert.assertEquals(13, value.getTuples().size());


    ByteArrayRef byteRef = new ByteArrayRef();


    // string
    Text cfText = new Text(cf), cqHolder = new Text();
    cqHolder.set("string");
    byte[] valueBytes = value.getValue(cfText, cqHolder);
    Assert.assertNotNull(valueBytes);


    byteRef.setData(valueBytes);
    LazyStringObjectInspector lazyStringOI = LazyPrimitiveObjectInspectorFactory
        .getLazyStringObjectInspector(false, (byte) 0);
    LazyString lazyString = (LazyString) LazyFactory.createLazyObject(lazyStringOI);
    lazyString.init(byteRef, 0, valueBytes.length);


    Assert.assertEquals(stringValue, lazyString.getWritableObject().toString());


    // boolean
    cqHolder.set("boolean");
    valueBytes = value.getValue(cfText, cqHolder);
    Assert.assertNotNull(valueBytes);


    byteRef.setData(valueBytes);
    LazyBooleanObjectInspector lazyBooleanOI = (LazyBooleanObjectInspector) LazyPrimitiveObjectInspectorFactory
        .getLazyObjectInspector(TypeInfoFactory
            .getPrimitiveTypeInfo(serdeConstants.BOOLEAN_TYPE_NAME));
    LazyBoolean lazyBoolean = (LazyBoolean) LazyFactory
        .createLazyPrimitiveBinaryClass(lazyBooleanOI);
    lazyBoolean.init(byteRef, 0, valueBytes.length);


    Assert.assertEquals(booleanValue, lazyBoolean.getWritableObject().get());


    // tinyint
    cqHolder.set("tinyint");
    valueBytes = value.getValue(cfText, cqHolder);
    Assert.assertNotNull(valueBytes);


    byteRef.setData(valueBytes);
    LazyByteObjectInspector lazyByteOI = (LazyByteObjectInspector) LazyPrimitiveObjectInspectorFactory
        .getLazyObjectInspector(TypeInfoFactory
            .getPrimitiveTypeInfo(serdeConstants.TINYINT_TYPE_NAME));
    LazyByte lazyByte = (LazyByte) LazyFactory.createLazyPrimitiveBinaryClass(lazyByteOI);
    lazyByte.init(byteRef, 0, valueBytes.length);


    Assert.assertEquals(tinyintValue, lazyByte.getWritableObject().get());


    // smallint
    cqHolder.set("smallint");
    valueBytes = value.getValue(cfText, cqHolder);
    Assert.assertNotNull(valueBytes);


    byteRef.setData(valueBytes);
    LazyShortObjectInspector lazyShortOI = (LazyShortObjectInspector) LazyPrimitiveObjectInspectorFactory
        .getLazyObjectInspector(TypeInfoFactory
            .getPrimitiveTypeInfo(serdeConstants.SMALLINT_TYPE_NAME));
    LazyShort lazyShort = (LazyShort) LazyFactory.createLazyPrimitiveBinaryClass(lazyShortOI);
    lazyShort.init(byteRef, 0, valueBytes.length);


    Assert.assertEquals(smallintValue, lazyShort.getWritableObject().get());


    // int
    cqHolder.set("int");
    valueBytes = value.getValue(cfText, cqHolder);
    Assert.assertNotNull(valueBytes);


    byteRef.setData(valueBytes);
    LazyIntObjectInspector lazyIntOI = (LazyIntObjectInspector) LazyPrimitiveObjectInspectorFactory
        .getLazyObjectInspector(TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.INT_TYPE_NAME));
    LazyInteger lazyInt = (LazyInteger) LazyFactory.createLazyPrimitiveBinaryClass(lazyIntOI);
    lazyInt.init(byteRef, 0, valueBytes.length);


    Assert.assertEquals(intValue, lazyInt.getWritableObject().get());


    // bigint
    cqHolder.set("bigint");
    valueBytes = value.getValue(cfText, cqHolder);
    Assert.assertNotNull(valueBytes);


    byteRef.setData(valueBytes);
    LazyLongObjectInspector lazyLongOI = (LazyLongObjectInspector) LazyPrimitiveObjectInspectorFactory
        .getLazyObjectInspector(TypeInfoFactory
            .getPrimitiveTypeInfo(serdeConstants.BIGINT_TYPE_NAME));
    LazyLong lazyLong = (LazyLong) LazyFactory.createLazyPrimitiveBinaryClass(lazyLongOI);
    lazyLong.init(byteRef, 0, valueBytes.length);


    Assert.assertEquals(bigintValue, lazyLong.getWritableObject().get());


    // float
    cqHolder.set("float");
    valueBytes = value.getValue(cfText, cqHolder);
    Assert.assertNotNull(valueBytes);


    byteRef.setData(valueBytes);
    LazyFloatObjectInspector lazyFloatOI = (LazyFloatObjectInspector) LazyPrimitiveObjectInspectorFactory
        .getLazyObjectInspector(TypeInfoFactory
            .getPrimitiveTypeInfo(serdeConstants.FLOAT_TYPE_NAME));
    LazyFloat lazyFloat = (LazyFloat) LazyFactory.createLazyPrimitiveBinaryClass(lazyFloatOI);
    lazyFloat.init(byteRef, 0, valueBytes.length);


    Assert.assertEquals(floatValue, lazyFloat.getWritableObject().get(), 0);


    // double
    cqHolder.set("double");
    valueBytes = value.getValue(cfText, cqHolder);
    Assert.assertNotNull(valueBytes);


    byteRef.setData(valueBytes);
    LazyDoubleObjectInspector lazyDoubleOI = (LazyDoubleObjectInspector) LazyPrimitiveObjectInspectorFactory
        .getLazyObjectInspector(TypeInfoFactory
            .getPrimitiveTypeInfo(serdeConstants.DOUBLE_TYPE_NAME));
    LazyDouble lazyDouble = (LazyDouble) LazyFactory.createLazyPrimitiveBinaryClass(lazyDoubleOI);
    lazyDouble.init(byteRef, 0, valueBytes.length);


    Assert.assertEquals(doubleValue, lazyDouble.getWritableObject().get(), 0);


    // decimal
    cqHolder.set("decimal");
    valueBytes = value.getValue(cfText, cqHolder);
    Assert.assertNotNull(valueBytes);


    byteRef.setData(valueBytes);
    ByteArrayInputStream bais = new ByteArrayInputStream(valueBytes);
    DataInputStream in = new DataInputStream(bais);
    decimalWritable.readFields(in);


    Assert.assertEquals(decimalValue, decimalWritable.getHiveDecimal());


    // date
    cqHolder.set("date");
    valueBytes = value.getValue(cfText, cqHolder);
    Assert.assertNotNull(valueBytes);


    byteRef.setData(valueBytes);
    bais = new ByteArrayInputStream(valueBytes);
    in = new DataInputStream(bais);
    dateWritable.readFields(in);


    Assert.assertEquals(dateValue, dateWritable.get());


    // timestamp
    cqHolder.set("timestamp");
    valueBytes = value.getValue(cfText, cqHolder);
    Assert.assertNotNull(valueBytes);


    byteRef.setData(valueBytes);
    bais = new ByteArrayInputStream(valueBytes);
    in = new DataInputStream(bais);
    timestampWritable.readFields(in);


    Assert.assertEquals(timestampValue, timestampWritable.getTimestamp());


    // char
    cqHolder.set("char");
    valueBytes = value.getValue(cfText, cqHolder);
    Assert.assertNotNull(valueBytes);

View Full Code Here

        }
      }
      System.arraycopy(src, 0, tgt, bytes - src.length, src.length); // Padding leading zeroes/ones.
      return new BytesWritable(tgt);
    case TIMESTAMP:
      return new TimestampWritable(((TimestampObjectInspector) inspector).getPrimitiveJavaObject(obj));
    case CHAR:
      String strippedValue = ((HiveCharObjectInspector) inspector).getPrimitiveJavaObject(obj).getStrippedValue();
      return new BytesWritable(Binary.fromString(strippedValue).getBytes());
    case VARCHAR:
      String value = ((HiveVarcharObjectInspector) inspector).getPrimitiveJavaObject(obj).getValue();

View Full Code Here

          public void assignObjectValue(Object val, int destIndex) throws HiveException {
            if (val == null) {
              assignNull(destIndex);
            }
            else {
              TimestampWritable bw = (TimestampWritable) val;
              Timestamp t = bw.getTimestamp();
              assignLong(TimestampUtils.getTimeNanoSec(t), destIndex);
            }
          }
        }.init(outputBatch, (LongColumnVector) destCol);
        break;
      case DATE:
        outVCA = new VectorLongColumnAssign() {
          @Override
          public void assignObjectValue(Object val, int destIndex) throws HiveException {
            if (val == null) {
              assignNull(destIndex);
            }
            else {
              DateWritable bw = (DateWritable) val;
              assignLong(bw.getDays(), destIndex);
            }
          }
        }.init(outputBatch, (LongColumnVector) destCol);
        break;
      default:
        throw new HiveException("Incompatible Long vector column and primitive category " +
            category);
      }
    }
    else if (destCol instanceof DoubleColumnVector) {
      switch(category) {
      case DOUBLE:
        outVCA = new VectorDoubleColumnAssign() {
          @Override
          public void assignObjectValue(Object val, int destIndex) throws HiveException {
            if (val == null) {
              assignNull(destIndex);
            }
            else {
              DoubleWritable bw = (DoubleWritable) val;
              assignDouble(bw.get(), destIndex);
            }
          }
        }.init(outputBatch, (DoubleColumnVector) destCol);
        break;
      case FLOAT:
        outVCA = new VectorDoubleColumnAssign() {
          @Override
          public void assignObjectValue(Object val, int destIndex) throws HiveException {
            if (val == null) {
              assignNull(destIndex);
            }
            else {
              FloatWritable bw = (FloatWritable) val;
              assignDouble(bw.get(), destIndex);
            }
          }
        }.init(outputBatch, (DoubleColumnVector) destCol);
        break;
      default:
        throw new HiveException("Incompatible Double vector column and primitive category " +
            category);
      }
    }
    else if (destCol instanceof BytesColumnVector) {
      switch(category) {
      case BINARY:
        outVCA = new VectorBytesColumnAssign() {
          @Override
          public void assignObjectValue(Object val, int destIndex) throws HiveException {
            if (val == null) {
              assignNull(destIndex);
            }
            else {
              BytesWritable bw = (BytesWritable) val;
              byte[] bytes = bw.getBytes();
              assignBytes(bytes, 0, bw.getLength(), destIndex);
            }
          }
        }.init(outputBatch, (BytesColumnVector) destCol);
        break;
      case STRING:
        outVCA = new VectorBytesColumnAssign() {
          @Override
          public void assignObjectValue(Object val, int destIndex) throws HiveException {
            if (val == null) {
              assignNull(destIndex);
            } else {
              Text bw = (Text) val;
              byte[] bytes = bw.getBytes();
              assignBytes(bytes, 0, bw.getLength(), destIndex);
            }
          }
        }.init(outputBatch, (BytesColumnVector) destCol);
        break;
      case VARCHAR:

View Full Code Here

              case TIMESTAMP:
                LongColumnVector tcv = (LongColumnVector) batch.cols[k];
                long timeInNanoSec = tcv.vector[rowIndex];
                Timestamp t = new Timestamp(0);
                TimestampUtils.assignTimeInNanoSec(timeInNanoSec, t);
                TimestampWritable tw = new TimestampWritable();
                tw.set(t);
                LazyTimestamp.writeUTF8(serializeVectorStream, tw);
                break;
              case DATE:
                LongColumnVector dacv = (LongColumnVector) batch.cols[k];
                DateWritable daw = new DateWritable((int) dacv.vector[rowIndex]);

View Full Code Here

    }


    @Override
    Object next(Object previous) throws IOException {
      super.next(previous);
      TimestampWritable result = null;
      if (valuePresent) {
        if (previous == null) {
          result = new TimestampWritable();
        } else {
          result = (TimestampWritable) previous;
        }
        Timestamp ts = new Timestamp(0);
        long millis = (data.next() + WriterImpl.BASE_TIMESTAMP) *
            WriterImpl.MILLIS_PER_SECOND;
        int newNanos = parseNanos(nanos.next());
        // fix the rounding when we divided by 1000.
        if (millis >= 0) {
          millis += newNanos / 1000000;
        } else {
          millis -= newNanos / 1000000;
        }
        ts.setTime(millis);
        ts.setNanos(newNanos);
        result.set(ts);
      }
      return result;
    }

View Full Code Here

        obj = next(obj);
        if (obj == null) {
          result.noNulls = false;
          result.isNull[i] = true;
        } else {
          TimestampWritable writable = (TimestampWritable) obj;
          Timestamp  timestamp = writable.getTimestamp();
          result.vector[i] = TimestampUtils.getTimeNanoSec(timestamp);
        }
      }


      return result;

View Full Code Here

            keyValue.add(iwKey.get());
          } else if (key instanceof LongWritable) {
            LongWritable lwKey = (LongWritable)key;
            keyValue.add(lwKey.get());
          } else if (key instanceof TimestampWritable) {
            TimestampWritable twKey = (TimestampWritable)key;
            keyValue.add(twKey.getTimestamp());
          } else if (key instanceof DoubleWritable) {
            DoubleWritable dwKey = (DoubleWritable)key;
            keyValue.add(dwKey.get());
          } else if (key instanceof FloatWritable) {
            FloatWritable fwKey = (FloatWritable)key;

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.hive.serde2.io.TimestampWritable

com.asakusafw.directio.hive.orc.OrcFileInput

com.asakusafw.directio.hive.serde.DataModelDriver

com.asakusafw.directio.hive.serde.DataModelDriverTest

com.asakusafw.directio.hive.serde.ValueSerdeFactoryTest

com.facebook.hive.orc.lazy.LazyDoubleTreeReader

com.facebook.hive.orc.lazy.LazyShortDictionaryTreeReader

com.facebook.hive.orc.lazy.LazyShortDirectTreeReader

com.facebook.hive.orc.lazy.OrcLazyStructObjectInspector

com.facebook.hive.orc.lazy.OrcLazyTimestamp

com.facebook.hive.orc.OrcOutputFormat$OrcRecordWriter

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.