Examples of org.apache.hadoop.hive.serde2.io.TimestampWritable

Package org.apache.hadoop.hive.serde2.io

Examples of org.apache.hadoop.hive.serde2.io.TimestampWritable

org.apache.hadoop.hive.serde2.io.TimestampWritable
TimestampWritable Writable equivalent of java.sq.Timestamp Timestamps are of the format YYYY-MM-DD HH:MM:SS.[fff...] We encode Unix timestamp in seconds in 4 bytes, using the MSB to signify whether the timestamp has a fractional portion. The fractional portion is reversed, and encoded as a VInt so timestamps with less precision use fewer bytes. 0.1 -> 1 0.01 -> 10 0.001 -> 100

          keyValue = iwKey.get();
        } else if (key instanceof LongWritable) {
          LongWritable lwKey = (LongWritable)key;
          keyValue = lwKey.get();
        } else if (key instanceof TimestampWritable) {
          TimestampWritable twKey = (TimestampWritable)key;
          keyValue = twKey.getTimestamp();
        } else if (key instanceof DoubleWritable) {
          DoubleWritable dwKey = (DoubleWritable)key;
          keyValue = dwKey.get();
        } else if (key instanceof FloatWritable) {
          FloatWritable fwKey = (FloatWritable)key;

View Full Code Here

    } else if (ti.equals(TypeInfoFactory.booleanTypeInfo)) {
      return new BooleanWritable( value == 0 ? false : true);
    } else if (ti.equals(TypeInfoFactory.timestampTypeInfo)) {
      Timestamp ts = new Timestamp(value);
      TimestampUtils.assignTimeInNanoSec(value, ts);
      TimestampWritable tw = new TimestampWritable(ts);
      return tw;
    }
    return null;
  }

View Full Code Here

    for (int i = 0; i < vectorSize; i++) {
      Writable w = (Writable) vew.writeValue(lcv, i);
      if (w != null) {
        Writable expected = getWritableValue(type, lcv.vector[i]);
        if (expected instanceof TimestampWritable) {
          TimestampWritable t1 = (TimestampWritable) expected;
          TimestampWritable t2 = (TimestampWritable) w;
          Assert.assertTrue(t1.getNanos() == t2.getNanos());
          Assert.assertTrue(t1.getSeconds() == t2.getSeconds());
          continue;
        }
        Assert.assertEquals(expected, w);
      } else {
        Assert.assertTrue(lcv.isNull[i]);

View Full Code Here

      values[i] = null;  // setValue() should be able to handle null input
      values[i] = vew.setValue(values[i], lcv, i);
      if (values[i] != null) {
        Writable expected = getWritableValue(type, lcv.vector[i]);
        if (expected instanceof TimestampWritable) {
          TimestampWritable t1 = (TimestampWritable) expected;
          TimestampWritable t2 = (TimestampWritable) values[i];
          Assert.assertTrue(t1.getNanos() == t2.getNanos());
          Assert.assertTrue(t1.getSeconds() == t2.getSeconds());
          continue;
        }
        Assert.assertEquals(expected, values[i]);
      } else {
        Assert.assertTrue(lcv.isNull[i]);

View Full Code Here

                                         .bufferSize(100)
                                         .blockPadding(false));
    OrcStruct row = new OrcStruct(3);
    OrcUnion union = new OrcUnion();
    row.setFieldValue(1, union);
    row.setFieldValue(0, new TimestampWritable(Timestamp.valueOf("2000-03-12 15:00:00")));
    HiveDecimal value = HiveDecimal.create("12345678.6547456");
    row.setFieldValue(2, new HiveDecimalWritable(value));
    union.set((byte) 0, new IntWritable(42));
    writer.addRow(row);
    row.setFieldValue(0, new TimestampWritable(Timestamp.valueOf("2000-03-20 12:00:00.123456789")));
    union.set((byte) 1, new Text("hello"));
    value = HiveDecimal.create("-5643.234");
    row.setFieldValue(2, new HiveDecimalWritable(value));
    writer.addRow(row);
    row.setFieldValue(0, null);
    row.setFieldValue(1, null);
    row.setFieldValue(2, null);
    writer.addRow(row);
    row.setFieldValue(1, union);
    union.set((byte) 0, null);
    writer.addRow(row);
    union.set((byte) 1, null);
    writer.addRow(row);
    union.set((byte) 0, new IntWritable(200000));
    row.setFieldValue(0, new TimestampWritable(Timestamp.valueOf("1900-01-01 00:00:00")));
    value = HiveDecimal.create("10000000000000000000");
    row.setFieldValue(2, new HiveDecimalWritable(value));
    writer.addRow(row);
    Random rand = new Random(42);
    for(int i=1900; i < 2200; ++i) {
      row.setFieldValue(0, new TimestampWritable(Timestamp.valueOf(i + "-05-05 12:34:56." + i)));
      if ((i & 1) == 0) {
        union.set((byte) 0, new IntWritable(i*i));
      } else {
        union.set((byte) 1, new Text(Integer.toString(i * i)));
      }
      value = HiveDecimal.create(new BigInteger(64, rand),
          rand.nextInt(18));
      row.setFieldValue(2, new HiveDecimalWritable(value));
      if (maxValue.compareTo(value) < 0) {
        maxValue = value;
      }
      writer.addRow(row);
    }
    // let's add a lot of constant rows to test the rle
    row.setFieldValue(0, null);
    union.set((byte) 0, new IntWritable(1732050807));
    row.setFieldValue(2, null);
    for(int i=0; i < 5000; ++i) {
      writer.addRow(row);
    }
    union.set((byte) 0, new IntWritable(0));
    writer.addRow(row);
    union.set((byte) 0, new IntWritable(10));
    writer.addRow(row);
    union.set((byte) 0, new IntWritable(138));
    writer.addRow(row);
    writer.close();
    Reader reader = OrcFile.createReader(testFilePath,
        OrcFile.readerOptions(conf).filesystem(fs));
    assertEquals(false, reader.getMetadataKeys().iterator().hasNext());
    assertEquals(5309, reader.getNumberOfRows());
    DecimalColumnStatistics stats =
        (DecimalColumnStatistics) reader.getStatistics()[5];
    assertEquals(303, stats.getNumberOfValues());
    assertEquals(HiveDecimal.create("-5643.234"), stats.getMinimum());
    assertEquals(maxValue, stats.getMaximum());
    // TODO: fix this
//    assertEquals(null,stats.getSum());
    int stripeCount = 0;
    int rowCount = 0;
    long currentOffset = -1;
    for(StripeInformation stripe: reader.getStripes()) {
      stripeCount += 1;
      rowCount += stripe.getNumberOfRows();
      if (currentOffset < 0) {
        currentOffset = stripe.getOffset() + stripe.getLength();
      } else {
        assertEquals(currentOffset, stripe.getOffset());
        currentOffset += stripe.getLength();
      }
    }
    assertEquals(reader.getNumberOfRows(), rowCount);
    assertEquals(2, stripeCount);
    assertEquals(reader.getContentLength(), currentOffset);
    RecordReader rows = reader.rows();
    assertEquals(0, rows.getRowNumber());
    assertEquals(0.0, rows.getProgress(), 0.000001);
    assertEquals(true, rows.hasNext());
    row = (OrcStruct) rows.next(null);
    assertEquals(1, rows.getRowNumber());
    inspector = reader.getObjectInspector();
    assertEquals("struct<time:timestamp,union:uniontype<int,string>,decimal:decimal(38,18)>",
        inspector.getTypeName());
    assertEquals(new TimestampWritable(Timestamp.valueOf("2000-03-12 15:00:00")),
        row.getFieldValue(0));
    union = (OrcUnion) row.getFieldValue(1);
    assertEquals(0, union.getTag());
    assertEquals(new IntWritable(42), union.getObject());
    assertEquals(new HiveDecimalWritable(HiveDecimal.create("12345678.6547456")),
        row.getFieldValue(2));
    row = (OrcStruct) rows.next(row);
    assertEquals(2, rows.getRowNumber());
    assertEquals(new TimestampWritable(Timestamp.valueOf("2000-03-20 12:00:00.123456789")),
        row.getFieldValue(0));
    assertEquals(1, union.getTag());
    assertEquals(new Text("hello"), union.getObject());
    assertEquals(new HiveDecimalWritable(HiveDecimal.create("-5643.234")),
        row.getFieldValue(2));
    row = (OrcStruct) rows.next(row);
    assertEquals(null, row.getFieldValue(0));
    assertEquals(null, row.getFieldValue(1));
    assertEquals(null, row.getFieldValue(2));
    row = (OrcStruct) rows.next(row);
    assertEquals(null, row.getFieldValue(0));
    union = (OrcUnion) row.getFieldValue(1);
    assertEquals(0, union.getTag());
    assertEquals(null, union.getObject());
    assertEquals(null, row.getFieldValue(2));
    row = (OrcStruct) rows.next(row);
    assertEquals(null, row.getFieldValue(0));
    assertEquals(1, union.getTag());
    assertEquals(null, union.getObject());
    assertEquals(null, row.getFieldValue(2));
    row = (OrcStruct) rows.next(row);
    assertEquals(new TimestampWritable(Timestamp.valueOf("1900-01-01 00:00:00")),
        row.getFieldValue(0));
    assertEquals(new IntWritable(200000), union.getObject());
    assertEquals(new HiveDecimalWritable(HiveDecimal.create("10000000000000000000")),
                 row.getFieldValue(2));
    rand = new Random(42);
    for(int i=1900; i < 2200; ++i) {
      row = (OrcStruct) rows.next(row);
      assertEquals(new TimestampWritable(Timestamp.valueOf(i + "-05-05 12:34:56." + i)),
          row.getFieldValue(0));
      if ((i & 1) == 0) {
        assertEquals(0, union.getTag());
        assertEquals(new IntWritable(i*i), union.getObject());
      } else {
        assertEquals(1, union.getTag());
        assertEquals(new Text(Integer.toString(i * i)), union.getObject());
      }
      assertEquals(new HiveDecimalWritable(HiveDecimal.create(new BigInteger(64, rand),
                                   rand.nextInt(18))), row.getFieldValue(2));
    }
    for(int i=0; i < 5000; ++i) {
      row = (OrcStruct) rows.next(row);
      assertEquals(new IntWritable(1732050807), union.getObject());
    }
    row = (OrcStruct) rows.next(row);
    assertEquals(new IntWritable(0), union.getObject());
    row = (OrcStruct) rows.next(row);
    assertEquals(new IntWritable(10), union.getObject());
    row = (OrcStruct) rows.next(row);
    assertEquals(new IntWritable(138), union.getObject());
    assertEquals(false, rows.hasNext());
    assertEquals(1.0, rows.getProgress(), 0.00001);
    assertEquals(reader.getNumberOfRows(), rows.getRowNumber());
    rows.seekToRow(1);
    row = (OrcStruct) rows.next(row);
    assertEquals(new TimestampWritable(Timestamp.valueOf("2000-03-20 12:00:00.123456789")),
        row.getFieldValue(0));
    assertEquals(1, union.getTag());
    assertEquals(new Text("hello"), union.getObject());
    assertEquals(new HiveDecimalWritable(HiveDecimal.create("-5643.234")), row.getFieldValue(2));
    rows.close();

View Full Code Here

        }
        return bw;
      }


      case TIMESTAMP:
        TimestampWritable t = (reuse == null ? new TimestampWritable() :
            (TimestampWritable) reuse);
        byte[] bytes = new byte[8];


        for (int i = 0; i < bytes.length; i++) {
          bytes[i] = buffer.read(invert);
        }
        t.setBinarySortable(bytes, 0);
        return t;


      case DECIMAL: {
        // See serialization of decimal for explanation (below)

View Full Code Here

        serializeBytes(buffer, toSer, ba.getLength(), invert);
        return;
      }
      case TIMESTAMP: {
        TimestampObjectInspector toi = (TimestampObjectInspector) poi;
        TimestampWritable t = toi.getPrimitiveWritableObject(o);
        byte[] data = t.getBinarySortable();
        for (int i = 0; i < data.length; i++) {
          buffer.write(data[i], invert);
        }
        return;
      }

View Full Code Here

public class LazyTimestamp extends LazyPrimitive<LazyTimestampObjectInspector, TimestampWritable> {
  static final private Log LOG = LogFactory.getLog(LazyTimestamp.class);


  public LazyTimestamp(LazyTimestampObjectInspector oi) {
    super(oi);
    data = new TimestampWritable();
  }

View Full Code Here

    data = new TimestampWritable();
  }


  public LazyTimestamp(LazyTimestamp copy) {
    super(copy);
    data = new TimestampWritable(copy.data);
  }

View Full Code Here

      }
      case BINARY:
        return ((BinaryObjectInspector) poi).getPrimitiveWritableObject(o).hashCode();


      case TIMESTAMP:
        TimestampWritable t = ((TimestampObjectInspector) poi)
            .getPrimitiveWritableObject(o);
        return t.hashCode();
      case DECIMAL:
        return ((HiveDecimalObjectInspector) poi).getPrimitiveWritableObject(o).hashCode();


      default: {
        throw new RuntimeException("Unknown type: "

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.hive.serde2.io.TimestampWritable

com.asakusafw.directio.hive.orc.OrcFileInput

com.asakusafw.directio.hive.serde.DataModelDriver

com.asakusafw.directio.hive.serde.DataModelDriverTest

com.asakusafw.directio.hive.serde.ValueSerdeFactoryTest

com.facebook.hive.orc.lazy.LazyDoubleTreeReader

com.facebook.hive.orc.lazy.LazyShortDictionaryTreeReader

com.facebook.hive.orc.lazy.LazyShortDirectTreeReader

com.facebook.hive.orc.lazy.OrcLazyStructObjectInspector

com.facebook.hive.orc.lazy.OrcLazyTimestamp

com.facebook.hive.orc.OrcOutputFormat$OrcRecordWriter

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.