Examples of org.apache.hadoop.hive.serde2.SerDe

org.apache.hadoop.hive.serde2.SerDe
A union of HiveDeserializer and HiveSerializer interface. If a developer wants his hive table to be read-only, then he just want to return both readable and writable, then

    StructObjectInspector rowOI1 = (StructObjectInspector) ObjectInspectorFactory
        .getReflectionObjectInspector(MyTestClass.class,
        ObjectInspectorOptions.JAVA);
    String fieldNames1 = ObjectInspectorUtils.getFieldNames(rowOI1);
    String fieldTypes1 = ObjectInspectorUtils.getFieldTypes(rowOI1);
    SerDe serde1 = getSerDe(fieldNames1, fieldTypes1);
    serde1.getObjectInspector();


    StructObjectInspector rowOI2 = (StructObjectInspector) ObjectInspectorFactory
        .getReflectionObjectInspector(MyTestClassSmaller.class,
        ObjectInspectorOptions.JAVA);
    String fieldNames2 = ObjectInspectorUtils.getFieldNames(rowOI2);
    String fieldTypes2 = ObjectInspectorUtils.getFieldTypes(rowOI2);
    SerDe serde2 = getSerDe(fieldNames2, fieldTypes2);
    ObjectInspector serdeOI2 = serde2.getObjectInspector();


    int num = 100;
    for (int itest = 0; itest < num; itest++) {
      int randField = r.nextInt(12);
      Byte b = randField > 0 ? null : Byte.valueOf((byte) r.nextInt());
      Short s = randField > 1 ? null : Short.valueOf((short) r.nextInt());
      Integer n = randField > 2 ? null : Integer.valueOf(r.nextInt());
      Long l = randField > 3 ? null : Long.valueOf(r.nextLong());
      Float f = randField > 4 ? null : Float.valueOf(r.nextFloat());
      Double d = randField > 5 ? null : Double.valueOf(r.nextDouble());
      String st = randField > 6 ? null : TestBinarySortableSerDe
          .getRandString(r);
      HiveDecimal bd = randField > 7 ? null : TestBinarySortableSerDe.getRandHiveDecimal(r);
      Date date = randField > 8 ? null : TestBinarySortableSerDe.getRandDate(r);
      MyTestInnerStruct is = randField > 9 ? null : new MyTestInnerStruct(r
          .nextInt(5) - 2, r.nextInt(5) - 2);
      List<Integer> li = randField > 10 ? null : TestBinarySortableSerDe
          .getRandIntegerArray(r);
      byte[] ba = TestBinarySortableSerDe.getRandBA(r, itest);
      MyTestClass input = new MyTestClass(b, s, n, l, f, d, st, bd, date, is, li, ba);
      BytesWritable bw = (BytesWritable) serde1.serialize(input, rowOI1);
      Object output = serde2.deserialize(bw);


      if (0 != compareDiffSizedStructs(input, rowOI1, output, serdeOI2)) {
        System.out.println("structs      = "
            + SerDeUtils.getJSONString(input, rowOI1));
        System.out.println("deserialized = "

View Full Code Here

    StructObjectInspector rowOI1 = (StructObjectInspector) ObjectInspectorFactory
        .getReflectionObjectInspector(MyTestClass.class,
        ObjectInspectorOptions.JAVA);
    String fieldNames1 = ObjectInspectorUtils.getFieldNames(rowOI1);
    String fieldTypes1 = ObjectInspectorUtils.getFieldTypes(rowOI1);
    SerDe serde1 = getSerDe(fieldNames1, fieldTypes1);
    serde1.getObjectInspector();


    StructObjectInspector rowOI2 = (StructObjectInspector) ObjectInspectorFactory
        .getReflectionObjectInspector(MyTestClassBigger.class,
        ObjectInspectorOptions.JAVA);
    String fieldNames2 = ObjectInspectorUtils.getFieldNames(rowOI2);
    String fieldTypes2 = ObjectInspectorUtils.getFieldTypes(rowOI2);
    SerDe serde2 = getSerDe(fieldNames2, fieldTypes2);
    ObjectInspector serdeOI2 = serde2.getObjectInspector();


    int num = 100;
    for (int itest = 0; itest < num; itest++) {
      int randField = r.nextInt(12);
      Byte b = randField > 0 ? null : Byte.valueOf((byte) r.nextInt());
      Short s = randField > 1 ? null : Short.valueOf((short) r.nextInt());
      Integer n = randField > 2 ? null : Integer.valueOf(r.nextInt());
      Long l = randField > 3 ? null : Long.valueOf(r.nextLong());
      Float f = randField > 4 ? null : Float.valueOf(r.nextFloat());
      Double d = randField > 5 ? null : Double.valueOf(r.nextDouble());
      String st = randField > 6 ? null : TestBinarySortableSerDe
          .getRandString(r);
      HiveDecimal bd = randField > 7 ? null : TestBinarySortableSerDe.getRandHiveDecimal(r);
      Date date = randField > 8 ? null : TestBinarySortableSerDe.getRandDate(r);
      MyTestInnerStruct is = randField > 9 ? null : new MyTestInnerStruct(r
          .nextInt(5) - 2, r.nextInt(5) - 2);
      List<Integer> li = randField > 10 ? null : TestBinarySortableSerDe
          .getRandIntegerArray(r);
      byte[] ba = TestBinarySortableSerDe.getRandBA(r, itest);
      MyTestClass input = new MyTestClass(b, s, n, l, f, d, st, bd, date, is, li,ba);
      BytesWritable bw = (BytesWritable) serde1.serialize(input, rowOI1);
      Object output = serde2.deserialize(bw);


      if (0 != compareDiffSizedStructs(input, rowOI1, output, serdeOI2)) {
        System.out.println("structs      = "
            + SerDeUtils.getJSONString(input, rowOI1));
        System.out.println("deserialized = "

View Full Code Here

    StructObjectInspector rowOI1 = (StructObjectInspector) ObjectInspectorFactory
        .getReflectionObjectInspector(MyTestClassSmaller.class,
        ObjectInspectorOptions.JAVA);
    String fieldNames1 = ObjectInspectorUtils.getFieldNames(rowOI1);
    String fieldTypes1 = ObjectInspectorUtils.getFieldTypes(rowOI1);
    SerDe serde1 = getSerDe(fieldNames1, fieldTypes1);
    serde1.getObjectInspector();


    StructObjectInspector rowOI2 = (StructObjectInspector) ObjectInspectorFactory
        .getReflectionObjectInspector(MyTestClass.class,
        ObjectInspectorOptions.JAVA);
    String fieldNames2 = ObjectInspectorUtils.getFieldNames(rowOI2);
    String fieldTypes2 = ObjectInspectorUtils.getFieldTypes(rowOI2);
    SerDe serde2 = getSerDe(fieldNames2, fieldTypes2);
    ObjectInspector serdeOI2 = serde2.getObjectInspector();


    int num = 100;
    for (int itest = 0; itest < num; itest++) {
      int randField = r.nextInt(9);
      Byte b = randField > 0 ? null : Byte.valueOf((byte) r.nextInt());
      Short s = randField > 1 ? null : Short.valueOf((short) r.nextInt());
      Integer n = randField > 2 ? null : Integer.valueOf(r.nextInt());
      Long l = randField > 3 ? null : Long.valueOf(r.nextLong());
      Float f = randField > 4 ? null : Float.valueOf(r.nextFloat());
      Double d = randField > 5 ? null : Double.valueOf(r.nextDouble());
      String st = randField > 6 ? null : TestBinarySortableSerDe
          .getRandString(r);
      HiveDecimal bd = randField > 7 ? null : TestBinarySortableSerDe.getRandHiveDecimal(r);
      Date date = randField > 7 ? null : TestBinarySortableSerDe.getRandDate(r);
      MyTestInnerStruct is = randField > 7 ? null : new MyTestInnerStruct(r
          .nextInt(5) - 2, r.nextInt(5) - 2);


      MyTestClassSmaller input = new MyTestClassSmaller(b, s, n, l, f, d, st, bd, date,
          is);
      BytesWritable bw = (BytesWritable) serde1.serialize(input, rowOI1);
      Object output = serde2.deserialize(bw);


      if (0 != compareDiffSizedStructs(input, rowOI1, output, serdeOI2)) {
        System.out.println("structs      = "
            + SerDeUtils.getJSONString(input, rowOI1));
        System.out.println("deserialized = "

View Full Code Here

    StructObjectInspector rowOI = (StructObjectInspector) ObjectInspectorFactory
        .getReflectionObjectInspector(MyTestClassBigger.class,
        ObjectInspectorOptions.JAVA);
    String fieldNames = ObjectInspectorUtils.getFieldNames(rowOI);
    String fieldTypes = ObjectInspectorUtils.getFieldTypes(rowOI);
    SerDe serde = getSerDe(fieldNames, fieldTypes);
    ObjectInspector serdeOI = serde.getObjectInspector();


    StructObjectInspector soi1 = (StructObjectInspector) serdeOI;
    List<? extends StructField> fields1 = soi1.getAllStructFieldRefs();
    LazyBinaryMapObjectInspector lazympoi = (LazyBinaryMapObjectInspector) fields1
        .get(12).getFieldObjectInspector();
    ObjectInspector lazympkeyoi = lazympoi.getMapKeyObjectInspector();
    ObjectInspector lazympvalueoi = lazympoi.getMapValueObjectInspector();


    StructObjectInspector soi2 = rowOI;
    List<? extends StructField> fields2 = soi2.getAllStructFieldRefs();
    MapObjectInspector inputmpoi = (MapObjectInspector) fields2.get(12)
        .getFieldObjectInspector();
    ObjectInspector inputmpkeyoi = inputmpoi.getMapKeyObjectInspector();
    ObjectInspector inputmpvalueoi = inputmpoi.getMapValueObjectInspector();


    int num = 100;
    for (int testi = 0; testi < num; testi++) {


      Map<String, List<MyTestInnerStruct>> mp = new LinkedHashMap<String, List<MyTestInnerStruct>>();


      int randFields = r.nextInt(10);
      for (int i = 0; i < randFields; i++) {
        String key = TestBinarySortableSerDe.getRandString(r);
        int randField = r.nextInt(10);
        List<MyTestInnerStruct> value = randField > 4 ? null
            : getRandStructArray(r);
        mp.put(key, value);
      }


      MyTestClassBigger input = new MyTestClassBigger(null, null, null, null,
                  null, null, null, null, null, null, null, null, mp);
      BytesWritable bw = (BytesWritable) serde.serialize(input, rowOI);
      Object output = serde.deserialize(bw);
      Object lazyobj = soi1.getStructFieldData(output, fields1.get(12));
      Map<?, ?> outputmp = lazympoi.getMap(lazyobj);


      if (outputmp.size() != mp.size()) {
        throw new RuntimeException("Map size changed from " + mp.size()

View Full Code Here

      BytesWritable value, boolean allowLazy) throws HiveException {
    if (allowLazy) {
      addLazy(context, value);
      return;
    }
    SerDe serde = context.getSerDe();
    boolean hasValues = isLazy() ? setRowLength(serde, 0) : (rowLength > 0);
    int rowCount = rowCount();
    if (hasValues) {
      listRealloc(array.length + rowLength);
      read(serde, value, rowCount);

View Full Code Here

      return;
    }
    Object[] lazyObjs = this.array;
    assert lazyObjs.length > 1;
    MapJoinObjectSerDeContext context = (MapJoinObjectSerDeContext)lazyObjs[0];
    SerDe serde = context.getSerDe();
    int rowCount = lazyObjs.length - 1;
    if (!setRowLength(serde, rowCount)) return;


    this.array = new Object[rowLength * rowCount];
    ByteBufferWritable writable = new ByteBufferWritable();

View Full Code Here

   * @param in Input stream.
   * @return Loaded table.
   */
  public MapJoinTableContainer load(ObjectInputStream in) 
      throws HiveException {
    SerDe keySerDe = keyContext.getSerDe();
    SerDe valueSerDe = valueContext.getSerDe();
    MapJoinTableContainer tableContainer;
    try {
      String name = in.readUTF();
      Map<String, String> metaData = (Map<String, String>) in.readObject();
      tableContainer = create(name, metaData);
    } catch (IOException e) {
      throw new HiveException("IO error while trying to create table container", e);
    } catch (ClassNotFoundException e) {
      throw new HiveException("Class Initialization error while trying to create table container", e);
    }
    try {
      Writable keyContainer = keySerDe.getSerializedClass().newInstance();
      Writable valueContainer = valueSerDe.getSerializedClass().newInstance();    
      int numKeys = in.readInt();
      for (int keyIndex = 0; keyIndex < numKeys; keyIndex++) {
        MapJoinKeyObject key = new MapJoinKeyObject();
        key.read(keyContext, in, keyContainer);
        MapJoinEagerRowContainer values = new MapJoinEagerRowContainer();

View Full Code Here

  private static final byte[] EMPTY_BYTE_ARRAY = new byte[0];


  @SuppressWarnings("deprecation")
  public static MapJoinKey read(Output output, MapJoinKey key, MapJoinObjectSerDeContext context,
      Writable writable, boolean mayReuseKey) throws SerDeException {
    SerDe serde = context.getSerDe();
    Object obj = serde.deserialize(writable);
    boolean useOptimized = useOptimizedKeyBasedOnPrev(key);
    if (useOptimized || key == null) {
      byte[] structBytes = serializeKey(output, obj, serde.getObjectInspector(), !useOptimized);
      if (structBytes != null) {
        return MapJoinKeyBytes.fromBytes(key, mayReuseKey, structBytes);
      } else if (useOptimized) {
        throw new SerDeException(
            "Failed to serialize " + obj + " even though optimized keys are used");
      }
    }
    MapJoinKeyObject result = mayReuseKey ? (MapJoinKeyObject)key : new MapJoinKeyObject();
    result.read(serde.getObjectInspector(), obj);
    return result;
  }

View Full Code Here

    }
  }


  @SuppressWarnings("unchecked")
  public void read(MapJoinObjectSerDeContext context, Writable currentValue) throws SerDeException {
    SerDe serde = context.getSerDe();
    List<Object> value = (List<Object>)ObjectInspectorUtils.copyToStandardObject(serde.deserialize(currentValue),
        serde.getObjectInspector(), ObjectInspectorCopyOption.WRITABLE);
    if(value == null) {
      addRow(toList(EMPTY_OBJECT_ARRAY));
    } else {
      Object[] valuesArray = value.toArray();
      if (context.hasFilterTag()) {

View Full Code Here

  }


  @Override
  public void write(MapJoinObjectSerDeContext context, ObjectOutputStream out)
  throws IOException, SerDeException {
    SerDe serde = context.getSerDe();
    ObjectInspector valueObjectInspector = context.getStandardOI();
    long numRows = rowCount();
    long numRowsWritten = 0L;
    out.writeLong(numRows);
    for (List<Object> row = first(); row != null; row = next()) {
      serde.serialize(row.toArray(), valueObjectInspector).write(out);
      ++numRowsWritten;      
    }
    if(numRows != rowCount()) {
      throw new ConcurrentModificationException("Values was modifified while persisting");
    }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.hive.serde2.SerDe

com.asakusafw.directio.hive.orc.OrcFileInput

com.asakusafw.directio.hive.serde.DataModelDriver

com.asakusafw.directio.hive.serde.DataModelDriverTest

com.asakusafw.directio.hive.serde.ValueSerdeFactoryTest

com.facebook.hive.orc.lazy.LazyDoubleTreeReader

com.facebook.hive.orc.lazy.LazyShortDictionaryTreeReader

com.facebook.hive.orc.lazy.LazyShortDirectTreeReader

com.facebook.hive.orc.lazy.OrcLazyStructObjectInspector

com.facebook.hive.orc.OrcOutputFormat$OrcRecordWriter

com.facebook.hive.orc.OrcStruct$OrcStructInspector

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.