Examples of org.apache.hadoop.hive.serde2.SerDe

org.apache.hadoop.hive.serde2.SerDe
A union of HiveDeserializer and HiveSerializer interface. If a developer wants his hive table to be read-only, then he just want to return both readable and writable, then

    StructObjectInspector rowOI1 = (StructObjectInspector) ObjectInspectorFactory
        .getReflectionObjectInspector(MyTestClass.class,
        ObjectInspectorOptions.JAVA);
    String fieldNames1 = ObjectInspectorUtils.getFieldNames(rowOI1);
    String fieldTypes1 = ObjectInspectorUtils.getFieldTypes(rowOI1);
    SerDe serde1 = getSerDe(fieldNames1, fieldTypes1);
    serde1.getObjectInspector();


    StructObjectInspector rowOI2 = (StructObjectInspector) ObjectInspectorFactory
        .getReflectionObjectInspector(MyTestClassSmaller.class,
        ObjectInspectorOptions.JAVA);
    String fieldNames2 = ObjectInspectorUtils.getFieldNames(rowOI2);
    String fieldTypes2 = ObjectInspectorUtils.getFieldTypes(rowOI2);
    SerDe serde2 = getSerDe(fieldNames2, fieldTypes2);
    ObjectInspector serdeOI2 = serde2.getObjectInspector();


    int num = 100;
    for (int itest = 0; itest < num; itest++) {
      int randField = r.nextInt(10);
      Byte b = randField > 0 ? null : Byte.valueOf((byte) r.nextInt());
      Short s = randField > 1 ? null : Short.valueOf((short) r.nextInt());
      Integer n = randField > 2 ? null : Integer.valueOf(r.nextInt());
      Long l = randField > 3 ? null : Long.valueOf(r.nextLong());
      Float f = randField > 4 ? null : Float.valueOf(r.nextFloat());
      Double d = randField > 5 ? null : Double.valueOf(r.nextDouble());
      String st = randField > 6 ? null : TestBinarySortableSerDe
          .getRandString(r);
      MyTestInnerStruct is = randField > 7 ? null : new MyTestInnerStruct(r
          .nextInt(5) - 2, r.nextInt(5) - 2);
      List<Integer> li = randField > 8 ? null : TestBinarySortableSerDe
          .getRandIntegerArray(r);
      byte[] ba = TestBinarySortableSerDe.getRandBA(r, itest);
      MyTestClass input = new MyTestClass(b, s, n, l, f, d, st, is, li, ba);
      BytesWritable bw = (BytesWritable) serde1.serialize(input, rowOI1);
      Object output = serde2.deserialize(bw);


      if (0 != compareDiffSizedStructs(input, rowOI1, output, serdeOI2)) {
        System.out.println("structs      = "
            + SerDeUtils.getJSONString(input, rowOI1));
        System.out.println("deserialized = "

View Full Code Here

    StructObjectInspector rowOI1 = (StructObjectInspector) ObjectInspectorFactory
        .getReflectionObjectInspector(MyTestClass.class,
        ObjectInspectorOptions.JAVA);
    String fieldNames1 = ObjectInspectorUtils.getFieldNames(rowOI1);
    String fieldTypes1 = ObjectInspectorUtils.getFieldTypes(rowOI1);
    SerDe serde1 = getSerDe(fieldNames1, fieldTypes1);
    serde1.getObjectInspector();


    StructObjectInspector rowOI2 = (StructObjectInspector) ObjectInspectorFactory
        .getReflectionObjectInspector(MyTestClassBigger.class,
        ObjectInspectorOptions.JAVA);
    String fieldNames2 = ObjectInspectorUtils.getFieldNames(rowOI2);
    String fieldTypes2 = ObjectInspectorUtils.getFieldTypes(rowOI2);
    SerDe serde2 = getSerDe(fieldNames2, fieldTypes2);
    ObjectInspector serdeOI2 = serde2.getObjectInspector();


    int num = 100;
    for (int itest = 0; itest < num; itest++) {
      int randField = r.nextInt(10);
      Byte b = randField > 0 ? null : Byte.valueOf((byte) r.nextInt());
      Short s = randField > 1 ? null : Short.valueOf((short) r.nextInt());
      Integer n = randField > 2 ? null : Integer.valueOf(r.nextInt());
      Long l = randField > 3 ? null : Long.valueOf(r.nextLong());
      Float f = randField > 4 ? null : Float.valueOf(r.nextFloat());
      Double d = randField > 5 ? null : Double.valueOf(r.nextDouble());
      String st = randField > 6 ? null : TestBinarySortableSerDe
          .getRandString(r);
      MyTestInnerStruct is = randField > 7 ? null : new MyTestInnerStruct(r
          .nextInt(5) - 2, r.nextInt(5) - 2);
      List<Integer> li = randField > 8 ? null : TestBinarySortableSerDe
          .getRandIntegerArray(r);
      byte[] ba = TestBinarySortableSerDe.getRandBA(r, itest);
      MyTestClass input = new MyTestClass(b, s, n, l, f, d, st, is, li,ba);
      BytesWritable bw = (BytesWritable) serde1.serialize(input, rowOI1);
      Object output = serde2.deserialize(bw);


      if (0 != compareDiffSizedStructs(input, rowOI1, output, serdeOI2)) {
        System.out.println("structs      = "
            + SerDeUtils.getJSONString(input, rowOI1));
        System.out.println("deserialized = "

View Full Code Here

    StructObjectInspector rowOI1 = (StructObjectInspector) ObjectInspectorFactory
        .getReflectionObjectInspector(MyTestClassSmaller.class,
        ObjectInspectorOptions.JAVA);
    String fieldNames1 = ObjectInspectorUtils.getFieldNames(rowOI1);
    String fieldTypes1 = ObjectInspectorUtils.getFieldTypes(rowOI1);
    SerDe serde1 = getSerDe(fieldNames1, fieldTypes1);
    serde1.getObjectInspector();


    StructObjectInspector rowOI2 = (StructObjectInspector) ObjectInspectorFactory
        .getReflectionObjectInspector(MyTestClass.class,
        ObjectInspectorOptions.JAVA);
    String fieldNames2 = ObjectInspectorUtils.getFieldNames(rowOI2);
    String fieldTypes2 = ObjectInspectorUtils.getFieldTypes(rowOI2);
    SerDe serde2 = getSerDe(fieldNames2, fieldTypes2);
    ObjectInspector serdeOI2 = serde2.getObjectInspector();


    int num = 100;
    for (int itest = 0; itest < num; itest++) {
      int randField = r.nextInt(9);
      Byte b = randField > 0 ? null : Byte.valueOf((byte) r.nextInt());
      Short s = randField > 1 ? null : Short.valueOf((short) r.nextInt());
      Integer n = randField > 2 ? null : Integer.valueOf(r.nextInt());
      Long l = randField > 3 ? null : Long.valueOf(r.nextLong());
      Float f = randField > 4 ? null : Float.valueOf(r.nextFloat());
      Double d = randField > 5 ? null : Double.valueOf(r.nextDouble());
      String st = randField > 6 ? null : TestBinarySortableSerDe
          .getRandString(r);
      MyTestInnerStruct is = randField > 7 ? null : new MyTestInnerStruct(r
          .nextInt(5) - 2, r.nextInt(5) - 2);


      MyTestClassSmaller input = new MyTestClassSmaller(b, s, n, l, f, d, st,
          is);
      BytesWritable bw = (BytesWritable) serde1.serialize(input, rowOI1);
      Object output = serde2.deserialize(bw);


      if (0 != compareDiffSizedStructs(input, rowOI1, output, serdeOI2)) {
        System.out.println("structs      = "
            + SerDeUtils.getJSONString(input, rowOI1));
        System.out.println("deserialized = "

View Full Code Here

    StructObjectInspector rowOI = (StructObjectInspector) ObjectInspectorFactory
        .getReflectionObjectInspector(MyTestClassBigger.class,
        ObjectInspectorOptions.JAVA);
    String fieldNames = ObjectInspectorUtils.getFieldNames(rowOI);
    String fieldTypes = ObjectInspectorUtils.getFieldTypes(rowOI);
    SerDe serde = getSerDe(fieldNames, fieldTypes);
    ObjectInspector serdeOI = serde.getObjectInspector();


    StructObjectInspector soi1 = (StructObjectInspector) serdeOI;
    List<? extends StructField> fields1 = soi1.getAllStructFieldRefs();
    LazyBinaryMapObjectInspector lazympoi = (LazyBinaryMapObjectInspector) fields1
        .get(10).getFieldObjectInspector();
    ObjectInspector lazympkeyoi = lazympoi.getMapKeyObjectInspector();
    ObjectInspector lazympvalueoi = lazympoi.getMapValueObjectInspector();


    StructObjectInspector soi2 = rowOI;
    List<? extends StructField> fields2 = soi2.getAllStructFieldRefs();
    MapObjectInspector inputmpoi = (MapObjectInspector) fields2.get(10)
        .getFieldObjectInspector();
    ObjectInspector inputmpkeyoi = inputmpoi.getMapKeyObjectInspector();
    ObjectInspector inputmpvalueoi = inputmpoi.getMapValueObjectInspector();


    int num = 100;
    for (int testi = 0; testi < num; testi++) {


      Map<String, List<MyTestInnerStruct>> mp = new LinkedHashMap<String, List<MyTestInnerStruct>>();


      int randFields = r.nextInt(10);
      for (int i = 0; i < randFields; i++) {
        String key = TestBinarySortableSerDe.getRandString(r);
        int randField = r.nextInt(10);
        List<MyTestInnerStruct> value = randField > 4 ? null
            : getRandStructArray(r);
        mp.put(key, value);
      }


      MyTestClassBigger input = new MyTestClassBigger(null, null, null, null,
          null, null, null, null, null, null, mp);
      BytesWritable bw = (BytesWritable) serde.serialize(input, rowOI);
      Object output = serde.deserialize(bw);
      Object lazyobj = soi1.getStructFieldData(output, fields1.get(10));
      Map<?, ?> outputmp = lazympoi.getMap(lazyobj);


      if (outputmp.size() != mp.size()) {
        throw new RuntimeException("Map size changed from " + mp.size()

View Full Code Here

            .deserialize(jobInfoString);
        StorerInfo storeInfo = jobInfo.getTableInfo().getStorerInfo();
        HCatStorageHandler storageHandler = HCatUtil.getStorageHandler(
            context.getConfiguration(), storeInfo);
        Class<? extends SerDe> serde = storageHandler.getSerDeClass();
        SerDe sd = (SerDe) ReflectionUtils.newInstance(serde,
            context.getConfiguration());
        context.getConfiguration().set("mapred.output.value.class",
            sd.getSerializedClass().getName());


        // When Dynamic partitioning is used, the RecordWriter instance initialized here isn't used. Can use null.
        // (That's because records can't be written until the values of the dynamic partitions are deduced.
        // By that time, a new local instance of RecordWriter, with the correct output-path, will be constructed.)
        RecordWriter<WritableComparable<?>, HCatRecord> rw =

View Full Code Here

    public void write(WritableComparable<?> key, HCatRecord value) throws IOException,
        InterruptedException {


        org.apache.hadoop.mapred.RecordWriter localWriter;
        ObjectInspector localObjectInspector;
        SerDe localSerDe;
        OutputJobInfo localJobInfo = null;


        if (dynamicPartitioningUsed) {
            // calculate which writer to use from the remaining values - this needs to be done before we delete cols
            List<String> dynamicPartValues = new ArrayList<String>();
            for (Integer colToAppend : dynamicPartCols) {
                dynamicPartValues.add(value.get(colToAppend).toString());
            }


            String dynKey = dynamicPartValues.toString();
            if (!baseDynamicWriters.containsKey(dynKey)) {
                if ((maxDynamicPartitions != -1) && (baseDynamicWriters.size() > maxDynamicPartitions)) {
                    throw new HCatException(ErrorType.ERROR_TOO_MANY_DYNAMIC_PTNS,
                        "Number of dynamic partitions being created "
                            + "exceeds configured max allowable partitions["
                            + maxDynamicPartitions
                            + "], increase parameter ["
                            + HiveConf.ConfVars.DYNAMICPARTITIONMAXPARTS.varname
                            + "] if needed.");
                }


                org.apache.hadoop.mapred.TaskAttemptContext currTaskContext = HCatMapRedUtil.createTaskAttemptContext(context);
                configureDynamicStorageHandler(currTaskContext, dynamicPartValues);
                localJobInfo = HCatBaseOutputFormat.getJobInfo(currTaskContext);


                //setup serDe
                SerDe currSerDe = ReflectionUtils.newInstance(storageHandler.getSerDeClass(), currTaskContext.getJobConf());
                try {
                    InternalUtil.initializeOutputSerDe(currSerDe, currTaskContext.getConfiguration(), localJobInfo);
                } catch (SerDeException e) {
                    throw new IOException("Failed to initialize SerDe", e);
                }

View Full Code Here

      boolean noFilter) {
    TableDesc desc = getSpillTableDesc(alias, spillTableDesc, conf, noFilter);
    if (desc == null) {
      return null;
    }
    SerDe sd = (SerDe) ReflectionUtils.newInstance(desc.getDeserializerClass(),
        null);
    try {
      sd.initialize(null, desc.getProperties());
    } catch (SerDeException e) {
      e.printStackTrace();
      return null;
    }
    return sd;

View Full Code Here

      List<ObjectInspector> structFieldObjectInspectors,
      Byte alias,int containerSize, Map<Byte, TableDesc> spillTableDesc,
      JoinDesc conf,boolean noFilter, Reporter reporter) throws HiveException {


    TableDesc tblDesc = JoinUtil.getSpillTableDesc(alias,spillTableDesc,conf, noFilter);
    SerDe serde = JoinUtil.getSpillSerDe(alias, spillTableDesc, conf, noFilter);


    if (serde == null) {
      containerSize = -1;
    }

View Full Code Here


  public void generateMapMetaData() throws HiveException, SerDeException {
    // generate the meta data for key
    // index for key is -1
    TableDesc keyTableDesc = conf.getKeyTblDesc();
    SerDe keySerializer = (SerDe) ReflectionUtils.newInstance(keyTableDesc.getDeserializerClass(),
        null);
    keySerializer.initialize(null, keyTableDesc.getProperties());
    MapJoinMetaData.put(Integer.valueOf(metadataKeyTag), new HashTableSinkObjectCtx(
        ObjectInspectorUtils.getStandardObjectInspector(keySerializer.getObjectInspector(),
            ObjectInspectorCopyOption.WRITABLE), keySerializer, keyTableDesc, hconf));


    // index for values is just alias
    for (int tag = 0; tag < order.length; tag++) {
      int alias = (int) order[tag];


      if (alias == this.bigTableAlias) {
        continue;
      }




      TableDesc valueTableDesc;
      if (conf.getNoOuterJoin()) {
        valueTableDesc = conf.getValueTblDescs().get(tag);
      } else {
        valueTableDesc = conf.getValueFilteredTblDescs().get(tag);
      }
      SerDe valueSerDe = (SerDe) ReflectionUtils.newInstance(valueTableDesc.getDeserializerClass(),
          null);
      valueSerDe.initialize(null, valueTableDesc.getProperties());


      MapJoinMetaData.put(Integer.valueOf(alias), new HashTableSinkObjectCtx(ObjectInspectorUtils
          .getStandardObjectInspector(valueSerDe.getObjectInspector(),
              ObjectInspectorCopyOption.WRITABLE), valueSerDe, valueTableDesc, hconf));
    }
  }

View Full Code Here

          .getProperties());
      StructObjectInspector structTblKeyInpector = ObjectInspectorFactory
          .getStandardStructObjectInspector(keyColNames, skewTableKeyInspectors);


      try {
        SerDe serializer = (SerDe) ReflectionUtils.newInstance(tblDesc.get(
            alias).getDeserializerClass(), null);
        serializer.initialize(null, tblDesc.get(alias).getProperties());
        tblSerializers.put((byte) i, serializer);
      } catch (SerDeException e) {
        LOG.error("Skewjoin will be disabled due to " + e.getMessage(), e);
        joinOp.handleSkewJoin = false;
        break;

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.hive.serde2.SerDe

com.asakusafw.directio.hive.orc.OrcFileInput

com.asakusafw.directio.hive.serde.DataModelDriver

com.asakusafw.directio.hive.serde.DataModelDriverTest

com.asakusafw.directio.hive.serde.ValueSerdeFactoryTest

com.facebook.hive.orc.lazy.LazyDoubleTreeReader

com.facebook.hive.orc.lazy.LazyShortDictionaryTreeReader

com.facebook.hive.orc.lazy.LazyShortDirectTreeReader

com.facebook.hive.orc.lazy.OrcLazyStructObjectInspector

com.facebook.hive.orc.OrcOutputFormat$OrcRecordWriter

com.facebook.hive.orc.OrcStruct$OrcStructInspector

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.