Examples of org.apache.hadoop.hive.serde2.lazy.ByteArrayRef

Package org.apache.hadoop.hive.serde2.lazy

Examples of org.apache.hadoop.hive.serde2.lazy.ByteArrayRef

org.apache.hadoop.hive.serde2.lazy.ByteArrayRef
ByteArrayRef stores a reference to a byte array. The LazyObject hierarchy uses a reference to a single ByteArrayRef, so that it's much faster to switch to the next row and release the reference to the old row (so that the system can do garbage collection if needed).

            .getRandString(r);
        MyTestInnerStruct is = randField > 7 ? null : new MyTestInnerStruct(r
            .nextInt(5) - 2, r.nextInt(5) - 2);
        List<Integer> li = randField > 8 ? null : TestBinarySortableSerDe
            .getRandIntegerArray(r);
        ByteArrayRef ba = TestBinarySortableSerDe.getRandBA(r, i);
        MyTestClass t = new MyTestClass(b, s, n, l, f, d, st, is, li,ba);
        rows[i] = t;
      }


      StructObjectInspector rowOI = (StructObjectInspector) ObjectInspectorFactory

View Full Code Here

          .getRandString(r);
      MyTestInnerStruct is = randField > 7 ? null : new MyTestInnerStruct(r
          .nextInt(5) - 2, r.nextInt(5) - 2);
      List<Integer> li = randField > 8 ? null : TestBinarySortableSerDe
          .getRandIntegerArray(r);
      ByteArrayRef ba  = TestBinarySortableSerDe.getRandBA(r, itest);
      Map<String, List<MyTestInnerStruct>> mp = new HashMap<String, List<MyTestInnerStruct>>();
      String key = TestBinarySortableSerDe.getRandString(r);
      List<MyTestInnerStruct> value = randField > 10 ? null
          : getRandStructArray(r);
      mp.put(key, value);

View Full Code Here

          .getRandString(r);
      MyTestInnerStruct is = randField > 7 ? null : new MyTestInnerStruct(r
          .nextInt(5) - 2, r.nextInt(5) - 2);
      List<Integer> li = randField > 8 ? null : TestBinarySortableSerDe
          .getRandIntegerArray(r);
      ByteArrayRef ba = TestBinarySortableSerDe.getRandBA(r, itest);
      MyTestClass input = new MyTestClass(b, s, n, l, f, d, st, is, li, ba);
      BytesWritable bw = (BytesWritable) serde1.serialize(input, rowOI1);
      Object output = serde2.deserialize(bw);


      if (0 != compareDiffSizedStructs(input, rowOI1, output, serdeOI2)) {

View Full Code Here

          .getRandString(r);
      MyTestInnerStruct is = randField > 7 ? null : new MyTestInnerStruct(r
          .nextInt(5) - 2, r.nextInt(5) - 2);
      List<Integer> li = randField > 8 ? null : TestBinarySortableSerDe
          .getRandIntegerArray(r);
      ByteArrayRef ba = TestBinarySortableSerDe.getRandBA(r, itest);
      MyTestClass input = new MyTestClass(b, s, n, l, f, d, st, is, li,ba);
      BytesWritable bw = (BytesWritable) serde1.serialize(input, rowOI1);
      Object output = serde2.deserialize(bw);


      if (0 != compareDiffSizedStructs(input, rowOI1, output, serdeOI2)) {

View Full Code Here

            .getRandString(r);
        MyTestInnerStruct is = randField > 7 ? null : new MyTestInnerStruct(r
            .nextInt(5) - 2, r.nextInt(5) - 2);
        List<Integer> li = randField > 8 ? null : TestBinarySortableSerDe
            .getRandIntegerArray(r);
        ByteArrayRef ba = TestBinarySortableSerDe.getRandBA(r, i);
        MyTestClass t = new MyTestClass(b, s, n, l, f, d, st, is, li, ba);
        rows[i] = t;
      }


      StructObjectInspector rowOI = (StructObjectInspector) ObjectInspectorFactory

View Full Code Here

    boolean fieldSkipped;
    ObjectInspector objectInspector;


    public FieldInfo(LazyObjectBase lazyObject, boolean fieldSkipped, ObjectInspector oi) {
      field = lazyObject;
      cachedByteArrayRef = new ByteArrayRef();
      objectInspector = oi;
      if (fieldSkipped) {
        this.fieldSkipped = true;
        inited = true;
      } else {

View Full Code Here

        }
        else if (hiveTypes[column] == HiveType.MAP || hiveTypes[column] == HiveType.LIST || hiveTypes[column] == HiveType.STRUCT) {
            // temporarily special case MAP, LIST, and STRUCT types as strings
            // TODO: create a real parser for these complex types when we implement data types
            LazyObject<? extends ObjectInspector> lazyObject = LazyFactory.createLazyObject(fieldInspectors[column]);
            ByteArrayRef byteArrayRef = new ByteArrayRef();
            byteArrayRef.setData(bytes);
            lazyObject.init(byteArrayRef, start, length);
            strings[column] = SerDeUtils.getJsonBytes(lazyObject.getObject(), fieldInspectors[column]);
            wasNull = false;
        }
        else {

View Full Code Here

        }
        else if (hiveTypes[column] == HiveType.MAP || hiveTypes[column] == HiveType.LIST || hiveTypes[column] == HiveType.STRUCT) {
            // temporarily special case MAP, LIST, and STRUCT types as strings
            // TODO: create a real parser for these complex types when we implement data types
            LazyObject<? extends ObjectInspector> lazyObject = LazyFactory.createLazyObject(fieldInspectors[column]);
            ByteArrayRef byteArrayRef = new ByteArrayRef();
            byteArrayRef.setData(bytes);
            lazyObject.init(byteArrayRef, start, length);
            strings[column] = SerDeUtils.getJSONString(lazyObject.getObject(), fieldInspectors[column]).getBytes(Charsets.UTF_8);
            wasNull = false;
        }
        else {

View Full Code Here

            nulls[column] = false;
            if (hiveTypes[column] == HiveType.MAP || hiveTypes[column] == HiveType.LIST || hiveTypes[column] == HiveType.STRUCT) {
                // temporarily special case MAP, LIST, and STRUCT types as strings
                // TODO: create a real parser for these complex types when we implement data types
                LazyBinaryObject<? extends ObjectInspector> lazyObject = LazyBinaryFactory.createLazyBinaryObject(fieldInspectors[column]);
                ByteArrayRef byteArrayRef = new ByteArrayRef();
                byteArrayRef.setData(bytes);
                lazyObject.init(byteArrayRef, start, length);
                strings[column] = SerDeUtils.getJsonBytes(lazyObject.getObject(), fieldInspectors[column]);
            }
            else {
                // TODO: zero length BINARY is not supported. See https://issues.apache.org/jira/browse/HIVE-2483

View Full Code Here

            nulls[column] = false;
            if (hiveTypes[column] == HiveType.MAP || hiveTypes[column] == HiveType.LIST || hiveTypes[column] == HiveType.STRUCT) {
                // temporarily special case MAP, LIST, and STRUCT types as strings
                // TODO: create a real parser for these complex types when we implement data types
                LazyBinaryObject<? extends ObjectInspector> lazyObject = LazyBinaryFactory.createLazyBinaryObject(fieldInspectors[column]);
                ByteArrayRef byteArrayRef = new ByteArrayRef();
                byteArrayRef.setData(bytes);
                lazyObject.init(byteArrayRef, start, length);
                slices[column] = Slices.wrappedBuffer(SerDeUtils.getJsonBytes(sessionTimeZone, lazyObject.getObject(), fieldInspectors[column]));
            }
            else {
                // TODO: zero length BINARY is not supported. See https://issues.apache.org/jira/browse/HIVE-2483

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.hive.serde2.lazy.ByteArrayRef

com.asakusafw.directio.hive.orc.OrcFileInput

com.asakusafw.directio.hive.serde.DataModelDriver

com.asakusafw.directio.hive.serde.DataModelDriverTest

com.asakusafw.directio.hive.serde.ValueSerdeFactoryTest

com.facebook.hive.orc.lazy.LazyDoubleTreeReader

com.facebook.hive.orc.lazy.LazyShortDictionaryTreeReader

com.facebook.hive.orc.lazy.LazyShortDirectTreeReader

com.facebook.hive.orc.lazy.OrcLazyStructObjectInspector

com.facebook.hive.orc.OrcOutputFormat$OrcRecordWriter

com.facebook.hive.orc.OrcStruct$OrcStructInspector

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.