Package org.apache.hadoop.hive.serde2.lazy

Examples of org.apache.hadoop.hive.serde2.lazy.ByteArrayRef

245
246
247
248
249
250
251
252
253
254
255
256
257
      // need to keep HCatStorer lean and go fast.
      Type type = hcatFS.getType();
      switch(type){

      case BINARY:
        ByteArrayRef ba = new ByteArrayRef();
        byte[] bytes = (null == pigObj) ? new byte[0] : ((DataByteArray)pigObj).get();
        ba.setData(bytes);
        return ba;

      case STRUCT:
        if (pigObj == null) {
          return null;
View Full Code Here
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
    LazyObject<?> [] fields = getFields();
    boolean [] fieldsInited = getFieldInited();

    if (!fieldsInited[fieldID]) {
      fieldsInited[fieldID] = true;
      ByteArrayRef ref = null;
      ColumnMapping colMap = columnsMapping.get(fieldID);

      if (colMap.hbaseRowKey) {
        ref = new ByteArrayRef();
        ref.setData(result.getRow());
      } else {
        if (colMap.qualifierName == null) {
          // it is a column family
          // primitive type for Map<Key, Value> can be stored in binary format
          ((LazyHBaseCellMap) fields[fieldID]).init(
              result, colMap.familyNameBytes, colMap.binaryStorage);
        } else {
          // it is a column i.e. a column-family with column-qualifier
          byte [] res = result.getValue(colMap.familyNameBytes, colMap.qualifierNameBytes);

          if (res == null) {
            return null;
          } else {
            ref = new ByteArrayRef();
            ref.setData(res);
          }
        }
      }

      if (ref != null) {
        fields[fieldID].init(ref, 0, ref.getData().length);
      }
    }

    return fields[fieldID].getObject();
  }
View Full Code Here
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
        LazyPrimitive<? extends ObjectInspector, ? extends Writable> key =
          LazyFactory.createLazyPrimitiveClass(
              (PrimitiveObjectInspector) lazyMoi.getMapKeyObjectInspector(),
              binaryStorage.get(0));

        ByteArrayRef keyRef = new ByteArrayRef();
        keyRef.setData(e.getKey());
        key.init(keyRef, 0, keyRef.getData().length);

        // Value
        LazyObject<?> value =
          LazyFactory.createLazyObject(lazyMoi.getMapValueObjectInspector(),
              binaryStorage.get(1));

        ByteArrayRef valueRef = new ByteArrayRef();
        valueRef.setData(e.getValue());
        value.init(valueRef, 0, valueRef.getData().length);

        // Put the key/value into the map
        cachedMap.put(key.getObject(), value.getObject());
      }
    }
View Full Code Here
598
599
600
601
602
603
604
605
606
607
608
609
   * @throws Throwable
   */
  public void testLazyBinaryObjectInspector() throws Throwable {

    //create input ByteArrayRef
    ByteArrayRef inpBARef = new ByteArrayRef();
    inpBARef.setData(inpBArray);

    AbstractPrimitiveLazyObjectInspector<?> binInspector = LazyPrimitiveObjectInspectorFactory
    .getLazyObjectInspector(TypeInfoFactory.binaryTypeInfo, false, (byte)0);

    //create LazyBinary initialed with inputBA
View Full Code Here
509
510
511
512
513
514
515
516
517
518
519
    if (c.isInstance(new Timestamp(0))){
      return javaObjectOverHead + javaSizePrimitiveType;
    }

    if (c.isInstance(new String()) || c.isInstance(new ByteArrayRef())) {
      if (aggrPositions[pos] == null) {
        aggrPositions[pos] = new ArrayList<Field>();
      }
      aggrPositions[pos].add(f);
      return javaObjectOverHead;
View Full Code Here
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
    LazyObject<?> [] fields = getFields();
    boolean [] fieldsInited = getFieldInited();

    if (!fieldsInited[fieldID]) {
      ByteArrayRef ref = null;
      ColumnMapping colMap = columnsMapping.get(fieldID);

      if (colMap.hbaseRowKey) {
        ref = new ByteArrayRef();
        ref.setData(result.getRow());
      } else {
        if (colMap.qualifierName == null) {
          // it is a column family
          // primitive type for Map<Key, Value> can be stored in binary format. Pass in the
          // qualifier prefix to cherry pick the qualifiers that match the prefix instead of picking
          // up everything
          ((LazyHBaseCellMap) fields[fieldID]).init(
              result, colMap.familyNameBytes, colMap.binaryStorage, colMap.qualifierPrefixBytes);
        } else {
          // it is a column i.e. a column-family with column-qualifier
          byte [] res = result.getValue(colMap.familyNameBytes, colMap.qualifierNameBytes);

          if (res == null) {
            return null;
          } else {
            ref = new ByteArrayRef();
            ref.setData(res);
          }
        }
      }

      if (ref != null) {
        fields[fieldID].init(ref, 0, ref.getData().length);

        // if it was a row key and we have been provided a custom composite key class, initialize it
        // with the bytes for the row key
        if (colMap.hbaseRowKey && compositeKeyObj != null) {
          ((LazyStruct) compositeKeyObj).init(ref, 0, ref.getData().length);
        }
      }
    }

    // Has to be set last because of HIVE-3179: NULL fields would not work otherwise
View Full Code Here
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
        LazyPrimitive<? extends ObjectInspector, ? extends Writable> key =
          LazyFactory.createLazyPrimitiveClass(
              (PrimitiveObjectInspector) lazyMoi.getMapKeyObjectInspector(),
              binaryStorage.get(0));

        ByteArrayRef keyRef = new ByteArrayRef();
        keyRef.setData(e.getKey());
        key.init(keyRef, 0, keyRef.getData().length);

        // Value
        LazyObject<?> value =
          LazyFactory.createLazyObject(lazyMoi.getMapValueObjectInspector(),
              binaryStorage.get(1));

        ByteArrayRef valueRef = new ByteArrayRef();
        valueRef.setData(e.getValue());
        value.init(valueRef, 0, valueRef.getData().length);

        // Put the key/value into the map
        cachedMap.put(key.getObject(), value.getObject());
      }
    }
View Full Code Here
85
86
87
88
89
90
91
92
93
94
95
96
97
98
    ObjectInspector fieldOI = oi.getAllStructFieldRefs().get(fieldID).getFieldObjectInspector();

    LazyObject<? extends ObjectInspector> lazyObject = LazyFactory
        .createLazyObject(fieldOI);

    ByteArrayRef ref = new ByteArrayRef();

    ref.setData(bytes);

    // initialize the lazy object
    lazyObject.init(ref, 0, ref.getData().length);

    return lazyObject;
  }
View Full Code Here
143
144
145
146
147
148
149
150
151
152
153
    boolean inited;
    boolean fieldSkipped;

    public FieldInfo(LazyObject lazyObject, boolean fieldSkipped) {
      field = lazyObject;
      cachedByteArrayRef = new ByteArrayRef();
      if (fieldSkipped) {
        this.fieldSkipped = true;
        inited = true;
      } else {
        inited = false;
View Full Code Here
134
135
136
137
138
139
140
141
142
143
144
   * Deserialize a table record to a lazybinary struct.
   */
  @Override
  public Object deserialize(Writable field) throws SerDeException {
    if (byteArrayRef == null) {
      byteArrayRef = new ByteArrayRef();
    }
    if (field instanceof BytesWritable) {
      BytesWritable b = (BytesWritable) field;
      if (b.getSize() == 0) {
        return null;
View Full Code Here
TOP

Related Classes of org.apache.hadoop.hive.serde2.lazy.ByteArrayRef

  • com.asakusafw.directio.hive.orc.OrcFileInput
  • com.asakusafw.directio.hive.serde.DataModelDriver
  • com.asakusafw.directio.hive.serde.DataModelDriverTest
  • com.asakusafw.directio.hive.serde.ValueSerdeFactoryTest
  • com.facebook.hive.orc.lazy.LazyDoubleTreeReader
  • com.facebook.hive.orc.lazy.LazyShortDictionaryTreeReader
  • com.facebook.hive.orc.lazy.LazyShortDirectTreeReader
  • com.facebook.hive.orc.lazy.OrcLazyStructObjectInspector
  • com.facebook.hive.orc.OrcOutputFormat$OrcRecordWriter
  • com.facebook.hive.orc.OrcStruct$OrcStructInspector
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.