Package org.apache.hadoop.hive.serde2.lazy

Examples of org.apache.hadoop.hive.serde2.lazy.ByteArrayRef

131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
    LazyObject<?> [] fields = getFields();
    boolean [] fieldsInited = getFieldInited();

    if (!fieldsInited[fieldID]) {
      ByteArrayRef ref = null;
      ColumnMapping colMap = columnsMapping.get(fieldID);

      if (colMap.hbaseRowKey) {
        ref = new ByteArrayRef();
        ref.setData(result.getRow());
      } else {
        if (colMap.qualifierName == null) {
          // it is a column family
          // primitive type for Map<Key, Value> can be stored in binary format. Pass in the
          // qualifier prefix to cherry pick the qualifiers that match the prefix instead of picking
          // up everything
          ((LazyHBaseCellMap) fields[fieldID]).init(
              result, colMap.familyNameBytes, colMap.binaryStorage, colMap.qualifierPrefixBytes);
        } else {
          // it is a column i.e. a column-family with column-qualifier
          byte [] res = result.getValue(colMap.familyNameBytes, colMap.qualifierNameBytes);

          if (res == null) {
            return null;
          } else {
            ref = new ByteArrayRef();
            ref.setData(res);
          }
        }
      }

      if (ref != null) {
        fields[fieldID].init(ref, 0, ref.getData().length);
      }
    }

    // Has to be set last because of HIVE-3179: NULL fields would not work otherwise
    fieldsInited[fieldID] = true;
View Full Code Here
497
498
499
500
501
502
503
504
505
506
507
    if (c.isInstance(new Timestamp(0))){
      return javaObjectOverHead + javaSizePrimitiveType;
    }

    if (c.isInstance(new String()) || c.isInstance(new ByteArrayRef())) {
      if (aggrPositions[pos] == null) {
        aggrPositions[pos] = new ArrayList<Field>();
      }
      aggrPositions[pos].add(f);
      return javaObjectOverHead;
View Full Code Here
146
147
148
149
150
151
152
153
154
155
156
   * Deserialize a table record to a lazybinary struct.
   */
  @Override
  public Object deserialize(Writable field) throws SerDeException {
    if (byteArrayRef == null) {
      byteArrayRef = new ByteArrayRef();
    }
    if (field instanceof BytesWritable) {
      BytesWritable b = (BytesWritable) field;
      if (b.getSize() == 0) {
        return null;
View Full Code Here
144
145
146
147
148
149
150
151
152
153
154
    boolean inited;
    boolean fieldSkipped;

    public FieldInfo(LazyObject lazyObject, boolean fieldSkipped) {
      field = lazyObject;
      cachedByteArrayRef = new ByteArrayRef();
      if (fieldSkipped) {
        this.fieldSkipped = true;
        inited = true;
      } else {
        inited = false;
View Full Code Here
87
88
89
90
91
92
93
94
95
96
97
        notSkippedColumnIDs.add(i);
    }
   
    for (int i = 0; i < num; i++) {
      fields[i] = LazyFactory.createLazyObject(fieldRefs.get(i).getFieldObjectInspector());
      cachedByteArrayRef[i] = new ByteArrayRef();
      if(!notSkippedColumnIDs.contains(i)){
        fieldSkipped[i] = true;
        inited[i] = true;
      } else
        inited[i] = false;
View Full Code Here
149
150
151
152
153
154
155
156
157
158
159
   * Deserialize a table record to a lazybinary struct.
   */
  @Override
  public Object deserialize(Writable field) throws SerDeException {
    if (byteArrayRef == null) {
      byteArrayRef = new ByteArrayRef();
    }
    if (field instanceof BytesWritable) {
      BytesWritable b = (BytesWritable) field;
      if (b.getLength() == 0) {
        return null;
View Full Code Here
585
586
587
588
589
590
591
592
593
594
595
596
   * @throws Throwable
   */
  public void testLazyBinaryObjectInspector() throws Throwable {

    //create input ByteArrayRef
    ByteArrayRef inpBARef = new ByteArrayRef();
    inpBARef.setData(inpBArray);

    AbstractPrimitiveLazyObjectInspector<?> binInspector = LazyPrimitiveObjectInspectorFactory
    .getLazyObjectInspector(PrimitiveCategory.BINARY, false, (byte)0);

    //create LazyBinary initialed with inputBA
View Full Code Here
517
518
519
520
521
522
523
524
525
526
527
    if (c.isInstance(new Timestamp(0))){
      return javaObjectOverHead + javaSizePrimitiveType;
    }

    if (c.isInstance(new String()) || c.isInstance(new ByteArrayRef())) {
      int idx = 0;
      varLenFields v = null;
      for (idx = 0; idx < aggrPositions.size(); idx++) {
        v = aggrPositions.get(idx);
        if (v.getAggrPos() == pos) {
View Full Code Here
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
   */
  private Object uncheckedGetField(int fieldID) {
    if (!getFieldInited()[fieldID]) {
      getFieldInited()[fieldID] = true;
     
      ByteArrayRef ref = null;
     
      String columnName = hbaseColumns.get(fieldID);
      if (columnName.equals(HBaseSerDe.HBASE_KEY_COL)) {
        ref = new ByteArrayRef();
        ref.setData(rowResult.getRow());
      } else {
        if (columnName.endsWith(":")) {
          // it is a column family
          ((LazyHBaseCellMap) getFields()[fieldID]).init(
            rowResult, columnName);
        } else {
          // it is a column
          if (rowResult.containsKey(columnName)) {
            ref = new ByteArrayRef();
            ref.setData(rowResult.get(columnName).getValue());
          } else {
            return null;
          }
        }
      }
      if (ref != null) {
        getFields()[fieldID].init(ref, 0, ref.getData().length);
      }
    }
    return getFields()[fieldID].getObject();
  }
View Full Code Here
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
       
        // Keys are always primitive
        LazyPrimitive<?, ?> key = LazyFactory.createLazyPrimitiveClass(
            (PrimitiveObjectInspector)
            ((MapObjectInspector) getInspector()).getMapKeyObjectInspector());
        ByteArrayRef keyRef = new ByteArrayRef();
        keyRef.setData(columnKey);
        key.init(
          keyRef, columnFamily.length, columnKey.length - columnFamily.length);
       
        // Value
        LazyObject value = LazyFactory.createLazyObject(
          ((MapObjectInspector) getInspector()).getMapValueObjectInspector());
        ByteArrayRef valueRef = new ByteArrayRef();
        valueRef.setData(columnValue);
        value.init(valueRef, 0, columnValue.length);
       
        // Put it into the map
        cachedMap.put(key.getObject(), value.getObject());
      }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.hive.serde2.lazy.ByteArrayRef

  • com.asakusafw.directio.hive.orc.OrcFileInput
  • com.asakusafw.directio.hive.serde.DataModelDriver
  • com.asakusafw.directio.hive.serde.DataModelDriverTest
  • com.asakusafw.directio.hive.serde.ValueSerdeFactoryTest
  • com.facebook.hive.orc.lazy.LazyDoubleTreeReader
  • com.facebook.hive.orc.lazy.LazyShortDictionaryTreeReader
  • com.facebook.hive.orc.lazy.LazyShortDirectTreeReader
  • com.facebook.hive.orc.lazy.OrcLazyStructObjectInspector
  • com.facebook.hive.orc.OrcOutputFormat$OrcRecordWriter
  • com.facebook.hive.orc.OrcStruct$OrcStructInspector
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.