Package org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe

Examples of org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.SerDeParameters

330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
    tableProperties.setProperty(serdeConstants.FIELD_DELIM, " ");
    tableProperties.setProperty(serdeConstants.LIST_COLUMNS, Joiner.on(',').join(names));
    tableProperties.setProperty(serdeConstants.LIST_COLUMN_TYPES, Joiner.on(',').join(types));
    AccumuloSerDeParameters accumuloSerDeParams = new AccumuloSerDeParameters(new Configuration(),
        tableProperties, AccumuloSerDe.class.getSimpleName());
    SerDeParameters serDeParams = accumuloSerDeParams.getSerDeParameters();

    AccumuloRowSerializer serializer = new AccumuloRowSerializer(0, serDeParams,
        accumuloSerDeParams.getColumnMappings(), AccumuloSerDeParameters.DEFAULT_VISIBILITY_LABEL,
        accumuloSerDeParams.getRowIdFactory());

    TypeInfo stringTypeInfo = TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME);
    LazyStringObjectInspector stringOI = (LazyStringObjectInspector) LazyFactory
        .createLazyObjectInspector(stringTypeInfo, new byte[] {0}, 0,
            serDeParams.getNullSequence(), serDeParams.isEscaped(), serDeParams.getEscapeChar());

    LazyMapObjectInspector mapOI = LazyObjectInspectorFactory.getLazySimpleMapObjectInspector(
        stringOI, stringOI, (byte) ',', (byte) ':', serDeParams.getNullSequence(),
        serDeParams.isEscaped(), serDeParams.getEscapeChar());

    LazySimpleStructObjectInspector structOI = (LazySimpleStructObjectInspector) LazyObjectInspectorFactory
        .getLazySimpleStructObjectInspector(Arrays.asList("row", "data"),
            Arrays.asList(stringOI, mapOI), (byte) ' ', serDeParams.getNullSequence(),
            serDeParams.isLastColumnTakesRest(), serDeParams.isEscaped(),
            serDeParams.getEscapeChar());

    LazyStruct struct = (LazyStruct) LazyFactory.createLazyObject(structOI);

    ByteArrayRef bytes = new ByteArrayRef();
    bytes.setData("row cq1:value1,cq2:value2".getBytes());
View Full Code Here
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
    tableProperties.setProperty(serdeConstants.LIST_COLUMNS, Joiner.on(',').join(names));
    tableProperties.setProperty(serdeConstants.LIST_COLUMN_TYPES, Joiner.on(',').join(types));
    tableProperties.setProperty(AccumuloSerDeParameters.DEFAULT_STORAGE_TYPE, ColumnEncoding.BINARY.getName());
    AccumuloSerDeParameters accumuloSerDeParams = new AccumuloSerDeParameters(new Configuration(),
        tableProperties, AccumuloSerDe.class.getSimpleName());
    SerDeParameters serDeParams = accumuloSerDeParams.getSerDeParameters();

    AccumuloRowSerializer serializer = new AccumuloRowSerializer(0, serDeParams,
        accumuloSerDeParams.getColumnMappings(), AccumuloSerDeParameters.DEFAULT_VISIBILITY_LABEL,
        accumuloSerDeParams.getRowIdFactory());

    TypeInfo stringTypeInfo = TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME);

    LazySimpleStructObjectInspector structOI = (LazySimpleStructObjectInspector) LazyFactory
        .createLazyStructInspector(Arrays.asList("row", "cq1", "cq2"),
            Arrays.asList(stringTypeInfo, stringTypeInfo, stringTypeInfo),
            serDeParams.getSeparators(), serDeParams.getNullSequence(),
            serDeParams.isLastColumnTakesRest(), serDeParams.isEscaped(),
            serDeParams.getEscapeChar());

    LazyStruct struct = (LazyStruct) LazyFactory.createLazyObject(structOI);

    ByteArrayRef bytes = new ByteArrayRef();
    bytes.setData("row value1 value2".getBytes());
View Full Code Here
61
62
63
64
65
66
67
68
69
70
71
72
73
74
        + columnTypes + "]";
  }

  @Override
  public void initialize(Configuration conf, Properties tbl) throws SerDeException {
    SerDeParameters serdeParams = new SerDeParameters();
    LazyUtils.extractColumnInfo(tbl, serdeParams, getClass().getName());
    columnNames = serdeParams.getColumnNames();
    columnTypes = serdeParams.getColumnTypes();

    cachedObjectInspector = LazyBinaryFactory.createColumnarStructInspector(
        columnNames, columnTypes);
    int size = columnTypes.size();
    List<Integer> notSkipIDs = new ArrayList<Integer>();
View Full Code Here
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
  private static final Logger log = LoggerFactory.getLogger(AccumuloSerDe.class);

  public void initialize(Configuration conf, Properties properties) throws SerDeException {
    accumuloSerDeParameters = new AccumuloSerDeParameters(conf, properties, getClass().getName());

    final SerDeParameters serDeParams = accumuloSerDeParameters.getSerDeParameters();
    final List<ColumnMapping> mappings = accumuloSerDeParameters.getColumnMappings();
    final List<TypeInfo> columnTypes = accumuloSerDeParameters.getHiveColumnTypes();
    final AccumuloRowIdFactory factory = accumuloSerDeParameters.getRowIdFactory();

    ArrayList<ObjectInspector> columnObjectInspectors = getColumnObjectInspectors(columnTypes, serDeParams, mappings, factory);

    cachedObjectInspector = LazyObjectInspectorFactory.getLazySimpleStructObjectInspector(
        serDeParams.getColumnNames(), columnObjectInspectors, serDeParams.getSeparators()[0],
        serDeParams.getNullSequence(), serDeParams.isLastColumnTakesRest(),
        serDeParams.isEscaped(), serDeParams.getEscapeChar());

    cachedRow = new LazyAccumuloRow((LazySimpleStructObjectInspector) cachedObjectInspector);

    serializer = new AccumuloRowSerializer(accumuloSerDeParameters.getRowIdOffset(),
        accumuloSerDeParameters.getSerDeParameters(), accumuloSerDeParameters.getColumnMappings(),
View Full Code Here
97
98
99
100
101
102
103
104
105
106
107
    LazySimpleSerDe serde = new LazySimpleSerDe();
    Configuration conf = new Configuration();
    Properties tbl = new Properties();
    tbl.setProperty(serdeConstants.LIST_COLUMNS, ObjectInspectorUtils.getFieldNames(oi1));
    tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES, ObjectInspectorUtils.getFieldTypes(oi1));
    SerDeParameters serdeParams = LazySimpleSerDe.initSerdeParams(conf, tbl,
        LazySimpleSerDe.class.getName());
    SerDeUtils.initializeSerDe(serde, conf, tbl, null);
    ObjectInspector oi2 = serde.getObjectInspector();

    Object o2 = serializeAndDeserialize(o1, oi1, serde, serdeParams);
View Full Code Here
151
152
153
154
155
156
157
158
159
160
161
    LazySimpleSerDe serde = new LazySimpleSerDe();
    Configuration conf = new Configuration();
    Properties tbl = new Properties();
    tbl.setProperty(serdeConstants.LIST_COLUMNS, ObjectInspectorUtils.getFieldNames(oi1));
    tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES, ObjectInspectorUtils.getFieldTypes(oi1));
    SerDeParameters serdeParams = LazySimpleSerDe.initSerdeParams(conf, tbl,
        LazySimpleSerDe.class.getName());
    SerDeUtils.initializeSerDe(serde, conf, tbl, null);
    ObjectInspector oi2 = serde.getObjectInspector();

    Object o2 = serializeAndDeserialize(o1, oi1, serde, serdeParams);
View Full Code Here
97
98
99
100
101
102
103
104
105
106
107
    LazySimpleSerDe serde = new LazySimpleSerDe();
    Configuration conf = new Configuration();
    Properties tbl = new Properties();
    tbl.setProperty(serdeConstants.LIST_COLUMNS, ObjectInspectorUtils.getFieldNames(oi1));
    tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES, ObjectInspectorUtils.getFieldTypes(oi1));
    SerDeParameters serdeParams = LazySimpleSerDe.initSerdeParams(conf, tbl,
        LazySimpleSerDe.class.getName());
    SerDeUtils.initializeSerDe(serde, conf, tbl, null);
    ObjectInspector oi2 = serde.getObjectInspector();

    Object o2 = serializeAndDeserialize(o1, oi1, serde, serdeParams);
View Full Code Here
151
152
153
154
155
156
157
158
159
160
161
    LazySimpleSerDe serde = new LazySimpleSerDe();
    Configuration conf = new Configuration();
    Properties tbl = new Properties();
    tbl.setProperty(serdeConstants.LIST_COLUMNS, ObjectInspectorUtils.getFieldNames(oi1));
    tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES, ObjectInspectorUtils.getFieldTypes(oi1));
    SerDeParameters serdeParams = LazySimpleSerDe.initSerdeParams(conf, tbl,
        LazySimpleSerDe.class.getName());
    SerDeUtils.initializeSerDe(serde, conf, tbl, null);
    ObjectInspector oi2 = serde.getObjectInspector();

    Object o2 = serializeAndDeserialize(o1, oi1, serde, serdeParams);
View Full Code Here
58
59
60
61
62
63
64
65
66
67
68
69
70
71
        + columnTypes + "]";
  }

  @Override
  public void initialize(Configuration conf, Properties tbl) throws SerDeException {
    SerDeParameters serdeParams = new SerDeParameters();
    LazyUtils.extractColumnInfo(tbl, serdeParams, getClass().getName());
    columnNames = serdeParams.getColumnNames();
    columnTypes = serdeParams.getColumnTypes();

    cachedObjectInspector = LazyBinaryFactory.createColumnarStructInspector(
        columnNames, columnTypes);
    int size = columnTypes.size();
    List<Integer> notSkipIDs = new ArrayList<Integer>();
View Full Code Here
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
                   StreamFactory writer,
                   boolean nullable, Configuration conf,
                   boolean useVInts, boolean lowMemoryMode,
                   MemoryEstimate memoryEstimate) throws IOException {
      super(columnId, inspector, writer, nullable, conf, useVInts, memoryEstimate);
      ListObjectInspector listObjectInspector = (ListObjectInspector) inspector;
      childrenWriters = new TreeWriter[1];
      childrenWriters[0] =
        createTreeWriter(listObjectInspector.getListElementObjectInspector(),
          writer, true, conf, useVInts, lowMemoryMode, memoryEstimate);
      lengths =
        new RunLengthIntegerWriter(writer.createStream(columnId,
            OrcProto.Stream.Kind.LENGTH), false, INT_BYTE_SIZE, useVInts);
      recordPosition(rowIndexPosition);
View Full Code Here
TOP

Related Classes of org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.SerDeParameters

  • com.asakusafw.directio.hive.orc.OrcFileInput
  • com.asakusafw.directio.hive.serde.DataModelDriver
  • com.asakusafw.directio.hive.serde.DataModelDriverTest
  • com.asakusafw.directio.hive.serde.ValueSerdeFactoryTest
  • com.facebook.hive.orc.lazy.LazyDoubleTreeReader
  • com.facebook.hive.orc.lazy.LazyShortDictionaryTreeReader
  • com.facebook.hive.orc.lazy.LazyShortDirectTreeReader
  • com.facebook.hive.orc.lazy.OrcLazyStructObjectInspector
  • com.facebook.hive.orc.OrcOutputFormat$OrcRecordWriter
  • com.facebook.hive.orc.OrcStruct$OrcStructInspector
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.