Examples of org.apache.hadoop.hive.serde2.columnar.ColumnarStruct

Package org.apache.hadoop.hive.serde2.columnar

Examples of org.apache.hadoop.hive.serde2.columnar.ColumnarStruct

org.apache.hadoop.hive.serde2.io.ByteWritable
ColumnarStruct is different from LazyStruct in that ColumnarStruct's field Object get parsed at its initialize time when call {@link #init(BytesRefArrayWritable cols)}, while LazyStruct parse fields in a lazy way.

        server.store("b", outputFile.getAbsolutePath(), storeString);


        //then
        Path outputPath = new Path(outputFile.getAbsolutePath()+"/part-m-00000.rc");


        ColumnarStruct struct = readRow(outputFile, outputPath, "f1 string,f2 array<string>");


        assertEquals(2, struct.getFieldsAsList().size());
        Object o =  struct.getField(0);
        assertEquals(LazyString.class, o.getClass());
        o =  struct.getField(1);
        assertEquals(LazyArray.class, o.getClass());


        LazyArray arr = (LazyArray)o;
        List<Object> values = arr.getList();
        for(Object value : values) {

View Full Code Here

        server.store("b", outputFile.getAbsolutePath(), storeString);


        //then
        Path outputPath = new Path(outputFile.getAbsolutePath()+"/part-m-00000.rc");


        ColumnarStruct struct = readRow(outputFile, outputPath, "f1 string,f2 map<string,string>");


        assertEquals(2, struct.getFieldsAsList().size());
        Object o =  struct.getField(0);
        assertEquals(LazyString.class, o.getClass());
        o =  struct.getField(1);
        assertEquals(LazyMap.class, o.getClass());


        LazyMap arr = (LazyMap)o;
        Map<Object,Object> values = arr.getMap();
        for(Entry<Object,Object> entry : values.entrySet()) {

View Full Code Here

        LongWritable key = rcFileRecordReader.createKey();
        BytesRefArrayWritable value = rcFileRecordReader.createValue();
        rcFileRecordReader.next(key, value);
        rcFileRecordReader.close();


        ColumnarStruct struct = readColumnarStruct(value, schema);
        return struct;
    }

View Full Code Here


  try {
      if (reader.nextKeyValue()) {


    BytesRefArrayWritable buff = reader.getCurrentValue();
    ColumnarStruct struct = readColumnarStruct(buff);


    tuple = readColumnarTuple(struct, reader.getSplitPath());
      }


  } catch (InterruptedException e) {

View Full Code Here

     *            BytesRefArrayWritable
     * @return ColumnarStruct
     */
    private ColumnarStruct readColumnarStruct(BytesRefArrayWritable buff) {
  // use ColumnarSerDe to deserialize row
  ColumnarStruct struct = null;
  try {
      struct = (ColumnarStruct) serde.deserialize(buff);
  } catch (SerDeException e) {
      LOG.error(e.toString(), e);
      throw new RuntimeException(e.toString(), e);

View Full Code Here

  @Override
  public Object getStructFieldData(Object data, StructField fieldRef) {
    if (data == null) {
      return null;
    }
    ColumnarStruct struct = (ColumnarStruct) data;
    MyField f = (MyField) fieldRef;


    int fieldID = f.getFieldID();
    assert (fieldID >= 0 && fieldID < fields.size());


    return struct.getField(fieldID);
  }

View Full Code Here

  @Override
  public List<Object> getStructFieldsDataAsList(Object data) {
    if (data == null) {
      return null;
    }
    ColumnarStruct struct = (ColumnarStruct) data;
    return struct.getFieldsAsList();
  }

View Full Code Here


  try {
      if (reader.nextKeyValue()) {


    BytesRefArrayWritable buff = reader.getCurrentValue();
    ColumnarStruct struct = readColumnarStruct(buff);


    tuple = readColumnarTuple(struct, reader.getSplitPath());
      }


  } catch (InterruptedException e) {

View Full Code Here

     *            BytesRefArrayWritable
     * @return ColumnarStruct
     */
    private ColumnarStruct readColumnarStruct(BytesRefArrayWritable buff) {
  // use ColumnarSerDe to deserialize row
  ColumnarStruct struct = null;
  try {
      struct = (ColumnarStruct) serde.deserialize(buff);
  } catch (SerDeException e) {
      LOG.error(e.toString(), e);
      throw new RuntimeException(e.toString(), e);

View Full Code Here

    return reader.loadIndeces(rowIndexEntries, updatedStartIndex);
  }


  @Override
  public Object next(Object previous) throws IOException {
    ByteWritable result = null;
    if (valuePresent) {
      if (previous == null) {
        result = new ByteWritable();
      } else {
        result = (ByteWritable) previous;
      }
      result.set(reader.next());
    }
    return result;
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.hive.serde2.columnar.ColumnarStruct

com.asakusafw.directio.hive.orc.OrcFileInput

com.asakusafw.directio.hive.serde.DataModelDriver

com.asakusafw.directio.hive.serde.DataModelDriverTest

com.asakusafw.directio.hive.serde.ValueSerdeFactoryTest

com.facebook.hive.orc.lazy.LazyDoubleTreeReader

com.facebook.hive.orc.lazy.LazyShortDictionaryTreeReader

com.facebook.hive.orc.lazy.LazyShortDirectTreeReader

com.facebook.hive.orc.lazy.OrcLazyStructObjectInspector

com.facebook.hive.orc.OrcOutputFormat$OrcRecordWriter

com.facebook.hive.orc.OrcStruct$OrcStructInspector

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.