Examples of org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter

Package org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters

Examples of org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter

org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector

      String skewedValue, ExprNodeColumnDesc keyCol) {
      ObjectInspector inputOI = TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(
        TypeInfoFactory.stringTypeInfo);
      ObjectInspector outputOI = TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(
        keyCol.getTypeInfo());
      Converter converter = ObjectInspectorConverters.getConverter(inputOI, outputOI);
      Object skewedValueObject = converter.convert(skewedValue);
      return new ExprNodeConstantDesc(keyCol.getTypeInfo(), skewedValueObject);
    }

View Full Code Here

public class TestObjectInspectorConverters extends TestCase {


  public void testObjectInspectorConverters() throws Throwable {
    try {
      // Boolean
      Converter booleanConverter = ObjectInspectorConverters.getConverter(
          PrimitiveObjectInspectorFactory.javaIntObjectInspector,
          PrimitiveObjectInspectorFactory.writableBooleanObjectInspector);
      assertEquals("BooleanConverter", new BooleanWritable(false),
          booleanConverter.convert(Integer.valueOf(0)));
      assertEquals("BooleanConverter", new BooleanWritable(true),
          booleanConverter.convert(Integer.valueOf(1)));


      // Byte
      Converter byteConverter = ObjectInspectorConverters.getConverter(
          PrimitiveObjectInspectorFactory.javaIntObjectInspector,
          PrimitiveObjectInspectorFactory.writableByteObjectInspector);
      assertEquals("ByteConverter", new ByteWritable((byte) 0), byteConverter
          .convert(Integer.valueOf(0)));
      assertEquals("ByteConverter", new ByteWritable((byte) 1), byteConverter
          .convert(Integer.valueOf(1)));


      // Short
      Converter shortConverter = ObjectInspectorConverters.getConverter(
          PrimitiveObjectInspectorFactory.javaIntObjectInspector,
          PrimitiveObjectInspectorFactory.writableShortObjectInspector);
      assertEquals("ShortConverter", new ShortWritable((short) 0),
          shortConverter.convert(Integer.valueOf(0)));
      assertEquals("ShortConverter", new ShortWritable((short) 1),
          shortConverter.convert(Integer.valueOf(1)));


      // Int
      Converter intConverter = ObjectInspectorConverters.getConverter(
          PrimitiveObjectInspectorFactory.javaIntObjectInspector,
          PrimitiveObjectInspectorFactory.writableIntObjectInspector);
      assertEquals("IntConverter", new IntWritable(0), intConverter
          .convert(Integer.valueOf(0)));
      assertEquals("IntConverter", new IntWritable(1), intConverter
          .convert(Integer.valueOf(1)));


      // Long
      Converter longConverter = ObjectInspectorConverters.getConverter(
          PrimitiveObjectInspectorFactory.javaIntObjectInspector,
          PrimitiveObjectInspectorFactory.writableLongObjectInspector);
      assertEquals("LongConverter", new LongWritable(0), longConverter
          .convert(Integer.valueOf(0)));
      assertEquals("LongConverter", new LongWritable(1), longConverter
          .convert(Integer.valueOf(1)));


      // Float
      Converter floatConverter = ObjectInspectorConverters.getConverter(
          PrimitiveObjectInspectorFactory.javaIntObjectInspector,
          PrimitiveObjectInspectorFactory.writableFloatObjectInspector);
      assertEquals("LongConverter", new FloatWritable(0), floatConverter
          .convert(Integer.valueOf(0)));
      assertEquals("LongConverter", new FloatWritable(1), floatConverter
          .convert(Integer.valueOf(1)));


      // Double
      Converter doubleConverter = ObjectInspectorConverters.getConverter(
          PrimitiveObjectInspectorFactory.javaIntObjectInspector,
          PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);
      assertEquals("DoubleConverter", new DoubleWritable(0), doubleConverter
          .convert(Integer.valueOf(0)));
      assertEquals("DoubleConverter", new DoubleWritable(1), doubleConverter
          .convert(Integer.valueOf(1)));


      // Text
      Converter textConverter = ObjectInspectorConverters.getConverter(
          PrimitiveObjectInspectorFactory.javaIntObjectInspector,
          PrimitiveObjectInspectorFactory.writableStringObjectInspector);
      assertEquals("TextConverter", new Text("0"), textConverter
          .convert(Integer.valueOf(0)));
      assertEquals("TextConverter", new Text("1"), textConverter
          .convert(Integer.valueOf(1)));
      textConverter = ObjectInspectorConverters.getConverter(
          PrimitiveObjectInspectorFactory.writableBinaryObjectInspector,
          PrimitiveObjectInspectorFactory.writableStringObjectInspector);
      assertEquals("TextConverter", new Text("hive"), textConverter
          .convert(new BytesWritable(new byte[]
              {(byte)'h', (byte)'i',(byte)'v',(byte)'e'})));
      textConverter = ObjectInspectorConverters.getConverter(
          PrimitiveObjectInspectorFactory.writableStringObjectInspector,
          PrimitiveObjectInspectorFactory.writableStringObjectInspector);
      assertEquals("TextConverter", new Text("hive"), textConverter
    .convert(new Text("hive")));
      textConverter = ObjectInspectorConverters.getConverter(
          PrimitiveObjectInspectorFactory.javaStringObjectInspector,
          PrimitiveObjectInspectorFactory.writableStringObjectInspector);
      assertEquals("TextConverter", new Text("hive"), textConverter
    .convert(new String("hive")));
      textConverter = ObjectInspectorConverters.getConverter(
          PrimitiveObjectInspectorFactory.javaHiveDecimalObjectInspector,
          PrimitiveObjectInspectorFactory.writableStringObjectInspector);
      assertEquals("TextConverter", new Text("100.001"), textConverter
    .convert(new HiveDecimal("100.001")));


      // Binary
      Converter baConverter = ObjectInspectorConverters.getConverter(
          PrimitiveObjectInspectorFactory.javaStringObjectInspector,
          PrimitiveObjectInspectorFactory.writableBinaryObjectInspector);
      assertEquals("BAConverter", new BytesWritable(new byte[]
          {(byte)'h', (byte)'i',(byte)'v',(byte)'e'}),
          baConverter.convert("hive"));
      baConverter = ObjectInspectorConverters.getConverter(
          PrimitiveObjectInspectorFactory.writableStringObjectInspector,
          PrimitiveObjectInspectorFactory.writableBinaryObjectInspector);
      assertEquals("BAConverter", new BytesWritable(new byte[]
          {(byte)'h', (byte)'i',(byte)'v',(byte)'e'}),
          baConverter.convert(new Text("hive")));
    } catch (Throwable e) {
      e.printStackTrace();
      throw e;
    }

View Full Code Here


        if (converters == null) {
          converters = new HashMap<ObjectInspector, Converter>();
        }


        Converter converter = converters.get(oi);
        if (converter == null) {
          converter = ObjectInspectorConverters.getConverter(oi,
              returnObjectInspector);
          converters.put(oi, converter);
        }
        converted = converter.convert(o);
      }
      return converted;
    }

View Full Code Here


      // Create the converters
      conversionNeeded = false;
      converters = new Converter[parameterOIs.length];
      for (int i = 0; i < parameterOIs.length; i++) {
        Converter pc = ObjectInspectorConverters.getConverter(parameterOIs[i],
            methodParameterOIs[i]);
        converters[i] = pc;
        // Conversion is needed?
        conversionNeeded = conversionNeeded
            || (!(pc instanceof IdentityConverter));

View Full Code Here

        return struct;
      }
    };


    // Test control case (cases match)
    StructField field = STRUCT_OI.getStructFieldRef(FIELD_0);
    Assert.assertEquals("a",
        ((StringObjectInspector) field.getFieldObjectInspector()).getPrimitiveJavaObject(
            STRUCT_OI.getStructFieldData(struct, field)));
    // Test upper case
    field = STRUCT_OI.getStructFieldRef(FIELD_0.toUpperCase());
    Assert.assertEquals("a",
        ((StringObjectInspector) field.getFieldObjectInspector()).getPrimitiveJavaObject(
            STRUCT_OI.getStructFieldData(struct, field)));
    // Test lower case (even if someone changes the value of FIELD_0 in the future either upper
    // or lower case should be different from the actual case)
    field = STRUCT_OI.getStructFieldRef(FIELD_0.toLowerCase());
    Assert.assertEquals("a",
        ((StringObjectInspector) field.getFieldObjectInspector()).getPrimitiveJavaObject(
            STRUCT_OI.getStructFieldData(struct, field)));
  }

View Full Code Here

  public void testCaseInsensitiveFieldsStruct() throws Exception {
    OrcStruct struct = new OrcStruct(Lists.newArrayList(FIELD_0));
    struct.setFieldValue(0, new Text("a"));


    // Test control case (cases match)
    StructField field = NON_LAZY_STRUCT_OI.getStructFieldRef(FIELD_0);
    Assert.assertEquals("a",
        ((StringObjectInspector) field.getFieldObjectInspector()).getPrimitiveJavaObject(
            NON_LAZY_STRUCT_OI.getStructFieldData(struct, field)));
    // Test upper case
    field = NON_LAZY_STRUCT_OI.getStructFieldRef(FIELD_0.toUpperCase());
    Assert.assertEquals("a",
        ((StringObjectInspector) field.getFieldObjectInspector()).getPrimitiveJavaObject(
            NON_LAZY_STRUCT_OI.getStructFieldData(struct, field)));
    // Test lower case (even if someone changes the value of FIELD_0 in the future either upper
    // or lower case should be different from the actual case)
    field = NON_LAZY_STRUCT_OI.getStructFieldRef(FIELD_0.toLowerCase());
    Assert.assertEquals("a",
        ((StringObjectInspector) field.getFieldObjectInspector()).getPrimitiveJavaObject(
            NON_LAZY_STRUCT_OI.getStructFieldData(struct, field)));
  }

View Full Code Here

    writer.close();
    OrcConf.setIntVar(conf, OrcConf.ConfVars.HIVE_ORC_READ_COMPRESSION_STRIDES, 2);
    OrcConf.setBoolVar(conf, OrcConf.ConfVars.HIVE_ORC_EAGER_HDFS_READ, false);
    Reader reader = OrcFile.createReader(fs, testFilePath, conf);


    StructObjectInspector readerInspector = (StructObjectInspector) reader.getObjectInspector();
    List<? extends StructField> fields = readerInspector.getAllStructFieldRefs();
    DoubleObjectInspector columnInspector =
        (DoubleObjectInspector) fields.get(0).getFieldObjectInspector();


    RecordReader rows = reader.rows(null);
    Object row = null;


    // Skip enough values to get to the 2nd index stride in the first chunk
    for (int i = 0; i < 40001; i++) {
      row = rows.next(row);
    }


    // This will set previousOffset to be the size of the first compression block and the
    // compressionOffset to some other value (doesn't matter what point is it's different from the
    // start of the compression block)
    assertEquals(values[40000], columnInspector.get(readerInspector.getStructFieldData(row,
        fields.get(0))));


    // Skip enough values to get to the 2nd index stride of the second chunk
    for (int i = 0; i < 80000; i++) {
      rows.next(row);
    }


    // When seek is called, previousOffset will equal newCompressedOffset since the former is the
    // the length of the first compression block and the latter is the length of the third
    // compression block (remember the chunks contain 2 index strides), so if we only check this
    // (or for some other reason) we will not adjust compressedIndex, we will read the wrong data
    assertEquals(values[120000], columnInspector.get(readerInspector.getStructFieldData(row, fields.get(0))));


    rows.close();
  }

View Full Code Here

        stats[7].toString());


    assertEquals("count: 2 min: bye max: hi", stats[9].toString());


    // check the inspectors
    StructObjectInspector readerInspector =
        (StructObjectInspector) reader.getObjectInspector();
    assertEquals(ObjectInspector.Category.STRUCT,
        readerInspector.getCategory());
    assertEquals("struct<boolean1:boolean,byte1:tinyint,short1:smallint,"
        + "int1:int,long1:bigint,float1:float,double1:double,bytes1:"
        + "binary,string1:string,middle:struct<list:array<struct<int1:int,"
        + "string1:string>>>,list:array<struct<int1:int,string1:string>>,"
        + "map:map<string,struct<int1:int,string1:string>>>",
        readerInspector.getTypeName());
    List<? extends StructField> fields =
        readerInspector.getAllStructFieldRefs();
    BooleanObjectInspector bo = (BooleanObjectInspector) readerInspector.
        getStructFieldRef("boolean1").getFieldObjectInspector();
    ByteObjectInspector by = (ByteObjectInspector) readerInspector.
        getStructFieldRef("byte1").getFieldObjectInspector();
    ShortObjectInspector sh = (ShortObjectInspector) readerInspector.
        getStructFieldRef("short1").getFieldObjectInspector();
    IntObjectInspector in = (IntObjectInspector) readerInspector.
        getStructFieldRef("int1").getFieldObjectInspector();
    LongObjectInspector lo = (LongObjectInspector) readerInspector.
        getStructFieldRef("long1").getFieldObjectInspector();
    FloatObjectInspector fl = (FloatObjectInspector) readerInspector.
        getStructFieldRef("float1").getFieldObjectInspector();
    DoubleObjectInspector dbl = (DoubleObjectInspector) readerInspector.
        getStructFieldRef("double1").getFieldObjectInspector();
    BinaryObjectInspector bi = (BinaryObjectInspector) readerInspector.
        getStructFieldRef("bytes1").getFieldObjectInspector();
    StringObjectInspector st = (StringObjectInspector) readerInspector.
        getStructFieldRef("string1").getFieldObjectInspector();
    StructObjectInspector mid = (StructObjectInspector) readerInspector.
        getStructFieldRef("middle").getFieldObjectInspector();
    List<? extends StructField> midFields =
        mid.getAllStructFieldRefs();
    ListObjectInspector midli =
        (ListObjectInspector) midFields.get(0).getFieldObjectInspector();
    StructObjectInspector inner = (StructObjectInspector)
        midli.getListElementObjectInspector();
    List<? extends StructField> inFields = inner.getAllStructFieldRefs();
    IntObjectInspector inner_in = (IntObjectInspector) inFields.get(0).getFieldObjectInspector();
    StringObjectInspector inner_st = (StringObjectInspector) inFields.get(1).getFieldObjectInspector();
    ListObjectInspector li = (ListObjectInspector) readerInspector.
        getStructFieldRef("list").getFieldObjectInspector();
    MapObjectInspector ma = (MapObjectInspector) readerInspector.
        getStructFieldRef("map").getFieldObjectInspector();
    StructObjectInspector lc = (StructObjectInspector)
        li.getListElementObjectInspector();
    StringObjectInspector mk = (StringObjectInspector)
        ma.getMapKeyObjectInspector();
    StructObjectInspector mv = (StructObjectInspector)
        ma.getMapValueObjectInspector();
    RecordReader rows = reader.rows(null);
    Object row = rows.next(null);
    assertNotNull(row);
    // check the contents of the first row

View Full Code Here

                     StreamFactory writer,
                     boolean nullable, Configuration conf,
                     boolean useVInts, boolean lowMemoryMode,
                     MemoryEstimate memoryEstimate) throws IOException {
      super(columnId, inspector, writer, nullable, conf, useVInts, memoryEstimate);
      StructObjectInspector structObjectInspector =
        (StructObjectInspector) inspector;
      fields = structObjectInspector.getAllStructFieldRefs();
      childrenWriters = new TreeWriter[fields.size()];
      for(int i=0; i < childrenWriters.length; ++i) {
        childrenWriters[i] = createTreeWriter(
          fields.get(i).getFieldObjectInspector(), writer, true, conf, useVInts,
          lowMemoryMode, memoryEstimate);

View Full Code Here


    @Override
    void write(Object obj) throws IOException {
      long rawDataSize = 0;
      if (obj != null) {
        StructObjectInspector insp = (StructObjectInspector) inspector;
        List<Object> fieldDataList = insp.getStructFieldsDataAsList(obj);


        for(int i = 0; i < fields.size(); ++i) {
          TreeWriter writer = childrenWriters[i];
          writer.write(fieldDataList.get(i));
          rawDataSize += writer.getRowRawDataSize();

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter

com.asakusafw.directio.hive.orc.OrcFileInput

com.asakusafw.directio.hive.serde.DataModelDriver

com.asakusafw.directio.hive.serde.DataModelDriverTest

com.asakusafw.directio.hive.serde.ValueSerdeFactoryTest

com.facebook.hive.orc.lazy.LazyDoubleTreeReader

com.facebook.hive.orc.lazy.LazyShortDictionaryTreeReader

com.facebook.hive.orc.lazy.LazyShortDirectTreeReader

com.facebook.hive.orc.lazy.OrcLazyStructObjectInspector

com.facebook.hive.orc.OrcOutputFormat$OrcRecordWriter

com.facebook.hive.orc.OrcStruct$OrcStructInspector

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.