Examples of com.linkedin.haivvreo.AvroObjectInspectorGenerator

com.linkedin.haivvreo.AvroObjectInspectorGenerator
An AvroObjectInspectorGenerator takes an Avro schema and creates the three data structures Hive needs to work with Avro-encoded data: * A list of the schema field names * A list of those fields equivalent types in Hive * An ObjectInspector capable of working with an instance of that datum.

        "  \"symbols\" : [\"SPADES\", \"HEARTS\", \"DIAMONDS\", \"CLUBS\"]\n" +
        "}";


    Schema s = Schema.parse(nonRecordSchema);
    try {
      new AvroObjectInspectorGenerator(s);
      fail("Should not be able to handle non-record Avro types");
    } catch(SerDeException sde) {
      assertTrue(sde.getMessage().startsWith("Schema for table must be of type RECORD"));
    }
  }

View Full Code Here

        "      \"name\":\"aNull\",\n" +
        "      \"type\":\"null\"\n" +
        "    }\n" +
        "  ]\n" +
        "}";
    AvroObjectInspectorGenerator aoig = new AvroObjectInspectorGenerator(Schema.parse(bunchOfPrimitives));


    String [] expectedColumnNames = {"aString", "anInt", "aBoolean", "aLong", "aFloat", "aDouble", "aNull"};
    verifyColumnNames(expectedColumnNames, aoig.getColumnNames());


    TypeInfo [] expectedColumnTypes = {STRING, INT, BOOLEAN, LONG, FLOAT, DOUBLE, VOID};
    verifyColumnTypes(expectedColumnTypes, aoig.getColumnTypes());


    // Rip apart the object inspector, making sure we got what we expect.
    final ObjectInspector oi = aoig.getObjectInspector();
    assertTrue(oi instanceof StandardStructObjectInspector);
    final StandardStructObjectInspector ssoi = (StandardStructObjectInspector)oi;
    List<? extends StructField> structFields = ssoi.getAllStructFieldRefs();
    assertEquals(expectedColumnNames.length, structFields.size());

View Full Code Here

  }


  @Test
  public void canHandleMapsWithPrimitiveValueTypes() throws SerDeException {
    Schema s = Schema.parse(MAP_WITH_PRIMITIVE_VALUE_TYPE);
    AvroObjectInspectorGenerator aoig = new AvroObjectInspectorGenerator(s);


    // Column names
    assertEquals(1, aoig.getColumnNames().size());
    assertEquals("aMap", aoig.getColumnNames().get(0));


    // Column types
    assertEquals(1, aoig.getColumnTypes().size());
    TypeInfo typeInfo = aoig.getColumnTypes().get(0);
    assertEquals(ObjectInspector.Category.MAP, typeInfo.getCategory());
    assertTrue(typeInfo instanceof MapTypeInfo);
    MapTypeInfo mapTypeInfo = (MapTypeInfo)typeInfo;


    assertEquals("bigint" /* == long in Avro */, mapTypeInfo.getMapValueTypeInfo().getTypeName());

View Full Code Here

  }


  @Test
  public void canHandleArrays() throws SerDeException {
    Schema s = Schema.parse(ARRAY_WITH_PRIMITIVE_ELEMENT_TYPE);
    AvroObjectInspectorGenerator aoig = new AvroObjectInspectorGenerator(s);


    // Column names
    assertEquals(1, aoig.getColumnNames().size());
    assertEquals("anArray", aoig.getColumnNames().get(0));


    // Column types
    assertEquals(1, aoig.getColumnTypes().size());
    TypeInfo typeInfo = aoig.getColumnTypes().get(0);
    assertEquals(ObjectInspector.Category.LIST, typeInfo.getCategory());
    assertTrue(typeInfo instanceof ListTypeInfo);
    ListTypeInfo listTypeInfo = (ListTypeInfo)typeInfo;


    assertEquals("string", listTypeInfo.getListElementTypeInfo().getTypeName());

View Full Code Here

  }


  @Test
  public void canHandleRecords() throws SerDeException {
    Schema s = Schema.parse(RECORD_SCHEMA);
    AvroObjectInspectorGenerator aoig = new AvroObjectInspectorGenerator(s);


    // Column names
    assertEquals(1, aoig.getColumnNames().size());
    assertEquals("aRecord", aoig.getColumnNames().get(0));


    // Column types
    assertEquals(1, aoig.getColumnTypes().size());
    TypeInfo typeInfo = aoig.getColumnTypes().get(0);
    assertEquals(ObjectInspector.Category.STRUCT, typeInfo.getCategory());
    assertTrue(typeInfo instanceof StructTypeInfo);
    StructTypeInfo structTypeInfo = (StructTypeInfo)typeInfo;


    // Check individual elements of subrecord

View Full Code Here

  }


  @Test
  public void canHandleUnions() throws SerDeException {
    Schema s = Schema.parse(UNION_SCHEMA);
    AvroObjectInspectorGenerator aoig = new AvroObjectInspectorGenerator(s);


    // Column names
    assertEquals(1, aoig.getColumnNames().size());
    assertEquals("aUnion", aoig.getColumnNames().get(0));


    // Column types
    assertEquals(1, aoig.getColumnTypes().size());
    TypeInfo typeInfo = aoig.getColumnTypes().get(0);
    assertTrue(typeInfo instanceof UnionTypeInfo);
    UnionTypeInfo uti = (UnionTypeInfo)typeInfo;


    // Check that the union has come out unscathed. No scathing of unions allowed.
    List<TypeInfo> typeInfos = uti.getAllUnionObjectTypeInfos();

View Full Code Here

  }


  @Test // Enums are one of two Avro types that Hive doesn't have any native support for.
  public void canHandleEnums() throws SerDeException {
    Schema s = Schema.parse(ENUM_SCHEMA);
    AvroObjectInspectorGenerator aoig = new AvroObjectInspectorGenerator(s);


    // Column names - we lose the enumness of this schema
    assertEquals(1, aoig.getColumnNames().size());
    assertEquals("baddies", aoig.getColumnNames().get(0));


    // Column types
    assertEquals(1, aoig.getColumnTypes().size());
    assertEquals(STRING, aoig.getColumnTypes().get(0));
  }

View Full Code Here


  @Test // Hive has no concept of Avro's fixed type.  Fixed -> arrays of bytes
  public void canHandleFixed() throws SerDeException {
    Schema s = Schema.parse(FIXED_SCHEMA);


    AvroObjectInspectorGenerator aoig = new AvroObjectInspectorGenerator(s);


    // Column names
    assertEquals(1, aoig.getColumnNames().size());
    assertEquals("hash", aoig.getColumnNames().get(0));


    // Column types
    assertEquals(1, aoig.getColumnTypes().size());
    TypeInfo typeInfo = aoig.getColumnTypes().get(0);
    assertTrue(typeInfo instanceof ListTypeInfo);
    ListTypeInfo listTypeInfo = (ListTypeInfo) typeInfo;
    assertTrue(listTypeInfo.getListElementTypeInfo() instanceof PrimitiveTypeInfo);
    assertEquals("tinyint", listTypeInfo.getListElementTypeInfo().getTypeName());
  }

View Full Code Here


  @Test // Avro considers bytes primitive, Hive doesn't. Make them list of tinyint.
  public void canHandleBytes() throws SerDeException {
    Schema s = Schema.parse(BYTES_SCHEMA);


    AvroObjectInspectorGenerator aoig = new AvroObjectInspectorGenerator(s);


    // Column names
    assertEquals(1, aoig.getColumnNames().size());
    assertEquals("bytesField", aoig.getColumnNames().get(0));


    // Column types
    assertEquals(1, aoig.getColumnTypes().size());
    TypeInfo typeInfo = aoig.getColumnTypes().get(0);
    assertTrue(typeInfo instanceof ListTypeInfo);
    ListTypeInfo listTypeInfo = (ListTypeInfo) typeInfo;
    assertTrue(listTypeInfo.getListElementTypeInfo() instanceof PrimitiveTypeInfo);
    assertEquals("tinyint", listTypeInfo.getListElementTypeInfo().getTypeName());
  }

View Full Code Here


  @Test // That Union[T, NULL] is converted to just T.
  public void convertsNullableTypes() throws SerDeException {
    Schema s = Schema.parse(NULLABLE_STRING_SCHEMA);


    AvroObjectInspectorGenerator aoig = new AvroObjectInspectorGenerator(s);
    assertEquals(1, aoig.getColumnNames().size());
    assertEquals("nullableString", aoig.getColumnNames().get(0));


    // Column types
    assertEquals(1, aoig.getColumnTypes().size());
    TypeInfo typeInfo = aoig.getColumnTypes().get(0);
    assertTrue(typeInfo instanceof PrimitiveTypeInfo);
    PrimitiveTypeInfo pti = (PrimitiveTypeInfo) typeInfo;
    // Verify the union has been hidden and just the main type has been returned.
    assertEquals(PrimitiveObjectInspector.PrimitiveCategory.STRING, pti.getPrimitiveCategory());
  }

View Full Code Here

0 1

TOP

Related Classes of com.linkedin.haivvreo.AvroObjectInspectorGenerator

com.linkedin.haivvreo.TestAvroObjectInspectorGenerator

org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.