Examples of org.apache.hadoop.hive.serde2.typeinfo.TypeInfo

Package org.apache.hadoop.hive.serde2.typeinfo

Examples of org.apache.hadoop.hive.serde2.typeinfo.TypeInfo

org.apache.hadoop.hive.serde2.typeinfo.TypeInfo
Stores information about a type (DDL). Always use the TypeInfoFactory to create new TypeInfo objects. We support 4 categories of types: 1. Primitive objects (String, Number, etc) 2. List objects (a list of objects of a single type) 3. Map objects (a map from objects of one type to objects of another type) 4. Struct objects (a list of fields with names and their own types)

    List<TypeInfo> columnTypes = serdeParams.getColumnTypes();


    for (int i = 0; i < columnsMapping.size(); i++) {


      ColumnMapping colMap = columnsMapping.get(i);
      TypeInfo colType = columnTypes.get(i);
      String mappingSpec = colMap.mappingSpec;
      String [] mapInfo = mappingSpec.split("#");
      String [] storageInfo = null;


      if (mapInfo.length == 2) {
        storageInfo = mapInfo[1].split(":");
      }


      if (storageInfo == null) {


        // use the table default storage specification
        if (colType.getCategory() == Category.PRIMITIVE) {
          if (!colType.getTypeName().equals(serdeConstants.STRING_TYPE_NAME)) {
            colMap.binaryStorage.add(tableBinaryStorage);
          } else {
            colMap.binaryStorage.add(false);
          }
        } else if (colType.getCategory() == Category.MAP) {
          TypeInfo keyTypeInfo = ((MapTypeInfo) colType).getMapKeyTypeInfo();
          TypeInfo valueTypeInfo = ((MapTypeInfo) colType).getMapValueTypeInfo();


          if (keyTypeInfo.getCategory() == Category.PRIMITIVE &&
              !keyTypeInfo.getTypeName().equals(serdeConstants.STRING_TYPE_NAME)) {
            colMap.binaryStorage.add(tableBinaryStorage);
          } else {
            colMap.binaryStorage.add(false);
          }


          if (valueTypeInfo.getCategory() == Category.PRIMITIVE &&
              !valueTypeInfo.getTypeName().equals(serdeConstants.STRING_TYPE_NAME)) {
            colMap.binaryStorage.add(tableBinaryStorage);
          } else {
            colMap.binaryStorage.add(false);
          }
        } else {
          colMap.binaryStorage.add(false);
        }


      } else if (storageInfo.length == 1) {
        // we have a storage specification for a primitive column type
        String storageOption = storageInfo[0];


        if ((colType.getCategory() == Category.MAP) ||
            !(storageOption.equals("-") || "string".startsWith(storageOption) ||
                "binary".startsWith(storageOption))) {
          throw new SerDeException("Error: A column storage specification is one of the following:"
              + " '-', a prefix of 'string', or a prefix of 'binary'. "
              + storageOption + " is not a valid storage option specification for "
              + serdeParams.getColumnNames().get(i));
        }


        if (colType.getCategory() == Category.PRIMITIVE &&
            !colType.getTypeName().equals(serdeConstants.STRING_TYPE_NAME)) {


          if ("-".equals(storageOption)) {
            colMap.binaryStorage.add(tableBinaryStorage);
          } else if ("binary".startsWith(storageOption)) {
            colMap.binaryStorage.add(true);
          } else {
              colMap.binaryStorage.add(false);
          }
        } else {
          colMap.binaryStorage.add(false);
        }


      } else if (storageInfo.length == 2) {
        // we have a storage specification for a map column type


        String keyStorage = storageInfo[0];
        String valStorage = storageInfo[1];


        if ((colType.getCategory() != Category.MAP) ||
            !(keyStorage.equals("-") || "string".startsWith(keyStorage) ||
                "binary".startsWith(keyStorage)) ||
            !(valStorage.equals("-") || "string".startsWith(valStorage) ||
                "binary".startsWith(valStorage))) {
          throw new SerDeException("Error: To specify a valid column storage type for a Map"
              + " column, use any two specifiers from '-', a prefix of 'string', "
              + " and a prefix of 'binary' separated by a ':'."
              + " Valid examples are '-:-', 's:b', etc. They specify the storage type for the"
              + " key and value parts of the Map<?,?> respectively."
              + " Invalid storage specification for column "
              + serdeParams.getColumnNames().get(i)
              + "; " + storageInfo[0] + ":" + storageInfo[1]);
        }


        TypeInfo keyTypeInfo = ((MapTypeInfo) colType).getMapKeyTypeInfo();
        TypeInfo valueTypeInfo = ((MapTypeInfo) colType).getMapValueTypeInfo();


        if (keyTypeInfo.getCategory() == Category.PRIMITIVE &&
            !keyTypeInfo.getTypeName().equals(serdeConstants.STRING_TYPE_NAME)) {


          if (keyStorage.equals("-")) {
            colMap.binaryStorage.add(tableBinaryStorage);
          } else if ("binary".startsWith(keyStorage)) {
            colMap.binaryStorage.add(true);
          } else {
            colMap.binaryStorage.add(false);
          }
        } else {
          colMap.binaryStorage.add(false);
        }


        if (valueTypeInfo.getCategory() == Category.PRIMITIVE &&
            !valueTypeInfo.getTypeName().equals(serdeConstants.STRING_TYPE_NAME)) {
          if (valStorage.equals("-")) {
            colMap.binaryStorage.add(tableBinaryStorage);
          } else if ("binary".startsWith(valStorage)) {
            colMap.binaryStorage.add(true);
          } else {

View Full Code Here

    // check that the "column-family:" is mapped to  Map<key,?>
    // where key extends LazyPrimitive<?, ?> and thus has type Category.PRIMITIVE
    for (int i = 0; i < columnsMapping.size(); i++) {
      ColumnMapping colMap = columnsMapping.get(i);
      if (colMap.qualifierName == null && !colMap.hbaseRowKey) {
        TypeInfo typeInfo = serdeParams.getColumnTypes().get(i);
        if ((typeInfo.getCategory() != Category.MAP) ||
          (((MapTypeInfo) typeInfo).getMapKeyTypeInfo().getCategory()
            !=  Category.PRIMITIVE)) {


          throw new SerDeException(
            serdeName + ": hbase column family '" + colMap.familyName
            + "' should be mapped to Map<? extends LazyPrimitive<?, ?>,?>, that is "
            + "the Key for the map should be of primitive type, but is mapped to "
            + typeInfo.getTypeName());
        }
      }
    }


    // Precondition: make sure this is done after the rest of the SerDe initialization is done.

View Full Code Here

        return null;
      }
      return "Constant " + value + " is not string type";
    } else if (expr instanceof ExprNodeColumnDesc) {
      // JDO filter now only support String typed literal -- see Filter.g and ExpressionTree.java
      TypeInfo type = expr.getTypeInfo();
      if (type.getTypeName().equals(serdeConstants.STRING_TYPE_NAME)) {
        String colName = ((ExprNodeColumnDesc)expr).getColumn();
        for (FieldSchema fs: tab.getPartCols()) {
          if (fs.getName().equals(colName)) {
            if (fs.getType().equals(serdeConstants.STRING_TYPE_NAME)) {
              return null;

View Full Code Here

    assertEquals(1, aoig.getColumnNames().size());
    assertEquals(fieldName, aoig.getColumnNames().get(0));


    // Column types
    assertEquals(1, aoig.getColumnTypes().size());
    TypeInfo typeInfo = aoig.getColumnTypes().get(0);
    assertEquals(ObjectInspector.Category.MAP, typeInfo.getCategory());
    assertTrue(typeInfo instanceof MapTypeInfo);
    MapTypeInfo mapTypeInfo = (MapTypeInfo)typeInfo;


    assertEquals("bigint" /* == long in Avro */, mapTypeInfo.getMapValueTypeInfo().getTypeName());
    assertEquals("string", mapTypeInfo.getMapKeyTypeInfo().getTypeName());

View Full Code Here

    assertEquals(1, aoig.getColumnNames().size());
    assertEquals("anArray", aoig.getColumnNames().get(0));


    // Column types
    assertEquals(1, aoig.getColumnTypes().size());
    TypeInfo typeInfo = aoig.getColumnTypes().get(0);
    assertEquals(ObjectInspector.Category.LIST, typeInfo.getCategory());
    assertTrue(typeInfo instanceof ListTypeInfo);
    ListTypeInfo listTypeInfo = (ListTypeInfo)typeInfo;


    assertEquals("string", listTypeInfo.getListElementTypeInfo().getTypeName());
  }

View Full Code Here

    assertEquals(1, aoig.getColumnNames().size());
    assertEquals("aRecord", aoig.getColumnNames().get(0));


    // Column types
    assertEquals(1, aoig.getColumnTypes().size());
    TypeInfo typeInfo = aoig.getColumnTypes().get(0);
    assertEquals(ObjectInspector.Category.STRUCT, typeInfo.getCategory());
    assertTrue(typeInfo instanceof StructTypeInfo);
    StructTypeInfo structTypeInfo = (StructTypeInfo)typeInfo;


    // Check individual elements of subrecord
    ArrayList<String> allStructFieldNames = structTypeInfo.getAllStructFieldNames();

View Full Code Here

    assertEquals(1, aoig.getColumnNames().size());
    assertEquals("aUnion", aoig.getColumnNames().get(0));


    // Column types
    assertEquals(1, aoig.getColumnTypes().size());
    TypeInfo typeInfo = aoig.getColumnTypes().get(0);
    assertTrue(typeInfo instanceof UnionTypeInfo);
    UnionTypeInfo uti = (UnionTypeInfo)typeInfo;


    // Check that the union has come out unscathed. No scathing of unions allowed.
    List<TypeInfo> typeInfos = uti.getAllUnionObjectTypeInfos();

View Full Code Here

    assertEquals(1, aoig.getColumnNames().size());
    assertEquals("hash", aoig.getColumnNames().get(0));


    // Column types
    assertEquals(1, aoig.getColumnTypes().size());
    TypeInfo typeInfo = aoig.getColumnTypes().get(0);
    assertTrue(typeInfo instanceof ListTypeInfo);
    ListTypeInfo listTypeInfo = (ListTypeInfo) typeInfo;
    assertTrue(listTypeInfo.getListElementTypeInfo() instanceof PrimitiveTypeInfo);
    assertEquals("tinyint", listTypeInfo.getListElementTypeInfo().getTypeName());
  }

View Full Code Here

    assertEquals(1, aoig.getColumnNames().size());
    assertEquals("bytesField", aoig.getColumnNames().get(0));


    // Column types
    assertEquals(1, aoig.getColumnTypes().size());
    TypeInfo typeInfo = aoig.getColumnTypes().get(0);
    assertTrue(typeInfo instanceof ListTypeInfo);
    ListTypeInfo listTypeInfo = (ListTypeInfo) typeInfo;
    assertTrue(listTypeInfo.getListElementTypeInfo() instanceof PrimitiveTypeInfo);
    assertEquals("tinyint", listTypeInfo.getListElementTypeInfo().getTypeName());
  }

View Full Code Here

    assertEquals(1, aoig.getColumnNames().size());
    assertEquals("nullableString", aoig.getColumnNames().get(0));


    // Column types
    assertEquals(1, aoig.getColumnTypes().size());
    TypeInfo typeInfo = aoig.getColumnTypes().get(0);
    assertTrue(typeInfo instanceof PrimitiveTypeInfo);
    PrimitiveTypeInfo pti = (PrimitiveTypeInfo) typeInfo;
    // Verify the union has been hidden and just the main type has been returned.
    assertEquals(PrimitiveObjectInspector.PrimitiveCategory.STRING, pti.getPrimitiveCategory());
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.hive.serde2.typeinfo.TypeInfo

com.asakusafw.directio.hive.orc.OrcFileInput

com.asakusafw.directio.hive.serde.DataModelDriver

com.asakusafw.directio.hive.serde.DataModelDriverTest

com.asakusafw.directio.hive.serde.ValueSerdeFactoryTest

com.cloudera.cdk.data.hcatalog.TestSchemaConversion

com.esri.hadoop.hive.serde.JsonSerde

com.facebook.hive.orc.lazy.LazyDoubleTreeReader

com.facebook.hive.orc.lazy.LazyShortDictionaryTreeReader

com.facebook.hive.orc.lazy.LazyShortDirectTreeReader

com.facebook.hive.orc.lazy.OrcLazyStructObjectInspector

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.