Examples of org.apache.hadoop.hive.serde2.typeinfo.TypeInfo

Package org.apache.hadoop.hive.serde2.typeinfo

Examples of org.apache.hadoop.hive.serde2.typeinfo.TypeInfo

org.apache.hadoop.hive.serde2.typeinfo.TypeInfo
Stores information about a type (DDL). Always use the TypeInfoFactory to create new TypeInfo objects. We support 4 categories of types: 1. Primitive objects (String, Number, etc) 2. List objects (a list of objects of a single type) 3. Map objects (a map from objects of one type to objects of another type) 4. Struct objects (a list of fields with names and their own types)

      // Put partial aggregation results in reduceValues
      int inputField = reduceKeys.size();


      for (Map.Entry<String, ASTNode> entry : aggregationTrees.entrySet()) {


        TypeInfo type = reduceSinkInputRowResolver.getColumnInfos().get(inputField).getType();
        reduceValues.add(new exprNodeColumnDesc(type, getColumnInternalName(inputField),
                                                "", false));
        inputField++;
        outputColumnNames.add(getColumnInternalName(reduceValues.size() - 1));
        String field = Utilities.ReduceField.VALUE.toString() + "." + getColumnInternalName(reduceValues.size() - 1);

View Full Code Here

    List<ASTNode> grpByExprs = getGroupByForClause(parseInfo, dest);
    for (int i = 0; i < grpByExprs.size(); ++i) {
      ASTNode grpbyExpr = grpByExprs.get(i);
      String field = getColumnInternalName(i);
      outputColumnNames.add(field);
      TypeInfo typeInfo = reduceSinkInputRowResolver2.get("", grpbyExpr.toStringTree()).getType();
      exprNodeColumnDesc inputExpr = new exprNodeColumnDesc(typeInfo, field, "", false);
      reduceKeys.add(inputExpr);
      ColumnInfo colInfo = new ColumnInfo(Utilities.ReduceField.KEY.toString() + "." + field,
           typeInfo, "", false);
      reduceSinkOutputRowResolver2.put("", grpbyExpr.toStringTree(),
                                       colInfo);
      colExprMap.put(colInfo.getInternalName(), inputExpr);
    }
    // Get partial aggregation results and store in reduceValues
    ArrayList<exprNodeDesc> reduceValues = new ArrayList<exprNodeDesc>();
    int inputField = reduceKeys.size();
    HashMap<String, ASTNode> aggregationTrees = parseInfo
        .getAggregationExprsForClause(dest);
    for (Map.Entry<String, ASTNode> entry : aggregationTrees.entrySet()) {
      String field = getColumnInternalName(inputField);
      ASTNode t = entry.getValue();
      TypeInfo typeInfo = reduceSinkInputRowResolver2.get("", t.toStringTree()).getType();
      reduceValues.add(new exprNodeColumnDesc(typeInfo, field, "", false));
      inputField++;
      String col = getColumnInternalName(reduceValues.size()-1);
      outputColumnNames.add(col);
      reduceSinkOutputRowResolver2.put("", t.toStringTree(),

View Full Code Here

    boolean isMetaDataSerDe = table_desc.getDeserializerClass().equals(MetadataTypedColumnsetSerDe.class);
    boolean isLazySimpleSerDe = table_desc.getDeserializerClass().equals(LazySimpleSerDe.class);
    if (!isMetaDataSerDe) {
      for (int i=0; i<columnNumber; i++) {
        ObjectInspector tableFieldOI = tableFields.get(i).getFieldObjectInspector();
        TypeInfo tableFieldTypeInfo = TypeInfoUtils.getTypeInfoFromObjectInspector(tableFieldOI);
        TypeInfo rowFieldTypeInfo = rowFields.get(i).getType();
        exprNodeDesc column = new exprNodeColumnDesc(rowFieldTypeInfo,
                                                     rowFields.get(i).getInternalName(), "", false);
        // LazySimpleSerDe can convert any types to String type using JSON-format.
        if (!tableFieldTypeInfo.equals(rowFieldTypeInfo)
            && !(isLazySimpleSerDe && tableFieldTypeInfo.getCategory().equals(Category.PRIMITIVE)

View Full Code Here

      keys.add(now.getKeyCols());
    }
    // implicit type conversion hierarchy
    for (int k = 0; k < keyLength; k++) {
      // Find the common class for type conversion
      TypeInfo commonType = keys.get(0).get(k).getTypeInfo();
      for(int i=1; i<right.length; i++) {
        TypeInfo a = commonType;
        TypeInfo b = keys.get(i).get(k).getTypeInfo();
        commonType = FunctionRegistry.getCommonClassForComparison(a, b);
        if (commonType == null) {
          throw new SemanticException("Cannot do equality join on different types: " + a.getTypeName() + " and " + b.getTypeName());
        }
      }
      // Add implicit type conversion if necessary
      for(int i=0; i<right.length; i++) {
        if (!commonType.equals(keys.get(i).get(k).getTypeInfo())) {

View Full Code Here

        Assert.assertEquals(STRING_CONST, (String) r.get(3));
    }


    @Test
    public void testGetWithName() throws Exception {
        TypeInfo ti = getTypeInfo();
        HCatRecord r = new LazyHCatRecord(getHCatRecord(), getObjectInspector(ti));
        HCatSchema schema = HCatSchemaUtils.getHCatSchema(ti)
                                              .get(0).getStructSubSchema();
        Assert.assertEquals(INT_CONST, ((Integer) r.get("an_int", schema)).intValue());
        Assert.assertEquals(LONG_CONST, ((Long) r.get("a_long", schema)).longValue());

View Full Code Here

      }
      }
    }
    case LIST: {
      ListTypeInfo ltype = (ListTypeInfo) type;
      TypeInfo etype = ltype.getListElementTypeInfo();


      // Create the list if needed
      ArrayList<Object> r = reuse == null ? new ArrayList<Object>()
          : (ArrayList<Object>) reuse;


      // Read the list
      int size = 0;
      while (true) {
        int more = buffer.read(invert);
        if (more == 0) {
          // \0 to terminate
          break;
        }
        // \1 followed by each element
        assert (more == 1);
        if (size == r.size()) {
          r.add(null);
        }
        r.set(size, deserialize(buffer, etype, invert, r.get(size)));
        size++;
      }
      // Remove additional elements if the list is reused
      while (r.size() > size) {
        r.remove(r.size() - 1);
      }
      return r;
    }
    case MAP: {
      MapTypeInfo mtype = (MapTypeInfo) type;
      TypeInfo ktype = mtype.getMapKeyTypeInfo();
      TypeInfo vtype = mtype.getMapValueTypeInfo();


      // Create the map if needed
      Map<Object, Object> r;
      if (reuse == null) {
        r = new HashMap<Object, Object>();

View Full Code Here

    List<? extends StructField> allStructFieldRefs = soi.getAllStructFieldRefs();
    List<Object> structFieldsDataAsList = soi.getStructFieldsDataAsList(o);


    for(int i  = 0; i < size; i++) {
      Field field = schema.getFields().get(i);
      TypeInfo typeInfo = columnTypes.get(i);
      StructField structFieldRef = allStructFieldRefs.get(i);
      Object structFieldData = structFieldsDataAsList.get(i);
      ObjectInspector fieldOI = structFieldRef.getFieldObjectInspector();


      Object val = serialize(typeInfo, fieldOI, structFieldData, field.schema());

View Full Code Here

    GenericData.Record record = new GenericData.Record(schema);
    ArrayList<TypeInfo> allStructFieldTypeInfos = typeInfo.getAllStructFieldTypeInfos();


    for(int i  = 0; i < size; i++) {
      Field field = schema.getFields().get(i);
      TypeInfo colTypeInfo = allStructFieldTypeInfos.get(i);
      StructField structFieldRef = allStructFieldRefs.get(i);
      Object structFieldData = structFieldsDataAsList.get(i);
      ObjectInspector fieldOI = structFieldRef.getFieldObjectInspector();


      Object val = serialize(colTypeInfo, fieldOI, structFieldData, field.schema());

View Full Code Here

      return serializeTransformedType(typeInfo, fieldOI, structFieldData, schema);
    
    List<?> list = fieldOI.getList(structFieldData);
    List<Object> deserialized = new ArrayList<Object>(list.size());


    TypeInfo listElementTypeInfo = typeInfo.getListElementTypeInfo();
    ObjectInspector listElementObjectInspector = fieldOI.getListElementObjectInspector();
    Schema elementType = schema.getElementType();


    for(int i = 0; i < list.size(); i++) {
      deserialized.add(i, serialize(listElementTypeInfo, listElementObjectInspector, list.get(i), elementType));

View Full Code Here

    if(!mapHasStringKey(fieldOI.getMapKeyObjectInspector()))
      throw new AvroSerdeException("Avro only supports maps with keys as Strings.  Current Map is: " + typeInfo.toString());


    ObjectInspector mapKeyObjectInspector = fieldOI.getMapKeyObjectInspector();
    ObjectInspector mapValueObjectInspector = fieldOI.getMapValueObjectInspector();
    TypeInfo mapKeyTypeInfo = typeInfo.getMapKeyTypeInfo();
    TypeInfo mapValueTypeInfo = typeInfo.getMapValueTypeInfo();
    Map<?,?> map = fieldOI.getMap(structFieldData);
    Schema valueType = schema.getValueType();


    Map<Object, Object> deserialized = new HashMap<Object, Object>(fieldOI.getMapSize(structFieldData));

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.hive.serde2.typeinfo.TypeInfo

com.asakusafw.directio.hive.orc.OrcFileInput

com.asakusafw.directio.hive.serde.DataModelDriver

com.asakusafw.directio.hive.serde.DataModelDriverTest

com.asakusafw.directio.hive.serde.ValueSerdeFactoryTest

com.cloudera.cdk.data.hcatalog.TestSchemaConversion

com.esri.hadoop.hive.serde.JsonSerde

com.facebook.hive.orc.lazy.LazyDoubleTreeReader

com.facebook.hive.orc.lazy.LazyShortDictionaryTreeReader

com.facebook.hive.orc.lazy.LazyShortDirectTreeReader

com.facebook.hive.orc.lazy.OrcLazyStructObjectInspector

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.