Package parquet.schema

Examples of parquet.schema.GroupType

124
125
126
127
128
129
130
131
  // An optional group containing multiple elements
  private static GroupType convertStructType(final String name, final StructTypeInfo typeInfo) {
    final List<String> columnNames = typeInfo.getAllStructFieldNames();
    final List<TypeInfo> columnTypes = typeInfo.getAllStructFieldTypeInfos();
    return new GroupType(Repetition.OPTIONAL, name, convertTypes(columnNames, columnTypes));

  }
View Full Code Here
140
141
142
143
144
145
146
    return ConversionPatterns.mapType(Repetition.OPTIONAL, name, keyType, valueType);
  }

  private static GroupType listWrapper(final String name, final OriginalType originalType,
      final GroupType groupType) {
    return new GroupType(Repetition.OPTIONAL, name, originalType, groupType);
  }
View Full Code Here
65
66
67
68
69
70
71
72
73
74
75
76
77
  private PrimitiveConverter getPrimitiveConverter(ColumnDescriptor path) {
    Type currentType = schema;
    Converter currentConverter = recordConverter;
    for (String fieldName : path.getPath()) {
      final GroupType groupType = currentType.asGroupType();
      int fieldIndex = groupType.getFieldIndex(fieldName);
      currentType = groupType.getType(fieldName);
      currentConverter = currentConverter.asGroupConverter().getConverter(fieldIndex);
    }
    PrimitiveConverter converter = currentConverter.asPrimitiveConverter();
    return converter;
  }
View Full Code Here
184
185
186
187
188
189
190
191
    Type type = schema.getType(Arrays.copyOfRange(pathSegments, 0, depth + 1));
    if (depth + 1 == pathSegments.length) {
      return type;
    } else {
      Preconditions.checkState(!type.isPrimitive());
      return new GroupType(type.getRepetition(), type.getName(), getType(pathSegments, depth + 1, schema));
    }
  }
View Full Code Here
163
164
165
166
167
168
169
170
171
172
173
      case MAP:
        List<parquet.schema.Type> types = Lists.newArrayList();
        for (MaterializedField childField : field.getChildren()) {
          types.add(getType(childField));
        }
        return new GroupType(dataMode == DataMode.REPEATED ? Repetition.REPEATED : Repetition.OPTIONAL, field.getLastName(), types);
      case LIST:
        throw new UnsupportedOperationException("Unsupported type " + minorType);
      default:
        return getPrimitiveType(field);
    }
View Full Code Here
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
  public static void main(String[] args) {
    MessageType message1;
    MessageType message2;

    PrimitiveType c = new PrimitiveType(Repetition.OPTIONAL, PrimitiveTypeName.INT32, "c");
    GroupType b = new GroupType(Repetition.REQUIRED, "b");
    GroupType a = new GroupType(Repetition.OPTIONAL, "a", b);
    message1 = new MessageType("root", a);

    PrimitiveType c2 = new PrimitiveType(Repetition.OPTIONAL, PrimitiveTypeName.INT32, "d");
    GroupType b2 = new GroupType(Repetition.OPTIONAL, "b", c2);
    GroupType a2 = new GroupType(Repetition.OPTIONAL, "a", b2);
    message2 = new MessageType("root", a2);

    MessageType message3 = message1.union(message2);

    StringBuilder builder = new StringBuilder();
View Full Code Here
402
403
404
405
406
407
408
409
410
411
412
              getPrimitive(schemaElement.getType()),
              name,
              originalType);
        }
      } else {
        result[i] = new GroupType(
            repetition,
            name,
            originalType,
            convertChildren(schema, schemaElement.getNum_children()));
      }
View Full Code Here
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
                    parquet.schema.Type parquetType = messageType.getFields().get(column.getHiveColumnIndex());
                    if (parquetType.isPrimitive()) {
                        converters.add(new ParquetPrimitiveColumnConverter(i));
                    }
                    else {
                        GroupType groupType = parquetType.asGroupType();
                        switch (groupType.getOriginalType()) {
                            case LIST:
                                converters.add(new ParquetJsonColumnConverter(new ParquetListJsonConverter(groupType.getName(), null, groupType), i));
                                break;
                            case MAP:
                            case MAP_KEY_VALUE: // original versions of Parquet have map and entry swapped
                                converters.add(new ParquetJsonColumnConverter(new ParquetMapJsonConverter(groupType.getName(), null, groupType), i));
                                break;
                            case UTF8:
                            case ENUM:
                                throw new IllegalArgumentException("Group column " + groupType.getName() + " type " + groupType.getOriginalType() + " not supported");
                        }
                    }
                }
            }
            this.converters = converters.build();
View Full Code Here
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
                        columnName,
                        MAP_KEY_VALUE,
                        entryType);
            }

            GroupType entryGroupType = entryType.asGroupType();
            checkArgument(entryGroupType.getFieldCount() == 2,
                    "Expected MAP column '%s' entry to have two fields, but has %s fields",
                    columnName,
                    entryGroupType.getFieldCount());
            checkArgument(entryGroupType.getFieldName(0).equals("key"),
                    "Expected MAP column '%s' entry field 0 to be named 'key', but is named %s",
                    columnName,
                    entryGroupType.getFieldName(0));
            checkArgument(entryGroupType.getFieldName(1).equals("value"),
                    "Expected MAP column '%s' entry field 1 to be named 'value', but is named %s",
                    columnName,
                    entryGroupType.getFieldName(1));
            checkArgument(entryGroupType.getType(0).isPrimitive(),
                    "Expected MAP column '%s' entry field 0 to be primitive, but is named %s",
                    columnName,
                    entryGroupType.getType(0));

            keyConverter = new ParquetMapKeyJsonConverter();
            valueConverter = createJsonConverter(columnName + ".value", null, entryGroupType.getFields().get(1));
        }
View Full Code Here
186
187
188
189
190
191
192
193
    Type type = schema.getType(Arrays.copyOfRange(pathSegments, 0, depth + 1));
    if (depth + 1 == pathSegments.length) {
      return type;
    } else {
      Preconditions.checkState(!type.isPrimitive());
      return new GroupType(type.getRepetition(), type.getName(), getType(pathSegments, depth + 1, schema));
    }
  }
View Full Code Here
TOP

Related Classes of parquet.schema.GroupType

  • com.facebook.presto.hive.ParquetHiveRecordCursor$ParquetMapEntryJsonConverter
  • com.facebook.presto.hive.ParquetHiveRecordCursor$PrestoReadSupport
  • org.apache.drill.exec.store.parquet.ParquetRecordWriter
  • org.apache.drill.exec.store.parquet2.DrillParquetReader
  • org.apache.drill.ParquetSchemaMerge
  • org.apache.hadoop.hive.ql.io.parquet.convert.HiveSchemaConverter
  • parquet.column.impl.ColumnReadStoreImpl
  • parquet.format.converter.ParquetMetadataConverter
  • parquet.io.InvalidRecordException
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.