Examples of parquet.hadoop.metadata.ParquetMetadata

parquet.hadoop.metadata.ParquetMetadata
Meta Data block stored in the footer of the file contains file level (Codec, Schema, ...) and block level (location, columns, record count, ...) meta data @author Julien Le Dem

  }


  public ParquetMetadata readParquetMetadata(InputStream from) throws IOException {
    FileMetaData fileMetaData = readFileMetaData(from);
    if (Log.DEBUG) LOG.debug(fileMetaData);
    ParquetMetadata parquetMetadata = fromParquetMetadata(fileMetaData);
    if (Log.DEBUG) LOG.debug(ParquetMetadata.toPrettyJSON(parquetMetadata));
    return parquetMetadata;
  }

View Full Code Here

    if (key_value_metadata != null) {
      for (KeyValue keyValue : key_value_metadata) {
        keyValueMetaData.put(keyValue.key, keyValue.value);
      }
    }
    return new ParquetMetadata(
        new parquet.hadoop.metadata.FileMetaData(messageType, keyValueMetaData, parquetMetadata.getCreated_by()),
        blocks);
  }

View Full Code Here

    }


    private ParquetRecordReader<Void> createParquetRecordReader(Configuration configuration, Path path, long start, long length, List<HiveColumnHandle> columns)
    {
        try {
            ParquetMetadata parquetMetadata = ParquetFileReader.readFooter(configuration, path);
            List<BlockMetaData> blocks = parquetMetadata.getBlocks();
            FileMetaData fileMetaData = parquetMetadata.getFileMetaData();


            PrestoReadSupport readSupport = new PrestoReadSupport(columns, parquetMetadata.getFileMetaData().getSchema());
            ReadContext readContext = readSupport.init(configuration, fileMetaData.getKeyValueMetaData(), fileMetaData.getSchema());


            List<BlockMetaData> splitGroup = new ArrayList<>();
            long splitStart = start;
            long splitLength = length;

View Full Code Here

                  footers.get(e.getPath()),
                  rowGroupScan.getColumns()
              )
          );
        } else {
          ParquetMetadata footer = footers.get(e.getPath());
          readers.add(new DrillParquetReader(footer, e, columns, conf));
        }
        if (rowGroupScan.getSelectionRoot() != null) {
          String[] r = rowGroupScan.getSelectionRoot().split("/");
          String[] p = e.getPath().split("/");

View Full Code Here

0 1 2

TOP

Related Classes of parquet.hadoop.metadata.ParquetMetadata

com.asakusafw.directio.hive.parquet.ParquetFileInput

com.facebook.presto.hive.ParquetHiveRecordCursor

org.apache.drill.exec.store.parquet.ParquetGroupScan

org.apache.drill.exec.store.parquet.ParquetRecordReaderTest

org.apache.drill.exec.store.parquet.ParquetScanBatchCreator

org.apache.hadoop.hive.ql.io.parquet.read.ParquetRecordReaderWrapper

parquet.format.converter.ParquetMetadataConverter

parquet.hadoop.ParquetFileReader

parquet.hadoop.ParquetFileWriter

parquet.hadoop.ParquetInputFormat

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.