Package parquet.hadoop

Examples of parquet.hadoop.CodecFactoryExposer


  private final ParquetStorageEngineConfig engineConfig;

  public ParquetStorageEngine(ParquetStorageEngineConfig configuration, DrillbitContext context){
    this.context = context;
    this.schemaProvider = new ParquetSchemaProvider(configuration, context.getConfig());
    codecFactoryExposer = new CodecFactoryExposer(schemaProvider.conf);
    this.engineConfig = configuration;
    this.footer = null;
  }
View Full Code Here


    this(name, context, fs, storageConfig, new ParquetFormatConfig());
  }
 
  public ParquetFormatPlugin(String name, DrillbitContext context, DrillFileSystem fs, StoragePluginConfig storageConfig, ParquetFormatConfig formatConfig){
    this.context = context;
    this.codecFactoryExposer = new CodecFactoryExposer(fs.getUnderlying().getConf());
    this.config = formatConfig;
    this.formatMatcher = new ParquetFormatMatcher(this, fs);
    this.storageConfig = storageConfig;
    this.fs = fs;
    this.name = name == null ? "parquet" : name;
View Full Code Here

      for (ColumnChunkMetaData md : footer.getBlocks().get(entry.getRowGroupIndex()).getColumns()) {
        paths.put(md.getPath(), md);
      }

      CodecFactoryExposer codecFactoryExposer = new CodecFactoryExposer(conf);
      FileSystem fs = FileSystem.get(conf);
      Path filePath = new Path(entry.getPath());

      BlockMetaData blockMetaData = footer.getBlocks().get(entry.getRowGroupIndex());

      recordCount = (int) blockMetaData.getRowCount();

      ColumnChunkIncReadStore pageReadStore = new ColumnChunkIncReadStore(recordCount,
              codecFactoryExposer.getCodecFactory(), fs, filePath);

      for (String[] path : schema.getPaths()) {
        Type type = schema.getType(path);
        if (type.isPrimitive()) {
          ColumnChunkMetaData md = paths.get(ColumnPath.get(path));
View Full Code Here

    FileSystem fs = new CachedSingleFileSystem(fileName);
    BufferAllocator allocator = new TopLevelAllocator();
    for(int i = 0; i < 25; i++){
      ParquetRecordReader rr = new ParquetRecordReader(context, 256000, fileName, 0, fs,
          new CodecFactoryExposer(dfsConfig), f.getParquetMetadata(), columns);
      TestOutputMutator mutator = new TestOutputMutator(allocator);
      rr.setup(mutator);
      Stopwatch watch = new Stopwatch();
      watch.start();
View Full Code Here

    FileSystem fs = new CachedSingleFileSystem(fileName);
    BufferAllocator allocator = new TopLevelAllocator();
    for(int i = 0; i < 25; i++){
      ParquetRecordReader rr = new ParquetRecordReader(context, 256000, fileName, 0, fs,
          new CodecFactoryExposer(dfsConfig), f.getParquetMetadata(), columns);
      TestOutputMutator mutator = new TestOutputMutator(allocator);
      rr.setup(mutator);
      Stopwatch watch = new Stopwatch();
      watch.start();
View Full Code Here

      for (ColumnChunkMetaData md : footer.getBlocks().get(entry.getRowGroupIndex()).getColumns()) {
        paths.put(md.getPath(), md);
      }

      CodecFactoryExposer codecFactoryExposer = new CodecFactoryExposer(conf);
      FileSystem fs = FileSystem.get(conf);
      Path filePath = new Path(entry.getPath());

      BlockMetaData blockMetaData = footer.getBlocks().get(entry.getRowGroupIndex());

      recordCount = (int) blockMetaData.getRowCount();

      pageReadStore = new ColumnChunkIncReadStore(recordCount,
              codecFactoryExposer.getCodecFactory(), operatorContext.getAllocator(), fs, filePath);

      for (String[] path : schema.getPaths()) {
        Type type = schema.getType(path);
        if (type.isPrimitive()) {
          ColumnChunkMetaData md = paths.get(ColumnPath.get(path));
View Full Code Here

    this(name, context, fs, storageConfig, new ParquetFormatConfig());
  }

  public ParquetFormatPlugin(String name, DrillbitContext context, DrillFileSystem fs, StoragePluginConfig storageConfig, ParquetFormatConfig formatConfig){
    this.context = context;
    this.codecFactoryExposer = new CodecFactoryExposer(fs.getUnderlying().getConf());
    this.config = formatConfig;
    this.formatMatcher = new ParquetFormatMatcher(this, fs);
    this.storageConfig = storageConfig;
    this.fs = fs;
    this.name = name == null ? "parquet" : name;
View Full Code Here

    FileSystem fs = new CachedSingleFileSystem(fileName);
    BufferAllocator allocator = new TopLevelAllocator();
    for(int i = 0; i < 25; i++) {
      ParquetRecordReader rr = new ParquetRecordReader(context, 256000, fileName, 0, fs,
          new CodecFactoryExposer(dfsConfig), f.getParquetMetadata(), columns);
      TestOutputMutator mutator = new TestOutputMutator(allocator);
      rr.setup(mutator);
      Stopwatch watch = new Stopwatch();
      watch.start();
View Full Code Here

      for (ColumnChunkMetaData md : footer.getBlocks().get(entry.getRowGroupIndex()).getColumns()) {
        paths.put(md.getPath(), md);
      }

      CodecFactoryExposer codecFactoryExposer = new CodecFactoryExposer(conf);
      FileSystem fs = FileSystem.get(conf);
      Path filePath = new Path(entry.getPath());

      BlockMetaData blockMetaData = footer.getBlocks().get(entry.getRowGroupIndex());

      recordCount = (int) blockMetaData.getRowCount();

      pageReadStore = new ColumnChunkIncReadStore(recordCount,
              codecFactoryExposer.getCodecFactory(), operatorContext.getAllocator(), fs, filePath);

      for (String[] path : schema.getPaths()) {
        Type type = schema.getType(path);
        if (type.isPrimitive()) {
          ColumnChunkMetaData md = paths.get(ColumnPath.get(path));
View Full Code Here

    this(name, context, fs, storageConfig, new ParquetFormatConfig());
  }
 
  public ParquetFormatPlugin(String name, DrillbitContext context, DrillFileSystem fs, StoragePluginConfig storageConfig, ParquetFormatConfig formatConfig){
    this.context = context;
    this.codecFactoryExposer = new CodecFactoryExposer(fs.getUnderlying().getConf());
    this.config = formatConfig;
    this.formatMatcher = new ParquetFormatMatcher(this, fs);
    this.storageConfig = storageConfig;
    this.fs = fs;
    this.name = name == null ? "parquet" : name;
View Full Code Here

TOP

Related Classes of parquet.hadoop.CodecFactoryExposer

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.