Package parquet.hadoop

Examples of parquet.hadoop.CodecFactoryExposer

63
64
65
66
67
68
69
70
71
  private final ParquetStorageEngineConfig engineConfig;

  public ParquetStorageEngine(ParquetStorageEngineConfig configuration, DrillbitContext context){
    this.context = context;
    this.schemaProvider = new ParquetSchemaProvider(configuration, context.getConfig());
    codecFactoryExposer = new CodecFactoryExposer(schemaProvider.conf);
    this.engineConfig = configuration;
    this.footer = null;
  }
View Full Code Here
76
77
78
79
80
81
82
83
84
85
86
    this(name, context, fs, storageConfig, new ParquetFormatConfig());
  }
 
  public ParquetFormatPlugin(String name, DrillbitContext context, DrillFileSystem fs, StoragePluginConfig storageConfig, ParquetFormatConfig formatConfig){
    this.context = context;
    this.codecFactoryExposer = new CodecFactoryExposer(fs.getUnderlying().getConf());
    this.config = formatConfig;
    this.formatMatcher = new ParquetFormatMatcher(this, fs);
    this.storageConfig = storageConfig;
    this.fs = fs;
    this.name = name == null ? "parquet" : name;
View Full Code Here
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
      for (ColumnChunkMetaData md : footer.getBlocks().get(entry.getRowGroupIndex()).getColumns()) {
        paths.put(md.getPath(), md);
      }

      CodecFactoryExposer codecFactoryExposer = new CodecFactoryExposer(conf);
      FileSystem fs = FileSystem.get(conf);
      Path filePath = new Path(entry.getPath());

      BlockMetaData blockMetaData = footer.getBlocks().get(entry.getRowGroupIndex());

      recordCount = (int) blockMetaData.getRowCount();

      ColumnChunkIncReadStore pageReadStore = new ColumnChunkIncReadStore(recordCount,
              codecFactoryExposer.getCodecFactory(), fs, filePath);

      for (String[] path : schema.getPaths()) {
        Type type = schema.getType(path);
        if (type.isPrimitive()) {
          ColumnChunkMetaData md = paths.get(ColumnPath.get(path));
View Full Code Here
512
513
514
515
516
517
518
519
520
521
522
    FileSystem fs = new CachedSingleFileSystem(fileName);
    BufferAllocator allocator = new TopLevelAllocator();
    for(int i = 0; i < 25; i++){
      ParquetRecordReader rr = new ParquetRecordReader(context, 256000, fileName, 0, fs,
          new CodecFactoryExposer(dfsConfig), f.getParquetMetadata(), columns);
      TestOutputMutator mutator = new TestOutputMutator(allocator);
      rr.setup(mutator);
      Stopwatch watch = new Stopwatch();
      watch.start();
View Full Code Here
578
579
580
581
582
583
584
585
586
587
588
    FileSystem fs = new CachedSingleFileSystem(fileName);
    BufferAllocator allocator = new TopLevelAllocator();
    for(int i = 0; i < 25; i++){
      ParquetRecordReader rr = new ParquetRecordReader(context, 256000, fileName, 0, fs,
          new CodecFactoryExposer(dfsConfig), f.getParquetMetadata(), columns);
      TestOutputMutator mutator = new TestOutputMutator(allocator);
      rr.setup(mutator);
      Stopwatch watch = new Stopwatch();
      watch.start();
View Full Code Here
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
      for (ColumnChunkMetaData md : footer.getBlocks().get(entry.getRowGroupIndex()).getColumns()) {
        paths.put(md.getPath(), md);
      }

      CodecFactoryExposer codecFactoryExposer = new CodecFactoryExposer(conf);
      FileSystem fs = FileSystem.get(conf);
      Path filePath = new Path(entry.getPath());

      BlockMetaData blockMetaData = footer.getBlocks().get(entry.getRowGroupIndex());

      recordCount = (int) blockMetaData.getRowCount();

      pageReadStore = new ColumnChunkIncReadStore(recordCount,
              codecFactoryExposer.getCodecFactory(), operatorContext.getAllocator(), fs, filePath);

      for (String[] path : schema.getPaths()) {
        Type type = schema.getType(path);
        if (type.isPrimitive()) {
          ColumnChunkMetaData md = paths.get(ColumnPath.get(path));
View Full Code Here
77
78
79
80
81
82
83
84
85
86
87
    this(name, context, fs, storageConfig, new ParquetFormatConfig());
  }

  public ParquetFormatPlugin(String name, DrillbitContext context, DrillFileSystem fs, StoragePluginConfig storageConfig, ParquetFormatConfig formatConfig){
    this.context = context;
    this.codecFactoryExposer = new CodecFactoryExposer(fs.getUnderlying().getConf());
    this.config = formatConfig;
    this.formatMatcher = new ParquetFormatMatcher(this, fs);
    this.storageConfig = storageConfig;
    this.fs = fs;
    this.name = name == null ? "parquet" : name;
View Full Code Here
630
631
632
633
634
635
636
637
638
639
640
    FileSystem fs = new CachedSingleFileSystem(fileName);
    BufferAllocator allocator = new TopLevelAllocator();
    for(int i = 0; i < 25; i++) {
      ParquetRecordReader rr = new ParquetRecordReader(context, 256000, fileName, 0, fs,
          new CodecFactoryExposer(dfsConfig), f.getParquetMetadata(), columns);
      TestOutputMutator mutator = new TestOutputMutator(allocator);
      rr.setup(mutator);
      Stopwatch watch = new Stopwatch();
      watch.start();
View Full Code Here
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
      for (ColumnChunkMetaData md : footer.getBlocks().get(entry.getRowGroupIndex()).getColumns()) {
        paths.put(md.getPath(), md);
      }

      CodecFactoryExposer codecFactoryExposer = new CodecFactoryExposer(conf);
      FileSystem fs = FileSystem.get(conf);
      Path filePath = new Path(entry.getPath());

      BlockMetaData blockMetaData = footer.getBlocks().get(entry.getRowGroupIndex());

      recordCount = (int) blockMetaData.getRowCount();

      pageReadStore = new ColumnChunkIncReadStore(recordCount,
              codecFactoryExposer.getCodecFactory(), operatorContext.getAllocator(), fs, filePath);

      for (String[] path : schema.getPaths()) {
        Type type = schema.getType(path);
        if (type.isPrimitive()) {
          ColumnChunkMetaData md = paths.get(ColumnPath.get(path));
View Full Code Here
78
79
80
81
82
83
84
85
86
87
88
    this(name, context, fs, storageConfig, new ParquetFormatConfig());
  }
 
  public ParquetFormatPlugin(String name, DrillbitContext context, DrillFileSystem fs, StoragePluginConfig storageConfig, ParquetFormatConfig formatConfig){
    this.context = context;
    this.codecFactoryExposer = new CodecFactoryExposer(fs.getUnderlying().getConf());
    this.config = formatConfig;
    this.formatMatcher = new ParquetFormatMatcher(this, fs);
    this.storageConfig = storageConfig;
    this.fs = fs;
    this.name = name == null ? "parquet" : name;
View Full Code Here
TOP

Related Classes of parquet.hadoop.CodecFactoryExposer

  • org.apache.drill.exec.store.parquet.ParquetFormatPlugin
  • org.apache.drill.exec.store.parquet.ParquetRecordReaderTest
  • org.apache.drill.exec.store.parquet.ParquetStorageEngine
  • org.apache.drill.exec.store.parquet2.DrillParquetReader
  • org.apache.hadoop.io.compress.CompressionCodec
  • org.apache.hadoop.io.compress.DirectDecompressionCodec
  • org.apache.hadoop.io.compress.DirectDecompressor
  • parquet.bytes.BytesInput
  • org.apache.drill.common.exceptions.DrillRuntimeException
  • java.nio.channels.WritableByteChannel
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.