Package org.apache.drill.exec.store.parquet.columnreaders

Examples of org.apache.drill.exec.store.parquet.columnreaders.ParquetRecordReader

511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
    int totalRowCount = 0;

    FileSystem fs = new CachedSingleFileSystem(fileName);
    BufferAllocator allocator = new TopLevelAllocator();
    for(int i = 0; i < 25; i++){
      ParquetRecordReader rr = new ParquetRecordReader(context, 256000, fileName, 0, fs,
          new CodecFactoryExposer(dfsConfig), f.getParquetMetadata(), columns);
      TestOutputMutator mutator = new TestOutputMutator(allocator);
      rr.setup(mutator);
      Stopwatch watch = new Stopwatch();
      watch.start();

      int rowCount = 0;
      while ((rowCount = rr.next()) > 0) {
        totalRowCount += rowCount;
      }
      System.out.println(String.format("Time completed: %s. ", watch.elapsed(TimeUnit.MILLISECONDS)));
      rr.cleanup();
    }

    allocator.close();
    System.out.println(String.format("Total row count %s", totalRowCount));
  }
View Full Code Here
104
105
106
107
108
109
110
111
112
113
114
          footers.put(e.getPath(),
              ParquetFileReader.readFooter( fs.getConf(), new Path(e.getPath())));
        }
        if (!context.getOptions().getOption(ExecConstants.PARQUET_NEW_RECORD_READER).bool_val && !isComplex(footers.get(e.getPath()))) {
          readers.add(
              new ParquetRecordReader(
                  context, e.getPath(), e.getRowGroupIndex(), fs,
                  rowGroupScan.getStorageEngine().getCodecFactoryExposer(),
                  footers.get(e.getPath()),
                  rowGroupScan.getColumns()
              )
View Full Code Here
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
    int totalRowCount = 0;

    FileSystem fs = new CachedSingleFileSystem(fileName);
    BufferAllocator allocator = new TopLevelAllocator();
    for(int i = 0; i < 25; i++){
      ParquetRecordReader rr = new ParquetRecordReader(context, 256000, fileName, 0, fs,
          new CodecFactoryExposer(dfsConfig), f.getParquetMetadata(), columns);
      TestOutputMutator mutator = new TestOutputMutator(allocator);
      rr.setup(mutator);
      Stopwatch watch = new Stopwatch();
      watch.start();

      int rowCount = 0;
      while ((rowCount = rr.next()) > 0) {
        totalRowCount += rowCount;
      }
      System.out.println(String.format("Time completed: %s. ", watch.elapsed(TimeUnit.MILLISECONDS)));
      rr.cleanup();
    }

    allocator.close();
    System.out.println(String.format("Total row count %s", totalRowCount));
  }
View Full Code Here
107
108
109
110
111
112
113
114
115
116
117
          footers.put(e.getPath(),
              ParquetFileReader.readFooter( fs.getConf(), new Path(e.getPath())));
        }
        if (!context.getOptions().getOption(ExecConstants.PARQUET_NEW_RECORD_READER).bool_val && !isComplex(footers.get(e.getPath()))) {
          readers.add(
              new ParquetRecordReader(
                  context, e.getPath(), e.getRowGroupIndex(), fs,
                  rowGroupScan.getStorageEngine().getCodecFactoryExposer(),
                  footers.get(e.getPath()),
                  rowGroupScan.getColumns()
              )
View Full Code Here
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
    int totalRowCount = 0;

    FileSystem fs = new CachedSingleFileSystem(fileName);
    BufferAllocator allocator = new TopLevelAllocator();
    for(int i = 0; i < 25; i++) {
      ParquetRecordReader rr = new ParquetRecordReader(context, 256000, fileName, 0, fs,
          new CodecFactoryExposer(dfsConfig), f.getParquetMetadata(), columns);
      TestOutputMutator mutator = new TestOutputMutator(allocator);
      rr.setup(mutator);
      Stopwatch watch = new Stopwatch();
      watch.start();

      int rowCount = 0;
      while ((rowCount = rr.next()) > 0) {
        totalRowCount += rowCount;
      }
      System.out.println(String.format("Time completed: %s. ", watch.elapsed(TimeUnit.MILLISECONDS)));
      rr.cleanup();
    }

    allocator.close();
    System.out.println(String.format("Total row count %s", totalRowCount));
  }
View Full Code Here
108
109
110
111
112
113
114
115
116
117
118
          footers.put(e.getPath(),
              ParquetFileReader.readFooter( fs.getConf(), new Path(e.getPath())));
        }
        if (!context.getOptions().getOption(ExecConstants.PARQUET_NEW_RECORD_READER).bool_val && !isComplex(footers.get(e.getPath()))) {
          readers.add(
              new ParquetRecordReader(
                  context, e.getPath(), e.getRowGroupIndex(), fs,
                  rowGroupScan.getStorageEngine().getCodecFactoryExposer(),
                  footers.get(e.getPath()),
                  rowGroupScan.getColumns()
              )
View Full Code Here
TOP

Related Classes of org.apache.drill.exec.store.parquet.columnreaders.ParquetRecordReader

  • org.apache.drill.common.expression.SchemaPath
  • org.apache.drill.common.types.TypeProtos.MajorType
  • org.apache.drill.exec.record.MaterializedField
  • org.apache.drill.exec.store.parquet.ParquetRecordReaderTest
  • org.apache.drill.exec.store.parquet.ParquetScanBatchCreator
  • org.apache.drill.exec.vector.ValueVector
  • org.apache.hadoop.fs.Path
  • parquet.column.ColumnDescriptor
  • parquet.format.ConvertedType
  • parquet.format.converter.ParquetMetadataConverter
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.