Package parquet.column.page

Examples of parquet.column.page.DictionaryPage

266
267
268
269
270
271
272
273
274
        // write only the part of the dict that we used
        for (int i = 0; i < lastUsedDictionarySize; i++) {
          Binary entry = binaryIterator.next();
          dictionaryEncoder.writeBytes(entry);
        }
        return new DictionaryPage(dictionaryEncoder.getBytes(), lastUsedDictionarySize, PLAIN_DICTIONARY);
      }
      return plainValuesWriter.createDictionaryPage();
    }
View Full Code Here
125
126
127
128
129
130
131
        throw new ParquetEncodingException("Only one dictionary page is allowed");
      }
      BytesInput dictionaryBytes = dictionaryPage.getBytes();
      int uncompressedSize = (int)dictionaryBytes.size();
      BytesInput compressedBytes = compressor.compress(dictionaryBytes);
      this.dictionaryPage = new DictionaryPage(BytesInput.copy(compressedBytes), uncompressedSize, dictionaryPage.getDictionarySize(), dictionaryPage.getEncoding());
    }
View Full Code Here
91
92
93
94
95
96
97
98
99
100
101
    public DictionaryPage readDictionaryPage() {
      if (compressedDictionaryPage == null) {
        return null;
      }
      try {
        return new DictionaryPage(
            decompressor.decompress(compressedDictionaryPage.getBytes(), compressedDictionaryPage.getUncompressedSize()),
            compressedDictionaryPage.getDictionarySize(),
            compressedDictionaryPage.getEncoding());
      } catch (IOException e) {
        throw new RuntimeException(e); // TODO: cleanup
View Full Code Here
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
     * Read all of the pages in a given column chunk.
     * @return the list of pages
     */
    public ColumnChunkPageReader readAllPages() throws IOException {
      List<Page> pagesInChunk = new ArrayList<Page>();
      DictionaryPage dictionaryPage = null;
      long valuesCountReadSoFar = 0;
      while (valuesCountReadSoFar < descriptor.metadata.getValueCount()) {
        PageHeader pageHeader = readPageHeader(this);
        switch (pageHeader.type) {
          case DICTIONARY_PAGE:
            // there is only one dictionary page per column chunk
            if (dictionaryPage != null) {
              throw new ParquetDecodingException("more than one dictionary page in column " + descriptor.col);
            }
            dictionaryPage =
                new DictionaryPage(
                    this.readAsBytesInput(pageHeader.compressed_page_size),
                    pageHeader.uncompressed_page_size,
                    pageHeader.dictionary_page_header.num_values,
                    parquetMetadataConverter.getEncoding(pageHeader.dictionary_page_header.encoding)
                    );
View Full Code Here
98
99
100
101
102
103
104
105
106
107
108
          if (pageHeader.getDictionary_page_header() == null) {
            in.seek(pos);
            return null;
          }
          dictionaryPage =
                  new DictionaryPage(
                          decompressor.decompress(BytesInput.from(in, pageHeader.compressed_page_size), pageHeader.getUncompressed_page_size()),
                          pageHeader.getDictionary_page_header().getNum_values(),
                          parquetMetadataConverter.getEncoding(pageHeader.dictionary_page_header.encoding)
                  );
        } catch (IOException e) {
View Full Code Here
128
129
130
131
132
133
134
135
136
137
138
          PageHeader pageHeader = Util.readPageHeader(in);
          switch (pageHeader.type) {
            case DICTIONARY_PAGE:
              if (dictionaryPage == null) {
                dictionaryPage =
                        new DictionaryPage(
                                decompressor.decompress(BytesInput.from(in, pageHeader.compressed_page_size), pageHeader.getUncompressed_page_size()),
                                pageHeader.uncompressed_page_size,
                                parquetMetadataConverter.getEncoding(pageHeader.dictionary_page_header.encoding)
                        );
              } else {
View Full Code Here
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
              uncompressedData,
              pageHeader.compressed_page_size,
              pageHeader.getUncompressed_page_size());
          compressedData.release();
        }
        DictionaryPage page = new DictionaryPage(
            bytesIn,
            pageHeader.uncompressed_page_size,
            pageHeader.dictionary_page_header.num_values,
            parquet.column.Encoding.valueOf(pageHeader.dictionary_page_header.encoding.name())
        );
        this.dictionary = page.getEncoding().initDictionary(parentStatus.columnDescriptor, page);
      }
    } catch (IOException e) {
      throw new ExecutionSetupException("Error opening or reading metadata for parquet file at location: "
        + path.getName(), e);
    }
View Full Code Here
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
              uncompressedData,
              pageHeader.compressed_page_size,
              pageHeader.getUncompressed_page_size());
          compressedData.release();
        }
        DictionaryPage page = new DictionaryPage(
            bytesIn,
            pageHeader.uncompressed_page_size,
            pageHeader.dictionary_page_header.num_values,
            parquet.column.Encoding.valueOf(pageHeader.dictionary_page_header.encoding.name())
        );
        this.dictionary = page.getEncoding().initDictionary(parentColumnReader.columnDescriptor, page);
      }
    } while (pageHeader.getType() == PageType.DICTIONARY_PAGE);

    //TODO: Handle buffer allocation exception
    BytesInput bytesIn;
View Full Code Here
99
100
101
102
103
104
105
106
107
108
109
          if (pageHeader.getDictionary_page_header() == null) {
            in.seek(pos);
            return null;
          }
          dictionaryPage =
                  new DictionaryPage(
                          decompressor.decompress(BytesInput.from(in, pageHeader.compressed_page_size), pageHeader.getUncompressed_page_size()),
                          pageHeader.getDictionary_page_header().getNum_values(),
                          parquetMetadataConverter.getEncoding(pageHeader.dictionary_page_header.encoding)
                  );
        } catch (IOException e) {
View Full Code Here
129
130
131
132
133
134
135
136
137
138
139
          PageHeader pageHeader = Util.readPageHeader(in);
          switch (pageHeader.type) {
            case DICTIONARY_PAGE:
              if (dictionaryPage == null) {
                dictionaryPage =
                        new DictionaryPage(
                                decompressor.decompress(BytesInput.from(in, pageHeader.compressed_page_size), pageHeader.getUncompressed_page_size()),
                                pageHeader.uncompressed_page_size,
                                parquetMetadataConverter.getEncoding(pageHeader.dictionary_page_header.encoding)
                        );
              } else {
View Full Code Here
TOP

Related Classes of parquet.column.page.DictionaryPage

  • org.apache.drill.exec.store.parquet.columnreaders.PageReader
  • parquet.column.impl.ColumnReaderImpl
  • parquet.column.impl.ColumnWriterImpl
  • parquet.column.values.dictionary.DictionaryValuesWriter$PlainBinaryDictionaryValuesWriter
  • parquet.column.values.dictionary.DictionaryValuesWriter$PlainDoubleDictionaryValuesWriter
  • parquet.column.values.dictionary.DictionaryValuesWriter$PlainFloatDictionaryValuesWriter
  • parquet.column.values.dictionary.DictionaryValuesWriter$PlainIntegerDictionaryValuesWriter
  • parquet.column.values.dictionary.DictionaryValuesWriter$PlainLongDictionaryValuesWriter
  • parquet.hadoop.ColumnChunkIncReadStore$ColumnChunkIncPageReader
  • parquet.hadoop.ColumnChunkPageReadStore$ColumnChunkPageReader
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.