Package com.facebook.hive.orc

Examples of com.facebook.hive.orc.InStream$CompressedStream


    dictionarySize = encodings.get(columnId).getDictionarySize();

    // read the lengths
    StreamName name = new StreamName(columnId, OrcProto.Stream.Kind.LENGTH);
    InStream in = streams.get(name);
    RunLengthIntegerReader lenReader = new RunLengthIntegerReader(in, false,
        WriterImpl.INT_BYTE_SIZE);
    int offset = 0;
    if (dictionaryOffsets == null ||
        dictionaryOffsets.length < dictionarySize + 1) {
      dictionaryOffsets = new int[dictionarySize + 1];
    }
    for(int i=0; i < dictionarySize; ++i) {
      dictionaryOffsets[i] = offset;
      offset += (int) lenReader.next();
    }
    dictionaryOffsets[dictionarySize] = offset;
    in.close();

    // read the dictionary blob
    name = new StreamName(columnId,
        OrcProto.Stream.Kind.DICTIONARY_DATA);
    in = streams.get(name);
    if (in.available() > 0) {
      dictionaryBuffer = new DynamicByteArray(dictionaryOffsets[dictionarySize],
          DUMMY_MEMORY_ESTIMATE);
      dictionaryBuffer.readAll(in);
    } else {
      dictionaryBuffer = null;
    }
    in.close();

    // set up the row reader
    name = new StreamName(columnId, OrcProto.Stream.Kind.DATA);
    reader = new RunLengthIntegerReader(streams.get(name), false, WriterImpl.INT_BYTE_SIZE);
    InStream inDictionaryStream = streams.get(new StreamName(columnId, OrcProto.Stream.Kind.IN_DICTIONARY));
    inDictionary = inDictionaryStream == null ? null : new BitFieldReader(inDictionaryStream);
    directReader = streams.get(new StreamName(columnId, OrcProto.Stream.Kind.STRIDE_DICTIONARY));
    InStream directLengthsStream = streams.get(new StreamName(columnId,
        OrcProto.Stream.Kind.STRIDE_DICTIONARY_LENGTH));
    directLengths = directLengthsStream == null ? null : new RunLengthIntegerReader(
        directLengthsStream, false, WriterImpl.INT_BYTE_SIZE);
    if (indexes[columnId] != null) {
      loadIndeces(indexes[columnId].getEntryList(), 0);
View Full Code Here


    this.previousRow = rowBaseInStripe;
    this.previousPresentRow = rowBaseInStripe;
    this.rowBaseInStripe = rowBaseInStripe;
    this.numNonNulls = 0;

    InStream in = streams.get(new StreamName(columnId,
        OrcProto.Stream.Kind.PRESENT));
    if (in == null) {
      present = null;
      valuePresent = true;
    } else {
View Full Code Here

    // read the dictionary blob
    dictionarySize = encodings.get(columnId).getDictionarySize();
    dictionaryValues = new long[dictionarySize];
    StreamName name = new StreamName(columnId,
        OrcProto.Stream.Kind.DICTIONARY_DATA);
    InStream in = streams.get(name);
    for(int i = 0; i < dictionarySize; ++i) {
      dictionaryValues[i] = SerializationUtils.readIntegerType(in, WriterImpl.INT_BYTE_SIZE,
          true, in.useVInts());
    }
    in.close();
    // set up the row reader
    name = new StreamName(columnId, OrcProto.Stream.Kind.DATA);
    reader = new RunLengthIntegerReader(streams.get(name), false, getNumBytes());
    InStream inDictionaryStream = streams.get(new StreamName(columnId,
        OrcProto.Stream.Kind.IN_DICTIONARY));
    inDictionary = inDictionaryStream == null ? null : new BitFieldReader(inDictionaryStream);
    if (indexes[columnId] != null) {
      loadIndeces(indexes[columnId].getEntryList(), 0);
    }
View Full Code Here

    dictionarySize = encodings.get(columnId).getDictionarySize();

    // read the lengths
    StreamName name = new StreamName(columnId, OrcProto.Stream.Kind.LENGTH);
    InStream in = streams.get(name);
    RunLengthIntegerReader lenReader = new RunLengthIntegerReader(in, false,
        WriterImpl.INT_BYTE_SIZE);
    int offset = 0;
    if (dictionaryOffsets == null ||
        dictionaryOffsets.length < dictionarySize + 1) {
      dictionaryOffsets = new int[dictionarySize + 1];
    }
    for(int i=0; i < dictionarySize; ++i) {
      dictionaryOffsets[i] = offset;
      offset += (int) lenReader.next();
    }
    dictionaryOffsets[dictionarySize] = offset;
    in.close();

    // read the dictionary blob
    name = new StreamName(columnId,
        OrcProto.Stream.Kind.DICTIONARY_DATA);
    in = streams.get(name);
    if (in.available() > 0) {
      dictionaryBuffer = new DynamicByteArray(dictionaryOffsets[dictionarySize]);
      dictionaryBuffer.readAll(in);
    } else {
      dictionaryBuffer = null;
    }
    in.close();

    // set up the row reader
    name = new StreamName(columnId, OrcProto.Stream.Kind.DATA);
    reader = new RunLengthIntegerReader(streams.get(name), false, WriterImpl.INT_BYTE_SIZE);
    InStream inDictionaryStream = streams.get(new StreamName(columnId, OrcProto.Stream.Kind.IN_DICTIONARY));
    inDictionary = inDictionaryStream == null ? null : new BitFieldReader(inDictionaryStream);
    directReader = streams.get(new StreamName(columnId, OrcProto.Stream.Kind.STRIDE_DICTIONARY));
    InStream directLengthsStream = streams.get(new StreamName(columnId,
        OrcProto.Stream.Kind.STRIDE_DICTIONARY_LENGTH));
    directLengths = directLengthsStream == null ? null : new RunLengthIntegerReader(
        directLengthsStream, false, WriterImpl.INT_BYTE_SIZE);
    if (indexes[columnId] != null) {
      loadIndeces(indexes[columnId].getEntryList(), 0);
View Full Code Here

TOP

Related Classes of com.facebook.hive.orc.InStream$CompressedStream

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.