* Licensed to the Apache Software Foundation (ASF) under one

* or more contributor license agreements.  See the NOTICE file

* distributed with this work for additional information

* regarding copyright ownership.  The ASF licenses this file

* to you under the Apache License, Version 2.0 (the

* "License"); you may not use this file except in compliance

* with the License.  You may obtain a copy of the License at

*

*     http://www.apache.org/licenses/LICENSE-2.0

*

* Unless required by applicable law or agreed to in writing, software

* distributed under the License is distributed on an "AS IS" BASIS,

* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

* See the License for the specific language governing permissions and

* limitations under the License.

*/

* An ORC file writer. The file is divided into stripes, which is the natural

* unit of work when reading. Each stripe is buffered in memory until the

* memory reaches the stripe size and then it is written out broken down by

* columns. Each column is written by a TreeWriter that is specific to that

* type of column. TreeWriters may have children TreeWriters that handle the

* sub-types. Each of the TreeWriters writes the column's data as a set of

* streams.

*

* This class is synchronized so that multi-threaded access is ok. In

* particular, because the MemoryManager is shared between writers, this class

* assumes that checkMemory may be called from a separate thread.

*/

  private static final Log LOG = LogFactory.getLog(WriterImpl.class);

  private static final int HDFS_BUFFER_SIZE = 256 * 1024;

  private static final int MIN_ROW_INDEX_STRIDE = 1000;

  public static final int SHORT_BYTE_SIZE = 2;

  public static final int INT_BYTE_SIZE = 4;

  public static final int LONG_BYTE_SIZE = 8;

  // Specifies how many index entries are created for a present stream

  public static final int UNCOMPRESSED_PRESENT_STREAM_INDEX_ENTRIES = 3;

  public static final int COMPRESSED_PRESENT_STREAM_INDEX_ENTRIES = 4;

  private final FileSystem fs;

  private final Path path;

  private final long stripeSize;

  private final int rowIndexStride;

  private final CompressionKind compress;

  private final CompressionCodec codec;

  private final int bufferSize;

  // the streams that make up the current stripe

  private final Map<StreamName, BufferedStream> streams =

    new TreeMap<StreamName, BufferedStream>();

  private FSDataOutputStream rawWriter = null;

  // the compressed metadata information outStream

  private OutStream writer = null;

  // a protobuf outStream around streamFactory

  private CodedOutputStream protobufWriter = null;

  private long headerLength;

  private int columnCount;

  private long rowCount = 0;

  private long rowsInStripe = 0;

  private int rowsInIndex = 0;

  private long rawDataSize = 0;

  private final List<OrcProto.StripeInformation> stripes =

    new ArrayList<OrcProto.StripeInformation>();

  private final Map<String, ByteString> userMetadata =

    new TreeMap<String, ByteString>();

  private final StreamFactory streamFactory = new StreamFactory();

  private final TreeWriter treeWriter;

  private final OrcProto.RowIndex.Builder rowIndex =

      OrcProto.RowIndex.newBuilder();

  private final boolean buildIndex;

  private final MemoryManager memoryManager;

  private final boolean useVInts;

  private final int dfsBytesPerChecksum;

  private final long maxDictSize;

  private final MemoryEstimate memoryEstimate = new MemoryEstimate();

  private final Configuration conf;

  WriterImpl(FileSystem fs,

             Path path,

             Configuration conf,

             ObjectInspector inspector,

             long stripeSize,

             CompressionKind compress,

             int bufferSize,

             int rowIndexStride,

             MemoryManager memoryManager) throws IOException {

    this.fs = fs;

    this.path = path;

    this.conf = conf;

    this.stripeSize = stripeSize;

    this.compress = compress;

    this.bufferSize = bufferSize;

    this.rowIndexStride = rowIndexStride;

    this.memoryManager = memoryManager;

    buildIndex = rowIndexStride > 0;

    codec = createCodec(compress, conf);

    useVInts = OrcConf.getBoolVar(conf, OrcConf.ConfVars.HIVE_ORC_USE_VINTS);

    treeWriter = createTreeWriter(inspector, streamFactory, false, conf, useVInts,

        memoryManager.isLowMemoryMode(), memoryEstimate);

    dfsBytesPerChecksum = conf.getInt("io.bytes.per.checksum", 512);

    if (buildIndex && rowIndexStride < MIN_ROW_INDEX_STRIDE) {

      throw new IllegalArgumentException("Row stride must be at least " +

          MIN_ROW_INDEX_STRIDE);

    }

    maxDictSize = OrcConf.getLongVar(conf, OrcConf.ConfVars.HIVE_ORC_MAX_DICTIONARY_SIZE);

    // ensure that we are able to handle callbacks before we register ourselves

    memoryManager.addWriter(path, stripeSize, this, memoryEstimate.getTotalMemory());

  }

  static CompressionCodec createCodec(CompressionKind kind) {

    // To be used for cases where we don't care about configuring the codec,

    // e.g. reads

    return createCodec(kind, null);

  }

  static CompressionCodec createCodec(CompressionKind kind, Configuration conf) {

    switch (kind) {

      case NONE:

        return null;

      case ZLIB:

        return new ZlibCodec(conf);

      case SNAPPY:

        return new SnappyCodec();

      case LZO:

        try {

          Class<? extends CompressionCodec> lzo =

              (Class<? extends CompressionCodec>)

                  Class.forName("com.facebook.hive.orc.LzoCodec");

          return lzo.newInstance();

        } catch (ClassNotFoundException e) {

          throw new IllegalArgumentException("LZO is not available.", e);

        } catch (InstantiationException e) {

          throw new IllegalArgumentException("Problem initializing LZO", e);

        } catch (IllegalAccessException e) {

          throw new IllegalArgumentException("Insufficient access to LZO", e);

        }

      default:

        throw new IllegalArgumentException("Unknown compression codec: " +

            kind);

    }

  }

  @Override

  public synchronized void enterLowMemoryMode() throws IOException {

    // Don't use dictionaries

    treeWriter.abandonDictionaries();

    // If the Zlib compression level is less than 6, raise it to 6 to compensate for the fact

    // we aren't using dictionaries

    if (codec != null && OrcConf.getIntVar(conf, ConfVars.HIVE_ORC_ZLIB_COMPRESSION_LEVEL) < 6) {

      OrcConf.setIntVar(conf, ConfVars.HIVE_ORC_ZLIB_COMPRESSION_LEVEL, 6);

      codec.reloadConfigurations(conf);

    }

  }

  /**

   * This class is used to hold the contents of streams as they are buffered.

   * The TreeWriters write to the outStream and the codec compresses the

   * data as buffers fill up and stores them in the output list. When the

   * stripe is being written, the whole stream is written to the file.

   */

  private class BufferedStream implements OutStream.OutputReceiver {

    private final OutStream outStream;

    private final List<ByteBuffer> output = new ArrayList<ByteBuffer>();

    BufferedStream(String name, int bufferSize,

                   CompressionCodec codec) throws IOException {

      outStream = new OutStream(name, bufferSize, codec, this, memoryEstimate);

    }

    /**

     * Receive a buffer from the compression codec.

     * @param buffer the buffer to save

     * @throws IOException

     */

    @Override

    public void output(ByteBuffer buffer) {

      output.add(buffer);

    }

    /**

     * Get the number of bytes in buffers that are allocated to this stream.

     * @return number of bytes in buffers

     */

    public long getBufferSize() {

      long result = 0;

      for(ByteBuffer buf: output) {

        result += buf.capacity();

      }

      return outStream.getBufferSize() + result;

    }

    /**

     * Flush the stream to the codec.

     * @throws IOException

     */

    public void flush(boolean reuseBuffer) throws IOException {

      outStream.flush(reuseBuffer);

    }

    /**

     * Clear all of the buffers.

     * @throws IOException

     */

    public void clear() throws IOException {

      outStream.clear();

      for(ByteBuffer buf: output) {

        memoryEstimate.decrementTotalMemory(buf.capacity());

      }

      output.clear();

    }

    /**

     * Check the state of suppress flag in output stream

     * @return value of suppress flag

     */

    public boolean isSuppressed() {

      return outStream.isSuppressed();

    }

    /**

     * Write the saved compressed buffers to the OutputStream.

     * @param out the stream to write to

     * @throws IOException

     */

    void spillTo(OutputStream out) throws IOException {

      for(ByteBuffer buffer: output) {

        out.write(buffer.array(), buffer.arrayOffset() + buffer.position(),

          buffer.remaining());

      }

    }

    @Override

    public String toString() {

      return outStream.toString();

    }

  }

  /**

   * An output receiver that writes the ByteBuffers to the output stream

   * as they are received.

   */

  private class DirectStream implements OutStream.OutputReceiver {

    private final FSDataOutputStream output;

    DirectStream(FSDataOutputStream output) {

      this.output = output;

    }

    @Override

    public void output(ByteBuffer buffer) throws IOException {

      output.write(buffer.array(), buffer.arrayOffset() + buffer.position(),

        buffer.remaining());

      memoryEstimate.decrementTotalMemory(buffer.capacity());

    }

  }

  static class RowIndexPositionRecorder implements PositionRecorder {

    private final OrcProto.RowIndexEntry.Builder builder;

    RowIndexPositionRecorder(OrcProto.RowIndexEntry.Builder builder) {

      this.builder = builder;

    }

    @Override

    public void addPosition(long position) {

      builder.addPositions(position);

    }

  }

  /**

   * Interface from the Writer to the TreeWriters. This limits the visibility

   * that the TreeWriters have into the Writer.

   */

  private class StreamFactory {

    /**

     * Create a stream to store part of a column.

     * @param column the column id for the stream

     * @param kind the kind of stream

     * @return The output outStream that the section needs to be written to.

     * @throws IOException

     */

    public OutStream createStream(int column,

                                               OrcProto.Stream.Kind kind

                                              ) throws IOException {

      StreamName name = new StreamName(column, kind);

      BufferedStream result = streams.get(name);

      if (result == null) {

        result = new BufferedStream(name.toString(), bufferSize, codec);

        streams.put(name, result);

      }

      return result.outStream;

    }

    /**

     * Get the next column id.

     * @return a number from 0 to the number of columns - 1

     */

    public int getNextColumnId() {

      return columnCount++;

    }

    /**

     * Get the stride rate of the row index.

     */

    public int getRowIndexStride() {

      return rowIndexStride;

    }

    /**

     * Should be building the row index.

     * @return true if we are building the index

     */

    public boolean buildIndex() {

      return buildIndex;

    }

    /**

     * Is the ORC file compressed?

     * @return are the streams compressed

     */

    public boolean isCompressed() {

      return codec != null;

    }

  }

  /**

   * The parent class of all of the writers for each column. Each column

   * is written by an instance of this class. The compound types (struct,

   * list, map, and union) have children tree writers that write the children

   * types.

   */

  private abstract static class TreeWriter {

    protected final int id;

    protected final ObjectInspector inspector;

    private final BitFieldWriter isPresent;

    private final boolean isCompressed;

    protected final ColumnStatisticsImpl indexStatistics;

    private final ColumnStatisticsImpl fileStatistics;

    protected TreeWriter[] childrenWriters;

    protected final RowIndexPositionRecorder rowIndexPosition;

    private final OrcProto.RowIndex.Builder rowIndex;

    private final OrcProto.RowIndexEntry.Builder rowIndexEntry;

    private final PositionedOutputStream rowIndexStream;

    private final Configuration conf;

    protected long stripeRawDataSize = 0;

    protected long rowRawDataSize = 0;

    protected final boolean useVInts;

    private int numStripes = 0;

    private boolean foundNulls;

    private OutStream isPresentOutStream;

    protected final MemoryEstimate memoryEstimate;

    /**

     * Create a tree writer

     * @param columnId the column id of the column to write

     * @param inspector the object inspector to use

     * @param streamFactory limited access to the Writer's data.

     * @param nullable can the value be null?

     * @throws IOException

     */

    TreeWriter(int columnId, ObjectInspector inspector,

               StreamFactory streamFactory,

               boolean nullable, Configuration conf,

               boolean useVInts, MemoryEstimate memoryEstimate) throws IOException {

      this.id = columnId;

      this.inspector = inspector;

      this.conf = conf;

      this.useVInts = useVInts;

      this.memoryEstimate = memoryEstimate;

      this.isCompressed = streamFactory.isCompressed();

      if (nullable) {

        isPresentOutStream = streamFactory.createStream(id, OrcProto.Stream.Kind.PRESENT);

        isPresent = new BitFieldWriter(isPresentOutStream, 1);

      } else {

        isPresent = null;

      }

      this.foundNulls = false;

      indexStatistics = ColumnStatisticsImpl.create(inspector);

      fileStatistics = ColumnStatisticsImpl.create(inspector);

      childrenWriters = new TreeWriter[0];

      rowIndex = OrcProto.RowIndex.newBuilder();

      rowIndexEntry = OrcProto.RowIndexEntry.newBuilder();

      rowIndexPosition = new RowIndexPositionRecorder(rowIndexEntry);

      if (streamFactory.buildIndex()) {

        rowIndexStream = streamFactory.createStream(id,

            OrcProto.Stream.Kind.ROW_INDEX);

      } else {

        rowIndexStream = null;

      }

    }

    protected int getNumStripes() {

      return numStripes;

    }

    protected OrcProto.RowIndex.Builder getRowIndex() {

      return rowIndex;

    }

    protected ColumnStatisticsImpl getFileStatistics() {

      return fileStatistics;

    }

    protected OrcProto.RowIndexEntry.Builder getRowIndexEntry() {

      return rowIndexEntry;

    }

    /**

     * Add a new value to the column.

     * @param obj

     * @throws IOException

     */

    abstract void write(Object obj) throws IOException;

    void write(Object obj, long rawDataSize) throws IOException{

      if (obj != null) {

        setRawDataSize(rawDataSize);

      } else {

        // Estimate the raw size of null as 1 byte

        setRawDataSize(RawDatasizeConst.NULL_SIZE);

      }

      flushRow(obj);

    }

    /**

     * Update the row count and mark the isPresent bit

     */

    void flushRow(Object obj) throws IOException {

      if (obj != null) {

        indexStatistics.increment();

      }

      if (isPresent != null) {

        isPresent.write(obj == null ? 0 : 1);

        if(obj == null) {

          foundNulls = true;

        }

      }

    }

    private void removeIsPresentPositions() {

      for(int i=0; i < rowIndex.getEntryCount(); ++i) {

        OrcProto.RowIndexEntry.Builder entry = rowIndex.getEntryBuilder(i);

        List<Long> positions = entry.getPositionsList();

        // bit streams use 3 positions if uncompressed, 4 if compressed

        positions = positions.subList(isCompressed ? COMPRESSED_PRESENT_STREAM_INDEX_ENTRIES :

          UNCOMPRESSED_PRESENT_STREAM_INDEX_ENTRIES, positions.size());

        entry.clearPositions();

        entry.addAllPositions(positions);

      }

    }

    /**

     * Sets the row raw data size and updates the stripe raw data size

     *

     * @param rawDataSize

     */

    protected void setRawDataSize(long rawDataSize) {

      rowRawDataSize = rawDataSize;

      stripeRawDataSize += rawDataSize;

    }

    /**

     * Write the stripe out to the file.

     * @param builder the stripe footer that contains the information about the

     *                layout of the stripe. The TreeWriter is required to update

     *                the footer with its information.

     * @param requiredIndexEntries the number of index entries that are

     *                             required. this is to check to make sure the

     *                             row index is well formed.

     * @throws IOException

     */

    void writeStripe(OrcProto.StripeFooter.Builder builder,

                     int requiredIndexEntries) throws IOException {

      numStripes++;

      if (isPresent != null) {

        isPresent.flush();

        // if no nulls are found in a stream, then suppress the stream

        if(!foundNulls) {

          isPresentOutStream.suppress();

          // since isPresent bitstream is suppressed, update the index to

          // remove the positions of the isPresent stream

          if (rowIndexStream != null) {

            removeIsPresentPositions();

          }

        }

      }

      // reset the flag for next stripe

      foundNulls = false;

      builder.addColumns(getEncoding());

      if (rowIndexStream != null) {

        if (rowIndex.getEntryCount() != requiredIndexEntries) {

          throw new IllegalArgumentException("Column has wrong number of " +

               "index entries found: " + rowIndex.getEntryCount() + " expected: " +

               requiredIndexEntries);

        }

        rowIndex.build().writeTo(rowIndexStream);

        rowIndexStream.flush();

      }

      rowIndex.clear();

      rowIndexEntry.clear();

      stripeRawDataSize = 0;

    }

    TreeWriter[] getChildrenWriters() {

      return childrenWriters;

    }

    /**

     * For all the TreeWriters that buffer rows, process

     * all the buffered rows.

     */

    void flush() throws IOException {

      for (TreeWriter writer : childrenWriters) {

        writer.flush();

      }

      return;

    }

    /**

     * Get the encoding for this column.

     * @return the information about the encoding of this column

     */

    OrcProto.ColumnEncoding getEncoding() {

      return OrcProto.ColumnEncoding.newBuilder().setKind(

          OrcProto.ColumnEncoding.Kind.DIRECT).build();

    }

    /**

     * Create a row index entry with the previous location and the current

     * index statistics. Also merges the index statistics into the file

     * statistics before they are cleared. Finally, it records the start of the

     * next index and ensures all of the children columns also create an entry.

     * @throws IOException

     */

    void createRowIndexEntry() throws IOException {

      fileStatistics.merge(indexStatistics);

      rowIndexEntry.setStatistics(indexStatistics.serialize());

      indexStatistics.reset();

      rowIndex.addEntry(rowIndexEntry);

      rowIndexEntry.clear();

      recordPosition(rowIndexPosition);

      for(TreeWriter child: childrenWriters) {

        child.createRowIndexEntry();

      }

    }

    /**

     * Record the current position in each of this column's streams.

     * @param recorder where should the locations be recorded

     * @throws IOException

     */

    void recordPosition(PositionRecorder recorder) throws IOException {

      if (isPresent != null) {

        isPresent.getPosition(recorder);

      }

    }

    public void abandonDictionaries() throws IOException {

      for (TreeWriter child: childrenWriters) {

        child.abandonDictionaries();

      }

    }

    long getStripeRawDataSize() {

      return stripeRawDataSize;

    }

    long getRowRawDataSize() {

      return rowRawDataSize;

    }

  }

  private static class BooleanTreeWriter extends TreeWriter {

    private final BitFieldWriter writer;

    BooleanTreeWriter(int columnId,

                      ObjectInspector inspector,

                      StreamFactory writer,

                      boolean nullable, Configuration conf,

                      boolean useVInts, boolean lowMemoryMode,

                      MemoryEstimate memoryEstimate) throws IOException {

      super(columnId, inspector, writer, nullable, conf, useVInts, memoryEstimate);

      PositionedOutputStream out = writer.createStream(id,

          OrcProto.Stream.Kind.DATA);

      this.writer = new BitFieldWriter(out, 1);

      recordPosition(rowIndexPosition);

    }

    @Override

    void write(Object obj) throws IOException {

      super.write(obj, RawDatasizeConst.BOOLEAN_SIZE);

      if (obj != null) {

        boolean val = ((BooleanObjectInspector) inspector).get(obj);

        indexStatistics.updateBoolean(val);

        writer.write(val ? 1 : 0);

      }

    }

    @Override

    void writeStripe(OrcProto.StripeFooter.Builder builder,

                     int requiredIndexEntries) throws IOException {

      super.writeStripe(builder, requiredIndexEntries);

      writer.flush();

      recordPosition(rowIndexPosition);

    }

    @Override

    void recordPosition(PositionRecorder recorder) throws IOException {

      super.recordPosition(recorder);

      writer.getPosition(recorder);

    }

  }

  private static class ByteTreeWriter extends TreeWriter {

    private final RunLengthByteWriter writer;

    ByteTreeWriter(int columnId,

                      ObjectInspector inspector,

                      StreamFactory writer,

                      boolean nullable, Configuration conf,

                      boolean useVInts, boolean lowMemoryMode,

                      MemoryEstimate memoryEstimate) throws IOException {

      super(columnId, inspector, writer, nullable, conf, useVInts, memoryEstimate);

      this.writer = new RunLengthByteWriter(writer.createStream(id,

          OrcProto.Stream.Kind.DATA));

      recordPosition(rowIndexPosition);

    }

    @Override

    void write(Object obj) throws IOException {

      super.write(obj, RawDatasizeConst.BYTE_SIZE);

      if (obj != null) {

        byte val = ((ByteObjectInspector) inspector).get(obj);

        indexStatistics.updateInteger(val);

        writer.write(val);

      }

    }

    @Override

    void writeStripe(OrcProto.StripeFooter.Builder builder,

                     int requiredIndexEntries) throws IOException {

      super.writeStripe(builder, requiredIndexEntries);

      writer.flush();

      recordPosition(rowIndexPosition);

    }

    @Override

    void recordPosition(PositionRecorder recorder) throws IOException {

      super.recordPosition(recorder);

      writer.getPosition(recorder);

    }

  }

  private static class IntegerTreeWriter extends TreeWriter {

    private final PositionedOutputStream output;

    private DynamicIntArray rows;

    private final PositionedOutputStream inDictionaryStream;

    private final BitFieldWriter inDictionary;

    private final List<OrcProto.RowIndexEntry> savedRowIndex =

        new ArrayList<OrcProto.RowIndexEntry>();

    private final boolean buildIndex;

    private final List<Long> rowIndexValueCount = new ArrayList<Long>();

    private final float dictionaryKeySizeThreshold;

    private IntDictionaryEncoder dictionary;

    private boolean useDictionaryEncoding = true;

    private final StreamFactory writer;

    private final int numBytes;

    private final Long[] buffer;

    private int bufferIndex = 0;

    private long bufferedBytes = 0;

    private final int recomputeStripeEncodingInterval;

    PositionedOutputStream rowOutput;

    private boolean abandonDictionaries = false;

    private final boolean sortKeys;

    private int dictionarySize;

    private final boolean useStrideDictionaries;

    IntegerTreeWriter(int columnId, ObjectInspector inspector, StreamFactory writerFactory,

        boolean nullable, Configuration conf, boolean useVInts, int numBytes,

        boolean lowMemoryMode, MemoryEstimate memoryEstimate) throws IOException {

      super(columnId, inspector, writerFactory, nullable, conf, useVInts, memoryEstimate);

      writer = writerFactory;

      sortKeys = OrcConf.getBoolVar(conf,

          OrcConf.ConfVars.HIVE_ORC_DICTIONARY_SORT_KEYS);

      useStrideDictionaries = OrcConf.getBoolVar(conf,

          OrcConf.ConfVars.HIVE_ORC_BUILD_STRIDE_DICTIONARY);

      this.numBytes = numBytes;

      recomputeStripeEncodingInterval = OrcConf.getIntVar(conf,

          OrcConf.ConfVars.HIVE_ORC_DICTIONARY_ENCODING_INTERVAL);

      if (!lowMemoryMode) {

        dictionary = new IntDictionaryEncoder(sortKeys, numBytes, useVInts, memoryEstimate);

        rows = new DynamicIntArray(memoryEstimate);

      } else {

        abandonDictionaries = true;

        rowOutput = writer.createStream(id,

            OrcProto.Stream.Kind.DATA);

        useDictionaryEncoding = false;

      }

      output = writer.createStream(id,

          OrcProto.Stream.Kind.DICTIONARY_DATA);

      inDictionaryStream = writer.createStream(id,

          OrcProto.Stream.Kind.IN_DICTIONARY);

      inDictionary = new BitFieldWriter(inDictionaryStream, 1);

      dictionaryKeySizeThreshold = OrcConf.getFloatVar(conf,

          OrcConf.ConfVars.HIVE_ORC_DICTIONARY_NUMERIC_KEY_SIZE_THRESHOLD);

      int bufferLength = OrcConf.getIntVar(conf, OrcConf.ConfVars.HIVE_ORC_ROW_BUFFER_SIZE);

      buffer = new Long[bufferLength];

      recordPosition(rowIndexPosition);

      rowIndexValueCount.add(0L);

      buildIndex = writer.buildIndex();

      if (buildIndex && lowMemoryMode) {

        rowOutput.getPosition(rowIndexPosition);

      }

    }

    boolean determineEncodingStripe() {

      return (getNumStripes() % recomputeStripeEncodingInterval) == 0 && !abandonDictionaries;

    }

    @Override

    void write(Object obj) throws IOException {

      if (obj != null) {

        switch (inspector.getCategory()) {

          case PRIMITIVE:

            switch (((PrimitiveObjectInspector) inspector).getPrimitiveCategory()) {

              case SHORT:

                buffer[bufferIndex++] = new Long(((ShortObjectInspector) inspector).get(obj));

                setRawDataSize(RawDatasizeConst.SHORT_SIZE);

                break;

              case INT:

                buffer[bufferIndex++] = new Long(((IntObjectInspector) inspector).get(obj));

                setRawDataSize(RawDatasizeConst.INT_SIZE);

                break;

              case LONG:

                buffer[bufferIndex++] = new Long(((LongObjectInspector) inspector).get(obj));

                setRawDataSize(RawDatasizeConst.LONG_SIZE);

                break;

              default:

                throw new IllegalArgumentException("Bad Category: Dictionary Encoding not available for " +

                    ((PrimitiveObjectInspector) inspector).getPrimitiveCategory());

            }

            break;

          default:

            throw new IllegalArgumentException("Bad Category: DictionaryEncoding not available for " + inspector.getCategory());

        }

        // Increment the total memory for the buffered long

        memoryEstimate.incrementTotalMemory(RawDatasizeConst.LONG_SIZE);

        bufferedBytes += RawDatasizeConst.LONG_SIZE;

      } else {

        buffer[bufferIndex++] = null;

        setRawDataSize(RawDatasizeConst.NULL_SIZE);

      }

      if (bufferIndex == buffer.length) {

        flush();

      }

    }

    @Override

    void flush() throws IOException {

      for (int i = 0; i < bufferIndex; i++) {

        Long val = buffer[i];

        buffer[i] = null;

        if (val != null) {

          if (useCarriedOverDirectEncoding()) {

            SerializationUtils.writeIntegerType(rowOutput,

                val, numBytes, true, useVInts);

          } else {

            rows.add(dictionary.add(val));

          }

          indexStatistics.updateInteger(val);

        }

        super.flushRow(val);

      }

      bufferIndex = 0;

      memoryEstimate.decrementTotalMemory(bufferedBytes);

      bufferedBytes = 0;

    }

    boolean getUseDictionaryEncoding() {

      return (rows.size() > 0 &&

          (float)(dictionary.size()) / (float)rows.size() <= dictionaryKeySizeThreshold);

    }

    /**

     * Returns true iff the encoding is not being determined using this stripe, and

     * the previously determined encoding was direct.

     */

    private boolean useCarriedOverDirectEncoding() {

      return !determineEncodingStripe() && !useDictionaryEncoding;

    }

    @Override

    void writeStripe(OrcProto.StripeFooter.Builder builder,

                     int requiredIndexEntries) throws IOException {

      if (determineEncodingStripe()) {

        useDictionaryEncoding = getUseDictionaryEncoding();

      }

      if (useDictionaryEncoding) {

        rowOutput =

          new RunLengthIntegerWriter(writer.createStream(id,

          OrcProto.Stream.Kind.DATA), false, INT_BYTE_SIZE, useVInts);

      } else if (determineEncodingStripe()){

        rowOutput = writer.createStream(id,

          OrcProto.Stream.Kind.DATA);

      }

      final long[] dumpOrder;

      final int[] counts;

      if (useDictionaryEncoding) {

        // Traverse the dictionary keys writing out the bytes and lengths; and

        // creating the map from the original order to the final sorted order.

        dumpOrder = new long[dictionary.size()];