Examples of org.apache.hadoop.hive.serde2.columnar.BytesRefArrayWritable

Package org.apache.hadoop.hive.serde2.columnar

Examples of org.apache.hadoop.hive.serde2.columnar.BytesRefArrayWritable

org.apache.hadoop.hive.serde2.columnar.BytesRefArrayWritable
BytesRefArrayWritable holds an array reference to BytesRefWritable, and is able to resize without recreating new array if not necessary.
Each BytesRefArrayWritable holds instance has a valid field, which is the desired valid number of BytesRefWritable it holds. resetValid can reset the valid, but it will not care the underlying BytesRefWritable.


  public int performRCFileReadFirstAndLastColumnTest(FileSystem fs, Path file,
      int allColumnsNumber, boolean chechCorrect) throws IOException {


    byte[][] checkBytes = null;
    BytesRefArrayWritable checkRow = new BytesRefArrayWritable(allColumnsNumber);
    if (chechCorrect) {
      this.resetRandomGenerators();
      checkBytes = new byte[allColumnsNumber][];
    }


    int actualReadCount = 0;


    java.util.ArrayList<Integer> readCols = new java.util.ArrayList<Integer>();
    readCols.add(Integer.valueOf(0));
    readCols.add(Integer.valueOf(allColumnsNumber - 1));
    HiveFileFormatUtils.setReadColumnIDs(conf, readCols);
    RCFile.Reader reader = new RCFile.Reader(fs, file, conf);


    LongWritable rowID = new LongWritable();
    BytesRefArrayWritable cols = new BytesRefArrayWritable();
    while (reader.next(rowID)) {
      reader.getCurrentRow(cols);
      boolean ok = true;
      if (chechCorrect) {
        this.nextRandomRow(checkBytes, checkRow);
        ok = ok && (checkRow.get(0).equals(cols.get(0)));
        ok = ok
            && checkRow.get(allColumnsNumber - 1).equals(
                cols.get(allColumnsNumber - 1));
      }
      if (!ok)
        throw new IllegalStateException("Compare read and write error.");
      actualReadCount++;
    }

View Full Code Here


  public int performRCFileFullyReadColumnTest(FileSystem fs, Path file,
      int allColumnsNumber, boolean chechCorrect) throws IOException {


    byte[][] checkBytes = null;
    BytesRefArrayWritable checkRow = new BytesRefArrayWritable(allColumnsNumber);
    if (chechCorrect) {
      this.resetRandomGenerators();
      checkBytes = new byte[allColumnsNumber][];
    }


    int actualReadCount = 0;


    HiveFileFormatUtils.setFullyReadColumns(conf);
    RCFile.Reader reader = new RCFile.Reader(fs, file, conf);


    LongWritable rowID = new LongWritable();
    BytesRefArrayWritable cols = new BytesRefArrayWritable();
    while (reader.next(rowID)) {
      reader.getCurrentRow(cols);
      boolean ok = true;
      if (chechCorrect) {
        this.nextRandomRow(checkBytes, checkRow);

View Full Code Here


      if (!(val instanceof BytesRefArrayWritable))
        throw new UnsupportedOperationException(
            "Currently the writer can only accept BytesRefArrayWritable");


      BytesRefArrayWritable columns = (BytesRefArrayWritable) val;
      int size = columns.size();
      for (int i = 0; i < size; i++) {
        BytesRefWritable cu = columns.get(i);
        int plainLen = cu.getLength();
        columnBufferSize += plainLen;
        columnValuePlainLength[i] += plainLen;
        columnBuffers[i].append(cu);
      }


      if (size < columnNumber) {
        for (int i = columns.size(); i < columnNumber; i++) {
          columnBuffers[i].append(BytesRefWritable.ZeroBytesRefWritable);
        }
      }


      bufferedRecords++;

View Full Code Here

      if (skippedColIDs[columnID]) {
        return null;
      }


      if (rest == null) {
        rest = new BytesRefArrayWritable();
      }


      rest.resetValid(recordsNumInValBuffer);


      if (!currentValue.inited)

View Full Code Here


    //share the code with RecordReader.
    FileSplit split = new FileSplit(fileName,start, length, new JobConf(conf));
    RCFileRecordReader recordReader = new RCFileRecordReader(conf, split);
    LongWritable key = new LongWritable();
    BytesRefArrayWritable value = new BytesRefArrayWritable();
    StringBuilder buf = new StringBuilder(STRING_BUFFER_SIZE); // extra capacity in case we overrun, to avoid resizing
    while (recordReader.next(key, value)) {
      printRecord(value, buf);
      recordCount++;
      if (verbose && (recordCount % RECORD_PRINT_INTERVAL) == 0) {

View Full Code Here

      Configuration conf = new Configuration();
      Properties tbl = createProperties();
      serDe.initialize(conf, tbl);


      // Data
      BytesRefArrayWritable braw = new BytesRefArrayWritable(8);
      String[] data = {"123", "456", "789", "1000", "5.3", "hive and hadoop", "1.", "NULL"};
      for (int i = 0; i < 8; i++) {
        braw.set(i, new BytesRefWritable(data[i].getBytes()));
      }
      // Test
      deserializeAndSerializeColumnar(serDe, braw, data);
      System.out.println("test: testColumnarSerde - OK");

View Full Code Here


    assertEquals("serialized size correct after deserialization", size, serDe.getSerDeStats()
        .getRawDataSize());
    assertNotSame(0, size);


    BytesRefArrayWritable serializedData = (BytesRefArrayWritable) serDe.serialize(row, oi);
    size = 0;
    for (int i = 0; i < serializedData.size(); i++) {
      size += serializedData.get(i).getLength();
    }


    assertEquals("serialized size correct after serialization", size, serDe.getSerDeStats()
        .getRawDataSize());
    assertNotSame(0, size);

View Full Code Here

      if (!(val instanceof BytesRefArrayWritable)) {
        throw new UnsupportedOperationException(
            "Currently the writer can only accept BytesRefArrayWritable");
      }


      BytesRefArrayWritable columns = (BytesRefArrayWritable) val;
      int size = columns.size();
      for (int i = 0; i < size; i++) {
        BytesRefWritable cu = columns.get(i);
        int plainLen = cu.getLength();
        columnBufferSize += plainLen;
        columnValuePlainLength[i] += plainLen;
        columnBuffers[i].append(cu);
      }


      if (size < columnNumber) {
        for (int i = columns.size(); i < columnNumber; i++) {
          columnBuffers[i].append(BytesRefWritable.ZeroBytesRefWritable);
        }
      }


      bufferedRecords++;

View Full Code Here

      if (selColIdx == -1) {
        return null;
      }


      if (rest == null) {
        rest = new BytesRefArrayWritable();
      }


      rest.resetValid(recordsNumInValBuffer);


      if (!currentValue.inited) {

View Full Code Here


            RCFileOutputFormat.setColumnNumber(conf, columnCount);
            RCFile.Writer writer = new RCFile.Writer(fs, conf, new Path(
                    file.getAbsolutePath()));


            BytesRefArrayWritable bytes = new BytesRefArrayWritable(columnCount);


            for (int c = 0; c < columnCount; c++) {
                bytes.set(c, new BytesRefWritable());
            }


            try {


                for (int r = 0; r < recordCounts; r++) {
                    // foreach row write n columns


                    for (int c = 0; c < columnCount; c++) {


                        byte[] stringbytes = String.valueOf(Math.random())
                                .getBytes();
                        bytes.get(c).set(stringbytes, 0, stringbytes.length);


                    }


                    writer.append(bytes);

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.hive.serde2.columnar.BytesRefArrayWritable

com.asakusafw.directio.hive.orc.OrcFileInput

com.asakusafw.directio.hive.serde.DataModelDriver

com.asakusafw.directio.hive.serde.DataModelDriverTest

com.asakusafw.directio.hive.serde.ValueSerdeFactoryTest

com.cloudera.cdk.morphline.hadoop.rcfile.ReadRCFileBuilder$ReadRCFile

com.facebook.hive.orc.lazy.LazyDoubleTreeReader

com.facebook.hive.orc.lazy.LazyShortDictionaryTreeReader

com.facebook.hive.orc.lazy.LazyShortDirectTreeReader

com.facebook.hive.orc.lazy.OrcLazyStructObjectInspector

com.facebook.hive.orc.OrcOutputFormat$OrcRecordWriter

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.