Examples of org.apache.hadoop.hive.serde2.columnar.BytesRefWritable

Package org.apache.hadoop.hive.serde2.columnar

Examples of org.apache.hadoop.hive.serde2.columnar.BytesRefWritable

org.apache.hadoop.hive.serde2.columnar.BytesRefWritable
BytesRefWritable referenced a section of byte array. It can be used to avoid unnecessary byte copy.

        byte[][] columnRandom;


        BytesRefArrayWritable bytes = new BytesRefArrayWritable(columnNum);
        columnRandom = new byte[columnNum][];
        for (int i = 0; i < columnNum; i++) {
            BytesRefWritable cu = new BytesRefWritable();
            bytes.set(i, cu);
        }


        for (int i = 0; i < rowCount; i++) {
            nextRandomRow(columnRandom, bytes, columnCount);

View Full Code Here

        byte[][] columnRandom;


        BytesRefArrayWritable bytes = new BytesRefArrayWritable(columnNum);
        columnRandom = new byte[columnNum][];
        for (int i = 0; i < columnNum; i++) {
            BytesRefWritable cu = new BytesRefWritable();
            bytes.set(i, cu);
        }


        for (int i = 0; i < rowCount; i++) {

View Full Code Here

            byteStream = new ByteStream.Output();
            rowWritable = new BytesRefArrayWritable();
            colValRefs = new BytesRefWritable[numColumns];


            for (int i = 0; i < numColumns; i++) {
                colValRefs[i] = new BytesRefWritable();
                rowWritable.set(i, colValRefs[i]);
            }
        }


        byteStream.reset();

View Full Code Here

                                              new Text("cat"),
                                              new Text("dog")),
                        new DefaultCodec());
    BytesRefArrayWritable bytes = new BytesRefArrayWritable(record_1.length);
    for (int i = 0; i < record_1.length; i++) {
      BytesRefWritable cu = new BytesRefWritable(record_1[i], 0,
          record_1[i].length);
      bytes.set(i, cu);
    }
    writer.append(bytes);
    bytes.clear();
    for (int i = 0; i < record_2.length; i++) {
      BytesRefWritable cu = new BytesRefWritable(record_2[i], 0,
          record_2[i].length);
      bytes.set(i, cu);
    }
    writer.append(bytes);
    writer.close();

View Full Code Here

    for (int recIdx = 0; recIdx < recCount; recIdx++) {
      for (int i = 0; i < record.length; i++) {
        record[i] = new Integer(rand.nextInt()).toString().getBytes("UTF-8");
      }
      for (int i = 0; i < record.length; i++) {
        BytesRefWritable cu = new BytesRefWritable(record[i], 0,
            record[i].length);
        bytes.set(i, cu);
      }
      writer.append(bytes);
      bytes.clear();

View Full Code Here

    RCFile.Writer writer = new RCFile.Writer(fs, conf, file, null,
        new DefaultCodec());


    BytesRefArrayWritable bytes = new BytesRefArrayWritable(fieldsData.length);
    for (int i = 0; i < fieldsData.length; i++) {
      BytesRefWritable cu = null;
      cu = new BytesRefWritable(fieldsData[i], 0, fieldsData[i].length);
      bytes.set(i, cu);
    }


    for (int i = 0; i < count; i++) {
      writer.append(bytes);

View Full Code Here


    RCFile.Writer writer = new RCFile.Writer(fs, cloneConf, testFile, null, codec);


    BytesRefArrayWritable bytes = new BytesRefArrayWritable(bytesArray.length);
    for (int i = 0; i < bytesArray.length; i++) {
      BytesRefWritable cu = null;
      cu = new BytesRefWritable(bytesArray[i], 0, bytesArray[i].length);
      bytes.set(i, cu);
    }
    for (int i = 0; i < writeCount; i++) {
      if (i == intervalRecordCount) {
        System.out.println("write position:" + writer.getLength());

View Full Code Here

            // this partition may contain fewer fields than what's declared in the schema
            // this happens when additional columns are added to the hive table after a partition has been created
            nulls[column] = true;
        }
        else {
            BytesRefWritable fieldData = value.unCheckedGet(hiveColumnIndexes[column]);


            byte[] bytes;
            try {
                bytes = fieldData.getData();
            }
            catch (IOException e) {
                throw Throwables.propagate(e);
            }


            int start = fieldData.getStart();
            int length = fieldData.getLength();


            parseBooleanColumn(column, bytes, start, length);
        }
    }

View Full Code Here

            // this partition may contain fewer fields than what's declared in the schema
            // this happens when additional columns are added to the hive table after a partition has been created
            nulls[column] = true;
        }
        else {
            BytesRefWritable fieldData = value.unCheckedGet(hiveColumnIndexes[column]);


            byte[] bytes;
            try {
                bytes = fieldData.getData();
            }
            catch (IOException e) {
                throw Throwables.propagate(e);
            }


            int start = fieldData.getStart();
            int length = fieldData.getLength();


            parseLongColumn(column, bytes, start, length);
        }
    }

View Full Code Here

            // this partition may contain fewer fields than what's declared in the schema
            // this happens when additional columns are added to the hive table after a partition has been created
            nulls[column] = true;
        }
        else {
            BytesRefWritable fieldData = value.unCheckedGet(hiveColumnIndexes[column]);


            byte[] bytes;
            try {
                bytes = fieldData.getData();
            }
            catch (IOException e) {
                throw Throwables.propagate(e);
            }


            int start = fieldData.getStart();
            int length = fieldData.getLength();


            parseDoubleColumn(column, bytes, start, length);
        }
    }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.hive.serde2.columnar.BytesRefWritable

com.asakusafw.directio.hive.orc.OrcFileInput

com.asakusafw.directio.hive.serde.DataModelDriver

com.asakusafw.directio.hive.serde.DataModelDriverTest

com.asakusafw.directio.hive.serde.ValueSerdeFactoryTest

com.cloudera.cdk.morphline.hadoop.rcfile.ReadRCFileBuilder$ReadRCFile

com.facebook.hive.orc.lazy.LazyDoubleTreeReader

com.facebook.hive.orc.lazy.LazyShortDictionaryTreeReader

com.facebook.hive.orc.lazy.LazyShortDirectTreeReader

com.facebook.hive.orc.lazy.OrcLazyStructObjectInspector

com.facebook.hive.orc.OrcOutputFormat$OrcRecordWriter

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.