Examples of BytesRefArrayWritable

org.apache.hadoop.hive.serde2.columnar.BytesRefArrayWritable
BytesRefArrayWritable holds an array reference to BytesRefWritable, and is able to resize without recreating new array if not necessary.
Each BytesRefArrayWritable holds instance has a valid field, which is the desired valid number of BytesRefWritable it holds. resetValid can reset the valid, but it will not care the underlying BytesRefWritable.
org.apache.tajo.storage.rcfile.BytesRefArrayWritable
BytesRefArrayWritable holds an array reference to BytesRefWritable, and is able to resize without recreating new array if not necessary.
Each BytesRefArrayWritable holds instance has a valid field, which is the desired valid number of BytesRefWritable it holds. resetValid can reset the valid, but it will not care the underlying BytesRefWritable.

Examples of org.apache.hadoop.hive.serde2.columnar.BytesRefArrayWritable


    byte [][][] records = getRecords();
    RCFileOutputFormat.setColumnNumber(conf, 8);
    RCFile.Writer writer = new RCFile.Writer(fs, conf, file, null, new DefaultCodec());


    BytesRefArrayWritable bytes = writeBytesToFile(records[0], writer);
    BytesRefArrayWritable bytes2 = writeBytesToFile(records[1], writer);


    writer.close();
    return new BytesRefArrayWritable[]{bytes,bytes2};
  }

View Full Code Here

Examples of org.apache.hadoop.hive.serde2.columnar.BytesRefArrayWritable

    writer.close();
    return new BytesRefArrayWritable[]{bytes,bytes2};
  }


  private BytesRefArrayWritable writeBytesToFile(byte[][] record, RCFile.Writer writer) throws IOException {
    BytesRefArrayWritable bytes = new BytesRefArrayWritable(record.length);
    for (int i = 0; i < record.length; i++) {
      BytesRefWritable cu = new BytesRefWritable(record[i], 0, record[i].length);
      bytes.set(i, cu);
    }
    writer.append(bytes);
    return bytes;
  }

View Full Code Here

Examples of org.apache.hadoop.hive.serde2.columnar.BytesRefArrayWritable

    RecordReader<?,?> rr = iF.createRecordReader(split,tac);
    rr.initialize(split, tac);
    HCatRecord[] tuples = getExpectedRecords();
    for(int j=0; j < 2; j++){
      Assert.assertTrue(rr.nextKeyValue());
      BytesRefArrayWritable w = (BytesRefArrayWritable)rr.getCurrentValue();
      Assert.assertEquals(bytesArr[j], w);
      HCatRecord t = sd.convertToHCatRecord(null,w);
      Assert.assertEquals(8, t.size());
      Assert.assertEquals(t,tuples[j]);
    }

View Full Code Here

Examples of org.apache.hadoop.hive.serde2.columnar.BytesRefArrayWritable

    RecordReader<?,?> rr = iF.createRecordReader(split,tac);
    rr.initialize(split, tac);
    HCatRecord[] tuples = getPrunedRecords();
    for(int j=0; j < 2; j++){
      Assert.assertTrue(rr.nextKeyValue());
      BytesRefArrayWritable w = (BytesRefArrayWritable)rr.getCurrentValue();
      Assert.assertFalse(bytesArr[j].equals(w));
      Assert.assertEquals(w.size(), 8);
      HCatRecord t = sd.convertToHCatRecord(null,w);
      Assert.assertEquals(5, t.size());
      Assert.assertEquals(t,tuples[j]);
    }
    assertFalse(rr.nextKeyValue());

View Full Code Here

Examples of org.apache.hadoop.hive.serde2.columnar.BytesRefArrayWritable

    RecordReader<?,?> rr = iF.createRecordReader(split,tac);
    rr.initialize(split, tac);
    HCatRecord[] tuples = getReorderedCols();
    for(int j=0; j < 2; j++){
      Assert.assertTrue(rr.nextKeyValue());
      BytesRefArrayWritable w = (BytesRefArrayWritable)rr.getCurrentValue();
      Assert.assertFalse(bytesArr[j].equals(w));
      Assert.assertEquals(w.size(), 8);
      HCatRecord t = sd.convertToHCatRecord(null,w);
      Assert.assertEquals(7, t.size());
      Assert.assertEquals(t,tuples[j]);
    }
    assertFalse(rr.nextKeyValue());

View Full Code Here

Examples of org.apache.hadoop.hive.serde2.columnar.BytesRefArrayWritable

      Configuration conf = new Configuration();
      Properties tbl = createProperties();
      serDe.initialize(conf, tbl);


      // Data
      BytesRefArrayWritable braw = new BytesRefArrayWritable(8);
      String[] data = {"123", "456", "789", "1000", "5.3", "hive and hadoop", "1.", "NULL"};
      for (int i = 0; i < 8; i++) {
        braw.set(i, new BytesRefWritable(data[i].getBytes()));
      }
      // Test
      deserializeAndSerializeColumnar(serDe, braw, data);
      System.out.println("test: testColumnarSerde - OK");

View Full Code Here

Examples of org.apache.hadoop.hive.serde2.columnar.BytesRefArrayWritable


    assertEquals("serialized size correct after deserialization", size, serDe.getSerDeStats()
        .getRawDataSize());
    assertNotSame(0, size);


    BytesRefArrayWritable serializedData = (BytesRefArrayWritable) serDe.serialize(row, oi);
    size = 0;
    for (int i = 0; i < serializedData.size(); i++) {
      size += serializedData.get(i).getLength();
    }


    assertEquals("serialized size correct after serialization", size, serDe.getSerDeStats()
        .getRawDataSize());
    assertNotSame(0, size);

View Full Code Here

Examples of org.apache.hadoop.hive.serde2.columnar.BytesRefArrayWritable

      if (skippedColIDs[columnID]) {
        return null;
      }


      if (rest == null) {
        rest = new BytesRefArrayWritable();
      }


      rest.resetValid(recordsNumInValBuffer);


      if (!currentValue.inited) {

View Full Code Here

Examples of org.apache.hadoop.hive.serde2.columnar.BytesRefArrayWritable


    this.start = in.getPosition();
    more = start < end;


    key = new LongWritable();
    value = new BytesRefArrayWritable();
  }

View Full Code Here

Examples of org.apache.tajo.storage.rcfile.BytesRefArrayWritable

    super(conf, meta, schema, fragment);


    this.start = fragment.getStartKey();
    this.end = start + fragment.getEndKey();
    key = new LongWritable();
    column = new BytesRefArrayWritable();


    String nullCharacters = StringEscapeUtils.unescapeJava(this.meta.getOption(NULL));
    if (StringUtils.isEmpty(nullCharacters)) {
      nullChars = NullDatum.get().asTextBytes();
    } else {

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.