Examples of org.apache.hadoop.io.BytesWritable

org.apache.hadoop.io.BytesWritable
A byte sequence that is usable as a key or value. It is resizable and distinguishes between the size of the seqeunce and the current capacity. The hash function is the front of the md5 of the buffer. The sort order is the same as memcmp. @author Doug Cutting

    return lengths.loadIndeces(rowIndexEntries, updatedStartIndex);
  }


  @Override
  public Object next(Object previous) throws IOException {
    BytesWritable result = null;
    if (valuePresent) {
      if (previous == null) {
        result = new BytesWritable();
      } else {
        result = (BytesWritable) previous;
      }
      int len = (int) lengths.next();
      result.setSize(len);
      int offset = 0;
      while (len > 0) {
        int written = stream.read(result.getBytes(), offset, len);
        if (written < 0) {
          throw new EOFException("Can't finish byte read from " + stream);
        }
        len -= written;
        offset += written;

View Full Code Here

  }


  public OrcLazyBinary(OrcLazyBinary copy) {
    super(copy);
    if (copy.previous != null) {
      BytesWritable copyPrevious = (BytesWritable) copy.previous;
      byte[] bytes = new byte[copyPrevious.getLength()];
      System.arraycopy(copyPrevious.getBytes(), 0, bytes, 0, copyPrevious.getLength());
      previous = new BytesWritable(bytes);
    }
  }

View Full Code Here

    }
    for(int i=0; i < count; ++i) {
      doubleValues[i] = rand.nextDouble();
      byte[] buf = new byte[20];
      rand.nextBytes(buf);
      byteValues[i] = new BytesWritable(buf);
    }
    for(int i=0; i < count; ++i) {
      ReallyBigRow bigrow = createRandomRow(intValues, doubleValues, stringValues,
          byteValues, words, i);
      writer.addRow(bigrow);

View Full Code Here

      assertNull(expected.double1);
    } else {
      assertEquals(expected.double1.doubleValue(), double1.get(), 0.0001);
    }


    BytesWritable bytes1 = (BytesWritable) ((OrcLazyBinary) row.getFieldValue(13)).materialize();
    if (bytes1 == null) {
      assertNull(expected.bytes1);
    } else {
      assertEquals(expected.bytes1, bytes1);
    }

View Full Code Here

    }
    for(int i=0; i < count; ++i) {
      doubleValues[i] = rand.nextDouble();
      byte[] buf = new byte[20];
      rand.nextBytes(buf);
      byteValues[i] = new BytesWritable(buf);
    }
    for(int i=0; i < count; ++i) {
      ReallyBigRow bigrow = createRandomRowWithNulls(intValues, doubleValues, stringValues,
          byteValues, words, i, numNulls);
      writer.addRow(bigrow);

View Full Code Here

    Short shortVal = intValues[i] % 12 == 0 ^ lotsOfNulls  ? null : (short) intValues[i];
    Integer intVal = intValues[i] % 13 == 0 ^ lotsOfNulls  ? null : (int) (Short.MAX_VALUE + i);
    Long longVal = intValues[i] % 14 == 0 ^ lotsOfNulls  ? null : (long) (Integer.MAX_VALUE + i);
    Float floatVal = intValues[i] % 15 == 0 ^ lotsOfNulls  ? null : (float) doubleValues[i];
    Double doubleVal = intValues[i] % 16 == 0 ^ lotsOfNulls  ? null : doubleValues[i];
    BytesWritable bytesVal = intValues[i] % 17 == 0 ^ lotsOfNulls  ? null : byteValues[i];
    String strVal = intValues[i] % 18 == 0 ^ lotsOfNulls  ? null : stringValues[i];
    InnerStruct inner = intValues[i] % 19 == 0 ^ lotsOfNulls  ? null : new InnerStruct(
        intValues[i] % 10 == 0 ^ lotsOfNulls  ? null : (int) intValues[i],
            intValues[i] % 11 == 0 ^ lotsOfNulls  ? null : stringValues[i]);
    InnerStruct inner2 = intValues[i] % 12 == 0 ^ lotsOfNulls  ? null : new InnerStruct(

View Full Code Here

    super(PrimitiveObjectInspectorUtils.binaryTypeEntry);
  }


  @Override
  public byte[] getPrimitiveJavaObject(Object o) {
    BytesWritable writable = getPrimitiveWritableObject(o);
    return writable == null ? null : LazyUtils.createByteArray(writable);
  }

View Full Code Here


            @Override
            public void doOperation(int index) throws Exception {
                try {


                    BytesWritable key = new BytesWritable();
                    BytesWritable value = new BytesWritable();


                    reader.next(key, value);
                    store.put(new ByteArray(ByteUtils.copy(key.get(), 0, key.getSize())),
                              Versioned.value(ByteUtils.copy(value.get(), 0, value.getSize())), null);
                } catch(ObsoleteVersionException e) {
                    obsoletes.incrementAndGet();
                }
            }
        };

View Full Code Here


            @Override
            public void doOperation(int index) throws Exception {
                try {


                    BytesWritable key = new BytesWritable();
                    BytesWritable value = new BytesWritable();


                    reader.next(key, value);
                    store.put(new ByteArray(ByteUtils.copy(key.get(), 0, key.getSize())),
                              Versioned.value(ByteUtils.copy(value.get(), 0, value.getSize())), null);
                } catch(ObsoleteVersionException e) {
                    obsoletes.incrementAndGet();
                }
            }
        };

View Full Code Here

            System.arraycopy(valBytes, 0, outputValue, offsetTillNow, valBytes.length);


            // Generate MR key - upper 8 bytes of 16 byte md5
            byte[] oldMd5 = ByteUtils.copy(md5er.digest(keyBytes), 0, 2 * ByteUtils.SIZE_OF_INT);
            ByteArray oldMd5ByteArray = new ByteArray(oldMd5);
            BytesWritable outputKey = new BytesWritable(oldMd5ToNewMd5.get(oldMd5ByteArray));


            int replicaType = 0;
            for(Integer partition: partitionList) {


                ByteUtils.writeInt(outputValue, partitionToNode[partition].getId(), 0);
                ByteUtils.writeInt(outputValue, partition, ByteUtils.SIZE_OF_INT);


                if(getSaveKeys()) {
                    ByteUtils.writeBytes(outputValue,
                                         replicaType,
                                         2 * ByteUtils.SIZE_OF_INT,
                                         ByteUtils.SIZE_OF_BYTE);
                }
                BytesWritable outputVal = new BytesWritable(outputValue);


                output.collect(outputKey, outputVal);
                replicaType++;


            }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.io.BytesWritable

com.backtype.cascading.tap.PailTap$PailScheme

com.backtype.hadoop.pail.SequenceFileFormat$SequenceFilePailRecordReader

com.cloudera.sqoop.testutil.ManagerCompatTestCase

com.facebook.hive.orc.OrcTestUtils

com.facebook.hive.orc.TestOrcFile

com.facebook.hive.orc.WriterImpl$BinaryTreeWriter

com.facebook.presto.hive.util.SerDeUtils

com.pinterest.secor.tools.LogFileVerifier

edu.yale.cs.hadoopdb.connector.DBInputSplit

hipi.imagebundle.SeqImageBundle

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.