Examples of org.apache.hadoop.io.BytesWritable$Comparator

org.apache.hadoop.io.BytesWritable
A Comparator optimized for BytesWritable.

        byte[] keyBytes = null, valueBytes;


        keyBytes = new byte[keyBuffer.remaining()];
        keyBuffer.get(keyBytes);


        BytesWritable key = new BytesWritable(keyBytes);


        ArrayList<BytesWritable> valueList = new ArrayList();


        while(iterator.hasNext()) {
            ByteBuffer writable = iterator.next().datum();
            writable.rewind();
            // BytesWritable writable = iterator.next();
            valueBytes = null;
            valueBytes = new byte[writable.remaining()];
            writable.get(valueBytes);


            BytesWritable value = new BytesWritable(valueBytes);
            valueList.add(value);


        }


        writer.write(key, valueList.iterator(), reporter);

View Full Code Here


        public void map(LongWritable lineNumber,
                        Text line,
                        OutputCollector<BytesWritable, BytesWritable> collector,
                        Reporter reporter) throws IOException {
            collector.collect(new BytesWritable(line.getBytes()), value);
        }

View Full Code Here

        public void configure(JobConf job) {
            StringBuilder builder = new StringBuilder();
            int size = job.getInt("value.size", -1);
            for(int i = 0; i < size; i++)
                builder.append('a');
            this.value = new BytesWritable(builder.toString().getBytes());
        }

View Full Code Here

        short numTuples = 0;
        ByteArrayOutputStream stream = new ByteArrayOutputStream();
        DataOutputStream valueStream = new DataOutputStream(stream);


        while(iterator.hasNext()) {
            BytesWritable writable = iterator.next();
            byte[] valueBytes = writable.get();
            int offsetTillNow = 0;


            // Read node Id
            if(this.nodeId == -1)
                this.nodeId = ByteUtils.readInt(valueBytes, offsetTillNow);
            offsetTillNow += ByteUtils.SIZE_OF_INT;


            // Read partition id
            if(this.partitionId == -1)
                this.partitionId = ByteUtils.readInt(valueBytes, offsetTillNow);
            offsetTillNow += ByteUtils.SIZE_OF_INT;


            // Read chunk id
            if(this.chunkId == -1)
                this.chunkId = ReadOnlyUtils.chunk(key.get(), getNumChunks());


            // Read replica type
            if(getSaveKeys()) {
                if(this.replicaType == -1)
                    this.replicaType = (int) ByteUtils.readBytes(valueBytes,
                                                                 offsetTillNow,
                                                                 ByteUtils.SIZE_OF_BYTE);
                offsetTillNow += ByteUtils.SIZE_OF_BYTE;
            }


            int valueLength = writable.getSize() - offsetTillNow;
            if(getSaveKeys()) {
                // Write ( key_length, value_length, key,
                // value )
                valueStream.write(valueBytes, offsetTillNow, valueLength);
            } else {

View Full Code Here

        short numTuples = 0;
        ByteArrayOutputStream stream = new ByteArrayOutputStream();
        DataOutputStream valueStream = new DataOutputStream(stream);


        while(iterator.hasNext()) {
            BytesWritable writable = iterator.next();
            byte[] valueBytes = writable.get();
            int offsetTillNow = 0;


            // Read node Id
            if(this.nodeId == -1)
                this.nodeId = ByteUtils.readInt(valueBytes, offsetTillNow);
            offsetTillNow += ByteUtils.SIZE_OF_INT;


            // Read partition id
            if(this.partitionId == -1)
                this.partitionId = ByteUtils.readInt(valueBytes, offsetTillNow);
            offsetTillNow += ByteUtils.SIZE_OF_INT;


            // Read replica type
            if(getSaveKeys()) {
                if(this.replicaType == -1)
                    this.replicaType = (int) ByteUtils.readBytes(valueBytes,
                                                                 offsetTillNow,
                                                                 ByteUtils.SIZE_OF_BYTE);
                offsetTillNow += ByteUtils.SIZE_OF_BYTE;
            }


            int valueLength = writable.getSize() - offsetTillNow;
            if(getSaveKeys()) {
                // Write ( key_length, value_length, key,
                // value )
                valueStream.write(valueBytes, offsetTillNow, valueLength);
            } else {

View Full Code Here


    @Override
    void write(Object obj) throws IOException {
      long rawDataSize = 0;
      if (obj != null) {
        BytesWritable val =
            ((BinaryObjectInspector) inspector).getPrimitiveWritableObject(obj);
        stream.write(val.getBytes(), 0, val.getLength());
        length.write(val.getLength());


        // Raw data size is the length of the BytesWritable, i.e. the number of bytes
        rawDataSize = val.getLength();
      }
      super.write(obj, rawDataSize);
    }

View Full Code Here


      @Override
      public Object next(Object previous) throws IOException {
        if (nextCalls == 0) {
          nextCalls++;
          return new BytesWritable("a".getBytes());
        }


        throw new IOException("next should only be called once");
      }

View Full Code Here

    }
    return result;
  }


  public static BytesWritable bytes(int... items) {
    BytesWritable result = new BytesWritable();
    result.setSize(items.length);
    for(int i=0; i < items.length; ++i) {
      result.getBytes()[i] = (byte) items[i];
    }
    return result;
  }

View Full Code Here

        byte[] keyBytes = null, valueBytes;


        keyBytes = new byte[keyBuffer.remaining()];
        keyBuffer.get(keyBytes);


        BytesWritable key = new BytesWritable(keyBytes);


        ArrayList<BytesWritable> valueList = new ArrayList();


        while(iterator.hasNext()) {
            ByteBuffer writable = iterator.next().datum();
            writable.rewind();
            // BytesWritable writable = iterator.next();
            valueBytes = null;
            valueBytes = new byte[writable.remaining()];
            writable.get(valueBytes);


            BytesWritable value = new BytesWritable(valueBytes);
            valueList.add(value);


        }


        writer.write(key, valueList.iterator(), reporter);

View Full Code Here

        this.keySerializer = Utils.nonNull(keySerializer);
        this.valueSerializer = Utils.nonNull(valueSerializer);
    }


    public void collect(K key, V value) throws IOException {
        innerCollector.collect(new BytesWritable(keySerializer.toBytes(key)),
                               new BytesWritable(valueSerializer.toBytes(value)));
    }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.io.BytesWritable$Comparator

com.backtype.cascading.tap.PailTap$PailScheme

com.backtype.hadoop.pail.SequenceFileFormat$SequenceFilePailRecordReader

com.cloudera.sqoop.testutil.ManagerCompatTestCase

com.facebook.hive.orc.OrcTestUtils

com.facebook.hive.orc.TestOrcFile

com.facebook.hive.orc.WriterImpl$BinaryTreeWriter

com.facebook.presto.hive.util.SerDeUtils

com.pinterest.secor.tools.LogFileVerifier

edu.yale.cs.hadoopdb.connector.DBInputSplit

fitnesse.testsystems.slim.results.SlimTestResult

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.