Examples of org.apache.hadoop.io.DataOutputBuffer

org.apache.hadoop.io.DataOutputBuffer
A reusable {@link DataOutput} implementation that writes to an in-memorybuffer.
This saves memory over creating a new DataOutputStream and ByteArrayOutputStream each time data is written.
Typical usage is something like the following:
```
 DataOutputBuffer buffer = new DataOutputBuffer(); while (... loop condition ...) { buffer.reset(); ... write buffer using DataOutput methods ... byte[] data = buffer.getData(); int dataLength = buffer.getLength(); ... write data to its ultimate destination ... } 
```

     */
    protected void writeScanMetrics() throws IOException {
      if (this.scanMetrics == null || scanMetricsPublished) {
        return;
      }
      final DataOutputBuffer d = new DataOutputBuffer();
      scanMetrics.write(d);
      scan.setAttribute(Scan.SCAN_ATTRIBUTES_METRICS_DATA, d.getData());
      scanMetricsPublished = true;
    }

View Full Code Here

     */
    private void writeHeader() throws IOException {
      out.write(HBaseServer.HEADER.array());
      out.write(HBaseServer.CURRENT_VERSION);
      //When there are more fields we can have ConnectionHeader Writable.
      DataOutputBuffer buf = new DataOutputBuffer();
      ObjectWritable.writeObject(buf, remoteId.getTicket(), 
                                 UserGroupInformation.class, conf);
      int bufLen = buf.getLength();
      out.writeInt(bufLen);
      out.write(buf.getData(), 0, bufLen);
    }

View Full Code Here

    public void sendParam(Call call) {
      if (shouldCloseConnection.get()) {
        return;
      }


      DataOutputBuffer d=null;
      try {
        synchronized (this.out) {
          if (LOG.isDebugEnabled())
            LOG.debug(getName() + " sending #" + call.id);
          
          //for serializing the
          //data to be written
          d = new DataOutputBuffer();
          d.writeInt(call.id);
          call.param.write(d);
          byte[] data = d.getData();
          int dataLength = d.getLength();
          out.writeInt(dataLength);      //first put the data length
          out.write(data, 0, dataLength);//write the data
          out.flush();
        }
      } catch(IOException e) {

View Full Code Here

    log.info("collected " + toSendSize + " chunks for post_"+thisPost);


    // Serialize each chunk in turn into it's own DataOutputBuffer and add that
    // buffer to serializedEvents
    for (Chunk c : toSend) {
      DataOutputBuffer b = new DataOutputBuffer(c.getSerializedSizeEstimate());
      try {
        c.write(b);
      } catch (IOException err) {
        log.error("serialization threw IOException", err);
      }

View Full Code Here

    ChunkBuilder cb = new ChunkBuilder();
    cb.addRecord("foo".getBytes());
    cb.addRecord("bar".getBytes());
    cb.addRecord("baz".getBytes());
    Chunk c = cb.getChunk();
    DataOutputBuffer ob = new DataOutputBuffer(c.getSerializedSizeEstimate());
    try {
      c.write(ob);
      DataInputBuffer ib = new DataInputBuffer();
      ib.reset(ob.getData(), c.getSerializedSizeEstimate());
      int version = ib.readInt();
      assertEquals(version, ChunkImpl.PROTOCOL_VERSION);
    } catch (IOException e) {
      e.printStackTrace();
      fail("Should nor raise any exception");

View Full Code Here

    ChunkBuilder cb = new ChunkBuilder();
    cb.addRecord("foo".getBytes());
    cb.addRecord("bar".getBytes());
    cb.addRecord("baz".getBytes());
    Chunk c = cb.getChunk();
    DataOutputBuffer ob = new DataOutputBuffer(c.getSerializedSizeEstimate());
    try {
      c.write(ob);
      DataInputBuffer ib = new DataInputBuffer();
      ib.reset(ob.getData(), c.getSerializedSizeEstimate());
      // change current chunkImpl version
      ChunkImpl.PROTOCOL_VERSION = ChunkImpl.PROTOCOL_VERSION + 1;
      ChunkImpl.read(ib);
      fail("Should have raised an IOexception");
    } catch (IOException e) {

View Full Code Here


      record.add(Record.tagsField, chunk.getTags());
      record.add(Record.sourceField, chunk.getSource());
      record.add(Record.applicationField, chunk.getStreamName());


      DataOutputBuffer ob = new DataOutputBuffer(chunk
          .getSerializedSizeEstimate());
      chunk.write(ob);
      record.add(Record.chunkDataField, new String(ob.getData()));
      record.add(Record.chunkExceptionField, ExceptionUtil
          .getStackTrace(throwable));
      output.collect(key, record);


      return record;

View Full Code Here

                       appAttemptID.toString());


    UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
    Credentials credentials =
      currentUser.getCredentials();
    DataOutputBuffer dob = new DataOutputBuffer();
    credentials.writeTokenStorageToStream(dob);
    dob.close();
    // Now remove the AM->RM token so that containers cannot access it.
    Iterator<Token<?>> iter = credentials.getAllTokens().iterator();
    while (iter.hasNext()) {
      Token<?> token = iter.next();
      log.info("Token {}", token.getKind());
      if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) {
        iter.remove();
      }
    }
    allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
    
    // set up secret manager
    secretManager = new ClientToAMTokenSecretManager(appAttemptID, null);


    // if not a secure cluster, extract the username -it will be

View Full Code Here

      if (firstKey == null) {
        Utils.writeVInt(out, 0);
        return;
      }


      DataOutputBuffer dob = new DataOutputBuffer();
      Utils.writeVInt(dob, firstKey.size());
      dob.write(firstKey.buffer());
      Utils.writeVInt(out, dob.size());
      out.write(dob.getData(), 0, dob.getLength());


      for (TFileIndexEntry entry : index) {
        dob.reset();
        entry.write(dob);
        Utils.writeVInt(out, dob.getLength());
        out.write(dob.getData(), 0, dob.getLength());
      }
    }

View Full Code Here

     */
    protected void writeScanMetrics() throws IOException {
      if (this.scanMetrics == null || scanMetricsPublished) {
        return;
      }
      final DataOutputBuffer d = new DataOutputBuffer();
      scanMetrics.write(d);
      scan.setAttribute(Scan.SCAN_ATTRIBUTES_METRICS_DATA, d.getData());
      scanMetricsPublished = true;
    }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.io.DataOutputBuffer

ch.uzh.ifi.ddis.hadoop.CsvInputFormat$CsvRecordReader

com.alibaba.jstorm.yarn.JStormOnYarn

com.alibaba.jstorm.yarn.StormAMRMClient

com.asakusafw.dmdl.java.emitter.driver.ModelInputDriverTest

com.asakusafw.runtime.io.util.WritableTestRoot

com.datasalt.pangool.BaseTest

com.datasalt.pangool.io.TestBitField

com.datasalt.pangool.io.TupleFile$Writer

com.datasalt.pangool.tuplemr.mapred.BinaryComparator

com.datasalt.pangool.tuplemr.mapred.SerializerComparator

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.