Examples of org.apache.hadoop.io.DataOutputBuffer

org.apache.hadoop.io.DataOutputBuffer
A reusable {@link DataOutput} implementation that writes to an in-memorybuffer.
This saves memory over creating a new DataOutputStream and ByteArrayOutputStream each time data is written.
Typical usage is something like the following:
```
 DataOutputBuffer buffer = new DataOutputBuffer(); while (... loop condition ...) { buffer.reset(); ... write buffer using DataOutput methods ... byte[] data = buffer.getData(); int dataLength = buffer.getLength(); ... write data to its ultimate destination ... } 
```

        = new IdxIndexDescriptor(qualifierName2, IdxQualifierType.LONG);


    descriptor.addIndexDescriptor(indexDescriptor1);
    descriptor.addIndexDescriptor(indexDescriptor2);


    DataOutputBuffer dataOutputBuffer = new DataOutputBuffer();
    descriptor.write(dataOutputBuffer);


    DataInputBuffer dataInputBuffer = new DataInputBuffer();
    dataInputBuffer.reset(dataOutputBuffer.getData(), dataOutputBuffer.getLength());


    IdxColumnDescriptor clonedDescriptor = new IdxColumnDescriptor();
    clonedDescriptor.readFields(dataInputBuffer);


    Assert.assertEquals("The expression was not the same after being written and read", descriptor, clonedDescriptor);

View Full Code Here

  public void testWritable() throws IOException {
    byte[] qualifierName1 = Bytes.toBytes("qualifer1");
    IdxIndexDescriptor descriptor
        = new IdxIndexDescriptor(qualifierName1, IdxQualifierType.INT);


    DataOutputBuffer dataOutputBuffer = new DataOutputBuffer();
    descriptor.write(dataOutputBuffer);


    DataInputBuffer dataInputBuffer = new DataInputBuffer();
    dataInputBuffer.reset(dataOutputBuffer.getData(), dataOutputBuffer.getLength());


    IdxIndexDescriptor clonedDescriptor = new IdxIndexDescriptor();
    clonedDescriptor.readFields(dataInputBuffer);


    Assert.assertEquals("The descriptor was not the same after being written and read", descriptor, clonedDescriptor);

View Full Code Here

   * @throws java.io.IOException if an io error occurs
   */
  public void testWritable() throws IOException {
    Expression expression = Expression.comparison("columnName1", "qualifier1", Comparison.Operator.EQ, Bytes.toBytes("value"));


    DataOutputBuffer dataOutputBuffer = new DataOutputBuffer();
    expression.write(dataOutputBuffer);


    DataInputBuffer dataInputBuffer = new DataInputBuffer();
    dataInputBuffer.reset(dataOutputBuffer.getData(), dataOutputBuffer.getLength());


    Expression clonedExpression = new Comparison();
    clonedExpression.readFields(dataInputBuffer);


    Assert.assertEquals("The expression was not the same after being written and read", expression, clonedExpression);

View Full Code Here

            Expression.comparison("columnName2", "qualifier2", Comparison.Operator.GT, Bytes.toBytes("value2")),
            Expression.comparison("columnName3", "qualifier3", Comparison.Operator.LT, Bytes.toBytes("value3"))
        )
    );


    DataOutputBuffer dataOutputBuffer = new DataOutputBuffer();
    expression.write(dataOutputBuffer);


    DataInputBuffer dataInputBuffer = new DataInputBuffer();
    dataInputBuffer.reset(dataOutputBuffer.getData(), dataOutputBuffer.getLength());


    Expression clonedExpression = new Or();
    clonedExpression.readFields(dataInputBuffer);


    Assert.assertEquals("The expression was not the same after being written and read", expression, clonedExpression);

View Full Code Here

      key = (WritableComparable)ReflectionUtils.newInstance(
          r.getKeyClass(), getConf());
      val = (Writable)ReflectionUtils.newInstance(
          r.getValueClass(), getConf());
      inbuf = new DataInputBuffer();
      outbuf = new DataOutputBuffer();
    }

View Full Code Here

      this.partitions = job.getNumReduceTasks();
      this.partitioner = (Partitioner)ReflectionUtils.newInstance(
                                                                  job.getPartitionerClass(), job);
      maxBufferSize = job.getInt("io.sort.mb", 100) * 1024 * 1024 / 2;
      this.sortSpillException = null;
      keyValBuffer = new DataOutputBuffer();


      this.job = job;
      this.reporter = reporter;
      this.comparator = job.getOutputKeyComparator();
      this.keyClass = job.getMapOutputKeyClass();

View Full Code Here

      if (sortSpillException != null) {
        throw (IOException) new IOException().initCause(sortSpillException);
      }
      
      if (keyValBuffer == null) {
        keyValBuffer = new DataOutputBuffer();
        sortImpl = new BufferSorter[partitions];
        for (int i = 0; i < partitions; i++)
          sortImpl[i] = (BufferSorter)ReflectionUtils.newInstance(
                            job.getClass("map.sort.class", MergeSorter.class, 
                                         BufferSorter.class), job);

View Full Code Here

        throw new RuntimeException(e);
      }


      DataInputBuffer keyIn = new DataInputBuffer();
      DataInputBuffer valIn = new DataInputBuffer();
      DataOutputBuffer valOut = new DataOutputBuffer();
      while (resultIter.next()) {
        keyIn.reset(resultIter.getKey().getData(), 
                    resultIter.getKey().getLength());
        key.readFields(keyIn);
        valOut.reset();
        (resultIter.getValue()).writeUncompressedBytes(valOut);
        valIn.reset(valOut.getData(), valOut.getLength());
        value.readFields(valIn);
        writer.append(key, value);
        reporter.progress();
      }
    }

View Full Code Here

   */
  private void writeSplitsFile(InputSplit[] splits, FSDataOutputStream out) throws IOException {
    out.write(SPLIT_FILE_HEADER);
    WritableUtils.writeVInt(out, CURRENT_SPLIT_FILE_VERSION);
    WritableUtils.writeVInt(out, splits.length);
    DataOutputBuffer buffer = new DataOutputBuffer();
    RawSplit rawSplit = new RawSplit();
    for(InputSplit split: splits) {
      rawSplit.setClassName(split.getClass().getName());
      buffer.reset();
      split.write(buffer);
      rawSplit.setBytes(buffer.getData(), 0, buffer.getLength());
      rawSplit.setLocations(split.getLocations());
      rawSplit.write(out);
    }
  }

View Full Code Here

          if (!outputFs.mkdirs(tmpDir)) {
            LOG.error("Mkdirs failed to create " + tmpDir.toString());
          }
        }


        DataOutputBuffer buffer = new DataOutputBuffer();
        for (int i = 0; i < splits.length; i++) {
          String mapId = jobId + "_map_" + idFormat.format(i);
          mapIds.add(mapId);
          buffer.reset();
          splits[i].write(buffer);
          BytesWritable split = new BytesWritable();
          split.set(buffer.getData(), 0, buffer.getLength());
          MapTask map = new MapTask(jobId, file.toString(), "tip_m_" + mapId, 
                                    mapId, i,
                                    splits[i].getClass().getName(),
                                    split);
          JobConf localConf = new JobConf(job);

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.io.DataOutputBuffer

ch.uzh.ifi.ddis.hadoop.CsvInputFormat$CsvRecordReader

com.alibaba.jstorm.yarn.JStormOnYarn

com.alibaba.jstorm.yarn.StormAMRMClient

com.asakusafw.dmdl.java.emitter.driver.ModelInputDriverTest

com.asakusafw.runtime.io.util.WritableTestRoot

com.datasalt.pangool.BaseTest

com.datasalt.pangool.io.TestBitField

com.datasalt.pangool.io.TupleFile$Writer

com.datasalt.pangool.tuplemr.mapred.BinaryComparator

com.datasalt.pangool.tuplemr.mapred.SerializerComparator

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.