Examples of org.apache.hadoop.io.DataInputBuffer

org.apache.hadoop.io.DataInputBuffer
A reusable {@link DataInput} implementation that reads from an in-memorybuffer.
This saves memory over creating a new DataInputStream and ByteArrayInputStream each time data is read.
Typical usage is something like the following:
```
 DataInputBuffer buffer = new DataInputBuffer(); while (... loop condition ...) { byte[] data = ... get data ...; int dataLength = ... get data length ...; buffer.reset(data, dataLength); ... read buffer using DataInput methods ... } 
```

  }


  private static void run(Query query) throws IOException {


    DataOutputBuffer buffer = new DataOutputBuffer();
    DataInputBuffer in = new DataInputBuffer();
    QueryWritable outQw = new QueryWritable();
    QueryWritable inQw = new QueryWritable();


    long s = System.nanoTime();
    int count = 100000;
    for (int i = 0; i < count; i++) {
      outQw.setQuery(query);
      outQw.write(buffer);


      in.reset(buffer.getData(), 0, buffer.getLength());
      inQw.readFields(in);


      buffer.reset();
    }
    long e = System.nanoTime();

View Full Code Here

      IOException wrap = new IOException("Split class " + splitClass + 
                                         " not found");
      wrap.initCause(exp);
      throw wrap;
    }
    DataInputBuffer splitBuffer = new DataInputBuffer();
    splitBuffer.reset(split.get(), 0, split.getSize());
    instantiatedSplit.readFields(splitBuffer);
    
    // if it is a file split, we can give more details
    if (instantiatedSplit instanceof FileSplit) {
      FileSplit fileSplit = (FileSplit) instantiatedSplit;

View Full Code Here

      r = new SequenceFile.Reader(fs, f.getPath(), getConf());
      key = (WritableComparable)ReflectionUtils.newInstance(
          r.getKeyClass(), getConf());
      val = (Writable)ReflectionUtils.newInstance(
          r.getValueClass(), getConf());
      inbuf = new DataInputBuffer();
      outbuf = new DataOutputBuffer();
    }

View Full Code Here

        value = (Writable)ReflectionUtils.newInstance(valClass, job);
      } catch (Exception e) {
        throw new RuntimeException(e);
      }


      DataInputBuffer keyIn = new DataInputBuffer();
      DataInputBuffer valIn = new DataInputBuffer();
      DataOutputBuffer valOut = new DataOutputBuffer();
      while (resultIter.next()) {
        keyIn.reset(resultIter.getKey().getData(), 
                    resultIter.getKey().getLength());
        key.readFields(keyIn);
        valOut.reset();
        (resultIter.getValue()).writeUncompressedBytes(valOut);
        valIn.reset(valOut.getData(), valOut.getLength());
        value.readFields(valIn);
        writer.append(key, value);
        reporter.progress();
      }
    }

View Full Code Here

      }
      return Arrays.copyOf(out.getData(), out.getLength());
    }


    public void fromBytes(byte bytes[]) throws IOException {
      DataInputBuffer inp = new DataInputBuffer();
      inp.reset(bytes, bytes.length);
      extent = new KeyExtent();
      extent.readFields(inp);
      timestamp = inp.readLong();
      server = inp.readUTF();
      filename = inp.readUTF();
      tabletId = inp.read();
      int count = inp.read();
      ArrayList<String> logSet = new ArrayList<String>(count);
      for (int i = 0; i < count; i++)
        logSet.add(inp.readUTF());
      this.logSet = logSet;
    }

View Full Code Here

  private Map<Text,MergeInfo> parseMerges(String merges) {
    if (merges == null)
      return null;
    try {
      Map<Text,MergeInfo> result = new HashMap<Text,MergeInfo>();
      DataInputBuffer buffer = new DataInputBuffer();
      byte[] data = Base64.decodeBase64(merges.getBytes(Constants.UTF8));
      buffer.reset(data, data.length);
      while (buffer.available() > 0) {
        MergeInfo mergeInfo = new MergeInfo();
        mergeInfo.readFields(buffer);
        result.put(mergeInfo.range.getTableId(), mergeInfo);
      }
      return result;

View Full Code Here

      try {
        String path = ZooUtil.getRoot(instance.getInstanceID()) + Constants.ZTABLES + "/" + tableId.toString() + "/merge";
        if (!ZooReaderWriter.getInstance().exists(path))
          return new MergeInfo();
        byte[] data = ZooReaderWriter.getInstance().getData(path, new Stat());
        DataInputBuffer in = new DataInputBuffer();
        in.reset(data, data.length);
        MergeInfo info = new MergeInfo();
        info.readFields(in);
        return info;
      } catch (KeeperException.NoNodeException ex) {
        log.info("Error reading merge state, it probably just finished");

View Full Code Here

      final String table = entry.getKey(), tableId = entry.getValue();
      String path = ZooUtil.getRoot(conn.getInstance().getInstanceID()) + Constants.ZTABLES + "/" + tableId + "/merge";
      MergeInfo info = new MergeInfo();
      if (ZooReaderWriter.getInstance().exists(path)) {
        byte[] data = ZooReaderWriter.getInstance().getData(path, new Stat());
        DataInputBuffer in = new DataInputBuffer();
        in.reset(data, data.length);
        info.readFields(in);
      }
      System.out.println(String.format("%25s  %10s %10s %s", table, info.state, info.operation, info.range));
    }
  }

View Full Code Here

        beginLocation = begin;
        endLocation = end;
        
        valTransferBuffer = new BytesWritable();
        keyBuffer = new byte[MAX_KEY_SIZE];
        keyDataInputStream = new DataInputBuffer();
        valueBufferInputStream = new ChunkDecoder();
        valueDataInputStream = new DataInputStream(valueBufferInputStream);
        
        if (beginLocation.compareTo(endLocation) >= 0) {
          currentLocation = new Location(endLocation);

View Full Code Here

      final int offset, final int length) throws IOException {
    if (bytes == null) {
      throw new IllegalArgumentException(
          "Can't build an object with empty bytes array");
    }
    DataInputBuffer in = new DataInputBuffer();
    List<EntityGroupInfo> egis = new ArrayList<EntityGroupInfo>();
    try {
      in.reset(bytes, offset, length);
      while (in.available() > 0) {
        EntityGroupInfo egi = parseFrom(in);
        egis.add(egi);
      }
    } finally {
      in.close();
    }
    return egis;
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.io.DataInputBuffer

com.asakusafw.dmdl.java.emitter.driver.ModelInputDriverTest

com.asakusafw.runtime.value.DecimalOptionTest

com.datasalt.pangool.BaseTest

com.datasalt.pangool.examples.avro.AvroTopicalWordCount$MyAvroComparator

com.datasalt.pangool.io.TestBitField

com.twitter.elephantbird.pig.load.SequenceFileLoader

com.urbanairship.datacube.CollectionWritableTest

org.apache.accumulo.core.data.Value

org.apache.accumulo.core.file.map.MySequenceFile$CompressedBytes

org.apache.accumulo.core.file.map.MySequenceFile$Reader

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.