Examples of org.apache.hadoop.io.DataInputBuffer

org.apache.hadoop.io.DataInputBuffer
A reusable {@link DataInput} implementation that reads from an in-memorybuffer.
This saves memory over creating a new DataInputStream and ByteArrayInputStream each time data is read.
Typical usage is something like the following:
```
 DataInputBuffer buffer = new DataInputBuffer(); while (... loop condition ...) { byte[] data = ... get data ...; int dataLength = ... get data length ...; buffer.reset(data, dataLength); ... read buffer using DataInput methods ... } 
```

      r = new SequenceFile.Reader(fs, f.getPath(), getConf());
      key = ReflectionUtils.newInstance(r.getKeyClass().asSubclass(WritableComparable.class),
                                        getConf());
      val = ReflectionUtils.newInstance(r.getValueClass().asSubclass(Writable.class),
                                        getConf());
      inbuf = new DataInputBuffer();
      outbuf = new DataOutputBuffer();
    }

View Full Code Here

   * @throws IOException
   */
  private static void decodeWritable(Writable obj, 
                                     String newValue) throws IOException {
    Base64 decoder = new Base64(0, null, true);
    DataInputBuffer buf = new DataInputBuffer();
    byte[] decoded = decoder.decode(newValue);
    buf.reset(decoded, decoded.length);
    obj.readFields(buf);
  }

View Full Code Here

    public void stress_read() throws Exception {
        int count = 10000000;
        DecimalOption value = new DecimalOption(new BigDecimal("3.14"));
        byte[] bytes = toBytes(value);
        DecimalOption buf = new DecimalOption();
        DataInputBuffer in = new DataInputBuffer();
        for (int i = 0; i < count; i++) {
            in.reset(bytes, bytes.length);
            buf.readFields(in);
            if (i == 0) {
                assertThat(buf, is(value));
            }
        }

View Full Code Here

    public void stress_read_large() throws Exception {
        int count = 10000000;
        DecimalOption value = new DecimalOption(new BigDecimal(Long.MAX_VALUE).multiply(BigDecimal.TEN));
        byte[] bytes = toBytes(value);
        DecimalOption buf = new DecimalOption();
        DataInputBuffer in = new DataInputBuffer();
        for (int i = 0; i < count; i++) {
            in.reset(bytes, bytes.length);
            buf.readFields(in);
            if (i == 0) {
                assertThat(buf, is(value));
            }
        }

View Full Code Here


        DataOutputBuffer output = new DataOutputBuffer();
        writable.write(output);


        Writable copy = (Writable) loader.newModel("Primitives").unwrap();
        DataInputBuffer input = new DataInputBuffer();
        input.reset(output.getData(), output.getLength());
        copy.readFields(input);


        assertThat(input.read(), is(-1));
        assertThat(writable, equalTo(copy));
    }

View Full Code Here

        byte[] results = Arrays.copyOfRange(out.getData(), 0, out.getLength());
        return results;
    }


    static <T extends Writable> T des(T writable, byte[] serialized) throws IOException {
        DataInputBuffer buf = new DataInputBuffer();
        buf.reset(serialized, serialized.length);
        writable.readFields(buf);
        return writable;
    }

View Full Code Here

        byte[] results = Arrays.copyOfRange(out.getData(), 0, out.getLength());
        return results;
    }


    static <T extends Writable> T des(T writable, byte[] serialized) throws IOException {
        DataInputBuffer buf = new DataInputBuffer();
        buf.reset(serialized, serialized.length);
        writable.readFields(buf);
        return writable;
    }

View Full Code Here

        }
        return Arrays.copyOf(buffer.getData(), buffer.getLength());
    }


    static <T extends Writable> T read(T writable, byte[] bytes) {
        DataInputBuffer buffer = new DataInputBuffer();
        buffer.reset(bytes, bytes.length);
        try {
            writable.readFields(buffer);
        } catch (IOException e) {
            throw new AssertionError(e);
        }

View Full Code Here

        }
        return Arrays.copyOf(buffer.getData(), buffer.getLength());
    }


    static <T extends Writable> T read(T writable, byte[] bytes) {
        DataInputBuffer buffer = new DataInputBuffer();
        buffer.reset(bytes, bytes.length);
        try {
            writable.readFields(buffer);
            assertThat("Enf of Stream", buffer.read(), is(-1));
        } catch (IOException e) {
            throw new AssertionError(e);
        }
        return writable;
    }

View Full Code Here

        object.set("sid", 3L);
        object.set("value", null);
        modelOut.write(object.unwrap());
        modelOut.close();


        DataInputBuffer input = new DataInputBuffer();
        input.reset(output.getData(), output.getLength());
        ModelInput<Object> modelIn = (ModelInput<Object>) type.getAnnotation(ModelInputLocation.class)
            .value()
            .getDeclaredConstructor(RecordParser.class)
            .newInstance(new TsvParser(new InputStreamReader(input, "UTF-8")));
        ModelWrapper copy = loader.newModel("Simple");


        modelIn.readTo(copy.unwrap());
        assertThat(copy.get("sid"), is((Object) 1L));
        assertThat(copy.get("value"), is((Object) new Text("hello")));


        modelIn.readTo(copy.unwrap());
        assertThat(copy.get("sid"), is((Object) 2L));
        assertThat(copy.get("value"), is((Object) new Text("world")));


        modelIn.readTo(copy.unwrap());
        assertThat(copy.get("sid"), is((Object) 3L));
        assertThat(copy.getOption("value").isNull(), is(true));


        assertThat(input.read(), is(-1));
        modelIn.close();
    }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.io.DataInputBuffer

com.asakusafw.dmdl.java.emitter.driver.ModelInputDriverTest

com.asakusafw.runtime.value.DecimalOptionTest

com.datasalt.pangool.BaseTest

com.datasalt.pangool.examples.avro.AvroTopicalWordCount$MyAvroComparator

com.datasalt.pangool.io.TestBitField

com.twitter.elephantbird.pig.load.SequenceFileLoader

com.urbanairship.datacube.CollectionWritableTest

org.apache.accumulo.core.data.Value

org.apache.accumulo.core.file.map.MySequenceFile$CompressedBytes

org.apache.accumulo.core.file.map.MySequenceFile$Reader

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.