Examples of org.apache.hadoop.io.BoundedByteArrayOutputStream

org.apache.hadoop.io.BoundedByteArrayOutputStream
A byte array backed output stream with a limit. The limit should be smaller than the buffer capacity. The object can be reused through reset API and choose different limits in each round.

    Assert.assertEquals("exception reporter invoked",
        0, reporter.getNumExceptions());
  }


  private void fillOutput(MapOutput<Text, Text> output) throws IOException {
    BoundedByteArrayOutputStream stream = output.getArrayStream();
    int count = stream.getLimit();
    for (int i=0; i < count; ++i) {
      stream.write(i);
    }
  }

View Full Code Here


  @Test
  //Test InMemoryWriter
  public void testInMemoryWriter() throws IOException {
    InMemoryWriter writer = null;
    BoundedByteArrayOutputStream bout = new BoundedByteArrayOutputStream(1024 * 1024);


    List<KVPair> data = KVDataGen.generateTestData(true, 0);


    //No RLE, No RepeatKeys, no compression
    writer = new InMemoryWriter(bout);
    writeTestFileUsingDataBuffer(writer, false, false, data, null);
    readUsingInMemoryReader(bout.getBuffer(), data);


    //No RLE, RepeatKeys, no compression
    bout.reset();
    writer = new InMemoryWriter(bout);
    writeTestFileUsingDataBuffer(writer, false, true, data, null);
    readUsingInMemoryReader(bout.getBuffer(), data);


    //RLE, No RepeatKeys, no compression
    bout.reset();
    writer = new InMemoryWriter(bout);
    writeTestFileUsingDataBuffer(writer, true, false, data, null);
    readUsingInMemoryReader(bout.getBuffer(), data);


    //RLE, RepeatKeys, no compression
    bout.reset();
    writer = new InMemoryWriter(bout);
    writeTestFileUsingDataBuffer(writer, true, true, data, null);
    readUsingInMemoryReader(bout.getBuffer(), data);
  }

View Full Code Here


    // Other type specific values


    if (type == Type.MEMORY) {
      // since we are passing an int from createMemoryMapOutput, its safe to cast to int
      this.byteStream = new BoundedByteArrayOutputStream((int)size);
      this.memory = byteStream.getBuffer();
    } else {
      this.byteStream = null;
      this.memory = null;
    }

View Full Code Here


  public MemoryFetchedInput(long actualSize, long compressedSize,
      InputAttemptIdentifier inputAttemptIdentifier,
      FetchedInputCallback callbackHandler) {
    super(Type.MEMORY, actualSize, compressedSize, inputAttemptIdentifier, callbackHandler);
    this.byteStream = new BoundedByteArrayOutputStream((int) actualSize);
  }

View Full Code Here

    this.id = ID.incrementAndGet();
    this.mapId = mapId;
    this.merger = merger;


    type = Type.MEMORY;
    byteStream = new BoundedByteArrayOutputStream(size);
    memory = byteStream.getBuffer();


    this.size = size;
    
    localFS = null;

View Full Code Here

    keySerializer.open(keyBuf);
    valueSerializer.open(valBuf);


    List<TezMerger.Segment> segments = new LinkedList<TezMerger.Segment>();
    for (int i = 0; i < numberOfStreams; i++) {
      BoundedByteArrayOutputStream bout = new BoundedByteArrayOutputStream(1024 * 1024);
      InMemoryWriter writer =
          new InMemoryWriter(bout);
      Map<Writable, Writable> data = createData();
      //write data
      for (Map.Entry<Writable, Writable> entry : data.entrySet()) {
        keySerializer.serialize(entry.getKey());
        valueSerializer.serialize(entry.getValue());
        keyIn.reset(keyBuf.getData(), 0, keyBuf.getLength());
        valIn.reset(valBuf.getData(), 0, valBuf.getLength());
        writer.append(keyIn, valIn);
        keyBuf.reset();
        valBuf.reset();
        keyIn.reset();
        valIn.reset();
      }
      IFile.Reader reader = new InMemoryReader(mergeManager, null, bout.getBuffer(), 0,
          bout.getBuffer().length);
      segments.add(new TezMerger.Segment(reader, true));


      data.clear();
      writer.close();
    }

View Full Code Here

    this.id = ID.incrementAndGet();
    this.mapId = mapId;
    this.merger = merger;


    type = Type.MEMORY;
    byteStream = new BoundedByteArrayOutputStream(size);
    memory = byteStream.getBuffer();


    this.size = size;
    
    localFS = null;

View Full Code Here

    this.id = ID.incrementAndGet();
    this.attemptIdentifier = attemptIdentifier;
    this.merger = merger;


    type = Type.MEMORY;
    byteStream = new BoundedByteArrayOutputStream(size);
    memory = byteStream.getBuffer();


    this.size = size;
    
    localFS = null;

View Full Code Here

      sizeMinBlock = minBlockSize;
      tfileMeta = new TFileMeta(comparator);
      tfileIndex = new TFileIndex(tfileMeta.getComparator());


      writerBCF = new BCFile.Writer(fsdos, compressName, conf);
      currentKeyBufferOS = new BoundedByteArrayOutputStream(MAX_KEY_SIZE);
      lastKeyBufferOS = new BoundedByteArrayOutputStream(MAX_KEY_SIZE);
      this.conf = conf;
    }

View Full Code Here

                lastLen) < 0) {
              throw new IOException("Keys are not added in sorted order");
            }
          }


          BoundedByteArrayOutputStream tmp = currentKeyBufferOS;
          currentKeyBufferOS = lastKeyBufferOS;
          lastKeyBufferOS = tmp;
          --errorCount;
        } finally {
          closed = true;

View Full Code Here

0 1 2

TOP

Related Classes of org.apache.hadoop.io.BoundedByteArrayOutputStream

org.apache.hadoop.io.file.tfile.TFile$Writer

org.apache.hadoop.io.file.tfile.TFile$Writer$KeyRegister

org.apache.hadoop.mapreduce.task.reduce.InMemoryMapOutput

org.apache.hadoop.mapreduce.task.reduce.MapOutput

org.apache.hadoop.mapreduce.task.reduce.TestMergeManager

org.apache.tez.runtime.library.common.shuffle.impl.MapOutput

org.apache.tez.runtime.library.common.sort.impl.TestIFile

org.apache.tez.runtime.library.common.TestValuesIterator

org.apache.tez.runtime.library.shuffle.common.MemoryFetchedInput

java.io.EOFException

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.