Examples of BoundedByteArrayOutputStream

org.apache.hadoop.io.BoundedByteArrayOutputStream
A byte array backed output stream with a limit. The limit should be smaller than the buffer capacity. The object can be reused through reset API and choose different limits in each round.
org.apache.hadoop.zebra.tfile.BoundedByteArrayOutputStream
A byte array backed output stream with a limit. The limit should be smaller than the buffer capacity. The object can be reused through reset API and choose different limits in each round.

Examples of org.apache.hadoop.io.BoundedByteArrayOutputStream

    Assert.assertEquals("exception reporter invoked",
        0, reporter.getNumExceptions());
  }


  private void fillOutput(MapOutput<Text, Text> output) throws IOException {
    BoundedByteArrayOutputStream stream = output.getArrayStream();
    int count = stream.getLimit();
    for (int i=0; i < count; ++i) {
      stream.write(i);
    }
  }

View Full Code Here

Examples of org.apache.hadoop.io.BoundedByteArrayOutputStream


  @Test
  //Test InMemoryWriter
  public void testInMemoryWriter() throws IOException {
    InMemoryWriter writer = null;
    BoundedByteArrayOutputStream bout = new BoundedByteArrayOutputStream(1024 * 1024);


    List<KVPair> data = KVDataGen.generateTestData(true, 0);


    //No RLE, No RepeatKeys, no compression
    writer = new InMemoryWriter(bout);
    writeTestFileUsingDataBuffer(writer, false, false, data, null);
    readUsingInMemoryReader(bout.getBuffer(), data);


    //No RLE, RepeatKeys, no compression
    bout.reset();
    writer = new InMemoryWriter(bout);
    writeTestFileUsingDataBuffer(writer, false, true, data, null);
    readUsingInMemoryReader(bout.getBuffer(), data);


    //RLE, No RepeatKeys, no compression
    bout.reset();
    writer = new InMemoryWriter(bout);
    writeTestFileUsingDataBuffer(writer, true, false, data, null);
    readUsingInMemoryReader(bout.getBuffer(), data);


    //RLE, RepeatKeys, no compression
    bout.reset();
    writer = new InMemoryWriter(bout);
    writeTestFileUsingDataBuffer(writer, true, true, data, null);
    readUsingInMemoryReader(bout.getBuffer(), data);
  }

View Full Code Here

Examples of org.apache.hadoop.io.BoundedByteArrayOutputStream


    // Other type specific values


    if (type == Type.MEMORY) {
      // since we are passing an int from createMemoryMapOutput, its safe to cast to int
      this.byteStream = new BoundedByteArrayOutputStream((int)size);
      this.memory = byteStream.getBuffer();
    } else {
      this.byteStream = null;
      this.memory = null;
    }

View Full Code Here

Examples of org.apache.hadoop.io.BoundedByteArrayOutputStream


  public MemoryFetchedInput(long actualSize, long compressedSize,
      InputAttemptIdentifier inputAttemptIdentifier,
      FetchedInputCallback callbackHandler) {
    super(Type.MEMORY, actualSize, compressedSize, inputAttemptIdentifier, callbackHandler);
    this.byteStream = new BoundedByteArrayOutputStream((int) actualSize);
  }

View Full Code Here

Examples of org.apache.hadoop.io.BoundedByteArrayOutputStream

    this.id = ID.incrementAndGet();
    this.mapId = mapId;
    this.merger = merger;


    type = Type.MEMORY;
    byteStream = new BoundedByteArrayOutputStream(size);
    memory = byteStream.getBuffer();


    this.size = size;
    
    localFS = null;

View Full Code Here

Examples of org.apache.hadoop.io.BoundedByteArrayOutputStream

    keySerializer.open(keyBuf);
    valueSerializer.open(valBuf);


    List<TezMerger.Segment> segments = new LinkedList<TezMerger.Segment>();
    for (int i = 0; i < numberOfStreams; i++) {
      BoundedByteArrayOutputStream bout = new BoundedByteArrayOutputStream(1024 * 1024);
      InMemoryWriter writer =
          new InMemoryWriter(bout);
      Map<Writable, Writable> data = createData();
      //write data
      for (Map.Entry<Writable, Writable> entry : data.entrySet()) {
        keySerializer.serialize(entry.getKey());
        valueSerializer.serialize(entry.getValue());
        keyIn.reset(keyBuf.getData(), 0, keyBuf.getLength());
        valIn.reset(valBuf.getData(), 0, valBuf.getLength());
        writer.append(keyIn, valIn);
        keyBuf.reset();
        valBuf.reset();
        keyIn.reset();
        valIn.reset();
      }
      IFile.Reader reader = new InMemoryReader(mergeManager, null, bout.getBuffer(), 0,
          bout.getBuffer().length);
      segments.add(new TezMerger.Segment(reader, true));


      data.clear();
      writer.close();
    }

View Full Code Here

Examples of org.apache.hadoop.io.BoundedByteArrayOutputStream

    this.id = ID.incrementAndGet();
    this.mapId = mapId;
    this.merger = merger;


    type = Type.MEMORY;
    byteStream = new BoundedByteArrayOutputStream(size);
    memory = byteStream.getBuffer();


    this.size = size;
    
    localFS = null;

View Full Code Here

Examples of org.apache.hadoop.io.BoundedByteArrayOutputStream

    this.id = ID.incrementAndGet();
    this.attemptIdentifier = attemptIdentifier;
    this.merger = merger;


    type = Type.MEMORY;
    byteStream = new BoundedByteArrayOutputStream(size);
    memory = byteStream.getBuffer();


    this.size = size;
    
    localFS = null;

View Full Code Here

Examples of org.apache.hadoop.io.BoundedByteArrayOutputStream

      sizeMinBlock = minBlockSize;
      tfileMeta = new TFileMeta(comparator);
      tfileIndex = new TFileIndex(tfileMeta.getComparator());


      writerBCF = new BCFile.Writer(fsdos, compressName, conf);
      currentKeyBufferOS = new BoundedByteArrayOutputStream(MAX_KEY_SIZE);
      lastKeyBufferOS = new BoundedByteArrayOutputStream(MAX_KEY_SIZE);
      this.conf = conf;
    }

View Full Code Here

Examples of org.apache.hadoop.io.BoundedByteArrayOutputStream

                lastLen) < 0) {
              throw new IOException("Keys are not added in sorted order");
            }
          }


          BoundedByteArrayOutputStream tmp = currentKeyBufferOS;
          currentKeyBufferOS = lastKeyBufferOS;
          lastKeyBufferOS = tmp;
          --errorCount;
        } finally {
          closed = true;

View Full Code Here

0 1 2 3

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.