Examples of org.apache.spark.network.buffer.ManagedBuffer

org.apache.spark.network.buffer.ManagedBuffer
This interface provides an immutable view for data in the form of bytes. The implementation should specify how the data is provided: - {@link FileSegmentManagedBuffer}: data backed by part of a file - {@link NioManagedBuffer}: data backed by a NIO ByteBuffer - {@link NettyManagedBuffer}: data backed by a Netty ByteBuf The concrete buffer implementation might be managed outside the JVM garbage collector. For example, in the case of {@link NettyManagedBuffer}, the buffers are reference counted. In that case, if the buffer is going to be passed around to a different thread, retain/release should be called.

          int myChunkIndex = (Integer) invocation.getArguments()[1];
          assertEquals(123, streamId);
          assertEquals(expectedChunkIndex.getAndIncrement(), myChunkIndex);


          ChunkReceivedCallback callback = (ChunkReceivedCallback) invocation.getArguments()[2];
          ManagedBuffer result = blockIterator.next();
          if (result != null) {
            callback.onSuccess(myChunkIndex, result);
          } else {
            callback.onFailure(myChunkIndex, new RuntimeException("Failed " + myChunkIndex));
          }

View Full Code Here

  @SuppressWarnings("unchecked")
  @Test
  public void testOpenShuffleBlocks() {
    RpcResponseCallback callback = mock(RpcResponseCallback.class);


    ManagedBuffer block0Marker = new NioManagedBuffer(ByteBuffer.wrap(new byte[3]));
    ManagedBuffer block1Marker = new NioManagedBuffer(ByteBuffer.wrap(new byte[7]));
    when(blockManager.getBlockData("app0", "exec1", "b0")).thenReturn(block0Marker);
    when(blockManager.getBlockData("app0", "exec1", "b1")).thenReturn(block1Marker);
    byte[] openBlocks = new OpenBlocks("app0", "exec1", new String[] { "b0", "b1" }).toByteArray();
    handler.receive(client, openBlocks, callback);
    verify(blockManager, times(1)).getBlockData("app0", "exec1", "b0");

View Full Code Here

    final String client = NettyUtils.getRemoteAddress(channel);
    streamIds.add(req.streamChunkId.streamId);


    logger.trace("Received req from {} to fetch block {}", client, req.streamChunkId);


    ManagedBuffer buf;
    try {
      buf = streamManager.getChunk(req.streamChunkId.streamId, req.streamChunkId.chunkIndex);
    } catch (Exception e) {
      logger.error(String.format(
        "Error opening block %s for request from %s", req.streamChunkId, client), e);

View Full Code Here

    } else if (!state.buffers.hasNext()) {
      throw new IllegalStateException(String.format(
        "Requested chunk index beyond end %s", chunkIndex));
    }
    state.curChunk += 1;
    ManagedBuffer nextChunk = state.buffers.next();


    if (!state.buffers.hasNext()) {
      logger.trace("Removing stream id {}", streamId);
      streams.remove(streamId);
    }

View Full Code Here

TOP

Related Classes of org.apache.spark.network.buffer.ManagedBuffer

org.apache.spark.network.server.OneForOneStreamManager

org.apache.spark.network.server.TransportRequestHandler

org.apache.spark.network.shuffle.ExternalShuffleBlockHandlerSuite

org.apache.spark.network.shuffle.OneForOneBlockFetcherSuite

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.