Package org.apache.spark.network.shuffle.protocol

Examples of org.apache.spark.network.shuffle.protocol.OpenBlocks

131
132
133
134
135
136
137
138
139
140
141
      public Void answer(InvocationOnMock invocationOnMock) throws Throwable {
        BlockTransferMessage message = BlockTransferMessage.Decoder.fromByteArray(
          (byte[]) invocationOnMock.getArguments()[0]);
        RpcResponseCallback callback = (RpcResponseCallback) invocationOnMock.getArguments()[1];
        callback.onSuccess(new StreamHandle(123, blocks.size()).toByteArray());
        assertEquals(new OpenBlocks("app-id", "exec-id", blockIds), message);
        return null;
      }
    }).when(client).sendRpc((byte[]) any(), (RpcResponseCallback) any());

    // Respond to each chunk request with a single buffer from our blocks array.
View Full Code Here
75
76
77
78
79
80
81
82
83
84
85
    ManagedBuffer block0Marker = new NioManagedBuffer(ByteBuffer.wrap(new byte[3]));
    ManagedBuffer block1Marker = new NioManagedBuffer(ByteBuffer.wrap(new byte[7]));
    when(blockManager.getBlockData("app0", "exec1", "b0")).thenReturn(block0Marker);
    when(blockManager.getBlockData("app0", "exec1", "b1")).thenReturn(block1Marker);
    byte[] openBlocks = new OpenBlocks("app0", "exec1", new String[] { "b0", "b1" }).toByteArray();
    handler.receive(client, openBlocks, callback);
    verify(blockManager, times(1)).getBlockData("app0", "exec1", "b0");
    verify(blockManager, times(1)).getBlockData("app0", "exec1", "b1");

    ArgumentCaptor<byte[]> response = ArgumentCaptor.forClass(byte[].class);
View Full Code Here
64
65
66
67
68
69
70
71
72
73
74
  @Override
  public void receive(TransportClient client, byte[] message, RpcResponseCallback callback) {
    BlockTransferMessage msgObj = BlockTransferMessage.Decoder.fromByteArray(message);

    if (msgObj instanceof OpenBlocks) {
      OpenBlocks msg = (OpenBlocks) msgObj;
      List<ManagedBuffer> blocks = Lists.newArrayList();

      for (String blockId : msg.blockIds) {
        blocks.add(blockManager.getBlockData(msg.appId, msg.execId, blockId));
      }
View Full Code Here
56
57
58
59
60
61
62
63
64
65
      String appId,
      String execId,
      String[] blockIds,
      BlockFetchingListener listener) {
    this.client = client;
    this.openMessage = new OpenBlocks(appId, execId, blockIds);
    this.blockIds = blockIds;
    this.listener = listener;
    this.chunkCallback = new ChunkCallback();
  }
View Full Code Here
TOP

Related Classes of org.apache.spark.network.shuffle.protocol.OpenBlocks

  • org.apache.spark.network.shuffle.ExternalShuffleBlockHandler
  • org.apache.spark.network.shuffle.ExternalShuffleBlockHandlerSuite
  • org.apache.spark.network.shuffle.OneForOneBlockFetcher
  • org.apache.spark.network.shuffle.OneForOneBlockFetcherSuite
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.