Package org.apache.spark.network.shuffle.protocol

Examples of org.apache.spark.network.shuffle.protocol.StreamHandle

130
131
132
133
134
135
136
137
138
139
140
      @Override
      public Void answer(InvocationOnMock invocationOnMock) throws Throwable {
        BlockTransferMessage message = BlockTransferMessage.Decoder.fromByteArray(
          (byte[]) invocationOnMock.getArguments()[0]);
        RpcResponseCallback callback = (RpcResponseCallback) invocationOnMock.getArguments()[1];
        callback.onSuccess(new StreamHandle(123, blocks.size()).toByteArray());
        assertEquals(new OpenBlocks("app-id", "exec-id", blockIds), message);
        return null;
      }
    }).when(client).sendRpc((byte[]) any(), (RpcResponseCallback) any());
View Full Code Here
84
85
86
87
88
89
90
91
92
93
94
    ArgumentCaptor<byte[]> response = ArgumentCaptor.forClass(byte[].class);
    verify(callback, times(1)).onSuccess(response.capture());
    verify(callback, never()).onFailure((Throwable) any());

    StreamHandle handle =
      (StreamHandle) BlockTransferMessage.Decoder.fromByteArray(response.getValue());
    assertEquals(2, handle.numChunks);

    ArgumentCaptor<Iterator> stream = ArgumentCaptor.forClass(Iterator.class);
    verify(streamManager, times(1)).registerStream(stream.capture());
View Full Code Here
72
73
74
75
76
77
78
79
80
81
82
      for (String blockId : msg.blockIds) {
        blocks.add(blockManager.getBlockData(msg.appId, msg.execId, blockId));
      }
      long streamId = streamManager.registerStream(blocks.iterator());
      logger.trace("Registered streamId {} with {} buffers", streamId, msg.blockIds.length);
      callback.onSuccess(new StreamHandle(streamId, msg.blockIds.length).toByteArray());

    } else if (msgObj instanceof RegisterExecutor) {
      RegisterExecutor msg = (RegisterExecutor) msgObj;
      blockManager.registerExecutor(msg.appId, msg.execId, msg.executorInfo);
      callback.onSuccess(new byte[0]);
View Full Code Here
TOP

Related Classes of org.apache.spark.network.shuffle.protocol.StreamHandle

  • org.apache.spark.network.shuffle.ExternalShuffleBlockHandler
  • org.apache.spark.network.shuffle.ExternalShuffleBlockHandlerSuite
  • org.apache.spark.network.shuffle.OneForOneBlockFetcherSuite
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.