Package org.apache.spark.network.client

Examples of org.apache.spark.network.client.TransportClient


   * that they were inserted in.
   *
   * If a block's buffer is "null", an exception will be thrown instead.
   */
  private BlockFetchingListener fetchBlocks(final LinkedHashMap<String, ManagedBuffer> blocks) {
    TransportClient client = mock(TransportClient.class);
    BlockFetchingListener listener = mock(BlockFetchingListener.class);
    final String[] blockIds = blocks.keySet().toArray(new String[blocks.size()]);
    OneForOneBlockFetcher fetcher =
      new OneForOneBlockFetcher(client, "app-id", "exec-id", blockIds, listener);

View Full Code Here


  public void testGoodClient() throws IOException {
    clientFactory = context.createClientFactory(
      Lists.<TransportClientBootstrap>newArrayList(
        new SaslClientBootstrap(conf, "app-id", new TestSecretKeyHolder("good-key"))));

    TransportClient client = clientFactory.createClient(TestUtils.getLocalHost(), server.getPort());
    String msg = "Hello, World!";
    byte[] resp = client.sendRpcSync(msg.getBytes(), 1000);
    assertEquals(msg, new String(resp)); // our rpc handler should just return the given msg
  }
View Full Code Here

  @Test
  public void testNoSaslClient() throws IOException {
    clientFactory = context.createClientFactory(
      Lists.<TransportClientBootstrap>newArrayList());

    TransportClient client = clientFactory.createClient(TestUtils.getLocalHost(), server.getPort());
    try {
      client.sendRpcSync(new byte[13], 1000);
      fail("Should have failed");
    } catch (Exception e) {
      assertTrue(e.getMessage(), e.getMessage().contains("Expected SaslMessage"));
    }

    try {
      // Guessing the right tag byte doesn't magically get you in...
      client.sendRpcSync(new byte[] { (byte) 0xEA }, 1000);
      fail("Should have failed");
    } catch (Exception e) {
      assertTrue(e.getMessage(), e.getMessage().contains("java.lang.IndexOutOfBoundsException"));
    }
  }
View Full Code Here

  }

  @Test
  public void createAndReuseBlockClients() throws IOException {
    TransportClientFactory factory = context.createClientFactory();
    TransportClient c1 = factory.createClient(TestUtils.getLocalHost(), server1.getPort());
    TransportClient c2 = factory.createClient(TestUtils.getLocalHost(), server1.getPort());
    TransportClient c3 = factory.createClient(TestUtils.getLocalHost(), server2.getPort());
    assertTrue(c1.isActive());
    assertTrue(c3.isActive());
    assertTrue(c1 == c2);
    assertTrue(c1 != c3);
    factory.close();
  }
View Full Code Here

  }

  @Test
  public void neverReturnInactiveClients() throws IOException, InterruptedException {
    TransportClientFactory factory = context.createClientFactory();
    TransportClient c1 = factory.createClient(TestUtils.getLocalHost(), server1.getPort());
    c1.close();

    long start = System.currentTimeMillis();
    while (c1.isActive() && (System.currentTimeMillis() - start) < 3000) {
      Thread.sleep(10);
    }
    assertFalse(c1.isActive());

    TransportClient c2 = factory.createClient(TestUtils.getLocalHost(), server1.getPort());
    assertFalse(c1 == c2);
    assertTrue(c2.isActive());
    factory.close();
  }
View Full Code Here

  }

  @Test
  public void closeBlockClientsWithFactory() throws IOException {
    TransportClientFactory factory = context.createClientFactory();
    TransportClient c1 = factory.createClient(TestUtils.getLocalHost(), server1.getPort());
    TransportClient c2 = factory.createClient(TestUtils.getLocalHost(), server2.getPort());
    assertTrue(c1.isActive());
    assertTrue(c2.isActive());
    factory.close();
    assertFalse(c1.isActive());
    assertFalse(c2.isActive());
  }
View Full Code Here

      RetryingBlockFetcher.BlockFetchStarter blockFetchStarter =
        new RetryingBlockFetcher.BlockFetchStarter() {
          @Override
          public void createAndStart(String[] blockIds, BlockFetchingListener listener)
              throws IOException {
            TransportClient client = clientFactory.createClient(host, port);
            new OneForOneBlockFetcher(client, appId, execId, blockIds, listener).start();
          }
        };

      int maxRetries = conf.maxIORetries();
View Full Code Here

      String host,
      int port,
      String execId,
      ExecutorShuffleInfo executorInfo) throws IOException {
    assert appId != null : "Called before init()";
    TransportClient client = clientFactory.createClient(host, port);
    byte[] registerMessage = new RegisterExecutor(appId, execId, executorInfo).toByteArray();
    client.sendRpcSync(registerMessage, 5000 /* timeoutMs */);
  }
View Full Code Here

      }
    }
  }

  private FetchResult fetchChunks(List<Integer> chunkIndices) throws Exception {
    TransportClient client = clientFactory.createClient(TestUtils.getLocalHost(), server.getPort());
    final Semaphore sem = new Semaphore(0);

    final FetchResult res = new FetchResult();
    res.successChunks = Collections.synchronizedSet(new HashSet<Integer>());
    res.failedChunks = Collections.synchronizedSet(new HashSet<Integer>());
    res.buffers = Collections.synchronizedList(new LinkedList<ManagedBuffer>());

    ChunkReceivedCallback callback = new ChunkReceivedCallback() {
      @Override
      public void onSuccess(int chunkIndex, ManagedBuffer buffer) {
        buffer.retain();
        res.successChunks.add(chunkIndex);
        res.buffers.add(buffer);
        sem.release();
      }

      @Override
      public void onFailure(int chunkIndex, Throwable e) {
        res.failedChunks.add(chunkIndex);
        sem.release();
      }
    };

    for (int chunkIndex : chunkIndices) {
      client.fetchChunk(STREAM_ID, chunkIndex, callback);
    }
    if (!sem.tryAcquire(chunkIndices.size(), 5, TimeUnit.SECONDS)) {
      fail("Timeout getting response from the server");
    }
    client.close();
    return res;
  }
View Full Code Here

    public Set<String> successMessages;
    public Set<String> errorMessages;
  }

  private RpcResult sendRPC(String ... commands) throws Exception {
    TransportClient client = clientFactory.createClient(TestUtils.getLocalHost(), server.getPort());
    final Semaphore sem = new Semaphore(0);

    final RpcResult res = new RpcResult();
    res.successMessages = Collections.synchronizedSet(new HashSet<String>());
    res.errorMessages = Collections.synchronizedSet(new HashSet<String>());

    RpcResponseCallback callback = new RpcResponseCallback() {
      @Override
      public void onSuccess(byte[] message) {
        res.successMessages.add(new String(message, Charsets.UTF_8));
        sem.release();
      }

      @Override
      public void onFailure(Throwable e) {
        res.errorMessages.add(e.getMessage());
        sem.release();
      }
    };

    for (String command : commands) {
      client.sendRpc(command.getBytes(Charsets.UTF_8), callback);
    }

    if (!sem.tryAcquire(commands.length, 5, TimeUnit.SECONDS)) {
      fail("Timeout getting response from the server");
    }
    client.close();
    return res;
  }
View Full Code Here

TOP

Related Classes of org.apache.spark.network.client.TransportClient

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.