Package org.apache.giraph.worker

Examples of org.apache.giraph.worker.WorkerInfo


  }

  @Override
  public void readFields(DataInput input) throws IOException {
    partitionId = input.readInt();
    workerInfo = new WorkerInfo();
    workerInfo.readFields(input);
    boolean hasPreviousWorkerInfo = input.readBoolean();
    if (hasPreviousWorkerInfo) {
      previousWorkerInfo = new WorkerInfo();
      previousWorkerInfo.readFields(input);
    }
    boolean hasCheckpointFilePrefix = input.readBoolean();
    if (hasCheckpointFilePrefix) {
      checkpointFilesPrefix = input.readUTF();
View Full Code Here


    } catch (InterruptedException e) {
      throw new IllegalStateException(
          "getWorkers: Got InterruptedStateException", e);
    }
    for (String workerInfoPath : workerInfoPathList) {
      WorkerInfo workerInfo = new WorkerInfo();
      WritableUtils.readFieldsFromZnode(
          getZkExt(), workerInfoPath, true, null, workerInfo);
      workerInfoList.add(workerInfo);
    }
    return workerInfoList;
View Full Code Here

      int mrTaskId = finalizedStream.readInt();

      DataInputStream metadataStream = fs.open(new Path(checkpointFile +
          "." + mrTaskId + CheckpointingUtils.CHECKPOINT_METADATA_POSTFIX));
      long partitions = metadataStream.readInt();
      WorkerInfo worker = workersMap.get(mrTaskId);
      for (long p = 0; p < partitions; ++p) {
        int partitionId = metadataStream.readInt();
        PartitionOwner partitionOwner = new BasicPartitionOwner(partitionId,
            worker);
        partitionOwners.add(partitionOwner);
View Full Code Here

    Collection<PartitionOwner> partitionOwners =
        masterGraphPartitioner.getCurrentPartitionOwners();
    String hostnameId =
        getHealthyHostnameIdFromPath(failedWorkerPath);
    for (PartitionOwner partitionOwner : partitionOwners) {
      WorkerInfo workerInfo = partitionOwner.getWorkerInfo();
      WorkerInfo previousWorkerInfo =
          partitionOwner.getPreviousWorkerInfo();
      if (workerInfo.getHostnameId().equals(hostnameId) ||
          ((previousWorkerInfo != null) &&
              previousWorkerInfo.getHostnameId().equals(hostnameId))) {
        LOG.warn("checkHealthyWorkerFailure: " +
            "at least one healthy worker went down " +
            "for superstep " + getSuperstep() + " - " +
            hostnameId + ", will try to restart from " +
            "checkpointed superstep " +
View Full Code Here

      // superstep mod availableWorkerInfos
      // Guaranteed to be different if the workers (and their order)
      // do not change
      long workerIndex = superstep % availableWorkerInfos.size();
      int i = 0;
      WorkerInfo chosenWorkerInfo = null;
      for (WorkerInfo workerInfo : availableWorkerInfos) {
        if (workerIndex == i) {
          chosenWorkerInfo = workerInfo;
        }
        ++i;
      }
      if (LOG.isInfoEnabled()) {
        LOG.info("generateChangedPartitionOwners: Chosen worker " +
                 "for superstep " + superstep + " is " +
                 chosenWorkerInfo);
      }

      List<PartitionOwner> partitionOwnerList = new ArrayList<PartitionOwner>();
      for (PartitionOwner partitionOwner :
        getCurrentPartitionOwners()) {
        WorkerInfo prevWorkerinfo =
          partitionOwner.getWorkerInfo().equals(chosenWorkerInfo) ?
            null : partitionOwner.getWorkerInfo();
        PartitionOwner tmpPartitionOwner =
          new BasicPartitionOwner(partitionOwner.getPartitionId(),
                                  chosenWorkerInfo,
View Full Code Here

  private void checkSendingTwoRequests() throws IOException {
    // Start the service
    serverData = MockUtils.createNewServerData(conf, context);
    serverData.prepareSuperstep();
    WorkerInfo workerInfo = new WorkerInfo();
    server = new NettyServer(conf,
        new WorkerRequestServerHandler.Factory(serverData), workerInfo,
            context, new MockExceptionHandler());
    server.start();
    workerInfo.setInetSocketAddress(server.getMyAddress());
    client = new NettyClient(context, conf, new WorkerInfo(),
        new MockExceptionHandler());
    client.connectAllAddresses(
        Lists.<WorkerInfo>newArrayList(workerInfo));

    // Send the request 2x, but should only be processed once
    WritableRequest request1 = getRequest();
    WritableRequest request2 = getRequest();
    client.sendWritableRequest(workerInfo.getTaskId(), request1);
    client.sendWritableRequest(workerInfo.getTaskId(), request2);
    client.waitAllRequests();

    // Stop the service
    client.stop();
    server.stop();
View Full Code Here

    when(context.getConfiguration()).thenReturn(conf);

    // Start the service
    serverData = MockUtils.createNewServerData(conf, context);
    serverData.prepareSuperstep();
    workerInfo = new WorkerInfo();
    server = new NettyServer(conf,
        new WorkerRequestServerHandler.Factory(serverData), workerInfo,
            context, new MockExceptionHandler());
    server.start();
    workerInfo.setInetSocketAddress(server.getMyAddress());
    client = new NettyClient(context, conf, new WorkerInfo(),
        new MockExceptionHandler());
    client.connectAllAddresses(
        Lists.<WorkerInfo>newArrayList(workerInfo));
  }
View Full Code Here

    Context context = mock(Context.class);
    when(context.getConfiguration()).thenReturn(conf);

    ServerData<IntWritable, IntWritable, IntWritable> serverData =
        MockUtils.createNewServerData(conf, context);
    WorkerInfo workerInfo = new WorkerInfo();
    NettyServer server =
        new NettyServer(conf,
            new WorkerRequestServerHandler.Factory(serverData), workerInfo,
            context, new MockExceptionHandler());
    server.start();
    workerInfo.setInetSocketAddress(server.getMyAddress());

    NettyClient client = new NettyClient(context, conf, new WorkerInfo(),
        new MockExceptionHandler());
    client.connectAllAddresses(
        Lists.<WorkerInfo>newArrayList(workerInfo));

    client.stop();
View Full Code Here

    ServerData<IntWritable, IntWritable, IntWritable> serverData =
        MockUtils.createNewServerData(conf, context);
   RequestServerHandler.Factory requestServerHandlerFactory =
       new WorkerRequestServerHandler.Factory(serverData);

    WorkerInfo workerInfo1 = new WorkerInfo();
    workerInfo1.setTaskId(1);
    NettyServer server1 =
        new NettyServer(conf, requestServerHandlerFactory, workerInfo1,
            context, new MockExceptionHandler());
    server1.start();
    workerInfo1.setInetSocketAddress(server1.getMyAddress());

    WorkerInfo workerInfo2 = new WorkerInfo();
    workerInfo1.setTaskId(2);
    NettyServer server2 =
        new NettyServer(conf, requestServerHandlerFactory, workerInfo2,
            context, new MockExceptionHandler());
    server2.start();
    workerInfo2.setInetSocketAddress(server2.getMyAddress());

    WorkerInfo workerInfo3 = new WorkerInfo();
    workerInfo1.setTaskId(3);
    NettyServer server3 =
        new NettyServer(conf, requestServerHandlerFactory, workerInfo3,
            context, new MockExceptionHandler());
    server3.start();
    workerInfo3.setInetSocketAddress(server3.getMyAddress());

    NettyClient client = new NettyClient(context, conf, new WorkerInfo(),
        new MockExceptionHandler());
    List<WorkerInfo> addresses = Lists.<WorkerInfo>newArrayList(workerInfo1,
        workerInfo2, workerInfo3);
    client.connectAllAddresses(addresses);
View Full Code Here

    Context context = mock(Context.class);
    when(context.getConfiguration()).thenReturn(conf);

    ServerData<IntWritable, IntWritable, IntWritable> serverData =
        MockUtils.createNewServerData(conf, context);
    WorkerInfo workerInfo = new WorkerInfo();
    NettyServer server = new NettyServer(conf,
        new WorkerRequestServerHandler.Factory(serverData), workerInfo,
            context, new MockExceptionHandler());
    server.start();
    workerInfo.setInetSocketAddress(server.getMyAddress());

    List<WorkerInfo> addresses = Lists.<WorkerInfo>newArrayList(workerInfo);
    NettyClient client1 = new NettyClient(context, conf, new WorkerInfo(),
        new MockExceptionHandler());
    client1.connectAllAddresses(addresses);
    NettyClient client2 = new NettyClient(context, conf, new WorkerInfo(),
        new MockExceptionHandler());
    client2.connectAllAddresses(addresses);
    NettyClient client3 = new NettyClient(context, conf, new WorkerInfo(),
        new MockExceptionHandler());
    client3.connectAllAddresses(addresses);

    client1.stop();
    client2.stop();
View Full Code Here

TOP

Related Classes of org.apache.giraph.worker.WorkerInfo

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.