Package org.apache.giraph.worker

Examples of org.apache.giraph.worker.WorkerInfo

388
389
390
391
392
393
394
395
396
397
398
    } catch (InterruptedException e) {
      throw new IllegalStateException(
          "getWorkers: Got InterruptedStateException", e);
    }
    for (String workerInfoPath : workerInfoPathList) {
      WorkerInfo workerInfo = new WorkerInfo();
      WritableUtils.readFieldsFromZnode(
          getZkExt(), workerInfoPath, true, null, workerInfo);
      workerInfoList.add(workerInfo);
    }
    return workerInfoList;
View Full Code Here
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
    Collection<PartitionOwner> partitionOwners =
        masterGraphPartitioner.getCurrentPartitionOwners();
    String hostnameId =
        getHealthyHostnameIdFromPath(failedWorkerPath);
    for (PartitionOwner partitionOwner : partitionOwners) {
      WorkerInfo workerInfo = partitionOwner.getWorkerInfo();
      WorkerInfo previousWorkerInfo =
          partitionOwner.getPreviousWorkerInfo();
      if (workerInfo.getHostnameId().equals(hostnameId) ||
          ((previousWorkerInfo != null) &&
              previousWorkerInfo.getHostnameId().equals(hostnameId))) {
        LOG.warn("checkHealthyWorkerFailure: " +
            "at least one healthy worker went down " +
            "for superstep " + getSuperstep() + " - " +
            hostnameId + ", will try to restart from " +
            "checkpointed superstep " +
View Full Code Here
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
  }

  @Override
  public void readFields(DataInput input) throws IOException {
    partitionId = input.readInt();
    workerInfo = new WorkerInfo();
    workerInfo.readFields(input);
    boolean hasPreviousWorkerInfo = input.readBoolean();
    if (hasPreviousWorkerInfo) {
      previousWorkerInfo = new WorkerInfo();
      previousWorkerInfo.readFields(input);
    }
    boolean hasCheckpointFilePrefix = input.readBoolean();
    if (hasCheckpointFilePrefix) {
      checkpointFilesPrefix = input.readUTF();
View Full Code Here
58
59
60
61
62
63
64
65
66
67
    usingRangePartitioner =
        SimpleLongRangePartitionerFactory.class.isAssignableFrom(
            conf.getGraphPartitionerClass());
    int numWorkers = conf.getMaxWorkers();
    List<WorkerInfo> workerInfos = Collections.nCopies(numWorkers,
        new WorkerInfo());
    numPartitions = PartitionUtils.computePartitionCount(workerInfos,
        numWorkers, conf);
    partitionSize = numVertices / numPartitions;
  }
View Full Code Here
53
54
55
56
57
58
59
60
61
62
        PseudoRandomInputFormatConstants.LOCAL_EDGES_MIN_RATIO,
        PseudoRandomInputFormatConstants.LOCAL_EDGES_MIN_RATIO_DEFAULT);
    this.numVertices = numVertices;
    int numWorkers = conf.getMaxWorkers();
    List<WorkerInfo> workerInfos = Collections.nCopies(numWorkers,
        new WorkerInfo());
    numPartitions = PartitionUtils.computePartitionCount(workerInfos,
        numWorkers, conf);
    partitionSize = numVertices / numPartitions;
  }
View Full Code Here
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
  }

  @Override
  public boolean sendAggregatedValue(String aggregatorName,
      Writable aggregatedValue) throws IOException {
    WorkerInfo owner =
        AggregatorUtils.getOwner(aggregatorName,
            serviceWorker.getWorkerInfoList());
    if (isThisWorker(owner)) {
      return false;
    } else {
      int currentSize = sendAggregatedValueCache.addAggregator(
          owner.getTaskId(), aggregatorName, aggregatedValue);
      if (currentSize >= maxBytesPerAggregatorRequest) {
        flushAggregatorsToWorker(owner);
      }
      return true;
    }
View Full Code Here
74
75
76
77
78
79
80
81
82
83
84
85
86
  @Override
  public void sendAggregator(String aggregatorName,
      Class<? extends Aggregator> aggregatorClass,
      Writable aggregatedValue) throws IOException {
    WorkerInfo owner =
        AggregatorUtils.getOwner(aggregatorName, service.getWorkerInfoList());
    int currentSize = sendAggregatorCache.addAggregator(owner.getTaskId(),
        aggregatorName, aggregatorClass, aggregatedValue);
    if (currentSize >= maxBytesPerAggregatorRequest) {
      flushAggregatorsToWorker(owner);
    }
  }
View Full Code Here
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
    masterInfo.readFields(input);

    int workerInfosSize = input.readInt();
    workerInfos = Lists.newArrayListWithCapacity(workerInfosSize);
    for (int i = 0; i < workerInfosSize; i++) {
      WorkerInfo workerInfo = new WorkerInfo();
      workerInfo.readFields(input);
      workerInfos.add(workerInfo);
    }

    Map<Integer, WorkerInfo> workerInfoMap = getAsWorkerInfoMap(workerInfos);
    int additionalWorkerInfos = input.readInt();
    for (int i = 0; i < additionalWorkerInfos; i++) {
      WorkerInfo workerInfo = new WorkerInfo();
      workerInfo.readFields(input);
      workerInfoMap.put(workerInfo.getTaskId(), workerInfo);
    }

    int partitionOwnersSize = input.readInt();
    partitionOwners = Lists.newArrayListWithCapacity(partitionOwnersSize);
    for (int i = 0; i < partitionOwnersSize; i++) {
View Full Code Here
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
      // superstep mod availableWorkerInfos
      // Guaranteed to be different if the workers (and their order)
      // do not change
      long workerIndex = superstep % availableWorkerInfos.size();
      int i = 0;
      WorkerInfo chosenWorkerInfo = null;
      for (WorkerInfo workerInfo : availableWorkerInfos) {
        if (workerIndex == i) {
          chosenWorkerInfo = workerInfo;
        }
        ++i;
      }
      if (LOG.isInfoEnabled()) {
        LOG.info("generateChangedPartitionOwners: Chosen worker " +
                 "for superstep " + superstep + " is " +
                 chosenWorkerInfo);
      }

      List<PartitionOwner> partitionOwnerList = new ArrayList<PartitionOwner>();
      for (PartitionOwner partitionOwner :
        getCurrentPartitionOwners()) {
        WorkerInfo prevWorkerinfo =
          partitionOwner.getWorkerInfo().equals(chosenWorkerInfo) ?
            null : partitionOwner.getWorkerInfo();
        PartitionOwner tmpPartitionOwner =
          new BasicPartitionOwner(partitionOwner.getPartitionId(),
                                  chosenWorkerInfo,
View Full Code Here
161
162
163
164
165
166
167
168
169
170
171
  @Override
  public boolean sendMessageRequest(I destVertexId, M message) {
    PartitionOwner owner =
        serviceWorker.getVertexPartitionOwner(destVertexId);
    WorkerInfo workerInfo = owner.getWorkerInfo();
    final int partitionId = owner.getPartitionId();
    if (LOG.isTraceEnabled()) {
      LOG.trace("sendMessageRequest: Send bytes (" + message.toString() +
          ") to " + destVertexId + " on worker " + workerInfo);
    }
View Full Code Here
TOP

Related Classes of org.apache.giraph.worker.WorkerInfo

  • org.apache.giraph.comm.ConnectionTest
  • org.apache.giraph.comm.netty.NettyMasterClient
  • org.apache.giraph.comm.netty.NettyWorkerAggregatorRequestProcessor
  • org.apache.giraph.comm.netty.NettyWorkerClientRequestProcessor
  • org.apache.giraph.comm.RequestFailureTest
  • org.apache.giraph.comm.RequestTest
  • org.apache.giraph.comm.SaslConnectionTest
  • org.apache.giraph.comm.SendMessageCache
  • org.apache.giraph.comm.SendOneMessageToManyCache
  • org.apache.giraph.graph.AddressesAndPartitionsWritable
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.