Package org.apache.giraph.comm.aggregators

Examples of org.apache.giraph.comm.aggregators.AllAggregatorServerData

52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
  }

  @Override
  public void doRequest(ServerData serverData) {
    DataInput input = getDataInput();
    AllAggregatorServerData aggregatorData = serverData.getAllAggregatorData();
    try {
      int numAggregators = input.readInt();
      for (int i = 0; i < numAggregators; i++) {
        String aggregatorName = input.readUTF();
        String aggregatorClassName = input.readUTF();
        if (aggregatorName.equals(AggregatorUtils.SPECIAL_COUNT_AGGREGATOR)) {
          LongWritable count = new LongWritable(0);
          count.readFields(input);
          aggregatorData.receivedRequestCountFromMaster(count.get(),
              getSenderTaskId());
        } else {
          Class<Aggregator<Writable>> aggregatorClass =
              AggregatorUtils.getAggregatorClass(aggregatorClassName);
          aggregatorData.registerAggregatorClass(aggregatorName,
              aggregatorClass);
          Writable aggregatorValue =
              aggregatorData.createAggregatorInitialValue(aggregatorName);
          aggregatorValue.readFields(input);
          aggregatorData.setAggregatorValue(aggregatorName, aggregatorValue);
          serverData.getOwnerAggregatorData().registerAggregator(
              aggregatorName, aggregatorClass);
        }
      }
    } catch (IOException e) {
      throw new IllegalStateException("doRequest: " +
          "IOException occurred while processing request", e);
    }
    aggregatorData.receivedRequestFromMaster(getData());
  }
View Full Code Here
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
  }

  @Override
  public void doRequest(ServerData serverData) {
    DataInput input = getDataInput();
    AllAggregatorServerData aggregatorData = serverData.getAllAggregatorData();
    try {
      int numAggregators = input.readInt();
      for (int i = 0; i < numAggregators; i++) {
        String aggregatorName = input.readUTF();
        String aggregatorClassName = input.readUTF();
        if (aggregatorName.equals(AggregatorUtils.SPECIAL_COUNT_AGGREGATOR)) {
          LongWritable count = new LongWritable(0);
          count.readFields(input);
          aggregatorData.receivedRequestCountFromWorker(count.get(),
              getSenderTaskId());
        } else {
          Class<Aggregator<Writable>> aggregatorClass =
              AggregatorUtils.getAggregatorClass(aggregatorClassName);
          aggregatorData.registerAggregatorClass(aggregatorName,
              aggregatorClass);
          Writable aggregatorValue =
              aggregatorData.createAggregatorInitialValue(aggregatorName);
          aggregatorValue.readFields(input);
          aggregatorData.setAggregatorValue(aggregatorName, aggregatorValue);
        }
      }
    } catch (IOException e) {
      throw new IllegalStateException("doRequest: " +
          "IOException occurred while processing request", e);
    }
    aggregatorData.receivedRequestFromWorker();
  }
View Full Code Here
105
106
107
108
109
110
111
      partitionStore =
          new SimplePartitionStore<I, V, E, M>(conf, context);
    }
    edgeStore = new EdgeStore<I, V, E, M>(service, conf, context);
    ownerAggregatorData = new OwnerAggregatorServerData(context, conf);
    allAggregatorData = new AllAggregatorServerData(context, conf);
  }
View Full Code Here
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
  public void prepareSuperstep(
      WorkerAggregatorRequestProcessor requestProcessor) {
    if (LOG.isDebugEnabled()) {
      LOG.debug("prepareSuperstep: Start preparing aggregators");
    }
    AllAggregatorServerData allAggregatorData =
        serviceWorker.getServerData().getAllAggregatorData();
    // Wait for my aggregators
    Iterable<byte[]> dataToDistribute =
        allAggregatorData.getDataFromMasterWhenReady(
            serviceWorker.getMasterInfo());
    try {
      // Distribute my aggregators
      requestProcessor.distributeAggregators(dataToDistribute);
    } catch (IOException e) {
      throw new IllegalStateException("prepareSuperstep: " +
          "IOException occurred while trying to distribute aggregators", e);
    }
    // Wait for all other aggregators and store them
    allAggregatorData.fillNextSuperstepMapsWhenReady(
        getOtherWorkerIdsSet(), previousAggregatedValueMap,
        currentAggregatorMap);
    allAggregatorData.reset();
    if (LOG.isDebugEnabled()) {
      LOG.debug("prepareSuperstep: Aggregators prepared");
    }
  }
View Full Code Here
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
    reducerMap.clear();

    if (LOG.isDebugEnabled()) {
      LOG.debug("prepareSuperstep: Start preparing aggregators");
    }
    AllAggregatorServerData allGlobalCommData =
        serviceWorker.getServerData().getAllAggregatorData();
    // Wait for my aggregators
    Iterable<byte[]> dataToDistribute =
        allGlobalCommData.getDataFromMasterWhenReady(
            serviceWorker.getMasterInfo());
    try {
      // Distribute my aggregators
      requestProcessor.distributeReducedValues(dataToDistribute);
    } catch (IOException e) {
      throw new IllegalStateException("prepareSuperstep: " +
          "IOException occurred while trying to distribute aggregators", e);
    }
    // Wait for all other aggregators and store them
    allGlobalCommData.fillNextSuperstepMapsWhenReady(
        getOtherWorkerIdsSet(), broadcastedMap,
        reducerMap);
    if (LOG.isDebugEnabled()) {
      LOG.debug("prepareSuperstep: Aggregators prepared");
    }
View Full Code Here
122
123
124
125
126
127
128
    }
    EdgeStoreFactory<I, V, E> edgeStoreFactory = conf.createEdgeStoreFactory();
    edgeStoreFactory.initialize(service, conf, context);
    edgeStore = edgeStoreFactory.newStore();
    ownerAggregatorData = new OwnerAggregatorServerData(context);
    allAggregatorData = new AllAggregatorServerData(context, conf);
  }
View Full Code Here
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
  }

  @Override
  public void doRequest(ServerData serverData) {
    DataInput input = getDataInput();
    AllAggregatorServerData aggregatorData = serverData.getAllAggregatorData();
    try {
      int num = input.readInt();
      for (int i = 0; i < num; i++) {
        String name = input.readUTF();
        GlobalCommType type = GlobalCommType.values()[input.readByte()];
        Writable value = WritableUtils.readWritableObject(input, conf);
        if (type == GlobalCommType.SPECIAL_COUNT) {
          aggregatorData.receivedRequestCountFromWorker(
              ((LongWritable) value).get(),
              getSenderTaskId());
        } else {
          aggregatorData.receiveValueFromMaster(name, type, value);
        }
      }
    } catch (IOException e) {
      throw new IllegalStateException("doRequest: " +
          "IOException occurred while processing request", e);
    }
    aggregatorData.receivedRequestFromWorker();
  }
View Full Code Here
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
  public void doRequest(ServerData serverData) {
    UnsafeByteArrayOutputStream reusedOut = new UnsafeByteArrayOutputStream();
    UnsafeReusableByteArrayInput reusedIn = new UnsafeReusableByteArrayInput();

    DataInput input = getDataInput();
    AllAggregatorServerData aggregatorData = serverData.getAllAggregatorData();
    try {
      int num = input.readInt();
      for (int i = 0; i < num; i++) {
        String name = input.readUTF();
        GlobalCommType type = GlobalCommType.values()[input.readByte()];
        Writable value = WritableUtils.readWritableObject(input, conf);
        if (type == GlobalCommType.SPECIAL_COUNT) {
          aggregatorData.receivedRequestCountFromMaster(
              ((LongWritable) value).get(),
              getSenderTaskId());
        } else {
          aggregatorData.receiveValueFromMaster(name, type, value);

          if (type == GlobalCommType.REDUCE_OPERATIONS) {
            ReduceOperation<Object, Writable> reduceOpCopy =
                (ReduceOperation<Object, Writable>)
                WritableUtils.createCopy(reusedOut, reusedIn, value, conf);

            serverData.getOwnerAggregatorData().registerReducer(
                name, reduceOpCopy);
          }
        }
      }
    } catch (IOException e) {
      throw new IllegalStateException("doRequest: " +
          "IOException occurred while processing request", e);
    }
    aggregatorData.receivedRequestFromMaster(getData());
  }
View Full Code Here
TOP

Related Classes of org.apache.giraph.comm.aggregators.AllAggregatorServerData

  • org.apache.giraph.comm.requests.SendAggregatorsToOwnerRequest
  • org.apache.giraph.comm.requests.SendAggregatorsToWorkerRequest
  • org.apache.giraph.comm.ServerData
  • org.apache.giraph.utils.TaskIdsPermitsBarrier
  • org.apache.giraph.worker.WorkerAggregatorHandler
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.