Examples of org.apache.giraph.comm.aggregators.OwnerAggregatorServerData

Package org.apache.giraph.comm.aggregators

Examples of org.apache.giraph.comm.aggregators.OwnerAggregatorServerData

org.apache.giraph.comm.aggregators.OwnerAggregatorServerData
Class for holding aggregators which current worker owns, and aggregating partial aggregator values from workers. Protocol: 1. Before the beginning of superstep, worker receives its aggregators from master, and these aggregators will be registered to this class. Multiple registrations can be called concurrently. 2. During the superstep, whenever a worker finishes computation, it will send partial aggregated values to worker owner. This class is used to help deserialize the arriving aggregator values, and aggregate the values at the destination owner worker; these can happen concurrently. (we know step 1. is finished before anything from step 2. happens because other workers can't start computation before they receive aggregators which this worker owns) 3. This class also tracks the number of partial aggregator requests which worker received. In the end of superstep, getMyAggregatorValuesWhenReady will be called to ensure everything was received and get the values which need to be sent to master. Because of this counting, in step 2. even if worker owns no aggregators, it will still send a message without aggregator data. 4. In the end we reset to prepare for the next superstep.

  }


  @Override
  public void doRequest(ServerData serverData) {
    DataInput input = getDataInput();
    OwnerAggregatorServerData aggregatorData =
        serverData.getOwnerAggregatorData();
    try {
      int numAggregators = input.readInt();
      for (int i = 0; i < numAggregators; i++) {
        String aggregatorName = input.readUTF();
        if (aggregatorName.equals(
            AggregatorUtils.SPECIAL_COUNT_AGGREGATOR)) {
          LongWritable count = new LongWritable(0);
          count.readFields(input);
          aggregatorData.receivedRequestCountFromWorker(count.get(),
              getSenderTaskId());
        } else {
          Writable aggregatedValue =
              aggregatorData.createAggregatorInitialValue(aggregatorName);
          aggregatedValue.readFields(input);
          aggregatorData.aggregate(aggregatorName, aggregatedValue);
        }
      }
    } catch (IOException e) {
      throw new IllegalStateException("doRequest: " +
          "IOException occurred while processing request", e);
    }
    aggregatorData.receivedRequestFromWorker();
  }

View Full Code Here

    } else {
      partitionStore =
          new SimplePartitionStore<I, V, E, M>(conf, context);
    }
    edgeStore = new EdgeStore<I, V, E, M>(service, conf, context);
    ownerAggregatorData = new OwnerAggregatorServerData(context, conf);
    allAggregatorData = new AllAggregatorServerData(context, conf);
  }

View Full Code Here

    if (LOG.isInfoEnabled()) {
      LOG.info("finishSuperstep: Start gathering aggregators, " +
          "workers will send their aggregated values " +
          "once they are done with superstep computation");
    }
    OwnerAggregatorServerData ownerAggregatorData =
        serviceWorker.getServerData().getOwnerAggregatorData();
    // First send partial aggregated values to their owners and determine
    // which aggregators belong to this worker
    for (Map.Entry<String, Aggregator<Writable>> entry :
        currentAggregatorMap.entrySet()) {
      try {
        boolean sent = requestProcessor.sendAggregatedValue(entry.getKey(),
            entry.getValue().getAggregatedValue());
        if (!sent) {
          // If it's my aggregator, add it directly
          ownerAggregatorData.aggregate(entry.getKey(),
              entry.getValue().getAggregatedValue());
        }
      } catch (IOException e) {
        throw new IllegalStateException("finishSuperstep: " +
            "IOException occurred while sending aggregator " +
            entry.getKey() + " to its owner", e);
      }
      progressable.progress();
    }
    try {
      // Flush
      requestProcessor.flush();
    } catch (IOException e) {
      throw new IllegalStateException("finishSuperstep: " +
          "IOException occurred while sending aggregators to owners", e);
    }


    // Wait to receive partial aggregated values from all other workers
    Iterable<Map.Entry<String, Writable>> myAggregators =
        ownerAggregatorData.getMyAggregatorValuesWhenReady(
            getOtherWorkerIdsSet());


    // Send final aggregated values to master
    AggregatedValueOutputStream aggregatorOutput =
        new AggregatedValueOutputStream();
    for (Map.Entry<String, Writable> entry : myAggregators) {
      try {
        int currentSize = aggregatorOutput.addAggregator(entry.getKey(),
            entry.getValue());
        if (currentSize > maxBytesPerAggregatorRequest) {
          requestProcessor.sendAggregatedValuesToMaster(
              aggregatorOutput.flush());
        }
        progressable.progress();
      } catch (IOException e) {
        throw new IllegalStateException("finishSuperstep: " +
            "IOException occurred while writing aggregator " +
            entry.getKey(), e);
      }
    }
    try {
      requestProcessor.sendAggregatedValuesToMaster(aggregatorOutput.flush());
    } catch (IOException e) {
      throw new IllegalStateException("finishSuperstep: " +
          "IOException occured while sending aggregators to master", e);
    }
    // Wait for master to receive aggregated values before proceeding
    serviceWorker.getWorkerClient().waitAllRequests();


    ownerAggregatorData.reset();
    if (LOG.isDebugEnabled()) {
      LOG.debug("finishSuperstep: Aggregators finished");
    }
  }

View Full Code Here

    if (LOG.isInfoEnabled()) {
      LOG.info("finishSuperstep: Start gathering aggregators, " +
          "workers will send their aggregated values " +
          "once they are done with superstep computation");
    }
    OwnerAggregatorServerData ownerGlobalCommData =
        serviceWorker.getServerData().getOwnerAggregatorData();
    // First send partial aggregated values to their owners and determine
    // which aggregators belong to this worker
    for (Map.Entry<String, Reducer<Object, Writable>> entry :
        reducerMap.entrySet()) {
      try {
        boolean sent = requestProcessor.sendReducedValue(entry.getKey(),
            entry.getValue().getCurrentValue());
        if (!sent) {
          // If it's my aggregator, add it directly
          ownerGlobalCommData.reduce(entry.getKey(),
              entry.getValue().getCurrentValue());
        }
      } catch (IOException e) {
        throw new IllegalStateException("finishSuperstep: " +
            "IOException occurred while sending aggregator " +
            entry.getKey() + " to its owner", e);
      }
      progressable.progress();
    }
    try {
      // Flush
      requestProcessor.flush();
    } catch (IOException e) {
      throw new IllegalStateException("finishSuperstep: " +
          "IOException occurred while sending aggregators to owners", e);
    }


    // Wait to receive partial aggregated values from all other workers
    Iterable<Map.Entry<String, Writable>> myReducedValues =
        ownerGlobalCommData.getMyReducedValuesWhenReady(
            getOtherWorkerIdsSet());


    // Send final aggregated values to master
    GlobalCommValueOutputStream globalOutput =
        new GlobalCommValueOutputStream(false);
    for (Map.Entry<String, Writable> entry : myReducedValues) {
      try {
        int currentSize = globalOutput.addValue(entry.getKey(),
            GlobalCommType.REDUCED_VALUE,
            entry.getValue());
        if (currentSize > maxBytesPerAggregatorRequest) {
          requestProcessor.sendReducedValuesToMaster(
              globalOutput.flush());
        }
        progressable.progress();
      } catch (IOException e) {
        throw new IllegalStateException("finishSuperstep: " +
            "IOException occurred while writing aggregator " +
            entry.getKey(), e);
      }
    }
    try {
      requestProcessor.sendReducedValuesToMaster(globalOutput.flush());
    } catch (IOException e) {
      throw new IllegalStateException("finishSuperstep: " +
          "IOException occured while sending aggregators to master", e);
    }
    // Wait for master to receive aggregated values before proceeding
    serviceWorker.getWorkerClient().waitAllRequests();


    ownerGlobalCommData.reset();
    if (LOG.isDebugEnabled()) {
      LOG.debug("finishSuperstep: Aggregators finished");
    }
  }

View Full Code Here

          new SimplePartitionStore<I, V, E>(conf, context);
    }
    EdgeStoreFactory<I, V, E> edgeStoreFactory = conf.createEdgeStoreFactory();
    edgeStoreFactory.initialize(service, conf, context);
    edgeStore = edgeStoreFactory.newStore();
    ownerAggregatorData = new OwnerAggregatorServerData(context);
    allAggregatorData = new AllAggregatorServerData(context, conf);
  }

View Full Code Here

  }


  @Override
  public void doRequest(ServerData serverData) {
    DataInput input = getDataInput();
    OwnerAggregatorServerData aggregatorData =
        serverData.getOwnerAggregatorData();
    try {
      int num = input.readInt();
      for (int i = 0; i < num; i++) {
        String name = input.readUTF();
        GlobalCommType type = GlobalCommType.values()[input.readByte()];
        if (type == GlobalCommType.SPECIAL_COUNT) {
          LongWritable value = new LongWritable();
          value.readFields(input);
          aggregatorData.receivedRequestCountFromWorker(
              value.get(),
              getSenderTaskId());
        } else if (type == GlobalCommType.REDUCED_VALUE) {
          Writable value = aggregatorData.createInitialValue(name);
          value.readFields(input);
          aggregatorData.reduce(name, value);
        } else {
          throw new IllegalStateException(
              "SendWorkerAggregatorsRequest received " + type);
        }
      }
    } catch (IOException e) {
      throw new IllegalStateException("doRequest: " +
          "IOException occurred while processing request", e);
    }
    aggregatorData.receivedRequestFromWorker();
  }

View Full Code Here

TOP

Related Classes of org.apache.giraph.comm.aggregators.OwnerAggregatorServerData

org.apache.giraph.comm.requests.SendWorkerAggregatorsRequest

org.apache.giraph.comm.ServerData

org.apache.giraph.utils.TaskIdsPermitsBarrier

org.apache.giraph.worker.WorkerAggregatorHandler

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.