Examples of org.apache.giraph.conf.ImmutableClassesGiraphConfiguration

org.apache.giraph.conf.ImmutableClassesGiraphConfiguration
The classes set here are immutable, the remaining configuration is mutable. Classes are immutable and final to provide the best performance for instantiation. Everything is thread-safe. @param < I> Vertex id @param < V> Vertex data @param < E> Edge data @param < M> Message data

   *         valid input splits.
   */
  private int createInputSplits(GiraphInputFormat inputFormat,
                                InputSplitPaths inputSplitPaths,
                                String inputSplitType) {
    ImmutableClassesGiraphConfiguration conf = getConfiguration();
    String logPrefix = "create" + inputSplitType + "InputSplits";
    // Only the 'master' should be doing this.  Wait until the number of
    // processes that have reported health exceeds the minimum percentage.
    // If the minimum percentage is not met, fail the job.  Otherwise
    // generate the input splits
    String inputSplitsPath = inputSplitPaths.getPath();
    try {
      if (getZkExt().exists(inputSplitsPath, false) != null) {
        LOG.info(inputSplitsPath + " already exists, no need to create");
        return Integer.parseInt(
            new String(getZkExt().getData(inputSplitsPath, false, null)));
      }
    } catch (KeeperException.NoNodeException e) {
      if (LOG.isInfoEnabled()) {
        LOG.info(logPrefix + ": Need to create the input splits at " +
            inputSplitsPath);
      }
    } catch (KeeperException e) {
      throw new IllegalStateException(logPrefix + ": KeeperException", e);
    } catch (InterruptedException e) {
      throw new IllegalStateException(logPrefix + ": InterruptedException", e);
    }


    // When creating znodes, in case the master has already run, resume
    // where it left off.
    List<WorkerInfo> healthyWorkerInfoList = checkWorkers();
    if (healthyWorkerInfoList == null) {
      setJobStateFailed("Not enough healthy workers to create input splits");
      return -1;
    }


    // Create at least as many splits as the total number of input threads.
    int minSplitCountHint = healthyWorkerInfoList.size() *
        conf.getNumInputSplitsThreads();


    // Note that the input splits may only be a sample if
    // INPUT_SPLIT_SAMPLE_PERCENT is set to something other than 100
    List<InputSplit> splitList = generateInputSplits(inputFormat,
        minSplitCountHint, inputSplitType);


    if (splitList.isEmpty()) {
      LOG.fatal(logPrefix + ": Failing job due to 0 input splits, " +
          "check input of " + inputFormat.getClass().getName() + "!");
      getContext().setStatus("Failing job due to 0 input splits, " +
          "check input of " + inputFormat.getClass().getName() + "!");
      setJobStateFailed("0 input splits");
    }
    if (minSplitCountHint > splitList.size()) {
      LOG.warn(logPrefix + ": Number of inputSplits=" +
          splitList.size() + " < " +
          minSplitCountHint +
          "=total number of input threads, " +
          "some threads will be not used");
    }


    // Write input splits to zookeeper in parallel
    int inputSplitThreadCount = conf.getInt(NUM_MASTER_ZK_INPUT_SPLIT_THREADS,
        DEFAULT_INPUT_SPLIT_THREAD_COUNT);
    if (LOG.isInfoEnabled()) {
      LOG.info(logPrefix + ": Starting to write input split data " +
          "to zookeeper with " + inputSplitThreadCount + " threads");
    }

View Full Code Here

   *
   * @param superstep Superstep to aggregate on
   * @return Global statistics aggregated on all worker statistics
   */
  private GlobalStats aggregateWorkerStats(long superstep) {
    ImmutableClassesGiraphConfiguration conf = getConfiguration();


    Class<? extends PartitionStats> partitionStatsClass =
        masterGraphPartitioner.createPartitionStats().getClass();
    GlobalStats globalStats = new GlobalStats();
    // Get the stats from the all the worker selected nodes
    String workerFinishedPath =
        getWorkerFinishedPath(getApplicationAttempt(), superstep);
    List<String> workerFinishedPathList = null;
    try {
      workerFinishedPathList =
          getZkExt().getChildrenExt(
              workerFinishedPath, false, false, true);
    } catch (KeeperException e) {
      throw new IllegalStateException(
          "aggregateWorkerStats: KeeperException", e);
    } catch (InterruptedException e) {
      throw new IllegalStateException(
          "aggregateWorkerStats: InterruptedException", e);
    }


    AggregatedMetrics aggregatedMetrics = new AggregatedMetrics();


    allPartitionStatsList.clear();
    for (String finishedPath : workerFinishedPathList) {
      String hostnamePartitionId = FilenameUtils.getName(finishedPath);
      JSONObject workerFinishedInfoObj = null;
      try {
        byte [] zkData =
            getZkExt().getData(finishedPath, false, null);
        workerFinishedInfoObj = new JSONObject(new String(zkData));
        List<PartitionStats> statsList =
            WritableUtils.readListFieldsFromByteArray(
                Base64.decode(workerFinishedInfoObj.getString(
                    JSONOBJ_PARTITION_STATS_KEY)),
                    partitionStatsClass,
                    conf);
        for (PartitionStats partitionStats : statsList) {
          globalStats.addPartitionStats(partitionStats);
          allPartitionStatsList.add(partitionStats);
        }
        globalStats.addMessageCount(
            workerFinishedInfoObj.getLong(
                JSONOBJ_NUM_MESSAGES_KEY));
        if (conf.metricsEnabled() &&
            workerFinishedInfoObj.has(JSONOBJ_METRICS_KEY)) {
          WorkerSuperstepMetrics workerMetrics = new WorkerSuperstepMetrics();
          WritableUtils.readFieldsFromByteArray(
              Base64.decode(
                  workerFinishedInfoObj.getString(
                      JSONOBJ_METRICS_KEY)),
              workerMetrics);
          aggregatedMetrics.add(workerMetrics, hostnamePartitionId);
        }
      } catch (JSONException e) {
        throw new IllegalStateException(
            "aggregateWorkerStats: JSONException", e);
      } catch (KeeperException e) {
        throw new IllegalStateException(
            "aggregateWorkerStats: KeeperException", e);
      } catch (InterruptedException e) {
        throw new IllegalStateException(
            "aggregateWorkerStats: InterruptedException", e);
      } catch (IOException e) {
        throw new IllegalStateException(
            "aggregateWorkerStats: IOException", e);
      }
    }


    if (conf.metricsEnabled()) {
      aggregatedMetrics.print(superstep, System.err);
    }


    if (LOG.isInfoEnabled()) {
      LOG.info("aggregateWorkerStats: Aggregation found " + globalStats +

View Full Code Here

    }
  }


  @Override
  public void cleanup() throws IOException {
    ImmutableClassesGiraphConfiguration conf = getConfiguration();


    // All master processes should denote they are done by adding special
    // znode.  Once the number of znodes equals the number of partitions
    // for workers and masters, the master will clean up the ZooKeeper
    // znodes associated with this job.

View Full Code Here

  private static Logger LOG = Logger.getLogger(BspOutputFormat.class);


  @Override
  public void checkOutputSpecs(JobContext context)
    throws IOException, InterruptedException {
    ImmutableClassesGiraphConfiguration conf =
        new ImmutableClassesGiraphConfiguration(context.getConfiguration());
    if (!conf.hasVertexOutputFormat()) {
      LOG.warn("checkOutputSpecs: ImmutableOutputCommiter" +
          " will not check anything");
      return;
    }
    conf.createVertexOutputFormat().checkOutputSpecs(context);
  }

View Full Code Here

  }


  @Override
  public OutputCommitter getOutputCommitter(TaskAttemptContext context)
    throws IOException, InterruptedException {
    ImmutableClassesGiraphConfiguration conf =
        new ImmutableClassesGiraphConfiguration(context.getConfiguration());
    if (!conf.hasVertexOutputFormat()) {
      LOG.warn("getOutputCommitter: Returning " +
          "ImmutableOutputCommiter (does nothing).");
      return new ImmutableOutputCommitter();
    }
    return conf.createVertexOutputFormat().getOutputCommitter(context);
  }

View Full Code Here

   * Constructor requiring classes
   *
   * @param conf Should have vertex and edge classes set.
   */
  public TestGraph(GiraphConfiguration conf) {
    this.conf = new ImmutableClassesGiraphConfiguration(conf);
  }

View Full Code Here

    containerId = cId; // future good stuff will need me to operate.
    appAttemptId = aId;
    progress = 0.0f;
    lastResponseId = new AtomicInteger(0);
    giraphConf =
      new ImmutableClassesGiraphConfiguration(new GiraphConfiguration());
    completedCount = new AtomicInteger(0);
    failedCount = new AtomicInteger(0);
    allocatedCount = new AtomicInteger(0);
    successfulCount = new AtomicInteger(0);
    rpc = YarnRPC.create(giraphConf);

View Full Code Here

  @Before
  public void setUp() throws IOException {
    // Setup the conf
    GiraphConfiguration tmpConf = new GiraphConfiguration();
    tmpConf.setVertexClass(TestVertex.class);
    conf = new ImmutableClassesGiraphConfiguration(tmpConf);


    context = mock(Context.class);
    when(context.getConfiguration()).thenReturn(conf);
  }

View Full Code Here


  /** Test if aggregators serialization captures everything */
  @Test
  public void testMasterAggregatorsSerialization() throws
      IllegalAccessException, InstantiationException, IOException {
    ImmutableClassesGiraphConfiguration conf =
        Mockito.mock(ImmutableClassesGiraphConfiguration.class);
    Mockito.when(conf.getAggregatorWriterClass()).thenReturn(
        TextAggregatorWriter.class);
    Progressable progressable = Mockito.mock(Progressable.class);
    MasterAggregatorHandler handler =
        new MasterAggregatorHandler(conf, progressable);

View Full Code Here

  /**
   * Initialize the MiniYARNCluster for the integration test.
   */
  private void initYarnCluster() {
    cluster = new MiniYARNCluster(TestYarnJob.class.getName(), 1, 1, 1);
    cluster.init(new ImmutableClassesGiraphConfiguration(conf));
    cluster.start();
  }

View Full Code Here

0 1 2 3 4 5

TOP

Related Classes of org.apache.giraph.conf.ImmutableClassesGiraphConfiguration

io.netty.handler.codec.compression.JdkZlibDecoder

io.netty.handler.codec.compression.JdkZlibEncoder

io.netty.handler.codec.compression.SnappyFramedDecoder

io.netty.handler.codec.compression.SnappyFramedEncoder

org.apache.giraph.aggregators.TestAggregatorsHandling

org.apache.giraph.bsp.BspOutputFormat

org.apache.giraph.comm.ConnectionTest

org.apache.giraph.comm.messages.TestIntFloatPrimitiveMessageStores

org.apache.giraph.comm.messages.TestLongDoublePrimitiveMessageStores

org.apache.giraph.comm.RequestFailureTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.