Examples of org.apache.giraph.job.GiraphJob

org.apache.giraph.job.GiraphJob
Generates an appropriate internal {@link Job} for using Giraph in Hadoop.Uses composition to avoid unwanted {@link Job} methods from exposureto the user.

    GiraphConstants.CHECKPOINT_DIRECTORY.set(conf,
        getTempPath("_singleFaultCheckpoints").toString());
    GiraphConstants.CLEANUP_CHECKPOINTS_AFTER_SUCCESS.set(conf, false);
    GiraphConstants.ZOOKEEPER_SESSION_TIMEOUT.set(conf, 10000);
    GiraphConstants.ZOOKEEPER_MIN_SESSION_TIMEOUT.set(conf, 10000);
    GiraphJob job = prepareJob(getCallingMethodName(), conf, outputPath);
    assertTrue(job.run(true));
  }

View Full Code Here

      FloatWritable, DoubleWritable, LongWritable> { }


  @Before
  public void setUp() {
    try {
      job = new GiraphJob("TestGiraphTransferRegulator");
    } catch (IOException e) {
      throw new RuntimeException("setUp: Failed", e);
    }
    job.getConfiguration().setComputationClass(TestComputation.class);
  }

View Full Code Here

        SimpleCheckpoint.SimpleCheckpointVertexWorkerContext.class);
    conf.setMasterComputeClass(
        SimpleCheckpoint.SimpleCheckpointVertexMasterCompute.class);
    conf.setVertexInputFormatClass(SimpleSuperstepVertexInputFormat.class);
    conf.setVertexOutputFormatClass(SimpleSuperstepVertexOutputFormat.class);
    GiraphJob job = prepareJob("testVertexBalancer", conf, outputPath);


    job.getConfiguration().set(
        PartitionBalancer.PARTITION_BALANCE_ALGORITHM,
        PartitionBalancer.VERTICES_BALANCE_ALGORITHM);


    assertTrue(job.run(true));
    FileSystem hdfs = FileSystem.get(job.getConfiguration());


    conf = new GiraphConfiguration();
    conf.setComputationClass(
        SimpleCheckpoint.SimpleCheckpointComputation.class);
    conf.setWorkerContextClass(
        SimpleCheckpoint.SimpleCheckpointVertexWorkerContext.class);
    conf.setMasterComputeClass(
        SimpleCheckpoint.SimpleCheckpointVertexMasterCompute.class);
    conf.setVertexInputFormatClass(SimpleSuperstepVertexInputFormat.class);
    conf.setVertexOutputFormatClass(SimpleSuperstepVertexOutputFormat.class);
    outputPath = getTempPath("testHashPartitioner");
    job = prepareJob("testHashPartitioner", conf, outputPath);
    assertTrue(job.run(true));
    verifyOutput(hdfs, outputPath);


    outputPath = getTempPath("testSuperstepHashPartitioner");
    conf = new GiraphConfiguration();
    conf.setComputationClass(
        SimpleCheckpoint.SimpleCheckpointComputation.class);
    conf.setWorkerContextClass(
        SimpleCheckpoint.SimpleCheckpointVertexWorkerContext.class);
    conf.setMasterComputeClass(
        SimpleCheckpoint.SimpleCheckpointVertexMasterCompute.class);
    conf.setVertexInputFormatClass(SimpleSuperstepVertexInputFormat.class);
    conf.setVertexOutputFormatClass(SimpleSuperstepVertexOutputFormat.class);
    job = prepareJob("testSuperstepHashPartitioner", conf, outputPath);


    job.getConfiguration().setGraphPartitionerFactoryClass(
        SuperstepHashPartitionerFactory.class);


    assertTrue(job.run(true));
    verifyOutput(hdfs, outputPath);


    job = new GiraphJob("testHashRangePartitioner");
    setupConfiguration(job);
    job.getConfiguration().setComputationClass(
        SimpleCheckpoint.SimpleCheckpointComputation.class);
    job.getConfiguration().setWorkerContextClass(
        SimpleCheckpoint.SimpleCheckpointVertexWorkerContext.class);
    job.getConfiguration().setMasterComputeClass(
        SimpleCheckpoint.SimpleCheckpointVertexMasterCompute.class);
    job.getConfiguration().setVertexInputFormatClass(
        SimpleSuperstepVertexInputFormat.class);
    job.getConfiguration().setVertexOutputFormatClass(
        SimpleSuperstepVertexOutputFormat.class);
    job.getConfiguration().setGraphPartitionerFactoryClass(
        HashRangePartitionerFactory.class);
    outputPath = getTempPath("testHashRangePartitioner");
    removeAndSetOutput(job, outputPath);
    assertTrue(job.run(true));
    verifyOutput(hdfs, outputPath);


    outputPath = getTempPath("testReverseIdSuperstepHashPartitioner");
    conf = new GiraphConfiguration();
    conf.setComputationClass(
        SimpleCheckpoint.SimpleCheckpointComputation.class);
    conf.setWorkerContextClass(
        SimpleCheckpoint.SimpleCheckpointVertexWorkerContext.class);
    conf.setMasterComputeClass(
        SimpleCheckpoint.SimpleCheckpointVertexMasterCompute.class);
    conf.setVertexInputFormatClass(SimpleSuperstepVertexInputFormat.class);
    conf.setVertexOutputFormatClass(SimpleSuperstepVertexOutputFormat.class);
    job = prepareJob("testReverseIdSuperstepHashPartitioner", conf,
        outputPath);
    job.getConfiguration().setGraphPartitionerFactoryClass(
        SuperstepHashPartitionerFactory.class);
    GeneratedVertexReader.REVERSE_ID_ORDER.set(job.getConfiguration(), true);
    assertTrue(job.run(true));
    verifyOutput(hdfs, outputPath);


    job = new GiraphJob("testSimpleRangePartitioner");
    setupConfiguration(job);
    job.getConfiguration().setComputationClass(
        SimpleCheckpoint.SimpleCheckpointComputation.class);
    job.getConfiguration().setWorkerContextClass(
        SimpleCheckpoint.SimpleCheckpointVertexWorkerContext.class);
    job.getConfiguration().setMasterComputeClass(
        SimpleCheckpoint.SimpleCheckpointVertexMasterCompute.class);
    job.getConfiguration().setVertexInputFormatClass(
        SimpleSuperstepVertexInputFormat.class);
    job.getConfiguration().setVertexOutputFormatClass(
        SimpleSuperstepVertexOutputFormat.class);


    job.getConfiguration().setGraphPartitionerFactoryClass(
        SimpleLongRangePartitionerFactory.class);
    long readerVertices =
        READER_VERTICES.getWithDefault(job.getConfiguration(), -1L);
    job.getConfiguration().setLong(
        GiraphConstants.PARTITION_VERTEX_KEY_SPACE_SIZE, readerVertices);


    outputPath = getTempPath("testSimpleRangePartitioner");
    removeAndSetOutput(job, outputPath);
    assertTrue(job.run(true));
    verifyOutput(hdfs, outputPath);
  }

View Full Code Here

          throws IOException, InterruptedException, ClassNotFoundException {
    GiraphConfiguration conf = new GiraphConfiguration();
    conf.setComputationClass(InfiniteLoopComputation.class);
    conf.setVertexInputFormatClass(SimplePageRankVertexInputFormat.class);
    conf.setVertexOutputFormatClass(SimplePageRankVertexOutputFormat.class);
    GiraphJob job = prepareJob(getCallingMethodName(), conf,
        getTempPath(getCallingMethodName()));
    job.getConfiguration().setMaxNumberOfSupersteps(3);
    assertTrue(job.run(true));
    if (!runningInDistributedMode()) {
      GiraphHadoopCounter superstepCounter =
          GiraphStats.getInstance().getSuperstepCounter();
      assertEquals(superstepCounter.getValue(), 3L);
    }

View Full Code Here

    conf.setComputationClass(AggregatorsTestComputation.class);
    conf.setVertexInputFormatClass(
        AggregatorsTestComputation.SimpleVertexInputFormat.class);
    conf.setEdgeInputFormatClass(
        AggregatorsTestComputation.SimpleEdgeInputFormat.class);
    GiraphJob job = prepareJob(getCallingMethodName(), conf);
    job.getConfiguration().setMasterComputeClass(
        AggregatorsTestComputation.AggregatorsTestMasterCompute.class);
    // test with aggregators split in a few requests
    job.getConfiguration().setInt(
        AggregatorUtils.MAX_BYTES_PER_AGGREGATOR_REQUEST, 50);
    assertTrue(job.run(true));
  }

View Full Code Here

        AggregatorsTestComputation.AggregatorsTestMasterCompute.class);
    conf.setVertexInputFormatClass(
        AggregatorsTestComputation.SimpleVertexInputFormat.class);
    conf.setEdgeInputFormatClass(
        AggregatorsTestComputation.SimpleEdgeInputFormat.class);
    GiraphJob job = prepareJob(getCallingMethodName(), conf, outputPath);


    GiraphConfiguration configuration = job.getConfiguration();
    GiraphConstants.CHECKPOINT_DIRECTORY.set(configuration, checkpointsDir.toString());
    GiraphConstants.CLEANUP_CHECKPOINTS_AFTER_SUCCESS.set(configuration, false);
    configuration.setCheckpointFrequency(4);


    assertTrue(job.run(true));


    // Restart the test from superstep 4
    System.out.println("testAggregatorsCheckpointing: Restarting from " +
        "superstep 4 with checkpoint path = " + checkpointsDir);
    outputPath = getTempPath(getCallingMethodName() + "Restarted");
    conf = new GiraphConfiguration();
    conf.setComputationClass(AggregatorsTestComputation.class);
    conf.setMasterComputeClass(
        AggregatorsTestComputation.AggregatorsTestMasterCompute.class);
    conf.setVertexInputFormatClass(
        AggregatorsTestComputation.SimpleVertexInputFormat.class);
    conf.setEdgeInputFormatClass(
        AggregatorsTestComputation.SimpleEdgeInputFormat.class);
    GiraphJob restartedJob = prepareJob(getCallingMethodName() + "Restarted",
        conf, outputPath);
    job.getConfiguration().setMasterComputeClass(
        SimpleCheckpoint.SimpleCheckpointVertexMasterCompute.class);
    GiraphConfiguration restartedJobConf = restartedJob.getConfiguration();
    GiraphConstants.CHECKPOINT_DIRECTORY.set(restartedJobConf,
        checkpointsDir.toString());
    restartedJobConf.setLong(GiraphConstants.RESTART_SUPERSTEP, 4);


    assertTrue(restartedJob.run(true));
  }

View Full Code Here

    conf.set("mapred.job.id", "restarted_test_job");
    if (restartFrom >= 0) {
      conf.set(GiraphConstants.RESTART_SUPERSTEP, Long.toString(restartFrom));
    }


    GiraphJob restartedJob = prepareJob(getCallingMethodName() + "Restarted",
        conf, outputPath);


    GiraphConstants.CHECKPOINT_DIRECTORY.set(restartedJob.getConfiguration(),
        checkpointsDir.toString());


    assertTrue(restartedJob.run(true));




    if (!runningInDistributedMode()) {
      long idSumRestarted =
          CheckpointVertexWorkerContext

View Full Code Here

    conf.setMasterComputeClass(
        CheckpointVertexMasterCompute.class);
    conf.setVertexInputFormatClass(SimpleSuperstepComputation.SimpleSuperstepVertexInputFormat.class);
    conf.setVertexOutputFormatClass(SimpleSuperstepComputation.SimpleSuperstepVertexOutputFormat.class);
    conf.set("mapred.job.id", TEST_JOB_ID);
    GiraphJob job = prepareJob(getCallingMethodName(), conf, outputPath);


    GiraphConfiguration configuration = job.getConfiguration();
    GiraphConstants.CHECKPOINT_DIRECTORY.set(configuration, checkpointsDir.toString());


    assertTrue(job.run(true));


    long idSum = 0;
    if (!runningInDistributedMode()) {
      FileStatus fileStatus = getSinglePartFileStatus(job.getConfiguration(),
          outputPath);
      idSum = CheckpointVertexWorkerContext
          .getFinalSum();
      LOG.info("testBspCheckpoint: idSum = " + idSum +
          " fileLen = " + fileStatus.getLen());

View Full Code Here

    }


    conf.setVertexInputFormatClass(InMemoryVertexInputFormat.class);


    // Create and configure the job to run the vertex
    GiraphJob job = new GiraphJob(conf, conf.getComputationName());


    InMemoryVertexInputFormat.setGraph(graph);


    int localZookeeperPort = findAvailablePort();

View Full Code Here

          tmpDir, "_checkpoints").toString();
    }
    GiraphConstants.CHECKPOINT_DIRECTORY.set(conf, checkpointsDir);


    // Create and configure the job to run the vertex
    GiraphJob job = new GiraphJob(conf, conf.getComputationName());


    Job internalJob = job.getInternalJob();
    if (conf.hasVertexInputFormat()) {
      GiraphFileInputFormat.setVertexInputPath(internalJob.getConfiguration(),
          new Path(vertexInputFile.toString()));
    }
    if (conf.hasEdgeInputFormat()) {
      GiraphFileInputFormat.setEdgeInputPath(internalJob.getConfiguration(),
          new Path(edgeInputFile.toString()));
    }
    FileOutputFormat.setOutputPath(job.getInternalJob(),
        new Path(outputDir.toString()));


    // Configure a local zookeeper instance
    Properties zkProperties = configLocalZooKeeper(zkDir, localZookeeperPort);

View Full Code Here

0 1 2 3 4 5 6

TOP

Related Classes of org.apache.giraph.job.GiraphJob

com.tinkerpop.gremlin.giraph.process.computer.GiraphGraphComputer

org.apache.giraph.aggregators.TestAggregatorsHandling

org.apache.giraph.benchmark.GiraphBenchmark

org.apache.giraph.BspCase

org.apache.giraph.conf.GiraphConfiguration

org.apache.giraph.conf.ImmutableClassesGiraphConfiguration

org.apache.giraph.examples.SimpleCheckpoint

org.apache.giraph.examples.SimpleCheckpointVertex

org.apache.giraph.examples.SimpleVertexWithWorkerContext

org.apache.giraph.examples.TestPageRank

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.