Examples of eu.stratosphere.nephele.jobgraph.JobTaskVertex

Package eu.stratosphere.nephele.jobgraph

Examples of eu.stratosphere.nephele.jobgraph.JobTaskVertex

eu.stratosphere.nephele.jobgraph.JobTaskVertex
A JobTaskVertex is the vertex type for regular tasks (with both input and output) in Nephele. Tasks running inside a JobTaskVertex must specify at least one record reader and one record writer.

    // input
    JobInputVertex vertices = createVerticesInput(jobGraph, verticesPath, numSubTasks, serializer, comparator);
    JobInputVertex edges = createEdgesInput(jobGraph, edgesPath, numSubTasks, serializer, comparator);


    // head
    JobTaskVertex head = createIterationHead(jobGraph, numSubTasks, serializer, comparator, pairComparator);
    TaskConfig headConfig = new TaskConfig(head.getConfiguration());
    headConfig.setWaitForSolutionSetUpdate();


    // intermediate
    JobTaskVertex intermediate = createIterationIntermediate(jobGraph, numSubTasks, serializer, comparator);
    TaskConfig intermediateConfig = new TaskConfig(intermediate.getConfiguration());


    // output and auxiliaries
    JobOutputVertex output = createOutput(jobGraph, resultPath, numSubTasks, serializer);
    JobOutputVertex fakeTail = createFakeTail(jobGraph, numSubTasks);
    JobOutputVertex sync = createSync(jobGraph, numSubTasks, maxIterations);


    // ------------------ the intermediate (ws update) ----------------------
    JobTaskVertex wsUpdateIntermediate =
      JobGraphUtils.createTask(IterationIntermediatePactTask.class, "WorksetUpdate", jobGraph,
        numSubTasks, numSubTasks);
    TaskConfig wsUpdateConfig = new TaskConfig(wsUpdateIntermediate.getConfiguration());
    {
      wsUpdateConfig.setIterationId(ITERATION_ID);
      wsUpdateConfig.setIsWorksetIteration();
      wsUpdateConfig.setIsWorksetUpdate();


      // inputs
      wsUpdateConfig.addInputToGroup(0);
      wsUpdateConfig.setInputSerializer(serializer, 0);


      // output
      wsUpdateConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
      wsUpdateConfig.setOutputComparator(comparator, 0);


      wsUpdateConfig.setOutputSerializer(serializer);


      // driver
      wsUpdateConfig.setDriver(JoinWithSolutionSetSecondDriver.class);
      wsUpdateConfig.setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_SECOND);
      wsUpdateConfig.setDriverComparator(comparator, 0);
      wsUpdateConfig.setDriverPairComparator(pairComparator);
      
      wsUpdateConfig.setStubWrapper(new UserCodeClassWrapper<UpdateComponentIdMatch>(
        UpdateComponentIdMatch.class));
    }


    // -------------------------- ss tail --------------------------------
    JobTaskVertex ssTail =
      JobGraphUtils.createTask(IterationTailPactTask.class, "IterationSolutionSetTail", jobGraph,
        numSubTasks, numSubTasks);
    TaskConfig ssTailConfig = new TaskConfig(ssTail.getConfiguration());
    {
      ssTailConfig.setIterationId(ITERATION_ID);
      ssTailConfig.setIsSolutionSetUpdate();
      ssTailConfig.setIsWorksetIteration();


      // inputs and driver
      ssTailConfig.addInputToGroup(0);
      ssTailConfig.setInputSerializer(serializer, 0);


      // output
      ssTailConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
      ssTailConfig.setOutputSerializer(serializer);


      // the driver
      ssTailConfig.setDriver(CollectorMapDriver.class);
      ssTailConfig.setDriverStrategy(DriverStrategy.COLLECTOR_MAP);
      ssTailConfig.setStubWrapper(new UserCodeClassWrapper<DummyMapper>(DummyMapper.class));
    }


    // edges


    JobGraphUtils.connect(vertices, head, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
    JobGraphUtils.connect(edges, head, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
    JobGraphUtils.connect(vertices, head, ChannelType.NETWORK, DistributionPattern.BIPARTITE);


    JobGraphUtils.connect(head, intermediate, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
    intermediateConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, numSubTasks);


    JobGraphUtils.connect(intermediate, wsUpdateIntermediate, ChannelType.NETWORK,
      DistributionPattern.POINTWISE);
    wsUpdateConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, 1);


    JobGraphUtils.connect(wsUpdateIntermediate, ssTail, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);
    ssTailConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, 1);


    JobGraphUtils.connect(head, output, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);


    JobGraphUtils.connect(ssTail, fakeTail, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);


    JobGraphUtils.connect(head, sync, ChannelType.NETWORK, DistributionPattern.POINTWISE);


    vertices.setVertexToShareInstancesWith(head);
    edges.setVertexToShareInstancesWith(head);


    intermediate.setVertexToShareInstancesWith(head);


    wsUpdateIntermediate.setVertexToShareInstancesWith(head);
    ssTail.setVertexToShareInstancesWith(head);


    output.setVertexToShareInstancesWith(head);
    sync.setVertexToShareInstancesWith(head);


    fakeTail.setVertexToShareInstancesWith(ssTail);

View Full Code Here

    // input
    JobInputVertex vertices = createVerticesInput(jobGraph, verticesPath, numSubTasks, serializer, comparator);
    JobInputVertex edges = createEdgesInput(jobGraph, edgesPath, numSubTasks, serializer, comparator);


    // head
    JobTaskVertex head = createIterationHead(jobGraph, numSubTasks, serializer, comparator, pairComparator);


    // intermediate
    JobTaskVertex intermediate = createIterationIntermediate(jobGraph, numSubTasks, serializer, comparator);
    TaskConfig intermediateConfig = new TaskConfig(intermediate.getConfiguration());


    // output and auxiliaries
    JobOutputVertex output = createOutput(jobGraph, resultPath, numSubTasks, serializer);
    JobOutputVertex fakeTail = createFakeTail(jobGraph, numSubTasks);
    JobOutputVertex sync = createSync(jobGraph, numSubTasks, maxIterations);


    // ------------------ the intermediate (ss update) ----------------------
    JobTaskVertex ssJoinIntermediate = JobGraphUtils.createTask(IterationIntermediatePactTask.class,
      "Solution Set Update", jobGraph, numSubTasks, numSubTasks);
    TaskConfig ssJoinIntermediateConfig = new TaskConfig(ssJoinIntermediate.getConfiguration());
    {
      ssJoinIntermediateConfig.setIterationId(ITERATION_ID);
      ssJoinIntermediateConfig.setIsSolutionSetUpdate();
      ssJoinIntermediateConfig.setIsSolutionSetUpdateWithoutReprobe();


      // inputs
      ssJoinIntermediateConfig.addInputToGroup(0);
      ssJoinIntermediateConfig.setInputSerializer(serializer, 0);


      // output
      ssJoinIntermediateConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
      ssJoinIntermediateConfig.setOutputComparator(comparator, 0);


      ssJoinIntermediateConfig.setOutputSerializer(serializer);


      // driver
      ssJoinIntermediateConfig.setDriver(JoinWithSolutionSetSecondDriver.class);
      ssJoinIntermediateConfig.setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_SECOND);
      ssJoinIntermediateConfig.setDriverComparator(comparator, 0);
      ssJoinIntermediateConfig.setDriverPairComparator(pairComparator);
      
      ssJoinIntermediateConfig.setStubWrapper(new UserCodeClassWrapper<UpdateComponentIdMatch>(UpdateComponentIdMatch.class));
    }


    // -------------------------- ws tail --------------------------------
    JobTaskVertex wsTail = JobGraphUtils.createTask(IterationTailPactTask.class, "IterationWorksetTail",
      jobGraph, numSubTasks, numSubTasks);
    TaskConfig wsTailConfig = new TaskConfig(wsTail.getConfiguration());
    {
      wsTailConfig.setIterationId(ITERATION_ID);
      wsTailConfig.setIsWorksetIteration();
      wsTailConfig.setIsWorksetUpdate();


      // inputs and driver
      wsTailConfig.addInputToGroup(0);
      wsTailConfig.setInputSerializer(serializer, 0);


      // output
      wsTailConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
      wsTailConfig.setOutputSerializer(serializer);


      // the driver
      wsTailConfig.setDriver(CollectorMapDriver.class);
      wsTailConfig.setDriverStrategy(DriverStrategy.COLLECTOR_MAP);
      wsTailConfig.setStubWrapper(new UserCodeClassWrapper<DummyMapper>(DummyMapper.class));
    }


    // --------------- the wiring ---------------------


    JobGraphUtils.connect(vertices, head, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
    JobGraphUtils.connect(edges, head, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
    JobGraphUtils.connect(vertices, head, ChannelType.NETWORK, DistributionPattern.BIPARTITE);


    JobGraphUtils.connect(head, intermediate, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
    intermediateConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, numSubTasks);


    JobGraphUtils.connect(intermediate, ssJoinIntermediate, ChannelType.NETWORK, DistributionPattern.POINTWISE);
    ssJoinIntermediateConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, 1);


    JobGraphUtils.connect(ssJoinIntermediate, wsTail, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);
    wsTailConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, 1);


    JobGraphUtils.connect(head, output, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);


    JobGraphUtils.connect(wsTail, fakeTail, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);


    JobGraphUtils.connect(head, sync, ChannelType.NETWORK, DistributionPattern.POINTWISE);


    vertices.setVertexToShareInstancesWith(head);
    edges.setVertexToShareInstancesWith(head);


    intermediate.setVertexToShareInstancesWith(head);


    ssJoinIntermediate.setVertexToShareInstancesWith(head);
    wsTail.setVertexToShareInstancesWith(head);


    output.setVertexToShareInstancesWith(head);
    sync.setVertexToShareInstancesWith(head);


    fakeTail.setVertexToShareInstancesWith(wsTail);

View Full Code Here


    return modelsInput;
  }


  private static JobTaskVertex createMapper(JobGraph jobGraph, int numSubTasks, TypeSerializerFactory<?> serializer) {
    JobTaskVertex pointsInput = JobGraphUtils.createTask(RegularPactTask.class, "Map[DotProducts]", jobGraph, numSubTasks, numSubTasks);


    {
      TaskConfig taskConfig = new TaskConfig(pointsInput.getConfiguration());


      taskConfig.setStubWrapper(new UserCodeClassWrapper<DotProducts>(DotProducts.class));
      taskConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
      taskConfig.setOutputSerializer(serializer);
      taskConfig.setDriver(CollectorMapDriver.class);

View Full Code Here

    JobGraph jobGraph = new JobGraph("Distance Builder");


    // -- vertices ---------------------------------------------------------------------------------------------
    JobInputVertex points = createPointsInput(jobGraph, pointsPath, numSubTasks, serializer);
    JobInputVertex models = createModelsInput(jobGraph, centersPath, numSubTasks, serializer);
    JobTaskVertex mapper = createMapper(jobGraph, numSubTasks, serializer);
    JobOutputVertex output = createOutput(jobGraph, resultPath, numSubTasks, serializer);


    // -- edges ------------------------------------------------------------------------------------------------
    JobGraphUtils.connect(points, mapper, ChannelType.NETWORK, DistributionPattern.POINTWISE);
    JobGraphUtils.connect(models, mapper, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
    JobGraphUtils.connect(mapper, output, ChannelType.NETWORK, DistributionPattern.POINTWISE);


    // -- instance sharing -------------------------------------------------------------------------------------
    points.setVertexToShareInstancesWith(output);
    models.setVertexToShareInstancesWith(output);
    mapper.setVertexToShareInstancesWith(output);


    return jobGraph;
  }

View Full Code Here

      final JobFileInputVertex i1 = new JobFileInputVertex("Input 1", jg);
      i1.setFileInputClass(FileLineReader.class);
      i1.setFilePath(new Path(new File(testDirectory).toURI()));


      // task vertex 1
      final JobTaskVertex t1 = new JobTaskVertex("Task 1", jg);
      t1.setTaskClass(ForwardTask.class);


      // task vertex 2
      final JobTaskVertex t2 = new JobTaskVertex("Task 2", jg);
      t2.setTaskClass(ForwardTask.class);


      // output vertex
      JobFileOutputVertex o1 = new JobFileOutputVertex("Output 1", jg);
      o1.setFileOutputClass(FileLineWriter.class);
      o1.setFilePath(new Path(outputFile.toURI()));


      t1.setVertexToShareInstancesWith(i1);
      t2.setVertexToShareInstancesWith(i1);
      o1.setVertexToShareInstancesWith(i1);


      // connect vertices
      try {
        i1.connectTo(t1, ChannelType.NETWORK);
        t1.connectTo(t2, ChannelType.IN_MEMORY);
        t2.connectTo(o1, ChannelType.IN_MEMORY);
      } catch (JobGraphDefinitionException e) {
        e.printStackTrace();
      }


      // add jar

View Full Code Here

      input.setFileInputClass(SelfCrossInputTask.class);
      input.setFilePath(new Path(inputFile1.toURI()));
      input.setNumberOfSubtasks(degreeOfParallelism);


      // cross vertex
      final JobTaskVertex cross = new JobTaskVertex(crossTaskName, jg);
      cross.setTaskClass(SelfCrossForwardTask.class);
      cross.setNumberOfSubtasks(degreeOfParallelism);


      // output vertex
      final JobFileOutputVertex output = new JobFileOutputVertex(outputTaskName, jg);
      output.setFileOutputClass(FileLineWriter.class);
      output.setFilePath(new Path(outputFile1.toURI()));
      output.setNumberOfSubtasks(degreeOfParallelism);


      // connect vertices
      input.connectTo(cross, ChannelType.IN_MEMORY, 0, 0,
        DistributionPattern.POINTWISE);
      input.connectTo(cross, ChannelType.NETWORK, 1, 1,
        DistributionPattern.BIPARTITE);
      cross.connectTo(output, ChannelType.IN_MEMORY, 0, 0,
        DistributionPattern.POINTWISE);


      LibraryCacheManager.register(jobID, new String[0]);


      // now convert job graph to execution graph

View Full Code Here

      final JobFileInputVertex i1 = new JobFileInputVertex("Input 1", jg);
      i1.setFileInputClass(FileLineReader.class);
      i1.setFilePath(new Path(inputFile.toURI()));


      // task vertex 1
      final JobTaskVertex t1 = new JobTaskVertex("Task with Exception", jg);
      t1.setTaskClass(ExceptionTask.class);


      // output vertex
      JobFileOutputVertex o1 = new JobFileOutputVertex("Output 1", jg);
      o1.setFileOutputClass(FileLineWriter.class);
      o1.setFilePath(new Path(outputFile.toURI()));


      t1.setVertexToShareInstancesWith(i1);
      o1.setVertexToShareInstancesWith(i1);


      // connect vertices
      i1.connectTo(t1, ChannelType.IN_MEMORY);
      t1.connectTo(o1, ChannelType.IN_MEMORY);


      // add jar
      jg.addJar(new Path(new File(ServerTestUtils.getTempDir() + File.separator + exceptionClassName + ".jar")
        .toURI()));

View Full Code Here

      input1.setFileInputClass(FileLineReader.class);
      input1.setFilePath(new Path(inputFile1.toURI()));
      input1.setNumberOfSubtasks(degreeOfParallelism);


      // forward vertex 1
      final JobTaskVertex forward1 = new JobTaskVertex("Forward 1", jg);
      forward1.setTaskClass(ForwardTask1Input1Output.class);
      forward1.setNumberOfSubtasks(degreeOfParallelism);


      // forward vertex 2
      final JobTaskVertex forward2 = new JobTaskVertex("Forward 2", jg);
      forward2.setTaskClass(ForwardTask1Input1Output.class);
      forward2.setNumberOfSubtasks(degreeOfParallelism);


      // forward vertex 3
      final JobTaskVertex forward3 = new JobTaskVertex("Forward 3", jg);
      forward3.setTaskClass(ForwardTask1Input1Output.class);
      forward3.setNumberOfSubtasks(degreeOfParallelism);


      // output vertex
      final JobFileOutputVertex output1 = new JobFileOutputVertex("Output 1", jg);
      output1.setFileOutputClass(FileLineWriter.class);
      output1.setFilePath(new Path(outputFile1.toURI()));
      output1.setNumberOfSubtasks(degreeOfParallelism);


      // connect vertices
      input1.connectTo(forward1, ChannelType.IN_MEMORY,
        DistributionPattern.POINTWISE);
      forward1.connectTo(forward2, ChannelType.IN_MEMORY,
        DistributionPattern.POINTWISE);
      forward2.connectTo(forward3, ChannelType.NETWORK,
        DistributionPattern.POINTWISE);
      forward3.connectTo(output1, ChannelType.IN_MEMORY);


      // setup instance sharing
      input1.setVertexToShareInstancesWith(forward1);
      forward1.setVertexToShareInstancesWith(forward2);
      forward2.setVertexToShareInstancesWith(forward3);
      forward3.setVertexToShareInstancesWith(output1);


      LibraryCacheManager.register(jobID, new String[0]);


      // now convert job graph to execution graph
      final ExecutionGraph eg = new ExecutionGraph(jg, INSTANCE_MANAGER);

View Full Code Here

      final JobFileInputVertex i1 = new JobFileInputVertex("Input 1", jg);
      i1.setFileInputClass(FileLineReader.class);
      i1.setFilePath(new Path(inputFile.toURI()));


      // task vertex 1
      final JobTaskVertex t1 = new JobTaskVertex("Task with Exception", jg);
      t1.setTaskClass(RuntimeExceptionTask.class);


      // output vertex
      JobFileOutputVertex o1 = new JobFileOutputVertex("Output 1", jg);
      o1.setFileOutputClass(FileLineWriter.class);
      o1.setFilePath(new Path(outputFile.toURI()));


      t1.setVertexToShareInstancesWith(i1);
      o1.setVertexToShareInstancesWith(i1);


      // connect vertices
      i1.connectTo(t1, ChannelType.IN_MEMORY);
      t1.connectTo(o1, ChannelType.IN_MEMORY);


      // add jar
      jg.addJar(new Path(new File(ServerTestUtils.getTempDir() + File.separator + runtimeExceptionClassName
        + ".jar").toURI()));

View Full Code Here

      final JobFileInputVertex i1 = new JobFileInputVertex("Input 1", jg);
      i1.setFileInputClass(FileLineReader.class);
      i1.setFilePath(new Path(inputFile.toURI()));


      // task vertex 1
      final JobTaskVertex t1 = new JobTaskVertex("Task 1", jg);
      t1.setTaskClass(ForwardTask.class);


      // task vertex 2
      final JobTaskVertex t2 = new JobTaskVertex("Task 2", jg);
      t2.setTaskClass(ForwardTask.class);


      // output vertex
      JobFileOutputVertex o1 = new JobFileOutputVertex("Output 1", jg);
      o1.setFileOutputClass(FileLineWriter.class);
      o1.setFilePath(new Path(outputFile.toURI()));


      t1.setVertexToShareInstancesWith(i1);
      t2.setVertexToShareInstancesWith(i1);
      o1.setVertexToShareInstancesWith(i1);


      // connect vertices
      try {
        i1.connectTo(t1, ChannelType.NETWORK);
        t1.connectTo(t2, ChannelType.IN_MEMORY);
        t2.connectTo(o1, ChannelType.IN_MEMORY);
      } catch (JobGraphDefinitionException e) {
        e.printStackTrace();
      }


      // add jar

View Full Code Here

0 1 2 3

TOP

Related Classes of eu.stratosphere.nephele.jobgraph.JobTaskVertex

eu.stratosphere.compiler.plantranslate.NepheleJobGraphGenerator

eu.stratosphere.nephele.executiongraph.ExecutionGraphTest

eu.stratosphere.nephele.jobmanager.JobManagerITCase

eu.stratosphere.test.broadcastvars.BroadcastVarsNepheleITCase

eu.stratosphere.test.broadcastvars.KMeansIterativeNepheleITCase

eu.stratosphere.test.iterative.nephele.ConnectedComponentsNepheleITCase

eu.stratosphere.test.iterative.nephele.customdanglingpagerank.CustomCompensatableDanglingPageRank

eu.stratosphere.test.iterative.nephele.customdanglingpagerank.CustomCompensatableDanglingPageRankWithCombiner

eu.stratosphere.test.iterative.nephele.danglingpagerank.CompensatableDanglingPageRank

eu.stratosphere.test.iterative.nephele.IterationWithChainingNepheleITCase

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.