Examples of eu.stratosphere.nephele.jobgraph.JobOutputVertex

Package eu.stratosphere.nephele.jobgraph

Examples of eu.stratosphere.nephele.jobgraph.JobOutputVertex

eu.stratosphere.nephele.jobgraph.JobOutputVertex
A JobOutputVertex is a specific subtype of a {@link AbstractJobOutputVertex} and is designedfor Nephele tasks which sink data in a not further specified way. As every job output vertex, a JobOutputVertex must not have any further output.

    JobTaskVertex head = createIterationHead(jobGraph, numSubTasks, serializer, comparator, pairComparator);


    JobTaskVertex intermediate = createIterationIntermediate(jobGraph, numSubTasks, serializer, comparator);
    TaskConfig intermediateConfig = new TaskConfig(intermediate.getConfiguration());


    JobOutputVertex output = createOutput(jobGraph, resultPath, numSubTasks, serializer);
    JobOutputVertex fakeTail = createFakeTail(jobGraph, numSubTasks);
    JobOutputVertex sync = createSync(jobGraph, numSubTasks, maxIterations);


    // --------------- the tail (solution set join) ---------------
    JobTaskVertex tail = JobGraphUtils.createTask(IterationTailPactTask.class, "IterationTail", jobGraph,
      numSubTasks, numSubTasks);
    TaskConfig tailConfig = new TaskConfig(tail.getConfiguration());
    {
      tailConfig.setIterationId(ITERATION_ID);


      tailConfig.setIsWorksetIteration();
      tailConfig.setIsWorksetUpdate();


      tailConfig.setIsSolutionSetUpdate();
      tailConfig.setIsSolutionSetUpdateWithoutReprobe();


      // inputs and driver
      tailConfig.addInputToGroup(0);
      tailConfig.setInputSerializer(serializer, 0);


      // output
      tailConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
      tailConfig.setOutputSerializer(serializer);


      // the driver
      tailConfig.setDriver(JoinWithSolutionSetSecondDriver.class);
      tailConfig.setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_SECOND);
      tailConfig.setDriverComparator(comparator, 0);
      tailConfig.setDriverPairComparator(pairComparator);
      
      tailConfig.setStubWrapper(new UserCodeClassWrapper<UpdateComponentIdMatch>(UpdateComponentIdMatch.class));
    }


    // -- edges ------------------------------------------------------------------------------------------------
    JobGraphUtils.connect(vertices, head, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
    JobGraphUtils.connect(edges, head, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
    JobGraphUtils.connect(vertices, head, ChannelType.NETWORK, DistributionPattern.BIPARTITE);


    JobGraphUtils.connect(head, intermediate, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
    intermediateConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, numSubTasks);


    JobGraphUtils.connect(intermediate, tail, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);
    tailConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, 1);


    JobGraphUtils.connect(head, output, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);
    JobGraphUtils.connect(tail, fakeTail, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);


    JobGraphUtils.connect(head, sync, ChannelType.NETWORK, DistributionPattern.POINTWISE);


    vertices.setVertexToShareInstancesWith(head);
    edges.setVertexToShareInstancesWith(head);


    intermediate.setVertexToShareInstancesWith(head);
    tail.setVertexToShareInstancesWith(head);


    output.setVertexToShareInstancesWith(head);
    sync.setVertexToShareInstancesWith(head);
    fakeTail.setVertexToShareInstancesWith(tail);


    return jobGraph;
  }

View Full Code Here

    // intermediate
    JobTaskVertex intermediate = createIterationIntermediate(jobGraph, numSubTasks, serializer, comparator);
    TaskConfig intermediateConfig = new TaskConfig(intermediate.getConfiguration());


    // output and auxiliaries
    JobOutputVertex output = createOutput(jobGraph, resultPath, numSubTasks, serializer);
    JobOutputVertex ssFakeTail = createFakeTail(jobGraph, numSubTasks);
    JobOutputVertex wsFakeTail = createFakeTail(jobGraph, numSubTasks);
    JobOutputVertex sync = createSync(jobGraph, numSubTasks, maxIterations);


    // ------------------ the intermediate (ss join) ----------------------
    JobTaskVertex ssJoinIntermediate = JobGraphUtils.createTask(IterationIntermediatePactTask.class,
      "Solution Set Join", jobGraph, numSubTasks, numSubTasks);
    TaskConfig ssJoinIntermediateConfig = new TaskConfig(ssJoinIntermediate.getConfiguration());
    {
      ssJoinIntermediateConfig.setIterationId(ITERATION_ID);


      // inputs
      ssJoinIntermediateConfig.addInputToGroup(0);
      ssJoinIntermediateConfig.setInputSerializer(serializer, 0);


      // output
      ssJoinIntermediateConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
      ssJoinIntermediateConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
      ssJoinIntermediateConfig.setOutputComparator(comparator, 0);
      ssJoinIntermediateConfig.setOutputComparator(comparator, 1);


      ssJoinIntermediateConfig.setOutputSerializer(serializer);


      // driver
      ssJoinIntermediateConfig.setDriver(JoinWithSolutionSetSecondDriver.class);
      ssJoinIntermediateConfig.setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_SECOND);
      ssJoinIntermediateConfig.setDriverComparator(comparator, 0);
      ssJoinIntermediateConfig.setDriverPairComparator(pairComparator);
      
      ssJoinIntermediateConfig.setStubWrapper(
        new UserCodeClassWrapper<UpdateComponentIdMatch>(UpdateComponentIdMatch.class));
    }


    // -------------------------- ss tail --------------------------------
    JobTaskVertex ssTail = JobGraphUtils.createTask(IterationTailPactTask.class, "IterationSolutionSetTail",
      jobGraph, numSubTasks, numSubTasks);
    TaskConfig ssTailConfig = new TaskConfig(ssTail.getConfiguration());
    {
      ssTailConfig.setIterationId(ITERATION_ID);
      ssTailConfig.setIsSolutionSetUpdate();
      ssTailConfig.setIsWorksetIteration();


      // inputs and driver
      ssTailConfig.addInputToGroup(0);
      ssTailConfig.setInputSerializer(serializer, 0);
      ssTailConfig.setInputAsynchronouslyMaterialized(0, true);
      ssTailConfig.setInputMaterializationMemory(0, MEM_PER_CONSUMER * JobGraphUtils.MEGABYTE);


      // output
      ssTailConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
      ssTailConfig.setOutputSerializer(serializer);


      // the driver
      ssTailConfig.setDriver(CollectorMapDriver.class);
      ssTailConfig.setDriverStrategy(DriverStrategy.COLLECTOR_MAP);
      ssTailConfig.setStubWrapper(new UserCodeClassWrapper<DummyMapper>(DummyMapper.class));
    }


    // -------------------------- ws tail --------------------------------
    JobTaskVertex wsTail = JobGraphUtils.createTask(IterationTailPactTask.class, "IterationWorksetTail",
      jobGraph, numSubTasks, numSubTasks);
    TaskConfig wsTailConfig = new TaskConfig(wsTail.getConfiguration());
    {
      wsTailConfig.setIterationId(ITERATION_ID);
      wsTailConfig.setIsWorksetIteration();
      wsTailConfig.setIsWorksetUpdate();


      // inputs and driver
      wsTailConfig.addInputToGroup(0);
      wsTailConfig.setInputSerializer(serializer, 0);


      // output
      wsTailConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
      wsTailConfig.setOutputSerializer(serializer);


      // the driver
      wsTailConfig.setDriver(CollectorMapDriver.class);
      wsTailConfig.setDriverStrategy(DriverStrategy.COLLECTOR_MAP);
      wsTailConfig.setStubWrapper(new UserCodeClassWrapper<DummyMapper>(DummyMapper.class));
    }


    // --------------- the wiring ---------------------


    JobGraphUtils.connect(vertices, head, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
    JobGraphUtils.connect(edges, head, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
    JobGraphUtils.connect(vertices, head, ChannelType.NETWORK, DistributionPattern.BIPARTITE);


    JobGraphUtils.connect(head, intermediate, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
    intermediateConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, numSubTasks);


    JobGraphUtils.connect(intermediate, ssJoinIntermediate, ChannelType.NETWORK, DistributionPattern.POINTWISE);
    ssJoinIntermediateConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, 1);


    JobGraphUtils.connect(ssJoinIntermediate, ssTail, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);
    ssTailConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, 1);


    JobGraphUtils.connect(ssJoinIntermediate, wsTail, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);
    wsTailConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, 1);


    JobGraphUtils.connect(head, output, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);


    JobGraphUtils.connect(ssTail, ssFakeTail, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);
    JobGraphUtils.connect(wsTail, wsFakeTail, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);


    JobGraphUtils.connect(head, sync, ChannelType.NETWORK, DistributionPattern.POINTWISE);


    vertices.setVertexToShareInstancesWith(head);
    edges.setVertexToShareInstancesWith(head);


    intermediate.setVertexToShareInstancesWith(head);


    ssJoinIntermediate.setVertexToShareInstancesWith(head);
    wsTail.setVertexToShareInstancesWith(head);


    output.setVertexToShareInstancesWith(head);
    sync.setVertexToShareInstancesWith(head);


    ssTail.setVertexToShareInstancesWith(wsTail);
    ssFakeTail.setVertexToShareInstancesWith(ssTail);
    wsFakeTail.setVertexToShareInstancesWith(wsTail);

View Full Code Here

    // intermediate
    JobTaskVertex intermediate = createIterationIntermediate(jobGraph, numSubTasks, serializer, comparator);
    TaskConfig intermediateConfig = new TaskConfig(intermediate.getConfiguration());


    // output and auxiliaries
    JobOutputVertex output = createOutput(jobGraph, resultPath, numSubTasks, serializer);
    JobOutputVertex fakeTail = createFakeTail(jobGraph, numSubTasks);
    JobOutputVertex sync = createSync(jobGraph, numSubTasks, maxIterations);


    // ------------------ the intermediate (ws update) ----------------------
    JobTaskVertex wsUpdateIntermediate =
      JobGraphUtils.createTask(IterationIntermediatePactTask.class, "WorksetUpdate", jobGraph,
        numSubTasks, numSubTasks);
    TaskConfig wsUpdateConfig = new TaskConfig(wsUpdateIntermediate.getConfiguration());
    {
      wsUpdateConfig.setIterationId(ITERATION_ID);
      wsUpdateConfig.setIsWorksetIteration();
      wsUpdateConfig.setIsWorksetUpdate();


      // inputs
      wsUpdateConfig.addInputToGroup(0);
      wsUpdateConfig.setInputSerializer(serializer, 0);


      // output
      wsUpdateConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
      wsUpdateConfig.setOutputComparator(comparator, 0);


      wsUpdateConfig.setOutputSerializer(serializer);


      // driver
      wsUpdateConfig.setDriver(JoinWithSolutionSetSecondDriver.class);
      wsUpdateConfig.setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_SECOND);
      wsUpdateConfig.setDriverComparator(comparator, 0);
      wsUpdateConfig.setDriverPairComparator(pairComparator);
      
      wsUpdateConfig.setStubWrapper(new UserCodeClassWrapper<UpdateComponentIdMatch>(
        UpdateComponentIdMatch.class));
    }


    // -------------------------- ss tail --------------------------------
    JobTaskVertex ssTail =
      JobGraphUtils.createTask(IterationTailPactTask.class, "IterationSolutionSetTail", jobGraph,
        numSubTasks, numSubTasks);
    TaskConfig ssTailConfig = new TaskConfig(ssTail.getConfiguration());
    {
      ssTailConfig.setIterationId(ITERATION_ID);
      ssTailConfig.setIsSolutionSetUpdate();
      ssTailConfig.setIsWorksetIteration();


      // inputs and driver
      ssTailConfig.addInputToGroup(0);
      ssTailConfig.setInputSerializer(serializer, 0);


      // output
      ssTailConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
      ssTailConfig.setOutputSerializer(serializer);


      // the driver
      ssTailConfig.setDriver(CollectorMapDriver.class);
      ssTailConfig.setDriverStrategy(DriverStrategy.COLLECTOR_MAP);
      ssTailConfig.setStubWrapper(new UserCodeClassWrapper<DummyMapper>(DummyMapper.class));
    }


    // edges


    JobGraphUtils.connect(vertices, head, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
    JobGraphUtils.connect(edges, head, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
    JobGraphUtils.connect(vertices, head, ChannelType.NETWORK, DistributionPattern.BIPARTITE);


    JobGraphUtils.connect(head, intermediate, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
    intermediateConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, numSubTasks);


    JobGraphUtils.connect(intermediate, wsUpdateIntermediate, ChannelType.NETWORK,
      DistributionPattern.POINTWISE);
    wsUpdateConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, 1);


    JobGraphUtils.connect(wsUpdateIntermediate, ssTail, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);
    ssTailConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, 1);


    JobGraphUtils.connect(head, output, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);


    JobGraphUtils.connect(ssTail, fakeTail, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);


    JobGraphUtils.connect(head, sync, ChannelType.NETWORK, DistributionPattern.POINTWISE);


    vertices.setVertexToShareInstancesWith(head);
    edges.setVertexToShareInstancesWith(head);


    intermediate.setVertexToShareInstancesWith(head);


    wsUpdateIntermediate.setVertexToShareInstancesWith(head);
    ssTail.setVertexToShareInstancesWith(head);


    output.setVertexToShareInstancesWith(head);
    sync.setVertexToShareInstancesWith(head);


    fakeTail.setVertexToShareInstancesWith(ssTail);


    return jobGraph;
  }

View Full Code Here

    // intermediate
    JobTaskVertex intermediate = createIterationIntermediate(jobGraph, numSubTasks, serializer, comparator);
    TaskConfig intermediateConfig = new TaskConfig(intermediate.getConfiguration());


    // output and auxiliaries
    JobOutputVertex output = createOutput(jobGraph, resultPath, numSubTasks, serializer);
    JobOutputVertex fakeTail = createFakeTail(jobGraph, numSubTasks);
    JobOutputVertex sync = createSync(jobGraph, numSubTasks, maxIterations);


    // ------------------ the intermediate (ss update) ----------------------
    JobTaskVertex ssJoinIntermediate = JobGraphUtils.createTask(IterationIntermediatePactTask.class,
      "Solution Set Update", jobGraph, numSubTasks, numSubTasks);
    TaskConfig ssJoinIntermediateConfig = new TaskConfig(ssJoinIntermediate.getConfiguration());
    {
      ssJoinIntermediateConfig.setIterationId(ITERATION_ID);
      ssJoinIntermediateConfig.setIsSolutionSetUpdate();
      ssJoinIntermediateConfig.setIsSolutionSetUpdateWithoutReprobe();


      // inputs
      ssJoinIntermediateConfig.addInputToGroup(0);
      ssJoinIntermediateConfig.setInputSerializer(serializer, 0);


      // output
      ssJoinIntermediateConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
      ssJoinIntermediateConfig.setOutputComparator(comparator, 0);


      ssJoinIntermediateConfig.setOutputSerializer(serializer);


      // driver
      ssJoinIntermediateConfig.setDriver(JoinWithSolutionSetSecondDriver.class);
      ssJoinIntermediateConfig.setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_SECOND);
      ssJoinIntermediateConfig.setDriverComparator(comparator, 0);
      ssJoinIntermediateConfig.setDriverPairComparator(pairComparator);
      
      ssJoinIntermediateConfig.setStubWrapper(new UserCodeClassWrapper<UpdateComponentIdMatch>(UpdateComponentIdMatch.class));
    }


    // -------------------------- ws tail --------------------------------
    JobTaskVertex wsTail = JobGraphUtils.createTask(IterationTailPactTask.class, "IterationWorksetTail",
      jobGraph, numSubTasks, numSubTasks);
    TaskConfig wsTailConfig = new TaskConfig(wsTail.getConfiguration());
    {
      wsTailConfig.setIterationId(ITERATION_ID);
      wsTailConfig.setIsWorksetIteration();
      wsTailConfig.setIsWorksetUpdate();


      // inputs and driver
      wsTailConfig.addInputToGroup(0);
      wsTailConfig.setInputSerializer(serializer, 0);


      // output
      wsTailConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
      wsTailConfig.setOutputSerializer(serializer);


      // the driver
      wsTailConfig.setDriver(CollectorMapDriver.class);
      wsTailConfig.setDriverStrategy(DriverStrategy.COLLECTOR_MAP);
      wsTailConfig.setStubWrapper(new UserCodeClassWrapper<DummyMapper>(DummyMapper.class));
    }


    // --------------- the wiring ---------------------


    JobGraphUtils.connect(vertices, head, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
    JobGraphUtils.connect(edges, head, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
    JobGraphUtils.connect(vertices, head, ChannelType.NETWORK, DistributionPattern.BIPARTITE);


    JobGraphUtils.connect(head, intermediate, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
    intermediateConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, numSubTasks);


    JobGraphUtils.connect(intermediate, ssJoinIntermediate, ChannelType.NETWORK, DistributionPattern.POINTWISE);
    ssJoinIntermediateConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, 1);


    JobGraphUtils.connect(ssJoinIntermediate, wsTail, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);
    wsTailConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, 1);


    JobGraphUtils.connect(head, output, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);


    JobGraphUtils.connect(wsTail, fakeTail, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);


    JobGraphUtils.connect(head, sync, ChannelType.NETWORK, DistributionPattern.POINTWISE);


    vertices.setVertexToShareInstancesWith(head);
    edges.setVertexToShareInstancesWith(head);


    intermediate.setVertexToShareInstancesWith(head);


    ssJoinIntermediate.setVertexToShareInstancesWith(head);
    wsTail.setVertexToShareInstancesWith(head);


    output.setVertexToShareInstancesWith(head);
    sync.setVertexToShareInstancesWith(head);


    fakeTail.setVertexToShareInstancesWith(wsTail);


    return jobGraph;
  }

View Full Code Here

0 1 2

TOP

Related Classes of eu.stratosphere.nephele.jobgraph.JobOutputVertex

eu.stratosphere.compiler.plantranslate.NepheleJobGraphGenerator

eu.stratosphere.nephele.jobmanager.scheduler.queue.QueueSchedulerTest

eu.stratosphere.test.broadcastvars.BroadcastVarsNepheleITCase

eu.stratosphere.test.broadcastvars.KMeansIterativeNepheleITCase

eu.stratosphere.test.iterative.nephele.ConnectedComponentsNepheleITCase

eu.stratosphere.test.iterative.nephele.customdanglingpagerank.CustomCompensatableDanglingPageRank

eu.stratosphere.test.iterative.nephele.customdanglingpagerank.CustomCompensatableDanglingPageRankWithCombiner

eu.stratosphere.test.iterative.nephele.danglingpagerank.CompensatableDanglingPageRank

eu.stratosphere.test.iterative.nephele.IterationWithChainingNepheleITCase

eu.stratosphere.test.iterative.nephele.JobGraphUtils

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.