Package eu.stratosphere.nephele.jobgraph

Examples of eu.stratosphere.nephele.jobgraph.JobOutputVertex


    tailConfig.setStubParameter("compensation.failingIteration", String.valueOf(failingIteration));
    tailConfig.setStubParameter("compensation.messageLoss", String.valueOf(messageLoss));
   
    // --------------- the output ---------------------

    JobOutputVertex output = JobGraphUtils.createFileOutput(jobGraph, "FinalOutput", degreeOfParallelism,
      numSubTasksPerInstance);
    TaskConfig outputConfig = new TaskConfig(output.getConfiguration());
    outputConfig.addInputToGroup(0);
    outputConfig.setInputSerializer(recSerializer, 0);
    outputConfig.setStubWrapper(new UserCodeClassWrapper<PageWithRankOutFormat>(PageWithRankOutFormat.class));
    outputConfig.setStubParameter(FileOutputFormat.FILE_PARAMETER_KEY, outputPath);
   
    // --------------- the auxiliaries ---------------------
   
    JobOutputVertex fakeTailOutput = JobGraphUtils.createFakeOutput(jobGraph, "FakeTailOutput",
      degreeOfParallelism, numSubTasksPerInstance);

    JobOutputVertex sync = JobGraphUtils.createSync(jobGraph, degreeOfParallelism);
    TaskConfig syncConfig = new TaskConfig(sync.getConfiguration());
    syncConfig.setNumberOfIterations(numIterations);
    syncConfig.addIterationAggregator(CompensatableDotProductCoGroup.AGGREGATOR_NAME, PageRankStatsAggregator.class);
    syncConfig.setConvergenceCriterion(CompensatableDotProductCoGroup.AGGREGATOR_NAME, DiffL1NormConvergenceCriterion.class);
    syncConfig.setIterationId(ITERATION_ID);
   
    // --------------- the wiring ---------------------

    JobGraphUtils.connect(pageWithRankInput, head, ChannelType.NETWORK, DistributionPattern.BIPARTITE);

    JobGraphUtils.connect(head, intermediate, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);
    intermediateConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, 1);
   
    JobGraphUtils.connect(adjacencyListInput, intermediate, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
   
    JobGraphUtils.connect(head, tail, ChannelType.NETWORK, DistributionPattern.POINTWISE);
    JobGraphUtils.connect(intermediate, tail, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
    tailConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, 1);
    tailConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(1, degreeOfParallelism);

    JobGraphUtils.connect(head, output, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);
    JobGraphUtils.connect(tail, fakeTailOutput, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);

    JobGraphUtils.connect(head, sync, ChannelType.NETWORK, DistributionPattern.POINTWISE);
   
    fakeTailOutput.setVertexToShareInstancesWith(tail);
    tail.setVertexToShareInstancesWith(head);
    pageWithRankInput.setVertexToShareInstancesWith(head);
    adjacencyListInput.setVertexToShareInstancesWith(head);
    intermediate.setVertexToShareInstancesWith(head);
    output.setVertexToShareInstancesWith(head);
    sync.setVertexToShareInstancesWith(head);

    return jobGraph;
  }
View Full Code Here


    }
   
    // --------------------------- create the sync task ---------------------------
    final TaskConfig syncConfig;
    {
      final JobOutputVertex sync = new JobOutputVertex("Sync (" +
            iterNode.getNodeName() + ")", this.jobGraph);
      sync.setOutputClass(IterationSynchronizationSinkTask.class);
      sync.setNumberOfSubtasks(1);
      this.auxVertices.add(sync);
     
      syncConfig = new TaskConfig(sync.getConfiguration());
      syncConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, headVertex.getNumberOfSubtasks());
 
      // set the number of iteration / convergence criterion for the sync
      final int maxNumIterations = iterNode.getIterationNode().getIterationContract().getMaximumNumberOfIterations();
      if (maxNumIterations < 1) {
        throw new CompilerException("Cannot create workset iteration with unspecified maximum number of iterations.");
      }
      syncConfig.setNumberOfIterations(maxNumIterations);
     
      // connect the sync task
      try {
        headVertex.connectTo(sync, ChannelType.NETWORK, DistributionPattern.POINTWISE);
      } catch (JobGraphDefinitionException e) {
        throw new CompilerException("Bug: Cannot connect head vertex to sync task.");
      }
    }
   
    // ----------------------------- create the iteration tails -----------------------------
    // ----------------------- for next workset and solution set delta-----------------------

    {
      // we have three possible cases:
      // 1) Two tails, one for workset update, one for solution set update
      // 2) One tail for workset update, solution set update happens in an intermediate task
      // 3) One tail for solution set update, workset update happens in an intermediate task
     
      final PlanNode nextWorksetNode = iterNode.getNextWorkSetPlanNode();
      final PlanNode solutionDeltaNode = iterNode.getSolutionSetDeltaPlanNode();
     
      final boolean hasWorksetTail = nextWorksetNode.getOutgoingChannels().isEmpty();
      final boolean hasSolutionSetTail = (!iterNode.isImmediateSolutionSetUpdate()) || (!hasWorksetTail);
     
      {
        // get the vertex for the workset update
        final TaskConfig worksetTailConfig;
        JobTaskVertex nextWorksetVertex = (JobTaskVertex) this.vertices.get(nextWorksetNode);
        if (nextWorksetVertex == null) {
          // nextWorksetVertex is chained
          TaskInChain taskInChain = this.chainedTasks.get(nextWorksetNode);
          if (taskInChain == null) {
            throw new CompilerException("Bug: Next workset node not found as vertex or chained task.");
          }
          nextWorksetVertex = (JobTaskVertex) taskInChain.getContainingVertex();
          worksetTailConfig = taskInChain.getTaskConfig();
        } else {
          worksetTailConfig = new TaskConfig(nextWorksetVertex.getConfiguration());
        }
       
        // mark the node to perform workset updates
        worksetTailConfig.setIsWorksetIteration();
        worksetTailConfig.setIsWorksetUpdate();
       
        if (hasWorksetTail) {
          nextWorksetVertex.setTaskClass(IterationTailPactTask.class);
         
          worksetTailConfig.setOutputSerializer(iterNode.getWorksetSerializer());
          worksetTailConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
         
          // create the fake output task
          JobOutputVertex fakeTail = new JobOutputVertex("Fake Tail", this.jobGraph);
          fakeTail.setOutputClass(FakeOutputTask.class);
          fakeTail.setNumberOfSubtasks(headVertex.getNumberOfSubtasks());
          fakeTail.setNumberOfSubtasksPerInstance(headVertex.getNumberOfSubtasksPerInstance());
          this.auxVertices.add(fakeTail);
         
          // connect the fake tail
          try {
            nextWorksetVertex.connectTo(fakeTail, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);
          } catch (JobGraphDefinitionException e) {
            throw new CompilerException("Bug: Cannot connect iteration tail vertex fake tail task");
          }
        }
      }
      {
        final TaskConfig solutionDeltaConfig;
        JobTaskVertex solutionDeltaVertex = (JobTaskVertex) this.vertices.get(solutionDeltaNode);
        if (solutionDeltaVertex == null) {
          // last op is chained
          TaskInChain taskInChain = this.chainedTasks.get(solutionDeltaNode);
          if (taskInChain == null) {
            throw new CompilerException("Bug: Solution Set Delta not found as vertex or chained task.");
          }
          solutionDeltaVertex = (JobTaskVertex) taskInChain.getContainingVertex();
          solutionDeltaConfig = taskInChain.getTaskConfig();
        } else {
          solutionDeltaConfig = new TaskConfig(solutionDeltaVertex.getConfiguration());
        }
       
        solutionDeltaConfig.setIsWorksetIteration();
        solutionDeltaConfig.setIsSolutionSetUpdate();
       
        if (hasSolutionSetTail) {
          solutionDeltaVertex.setTaskClass(IterationTailPactTask.class);
         
          solutionDeltaConfig.setOutputSerializer(iterNode.getSolutionSetSerializer());
          solutionDeltaConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
 
          // create the fake output task
          JobOutputVertex fakeTail = new JobOutputVertex("Fake Tail", this.jobGraph);
          fakeTail.setOutputClass(FakeOutputTask.class);
          fakeTail.setNumberOfSubtasks(headVertex.getNumberOfSubtasks());
          fakeTail.setNumberOfSubtasksPerInstance(headVertex.getNumberOfSubtasksPerInstance());
          this.auxVertices.add(fakeTail);
         
          // connect the fake tail
          try {
            solutionDeltaVertex.connectTo(fakeTail, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);
View Full Code Here

    config.setOutputSerializer(node.getSerializer());
    return vertex;
  }

  private AbstractJobOutputVertex createDataSinkVertex(SinkPlanNode node) throws CompilerException {
    final JobOutputVertex vertex = new JobOutputVertex(node.getNodeName(), this.jobGraph);
    final TaskConfig config = new TaskConfig(vertex.getConfiguration());
   
    vertex.setOutputClass(DataSinkTask.class);
    vertex.getConfiguration().setInteger(DataSinkTask.DEGREE_OF_PARALLELISM_KEY, node.getDegreeOfParallelism());
   
    // set user code
    config.setStubWrapper(node.getPactContract().getUserCodeWrapper());
    config.setStubParameters(node.getPactContract().getParameters());
   
View Full Code Here

      throw new CompilerException("Bug: No memory has been assigned to the iteration back channel.");
    }
    headConfig.setBackChannelMemory(memForBackChannel);
   
    // --------------------------- create the sync task ---------------------------
    final JobOutputVertex sync = new JobOutputVertex("Sync(" +
          bulkNode.getNodeName() + ")", this.jobGraph);
    sync.setOutputClass(IterationSynchronizationSinkTask.class);
    sync.setNumberOfSubtasks(1);
    this.auxVertices.add(sync);
   
    final TaskConfig syncConfig = new TaskConfig(sync.getConfiguration());
    syncConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, headVertex.getNumberOfSubtasks());

    // set the number of iteration / convergence criterion for the sync
    final int maxNumIterations = bulkNode.getIterationNode().getIterationContract().getMaximumNumberOfIterations();
    if (maxNumIterations < 1) {
      throw new CompilerException("Cannot create bulk iteration with unspecified maximum number of iterations.");
    }
    syncConfig.setNumberOfIterations(maxNumIterations);
   
    // connect the sync task
    try {
      headVertex.connectTo(sync, ChannelType.NETWORK, DistributionPattern.POINTWISE);
    } catch (JobGraphDefinitionException e) {
      throw new CompilerException("Bug: Cannot connect head vertex to sync task.");
    }
   
   
    // ----------------------------- create the iteration tail ------------------------------
   
    final PlanNode rootOfTerminationCriterion = bulkNode.getRootOfTerminationCriterion();
    final PlanNode rootOfStepFunction = bulkNode.getRootOfStepFunction();
    final TaskConfig tailConfig;
   
    JobTaskVertex rootOfStepFunctionVertex = (JobTaskVertex) this.vertices.get(rootOfStepFunction);
    if (rootOfStepFunctionVertex == null) {
      // last op is chained
      final TaskInChain taskInChain = this.chainedTasks.get(rootOfStepFunction);
      if (taskInChain == null) {
        throw new CompilerException("Bug: Tail of step function not found as vertex or chained task.");
      }
      rootOfStepFunctionVertex = (JobTaskVertex) taskInChain.getContainingVertex();

      // the fake channel is statically typed to pact record. no data is sent over this channel anyways.
      tailConfig = taskInChain.getTaskConfig();
    } else {
      tailConfig = new TaskConfig(rootOfStepFunctionVertex.getConfiguration());
    }
   
    tailConfig.setIsWorksetUpdate();
   
    // No following termination criterion
    if(rootOfStepFunction.getOutgoingChannels().isEmpty()) {
     
      rootOfStepFunctionVertex.setTaskClass(IterationTailPactTask.class);
     
      tailConfig.setOutputSerializer(bulkNode.getSerializerForIterationChannel());
      tailConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
     
      // create the fake output task
      JobOutputVertex fakeTail = new JobOutputVertex("Fake Tail", this.jobGraph);
      fakeTail.setOutputClass(FakeOutputTask.class);
      fakeTail.setNumberOfSubtasks(headVertex.getNumberOfSubtasks());
      fakeTail.setNumberOfSubtasksPerInstance(headVertex.getNumberOfSubtasksPerInstance());
      this.auxVertices.add(fakeTail);
     
      // connect the fake tail
      try {
        rootOfStepFunctionVertex.connectTo(fakeTail, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);
      } catch (JobGraphDefinitionException e) {
        throw new CompilerException("Bug: Cannot connect iteration tail vertex fake tail task");
      }
     
    }
   
   
    // create the fake output task for termination criterion, if needed
    final TaskConfig tailConfigOfTerminationCriterion;
    // If we have a termination criterion and it is not an intermediate node
    if(rootOfTerminationCriterion != null && rootOfTerminationCriterion.getOutgoingChannels().isEmpty()) {
      JobTaskVertex rootOfTerminationCriterionVertex = (JobTaskVertex) this.vertices.get(rootOfTerminationCriterion);
     
     
      if (rootOfTerminationCriterionVertex == null) {
        // last op is chained
        final TaskInChain taskInChain = this.chainedTasks.get(rootOfTerminationCriterion);
        if (taskInChain == null) {
          throw new CompilerException("Bug: Tail of termination criterion not found as vertex or chained task.");
        }
        rootOfTerminationCriterionVertex = (JobTaskVertex) taskInChain.getContainingVertex();

        // the fake channel is statically typed to pact record. no data is sent over this channel anyways.
        tailConfigOfTerminationCriterion = taskInChain.getTaskConfig();
      } else {
        tailConfigOfTerminationCriterion = new TaskConfig(rootOfTerminationCriterionVertex.getConfiguration());
      }
     
      rootOfTerminationCriterionVertex.setTaskClass(IterationTailPactTask.class);
      // Hack
      tailConfigOfTerminationCriterion.setIsSolutionSetUpdate();
      tailConfigOfTerminationCriterion.setOutputSerializer(bulkNode.getSerializerForIterationChannel());
      tailConfigOfTerminationCriterion.addOutputShipStrategy(ShipStrategyType.FORWARD);
     
      JobOutputVertex fakeTailTerminationCriterion = new JobOutputVertex("Fake Tail for Termination Criterion", this.jobGraph);
      fakeTailTerminationCriterion.setOutputClass(FakeOutputTask.class);
      fakeTailTerminationCriterion.setNumberOfSubtasks(headVertex.getNumberOfSubtasks());
      fakeTailTerminationCriterion.setNumberOfSubtasksPerInstance(headVertex.getNumberOfSubtasksPerInstance());
      this.auxVertices.add(fakeTailTerminationCriterion);
   
      // connect the fake tail
      try {
        rootOfTerminationCriterionVertex.connectTo(fakeTailTerminationCriterion, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);
View Full Code Here

      tailConfig.addChainedTask(ChainedCollectorMapDriver.class, chainedMapperConfig, "Chained ID Mapper");
    }

    // - output ----------------------------------------------------------------------------------------------------
    JobOutputVertex output = JobGraphUtils.createFileOutput(jobGraph, "Output", numSubTasks, numSubTasks);
    TaskConfig outputConfig = new TaskConfig(output.getConfiguration());
    {
      outputConfig.addInputToGroup(0);
      outputConfig.setInputSerializer(serializer, 0);

      outputConfig.setStubWrapper(new UserCodeClassWrapper<PointOutFormat>(PointOutFormat.class));
      outputConfig.setStubParameter(FileOutputFormat.FILE_PARAMETER_KEY, outputPath);
    }

    // - fake tail -------------------------------------------------------------------------------------------------
    JobOutputVertex fakeTail = JobGraphUtils.createFakeOutput(jobGraph, "Fake Tail", numSubTasks, numSubTasks);

    // - sync ------------------------------------------------------------------------------------------------------
    JobOutputVertex sync = JobGraphUtils.createSync(jobGraph, numSubTasks);
    TaskConfig syncConfig = new TaskConfig(sync.getConfiguration());
    syncConfig.setNumberOfIterations(maxIterations);
    syncConfig.setIterationId(ITERATION_ID);

    // --------------------------------------------------------------------------------------------------------------
    // 2. EDGES
    // --------------------------------------------------------------------------------------------------------------
    JobGraphUtils.connect(input, head, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);

    JobGraphUtils.connect(head, tail, ChannelType.IN_MEMORY, DistributionPattern.BIPARTITE);
    tailConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, numSubTasks);

    JobGraphUtils.connect(head, output, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);

    JobGraphUtils.connect(head, sync, ChannelType.NETWORK, DistributionPattern.POINTWISE);

    JobGraphUtils.connect(tail, fakeTail, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);

    // --------------------------------------------------------------------------------------------------------------
    // 3. INSTANCE SHARING
    // --------------------------------------------------------------------------------------------------------------
    input.setVertexToShareInstancesWith(head);

    tail.setVertexToShareInstancesWith(head);

    output.setVertexToShareInstancesWith(head);

    sync.setVertexToShareInstancesWith(head);

    fakeTail.setVertexToShareInstancesWith(tail);

    return jobGraph;
  }
View Full Code Here

    taskVertex.setNumberOfSubtasksPerInstance(numSubtasksPerInstance);
    return taskVertex;
  }

  public static JobOutputVertex createSync(JobGraph jobGraph, int degreeOfParallelism) {
    JobOutputVertex sync = new JobOutputVertex("BulkIterationSync", jobGraph);
    sync.setOutputClass(IterationSynchronizationSinkTask.class);
    sync.setNumberOfSubtasks(1);
    TaskConfig syncConfig = new TaskConfig(sync.getConfiguration());
    syncConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, degreeOfParallelism);
    return sync;
  }
View Full Code Here

  }

  public static JobOutputVertex createFakeOutput(JobGraph jobGraph, String name, int degreeOfParallelism,
      int numSubTasksPerInstance)
  {
    JobOutputVertex outputVertex = new JobOutputVertex(name, jobGraph);
    outputVertex.setOutputClass(FakeOutputTask.class);
    outputVertex.setNumberOfSubtasks(degreeOfParallelism);
    outputVertex.setNumberOfSubtasksPerInstance(numSubTasksPerInstance);
    return outputVertex;
  }
View Full Code Here

  }

  public static JobOutputVertex createFileOutput(JobGraph jobGraph, String name, int degreeOfParallelism,
      int numSubTasksPerInstance)
  {
    JobOutputVertex sinkVertex = new JobOutputVertex(name, jobGraph);
    sinkVertex.setOutputClass(DataSinkTask.class);
    sinkVertex.setNumberOfSubtasks(degreeOfParallelism);
    sinkVertex.setNumberOfSubtasksPerInstance(numSubTasksPerInstance);
    return sinkVertex;
  }
View Full Code Here

    return modelsInput;
  }

  private static JobOutputVertex createOutput(JobGraph jobGraph, String resultPath, int numSubTasks, TypeSerializerFactory<?> serializer) {
   
    JobOutputVertex output = JobGraphUtils.createFileOutput(jobGraph, "Output", numSubTasks, numSubTasks);

    {
      TaskConfig taskConfig = new TaskConfig(output.getConfiguration());
      taskConfig.addInputToGroup(0);
      taskConfig.setInputSerializer(serializer, 0);

      PointOutFormat outFormat = new PointOutFormat();
      outFormat.setOutputFilePath(new Path(resultPath));
View Full Code Here

   
    return tail;
  }
 
  private static JobOutputVertex createSync(JobGraph jobGraph, int numIterations, int dop) {
    JobOutputVertex sync = JobGraphUtils.createSync(jobGraph, dop);
    TaskConfig syncConfig = new TaskConfig(sync.getConfiguration());
    syncConfig.setNumberOfIterations(numIterations);
    syncConfig.setIterationId(ITERATION_ID);
    return sync;
  }
View Full Code Here

TOP

Related Classes of eu.stratosphere.nephele.jobgraph.JobOutputVertex

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.