Examples of eu.stratosphere.nephele.jobgraph.JobTaskVertex

Package eu.stratosphere.nephele.jobgraph

Examples of eu.stratosphere.nephele.jobgraph.JobTaskVertex

eu.stratosphere.nephele.jobgraph.JobTaskVertex
A JobTaskVertex is the vertex type for regular tasks (with both input and output) in Nephele. Tasks running inside a JobTaskVertex must specify at least one record reader and one record writer.

    adjacencyListInputConfig.addOutputShipStrategy(ShipStrategyType.PARTITION_HASH);
    adjacencyListInputConfig.setOutputSerializer(recSerializer);
    adjacencyListInputConfig.setOutputComparator(fieldZeroComparator, 0);


    // --------------- the head ---------------------
    JobTaskVertex head = JobGraphUtils.createTask(IterationHeadPactTask.class, "IterationHead", jobGraph,
      degreeOfParallelism, numSubTasksPerInstance);
    TaskConfig headConfig = new TaskConfig(head.getConfiguration());
    headConfig.setIterationId(ITERATION_ID);
    
    // initial input / partial solution
    headConfig.addInputToGroup(0);
    headConfig.setIterationHeadPartialSolutionOrWorksetInputIndex(0);
    headConfig.setInputSerializer(recSerializer, 0);
    headConfig.setInputComparator(fieldZeroComparator, 0);
    headConfig.setInputLocalStrategy(0, LocalStrategy.SORT);
    headConfig.setMemoryInput(0, minorConsumer * JobGraphUtils.MEGABYTE);
    headConfig.setFilehandlesInput(0, NUM_FILE_HANDLES_PER_SORT);
    headConfig.setSpillingThresholdInput(0, SORT_SPILL_THRESHOLD);
    
    // back channel / iterations
    headConfig.setBackChannelMemory(minorConsumer * JobGraphUtils.MEGABYTE);
    
    // output into iteration
    headConfig.setOutputSerializer(recSerializer);
    headConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
    headConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
    
    // final output
    TaskConfig headFinalOutConfig = new TaskConfig(new Configuration());
    headFinalOutConfig.setOutputSerializer(recSerializer);
    headFinalOutConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
    headConfig.setIterationHeadFinalOutputConfig(headFinalOutConfig);
    
    // the sync
    headConfig.setIterationHeadIndexOfSyncOutput(3);
    headConfig.setNumberOfIterations(numIterations);
    
    // the driver 
    headConfig.setDriver(CollectorMapDriver.class);
    headConfig.setDriverStrategy(DriverStrategy.COLLECTOR_MAP);
    headConfig.setStubWrapper(new UserCodeClassWrapper<CompensatingMap>(CompensatingMap.class));
    headConfig.setStubParameter("pageRank.numVertices", String.valueOf(numVertices));
    headConfig.setStubParameter("compensation.failingWorker", failingWorkers);
    headConfig.setStubParameter("compensation.failingIteration", String.valueOf(failingIteration));
    headConfig.setStubParameter("compensation.messageLoss", String.valueOf(messageLoss));
    headConfig.addIterationAggregator(CompensatableDotProductCoGroup.AGGREGATOR_NAME, PageRankStatsAggregator.class);


    // --------------- the join ---------------------
    
    JobTaskVertex intermediate = JobGraphUtils.createTask(IterationIntermediatePactTask.class,
      "IterationIntermediate", jobGraph, degreeOfParallelism, numSubTasksPerInstance);
    TaskConfig intermediateConfig = new TaskConfig(intermediate.getConfiguration());
    intermediateConfig.setIterationId(ITERATION_ID);
//    intermediateConfig.setDriver(RepeatableHashjoinMatchDriverWithCachedBuildside.class);
    intermediateConfig.setDriver(BuildSecondCachedMatchDriver.class);
    intermediateConfig.setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_SECOND);
    intermediateConfig.setMemoryDriver(matchMemory * JobGraphUtils.MEGABYTE);
    intermediateConfig.addInputToGroup(0);
    intermediateConfig.addInputToGroup(1);
    intermediateConfig.setInputSerializer(recSerializer, 0);
    intermediateConfig.setInputSerializer(recSerializer, 1);
    intermediateConfig.setDriverComparator(fieldZeroComparator, 0);
    intermediateConfig.setDriverComparator(fieldZeroComparator, 1);
    intermediateConfig.setDriverPairComparator(pairComparatorFactory);
    
    intermediateConfig.setOutputSerializer(recSerializer);
    intermediateConfig.addOutputShipStrategy(ShipStrategyType.PARTITION_HASH);
    intermediateConfig.setOutputComparator(fieldZeroComparator, 0);
    
    intermediateConfig.setStubWrapper(new UserCodeClassWrapper<CompensatableDotProductMatch>(CompensatableDotProductMatch.class));
    intermediateConfig.setStubParameter("pageRank.numVertices", String.valueOf(numVertices));
    intermediateConfig.setStubParameter("compensation.failingWorker", failingWorkers);
    intermediateConfig.setStubParameter("compensation.failingIteration", String.valueOf(failingIteration));
    intermediateConfig.setStubParameter("compensation.messageLoss", String.valueOf(messageLoss));


    // ---------------- the tail (co group) --------------------
    
    JobTaskVertex tail = JobGraphUtils.createTask(IterationTailPactTask.class, "IterationTail", jobGraph,
      degreeOfParallelism, numSubTasksPerInstance);
    TaskConfig tailConfig = new TaskConfig(tail.getConfiguration());
    tailConfig.setIterationId(ITERATION_ID);
        tailConfig.setIsWorksetUpdate();
    // TODO we need to combine!
    
    // inputs and driver
    tailConfig.setDriver(CoGroupDriver.class);
    tailConfig.setDriverStrategy(DriverStrategy.CO_GROUP);
    tailConfig.addInputToGroup(0);
    tailConfig.addInputToGroup(1);
    tailConfig.setInputSerializer(recSerializer, 0);
    tailConfig.setInputSerializer(recSerializer, 1);
    tailConfig.setDriverComparator(fieldZeroComparator, 0);
    tailConfig.setDriverComparator(fieldZeroComparator, 1);
    tailConfig.setDriverPairComparator(pairComparatorFactory);
    tailConfig.setInputAsynchronouslyMaterialized(0, true);
    tailConfig.setInputMaterializationMemory(0, minorConsumer * JobGraphUtils.MEGABYTE);
    tailConfig.setInputLocalStrategy(1, LocalStrategy.SORT);
    tailConfig.setInputComparator(fieldZeroComparator, 1);
    tailConfig.setMemoryInput(1, coGroupSortMemory * JobGraphUtils.MEGABYTE);
    tailConfig.setFilehandlesInput(1, NUM_FILE_HANDLES_PER_SORT);
    tailConfig.setSpillingThresholdInput(1, SORT_SPILL_THRESHOLD);
    
    // output
    tailConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
    tailConfig.setOutputSerializer(recSerializer);
    
    // the stub
    tailConfig.setStubWrapper(new UserCodeClassWrapper<CompensatableDotProductCoGroup>(CompensatableDotProductCoGroup.class));
    tailConfig.setStubParameter("pageRank.numVertices", String.valueOf(numVertices));
    tailConfig.setStubParameter("pageRank.numDanglingVertices", String.valueOf(numDanglingVertices));
    tailConfig.setStubParameter("compensation.failingWorker", failingWorkers);
    tailConfig.setStubParameter("compensation.failingIteration", String.valueOf(failingIteration));
    tailConfig.setStubParameter("compensation.messageLoss", String.valueOf(messageLoss));
    
    // --------------- the output ---------------------


    JobOutputVertex output = JobGraphUtils.createFileOutput(jobGraph, "FinalOutput", degreeOfParallelism,
      numSubTasksPerInstance);
    TaskConfig outputConfig = new TaskConfig(output.getConfiguration());
    outputConfig.addInputToGroup(0);
    outputConfig.setInputSerializer(recSerializer, 0);
    outputConfig.setStubWrapper(new UserCodeClassWrapper<PageWithRankOutFormat>(PageWithRankOutFormat.class));
    outputConfig.setStubParameter(FileOutputFormat.FILE_PARAMETER_KEY, outputPath);
    
    // --------------- the auxiliaries ---------------------
    
    JobOutputVertex fakeTailOutput = JobGraphUtils.createFakeOutput(jobGraph, "FakeTailOutput",
      degreeOfParallelism, numSubTasksPerInstance);


    JobOutputVertex sync = JobGraphUtils.createSync(jobGraph, degreeOfParallelism);
    TaskConfig syncConfig = new TaskConfig(sync.getConfiguration());
    syncConfig.setNumberOfIterations(numIterations);
    syncConfig.addIterationAggregator(CompensatableDotProductCoGroup.AGGREGATOR_NAME, PageRankStatsAggregator.class);
    syncConfig.setConvergenceCriterion(CompensatableDotProductCoGroup.AGGREGATOR_NAME, DiffL1NormConvergenceCriterion.class);
    syncConfig.setIterationId(ITERATION_ID);
    
    // --------------- the wiring ---------------------


    JobGraphUtils.connect(pageWithRankInput, head, ChannelType.NETWORK, DistributionPattern.BIPARTITE);


    JobGraphUtils.connect(head, intermediate, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);
    intermediateConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, 1);
    
    JobGraphUtils.connect(adjacencyListInput, intermediate, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
    
    JobGraphUtils.connect(head, tail, ChannelType.NETWORK, DistributionPattern.POINTWISE);
    JobGraphUtils.connect(intermediate, tail, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
    tailConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, 1);
    tailConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(1, degreeOfParallelism);


    JobGraphUtils.connect(head, output, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);
    JobGraphUtils.connect(tail, fakeTailOutput, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);


    JobGraphUtils.connect(head, sync, ChannelType.NETWORK, DistributionPattern.POINTWISE);
    
    fakeTailOutput.setVertexToShareInstancesWith(tail);
    tail.setVertexToShareInstancesWith(head);
    pageWithRankInput.setVertexToShareInstancesWith(head);
    adjacencyListInput.setVertexToShareInstancesWith(head);
    intermediate.setVertexToShareInstancesWith(head);
    output.setVertexToShareInstancesWith(head);
    sync.setVertexToShareInstancesWith(head);

View Full Code Here

          chaining = false;
        }
      }
    }
    
    final JobTaskVertex vertex;
    final TaskConfig config;
    
    if (chaining) {
      vertex = null;
      config = new TaskConfig(new Configuration());
      this.chainedTasks.put(node, new TaskInChain(ds.getPushChainDriverClass(), config, taskName));
    } else {
      // create task vertex
      vertex = new JobTaskVertex(taskName, this.jobGraph);
      vertex.setTaskClass( (this.currentIteration != null && node.isOnDynamicPath()) ? IterationIntermediatePactTask.class : RegularPactTask.class);
      
      config = new TaskConfig(vertex.getConfiguration());
      config.setDriver(ds.getDriverClass());
    }
    
    // set user code
    config.setStubWrapper(node.getPactContract().getUserCodeWrapper());

View Full Code Here

  }


  private JobTaskVertex createDualInputVertex(DualInputPlanNode node) throws CompilerException {
    final String taskName = node.getNodeName();
    final DriverStrategy ds = node.getDriverStrategy();
    final JobTaskVertex vertex = new JobTaskVertex(taskName, this.jobGraph);
    final TaskConfig config = new TaskConfig(vertex.getConfiguration());
    vertex.setTaskClass( (this.currentIteration != null && node.isOnDynamicPath()) ? IterationIntermediatePactTask.class : RegularPactTask.class);
    
    // set user code
    config.setStubWrapper(node.getPactContract().getUserCodeWrapper());
    config.setStubParameters(node.getPactContract().getParameters());

View Full Code Here

    } else {
      merge = false;
    }
    
    // create or adopt the head vertex
    final JobTaskVertex toReturn;
    final JobTaskVertex headVertex;
    final TaskConfig headConfig;
    if (merge) {
      final PlanNode successor = pspn.getOutgoingChannels().get(0).getTarget();
      headVertex = (JobTaskVertex) this.vertices.get(successor);
      
      if (headVertex == null) {
        throw new CompilerException(
          "Bug: Trying to merge solution set with its sucessor, but successor has not been created.");
      }
      
      // reset the vertex type to iteration head
      headVertex.setTaskClass(IterationHeadPactTask.class);
      headConfig = new TaskConfig(headVertex.getConfiguration());
      toReturn = null;
    } else {
      // instantiate the head vertex and give it a no-op driver as the driver strategy.
      // everything else happens in the post visit, after the input (the initial partial solution)
      // is connected.
      headVertex = new JobTaskVertex("PartialSolution ("+iteration.getNodeName()+")", this.jobGraph);
      headVertex.setTaskClass(IterationHeadPactTask.class);
      headConfig = new TaskConfig(headVertex.getConfiguration());
      headConfig.setDriver(NoOpDriver.class);
      toReturn = headVertex;
    }
    
    // create the iteration descriptor and the iteration to it

View Full Code Here

    } else {
      merge = false;
    }
    
    // create or adopt the head vertex
    final JobTaskVertex toReturn;
    final JobTaskVertex headVertex;
    final TaskConfig headConfig;
    if (merge) {
      final PlanNode successor = wspn.getOutgoingChannels().get(0).getTarget();
      headVertex = (JobTaskVertex) this.vertices.get(successor);
      
      if (headVertex == null) {
        throw new CompilerException(
          "Bug: Trying to merge solution set with its sucessor, but successor has not been created.");
      }
      
      // reset the vertex type to iteration head
      headVertex.setTaskClass(IterationHeadPactTask.class);
      headConfig = new TaskConfig(headVertex.getConfiguration());
      toReturn = null;
    } else {
      // instantiate the head vertex and give it a no-op driver as the driver strategy.
      // everything else happens in the post visit, after the input (the initial partial solution)
      // is connected.
      headVertex = new JobTaskVertex("IterationHead("+iteration.getNodeName()+")", this.jobGraph);
      headVertex.setTaskClass(IterationHeadPactTask.class);
      headConfig = new TaskConfig(headVertex.getConfiguration());
      headConfig.setDriver(NoOpDriver.class);
      toReturn = headVertex;
    }
    
    // create the iteration descriptor and the iteration to it

View Full Code Here

  }
  
  private void finalizeBulkIteration(IterationDescriptor descr) {
    
    final BulkIterationPlanNode bulkNode = (BulkIterationPlanNode) descr.getIterationNode();
    final JobTaskVertex headVertex = descr.getHeadTask();
    final TaskConfig headConfig = new TaskConfig(headVertex.getConfiguration());
    final TaskConfig headFinalOutputConfig = descr.getHeadFinalResultConfig();
    
    // ------------ finalize the head config with the final outputs and the sync gate ------------
    final int numStepFunctionOuts = headConfig.getNumOutputs();
    final int numFinalOuts = headFinalOutputConfig.getNumOutputs();
    headConfig.setIterationHeadFinalOutputConfig(headFinalOutputConfig);
    headConfig.setIterationHeadIndexOfSyncOutput(numStepFunctionOuts + numFinalOuts);
    final long memForBackChannel = bulkNode.getMemoryPerSubTask();
    if (memForBackChannel <= 0) {
      throw new CompilerException("Bug: No memory has been assigned to the iteration back channel.");
    }
    headConfig.setBackChannelMemory(memForBackChannel);
    
    // --------------------------- create the sync task ---------------------------
    final JobOutputVertex sync = new JobOutputVertex("Sync(" +
          bulkNode.getNodeName() + ")", this.jobGraph);
    sync.setOutputClass(IterationSynchronizationSinkTask.class);
    sync.setNumberOfSubtasks(1);
    this.auxVertices.add(sync);
    
    final TaskConfig syncConfig = new TaskConfig(sync.getConfiguration());
    syncConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, headVertex.getNumberOfSubtasks());


    // set the number of iteration / convergence criterion for the sync
    final int maxNumIterations = bulkNode.getIterationNode().getIterationContract().getMaximumNumberOfIterations();
    if (maxNumIterations < 1) {
      throw new CompilerException("Cannot create bulk iteration with unspecified maximum number of iterations.");
    }
    syncConfig.setNumberOfIterations(maxNumIterations);
    
    // connect the sync task
    try {
      headVertex.connectTo(sync, ChannelType.NETWORK, DistributionPattern.POINTWISE);
    } catch (JobGraphDefinitionException e) {
      throw new CompilerException("Bug: Cannot connect head vertex to sync task.");
    }
    
    
    // ----------------------------- create the iteration tail ------------------------------
    
    final PlanNode rootOfTerminationCriterion = bulkNode.getRootOfTerminationCriterion();
    final PlanNode rootOfStepFunction = bulkNode.getRootOfStepFunction();
    final TaskConfig tailConfig;
    
    JobTaskVertex rootOfStepFunctionVertex = (JobTaskVertex) this.vertices.get(rootOfStepFunction);
    if (rootOfStepFunctionVertex == null) {
      // last op is chained
      final TaskInChain taskInChain = this.chainedTasks.get(rootOfStepFunction);
      if (taskInChain == null) {
        throw new CompilerException("Bug: Tail of step function not found as vertex or chained task.");
      }
      rootOfStepFunctionVertex = (JobTaskVertex) taskInChain.getContainingVertex();


      // the fake channel is statically typed to pact record. no data is sent over this channel anyways.
      tailConfig = taskInChain.getTaskConfig();
    } else {
      tailConfig = new TaskConfig(rootOfStepFunctionVertex.getConfiguration());
    }
    
    tailConfig.setIsWorksetUpdate();
    
    // No following termination criterion
    if(rootOfStepFunction.getOutgoingChannels().isEmpty()) {
      
      rootOfStepFunctionVertex.setTaskClass(IterationTailPactTask.class);
      
      tailConfig.setOutputSerializer(bulkNode.getSerializerForIterationChannel());
      tailConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
      
      // create the fake output task
      JobOutputVertex fakeTail = new JobOutputVertex("Fake Tail", this.jobGraph);
      fakeTail.setOutputClass(FakeOutputTask.class);
      fakeTail.setNumberOfSubtasks(headVertex.getNumberOfSubtasks());
      fakeTail.setNumberOfSubtasksPerInstance(headVertex.getNumberOfSubtasksPerInstance());
      this.auxVertices.add(fakeTail);
      
      // connect the fake tail
      try {
        rootOfStepFunctionVertex.connectTo(fakeTail, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);
      } catch (JobGraphDefinitionException e) {
        throw new CompilerException("Bug: Cannot connect iteration tail vertex fake tail task");
      }
      
    }
    
    
    // create the fake output task for termination criterion, if needed
    final TaskConfig tailConfigOfTerminationCriterion;
    // If we have a termination criterion and it is not an intermediate node
    if(rootOfTerminationCriterion != null && rootOfTerminationCriterion.getOutgoingChannels().isEmpty()) {
      JobTaskVertex rootOfTerminationCriterionVertex = (JobTaskVertex) this.vertices.get(rootOfTerminationCriterion);
      
      
      if (rootOfTerminationCriterionVertex == null) {
        // last op is chained
        final TaskInChain taskInChain = this.chainedTasks.get(rootOfTerminationCriterion);
        if (taskInChain == null) {
          throw new CompilerException("Bug: Tail of termination criterion not found as vertex or chained task.");
        }
        rootOfTerminationCriterionVertex = (JobTaskVertex) taskInChain.getContainingVertex();


        // the fake channel is statically typed to pact record. no data is sent over this channel anyways.
        tailConfigOfTerminationCriterion = taskInChain.getTaskConfig();
      } else {
        tailConfigOfTerminationCriterion = new TaskConfig(rootOfTerminationCriterionVertex.getConfiguration());
      }
      
      rootOfTerminationCriterionVertex.setTaskClass(IterationTailPactTask.class);
      // Hack
      tailConfigOfTerminationCriterion.setIsSolutionSetUpdate();
      tailConfigOfTerminationCriterion.setOutputSerializer(bulkNode.getSerializerForIterationChannel());
      tailConfigOfTerminationCriterion.addOutputShipStrategy(ShipStrategyType.FORWARD);
      
      JobOutputVertex fakeTailTerminationCriterion = new JobOutputVertex("Fake Tail for Termination Criterion", this.jobGraph);
      fakeTailTerminationCriterion.setOutputClass(FakeOutputTask.class);
      fakeTailTerminationCriterion.setNumberOfSubtasks(headVertex.getNumberOfSubtasks());
      fakeTailTerminationCriterion.setNumberOfSubtasksPerInstance(headVertex.getNumberOfSubtasksPerInstance());
      this.auxVertices.add(fakeTailTerminationCriterion);
    
      // connect the fake tail
      try {
        rootOfTerminationCriterionVertex.connectTo(fakeTailTerminationCriterion, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);
      } catch (JobGraphDefinitionException e) {
        throw new CompilerException("Bug: Cannot connect iteration tail vertex fake tail task for termination criterion");
      }
      
      // tell the head that it needs to wait for the solution set updates

View Full Code Here

    }
  }
  
  private void finalizeWorksetIteration(IterationDescriptor descr) {
    final WorksetIterationPlanNode iterNode = (WorksetIterationPlanNode) descr.getIterationNode();
    final JobTaskVertex headVertex = descr.getHeadTask();
    final TaskConfig headConfig = new TaskConfig(headVertex.getConfiguration());
    final TaskConfig headFinalOutputConfig = descr.getHeadFinalResultConfig();
    
    // ------------ finalize the head config with the final outputs and the sync gate ------------
    {
      final int numStepFunctionOuts = headConfig.getNumOutputs();
      final int numFinalOuts = headFinalOutputConfig.getNumOutputs();
      headConfig.setIterationHeadFinalOutputConfig(headFinalOutputConfig);
      headConfig.setIterationHeadIndexOfSyncOutput(numStepFunctionOuts + numFinalOuts);
      final long mem = iterNode.getMemoryPerSubTask();
      if (mem <= 0) {
        throw new CompilerException("Bug: No memory has been assigned to the workset iteration.");
      }
      
      headConfig.setIsWorksetIteration();
      headConfig.setBackChannelMemory(mem / 2);
      headConfig.setSolutionSetMemory(mem / 2);
      
      // set the solution set serializer and comparator
      headConfig.setSolutionSetSerializer(iterNode.getSolutionSetSerializer());
      headConfig.setSolutionSetComparator(iterNode.getSolutionSetComparator());
    }
    
    // --------------------------- create the sync task ---------------------------
    final TaskConfig syncConfig;
    {
      final JobOutputVertex sync = new JobOutputVertex("Sync (" +
            iterNode.getNodeName() + ")", this.jobGraph);
      sync.setOutputClass(IterationSynchronizationSinkTask.class);
      sync.setNumberOfSubtasks(1);
      this.auxVertices.add(sync);
      
      syncConfig = new TaskConfig(sync.getConfiguration());
      syncConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, headVertex.getNumberOfSubtasks());
  
      // set the number of iteration / convergence criterion for the sync
      final int maxNumIterations = iterNode.getIterationNode().getIterationContract().getMaximumNumberOfIterations();
      if (maxNumIterations < 1) {
        throw new CompilerException("Cannot create workset iteration with unspecified maximum number of iterations.");
      }
      syncConfig.setNumberOfIterations(maxNumIterations);
      
      // connect the sync task
      try {
        headVertex.connectTo(sync, ChannelType.NETWORK, DistributionPattern.POINTWISE);
      } catch (JobGraphDefinitionException e) {
        throw new CompilerException("Bug: Cannot connect head vertex to sync task.");
      }
    }
    
    // ----------------------------- create the iteration tails -----------------------------
    // ----------------------- for next workset and solution set delta-----------------------


    {
      // we have three possible cases:
      // 1) Two tails, one for workset update, one for solution set update
      // 2) One tail for workset update, solution set update happens in an intermediate task
      // 3) One tail for solution set update, workset update happens in an intermediate task
      
      final PlanNode nextWorksetNode = iterNode.getNextWorkSetPlanNode();
      final PlanNode solutionDeltaNode = iterNode.getSolutionSetDeltaPlanNode();
      
      final boolean hasWorksetTail = nextWorksetNode.getOutgoingChannels().isEmpty();
      final boolean hasSolutionSetTail = (!iterNode.isImmediateSolutionSetUpdate()) || (!hasWorksetTail);
      
      {
        // get the vertex for the workset update
        final TaskConfig worksetTailConfig;
        JobTaskVertex nextWorksetVertex = (JobTaskVertex) this.vertices.get(nextWorksetNode);
        if (nextWorksetVertex == null) {
          // nextWorksetVertex is chained
          TaskInChain taskInChain = this.chainedTasks.get(nextWorksetNode);
          if (taskInChain == null) {
            throw new CompilerException("Bug: Next workset node not found as vertex or chained task.");
          }
          nextWorksetVertex = (JobTaskVertex) taskInChain.getContainingVertex();
          worksetTailConfig = taskInChain.getTaskConfig();
        } else {
          worksetTailConfig = new TaskConfig(nextWorksetVertex.getConfiguration());
        }
        
        // mark the node to perform workset updates
        worksetTailConfig.setIsWorksetIteration();
        worksetTailConfig.setIsWorksetUpdate();
        
        if (hasWorksetTail) {
          nextWorksetVertex.setTaskClass(IterationTailPactTask.class);
          
          worksetTailConfig.setOutputSerializer(iterNode.getWorksetSerializer());
          worksetTailConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
          
          // create the fake output task
          JobOutputVertex fakeTail = new JobOutputVertex("Fake Tail", this.jobGraph);
          fakeTail.setOutputClass(FakeOutputTask.class);
          fakeTail.setNumberOfSubtasks(headVertex.getNumberOfSubtasks());
          fakeTail.setNumberOfSubtasksPerInstance(headVertex.getNumberOfSubtasksPerInstance());
          this.auxVertices.add(fakeTail);
          
          // connect the fake tail
          try {
            nextWorksetVertex.connectTo(fakeTail, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);
          } catch (JobGraphDefinitionException e) {
            throw new CompilerException("Bug: Cannot connect iteration tail vertex fake tail task");
          }
        }
      }
      {
        final TaskConfig solutionDeltaConfig;
        JobTaskVertex solutionDeltaVertex = (JobTaskVertex) this.vertices.get(solutionDeltaNode);
        if (solutionDeltaVertex == null) {
          // last op is chained
          TaskInChain taskInChain = this.chainedTasks.get(solutionDeltaNode);
          if (taskInChain == null) {
            throw new CompilerException("Bug: Solution Set Delta not found as vertex or chained task.");
          }
          solutionDeltaVertex = (JobTaskVertex) taskInChain.getContainingVertex();
          solutionDeltaConfig = taskInChain.getTaskConfig();
        } else {
          solutionDeltaConfig = new TaskConfig(solutionDeltaVertex.getConfiguration());
        }
        
        solutionDeltaConfig.setIsWorksetIteration();
        solutionDeltaConfig.setIsSolutionSetUpdate();
        
        if (hasSolutionSetTail) {
          solutionDeltaVertex.setTaskClass(IterationTailPactTask.class);
          
          solutionDeltaConfig.setOutputSerializer(iterNode.getSolutionSetSerializer());
          solutionDeltaConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
  
          // create the fake output task
          JobOutputVertex fakeTail = new JobOutputVertex("Fake Tail", this.jobGraph);
          fakeTail.setOutputClass(FakeOutputTask.class);
          fakeTail.setNumberOfSubtasks(headVertex.getNumberOfSubtasks());
          fakeTail.setNumberOfSubtasksPerInstance(headVertex.getNumberOfSubtasksPerInstance());
          this.auxVertices.add(fakeTail);
          
          // connect the fake tail
          try {
            solutionDeltaVertex.connectTo(fakeTail, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);
          } catch (JobGraphDefinitionException e) {
            throw new CompilerException("Bug: Cannot connect iteration tail vertex fake tail task");
          }
          
          // tell the head that it needs to wait for the solution set updates

View Full Code Here

      inputConfig.setOutputSerializer(serializer);
      inputConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
    }


    // - head ------------------------------------------------------------------------------------------------------
    JobTaskVertex head = JobGraphUtils.createTask(
      IterationHeadPactTask.class, "Iteration Head", jobGraph, numSubTasks, numSubTasks);
    TaskConfig headConfig = new TaskConfig(head.getConfiguration());
    {
      headConfig.setIterationId(ITERATION_ID);


      // input to iteration head
      headConfig.addInputToGroup(0);
      headConfig.setInputSerializer(serializer, 0);
      headConfig.setInputLocalStrategy(0, LocalStrategy.NONE);
      headConfig.setIterationHeadPartialSolutionOrWorksetInputIndex(0);


      // output into iteration
      headConfig.setOutputSerializer(serializer);
      headConfig.addOutputShipStrategy(ShipStrategyType.PARTITION_HASH);
      headConfig.setOutputComparator(comparator, 0);


      // final output
      TaskConfig headFinalOutConfig = new TaskConfig(new Configuration());
      headFinalOutConfig.setOutputSerializer(serializer);
      headFinalOutConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
      headConfig.setIterationHeadFinalOutputConfig(headFinalOutConfig);


      // the sync
      headConfig.setIterationHeadIndexOfSyncOutput(2);


      // driver
      headConfig.setDriver(CollectorMapDriver.class);
      headConfig.setDriverStrategy(DriverStrategy.COLLECTOR_MAP);
      headConfig.setStubWrapper(new UserCodeClassWrapper<DummyMapper>(DummyMapper.class));


      // back channel
      headConfig.setBackChannelMemory(MEM_PER_CONSUMER * JobGraphUtils.MEGABYTE);
    }


    // - tail ------------------------------------------------------------------------------------------------------
    JobTaskVertex tail = JobGraphUtils.createTask(
      IterationTailPactTask.class, "Chained Iteration Tail", jobGraph, numSubTasks, numSubTasks);
    TaskConfig tailConfig = new TaskConfig(tail.getConfiguration());
    {
      tailConfig.setIterationId(ITERATION_ID);


      // inputs and driver
      tailConfig.addInputToGroup(0);
      tailConfig.setInputSerializer(serializer, 0);


      // output
      tailConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
      tailConfig.setOutputSerializer(serializer);


      // the driver
      tailConfig.setDriver(GroupReduceDriver.class);
      tailConfig.setDriverStrategy(DriverStrategy.SORTED_GROUP_REDUCE);
      tailConfig.setDriverComparator(comparator, 0);
      tailConfig.setStubWrapper(new UserCodeClassWrapper<DummyReducer>(DummyReducer.class));


      // chained mapper
      TaskConfig chainedMapperConfig = new TaskConfig(new Configuration());
      chainedMapperConfig.setDriverStrategy(DriverStrategy.COLLECTOR_MAP);
      chainedMapperConfig.setStubWrapper(new UserCodeClassWrapper<IncrementCoordinatesMapper>(
        IncrementCoordinatesMapper.class));


      chainedMapperConfig.setInputLocalStrategy(0, LocalStrategy.NONE);
      chainedMapperConfig.setInputSerializer(serializer, 0);


      chainedMapperConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
      chainedMapperConfig.setOutputSerializer(serializer);


      chainedMapperConfig.setIsWorksetUpdate();


      tailConfig.addChainedTask(ChainedCollectorMapDriver.class, chainedMapperConfig, "Chained ID Mapper");
    }


    // - output ----------------------------------------------------------------------------------------------------
    JobOutputVertex output = JobGraphUtils.createFileOutput(jobGraph, "Output", numSubTasks, numSubTasks);
    TaskConfig outputConfig = new TaskConfig(output.getConfiguration());
    {
      outputConfig.addInputToGroup(0);
      outputConfig.setInputSerializer(serializer, 0);


      outputConfig.setStubWrapper(new UserCodeClassWrapper<PointOutFormat>(PointOutFormat.class));
      outputConfig.setStubParameter(FileOutputFormat.FILE_PARAMETER_KEY, outputPath);
    }


    // - fake tail -------------------------------------------------------------------------------------------------
    JobOutputVertex fakeTail = JobGraphUtils.createFakeOutput(jobGraph, "Fake Tail", numSubTasks, numSubTasks);


    // - sync ------------------------------------------------------------------------------------------------------
    JobOutputVertex sync = JobGraphUtils.createSync(jobGraph, numSubTasks);
    TaskConfig syncConfig = new TaskConfig(sync.getConfiguration());
    syncConfig.setNumberOfIterations(maxIterations);
    syncConfig.setIterationId(ITERATION_ID);


    // --------------------------------------------------------------------------------------------------------------
    // 2. EDGES
    // --------------------------------------------------------------------------------------------------------------
    JobGraphUtils.connect(input, head, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);


    JobGraphUtils.connect(head, tail, ChannelType.IN_MEMORY, DistributionPattern.BIPARTITE);
    tailConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, numSubTasks);


    JobGraphUtils.connect(head, output, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);


    JobGraphUtils.connect(head, sync, ChannelType.NETWORK, DistributionPattern.POINTWISE);


    JobGraphUtils.connect(tail, fakeTail, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);


    // --------------------------------------------------------------------------------------------------------------
    // 3. INSTANCE SHARING
    // --------------------------------------------------------------------------------------------------------------
    input.setVertexToShareInstancesWith(head);


    tail.setVertexToShareInstancesWith(head);


    output.setVertexToShareInstancesWith(head);


    sync.setVertexToShareInstancesWith(head);

View Full Code Here

  }


  public static JobTaskVertex createTask(@SuppressWarnings("rawtypes") Class<? extends RegularPactTask> task, String name, JobGraph graph,
      int degreeOfParallelism, int numSubtasksPerInstance)
  {
    JobTaskVertex taskVertex = new JobTaskVertex(name, graph);
    taskVertex.setTaskClass(task);
    taskVertex.setNumberOfSubtasks(degreeOfParallelism);
    taskVertex.setNumberOfSubtasksPerInstance(numSubtasksPerInstance);
    return taskVertex;
  }

View Full Code Here


    return output;
  }
  
  private static JobTaskVertex createIterationHead(JobGraph jobGraph, int numSubTasks, TypeSerializerFactory<?> serializer) {
    JobTaskVertex head = JobGraphUtils.createTask(IterationHeadPactTask.class, "Iteration Head", jobGraph, numSubTasks, numSubTasks);


    TaskConfig headConfig = new TaskConfig(head.getConfiguration());
    headConfig.setIterationId(ITERATION_ID);
    
    // initial input / partial solution
    headConfig.addInputToGroup(0);
    headConfig.setIterationHeadPartialSolutionOrWorksetInputIndex(0);

View Full Code Here

0 1 2 3

TOP

Related Classes of eu.stratosphere.nephele.jobgraph.JobTaskVertex

eu.stratosphere.compiler.plantranslate.NepheleJobGraphGenerator

eu.stratosphere.nephele.executiongraph.ExecutionGraphTest

eu.stratosphere.nephele.jobmanager.JobManagerITCase

eu.stratosphere.test.broadcastvars.BroadcastVarsNepheleITCase

eu.stratosphere.test.broadcastvars.KMeansIterativeNepheleITCase

eu.stratosphere.test.iterative.nephele.ConnectedComponentsNepheleITCase

eu.stratosphere.test.iterative.nephele.customdanglingpagerank.CustomCompensatableDanglingPageRank

eu.stratosphere.test.iterative.nephele.customdanglingpagerank.CustomCompensatableDanglingPageRankWithCombiner

eu.stratosphere.test.iterative.nephele.danglingpagerank.CompensatableDanglingPageRank

eu.stratosphere.test.iterative.nephele.IterationWithChainingNepheleITCase

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.