Package org.apache.flink.compiler.plan

Examples of org.apache.flink.compiler.plan.BulkIterationPlanNode


      }
      else if (node instanceof SourcePlanNode) {
        vertex = createDataSourceVertex((SourcePlanNode) node);
      }
      else if (node instanceof BulkIterationPlanNode) {
        BulkIterationPlanNode iterationNode = (BulkIterationPlanNode) node;
        // for the bulk iteration, we skip creating anything for now. we create the graph
        // for the step function in the post visit.
       
        // check that the root of the step function has the same DOP as the iteration.
        // because the tail must have the same DOP as the head, we can only merge the last
        // operator with the tail, if they have the same DOP. not merging is currently not
        // implemented
        PlanNode root = iterationNode.getRootOfStepFunction();
        if (root.getDegreeOfParallelism() != node.getDegreeOfParallelism())
        {
          throw new CompilerException("Error: The final operator of the step " +
              "function has a different degree of parallelism than the iteration operator itself.");
        }
       
        IterationDescriptor descr = new IterationDescriptor(iterationNode, this.iterationIdEnumerator++);
        this.iterations.put(iterationNode, descr);
        vertex = null;
      }
      else if (node instanceof WorksetIterationPlanNode) {
        WorksetIterationPlanNode iterationNode = (WorksetIterationPlanNode) node;

        // we have the same constraints as for the bulk iteration
        PlanNode nextWorkSet = iterationNode.getNextWorkSetPlanNode();
        PlanNode solutionSetDelta  = iterationNode.getSolutionSetDeltaPlanNode();
       
        if (nextWorkSet.getDegreeOfParallelism() != node.getDegreeOfParallelism())
        {
          throw new CompilerException("It is currently not supported that the final operator of the step " +
              "function has a different degree of parallelism than the iteration operator itself.");
        }
        if (solutionSetDelta.getDegreeOfParallelism() != node.getDegreeOfParallelism())
        {
          throw new CompilerException("It is currently not supported that the final operator of the step " +
              "function has a different degree of parallelism than the iteration operator itself.");
        }
       
        IterationDescriptor descr = new IterationDescriptor(iterationNode, this.iterationIdEnumerator++);
        this.iterations.put(iterationNode, descr);
        vertex = null;
      }
      else if (node instanceof SingleInputPlanNode) {
        vertex = createSingleInputVertex((SingleInputPlanNode) node);
      }
      else if (node instanceof DualInputPlanNode) {
        vertex = createDualInputVertex((DualInputPlanNode) node);
      }
      else if (node instanceof NAryUnionPlanNode) {
        // skip the union for now
        vertex = null;
      }
      else if (node instanceof BulkPartialSolutionPlanNode) {
        // create a head node (or not, if it is merged into its successor)
        vertex = createBulkIterationHead((BulkPartialSolutionPlanNode) node);
      }
      else if (node instanceof SolutionSetPlanNode) {
        // this represents an access into the solution set index.
        // we do not create a vertex for the solution set here (we create the head at the workset place holder)
       
        // we adjust the joins / cogroups that go into the solution set here
        for (Channel c : node.getOutgoingChannels()) {
          DualInputPlanNode target = (DualInputPlanNode) c.getTarget();
          AbstractJobVertex accessingVertex = this.vertices.get(target);
          TaskConfig conf = new TaskConfig(accessingVertex.getConfiguration());
          int inputNum = c == target.getInput1() ? 0 : c == target.getInput2() ? 1 : -1;
         
          // sanity checks
          if (inputNum == -1) {
            throw new CompilerException();
          }
         
          // adjust the driver
          if (conf.getDriver().equals(MatchDriver.class)) {
            conf.setDriver(inputNum == 0 ? JoinWithSolutionSetFirstDriver.class : JoinWithSolutionSetSecondDriver.class);
          }
          else if (conf.getDriver().equals(CoGroupDriver.class)) {
            conf.setDriver(inputNum == 0 ? CoGroupWithSolutionSetFirstDriver.class : CoGroupWithSolutionSetSecondDriver.class);
          }
          else {
            throw new CompilerException("Found join with solution set using incompatible operator (only Join/CoGroup are valid).");
          }
        }
       
        // make sure we do not visit this node again. for that, we add a 'already seen' entry into one of the sets
        this.chainedTasks.put(node, ALREADY_VISITED_PLACEHOLDER);
       
        vertex = null;
      }
      else if (node instanceof WorksetPlanNode) {
        // create the iteration head here
        vertex = createWorksetIterationHead((WorksetPlanNode) node);
      }
      else {
        throw new CompilerException("Unrecognized node type: " + node.getClass().getName());
      }
    }
    catch (Exception e) {
      throw new CompilerException("Error translating node '" + node + "': " + e.getMessage(), e);
    }
   
    // check if a vertex was created, or if it was chained or skipped
    if (vertex != null) {
      // set degree of parallelism
      int pd = node.getDegreeOfParallelism();
      vertex.setParallelism(pd);
     
      vertex.setSlotSharingGroup(sharingGroup);
     
      // check whether this vertex is part of an iteration step function
      if (this.currentIteration != null) {
        // check that the task has the same DOP as the iteration as such
        PlanNode iterationNode = (PlanNode) this.currentIteration;
        if (iterationNode.getDegreeOfParallelism() < pd) {
          throw new CompilerException("Error: All functions that are part of an iteration must have the same, or a lower, degree-of-parallelism than the iteration operator.");
        }

        // store the id of the iterations the step functions participate in
        IterationDescriptor descr = this.iterations.get(this.currentIteration);
View Full Code Here


    }
    else if (inputPlanNode instanceof BulkPartialSolutionPlanNode) {
      if (this.vertices.get(inputPlanNode) == null) {
        // merged iteration head
        final BulkPartialSolutionPlanNode pspn = (BulkPartialSolutionPlanNode) inputPlanNode;
        final BulkIterationPlanNode iterationNode = pspn.getContainingIterationNode();
       
        // check if the iteration's input is a union
        if (iterationNode.getInput().getSource() instanceof NAryUnionPlanNode) {
          allInChannels = ((NAryUnionPlanNode) iterationNode.getInput().getSource()).getInputs().iterator();
        } else {
          allInChannels = Collections.singletonList(iterationNode.getInput()).iterator();
        }
       
        // also, set the index of the gate with the partial solution
        targetVertexConfig.setIterationHeadPartialSolutionOrWorksetInputIndex(inputIndex);
      } else {
        // standalone iteration head
        allInChannels = Collections.singletonList(input).iterator();
      }
    } else if (inputPlanNode instanceof WorksetPlanNode) {
      if (this.vertices.get(inputPlanNode) == null) {
        // merged iteration head
        final WorksetPlanNode wspn = (WorksetPlanNode) inputPlanNode;
        final WorksetIterationPlanNode iterationNode = wspn.getContainingIterationNode();
       
        // check if the iteration's input is a union
        if (iterationNode.getInput2().getSource() instanceof NAryUnionPlanNode) {
          allInChannels = ((NAryUnionPlanNode) iterationNode.getInput2().getSource()).getInputs().iterator();
        } else {
          allInChannels = Collections.singletonList(iterationNode.getInput2()).iterator();
        }
       
        // also, set the index of the gate with the partial solution
        targetVertexConfig.setIterationHeadPartialSolutionOrWorksetInputIndex(inputIndex);
      } else {
View Full Code Here

        }
      }
      // cannot chain the nodes that produce the next workset in a bulk iteration if a termination criterion follows
      if (this.currentIteration != null && this.currentIteration instanceof BulkIterationPlanNode)
      {
        BulkIterationPlanNode wspn = (BulkIterationPlanNode) this.currentIteration;
        if (node == wspn.getRootOfTerminationCriterion() && wspn.getRootOfStepFunction() == pred){
          chaining = false;
        }else if(node.getOutgoingChannels().size() > 0 &&(wspn.getRootOfStepFunction() == pred ||
            wspn.getRootOfTerminationCriterion() == pred)) {
          chaining = false;
        }
      }
    }
   
View Full Code Here

    return vertex;
  }
 
  private AbstractJobVertex createBulkIterationHead(BulkPartialSolutionPlanNode pspn) {
    // get the bulk iteration that corresponds to this partial solution node
    final BulkIterationPlanNode iteration = pspn.getContainingIterationNode();
   
    // check whether we need an individual vertex for the partial solution, or whether we
    // attach ourselves to the vertex of the parent node. We can combine the head with a node of
    // the step function, if
    // 1) There is one parent that the partial solution connects to via a forward pattern and no
    //    local strategy
    // 2) DOP and the number of subtasks per instance does not change
    // 3) That successor is not a union
    // 4) That successor is not itself the last node of the step function
    // 5) There is no local strategy on the edge for the initial partial solution, as
    //    this translates to a local strategy that would only be executed in the first iteration
   
    final boolean merge;
    if (mergeIterationAuxTasks && pspn.getOutgoingChannels().size() == 1) {
      final Channel c = pspn.getOutgoingChannels().get(0);
      final PlanNode successor = c.getTarget();
      merge = c.getShipStrategy() == ShipStrategyType.FORWARD &&
          c.getLocalStrategy() == LocalStrategy.NONE &&
          c.getTempMode() == TempMode.NONE &&
          successor.getDegreeOfParallelism() == pspn.getDegreeOfParallelism() &&
          !(successor instanceof NAryUnionPlanNode) &&
          successor != iteration.getRootOfStepFunction() &&
          iteration.getInput().getLocalStrategy() == LocalStrategy.NONE;
    } else {
      merge = false;
    }
   
    // create or adopt the head vertex
    final AbstractJobVertex toReturn;
    final AbstractJobVertex headVertex;
    final TaskConfig headConfig;
    if (merge) {
      final PlanNode successor = pspn.getOutgoingChannels().get(0).getTarget();
      headVertex = (AbstractJobVertex) this.vertices.get(successor);
     
      if (headVertex == null) {
        throw new CompilerException(
          "Bug: Trying to merge solution set with its sucessor, but successor has not been created.");
      }
     
      // reset the vertex type to iteration head
      headVertex.setInvokableClass(IterationHeadPactTask.class);
      headConfig = new TaskConfig(headVertex.getConfiguration());
      toReturn = null;
    } else {
      // instantiate the head vertex and give it a no-op driver as the driver strategy.
      // everything else happens in the post visit, after the input (the initial partial solution)
      // is connected.
      headVertex = new AbstractJobVertex("PartialSolution ("+iteration.getNodeName()+")");
      headVertex.setInvokableClass(IterationHeadPactTask.class);
      headConfig = new TaskConfig(headVertex.getConfiguration());
      headConfig.setDriver(NoOpDriver.class);
      toReturn = headVertex;
    }
View Full Code Here

    }
  }
 
  private void finalizeBulkIteration(IterationDescriptor descr) {
   
    final BulkIterationPlanNode bulkNode = (BulkIterationPlanNode) descr.getIterationNode();
    final AbstractJobVertex headVertex = descr.getHeadTask();
    final TaskConfig headConfig = new TaskConfig(headVertex.getConfiguration());
    final TaskConfig headFinalOutputConfig = descr.getHeadFinalResultConfig();
   
    // ------------ finalize the head config with the final outputs and the sync gate ------------
    final int numStepFunctionOuts = headConfig.getNumOutputs();
    final int numFinalOuts = headFinalOutputConfig.getNumOutputs();
   
    if (numStepFunctionOuts == 0) {
      throw new CompilerException("The iteration has no operation inside the step function.");
    }
   
    headConfig.setIterationHeadFinalOutputConfig(headFinalOutputConfig);
    headConfig.setIterationHeadIndexOfSyncOutput(numStepFunctionOuts + numFinalOuts);
    final double relativeMemForBackChannel = bulkNode.getRelativeMemoryPerSubTask();
    if (relativeMemForBackChannel <= 0) {
      throw new CompilerException("Bug: No memory has been assigned to the iteration back channel.");
    }
    headConfig.setRelativeBackChannelMemory(relativeMemForBackChannel);
   
    // --------------------------- create the sync task ---------------------------
    final AbstractJobVertex sync = new AbstractJobVertex("Sync(" + bulkNode.getNodeName() + ")");
    sync.setInvokableClass(IterationSynchronizationSinkTask.class);
    sync.setParallelism(1);
    this.auxVertices.add(sync);
   
    final TaskConfig syncConfig = new TaskConfig(sync.getConfiguration());
    syncConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, headVertex.getParallelism());

    // set the number of iteration / convergence criterion for the sync
    final int maxNumIterations = bulkNode.getIterationNode().getIterationContract().getMaximumNumberOfIterations();
    if (maxNumIterations < 1) {
      throw new CompilerException("Cannot create bulk iteration with unspecified maximum number of iterations.");
    }
    syncConfig.setNumberOfIterations(maxNumIterations);
   
    // connect the sync task
    sync.connectNewDataSetAsInput(headVertex, DistributionPattern.POINTWISE);
   
    // ----------------------------- create the iteration tail ------------------------------
   
    final PlanNode rootOfTerminationCriterion = bulkNode.getRootOfTerminationCriterion();
    final PlanNode rootOfStepFunction = bulkNode.getRootOfStepFunction();
    final TaskConfig tailConfig;
   
    AbstractJobVertex rootOfStepFunctionVertex = (AbstractJobVertex) this.vertices.get(rootOfStepFunction);
    if (rootOfStepFunctionVertex == null) {
      // last op is chained
      final TaskInChain taskInChain = this.chainedTasks.get(rootOfStepFunction);
      if (taskInChain == null) {
        throw new CompilerException("Bug: Tail of step function not found as vertex or chained task.");
      }
      rootOfStepFunctionVertex = (AbstractJobVertex) taskInChain.getContainingVertex();

      // the fake channel is statically typed to pact record. no data is sent over this channel anyways.
      tailConfig = taskInChain.getTaskConfig();
    } else {
      tailConfig = new TaskConfig(rootOfStepFunctionVertex.getConfiguration());
    }
   
    tailConfig.setIsWorksetUpdate();
   
    // No following termination criterion
    if (rootOfStepFunction.getOutgoingChannels().isEmpty()) {
     
      rootOfStepFunctionVertex.setInvokableClass(IterationTailPactTask.class);
     
      tailConfig.setOutputSerializer(bulkNode.getSerializerForIterationChannel());
    }
   
   
    // create the fake output task for termination criterion, if needed
    final TaskConfig tailConfigOfTerminationCriterion;
    // If we have a termination criterion and it is not an intermediate node
    if(rootOfTerminationCriterion != null && rootOfTerminationCriterion.getOutgoingChannels().isEmpty()) {
      AbstractJobVertex rootOfTerminationCriterionVertex = (AbstractJobVertex) this.vertices.get(rootOfTerminationCriterion);
     
     
      if (rootOfTerminationCriterionVertex == null) {
        // last op is chained
        final TaskInChain taskInChain = this.chainedTasks.get(rootOfTerminationCriterion);
        if (taskInChain == null) {
          throw new CompilerException("Bug: Tail of termination criterion not found as vertex or chained task.");
        }
        rootOfTerminationCriterionVertex = (AbstractJobVertex) taskInChain.getContainingVertex();

        // the fake channel is statically typed to pact record. no data is sent over this channel anyways.
        tailConfigOfTerminationCriterion = taskInChain.getTaskConfig();
      } else {
        tailConfigOfTerminationCriterion = new TaskConfig(rootOfTerminationCriterionVertex.getConfiguration());
      }
     
      rootOfTerminationCriterionVertex.setInvokableClass(IterationTailPactTask.class);
      // Hack
      tailConfigOfTerminationCriterion.setIsSolutionSetUpdate();
      tailConfigOfTerminationCriterion.setOutputSerializer(bulkNode.getSerializerForIterationChannel());
     
      // tell the head that it needs to wait for the solution set updates
      headConfig.setWaitForSolutionSetUpdate();
    }
   
    // ------------------- register the aggregators -------------------
    AggregatorRegistry aggs = bulkNode.getIterationNode().getIterationContract().getAggregators();
    Collection<AggregatorWithName<?>> allAggregators = aggs.getAllRegisteredAggregators();
   
    headConfig.addIterationAggregators(allAggregators);
    syncConfig.addIterationAggregators(allAggregators);
   
View Full Code Here

    }
   
    // 5) Create a candidate for the Iteration Node for every remaining plan of the step function.
    if (terminationCriterion == null) {
      for (PlanNode candidate : candidates) {
        BulkIterationPlanNode node = new BulkIterationPlanNode(this, "BulkIteration ("+this.getPactContract().getName()+")", in, pspn, candidate);
        GlobalProperties gProps = candidate.getGlobalProperties().clone();
        LocalProperties lProps = candidate.getLocalProperties().clone();
        node.initProperties(gProps, lProps);
        target.add(node);
      }
    }
    else if (candidates.size() > 0) {
      List<PlanNode> terminationCriterionCandidates = this.terminationCriterion.getAlternativePlans(estimator);

      SingleRootJoiner singleRoot = (SingleRootJoiner) this.singleRoot;
     
      for (PlanNode candidate : candidates) {
        for (PlanNode terminationCandidate : terminationCriterionCandidates) {
          if (singleRoot.areBranchCompatible(candidate, terminationCandidate)) {
            BulkIterationPlanNode node = new BulkIterationPlanNode(this, "BulkIteration ("+this.getPactContract().getName()+")", in, pspn, candidate, terminationCandidate);
            GlobalProperties gProps = candidate.getGlobalProperties().clone();
            LocalProperties lProps = candidate.getLocalProperties().clone();
            node.initProperties(gProps, lProps);
            target.add(node);
           
          }
        }
      }
View Full Code Here

      // get the plan and compile it
      Plan p = env.createProgramPlan();
      OptimizedPlan op = compileNoStats(p);
     
      SinkPlanNode sinkPlanNode = (SinkPlanNode) op.getDataSinks().iterator().next();
      BulkIterationPlanNode iterPlanNode = (BulkIterationPlanNode) sinkPlanNode.getInput().getSource();
     
      // check that the partitioning is pushed out of the first loop
      Assert.assertEquals(ShipStrategyType.PARTITION_HASH, iterPlanNode.getInput().getShipStrategy());
      Assert.assertEquals(LocalStrategy.NONE, iterPlanNode.getInput().getLocalStrategy());
     
      BulkPartialSolutionPlanNode partSolPlanNode = iterPlanNode.getPartialSolutionPlanNode();
      Assert.assertEquals(ShipStrategyType.FORWARD, partSolPlanNode.getOutgoingChannels().get(0).getShipStrategy());
    }
    catch (Exception e) {
      e.printStackTrace();
      fail(e.getMessage());
View Full Code Here

    final SinkPlanNode sink = or.getNode(SINK);
    final SingleInputPlanNode reducer = or.getNode(REDUCER_NAME);
    final SingleInputPlanNode combiner = (SingleInputPlanNode) reducer.getPredecessor();
    final SingleInputPlanNode mapper = or.getNode(MAPPER_NAME);
   
    final BulkIterationPlanNode iter = or.getNode(ITERATION_NAME);
   
    // -------------------- outside the loop -----------------------
   
    // check the sink
    assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
    assertEquals(LocalStrategy.NONE, sink.getInput().getLocalStrategy());
   
    // check the iteration
    assertEquals(ShipStrategyType.FORWARD, iter.getInput().getShipStrategy());
    assertEquals(LocalStrategy.NONE, iter.getInput().getLocalStrategy());
   
   
    // -------------------- inside the loop -----------------------
   
    // check the mapper
View Full Code Here

    if (visitable instanceof BulkIterationPlanNode) {
     
      DeadlockPreventer dp = new DeadlockPreventer();
      List<PlanNode> planSinks = new ArrayList<PlanNode>(1);
     
      BulkIterationPlanNode pspn = (BulkIterationPlanNode) visitable;
      planSinks.add(pspn.getRootOfStepFunction());
     
      dp.resolveDeadlocks(planSinks);
     
    }
    else if (visitable instanceof WorksetIterationPlanNode) {
     
      DeadlockPreventer dp = new DeadlockPreventer();
      List<PlanNode> planSinks = new ArrayList<PlanNode>(2);
     
      WorksetIterationPlanNode pspn = (WorksetIterationPlanNode) visitable;
      planSinks.add(pspn.getSolutionSetDeltaPlanNode());
      planSinks.add(pspn.getNextWorkSetPlanNode());
     
      dp.resolveDeadlocks(planSinks);
     
    }
  }
View Full Code Here

    else if (node instanceof SourcePlanNode) {
      TypeInformation<?> typeInfo = getTypeInfoFromSource((SourcePlanNode) node);
      ((SourcePlanNode) node).setSerializer(createSerializer(typeInfo));
    }
    else if (node instanceof BulkIterationPlanNode) {
      BulkIterationPlanNode iterationNode = (BulkIterationPlanNode) node;

      if (iterationNode.getRootOfStepFunction() instanceof NAryUnionPlanNode) {
        throw new CompilerException("Optimizer cannot compile an iteration step function where next partial solution is created by a Union node.");
      }
     
      // traverse the termination criterion for the first time. create schema only, no utilities. Needed in case of intermediate termination criterion
      if (iterationNode.getRootOfTerminationCriterion() != null) {
        SingleInputPlanNode addMapper = (SingleInputPlanNode) iterationNode.getRootOfTerminationCriterion();
        traverseChannel(addMapper.getInput());
      }

      BulkIterationBase<?> operator = (BulkIterationBase<?>) iterationNode.getPactContract();

      // set the serializer
      iterationNode.setSerializerForIterationChannel(createSerializer(operator.getOperatorInfo().getOutputType()));

      // done, we can now propagate our info down
      traverseChannel(iterationNode.getInput());
      traverse(iterationNode.getRootOfStepFunction());
    }
    else if (node instanceof WorksetIterationPlanNode) {
      WorksetIterationPlanNode iterationNode = (WorksetIterationPlanNode) node;
     
      if (iterationNode.getNextWorkSetPlanNode() instanceof NAryUnionPlanNode) {
        throw new CompilerException("Optimizer cannot compile a workset iteration step function where the next workset is produced by a Union node.");
      }
      if (iterationNode.getSolutionSetDeltaPlanNode() instanceof NAryUnionPlanNode) {
        throw new CompilerException("Optimizer cannot compile a workset iteration step function where the solution set delta is produced by a Union node.");
      }
     
      DeltaIterationBase<?, ?> operator = (DeltaIterationBase<?, ?>) iterationNode.getPactContract();
     
      // set the serializers and comparators for the workset iteration
      iterationNode.setSolutionSetSerializer(createSerializer(operator.getOperatorInfo().getFirstInputType()));
      iterationNode.setWorksetSerializer(createSerializer(operator.getOperatorInfo().getSecondInputType()));
      iterationNode.setSolutionSetComparator(createComparator(operator.getOperatorInfo().getFirstInputType(),
          iterationNode.getSolutionSetKeyFields(), getSortOrders(iterationNode.getSolutionSetKeyFields(), null)));
     
      // traverse the inputs
      traverseChannel(iterationNode.getInput1());
      traverseChannel(iterationNode.getInput2());
     
      // traverse the step function
      traverse(iterationNode.getSolutionSetDeltaPlanNode());
      traverse(iterationNode.getNextWorkSetPlanNode());
    }
    else if (node instanceof SingleInputPlanNode) {
      SingleInputPlanNode sn = (SingleInputPlanNode) node;
     
      if (!(sn.getOptimizerNode().getPactContract() instanceof SingleInputOperator)) {
View Full Code Here

TOP

Related Classes of org.apache.flink.compiler.plan.BulkIterationPlanNode

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.