Examples of OptimizerNode


Examples of org.apache.flink.compiler.dag.OptimizerNode

      if (this.branchPlan == null) {
        this.branchPlan = new HashMap<OptimizerNode, PlanNode>(8);
      }
 
      for (UnclosedBranchDescriptor uc : this.template.getOpenBranches()) {
        OptimizerNode brancher = uc.getBranchingNode();
        PlanNode selectedCandidate = null;
 
        if (branchPlan1 != null) {
          // predecessor 1 has branching children, see if it got the branch we are looking for
          selectedCandidate = branchPlan1.get(brancher);
View Full Code Here

Examples of org.apache.flink.compiler.dag.OptimizerNode

      if (this.branchPlan == null) {
        throw new CompilerException("Branching and rejoining logic did not find a candidate for the branching point.");
      }
 
      for (UnclosedBranchDescriptor uc : this.template.getOpenBranches()) {
        OptimizerNode brancher = uc.getBranchingNode();
        if (this.branchPlan.get(brancher) == null) {
          throw new CompilerException("Branching and rejoining logic did not find a candidate for the branching point.");
        }
      }
    }
View Full Code Here

Examples of org.apache.flink.compiler.dag.OptimizerNode

    program.accept(graphCreator);

    // if we have a plan with multiple data sinks, add logical optimizer nodes that have two data-sinks as children
    // each until we have only a single root node. This allows to transparently deal with the nodes with
    // multiple outputs
    OptimizerNode rootNode;
    if (graphCreator.sinks.size() == 1) {
      rootNode = graphCreator.sinks.get(0);
    } else if (graphCreator.sinks.size() > 1) {
      Iterator<DataSinkNode> iter = graphCreator.sinks.iterator();
      rootNode = iter.next();

      while (iter.hasNext()) {
        rootNode = new SinkJoiner(rootNode, iter.next());
      }
    } else {
      throw new CompilerException("Bug: The optimizer plan representation has no sinks.");
    }

    // now that we have all nodes created and recorded which ones consume memory, tell the nodes their minimal
    // guaranteed memory, for further cost estimations. we assume an equal distribution of memory among consumer tasks
   
    rootNode.accept(new IdAndEstimatesVisitor(this.statistics));
   
    // Now that the previous step is done, the next step is to traverse the graph again for the two
    // steps that cannot directly be performed during the plan enumeration, because we are dealing with DAGs
    // rather than a trees. That requires us to deviate at some points from the classical DB optimizer algorithms.
    //
    // 1) propagate the interesting properties top-down through the graph
    // 2) Track information about nodes with multiple outputs that are later on reconnected in a node with
    // multiple inputs.
    InterestingPropertyVisitor propsVisitor = new InterestingPropertyVisitor(this.costEstimator);
    rootNode.accept(propsVisitor);
   
    BranchesVisitor branchingVisitor = new BranchesVisitor();
    rootNode.accept(branchingVisitor);
   
    // perform a sanity check: the root may not have any unclosed branches
    if (rootNode.getOpenBranches() != null && rootNode.getOpenBranches().size() > 0) {
      throw new CompilerException("Bug: Logic for branching plans (non-tree plans) has an error, and does not " +
          "track the re-joining of branches correctly.");
    }

    // the final step is now to generate the actual plan alternatives
    List<PlanNode> bestPlan = rootNode.getAlternativePlans(this.costEstimator);

    if (bestPlan.size() != 1) {
      throw new CompilerException("Error in compiler: more than one best plan was created!");
    }
View Full Code Here

Examples of org.apache.flink.compiler.dag.OptimizerNode

    }
  }

  private void mergeBranchPlanMaps() {
    for(OptimizerNode.UnclosedBranchDescriptor desc: template.getOpenBranches()){
      OptimizerNode brancher = desc.getBranchingNode();

      if(branchPlan == null) {
        branchPlan = new HashMap<OptimizerNode, PlanNode>(6);
      }
     
View Full Code Here

Examples of org.apache.flink.compiler.dag.OptimizerNode

      // check if we have been here before
      if (this.con2node.containsKey(c)) {
        return false;
      }

      final OptimizerNode n;

      // create a node for the operator (or sink or source) if we have not been here before
      if (c instanceof GenericDataSinkBase) {
        DataSinkNode dsn = new DataSinkNode((GenericDataSinkBase<?>) c);
        this.sinks.add(dsn);
        n = dsn;
      }
      else if (c instanceof GenericDataSourceBase) {
        DataSourceNode dsn = new DataSourceNode((GenericDataSourceBase<?, ?>) c);
        this.sources.add(dsn);
        n = dsn;
      }
      else if (c instanceof MapOperatorBase) {
        n = new MapNode((MapOperatorBase<?, ?, ?>) c);
      }
      else if (c instanceof MapPartitionOperatorBase) {
        n = new MapPartitionNode((MapPartitionOperatorBase<?, ?, ?>) c);
      }
      else if (c instanceof org.apache.flink.api.common.operators.base.CollectorMapOperatorBase) {
        n = new CollectorMapNode((org.apache.flink.api.common.operators.base.CollectorMapOperatorBase<?, ?, ?>) c);
      }
      else if (c instanceof FlatMapOperatorBase) {
        n = new FlatMapNode((FlatMapOperatorBase<?, ?, ?>) c);
      }
      else if (c instanceof FilterOperatorBase) {
        n = new FilterNode((FilterOperatorBase<?, ?>) c);
      }
      else if (c instanceof ReduceOperatorBase) {
        n = new ReduceNode((ReduceOperatorBase<?, ?>) c);
      }
      else if (c instanceof GroupReduceOperatorBase) {
        n = new GroupReduceNode((GroupReduceOperatorBase<?, ?, ?>) c);
      }
      else if (c instanceof JoinOperatorBase) {
        n = new MatchNode((JoinOperatorBase<?, ?, ?, ?>) c);
      }
      else if (c instanceof CoGroupOperatorBase) {
        n = new CoGroupNode((CoGroupOperatorBase<?, ?, ?, ?>) c);
      }
      else if (c instanceof CrossOperatorBase) {
        n = new CrossNode((CrossOperatorBase<?, ?, ?, ?>) c);
      }
      else if (c instanceof BulkIterationBase) {
        n = new BulkIterationNode((BulkIterationBase<?>) c);
      }
      else if (c instanceof DeltaIterationBase) {
        n = new WorksetIterationNode((DeltaIterationBase<?, ?>) c);
      }
      else if (c instanceof Union){
        n = new BinaryUnionNode((Union<?>) c);
      }
      else if (c instanceof PartitionOperatorBase) {
        n = new PartitionNode((PartitionOperatorBase<?>) c);
      }
      else if (c instanceof PartialSolutionPlaceHolder) {
        if (this.parent == null) {
          throw new InvalidProgramException("It is currently not supported to create data sinks inside iterations.");
        }
       
        final PartialSolutionPlaceHolder<?> holder = (PartialSolutionPlaceHolder<?>) c;
        final BulkIterationBase<?> enclosingIteration = holder.getContainingBulkIteration();
        final BulkIterationNode containingIterationNode =
              (BulkIterationNode) this.parent.con2node.get(enclosingIteration);
       
        // catch this for the recursive translation of step functions
        BulkPartialSolutionNode p = new BulkPartialSolutionNode(holder, containingIterationNode);
        p.setDegreeOfParallelism(containingIterationNode.getDegreeOfParallelism());
        n = p;
      }
      else if (c instanceof WorksetPlaceHolder) {
        if (this.parent == null) {
          throw new InvalidProgramException("It is currently not supported to create data sinks inside iterations.");
        }
       
        final WorksetPlaceHolder<?> holder = (WorksetPlaceHolder<?>) c;
        final DeltaIterationBase<?, ?> enclosingIteration = holder.getContainingWorksetIteration();
        final WorksetIterationNode containingIterationNode =
              (WorksetIterationNode) this.parent.con2node.get(enclosingIteration);
       
        // catch this for the recursive translation of step functions
        WorksetNode p = new WorksetNode(holder, containingIterationNode);
        p.setDegreeOfParallelism(containingIterationNode.getDegreeOfParallelism());
        n = p;
      }
      else if (c instanceof SolutionSetPlaceHolder) {
        if (this.parent == null) {
          throw new InvalidProgramException("It is currently not supported to create data sinks inside iterations.");
        }
       
        final SolutionSetPlaceHolder<?> holder = (SolutionSetPlaceHolder<?>) c;
        final DeltaIterationBase<?, ?> enclosingIteration = holder.getContainingWorksetIteration();
        final WorksetIterationNode containingIterationNode =
              (WorksetIterationNode) this.parent.con2node.get(enclosingIteration);
       
        // catch this for the recursive translation of step functions
        SolutionSetNode p = new SolutionSetNode(holder, containingIterationNode);
        p.setDegreeOfParallelism(containingIterationNode.getDegreeOfParallelism());
        n = p;
      }
      else {
        throw new IllegalArgumentException("Unknown operator type: " + c);
      }

      this.con2node.put(c, n);
     
      // set the parallelism only if it has not been set before. some nodes have a fixed DOP, such as the
      // key-less reducer (all-reduce)
      if (n.getDegreeOfParallelism() < 1) {
        // set the degree of parallelism
        int par = c.getDegreeOfParallelism();
        if (par > 0) {
          if (this.forceDOP && par != this.defaultParallelism) {
            par = this.defaultParallelism;
            LOG.warn("The degree-of-parallelism of nested Dataflows (such as step functions in iterations) is " +
              "currently fixed to the degree-of-parallelism of the surrounding operator (the iteration).");
          }
        } else {
          par = this.defaultParallelism;
        }
        n.setDegreeOfParallelism(par);
      }

      return true;
    }
View Full Code Here

Examples of org.apache.flink.compiler.dag.OptimizerNode

    }

    @Override
    public void postVisit(Operator<?> c) {
     
      OptimizerNode n = this.con2node.get(c);

      // first connect to the predecessors
      n.setInput(this.con2node);
      n.setBroadcastInputs(this.con2node);
     
      // if the node represents a bulk iteration, we recursively translate the data flow now
      if (n instanceof BulkIterationNode) {
        final BulkIterationNode iterNode = (BulkIterationNode) n;
        final BulkIterationBase<?> iter = iterNode.getIterationContract();

        // pass a copy of the no iterative part into the iteration translation,
        // in case the iteration references its closure
        HashMap<Operator<?>, OptimizerNode> closure = new HashMap<Operator<?>, OptimizerNode>(con2node);

        // first, recursively build the data flow for the step function
        final GraphCreatingVisitor recursiveCreator = new GraphCreatingVisitor(this, true,
          iterNode.getDegreeOfParallelism(), closure);
       
        BulkPartialSolutionNode partialSolution = null;
       
        iter.getNextPartialSolution().accept(recursiveCreator);
       
        partialSolution =  (BulkPartialSolutionNode) recursiveCreator.con2node.get(iter.getPartialSolution());
        OptimizerNode rootOfStepFunction = recursiveCreator.con2node.get(iter.getNextPartialSolution());
        if (partialSolution == null) {
          throw new CompilerException("Error: The step functions result does not depend on the partial solution.");
        }
       
       
        OptimizerNode terminationCriterion = null;
       
        if (iter.getTerminationCriterion() != null) {
          terminationCriterion = recursiveCreator.con2node.get(iter.getTerminationCriterion());
         
          // no intermediate node yet, traverse from the termination criterion to build the missing parts
          if (terminationCriterion == null) {
            iter.getTerminationCriterion().accept(recursiveCreator);
            terminationCriterion = recursiveCreator.con2node.get(iter.getTerminationCriterion());
          }
        }
       
        iterNode.setPartialSolution(partialSolution);
        iterNode.setNextPartialSolution(rootOfStepFunction, terminationCriterion);
       
        // go over the contained data flow and mark the dynamic path nodes
        StaticDynamicPathIdentifier identifier = new StaticDynamicPathIdentifier(iterNode.getCostWeight());
        rootOfStepFunction.accept(identifier);
        if(terminationCriterion != null){
          terminationCriterion.accept(identifier);
        }
      }
      else if (n instanceof WorksetIterationNode) {
        final WorksetIterationNode iterNode = (WorksetIterationNode) n;
        final DeltaIterationBase<?, ?> iter = iterNode.getIterationContract();

        // we need to ensure that both the next-workset and the solution-set-delta depend on the workset. One check is for free
        // during the translation, we do the other check here as a pre-condition
        {
          WorksetFinder wsf = new WorksetFinder();
          iter.getNextWorkset().accept(wsf);
          if (!wsf.foundWorkset) {
            throw new CompilerException("In the given program, the next workset does not depend on the workset. This is a prerequisite in delta iterations.");
          }
        }
       
        // calculate the closure of the anonymous function
        HashMap<Operator<?>, OptimizerNode> closure = new HashMap<Operator<?>, OptimizerNode>(con2node);

        // first, recursively build the data flow for the step function
        final GraphCreatingVisitor recursiveCreator = new GraphCreatingVisitor(this, true, iterNode.getDegreeOfParallelism(), closure);
       
        // descend from the solution set delta. check that it depends on both the workset
        // and the solution set. If it does depend on both, this descend should create both nodes
        iter.getSolutionSetDelta().accept(recursiveCreator);
       
        final WorksetNode worksetNode = (WorksetNode) recursiveCreator.con2node.get(iter.getWorkset());
       
        if (worksetNode == null) {
          throw new CompilerException("In the given program, the solution set delta does not depend on the workset. This is a prerequisite in delta iterations.");
        }
       
        iter.getNextWorkset().accept(recursiveCreator);
       
        SolutionSetNode solutionSetNode = (SolutionSetNode) recursiveCreator.con2node.get(iter.getSolutionSet());
       
        if (solutionSetNode == null || solutionSetNode.getOutgoingConnections() == null || solutionSetNode.getOutgoingConnections().isEmpty()) {
          solutionSetNode = new SolutionSetNode((SolutionSetPlaceHolder<?>) iter.getSolutionSet(), iterNode);
        }
        else {
          for (PactConnection conn : solutionSetNode.getOutgoingConnections()) {
            OptimizerNode successor = conn.getTarget();
         
            if (successor.getClass() == MatchNode.class) {
              // find out which input to the match the solution set is
              MatchNode mn = (MatchNode) successor;
              if (mn.getFirstPredecessorNode() == solutionSetNode) {
                mn.makeJoinWithSolutionSet(0);
              } else if (mn.getSecondPredecessorNode() == solutionSetNode) {
                mn.makeJoinWithSolutionSet(1);
              } else {
                throw new CompilerException();
              }
            }
            else if (successor.getClass() == CoGroupNode.class) {
              CoGroupNode cg = (CoGroupNode) successor;
              if (cg.getFirstPredecessorNode() == solutionSetNode) {
                cg.makeCoGroupWithSolutionSet(0);
              } else if (cg.getSecondPredecessorNode() == solutionSetNode) {
                cg.makeCoGroupWithSolutionSet(1);
              } else {
                throw new CompilerException();
              }
            }
            else {
              throw new CompilerException("Error: The only operations allowed on the solution set are Join and CoGroup.");
            }
          }
        }
       
        final OptimizerNode nextWorksetNode = recursiveCreator.con2node.get(iter.getNextWorkset());
        final OptimizerNode solutionSetDeltaNode = recursiveCreator.con2node.get(iter.getSolutionSetDelta());
       
        // set the step function nodes to the iteration node
        iterNode.setPartialSolution(solutionSetNode, worksetNode);
        iterNode.setNextPartialSolution(solutionSetDeltaNode, nextWorksetNode);
       
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.