Examples of org.apache.flink.compiler.dag.OptimizerNode

org.apache.flink.compiler.dag.OptimizerNode
This class represents a node in the optimizer's internal representation of the PACT plan. It contains extra information about estimates, hints and data properties.

        first = false;
      };
    }
    
    // check if this node should be skipped from the dump
    final OptimizerNode n = node.getOptimizerNode();
    
    // ------------------ dump after the ascend ---------------------
    // start a new node and output node id
    if (!first) {
      writer.print(",\n");  
    }
    // open the node
    writer.print("\t{\n");
    
    // recurse, it is is an iteration node
    if (node instanceof BulkIterationNode || node instanceof BulkIterationPlanNode) {
      
      DumpableNode<?> innerChild = node instanceof BulkIterationNode ?
          ((BulkIterationNode) node).getNextPartialSolution() :
          ((BulkIterationPlanNode) node).getRootOfStepFunction();
          
      DumpableNode<?> begin = node instanceof BulkIterationNode ?
        ((BulkIterationNode) node).getPartialSolution() :
        ((BulkIterationPlanNode) node).getPartialSolutionPlanNode();
      
      writer.print("\t\t\"step_function\": [\n");
      
      visit(innerChild, writer, true);
      
      writer.print("\n\t\t],\n");
      writer.print("\t\t\"partial_solution\": " + this.nodeIds.get(begin) + ",\n");
      writer.print("\t\t\"next_partial_solution\": " + this.nodeIds.get(innerChild) + ",\n");
    } else if (node instanceof WorksetIterationNode || node instanceof WorksetIterationPlanNode) {
      
      DumpableNode<?> worksetRoot = node instanceof WorksetIterationNode ?
          ((WorksetIterationNode) node).getNextWorkset() :
          ((WorksetIterationPlanNode) node).getNextWorkSetPlanNode();
      DumpableNode<?> solutionDelta = node instanceof WorksetIterationNode ?
          ((WorksetIterationNode) node).getSolutionSetDelta() :
          ((WorksetIterationPlanNode) node).getSolutionSetDeltaPlanNode();
          
      DumpableNode<?> workset = node instanceof WorksetIterationNode ?
            ((WorksetIterationNode) node).getWorksetNode() :
            ((WorksetIterationPlanNode) node).getWorksetPlanNode();
      DumpableNode<?> solutionSet = node instanceof WorksetIterationNode ?
            ((WorksetIterationNode) node).getSolutionSetNode() :
            ((WorksetIterationPlanNode) node).getSolutionSetPlanNode();
      
      writer.print("\t\t\"step_function\": [\n");
      
      visit(worksetRoot, writer, true);
      visit(solutionDelta, writer, false);
      
      writer.print("\n\t\t],\n");
      writer.print("\t\t\"workset\": " + this.nodeIds.get(workset) + ",\n");
      writer.print("\t\t\"solution_set\": " + this.nodeIds.get(solutionSet) + ",\n");
      writer.print("\t\t\"next_workset\": " + this.nodeIds.get(worksetRoot) + ",\n");
      writer.print("\t\t\"solution_delta\": " + this.nodeIds.get(solutionDelta) + ",\n");
    }
    
    // print the id
    writer.print("\t\t\"id\": " + this.nodeIds.get(node));


    
    final String type;
    String contents;
    if (n instanceof DataSinkNode) {
      type = "sink";
      contents = n.getPactContract().toString();
    } else if (n instanceof DataSourceNode) {
      type = "source";
      contents = n.getPactContract().toString();
    }
    else if (n instanceof BulkIterationNode) {
      type = "bulk_iteration";
      contents = n.getPactContract().getName();
    }
    else if (n instanceof WorksetIterationNode) {
      type = "workset_iteration";
      contents = n.getPactContract().getName();
    }
    else if (n instanceof BinaryUnionNode) {
      type = "pact";
      contents = "";
    }
    else {
      type = "pact";
      contents = n.getPactContract().getName();
    }
    
    contents = StringUtils.showControlCharacters(contents);
    if (encodeForHTML) {
      contents = StringEscapeUtils.escapeHtml4(contents);
      contents = contents.replace("\\", "&#92;");
    }
    
    
    String name = n.getName();
    if (name.equals("Reduce") && (node instanceof SingleInputPlanNode) && 
        ((SingleInputPlanNode) node).getDriverStrategy() == DriverStrategy.SORTED_GROUP_COMBINE) {
      name = "Combine";
    }
    
    // output the type identifier
    writer.print(",\n\t\t\"type\": \"" + type + "\"");
    
    // output node name
    writer.print(",\n\t\t\"pact\": \"" + name + "\"");
    
    // output node contents
    writer.print(",\n\t\t\"contents\": \"" + contents + "\"");


    // degree of parallelism
    writer.print(",\n\t\t\"parallelism\": \""
      + (n.getDegreeOfParallelism() >= 1 ? n.getDegreeOfParallelism() : "default") + "\"");
    
    // output node predecessors
    Iterator<? extends DumpableConnection<?>> inConns = node.getDumpableInputs().iterator();
    String child1name = "", child2name = "";


    if (inConns != null && inConns.hasNext()) {
      // start predecessor list
      writer.print(",\n\t\t\"predecessors\": [");
      int inputNum = 0;
      
      while (inConns.hasNext()) {
        final DumpableConnection<?> inConn = inConns.next();
        final DumpableNode<?> source = inConn.getSource();
        writer.print(inputNum == 0 ? "\n" : ",\n");
        if (inputNum == 0) {
          child1name += child1name.length() > 0 ? ", " : ""; 
          child1name += source.getOptimizerNode().getPactContract().getName();
        } else if (inputNum == 1) {
          child2name += child2name.length() > 0 ? ", " : ""; 
          child2name = source.getOptimizerNode().getPactContract().getName();
        }


        // output predecessor id
        writer.print("\t\t\t{\"id\": " + this.nodeIds.get(source));


        // output connection side
        if (inConns.hasNext() || inputNum > 0) {
          writer.print(", \"side\": \"" + (inputNum == 0 ? "first" : "second") + "\"");
        }
        // output shipping strategy and channel type
        final Channel channel = (inConn instanceof Channel) ? (Channel) inConn : null; 
        final ShipStrategyType shipType = channel != null ? channel.getShipStrategy() :
            ((PactConnection) inConn).getShipStrategy();
          
        String shipStrategy = null;
        if (shipType != null) {
          switch (shipType) {
          case NONE:
            // nothing
            break;
          case FORWARD:
            shipStrategy = "Forward";
            break;
          case BROADCAST:
            shipStrategy = "Broadcast";
            break;
          case PARTITION_HASH:
            shipStrategy = "Hash Partition";
            break;
          case PARTITION_RANGE:
            shipStrategy = "Range Partition";
            break;
          case PARTITION_RANDOM:
            shipStrategy = "Redistribute";
            break;
          case PARTITION_FORCED_REBALANCE:
            shipStrategy = "Rebalance";
            break;
          default:
            throw new CompilerException("Unknown ship strategy '" + inConn.getShipStrategy().name()
              + "' in JSON generator.");
          }
        }
        
        if (channel != null && channel.getShipStrategyKeys() != null && channel.getShipStrategyKeys().size() > 0) {
          shipStrategy += " on " + (channel.getShipStrategySortOrder() == null ?
              channel.getShipStrategyKeys().toString() :
              Utils.createOrdering(channel.getShipStrategyKeys(), channel.getShipStrategySortOrder()).toString());
        }


        if (shipStrategy != null) {
          writer.print(", \"ship_strategy\": \"" + shipStrategy + "\"");
        }
        
        if (channel != null) {
          String localStrategy = null;
          switch (channel.getLocalStrategy()) {
          case NONE:
            break;
          case SORT:
            localStrategy = "Sort";
            break;
          case COMBININGSORT:
            localStrategy = "Sort (combining)";
            break;
          default:
            throw new CompilerException("Unknown local strategy " + channel.getLocalStrategy().name());
          }
          
          if (channel != null && channel.getLocalStrategyKeys() != null && channel.getLocalStrategyKeys().size() > 0) {
            localStrategy += " on " + (channel.getLocalStrategySortOrder() == null ?
                channel.getLocalStrategyKeys().toString() :
                Utils.createOrdering(channel.getLocalStrategyKeys(), channel.getLocalStrategySortOrder()).toString());
          }
          
          if (localStrategy != null) {
            writer.print(", \"local_strategy\": \"" + localStrategy + "\"");
          }
          
          if (channel != null && channel.getTempMode() != TempMode.NONE) {
            String tempMode = channel.getTempMode().toString();
            writer.print(", \"temp_mode\": \"" + tempMode + "\"");
          }
        }
        
        writer.print('}');
        inputNum++;
      }
      // finish predecessors
      writer.print("\n\t\t]");
    }
    
    //---------------------------------------------------------------------------------------
    // the part below here is relevant only to plan nodes with concrete strategies, etc
    //---------------------------------------------------------------------------------------


    final PlanNode p = node.getPlanNode();
    if (p == null) {
      // finish node
      writer.print("\n\t}");
      return true;
    }
    // local strategy
    String locString = null;
    if (p.getDriverStrategy() != null) {
      switch (p.getDriverStrategy()) {
      case NONE:
      case BINARY_NO_OP:
        break;
        
      case UNARY_NO_OP:
        locString = "No-Op";
        break;
        
      case COLLECTOR_MAP:
      case MAP:
        locString = "Map";
        break;
        
      case FLAT_MAP:
        locString = "FlatMap";
        break;
        
      case MAP_PARTITION:
        locString = "Map Partition";
        break;
      
      case ALL_REDUCE:
        locString = "Reduce All";
        break;
      
      case ALL_GROUP_REDUCE:
      case ALL_GROUP_COMBINE:
        locString = "Group Reduce All";
        break;
        
      case SORTED_REDUCE:
        locString = "Sorted Reduce";
        break;
        
      case SORTED_PARTIAL_REDUCE:
        locString = "Sorted Combine/Reduce";
        break;


      case SORTED_GROUP_REDUCE:
        locString = "Sorted Group Reduce";
        break;
        
      case SORTED_GROUP_COMBINE:
        locString = "Sorted Combine";
        break;


      case HYBRIDHASH_BUILD_FIRST:
        locString = "Hybrid Hash (build: " + child1name + ")";
        break;
      case HYBRIDHASH_BUILD_SECOND:
        locString = "Hybrid Hash (build: " + child2name + ")";
        break;
        
      case HYBRIDHASH_BUILD_FIRST_CACHED:
        locString = "Hybrid Hash (CACHED) (build: " + child1name + ")";
        break;
      case HYBRIDHASH_BUILD_SECOND_CACHED:
        locString = "Hybrid Hash (CACHED) (build: " + child2name + ")";
        break;


      case NESTEDLOOP_BLOCKED_OUTER_FIRST:
        locString = "Nested Loops (Blocked Outer: " + child1name + ")";
        break;
      case NESTEDLOOP_BLOCKED_OUTER_SECOND:
        locString = "Nested Loops (Blocked Outer: " + child2name + ")";
        break;
      case NESTEDLOOP_STREAMED_OUTER_FIRST:
        locString = "Nested Loops (Streamed Outer: " + child1name + ")";
        break;
      case NESTEDLOOP_STREAMED_OUTER_SECOND:
        locString = "Nested Loops (Streamed Outer: " + child2name + ")";
        break;


      case MERGE:
        locString = "Merge";
        break;


      case CO_GROUP:
        locString = "Co-Group";
        break;


      default:
        locString = p.getDriverStrategy().name();
        break;
      }


      if (locString != null) {
        writer.print(",\n\t\t\"driver_strategy\": \"");
        writer.print(locString);
        writer.print("\"");
      }
    }
    
    {
      // output node global properties
      final GlobalProperties gp = p.getGlobalProperties();


      writer.print(",\n\t\t\"global_properties\": [\n");


      addProperty(writer, "Partitioning", gp.getPartitioning().name(), true);
      if (gp.getPartitioningFields() != null) {
        addProperty(writer, "Partitioned on", gp.getPartitioningFields().toString(), false);
      }
      if (gp.getPartitioningOrdering() != null) {
        addProperty(writer, "Partitioning Order", gp.getPartitioningOrdering().toString(), false);  
      }
      else {
        addProperty(writer, "Partitioning Order", "(none)", false);
      }
      if (n.getUniqueFields() == null || n.getUniqueFields().size() == 0) {
        addProperty(writer, "Uniqueness", "not unique", false);
      }
      else {
        addProperty(writer, "Uniqueness", n.getUniqueFields().toString(), false);  
      }


      writer.print("\n\t\t]");
    }


    {
      // output node local properties
      LocalProperties lp = p.getLocalProperties();


      writer.print(",\n\t\t\"local_properties\": [\n");


      if (lp.getOrdering() != null) {
        addProperty(writer, "Order", lp.getOrdering().toString(), true);  
      }
      else {
        addProperty(writer, "Order", "(none)", true);
      }
      if (lp.getGroupedFields() != null && lp.getGroupedFields().size() > 0) {
        addProperty(writer, "Grouped on", lp.getGroupedFields().toString(), false);
      } else {
        addProperty(writer, "Grouping", "not grouped", false);  
      }
      if (n.getUniqueFields() == null || n.getUniqueFields().size() == 0) {
        addProperty(writer, "Uniqueness", "not unique", false);
      }
      else {
        addProperty(writer, "Uniqueness", n.getUniqueFields().toString(), false);  
      }


      writer.print("\n\t\t]");
    }


    // output node size estimates
    writer.print(",\n\t\t\"estimates\": [\n");


    addProperty(writer, "Est. Output Size", n.getEstimatedOutputSize() == -1 ? "(unknown)"
      : formatNumber(n.getEstimatedOutputSize(), "B"), true);
    addProperty(writer, "Est. Cardinality", n.getEstimatedNumRecords() == -1 ? "(unknown)"
      : formatNumber(n.getEstimatedNumRecords()), false);


    writer.print("\t\t]");


    // output node cost
    if (p.getNodeCosts() != null) {
      writer.print(",\n\t\t\"costs\": [\n");


      addProperty(writer, "Network", p.getNodeCosts().getNetworkCost() == -1 ? "(unknown)"
        : formatNumber(p.getNodeCosts().getNetworkCost(), "B"), true);
      addProperty(writer, "Disk I/O", p.getNodeCosts().getDiskCost() == -1 ? "(unknown)"
        : formatNumber(p.getNodeCosts().getDiskCost(), "B"), false);
      addProperty(writer, "CPU", p.getNodeCosts().getCpuCost() == -1 ? "(unknown)"
        : formatNumber(p.getNodeCosts().getCpuCost(), ""), false);


      addProperty(writer, "Cumulative Network",
        p.getCumulativeCosts().getNetworkCost() == -1 ? "(unknown)" : formatNumber(p
          .getCumulativeCosts().getNetworkCost(), "B"), false);
      addProperty(writer, "Cumulative Disk I/O",
        p.getCumulativeCosts().getDiskCost() == -1 ? "(unknown)" : formatNumber(p
          .getCumulativeCosts().getDiskCost(), "B"), false);
      addProperty(writer, "Cumulative CPU",
        p.getCumulativeCosts().getCpuCost() == -1 ? "(unknown)" : formatNumber(p
          .getCumulativeCosts().getCpuCost(), ""), false);


      writer.print("\n\t\t]");
    }


    // output the node compiler hints
    if (n.getPactContract().getCompilerHints() != null) {
      CompilerHints hints = n.getPactContract().getCompilerHints();
      CompilerHints defaults = new CompilerHints();


      String size = hints.getOutputSize() == defaults.getOutputSize() ? "(none)" : String.valueOf(hints.getOutputSize());
      String card = hints.getOutputCardinality() == defaults.getOutputCardinality() ? "(none)" : String.valueOf(hints.getOutputCardinality());
      String width = hints.getAvgOutputRecordSize() == defaults.getAvgOutputRecordSize() ? "(none)" : String.valueOf(hints.getAvgOutputRecordSize());

View Full Code Here

      if (this.branchPlan == null) {
        this.branchPlan = new HashMap<OptimizerNode, PlanNode>(8);
      }


      for (OptimizerNode.UnclosedBranchDescriptor uc : this.template.getOpenBranches()) {
        OptimizerNode brancher = uc.getBranchingNode();
        PlanNode selectedCandidate = null;


        if (branchPlan1 != null) {
          // predecessor 1 has branching children, see if it got the branch we are looking for
          selectedCandidate = branchPlan1.get(brancher);

View Full Code Here

      if (this.branchPlan == null) {
        this.branchPlan = new HashMap<OptimizerNode, PlanNode>(8);
      }
  
      for (UnclosedBranchDescriptor uc : this.template.getOpenBranches()) {
        OptimizerNode brancher = uc.getBranchingNode();
        PlanNode selectedCandidate = null;
  
        if (branchPlan1 != null) {
          // predecessor 1 has branching children, see if it got the branch we are looking for
          selectedCandidate = branchPlan1.get(brancher);

View Full Code Here

      if (this.branchPlan == null) {
        throw new CompilerException("Branching and rejoining logic did not find a candidate for the branching point.");
      }
  
      for (UnclosedBranchDescriptor uc : this.template.getOpenBranches()) {
        OptimizerNode brancher = uc.getBranchingNode();
        if (this.branchPlan.get(brancher) == null) {
          throw new CompilerException("Branching and rejoining logic did not find a candidate for the branching point.");
        }
      }
    }

View Full Code Here

    program.accept(graphCreator);


    // if we have a plan with multiple data sinks, add logical optimizer nodes that have two data-sinks as children
    // each until we have only a single root node. This allows to transparently deal with the nodes with
    // multiple outputs
    OptimizerNode rootNode;
    if (graphCreator.sinks.size() == 1) {
      rootNode = graphCreator.sinks.get(0);
    } else if (graphCreator.sinks.size() > 1) {
      Iterator<DataSinkNode> iter = graphCreator.sinks.iterator();
      rootNode = iter.next();


      while (iter.hasNext()) {
        rootNode = new SinkJoiner(rootNode, iter.next());
      }
    } else {
      throw new CompilerException("Bug: The optimizer plan representation has no sinks.");
    }


    // now that we have all nodes created and recorded which ones consume memory, tell the nodes their minimal
    // guaranteed memory, for further cost estimations. we assume an equal distribution of memory among consumer tasks
    
    rootNode.accept(new IdAndEstimatesVisitor(this.statistics));
    
    // Now that the previous step is done, the next step is to traverse the graph again for the two
    // steps that cannot directly be performed during the plan enumeration, because we are dealing with DAGs
    // rather than a trees. That requires us to deviate at some points from the classical DB optimizer algorithms.
    //
    // 1) propagate the interesting properties top-down through the graph
    // 2) Track information about nodes with multiple outputs that are later on reconnected in a node with
    // multiple inputs.
    InterestingPropertyVisitor propsVisitor = new InterestingPropertyVisitor(this.costEstimator);
    rootNode.accept(propsVisitor);
    
    BranchesVisitor branchingVisitor = new BranchesVisitor();
    rootNode.accept(branchingVisitor);
    
    // perform a sanity check: the root may not have any unclosed branches
    if (rootNode.getOpenBranches() != null && rootNode.getOpenBranches().size() > 0) {
      throw new CompilerException("Bug: Logic for branching plans (non-tree plans) has an error, and does not " +
          "track the re-joining of branches correctly.");
    }


    // the final step is now to generate the actual plan alternatives
    List<PlanNode> bestPlan = rootNode.getAlternativePlans(this.costEstimator);


    if (bestPlan.size() != 1) {
      throw new CompilerException("Error in compiler: more than one best plan was created!");
    }

View Full Code Here

    }
  }


  private void mergeBranchPlanMaps() {
    for(OptimizerNode.UnclosedBranchDescriptor desc: template.getOpenBranches()){
      OptimizerNode brancher = desc.getBranchingNode();


      if(branchPlan == null) {
        branchPlan = new HashMap<OptimizerNode, PlanNode>(6);
      }

View Full Code Here

      // check if we have been here before
      if (this.con2node.containsKey(c)) {
        return false;
      }


      final OptimizerNode n;


      // create a node for the operator (or sink or source) if we have not been here before
      if (c instanceof GenericDataSinkBase) {
        DataSinkNode dsn = new DataSinkNode((GenericDataSinkBase<?>) c);
        this.sinks.add(dsn);
        n = dsn;
      }
      else if (c instanceof GenericDataSourceBase) {
        DataSourceNode dsn = new DataSourceNode((GenericDataSourceBase<?, ?>) c);
        this.sources.add(dsn);
        n = dsn;
      }
      else if (c instanceof MapOperatorBase) {
        n = new MapNode((MapOperatorBase<?, ?, ?>) c);
      }
      else if (c instanceof MapPartitionOperatorBase) {
        n = new MapPartitionNode((MapPartitionOperatorBase<?, ?, ?>) c);
      }
      else if (c instanceof org.apache.flink.api.common.operators.base.CollectorMapOperatorBase) {
        n = new CollectorMapNode((org.apache.flink.api.common.operators.base.CollectorMapOperatorBase<?, ?, ?>) c);
      }
      else if (c instanceof FlatMapOperatorBase) {
        n = new FlatMapNode((FlatMapOperatorBase<?, ?, ?>) c);
      }
      else if (c instanceof FilterOperatorBase) {
        n = new FilterNode((FilterOperatorBase<?, ?>) c);
      }
      else if (c instanceof ReduceOperatorBase) {
        n = new ReduceNode((ReduceOperatorBase<?, ?>) c);
      }
      else if (c instanceof GroupReduceOperatorBase) {
        n = new GroupReduceNode((GroupReduceOperatorBase<?, ?, ?>) c);
      }
      else if (c instanceof JoinOperatorBase) {
        n = new MatchNode((JoinOperatorBase<?, ?, ?, ?>) c);
      }
      else if (c instanceof CoGroupOperatorBase) {
        n = new CoGroupNode((CoGroupOperatorBase<?, ?, ?, ?>) c);
      }
      else if (c instanceof CrossOperatorBase) {
        n = new CrossNode((CrossOperatorBase<?, ?, ?, ?>) c);
      }
      else if (c instanceof BulkIterationBase) {
        n = new BulkIterationNode((BulkIterationBase<?>) c);
      }
      else if (c instanceof DeltaIterationBase) {
        n = new WorksetIterationNode((DeltaIterationBase<?, ?>) c);
      }
      else if (c instanceof Union){
        n = new BinaryUnionNode((Union<?>) c);
      }
      else if (c instanceof PartitionOperatorBase) {
        n = new PartitionNode((PartitionOperatorBase<?>) c);
      }
      else if (c instanceof PartialSolutionPlaceHolder) {
        if (this.parent == null) {
          throw new InvalidProgramException("It is currently not supported to create data sinks inside iterations.");
        }
        
        final PartialSolutionPlaceHolder<?> holder = (PartialSolutionPlaceHolder<?>) c;
        final BulkIterationBase<?> enclosingIteration = holder.getContainingBulkIteration();
        final BulkIterationNode containingIterationNode =
              (BulkIterationNode) this.parent.con2node.get(enclosingIteration);
        
        // catch this for the recursive translation of step functions
        BulkPartialSolutionNode p = new BulkPartialSolutionNode(holder, containingIterationNode);
        p.setDegreeOfParallelism(containingIterationNode.getDegreeOfParallelism());
        n = p;
      }
      else if (c instanceof WorksetPlaceHolder) {
        if (this.parent == null) {
          throw new InvalidProgramException("It is currently not supported to create data sinks inside iterations.");
        }
        
        final WorksetPlaceHolder<?> holder = (WorksetPlaceHolder<?>) c;
        final DeltaIterationBase<?, ?> enclosingIteration = holder.getContainingWorksetIteration();
        final WorksetIterationNode containingIterationNode =
              (WorksetIterationNode) this.parent.con2node.get(enclosingIteration);
        
        // catch this for the recursive translation of step functions
        WorksetNode p = new WorksetNode(holder, containingIterationNode);
        p.setDegreeOfParallelism(containingIterationNode.getDegreeOfParallelism());
        n = p;
      }
      else if (c instanceof SolutionSetPlaceHolder) {
        if (this.parent == null) {
          throw new InvalidProgramException("It is currently not supported to create data sinks inside iterations.");
        }
        
        final SolutionSetPlaceHolder<?> holder = (SolutionSetPlaceHolder<?>) c;
        final DeltaIterationBase<?, ?> enclosingIteration = holder.getContainingWorksetIteration();
        final WorksetIterationNode containingIterationNode =
              (WorksetIterationNode) this.parent.con2node.get(enclosingIteration);
        
        // catch this for the recursive translation of step functions
        SolutionSetNode p = new SolutionSetNode(holder, containingIterationNode);
        p.setDegreeOfParallelism(containingIterationNode.getDegreeOfParallelism());
        n = p;
      }
      else {
        throw new IllegalArgumentException("Unknown operator type: " + c);
      }


      this.con2node.put(c, n);
      
      // set the parallelism only if it has not been set before. some nodes have a fixed DOP, such as the
      // key-less reducer (all-reduce)
      if (n.getDegreeOfParallelism() < 1) {
        // set the degree of parallelism
        int par = c.getDegreeOfParallelism();
        if (par > 0) {
          if (this.forceDOP && par != this.defaultParallelism) {
            par = this.defaultParallelism;
            LOG.warn("The degree-of-parallelism of nested Dataflows (such as step functions in iterations) is " +
              "currently fixed to the degree-of-parallelism of the surrounding operator (the iteration).");
          }
        } else {
          par = this.defaultParallelism;
        }
        n.setDegreeOfParallelism(par);
      }


      return true;
    }

View Full Code Here

    }


    @Override
    public void postVisit(Operator<?> c) {
      
      OptimizerNode n = this.con2node.get(c);


      // first connect to the predecessors
      n.setInput(this.con2node);
      n.setBroadcastInputs(this.con2node);
      
      // if the node represents a bulk iteration, we recursively translate the data flow now
      if (n instanceof BulkIterationNode) {
        final BulkIterationNode iterNode = (BulkIterationNode) n;
        final BulkIterationBase<?> iter = iterNode.getIterationContract();


        // pass a copy of the no iterative part into the iteration translation,
        // in case the iteration references its closure
        HashMap<Operator<?>, OptimizerNode> closure = new HashMap<Operator<?>, OptimizerNode>(con2node);


        // first, recursively build the data flow for the step function
        final GraphCreatingVisitor recursiveCreator = new GraphCreatingVisitor(this, true,
          iterNode.getDegreeOfParallelism(), closure);
        
        BulkPartialSolutionNode partialSolution = null;
        
        iter.getNextPartialSolution().accept(recursiveCreator);
        
        partialSolution =  (BulkPartialSolutionNode) recursiveCreator.con2node.get(iter.getPartialSolution());
        OptimizerNode rootOfStepFunction = recursiveCreator.con2node.get(iter.getNextPartialSolution());
        if (partialSolution == null) {
          throw new CompilerException("Error: The step functions result does not depend on the partial solution.");
        }
        
        
        OptimizerNode terminationCriterion = null;
        
        if (iter.getTerminationCriterion() != null) {
          terminationCriterion = recursiveCreator.con2node.get(iter.getTerminationCriterion());
          
          // no intermediate node yet, traverse from the termination criterion to build the missing parts
          if (terminationCriterion == null) {
            iter.getTerminationCriterion().accept(recursiveCreator);
            terminationCriterion = recursiveCreator.con2node.get(iter.getTerminationCriterion());
          }
        }
        
        iterNode.setPartialSolution(partialSolution);
        iterNode.setNextPartialSolution(rootOfStepFunction, terminationCriterion);
        
        // go over the contained data flow and mark the dynamic path nodes
        StaticDynamicPathIdentifier identifier = new StaticDynamicPathIdentifier(iterNode.getCostWeight());
        rootOfStepFunction.accept(identifier);
        if(terminationCriterion != null){
          terminationCriterion.accept(identifier);
        }
      }
      else if (n instanceof WorksetIterationNode) {
        final WorksetIterationNode iterNode = (WorksetIterationNode) n;
        final DeltaIterationBase<?, ?> iter = iterNode.getIterationContract();


        // we need to ensure that both the next-workset and the solution-set-delta depend on the workset. One check is for free
        // during the translation, we do the other check here as a pre-condition
        {
          WorksetFinder wsf = new WorksetFinder();
          iter.getNextWorkset().accept(wsf);
          if (!wsf.foundWorkset) {
            throw new CompilerException("In the given program, the next workset does not depend on the workset. This is a prerequisite in delta iterations.");
          }
        }
        
        // calculate the closure of the anonymous function
        HashMap<Operator<?>, OptimizerNode> closure = new HashMap<Operator<?>, OptimizerNode>(con2node);


        // first, recursively build the data flow for the step function
        final GraphCreatingVisitor recursiveCreator = new GraphCreatingVisitor(this, true, iterNode.getDegreeOfParallelism(), closure);
        
        // descend from the solution set delta. check that it depends on both the workset
        // and the solution set. If it does depend on both, this descend should create both nodes
        iter.getSolutionSetDelta().accept(recursiveCreator);
        
        final WorksetNode worksetNode = (WorksetNode) recursiveCreator.con2node.get(iter.getWorkset());
        
        if (worksetNode == null) {
          throw new CompilerException("In the given program, the solution set delta does not depend on the workset. This is a prerequisite in delta iterations.");
        }
        
        iter.getNextWorkset().accept(recursiveCreator);
        
        SolutionSetNode solutionSetNode = (SolutionSetNode) recursiveCreator.con2node.get(iter.getSolutionSet());
        
        if (solutionSetNode == null || solutionSetNode.getOutgoingConnections() == null || solutionSetNode.getOutgoingConnections().isEmpty()) {
          solutionSetNode = new SolutionSetNode((SolutionSetPlaceHolder<?>) iter.getSolutionSet(), iterNode);
        }
        else {
          for (PactConnection conn : solutionSetNode.getOutgoingConnections()) {
            OptimizerNode successor = conn.getTarget();
          
            if (successor.getClass() == MatchNode.class) {
              // find out which input to the match the solution set is
              MatchNode mn = (MatchNode) successor;
              if (mn.getFirstPredecessorNode() == solutionSetNode) {
                mn.makeJoinWithSolutionSet(0);
              } else if (mn.getSecondPredecessorNode() == solutionSetNode) {
                mn.makeJoinWithSolutionSet(1);
              } else {
                throw new CompilerException();
              }
            }
            else if (successor.getClass() == CoGroupNode.class) {
              CoGroupNode cg = (CoGroupNode) successor;
              if (cg.getFirstPredecessorNode() == solutionSetNode) {
                cg.makeCoGroupWithSolutionSet(0);
              } else if (cg.getSecondPredecessorNode() == solutionSetNode) {
                cg.makeCoGroupWithSolutionSet(1);
              } else {
                throw new CompilerException();
              }
            }
            else {
              throw new CompilerException("Error: The only operations allowed on the solution set are Join and CoGroup.");
            }
          }
        }
        
        final OptimizerNode nextWorksetNode = recursiveCreator.con2node.get(iter.getNextWorkset());
        final OptimizerNode solutionSetDeltaNode = recursiveCreator.con2node.get(iter.getSolutionSetDelta());
        
        // set the step function nodes to the iteration node
        iterNode.setPartialSolution(solutionSetNode, worksetNode);
        iterNode.setNextPartialSolution(solutionSetDeltaNode, nextWorksetNode);

View Full Code Here

TOP

Related Classes of org.apache.flink.compiler.dag.OptimizerNode

org.apache.flink.api.common.operators.CompilerHints

org.apache.flink.compiler.dataproperties.InterestingProperties

org.apache.flink.compiler.PactCompiler

org.apache.flink.compiler.PactCompiler$GraphCreatingVisitor

org.apache.flink.compiler.plan.BulkIterationPlanNode

org.apache.flink.compiler.plan.PlanNode

org.apache.flink.compiler.plan.WorksetIterationPlanNode

org.apache.flink.compiler.plandump.PlanJSONDumpGenerator

org.apache.flink.compiler.CompilerException

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.