Examples of eu.stratosphere.compiler.plan.PlanNode

eu.stratosphere.compiler.plan.PlanNode
The representation of a data exchange between to operators. The data exchange can realize a shipping strategy, which established global properties, and a local strategy, which establishes local properties.
Because we currently deal only with plans where the operator order is fixed, many properties are equal among candidates and are determined prior to the enumeration (such as for example constant/dynamic path membership). Hence, many methods will delegate to the {@code OptimizerNode} that represents the node this candidate wascreated for.

  }


  protected void instantiateCandidate(OperatorDescriptorSingle dps, Channel in, List<Set<? extends NamedChannel>> broadcastPlanChannels,
      List<PlanNode> target, CostEstimator estimator, RequestedGlobalProperties globPropsReq, RequestedLocalProperties locPropsReq)
  {
    final PlanNode inputSource = in.getSource();
    
    for (List<NamedChannel> broadcastChannelsCombination: Sets.cartesianProduct(broadcastPlanChannels)) {
      
      boolean validCombination = true;
      
      // check whether the broadcast inputs use the same plan candidate at the branching point
      for (int i = 0; i < broadcastChannelsCombination.size(); i++) {
        NamedChannel nc = broadcastChannelsCombination.get(i);
        PlanNode bcSource = nc.getSource();
        
        // check branch compatibility against input
        if (!areBranchCompatible(bcSource, inputSource)) {
          validCombination = false;
          break;
        }
        
        // check branch compatibility against all other broadcast variables
        for (int k = 0; k < i; k++) {
          PlanNode otherBcSource = broadcastChannelsCombination.get(k).getSource();
          
          if (!areBranchCompatible(bcSource, otherBcSource)) {
            validCombination = false;
            break;
          }

View Full Code Here

    List<PlanNode> candidates = this.nextPartialSolution.getAlternativePlans(estimator);
    
    // 4) Throw away all that are not compatible with the properties currently requested to the
    //    initial partial solution
    for (Iterator<PlanNode> planDeleter = candidates.iterator(); planDeleter.hasNext(); ) {
      PlanNode candidate = planDeleter.next();
      if (!(globPropsReq.isMetBy(candidate.getGlobalProperties()) && locPropsReq.isMetBy(candidate.getLocalProperties()))) {
        planDeleter.remove();
      }
    }
    
    // 5) Create a candidate for the Iteration Node for every remaining plan of the step function.
    if (terminationCriterion == null) {
      for (PlanNode candidate : candidates) {
        BulkIterationPlanNode node = new BulkIterationPlanNode(this, "BulkIteration ("+this.getPactContract().getName()+")", in, pspn, candidate);
        GlobalProperties gProps = candidate.getGlobalProperties().clone();
        LocalProperties lProps = candidate.getLocalProperties().clone();
        node.initProperties(gProps, lProps);
        target.add(node);
      }
    }
    else if(candidates.size() > 0) {
      List<PlanNode> terminationCriterionCandidates = this.terminationCriterion.getAlternativePlans(estimator);


      SingleRootJoiner singleRoot = (SingleRootJoiner) this.singleRoot;
      
      for (PlanNode candidate : candidates) {
        for(PlanNode terminationCandidate : terminationCriterionCandidates) {
          if (singleRoot.areBranchCompatible(candidate, terminationCandidate)) {
            BulkIterationPlanNode node = new BulkIterationPlanNode(this, "BulkIteration ("+this.getPactContract().getName()+")", in, pspn, candidate, terminationCandidate);
            GlobalProperties gProps = candidate.getGlobalProperties().clone();
            LocalProperties lProps = candidate.getLocalProperties().clone();
            node.initProperties(gProps, lProps);
            target.add(node);
            
          }
        }

View Full Code Here

      Comparator<PlanNode> sorter = new Comparator<PlanNode>() {
        
        @Override
        public int compare(PlanNode o1, PlanNode o2) {
          for (int i = 0; i < branchDeterminers.length; i++) {
            PlanNode n1 = o1.getCandidateAtBranchPoint(branchDeterminers[i]);
            PlanNode n2 = o2.getCandidateAtBranchPoint(branchDeterminers[i]);
            int hash1 = System.identityHashCode(n1);
            int hash2 = System.identityHashCode(n2);
            
            if (hash1 != hash2) {
              return hash1 - hash2;
            }
          }
          return 0;
        }
      };
      Collections.sort(plans, sorter);
      
      List<PlanNode> result = new ArrayList<PlanNode>();
      List<PlanNode> turn = new ArrayList<PlanNode>();
      
      final PlanNode[] determinerChoice = new PlanNode[branchDeterminers.length];


      while (!plans.isEmpty()) {
        // take one as the determiner
        turn.clear();
        PlanNode determiner = plans.remove(plans.size() - 1);
        turn.add(determiner);
        
        for (int i = 0; i < determinerChoice.length; i++) {
          determinerChoice[i] = determiner.getCandidateAtBranchPoint(branchDeterminers[i]);
        }


        // go backwards through the plans and find all that are equal
        boolean stillEqual = true;
        for (int k = plans.size() - 1; k >= 0 && stillEqual; k--) {
          PlanNode toCheck = plans.get(k);
          
          for (int i = 0; i < branchDeterminers.length; i++) {
            PlanNode checkerChoice = toCheck.getCandidateAtBranchPoint(branchDeterminers[i]);
          
            if (checkerChoice != determinerChoice[i]) {
              // not the same anymore
              stillEqual = false;
              break;

View Full Code Here

    
    final PlanNode[][] toKeep = new PlanNode[gps.length][];
    final PlanNode[] cheapestForGlobal = new PlanNode[gps.length];
    
    
    PlanNode cheapest = null; // the overall cheapest plan


    // go over all plans from the list
    for (PlanNode candidate : plans) {
      // check if that plan is the overall cheapest
      if (cheapest == null || (cheapest.getCumulativeCosts().compareTo(candidate.getCumulativeCosts()) > 0)) {
        cheapest = candidate;
      }


      // find the interesting global properties that this plan matches
      for (int i = 0; i < gps.length; i++) {
        if (gps[i].isMetBy(candidate.getGlobalProperties())) {
          // the candidate meets the global property requirements. That means
          // it has a chance that its local properties are re-used (they would be
          // destroyed if global properties need to be established)
          
          if (cheapestForGlobal[i] == null || (cheapestForGlobal[i].getCumulativeCosts().compareTo(candidate.getCumulativeCosts()) > 0)) {
            cheapestForGlobal[i] = candidate;
          }
          
          final PlanNode[] localMatches;
          if (toKeep[i] == null) {
            localMatches = new PlanNode[lps.length];
            toKeep[i] = localMatches;
          } else {
            localMatches = toKeep[i];
          }
          
          for (int k = 0; k < lps.length; k++) {
            if (lps[k].isMetBy(candidate.getLocalProperties())) {
              final PlanNode previous = localMatches[k];
              if (previous == null || previous.getCumulativeCosts().compareTo(candidate.getCumulativeCosts()) > 0) {
                // this one is cheaper!
                localMatches[k] = candidate;
              }
            }
          }
        }
      }
    }


    // all plans are set now
    plans.clear();


    // add the cheapest plan
    if (cheapest != null) {
      plans.add(cheapest);
      cheapest.setPruningMarker(); // remember that that plan is in the set
    }
    
    // skip the top down delta cost check for now (TODO: implement this)
    // add all others, which are optimal for some interesting properties
    for (int i = 0; i < gps.length; i++) {
      if (toKeep[i] != null) {
        final PlanNode[] localMatches = toKeep[i];
        for (int k = 0; k < localMatches.length; k++) {
          final PlanNode n = localMatches[k];
          if (n != null && !n.isPruneMarkerSet()) {
            n.setPruningMarker();
            plans.add(n);
          }
        }
      }
      if (cheapestForGlobal[i] != null) {
        final PlanNode n = cheapestForGlobal[i];
        if (!n.isPruneMarkerSet()) {
          n.setPruningMarker();
          plans.add(n);
        }
      }
    }
  }

View Full Code Here

    if (this.hereJoinedBranches == null || this.hereJoinedBranches.isEmpty()) {
      return true;
    }


    for (OptimizerNode joinedBrancher : hereJoinedBranches) {
      final PlanNode branch1Cand = plan1.getCandidateAtBranchPoint(joinedBrancher);
      final PlanNode branch2Cand = plan2.getCandidateAtBranchPoint(joinedBrancher);
      
      if (branch1Cand != null && branch2Cand != null && branch1Cand != branch2Cand) {
        return false;
      }
    }

View Full Code Here

      if (visitable instanceof BinaryUnionPlanNode) {
        final BinaryUnionPlanNode unionNode = (BinaryUnionPlanNode) visitable;
        final Channel in1 = unionNode.getInput1();
        final Channel in2 = unionNode.getInput2();
      
        PlanNode newUnionNode;
        
        // if any input is cached, we keep this as a binary union and do not collapse it into a
        // n-ary union
//        if (in1.getTempMode().isCached() || in2.getTempMode().isCached()) {
//          // replace this node by an explicit operator

View Full Code Here

    
    //---------------------------------------------------------------------------------------
    // the part below here is relevant only to plan nodes with concrete strategies, etc
    //---------------------------------------------------------------------------------------


    final PlanNode p = node.getPlanNode();
    if (p == null) {
      // finish node
      writer.print("\n\t}");
      return;
    }
    // local strategy
    String locString = null;
    if (p.getDriverStrategy() != null) {
      switch (p.getDriverStrategy()) {
      case NONE:
      case BINARY_NO_OP:
        break;
        
      case UNARY_NO_OP:
        locString = "No-Op";
        break;
        
      case COLLECTOR_MAP:
      case MAP:
      case FLAT_MAP:
        locString = "Map";
        break;
      
      case ALL_REDUCE:
        locString = "Reduce All";
        break;
      
      case ALL_GROUP_REDUCE:
      case ALL_GROUP_COMBINE:
        locString = "Group Reduce All";
        break;
        
      case SORTED_REDUCE:
        locString = "Sorted Reduce";
        break;
        
      case SORTED_PARTIAL_REDUCE:
        locString = "Sorted Combine/Reduce";
        break;


      case SORTED_GROUP_REDUCE:
        locString = "Sorted Group Reduce";
        break;
        
      case SORTED_GROUP_COMBINE:
        locString = "Sorted Combine";
        break;


      case HYBRIDHASH_BUILD_FIRST:
        locString = "Hybrid Hash (build: " + child1name + ")";
        break;
      case HYBRIDHASH_BUILD_SECOND:
        locString = "Hybrid Hash (build: " + child2name + ")";
        break;


      case NESTEDLOOP_BLOCKED_OUTER_FIRST:
        locString = "Nested Loops (Blocked Outer: " + child1name + ")";
        break;
      case NESTEDLOOP_BLOCKED_OUTER_SECOND:
        locString = "Nested Loops (Blocked Outer: " + child2name + ")";
        break;
      case NESTEDLOOP_STREAMED_OUTER_FIRST:
        locString = "Nested Loops (Streamed Outer: " + child1name + ")";
        break;
      case NESTEDLOOP_STREAMED_OUTER_SECOND:
        locString = "Nested Loops (Streamed Outer: " + child2name + ")";
        break;


      case MERGE:
        locString = "Merge";
        break;


      case CO_GROUP:
        locString = "Co-Group";
        break;


      default:
        throw new CompilerException("Unknown local strategy '" + p.getDriverStrategy().name()
          + "' in JSON generator.");
      }


      if (locString != null) {
        writer.print(",\n\t\t\"driver_strategy\": \"");
        writer.print(locString);
        writer.print("\"");
      }
    }
    
    {
      // output node global properties
      final GlobalProperties gp = p.getGlobalProperties();


      writer.print(",\n\t\t\"global_properties\": [\n");


      addProperty(writer, "Partitioning", gp.getPartitioning().name(), true);
      if (gp.getPartitioningFields() != null) {
        addProperty(writer, "Partitioned on", gp.getPartitioningFields().toString(), false);
      }
      if (gp.getPartitioningOrdering() != null) {
        addProperty(writer, "Partitioning Order", gp.getPartitioningOrdering().toString(), false);  
      }
      else {
        addProperty(writer, "Partitioning Order", "(none)", false);
      }
      if (n.getUniqueFields() == null || n.getUniqueFields().size() == 0) {
        addProperty(writer, "Uniqueness", "not unique", false);
      }
      else {
        addProperty(writer, "Uniqueness", n.getUniqueFields().toString(), false);  
      }


      writer.print("\n\t\t]");
    }


    {
      // output node local properties
      LocalProperties lp = p.getLocalProperties();


      writer.print(",\n\t\t\"local_properties\": [\n");


      if (lp.getOrdering() != null) {
        addProperty(writer, "Order", lp.getOrdering().toString(), true);  
      }
      else {
        addProperty(writer, "Order", "(none)", true);
      }
      if (lp.getGroupedFields() != null && lp.getGroupedFields().size() > 0) {
        addProperty(writer, "Grouped on", lp.getGroupedFields().toString(), false);
      } else {
        addProperty(writer, "Grouping", "not grouped", false);  
      }
      if (n.getUniqueFields() == null || n.getUniqueFields().size() == 0) {
        addProperty(writer, "Uniqueness", "not unique", false);
      }
      else {
        addProperty(writer, "Uniqueness", n.getUniqueFields().toString(), false);  
      }


      writer.print("\n\t\t]");
    }


    // output node size estimates
    writer.print(",\n\t\t\"estimates\": [\n");


    addProperty(writer, "Est. Output Size", n.getEstimatedOutputSize() == -1 ? "(unknown)"
      : formatNumber(n.getEstimatedOutputSize(), "B"), true);
    addProperty(writer, "Est. Cardinality", n.getEstimatedNumRecords() == -1 ? "(unknown)"
      : formatNumber(n.getEstimatedNumRecords()), false);


    writer.print("\t\t]");


    // output node cost
    if (p.getNodeCosts() != null) {
      writer.print(",\n\t\t\"costs\": [\n");


      addProperty(writer, "Network", p.getNodeCosts().getNetworkCost() == -1 ? "(unknown)"
        : formatNumber(p.getNodeCosts().getNetworkCost(), "B"), true);
      addProperty(writer, "Disk I/O", p.getNodeCosts().getDiskCost() == -1 ? "(unknown)"
        : formatNumber(p.getNodeCosts().getDiskCost(), "B"), false);
      addProperty(writer, "CPU", p.getNodeCosts().getCpuCost() == -1 ? "(unknown)"
        : formatNumber(p.getNodeCosts().getCpuCost(), ""), false);


      addProperty(writer, "Cumulative Network",
        p.getCumulativeCosts().getNetworkCost() == -1 ? "(unknown)" : formatNumber(p
          .getCumulativeCosts().getNetworkCost(), "B"), false);
      addProperty(writer, "Cumulative Disk I/O",
        p.getCumulativeCosts().getDiskCost() == -1 ? "(unknown)" : formatNumber(p
          .getCumulativeCosts().getDiskCost(), "B"), false);
      addProperty(writer, "Cumulative CPU",
        p.getCumulativeCosts().getCpuCost() == -1 ? "(unknown)" : formatNumber(p
          .getCumulativeCosts().getCpuCost(), ""), false);


      writer.print("\n\t\t]");
    }

View Full Code Here

    }
  }
  
  private void traverseChannel(Channel channel) {
    
    PlanNode source = channel.getSource();
    Operator<?> javaOp = source.getPactContract();
    
//    if (!(javaOp instanceof BulkIteration) && !(javaOp instanceof JavaPlanNode)) {
//      throw new RuntimeException("Wrong operator type found in post pass: " + javaOp);
//    }


    TypeInformation<?> type = javaOp.getOperatorInfo().getOutputType();




    if(javaOp instanceof GroupReduceOperatorBase &&
        (source.getDriverStrategy() == DriverStrategy.SORTED_GROUP_COMBINE || source.getDriverStrategy() == DriverStrategy.ALL_GROUP_COMBINE)) {
      GroupReduceOperatorBase<?, ?, ?> groupNode = (GroupReduceOperatorBase<?, ?, ?>) javaOp;
      type = groupNode.getInput().getOperatorInfo().getOutputType();
    }
    else if(javaOp instanceof PlanUnwrappingReduceGroupOperator &&
        source.getDriverStrategy().equals(DriverStrategy.SORTED_GROUP_COMBINE)) {
      PlanUnwrappingReduceGroupOperator<?, ?, ?> groupNode = (PlanUnwrappingReduceGroupOperator<?, ?, ?>) javaOp;
      type = groupNode.getInput().getOperatorInfo().getOutputType();
    }
    
    // the serializer always exists

View Full Code Here

      throw new CompilerException("Error in compiler: more than one best plan was created!");
    }


    // check if the best plan's root is a data sink (single sink plan)
    // if so, directly take it. if it is a sink joiner node, get its contained sinks
    PlanNode bestPlanRoot = bestPlan.get(0);
    List<SinkPlanNode> bestPlanSinks = new ArrayList<SinkPlanNode>(4);


    if (bestPlanRoot instanceof SinkPlanNode) {
      bestPlanSinks.add((SinkPlanNode) bestPlanRoot);
    } else if (bestPlanRoot instanceof SinkJoinerPlanNode) {

View Full Code Here

  protected void instantiate(OperatorDescriptorDual operator, Channel in1, Channel in2,
      List<Set<? extends NamedChannel>> broadcastPlanChannels, List<PlanNode> target, CostEstimator estimator,
      RequestedGlobalProperties globPropsReq1, RequestedGlobalProperties globPropsReq2,
      RequestedLocalProperties locPropsReq1, RequestedLocalProperties locPropsReq2)
  {
    final PlanNode inputSource1 = in1.getSource();
    final PlanNode inputSource2 = in2.getSource();
    
    for (List<NamedChannel> broadcastChannelsCombination: Sets.cartesianProduct(broadcastPlanChannels)) {
      
      boolean validCombination = true;
      
      // check whether the broadcast inputs use the same plan candidate at the branching point
      for (int i = 0; i < broadcastChannelsCombination.size(); i++) {
        NamedChannel nc = broadcastChannelsCombination.get(i);
        PlanNode bcSource = nc.getSource();
        
        if (!(areBranchCompatible(bcSource, inputSource1) || areBranchCompatible(bcSource, inputSource2))) {
          validCombination = false;
          break;
        }
        
        // check branch compatibility against all other broadcast variables
        for (int k = 0; k < i; k++) {
          PlanNode otherBcSource = broadcastChannelsCombination.get(k).getSource();
          
          if (!areBranchCompatible(bcSource, otherBcSource)) {
            validCombination = false;
            break;
          }

View Full Code Here

0 1 2

TOP

Related Classes of eu.stratosphere.compiler.plan.PlanNode

eu.stratosphere.compiler.costs.Costs

eu.stratosphere.compiler.dag.BulkIterationNode

eu.stratosphere.compiler.dag.OptimizerNode

eu.stratosphere.compiler.dag.SingleInputNode

eu.stratosphere.compiler.dag.TwoInputNode

eu.stratosphere.compiler.dag.WorksetIterationNode

eu.stratosphere.compiler.PactCompiler

eu.stratosphere.compiler.PactCompiler$BinaryUnionReplacer

eu.stratosphere.compiler.plandump.PlanJSONDumpGenerator

eu.stratosphere.compiler.plantranslate.NepheleJobGraphGenerator

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.