Package org.apache.flink.compiler.dataproperties

Examples of org.apache.flink.compiler.dataproperties.GlobalProperties


    return new DualInputPlanNode(node, "CoGroup ("+node.getPactContract().getName()+")", in1, in2, DriverStrategy.CO_GROUP, this.keys1, this.keys2, inputOrders);
  }
 
  @Override
  public GlobalProperties computeGlobalProperties(GlobalProperties in1, GlobalProperties in2) {
    GlobalProperties gp = GlobalProperties.combine(in1, in2);
    if (gp.getUniqueFieldCombination() != null && gp.getUniqueFieldCombination().size() > 0 &&
          gp.getPartitioning() == PartitioningProperty.RANDOM)
    {
      gp.setAnyPartitioning(gp.getUniqueFieldCombination().iterator().next().toFieldList());
    }
    gp.clearUniqueFieldCombinations();
    return gp;
  }
View Full Code Here


      List<PlanNode> newCandidates = new ArrayList<PlanNode>();
     
      for (Iterator<PlanNode> planDeleter = candidates.iterator(); planDeleter.hasNext(); ) {
        PlanNode candidate = planDeleter.next();
       
        GlobalProperties atEndGlobal = candidate.getGlobalProperties();
        LocalProperties atEndLocal = candidate.getLocalProperties();
       
        FeedbackPropertiesMeetRequirementsReport report = candidate.checkPartialSolutionPropertiesMet(pspn, atEndGlobal, atEndLocal);
        if (report == FeedbackPropertiesMeetRequirementsReport.NO_PARTIAL_SOLUTION) {
          ; // depends only through broadcast variable on the partial solution
        }
        else if (report == FeedbackPropertiesMeetRequirementsReport.NOT_MET) {
          // attach a no-op node through which we create the properties of the original input
          Channel toNoOp = new Channel(candidate);
          globPropsReq.parameterizeChannel(toNoOp, false);
          locPropsReq.parameterizeChannel(toNoOp);
         
          UnaryOperatorNode rebuildPropertiesNode = new UnaryOperatorNode("Rebuild Partial Solution Properties", FieldList.EMPTY_LIST);
          rebuildPropertiesNode.setDegreeOfParallelism(candidate.getDegreeOfParallelism());
         
          SingleInputPlanNode rebuildPropertiesPlanNode = new SingleInputPlanNode(rebuildPropertiesNode, "Rebuild Partial Solution Properties", toNoOp, DriverStrategy.UNARY_NO_OP);
          rebuildPropertiesPlanNode.initProperties(toNoOp.getGlobalProperties(), toNoOp.getLocalProperties());
          estimator.costOperator(rebuildPropertiesPlanNode);
           
          GlobalProperties atEndGlobalModified = rebuildPropertiesPlanNode.getGlobalProperties();
          LocalProperties atEndLocalModified = rebuildPropertiesPlanNode.getLocalProperties();
           
          if (!(atEndGlobalModified.equals(atEndGlobal) && atEndLocalModified.equals(atEndLocal))) {
            FeedbackPropertiesMeetRequirementsReport report2 = candidate.checkPartialSolutionPropertiesMet(pspn, atEndGlobalModified, atEndLocalModified);
           
            if (report2 != FeedbackPropertiesMeetRequirementsReport.NOT_MET) {
              newCandidates.add(rebuildPropertiesPlanNode);
            }
          }
         
          planDeleter.remove();
        }
      }
    }
   
    if (candidates.isEmpty()) {
      return;
    }
   
    // 5) Create a candidate for the Iteration Node for every remaining plan of the step function.
    if (terminationCriterion == null) {
      for (PlanNode candidate : candidates) {
        BulkIterationPlanNode node = new BulkIterationPlanNode(this, "BulkIteration ("+this.getPactContract().getName()+")", in, pspn, candidate);
        GlobalProperties gProps = candidate.getGlobalProperties().clone();
        LocalProperties lProps = candidate.getLocalProperties().clone();
        node.initProperties(gProps, lProps);
        target.add(node);
      }
    }
    else if (candidates.size() > 0) {
      List<PlanNode> terminationCriterionCandidates = this.terminationCriterion.getAlternativePlans(estimator);

      SingleRootJoiner singleRoot = (SingleRootJoiner) this.singleRoot;
     
      for (PlanNode candidate : candidates) {
        for (PlanNode terminationCandidate : terminationCriterionCandidates) {
          if (singleRoot.areBranchCompatible(candidate, terminationCandidate)) {
            BulkIterationPlanNode node = new BulkIterationPlanNode(this, "BulkIteration ("+this.getPactContract().getName()+")", in, pspn, candidate, terminationCandidate);
            GlobalProperties gProps = candidate.getGlobalProperties().clone();
            LocalProperties lProps = candidate.getLocalProperties().clone();
            node.initProperties(gProps, lProps);
            target.add(node);
           
          }
View Full Code Here

    return pairs;
  }

  @Override
  public GlobalProperties computeGlobalProperties(GlobalProperties in1, GlobalProperties in2) {
    GlobalProperties gp = GlobalProperties.combine(in1, in2);
    if (gp.getUniqueFieldCombination() != null && gp.getUniqueFieldCombination().size() > 0 &&
          gp.getPartitioning() == PartitioningProperty.RANDOM)
    {
      gp.setAnyPartitioning(gp.getUniqueFieldCombination().iterator().next().toFieldList());
    }
    gp.clearUniqueFieldCombinations();
    return gp;
  }
View Full Code Here

    final int[] ssKeys = iteration.getSolutionSetKeyFields();
    if (ssKeys == null || ssKeys.length == 0) {
      throw new CompilerException("Invalid WorksetIteration: No key fields defined for the solution set.");
    }
    this.solutionSetKeyFields = new FieldList(ssKeys);
    this.partitionedProperties = new GlobalProperties();
    this.partitionedProperties.setHashPartitioned(this.solutionSetKeyFields);
   
    int weight = iteration.getMaximumNumberOfIterations() > 0 ?
      iteration.getMaximumNumberOfIterations() : DEFAULT_COST_WEIGHT;
     
View Full Code Here

      List<PlanNode> newCandidates = new ArrayList<PlanNode>();
     
      for (Iterator<PlanNode> planDeleter = worksetCandidates.iterator(); planDeleter.hasNext(); ) {
        PlanNode candidate = planDeleter.next();
       
        GlobalProperties atEndGlobal = candidate.getGlobalProperties();
        LocalProperties atEndLocal = candidate.getLocalProperties();
       
        FeedbackPropertiesMeetRequirementsReport report = candidate.checkPartialSolutionPropertiesMet(wspn, atEndGlobal, atEndLocal);
        if (report == FeedbackPropertiesMeetRequirementsReport.NO_PARTIAL_SOLUTION) {
          ; // depends only through broadcast variable on the workset solution
        }
        else if (report == FeedbackPropertiesMeetRequirementsReport.NOT_MET) {
          // attach a no-op node through which we create the properties of the original input
          Channel toNoOp = new Channel(candidate);
          globPropsReqWorkset.parameterizeChannel(toNoOp, false);
          locPropsReqWorkset.parameterizeChannel(toNoOp);
         
          UnaryOperatorNode rebuildWorksetPropertiesNode = new UnaryOperatorNode("Rebuild Workset Properties", FieldList.EMPTY_LIST);
         
          rebuildWorksetPropertiesNode.setDegreeOfParallelism(candidate.getDegreeOfParallelism());
         
          SingleInputPlanNode rebuildWorksetPropertiesPlanNode = new SingleInputPlanNode(rebuildWorksetPropertiesNode, "Rebuild Workset Properties", toNoOp, DriverStrategy.UNARY_NO_OP);
          rebuildWorksetPropertiesPlanNode.initProperties(toNoOp.getGlobalProperties(), toNoOp.getLocalProperties());
          estimator.costOperator(rebuildWorksetPropertiesPlanNode);
           
          GlobalProperties atEndGlobalModified = rebuildWorksetPropertiesPlanNode.getGlobalProperties();
          LocalProperties atEndLocalModified = rebuildWorksetPropertiesPlanNode.getLocalProperties();
           
          if (!(atEndGlobalModified.equals(atEndGlobal) && atEndLocalModified.equals(atEndLocal))) {
            FeedbackPropertiesMeetRequirementsReport report2 = candidate.checkPartialSolutionPropertiesMet(wspn, atEndGlobalModified, atEndLocalModified);
           
            if (report2 != FeedbackPropertiesMeetRequirementsReport.NOT_MET) {
              newCandidates.add(rebuildWorksetPropertiesPlanNode);
            }
          }
         
          // remove the original operator and add the modified candidate
          planDeleter.remove();
         
        }
      }
     
      worksetCandidates.addAll(newCandidates);
    }
   
    if (worksetCandidates.isEmpty()) {
      return;
    }
   
    // sanity check the solution set delta
    for (Iterator<PlanNode> deltaPlans = solutionSetDeltaCandidates.iterator(); deltaPlans.hasNext(); ) {
      SingleInputPlanNode candidate = (SingleInputPlanNode) deltaPlans.next();
      GlobalProperties gp = candidate.getGlobalProperties();
     
      if (gp.getPartitioning() != PartitioningProperty.HASH_PARTITIONED || gp.getPartitioningFields() == null ||
          !gp.getPartitioningFields().equals(this.solutionSetKeyFields))
      {
        throw new CompilerException("Bug: The solution set delta is not partitioned.");
      }
    }
   
    // 5) Create a candidate for the Iteration Node for every remaining plan of the step function.
   
    final GlobalProperties gp = new GlobalProperties();
    gp.setHashPartitioned(this.solutionSetKeyFields);
    gp.addUniqueFieldCombination(this.solutionSetKeyFields);
   
    LocalProperties lp = LocalProperties.EMPTY.addUniqueFields(this.solutionSetKeyFields);
   
    // take all combinations of solution set delta and workset plans
    for (PlanNode solutionSetCandidate : solutionSetDeltaCandidates) {
View Full Code Here

    return new DualInputPlanNode(node, "Cross("+node.getPactContract().getName()+")", in1, in2, getStrategy());
  }
 
  @Override
  public GlobalProperties computeGlobalProperties(GlobalProperties in1, GlobalProperties in2) {
    GlobalProperties gp = GlobalProperties.combine(in1, in2);
    if (gp.getUniqueFieldCombination() != null && gp.getUniqueFieldCombination().size() > 0 &&
          gp.getPartitioning() == PartitioningProperty.RANDOM)
    {
      gp.setAnyPartitioning(gp.getUniqueFieldCombination().iterator().next().toFieldList());
    }
    gp.clearUniqueFieldCombinations();
    return gp;
  }
View Full Code Here

      placePipelineBreakersIfNecessary(operator.getStrategy(), in1, in2);
     
      DualInputPlanNode node = operator.instantiate(in1, in2, this);
      node.setBroadcastInputs(broadcastChannelsCombination);
     
      GlobalProperties gp1 = in1.getGlobalProperties().clone().filterByNodesConstantSet(this, 0);
      GlobalProperties gp2 = in2.getGlobalProperties().clone().filterByNodesConstantSet(this, 1);
      GlobalProperties combined = operator.computeGlobalProperties(gp1, gp2);

      LocalProperties lp1 = in1.getLocalProperties().clone().filterByNodesConstantSet(this, 0);
      LocalProperties lp2 = in2.getLocalProperties().clone().filterByNodesConstantSet(this, 1);
      LocalProperties locals = operator.computeLocalProperties(lp1, lp2);
     
View Full Code Here

      }
    }
   
    {
      // output node global properties
      final GlobalProperties gp = p.getGlobalProperties();

      writer.print(",\n\t\t\"global_properties\": [\n");

      addProperty(writer, "Partitioning", gp.getPartitioning().name(), true);
      if (gp.getPartitioningFields() != null) {
        addProperty(writer, "Partitioned on", gp.getPartitioningFields().toString(), false);
      }
      if (gp.getPartitioningOrdering() != null) {
        addProperty(writer, "Partitioning Order", gp.getPartitioningOrdering().toString(), false)
      }
      else {
        addProperty(writer, "Partitioning Order", "(none)", false);
      }
      if (n.getUniqueFields() == null || n.getUniqueFields().size() == 0) {
View Full Code Here

   * @param template The template optimizer node that this candidate is created for.
   */
  public SourcePlanNode(DataSourceNode template, String nodeName) {
    super(template, nodeName, DriverStrategy.NONE);
   
    this.globalProps = new GlobalProperties();
    this.localProps = new LocalProperties();
    updatePropertiesWithUniqueSets(template.getUniqueFields());
  }
View Full Code Here

     
      final SingleInputPlanNode node = dps.instantiate(in, this);
      node.setBroadcastInputs(broadcastChannelsCombination);
     
      // compute how the strategy affects the properties
      GlobalProperties gProps = in.getGlobalProperties().clone();
      LocalProperties lProps = in.getLocalProperties().clone();
      gProps = dps.computeGlobalProperties(gProps);
      lProps = dps.computeLocalProperties(lProps);
     
      // filter by the user code field copies
      gProps = gProps.filterByNodesConstantSet(this, 0);
      lProps = lProps.filterByNodesConstantSet(this, 0);
     
      // apply
      node.initProperties(gProps, lProps);
      node.updatePropertiesWithUniqueSets(getUniqueFields());
View Full Code Here

TOP

Related Classes of org.apache.flink.compiler.dataproperties.GlobalProperties

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.