Examples of GlobalProperties

eu.stratosphere.compiler.dataproperties.GlobalProperties
This class represents global properties of the data at a certain point in the plan. Global properties are properties that describe data across different partitions.
Currently, the properties are the following: A partitioning type (ANY, HASH, RANGE), and EITHER an ordering (for range partitioning) or an FieldSet with the hash partitioning columns.
net.alteiar.campaign.player.tools.GlobalProperties
@author Cody Stoutenburg
org.apache.flink.compiler.dataproperties.GlobalProperties
This class represents global properties of the data at a certain point in the plan. Global properties are properties that describe data across different partitions.
Currently, the properties are the following: A partitioning type (ANY, HASH, RANGE), and EITHER an ordering (for range partitioning) or an FieldSet with the hash partitioning columns.

Examples of org.apache.flink.compiler.dataproperties.GlobalProperties

      toAfterJoin.setLocalStrategy(LocalStrategy.NONE);
      SingleInputPlanNode afterJoin = new SingleInputPlanNode(getMapNode(), "After Join Mapper", toAfterJoin, DriverStrategy.MAP);
      
      // no properties from the partial solution, no required properties
      {
        GlobalProperties gp = new GlobalProperties();
        LocalProperties lp = LocalProperties.EMPTY;
        
        FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
        assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
      }
      
      // some properties from the partial solution, no required properties
      {
        GlobalProperties gp = new GlobalProperties();
        gp.setHashPartitioned(new FieldList(0));
        LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
        
        FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
        assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
      }
      
      // test requirements on one input and met
      {
        GlobalProperties gp = new GlobalProperties();
        gp.setHashPartitioned(new FieldList(0));
        LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
        
        RequestedGlobalProperties rgp = new RequestedGlobalProperties();
        rgp.setHashPartitioned(new FieldList(0));
        
        RequestedLocalProperties rlp = new RequestedLocalProperties();
        rlp.setGroupedFields(new FieldList(2, 1));
        
        toJoin1.setRequiredGlobalProps(rgp);
        toJoin1.setRequiredLocalProps(rlp);
        
        FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
        assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
      }
      
      // test requirements on both input and met
      {
        GlobalProperties gp = new GlobalProperties();
        gp.setHashPartitioned(new FieldList(0));
        LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
        
        RequestedGlobalProperties rgp = new RequestedGlobalProperties();
        rgp.setHashPartitioned(new FieldList(0));
        
        RequestedLocalProperties rlp = new RequestedLocalProperties();
        rlp.setGroupedFields(new FieldList(2, 1));
        
        toJoin1.setRequiredGlobalProps(rgp);
        toJoin1.setRequiredLocalProps(rlp);
        
        toJoin2.setRequiredGlobalProps(rgp);
        toJoin2.setRequiredLocalProps(rlp);
        
        FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
        assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
      }
      
      // test requirements on both inputs, one not met
      {
        GlobalProperties gp = new GlobalProperties();
        gp.setHashPartitioned(new FieldList(0));
        LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
        
        RequestedGlobalProperties rgp1 = new RequestedGlobalProperties();
        rgp1.setHashPartitioned(new FieldList(0));
        
        RequestedLocalProperties rlp1 = new RequestedLocalProperties();
        rlp1.setGroupedFields(new FieldList(2, 1));
        
        RequestedGlobalProperties rgp2 = new RequestedGlobalProperties();
        rgp2.setHashPartitioned(new FieldList(1));
        
        RequestedLocalProperties rlp2 = new RequestedLocalProperties();
        rlp2.setGroupedFields(new FieldList(0, 3));
        
        toJoin1.setRequiredGlobalProps(rgp1);
        toJoin1.setRequiredLocalProps(rlp1);
        
        toJoin2.setRequiredGlobalProps(rgp2);
        toJoin2.setRequiredLocalProps(rlp2);
        
        FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
        assertEquals(NOT_MET, report);
      }
      
      // test override on both inputs, later requirement ignored
      {
        GlobalProperties gp = new GlobalProperties();
        gp.setHashPartitioned(new FieldList(0));
        LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
        
        RequestedGlobalProperties rgp = new RequestedGlobalProperties();
        rgp.setHashPartitioned(new FieldList(1));
        
        RequestedLocalProperties rlp = new RequestedLocalProperties();
        rlp.setGroupedFields(new FieldList(0, 3));
        
        toJoin1.setRequiredGlobalProps(null);
        toJoin1.setRequiredLocalProps(null);
        
        toJoin2.setRequiredGlobalProps(null);
        toJoin2.setRequiredLocalProps(null);
        
        toJoin1.setShipStrategy(ShipStrategyType.PARTITION_HASH, new FieldList(88));
        toJoin2.setShipStrategy(ShipStrategyType.BROADCAST);
        
        toAfterJoin.setRequiredGlobalProps(rgp);
        toAfterJoin.setRequiredLocalProps(rlp);
        
        FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
        assertEquals(MET, report);
      }
      
      // test override on one inputs, later requirement met
      {
        GlobalProperties gp = new GlobalProperties();
        gp.setHashPartitioned(new FieldList(0));
        LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
        
        RequestedGlobalProperties rgp = new RequestedGlobalProperties();
        rgp.setHashPartitioned(new FieldList(0));
        
        RequestedLocalProperties rlp = new RequestedLocalProperties();
        rlp.setGroupedFields(new FieldList(2, 1));
        
        toJoin1.setShipStrategy(ShipStrategyType.PARTITION_HASH, new FieldList(88));
        toJoin2.setShipStrategy(ShipStrategyType.FORWARD);
        
        toAfterJoin.setRequiredGlobalProps(rgp);
        toAfterJoin.setRequiredLocalProps(rlp);
        
        FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
        assertEquals(PENDING, report);
      }
      
      // test override on one input, later requirement not met
      {
        GlobalProperties gp = new GlobalProperties();
        gp.setHashPartitioned(new FieldList(0));
        LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
        
        RequestedGlobalProperties rgp = new RequestedGlobalProperties();
        rgp.setHashPartitioned(new FieldList(3));
        
        RequestedLocalProperties rlp = new RequestedLocalProperties();
        rlp.setGroupedFields(new FieldList(77, 69));
        
        toJoin1.setShipStrategy(ShipStrategyType.PARTITION_HASH, new FieldList(88));
        toJoin2.setShipStrategy(ShipStrategyType.FORWARD);
        
        toAfterJoin.setRequiredGlobalProps(rgp);
        toAfterJoin.setRequiredLocalProps(rlp);
        
        FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
        assertEquals(NOT_MET, report);
      }
      
      // test override on one input locally, later global requirement not met
      {
        GlobalProperties gp = new GlobalProperties();
        gp.setHashPartitioned(new FieldList(0));
        LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
        
        RequestedGlobalProperties rgp = new RequestedGlobalProperties();
        rgp.setHashPartitioned(new FieldList(3));

View Full Code Here

Examples of org.apache.flink.compiler.dataproperties.GlobalProperties

              c2.adjustGlobalPropertiesForFullParallelismChange();
            }
          }
          
          // get the global properties and clear unique fields (not preserved anyways during the union)
          GlobalProperties p1 = c1.getGlobalProperties();
          GlobalProperties p2 = c2.getGlobalProperties();
          p1.clearUniqueFieldCombinations();
          p2.clearUniqueFieldCombinations();
          
          // adjust the partitionings, if they exist but are not equal. this may happen when both channels have a
          // partitioning that fulfills the requirements, but both are incompatible. For example may a property requirement
          // be ANY_PARTITIONING on fields (0) and one channel is range partitioned on that field, the other is hash
          // partitioned on that field. 
          if (!igps.isTrivial() && !(p1.equals(p2))) {
            if (c1.getShipStrategy() == ShipStrategyType.FORWARD && c2.getShipStrategy() != ShipStrategyType.FORWARD) {
              // adjust c2 to c1
              c2 = c2.clone();
              p1.parameterizeChannel(c2,dopChange2);
            } else if (c2.getShipStrategy() == ShipStrategyType.FORWARD && c1.getShipStrategy() != ShipStrategyType.FORWARD) {
              // adjust c1 to c2
              c1 = c1.clone();
              p2.parameterizeChannel(c1,dopChange1);
            } else if (c1.getShipStrategy() == ShipStrategyType.FORWARD && c2.getShipStrategy() == ShipStrategyType.FORWARD) {
              boolean adjustC1 = c1.getEstimatedOutputSize() <= 0 || c2.getEstimatedOutputSize() <= 0 ||
                  c1.getEstimatedOutputSize() <= c2.getEstimatedOutputSize();
              if (adjustC1) {
                c2 = c2.clone();
                p1.parameterizeChannel(c2, dopChange2);
              } else {
                c1 = c1.clone();
                p2.parameterizeChannel(c1, dopChange1);
              }
            } else {
              // this should never happen, as it implies both realize a different strategy, which is
              // excluded by the check that the required strategies must match
              throw new CompilerException("Bug in Plan Enumeration for Union Node.");

View Full Code Here

Examples of org.apache.flink.compiler.dataproperties.GlobalProperties

    return Collections.singletonList(new RequestedLocalProperties());
  }
  
  @Override
  public GlobalProperties computeGlobalProperties(GlobalProperties gProps) {
    return new GlobalProperties();
  }

View Full Code Here

Examples of org.apache.flink.compiler.dataproperties.GlobalProperties

    return new BinaryUnionPlanNode((BinaryUnionNode) node, in1, in2);
  }


  @Override
  public GlobalProperties computeGlobalProperties(GlobalProperties in1, GlobalProperties in2) {
    GlobalProperties newProps = new GlobalProperties();
    
    if (in1.getPartitioning() == PartitioningProperty.HASH_PARTITIONED &&
      in2.getPartitioning() == PartitioningProperty.HASH_PARTITIONED &&
      in1.getPartitioningFields().equals(in2.getPartitioningFields()))
    {
      newProps.setHashPartitioned(in1.getPartitioningFields());
    }
    
    return newProps;
  }

View Full Code Here

Examples of org.apache.flink.compiler.dataproperties.GlobalProperties

    return new DualInputPlanNode(node, "CoGroup ("+node.getPactContract().getName()+")", in1, in2, DriverStrategy.CO_GROUP, this.keys1, this.keys2, inputOrders);
  }
  
  @Override
  public GlobalProperties computeGlobalProperties(GlobalProperties in1, GlobalProperties in2) {
    GlobalProperties gp = GlobalProperties.combine(in1, in2);
    if (gp.getUniqueFieldCombination() != null && gp.getUniqueFieldCombination().size() > 0 &&
          gp.getPartitioning() == PartitioningProperty.RANDOM)
    {
      gp.setAnyPartitioning(gp.getUniqueFieldCombination().iterator().next().toFieldList());
    }
    gp.clearUniqueFieldCombinations();
    return gp;
  }

View Full Code Here

Examples of org.apache.flink.compiler.dataproperties.GlobalProperties

      List<PlanNode> newCandidates = new ArrayList<PlanNode>();
      
      for (Iterator<PlanNode> planDeleter = candidates.iterator(); planDeleter.hasNext(); ) {
        PlanNode candidate = planDeleter.next();
        
        GlobalProperties atEndGlobal = candidate.getGlobalProperties();
        LocalProperties atEndLocal = candidate.getLocalProperties();
        
        FeedbackPropertiesMeetRequirementsReport report = candidate.checkPartialSolutionPropertiesMet(pspn, atEndGlobal, atEndLocal);
        if (report == FeedbackPropertiesMeetRequirementsReport.NO_PARTIAL_SOLUTION) {
          ; // depends only through broadcast variable on the partial solution
        }
        else if (report == FeedbackPropertiesMeetRequirementsReport.NOT_MET) {
          // attach a no-op node through which we create the properties of the original input
          Channel toNoOp = new Channel(candidate);
          globPropsReq.parameterizeChannel(toNoOp, false);
          locPropsReq.parameterizeChannel(toNoOp);
          
          UnaryOperatorNode rebuildPropertiesNode = new UnaryOperatorNode("Rebuild Partial Solution Properties", FieldList.EMPTY_LIST);
          rebuildPropertiesNode.setDegreeOfParallelism(candidate.getDegreeOfParallelism());
          
          SingleInputPlanNode rebuildPropertiesPlanNode = new SingleInputPlanNode(rebuildPropertiesNode, "Rebuild Partial Solution Properties", toNoOp, DriverStrategy.UNARY_NO_OP);
          rebuildPropertiesPlanNode.initProperties(toNoOp.getGlobalProperties(), toNoOp.getLocalProperties());
          estimator.costOperator(rebuildPropertiesPlanNode);
            
          GlobalProperties atEndGlobalModified = rebuildPropertiesPlanNode.getGlobalProperties();
          LocalProperties atEndLocalModified = rebuildPropertiesPlanNode.getLocalProperties();
            
          if (!(atEndGlobalModified.equals(atEndGlobal) && atEndLocalModified.equals(atEndLocal))) {
            FeedbackPropertiesMeetRequirementsReport report2 = candidate.checkPartialSolutionPropertiesMet(pspn, atEndGlobalModified, atEndLocalModified);
            
            if (report2 != FeedbackPropertiesMeetRequirementsReport.NOT_MET) {
              newCandidates.add(rebuildPropertiesPlanNode);
            }
          }
          
          planDeleter.remove();
        }
      }
    }
    
    if (candidates.isEmpty()) {
      return;
    }
    
    // 5) Create a candidate for the Iteration Node for every remaining plan of the step function.
    if (terminationCriterion == null) {
      for (PlanNode candidate : candidates) {
        BulkIterationPlanNode node = new BulkIterationPlanNode(this, "BulkIteration ("+this.getPactContract().getName()+")", in, pspn, candidate);
        GlobalProperties gProps = candidate.getGlobalProperties().clone();
        LocalProperties lProps = candidate.getLocalProperties().clone();
        node.initProperties(gProps, lProps);
        target.add(node);
      }
    }
    else if (candidates.size() > 0) {
      List<PlanNode> terminationCriterionCandidates = this.terminationCriterion.getAlternativePlans(estimator);


      SingleRootJoiner singleRoot = (SingleRootJoiner) this.singleRoot;
      
      for (PlanNode candidate : candidates) {
        for (PlanNode terminationCandidate : terminationCriterionCandidates) {
          if (singleRoot.areBranchCompatible(candidate, terminationCandidate)) {
            BulkIterationPlanNode node = new BulkIterationPlanNode(this, "BulkIteration ("+this.getPactContract().getName()+")", in, pspn, candidate, terminationCandidate);
            GlobalProperties gProps = candidate.getGlobalProperties().clone();
            LocalProperties lProps = candidate.getLocalProperties().clone();
            node.initProperties(gProps, lProps);
            target.add(node);
            
          }

View Full Code Here

Examples of org.apache.flink.compiler.dataproperties.GlobalProperties

    return pairs;
  }


  @Override
  public GlobalProperties computeGlobalProperties(GlobalProperties in1, GlobalProperties in2) {
    GlobalProperties gp = GlobalProperties.combine(in1, in2);
    if (gp.getUniqueFieldCombination() != null && gp.getUniqueFieldCombination().size() > 0 &&
          gp.getPartitioning() == PartitioningProperty.RANDOM)
    {
      gp.setAnyPartitioning(gp.getUniqueFieldCombination().iterator().next().toFieldList());
    }
    gp.clearUniqueFieldCombinations();
    return gp;
  }

View Full Code Here

Examples of org.apache.flink.compiler.dataproperties.GlobalProperties

    final int[] ssKeys = iteration.getSolutionSetKeyFields();
    if (ssKeys == null || ssKeys.length == 0) {
      throw new CompilerException("Invalid WorksetIteration: No key fields defined for the solution set.");
    }
    this.solutionSetKeyFields = new FieldList(ssKeys);
    this.partitionedProperties = new GlobalProperties();
    this.partitionedProperties.setHashPartitioned(this.solutionSetKeyFields);
    
    int weight = iteration.getMaximumNumberOfIterations() > 0 ? 
      iteration.getMaximumNumberOfIterations() : DEFAULT_COST_WEIGHT;

View Full Code Here

Examples of org.apache.flink.compiler.dataproperties.GlobalProperties

      List<PlanNode> newCandidates = new ArrayList<PlanNode>();
      
      for (Iterator<PlanNode> planDeleter = worksetCandidates.iterator(); planDeleter.hasNext(); ) {
        PlanNode candidate = planDeleter.next();
        
        GlobalProperties atEndGlobal = candidate.getGlobalProperties();
        LocalProperties atEndLocal = candidate.getLocalProperties();
        
        FeedbackPropertiesMeetRequirementsReport report = candidate.checkPartialSolutionPropertiesMet(wspn, atEndGlobal, atEndLocal);
        if (report == FeedbackPropertiesMeetRequirementsReport.NO_PARTIAL_SOLUTION) {
          ; // depends only through broadcast variable on the workset solution
        }
        else if (report == FeedbackPropertiesMeetRequirementsReport.NOT_MET) {
          // attach a no-op node through which we create the properties of the original input
          Channel toNoOp = new Channel(candidate);
          globPropsReqWorkset.parameterizeChannel(toNoOp, false);
          locPropsReqWorkset.parameterizeChannel(toNoOp);
          
          UnaryOperatorNode rebuildWorksetPropertiesNode = new UnaryOperatorNode("Rebuild Workset Properties", FieldList.EMPTY_LIST);
          
          rebuildWorksetPropertiesNode.setDegreeOfParallelism(candidate.getDegreeOfParallelism());
          
          SingleInputPlanNode rebuildWorksetPropertiesPlanNode = new SingleInputPlanNode(rebuildWorksetPropertiesNode, "Rebuild Workset Properties", toNoOp, DriverStrategy.UNARY_NO_OP);
          rebuildWorksetPropertiesPlanNode.initProperties(toNoOp.getGlobalProperties(), toNoOp.getLocalProperties());
          estimator.costOperator(rebuildWorksetPropertiesPlanNode);
            
          GlobalProperties atEndGlobalModified = rebuildWorksetPropertiesPlanNode.getGlobalProperties();
          LocalProperties atEndLocalModified = rebuildWorksetPropertiesPlanNode.getLocalProperties();
            
          if (!(atEndGlobalModified.equals(atEndGlobal) && atEndLocalModified.equals(atEndLocal))) {
            FeedbackPropertiesMeetRequirementsReport report2 = candidate.checkPartialSolutionPropertiesMet(wspn, atEndGlobalModified, atEndLocalModified);
            
            if (report2 != FeedbackPropertiesMeetRequirementsReport.NOT_MET) {
              newCandidates.add(rebuildWorksetPropertiesPlanNode);
            }
          }
          
          // remove the original operator and add the modified candidate
          planDeleter.remove();
          
        }
      }
      
      worksetCandidates.addAll(newCandidates);
    }
    
    if (worksetCandidates.isEmpty()) {
      return;
    }
    
    // sanity check the solution set delta
    for (Iterator<PlanNode> deltaPlans = solutionSetDeltaCandidates.iterator(); deltaPlans.hasNext(); ) {
      SingleInputPlanNode candidate = (SingleInputPlanNode) deltaPlans.next();
      GlobalProperties gp = candidate.getGlobalProperties();
      
      if (gp.getPartitioning() != PartitioningProperty.HASH_PARTITIONED || gp.getPartitioningFields() == null ||
          !gp.getPartitioningFields().equals(this.solutionSetKeyFields))
      {
        throw new CompilerException("Bug: The solution set delta is not partitioned.");
      }
    }
    
    // 5) Create a candidate for the Iteration Node for every remaining plan of the step function.
    
    final GlobalProperties gp = new GlobalProperties();
    gp.setHashPartitioned(this.solutionSetKeyFields);
    gp.addUniqueFieldCombination(this.solutionSetKeyFields);
    
    LocalProperties lp = LocalProperties.EMPTY.addUniqueFields(this.solutionSetKeyFields);
    
    // take all combinations of solution set delta and workset plans
    for (PlanNode solutionSetCandidate : solutionSetDeltaCandidates) {

View Full Code Here

Examples of org.apache.flink.compiler.dataproperties.GlobalProperties

    return new DualInputPlanNode(node, "Cross("+node.getPactContract().getName()+")", in1, in2, getStrategy());
  }
  
  @Override
  public GlobalProperties computeGlobalProperties(GlobalProperties in1, GlobalProperties in2) {
    GlobalProperties gp = GlobalProperties.combine(in1, in2);
    if (gp.getUniqueFieldCombination() != null && gp.getUniqueFieldCombination().size() > 0 &&
          gp.getPartitioning() == PartitioningProperty.RANDOM)
    {
      gp.setAnyPartitioning(gp.getUniqueFieldCombination().iterator().next().toFieldList());
    }
    gp.clearUniqueFieldCombinations();
    return gp;
  }

View Full Code Here

0 1 2 3

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.