Examples of org.apache.flink.compiler.costs.Costs

org.apache.flink.compiler.costs.Costs
Simple class to represent the costs of an operation. The costs are currently tracking, network, I/O and CPU costs. Costs are composed of two parts of cost contributors:
1. Quantifiable costs. Those costs are used when estimates are available and track a quantifiable measure, such as the number of bytes for network or I/O
2. Heuristic costs. Those costs are used when no estimates are available. They can be used to track that an operator used a special operation which is heuristically considered more expensive than another operation.
The quantifiable costs may frequently be unknown, which is represented by a {@code -1} as a value for the unknowncomponents of the cost. In that case, all operations' costs are unknown and hence it is not decidable which operation to favor during pruning. In that case, the heuristic costs should contain a value to make sure that operators with different strategies are comparable, even in the absence of estimates. The heuristic costs are hence the system's mechanism of realizing pruning heuristics that favor some operations over others.

    testShipStrategyCombinationsWithUnknowns(MEDIUM_ESTIMATES);
    testShipStrategyCombinationsWithUnknowns(BIG_ESTIMATES);
  }
  
  private void testShipStrategyCombinationsWithUnknowns(EstimateProvider knownEstimates) {
    Costs hashBoth = new Costs();
    Costs bcKnown10 = new Costs();
    Costs bcUnknown10 = new Costs();
    Costs bcKnown1000 = new Costs();
    Costs bcUnknown1000 = new Costs();
    
    costEstimator.addHashPartitioningCost(knownEstimates, hashBoth);
    costEstimator.addHashPartitioningCost(UNKNOWN_ESTIMATES, hashBoth);
    
    costEstimator.addBroadcastCost(knownEstimates, 10, bcKnown10);

View Full Code Here

  @Test
  public void testJoinCostFormulasPlain() {
    
    // hash join costs
    
    Costs hashBothSmall = new Costs();
    Costs hashBothLarge = new Costs();
    Costs hashSmallBuild = new Costs();
    Costs hashLargeBuild = new Costs();
    
    costEstimator.addHybridHashCosts(SMALL_ESTIMATES, BIG_ESTIMATES, hashSmallBuild, 1);
    costEstimator.addHybridHashCosts(BIG_ESTIMATES, SMALL_ESTIMATES, hashLargeBuild, 1);
    costEstimator.addHybridHashCosts(SMALL_ESTIMATES, SMALL_ESTIMATES, hashBothSmall, 1);
    costEstimator.addHybridHashCosts(BIG_ESTIMATES, BIG_ESTIMATES, hashBothLarge, 1);


    assertTrue(hashBothSmall.compareTo(hashSmallBuild) < 0);
    assertTrue(hashSmallBuild.compareTo(hashLargeBuild) < 0);
    assertTrue(hashLargeBuild.compareTo(hashBothLarge) < 0);
    
    // merge join costs
    
    Costs mergeBothSmall = new Costs();
    Costs mergeBothLarge = new Costs();
    Costs mergeSmallFirst = new Costs();
    Costs mergeSmallSecond = new Costs();
    
    costEstimator.addLocalSortCost(SMALL_ESTIMATES, mergeSmallFirst);
    costEstimator.addLocalSortCost(BIG_ESTIMATES, mergeSmallFirst);
    costEstimator.addLocalMergeCost(SMALL_ESTIMATES, BIG_ESTIMATES, mergeSmallFirst, 1);
    
    costEstimator.addLocalSortCost(BIG_ESTIMATES, mergeSmallSecond);
    costEstimator.addLocalSortCost(SMALL_ESTIMATES, mergeSmallSecond);
    costEstimator.addLocalMergeCost(BIG_ESTIMATES, SMALL_ESTIMATES, mergeSmallSecond, 1);
    
    costEstimator.addLocalSortCost(SMALL_ESTIMATES, mergeBothSmall);
    costEstimator.addLocalSortCost(SMALL_ESTIMATES, mergeBothSmall);
    costEstimator.addLocalMergeCost(SMALL_ESTIMATES, SMALL_ESTIMATES, mergeBothSmall, 1);
    
    costEstimator.addLocalSortCost(BIG_ESTIMATES, mergeBothLarge);
    costEstimator.addLocalSortCost(BIG_ESTIMATES, mergeBothLarge);
    costEstimator.addLocalMergeCost(BIG_ESTIMATES, BIG_ESTIMATES, mergeBothLarge, 1);
    
    
    assertTrue(mergeBothSmall.compareTo(mergeSmallFirst) < 0);
    assertTrue(mergeBothSmall.compareTo(mergeSmallSecond) < 0);
    assertTrue(mergeSmallFirst.compareTo(mergeSmallSecond) == 0);
    assertTrue(mergeSmallFirst.compareTo(mergeBothLarge) < 0);
    assertTrue(mergeSmallSecond.compareTo(mergeBothLarge) < 0);
    
    // compare merge join and hash join costs
    
    assertTrue(hashBothSmall.compareTo(mergeBothSmall) < 0);
    assertTrue(hashBothLarge.compareTo(mergeBothLarge) < 0);

View Full Code Here

    testJoinCostFormulasWithWeights(BIG_ESTIMATES, MEDIUM_ESTIMATES);
    testJoinCostFormulasWithWeights(MEDIUM_ESTIMATES, BIG_ESTIMATES);
  }
  
  private void testJoinCostFormulasWithWeights(EstimateProvider e1, EstimateProvider e2) {
    Costs hf1 = new Costs();
    Costs hf5 = new Costs();
    Costs hs1 = new Costs();
    Costs hs5 = new Costs();
    Costs mm1 = new Costs();
    Costs mm5 = new Costs();
    
    costEstimator.addHybridHashCosts(e1, e2, hf1, 1);
    costEstimator.addHybridHashCosts(e1, e2, hf5, 5);
    costEstimator.addHybridHashCosts(e2, e1, hs1, 1);
    costEstimator.addHybridHashCosts(e2, e1, hs5, 5);
    
    costEstimator.addLocalSortCost(e1, mm1);
    costEstimator.addLocalSortCost(e2, mm1);
    costEstimator.addLocalMergeCost(e1, e2, mm1, 1);
    
    costEstimator.addLocalSortCost(e1, mm5);
    costEstimator.addLocalSortCost(e2, mm5);
    mm5.multiplyWith(5);
    costEstimator.addLocalMergeCost(e1, e2, mm5, 5);
    
    // weight 1 versus weight 5
    assertTrue(hf1.compareTo(hf5) < 0);
    assertTrue(hs1.compareTo(hs5) < 0);

View Full Code Here

  // --------------------------------------------------------------------------------------------
  
  @Test
  public void testHashJoinCostFormulasWithCaches() {
    
    Costs hashBothUnknown10 = new Costs();
    Costs hashBothUnknownCached10 = new Costs();
    
    Costs hashBothSmall10 = new Costs();
    Costs hashBothSmallCached10 = new Costs();
    
    Costs hashSmallLarge10 = new Costs();
    Costs hashSmallLargeCached10 = new Costs();
    
    Costs hashLargeSmall10 = new Costs();
    Costs hashLargeSmallCached10 = new Costs();
    
    Costs hashLargeSmall1 = new Costs();
    Costs hashLargeSmallCached1 = new Costs();
    
    costEstimator.addHybridHashCosts(UNKNOWN_ESTIMATES, UNKNOWN_ESTIMATES, hashBothUnknown10, 10);
    costEstimator.addCachedHybridHashCosts(UNKNOWN_ESTIMATES, UNKNOWN_ESTIMATES, hashBothUnknownCached10, 10);
    
    costEstimator.addHybridHashCosts(MEDIUM_ESTIMATES, MEDIUM_ESTIMATES, hashBothSmall10, 10);

View Full Code Here

0 1

TOP

Related Classes of org.apache.flink.compiler.costs.Costs

org.apache.flink.compiler.costs.DefaultCostEstimatorTest

org.apache.flink.compiler.dag.DataSourceNode

org.apache.flink.compiler.operators.AllGroupWithPartialPreGroupProperties

org.apache.flink.compiler.operators.AllReduceProperties

org.apache.flink.compiler.operators.GroupReduceWithCombineProperties

org.apache.flink.compiler.operators.ReduceProperties

org.apache.flink.compiler.plan.PlanNode

org.apache.flink.compiler.plan.SinkJoinerPlanNode

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.