Examples of org.apache.flink.compiler.plan.Channel

org.apache.flink.compiler.plan.Channel

    try {
      OptimizedPlan oPlan = compileNoStats(plan);
      OptimizerPlanNodeResolver resolver = new OptimizerPlanNodeResolver(oPlan);
      
      DualInputPlanNode crossPlanNode = resolver.getNode("Cross");
      Channel in1 = crossPlanNode.getInput1();
      Channel in2 = crossPlanNode.getInput2();
      
      assertEquals(ShipStrategyType.FORWARD, in1.getShipStrategy());
      assertEquals(ShipStrategyType.BROADCAST, in2.getShipStrategy());
    } catch(CompilerException ce) {
      ce.printStackTrace();
      fail("The pact compiler is unable to compile this plan correctly.");
    }
  }

View Full Code Here

    try {
      OptimizedPlan oPlan = compileNoStats(plan);
      OptimizerPlanNodeResolver resolver = new OptimizerPlanNodeResolver(oPlan);
      
      DualInputPlanNode crossPlanNode = resolver.getNode("Cross");
      Channel in1 = crossPlanNode.getInput1();
      Channel in2 = crossPlanNode.getInput2();
      
      assertEquals(ShipStrategyType.BROADCAST, in1.getShipStrategy());
      assertEquals(ShipStrategyType.FORWARD, in2.getShipStrategy());
    } catch(CompilerException ce) {
      ce.printStackTrace();
      fail("The pact compiler is unable to compile this plan correctly.");
    }
  }

View Full Code Here

      SourcePlanNode target = new SourcePlanNode(getSourceNode(), "Partial Solution");


      SourcePlanNode source1 = new SourcePlanNode(getSourceNode(), "Source 1");
      SourcePlanNode source2 = new SourcePlanNode(getSourceNode(), "Source 2");
      
      Channel toMap1 = new Channel(source1);
      toMap1.setShipStrategy(ShipStrategyType.FORWARD);
      toMap1.setLocalStrategy(LocalStrategy.NONE);
      SingleInputPlanNode map1 = new SingleInputPlanNode(getMapNode(), "Mapper 1", toMap1, DriverStrategy.MAP);
      
      Channel toMap2 = new Channel(source2);
      toMap2.setShipStrategy(ShipStrategyType.FORWARD);
      toMap2.setLocalStrategy(LocalStrategy.NONE);
      SingleInputPlanNode map2 = new SingleInputPlanNode(getMapNode(), "Mapper 2", toMap2, DriverStrategy.MAP);
      
      Channel toJoin1 = new Channel(map1);
      Channel toJoin2 = new Channel(map2);
      
      toJoin1.setShipStrategy(ShipStrategyType.FORWARD);
      toJoin1.setLocalStrategy(LocalStrategy.NONE);
      toJoin2.setShipStrategy(ShipStrategyType.FORWARD);
      toJoin2.setLocalStrategy(LocalStrategy.NONE);
      
      DualInputPlanNode join = new DualInputPlanNode(getJoinNode(), "Join", toJoin1, toJoin2, DriverStrategy.HYBRIDHASH_BUILD_FIRST);
      
      FeedbackPropertiesMeetRequirementsReport report = join.checkPartialSolutionPropertiesMet(target, new GlobalProperties(), new LocalProperties());
      assertEquals(NO_PARTIAL_SOLUTION, report);

View Full Code Here

  public void testTwoOperatorsOneIndependent() {
    try {
      SourcePlanNode target = new SourcePlanNode(getSourceNode(), "Partial Solution");
      SourcePlanNode source = new SourcePlanNode(getSourceNode(), "Other Source");
      
      Channel toMap1 = new Channel(target);
      toMap1.setShipStrategy(ShipStrategyType.FORWARD);
      toMap1.setLocalStrategy(LocalStrategy.NONE);
      SingleInputPlanNode map1 = new SingleInputPlanNode(getMapNode(), "Mapper 1", toMap1, DriverStrategy.MAP);
      
      Channel toMap2 = new Channel(source);
      toMap2.setShipStrategy(ShipStrategyType.FORWARD);
      toMap2.setLocalStrategy(LocalStrategy.NONE);
      SingleInputPlanNode map2 = new SingleInputPlanNode(getMapNode(), "Mapper 2", toMap2, DriverStrategy.MAP);
      
      Channel toJoin1 = new Channel(map1);
      Channel toJoin2 = new Channel(map2);
      
      DualInputPlanNode join = new DualInputPlanNode(getJoinNode(), "Join", toJoin1, toJoin2, DriverStrategy.HYBRIDHASH_BUILD_FIRST);
      
      Channel toAfterJoin = new Channel(join);
      toAfterJoin.setShipStrategy(ShipStrategyType.FORWARD);
      toAfterJoin.setLocalStrategy(LocalStrategy.NONE);
      SingleInputPlanNode afterJoin = new SingleInputPlanNode(getMapNode(), "After Join Mapper", toAfterJoin, DriverStrategy.MAP);
      
      // attach some properties to the non-relevant input
      {
        toMap2.setShipStrategy(ShipStrategyType.BROADCAST);
        toMap2.setLocalStrategy(LocalStrategy.SORT, new FieldList(2, 7), new boolean[] {true, true});
        
        RequestedGlobalProperties joinGp = new RequestedGlobalProperties();
        joinGp.setFullyReplicated();
        
        RequestedLocalProperties joinLp = new RequestedLocalProperties();
        joinLp.setOrdering(new Ordering(2, null, Order.ASCENDING).appendOrdering(7, null, Order.ASCENDING));
        
        toJoin2.setShipStrategy(ShipStrategyType.FORWARD);
        toJoin2.setLocalStrategy(LocalStrategy.NONE);
        toJoin2.setRequiredGlobalProps(joinGp);
        toJoin2.setRequiredLocalProps(joinLp);
      }
      
      // ------------------------------------------------------------------------------------
      
      // no properties from the partial solution, no required properties
      {
        toJoin1.setShipStrategy(ShipStrategyType.FORWARD);
        toJoin1.setLocalStrategy(LocalStrategy.NONE);
        
        GlobalProperties gp = new GlobalProperties();
        LocalProperties lp = LocalProperties.EMPTY;
        
        FeedbackPropertiesMeetRequirementsReport report = join.checkPartialSolutionPropertiesMet(target, gp, lp);
        assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
      }
      
      // some properties from the partial solution, no required properties
      {
        toJoin1.setShipStrategy(ShipStrategyType.FORWARD);
        toJoin1.setLocalStrategy(LocalStrategy.NONE);
        
        GlobalProperties gp = new GlobalProperties();
        gp.setHashPartitioned(new FieldList(0));
        LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
        
        FeedbackPropertiesMeetRequirementsReport report = join.checkPartialSolutionPropertiesMet(target, gp, lp);
        assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
      }
      
      
      // produced properties match relevant input
      {
        GlobalProperties gp = new GlobalProperties();
        gp.setHashPartitioned(new FieldList(0));
        LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
        
        RequestedGlobalProperties rgp = new RequestedGlobalProperties();
        rgp.setHashPartitioned(new FieldList(0));
        
        RequestedLocalProperties rlp = new RequestedLocalProperties();
        rlp.setGroupedFields(new FieldList(2));
        
        toJoin1.setRequiredGlobalProps(rgp);
        toJoin1.setRequiredLocalProps(rlp);
        
        toJoin1.setShipStrategy(ShipStrategyType.FORWARD);
        toJoin1.setLocalStrategy(LocalStrategy.NONE);
        
        FeedbackPropertiesMeetRequirementsReport report = join.checkPartialSolutionPropertiesMet(target, gp, lp);
        assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
      }
      
      // produced properties do not match relevant input
      {
        GlobalProperties gp = new GlobalProperties();
        gp.setHashPartitioned(new FieldList(0));
        LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
        
        RequestedGlobalProperties rgp = new RequestedGlobalProperties();
        rgp.setHashPartitioned(new FieldList(0));
        
        RequestedLocalProperties rlp = new RequestedLocalProperties();
        rlp.setGroupedFields(new FieldList(1, 2, 3));
        
        toJoin1.setRequiredGlobalProps(rgp);
        toJoin1.setRequiredLocalProps(rlp);
        
        toJoin1.setShipStrategy(ShipStrategyType.FORWARD);
        toJoin1.setLocalStrategy(LocalStrategy.NONE);
        
        FeedbackPropertiesMeetRequirementsReport report = join.checkPartialSolutionPropertiesMet(target, gp, lp);
        assertEquals(NOT_MET, report);
      }
      
      // produced properties overridden before join
      {
        GlobalProperties gp = new GlobalProperties();
        gp.setHashPartitioned(new FieldList(0));
        LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
        
        RequestedGlobalProperties rgp = new RequestedGlobalProperties();
        rgp.setHashPartitioned(new FieldList(0));
        
        RequestedLocalProperties rlp = new RequestedLocalProperties();
        rlp.setGroupedFields(new FieldList(2, 1));
        
        toMap1.setRequiredGlobalProps(rgp);
        toMap1.setRequiredLocalProps(rlp);
        
        toJoin1.setRequiredGlobalProps(null);
        toJoin1.setRequiredLocalProps(null);
        
        toJoin1.setShipStrategy(ShipStrategyType.PARTITION_HASH, new FieldList(2, 1));
        toJoin1.setLocalStrategy(LocalStrategy.SORT, new FieldList(7, 3), new boolean[] {true, false});
        
        FeedbackPropertiesMeetRequirementsReport report = join.checkPartialSolutionPropertiesMet(target, gp, lp);
        assertEquals(MET, report);
      }
      
      // produced properties before join match, after join match as well
      {
        GlobalProperties gp = new GlobalProperties();
        gp.setHashPartitioned(new FieldList(0));
        LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
        
        RequestedGlobalProperties rgp = new RequestedGlobalProperties();
        rgp.setHashPartitioned(new FieldList(0));
        
        RequestedLocalProperties rlp = new RequestedLocalProperties();
        rlp.setGroupedFields(new FieldList(2, 1));
        
        toMap1.setRequiredGlobalProps(null);
        toMap1.setRequiredLocalProps(null);
        
        toJoin1.setShipStrategy(ShipStrategyType.FORWARD);
        toJoin1.setLocalStrategy(LocalStrategy.NONE);
        
        toJoin1.setRequiredGlobalProps(rgp);
        toJoin1.setRequiredLocalProps(rlp);
      
        toAfterJoin.setShipStrategy(ShipStrategyType.FORWARD);
        toAfterJoin.setLocalStrategy(LocalStrategy.NONE);
        
        toAfterJoin.setRequiredGlobalProps(rgp);
        toAfterJoin.setRequiredLocalProps(rlp);
        
        FeedbackPropertiesMeetRequirementsReport report = join.checkPartialSolutionPropertiesMet(target, gp, lp);
        assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
      }
      
      // produced properties before join match, after join do not match
      {
        GlobalProperties gp = new GlobalProperties();
        gp.setHashPartitioned(new FieldList(0));
        LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
        
        RequestedGlobalProperties rgp1 = new RequestedGlobalProperties();
        rgp1.setHashPartitioned(new FieldList(0));
        
        RequestedGlobalProperties rgp2 = new RequestedGlobalProperties();
        rgp2.setHashPartitioned(new FieldList(3));
        
        RequestedLocalProperties rlp1 = new RequestedLocalProperties();
        rlp1.setGroupedFields(new FieldList(2, 1));
        
        RequestedLocalProperties rlp2 = new RequestedLocalProperties();
        rlp2.setGroupedFields(new FieldList(3, 4));
        
        toJoin1.setRequiredGlobalProps(rgp1);
        toJoin1.setRequiredLocalProps(rlp1);
      
        toAfterJoin.setShipStrategy(ShipStrategyType.FORWARD);
        toAfterJoin.setLocalStrategy(LocalStrategy.NONE);
        
        toAfterJoin.setRequiredGlobalProps(rgp2);
        toAfterJoin.setRequiredLocalProps(rlp2);
        
        FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
        assertEquals(NOT_MET, report);
      }
      
      // produced properties are overridden, does not matter that they do not match
      {
        GlobalProperties gp = new GlobalProperties();
        gp.setAnyPartitioning(new FieldList(0));
        LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
        
        RequestedGlobalProperties rgp = new RequestedGlobalProperties();
        rgp.setHashPartitioned(new FieldList(1));
        
        RequestedLocalProperties rlp = new RequestedLocalProperties();
        rlp.setGroupedFields(new FieldList(1, 2, 3));
        
        toJoin1.setRequiredGlobalProps(null);
        toJoin1.setRequiredLocalProps(null);
        
        toJoin1.setShipStrategy(ShipStrategyType.PARTITION_HASH, new FieldList(2, 1));
        toJoin1.setLocalStrategy(LocalStrategy.SORT, new FieldList(7, 3), new boolean[] {true, false});
        
        toAfterJoin.setRequiredGlobalProps(rgp);
        toAfterJoin.setRequiredLocalProps(rlp);
        
        FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
        assertEquals(MET, report);
      }
      
      // local property overridden before join, local property mismatch after join not relevant
      {
        GlobalProperties gp = new GlobalProperties();
        gp.setAnyPartitioning(new FieldList(0));
        LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
        
        RequestedLocalProperties rlp = new RequestedLocalProperties();
        rlp.setGroupedFields(new FieldList(1, 2, 3));
        
        toJoin1.setRequiredGlobalProps(null);
        toJoin1.setRequiredLocalProps(null);
        
        toJoin1.setShipStrategy(ShipStrategyType.FORWARD);
        toJoin1.setLocalStrategy(LocalStrategy.SORT, new FieldList(7, 3), new boolean[] {true, false});
        
        toAfterJoin.setRequiredGlobalProps(null);
        toAfterJoin.setRequiredLocalProps(rlp);
        
        FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
        assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
      }
      
      // local property overridden before join, global property mismatch after join void the match
      {
        GlobalProperties gp = new GlobalProperties();
        gp.setAnyPartitioning(new FieldList(0));
        LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
        
        RequestedGlobalProperties rgp = new RequestedGlobalProperties();
        rgp.setHashPartitioned(new FieldList(1));
        
        RequestedLocalProperties rlp = new RequestedLocalProperties();
        rlp.setGroupedFields(new FieldList(1, 2, 3));
        
        toJoin1.setRequiredGlobalProps(null);
        toJoin1.setRequiredLocalProps(null);
        
        toJoin1.setShipStrategy(ShipStrategyType.FORWARD);
        toJoin1.setLocalStrategy(LocalStrategy.SORT, new FieldList(7, 3), new boolean[] {true, false});
        
        toAfterJoin.setRequiredGlobalProps(rgp);
        toAfterJoin.setRequiredLocalProps(rlp);
        
        FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
        assertEquals(NOT_MET, report);
      }
    }

View Full Code Here

  @Test
  public void testTwoOperatorsBothDependent() {
    try {
      SourcePlanNode target = new SourcePlanNode(getSourceNode(), "Partial Solution");
      
      Channel toMap1 = new Channel(target);
      toMap1.setShipStrategy(ShipStrategyType.FORWARD);
      toMap1.setLocalStrategy(LocalStrategy.NONE);
      SingleInputPlanNode map1 = new SingleInputPlanNode(getMapNode(), "Mapper 1", toMap1, DriverStrategy.MAP);
      
      Channel toMap2 = new Channel(target);
      toMap2.setShipStrategy(ShipStrategyType.FORWARD);
      toMap2.setLocalStrategy(LocalStrategy.NONE);
      SingleInputPlanNode map2 = new SingleInputPlanNode(getMapNode(), "Mapper 2", toMap2, DriverStrategy.MAP);
      
      Channel toJoin1 = new Channel(map1);
      toJoin1.setShipStrategy(ShipStrategyType.FORWARD);
      toJoin1.setLocalStrategy(LocalStrategy.NONE);
      
      Channel toJoin2 = new Channel(map2);
      toJoin2.setShipStrategy(ShipStrategyType.FORWARD);
      toJoin2.setLocalStrategy(LocalStrategy.NONE);
      
      DualInputPlanNode join = new DualInputPlanNode(getJoinNode(), "Join", toJoin1, toJoin2, DriverStrategy.HYBRIDHASH_BUILD_FIRST);
      
      Channel toAfterJoin = new Channel(join);
      toAfterJoin.setShipStrategy(ShipStrategyType.FORWARD);
      toAfterJoin.setLocalStrategy(LocalStrategy.NONE);
      SingleInputPlanNode afterJoin = new SingleInputPlanNode(getMapNode(), "After Join Mapper", toAfterJoin, DriverStrategy.MAP);
      
      // no properties from the partial solution, no required properties
      {
        GlobalProperties gp = new GlobalProperties();
        LocalProperties lp = LocalProperties.EMPTY;
        
        FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
        assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
      }
      
      // some properties from the partial solution, no required properties
      {
        GlobalProperties gp = new GlobalProperties();
        gp.setHashPartitioned(new FieldList(0));
        LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
        
        FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
        assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
      }
      
      // test requirements on one input and met
      {
        GlobalProperties gp = new GlobalProperties();
        gp.setHashPartitioned(new FieldList(0));
        LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
        
        RequestedGlobalProperties rgp = new RequestedGlobalProperties();
        rgp.setHashPartitioned(new FieldList(0));
        
        RequestedLocalProperties rlp = new RequestedLocalProperties();
        rlp.setGroupedFields(new FieldList(2, 1));
        
        toJoin1.setRequiredGlobalProps(rgp);
        toJoin1.setRequiredLocalProps(rlp);
        
        FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
        assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
      }
      
      // test requirements on both input and met
      {
        GlobalProperties gp = new GlobalProperties();
        gp.setHashPartitioned(new FieldList(0));
        LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
        
        RequestedGlobalProperties rgp = new RequestedGlobalProperties();
        rgp.setHashPartitioned(new FieldList(0));
        
        RequestedLocalProperties rlp = new RequestedLocalProperties();
        rlp.setGroupedFields(new FieldList(2, 1));
        
        toJoin1.setRequiredGlobalProps(rgp);
        toJoin1.setRequiredLocalProps(rlp);
        
        toJoin2.setRequiredGlobalProps(rgp);
        toJoin2.setRequiredLocalProps(rlp);
        
        FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
        assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
      }
      
      // test requirements on both inputs, one not met
      {
        GlobalProperties gp = new GlobalProperties();
        gp.setHashPartitioned(new FieldList(0));
        LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
        
        RequestedGlobalProperties rgp1 = new RequestedGlobalProperties();
        rgp1.setHashPartitioned(new FieldList(0));
        
        RequestedLocalProperties rlp1 = new RequestedLocalProperties();
        rlp1.setGroupedFields(new FieldList(2, 1));
        
        RequestedGlobalProperties rgp2 = new RequestedGlobalProperties();
        rgp2.setHashPartitioned(new FieldList(1));
        
        RequestedLocalProperties rlp2 = new RequestedLocalProperties();
        rlp2.setGroupedFields(new FieldList(0, 3));
        
        toJoin1.setRequiredGlobalProps(rgp1);
        toJoin1.setRequiredLocalProps(rlp1);
        
        toJoin2.setRequiredGlobalProps(rgp2);
        toJoin2.setRequiredLocalProps(rlp2);
        
        FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
        assertEquals(NOT_MET, report);
      }
      
      // test override on both inputs, later requirement ignored
      {
        GlobalProperties gp = new GlobalProperties();
        gp.setHashPartitioned(new FieldList(0));
        LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
        
        RequestedGlobalProperties rgp = new RequestedGlobalProperties();
        rgp.setHashPartitioned(new FieldList(1));
        
        RequestedLocalProperties rlp = new RequestedLocalProperties();
        rlp.setGroupedFields(new FieldList(0, 3));
        
        toJoin1.setRequiredGlobalProps(null);
        toJoin1.setRequiredLocalProps(null);
        
        toJoin2.setRequiredGlobalProps(null);
        toJoin2.setRequiredLocalProps(null);
        
        toJoin1.setShipStrategy(ShipStrategyType.PARTITION_HASH, new FieldList(88));
        toJoin2.setShipStrategy(ShipStrategyType.BROADCAST);
        
        toAfterJoin.setRequiredGlobalProps(rgp);
        toAfterJoin.setRequiredLocalProps(rlp);
        
        FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
        assertEquals(MET, report);
      }
      
      // test override on one inputs, later requirement met
      {
        GlobalProperties gp = new GlobalProperties();
        gp.setHashPartitioned(new FieldList(0));
        LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
        
        RequestedGlobalProperties rgp = new RequestedGlobalProperties();
        rgp.setHashPartitioned(new FieldList(0));
        
        RequestedLocalProperties rlp = new RequestedLocalProperties();
        rlp.setGroupedFields(new FieldList(2, 1));
        
        toJoin1.setShipStrategy(ShipStrategyType.PARTITION_HASH, new FieldList(88));
        toJoin2.setShipStrategy(ShipStrategyType.FORWARD);
        
        toAfterJoin.setRequiredGlobalProps(rgp);
        toAfterJoin.setRequiredLocalProps(rlp);
        
        FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
        assertEquals(PENDING, report);
      }
      
      // test override on one input, later requirement not met
      {
        GlobalProperties gp = new GlobalProperties();
        gp.setHashPartitioned(new FieldList(0));
        LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
        
        RequestedGlobalProperties rgp = new RequestedGlobalProperties();
        rgp.setHashPartitioned(new FieldList(3));
        
        RequestedLocalProperties rlp = new RequestedLocalProperties();
        rlp.setGroupedFields(new FieldList(77, 69));
        
        toJoin1.setShipStrategy(ShipStrategyType.PARTITION_HASH, new FieldList(88));
        toJoin2.setShipStrategy(ShipStrategyType.FORWARD);
        
        toAfterJoin.setRequiredGlobalProps(rgp);
        toAfterJoin.setRequiredLocalProps(rlp);
        
        FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
        assertEquals(NOT_MET, report);
      }
      
      // test override on one input locally, later global requirement not met
      {
        GlobalProperties gp = new GlobalProperties();
        gp.setHashPartitioned(new FieldList(0));
        LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1));
        
        RequestedGlobalProperties rgp = new RequestedGlobalProperties();
        rgp.setHashPartitioned(new FieldList(3));
        
        
        toJoin1.setShipStrategy(ShipStrategyType.FORWARD);
        toJoin1.setLocalStrategy(LocalStrategy.SORT, new FieldList(3), new boolean[] { false });
        
        toJoin2.setShipStrategy(ShipStrategyType.FORWARD);
        toJoin1.setLocalStrategy(LocalStrategy.NONE);
        
        toAfterJoin.setRequiredGlobalProps(rgp);
        toAfterJoin.setRequiredLocalProps(null);
        
        FeedbackPropertiesMeetRequirementsReport report = afterJoin.checkPartialSolutionPropertiesMet(target, gp, lp);
        assertEquals(NOT_MET, report);
      }
    }

View Full Code Here

      
      @Override
      public boolean preVisit(PlanNode visitable) {
        if (visitable instanceof DualInputPlanNode) {
          DualInputPlanNode node = (DualInputPlanNode) visitable;
          Channel c1 = node.getInput1();
          Channel c2 = node.getInput2();
          
          Assert.assertEquals("Incompatible shipping strategy chosen for match", ShipStrategyType.FORWARD, c1.getShipStrategy());
          Assert.assertEquals("Incompatible shipping strategy chosen for match", ShipStrategyType.PARTITION_HASH, c2.getShipStrategy());
          return false;
        }
        return true;
      }

View Full Code Here

    
    // verify the strategies
    Assert.assertEquals(ShipStrategyType.FORWARD, sinkNode.getInput().getShipStrategy());
    Assert.assertEquals(ShipStrategyType.PARTITION_HASH, reducer.getInput().getShipStrategy());
    
    Channel c = reducer.getInput();
    Assert.assertEquals(LocalStrategy.SORT, c.getLocalStrategy());
    
    FieldList ship = new FieldList(2);
    FieldList local = new FieldList(2, 5);
    Assert.assertEquals(ship, c.getShipStrategyKeys());
    Assert.assertEquals(local, c.getLocalStrategyKeys());
    Assert.assertTrue(c.getLocalStrategySortOrder()[0] == reducer.getSortOrders(0)[0]);
    
    // check that we indeed sort descending
    Assert.assertTrue(c.getLocalStrategySortOrder()[1] == groupOrder.getFieldSortDirections()[0]);
  }

View Full Code Here

    // verify the strategies
    Assert.assertEquals(ShipStrategyType.FORWARD, sinkNode.getInput().getShipStrategy());
    Assert.assertEquals(ShipStrategyType.PARTITION_HASH, coGroupNode.getInput1().getShipStrategy());
    Assert.assertEquals(ShipStrategyType.PARTITION_HASH, coGroupNode.getInput2().getShipStrategy());
    
    Channel c1 = coGroupNode.getInput1();
    Channel c2 = coGroupNode.getInput2();
    
    Assert.assertEquals(LocalStrategy.SORT, c1.getLocalStrategy());
    Assert.assertEquals(LocalStrategy.SORT, c2.getLocalStrategy());
    
    FieldList ship1 = new FieldList(new int[] {3, 0});
    FieldList ship2 = new FieldList(new int[] {6, 0});
    
    FieldList local1 = new FieldList(new int[] {3, 0, 5});
    FieldList local2 = new FieldList(new int[] {6, 0, 1, 4});
    
    Assert.assertEquals(ship1, c1.getShipStrategyKeys());
    Assert.assertEquals(ship2, c2.getShipStrategyKeys());
    Assert.assertEquals(local1, c1.getLocalStrategyKeys());
    Assert.assertEquals(local2, c2.getLocalStrategyKeys());
    
    Assert.assertTrue(c1.getLocalStrategySortOrder()[0] == coGroupNode.getSortOrders()[0]);
    Assert.assertTrue(c1.getLocalStrategySortOrder()[1] == coGroupNode.getSortOrders()[1]);
    Assert.assertTrue(c2.getLocalStrategySortOrder()[0] == coGroupNode.getSortOrders()[0]);
    Assert.assertTrue(c2.getLocalStrategySortOrder()[1] == coGroupNode.getSortOrders()[1]);
    
    // check that the local group orderings are correct
    Assert.assertTrue(c1.getLocalStrategySortOrder()[2] == groupOrder1.getFieldSortDirections()[0]);
    Assert.assertTrue(c2.getLocalStrategySortOrder()[2] == groupOrder2.getFieldSortDirections()[0]);
    Assert.assertTrue(c2.getLocalStrategySortOrder()[3] == groupOrder2.getFieldSortDirections()[1]);
  }

View Full Code Here

    final long NUM_RECORD = 1001;
    final long SIZE = 467131;
    
    DataSourceNode source = getSourceNode();
    SourcePlanNode planNode = new SourcePlanNode(source, "test node");
    Channel channel = new Channel(planNode);


    // no estimates here
    Assert.assertEquals(-1, channel.getEstimatedOutputSize());
    Assert.assertEquals(-1, channel.getEstimatedNumRecords());
    
    // set estimates
    source.setEstimatedNumRecords(NUM_RECORD);
    source.setEstimatedOutputSize(SIZE);
    Assert.assertEquals(SIZE, channel.getEstimatedOutputSize());
    Assert.assertEquals(NUM_RECORD, channel.getEstimatedNumRecords());
  }

View Full Code Here

    
    final int REPLICATION = 23;
    
    DataSourceNode source = getSourceNode();
    SourcePlanNode planNode = new SourcePlanNode(source, "test node");
    Channel channel = new Channel(planNode);
    channel.setReplicationFactor(REPLICATION);


    // no estimates here
    Assert.assertEquals(-1, channel.getEstimatedOutputSize());
    Assert.assertEquals(-1, channel.getEstimatedNumRecords());
    
    // set estimates
    source.setEstimatedNumRecords(NUM_RECORD);
    source.setEstimatedOutputSize(SIZE);
    Assert.assertEquals(SIZE * REPLICATION, channel.getEstimatedOutputSize());
    Assert.assertEquals(NUM_RECORD * REPLICATION, channel.getEstimatedNumRecords());
  }

View Full Code Here

0 1 2 3

TOP

Related Classes of org.apache.flink.compiler.plan.Channel

org.apache.flink.compiler.AdditionalOperatorsTest

org.apache.flink.compiler.CoGroupSolutionSetFirstTest

org.apache.flink.compiler.costs.CostEstimator

org.apache.flink.compiler.dag.BinaryUnionNode

org.apache.flink.compiler.dag.BulkIterationNode

org.apache.flink.compiler.dag.DataSinkNode

org.apache.flink.compiler.dag.SingleInputNode

org.apache.flink.compiler.dag.TwoInputNode

org.apache.flink.compiler.dag.WorksetIterationNode

org.apache.flink.compiler.dataproperties.LocalProperties

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.