Package org.apache.flink.compiler.plan

Examples of org.apache.flink.compiler.plan.SinkPlanNode


          .print();
       
       
        Plan p = env.createProgramPlan();
        OptimizedPlan op = compileNoStats(p);
        SinkPlanNode sink = op.getDataSinks().iterator().next();
        DualInputPlanNode mapper = (DualInputPlanNode) sink.getInput().getSource();
       
        assertTrue(mapper.getInput1().getTempMode().breaksPipeline());
      }
     
      {
        ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        env.setDegreeOfParallelism(64);
       
        DataSet<Long> initialSource = env.generateSequence(1, 10);
       
        Configuration conf= new Configuration();
        conf.setString(PactCompiler.HINT_LOCAL_STRATEGY, PactCompiler.HINT_LOCAL_STRATEGY_NESTEDLOOP_BLOCKED_OUTER_SECOND);
        initialSource
          .map(new IdentityMapper<Long>())
          .cross(initialSource).withParameters(conf)
          .print();
       
       
        Plan p = env.createProgramPlan();
        OptimizedPlan op = compileNoStats(p);
       
        SinkPlanNode sink = op.getDataSinks().iterator().next();
        DualInputPlanNode mapper = (DualInputPlanNode) sink.getInput().getSource();
       
        assertTrue(mapper.getInput2().getTempMode().breaksPipeline());
      }
     
      {
        ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        env.setDegreeOfParallelism(64);
       
        DataSet<Long> initialSource = env.generateSequence(1, 10);
       
        Configuration conf= new Configuration();
        conf.setString(PactCompiler.HINT_LOCAL_STRATEGY, PactCompiler.HINT_LOCAL_STRATEGY_NESTEDLOOP_STREAMED_OUTER_FIRST);
        initialSource
          .map(new IdentityMapper<Long>())
          .cross(initialSource).withParameters(conf)
          .print();
       
       
        Plan p = env.createProgramPlan();
        OptimizedPlan op = compileNoStats(p);
       
        SinkPlanNode sink = op.getDataSinks().iterator().next();
        DualInputPlanNode mapper = (DualInputPlanNode) sink.getInput().getSource();
       
        assertTrue(mapper.getInput1().getTempMode().breaksPipeline());
      }
     
      {
        ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        env.setDegreeOfParallelism(64);
       
        DataSet<Long> initialSource = env.generateSequence(1, 10);
       
        Configuration conf= new Configuration();
        conf.setString(PactCompiler.HINT_LOCAL_STRATEGY, PactCompiler.HINT_LOCAL_STRATEGY_NESTEDLOOP_STREAMED_OUTER_SECOND);
        initialSource
          .map(new IdentityMapper<Long>())
          .cross(initialSource).withParameters(conf)
          .print();
       
       
        Plan p = env.createProgramPlan();
        OptimizedPlan op = compileNoStats(p);
       
        SinkPlanNode sink = op.getDataSinks().iterator().next();
        DualInputPlanNode mapper = (DualInputPlanNode) sink.getInput().getSource();
       
        assertTrue(mapper.getInput2().getTempMode().breaksPipeline());
      }
    }
    catch (Exception e) {
View Full Code Here


        .print();
     
      Plan p = env.createProgramPlan();
      OptimizedPlan op = compileNoStats(p);
     
      SinkPlanNode sink = op.getDataSinks().iterator().next();
     
      SingleInputPlanNode agg2Reducer = (SingleInputPlanNode) sink.getInput().getSource();
      SingleInputPlanNode agg2Combiner = (SingleInputPlanNode) agg2Reducer.getInput().getSource();
      SingleInputPlanNode agg1Reducer = (SingleInputPlanNode) agg2Combiner.getInput().getSource();
     
      assertEquals(ShipStrategyType.PARTITION_HASH, agg2Reducer.getInput().getShipStrategy());
      assertEquals(new FieldList(0), agg2Reducer.getInput().getShipStrategyKeys());
View Full Code Here

        .print();
     
      Plan p = env.createProgramPlan();
      OptimizedPlan op = compileNoStats(p);
     
      SinkPlanNode sink = op.getDataSinks().iterator().next();
     
      SingleInputPlanNode agg2Reducer = (SingleInputPlanNode) sink.getInput().getSource();
      SingleInputPlanNode agg1Reducer = (SingleInputPlanNode) agg2Reducer.getInput().getSource();
     
      assertEquals(ShipStrategyType.FORWARD, agg2Reducer.getInput().getShipStrategy());
     
      assertEquals(ShipStrategyType.PARTITION_HASH, agg1Reducer.getInput().getShipStrategy());
View Full Code Here

      OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(op);
     
      // get the original nodes
      SourcePlanNode sourceNode = resolver.getNode("source");
      SingleInputPlanNode reduceNode = resolver.getNode("reducer");
      SinkPlanNode sinkNode = resolver.getNode("sink");
     
      // get the combiner
      SingleInputPlanNode combineNode = (SingleInputPlanNode) reduceNode.getInput().getSource();
     
      // check wiring
      assertEquals(sourceNode, combineNode.getInput().getSource());
      assertEquals(reduceNode, sinkNode.getInput().getSource());
     
      // check that both reduce and combiner have the same strategy
      assertEquals(DriverStrategy.SORTED_GROUP_REDUCE, reduceNode.getDriverStrategy());
      assertEquals(DriverStrategy.SORTED_GROUP_COMBINE, combineNode.getDriverStrategy());
     
      // check the keys
      assertEquals(new FieldList(1), reduceNode.getKeys(0));
      assertEquals(new FieldList(1), combineNode.getKeys(0));
      assertEquals(new FieldList(1), combineNode.getKeys(1));
      assertEquals(new FieldList(1), reduceNode.getInput().getLocalStrategyKeys());
     
      // check DOP
      assertEquals(6, sourceNode.getDegreeOfParallelism());
      assertEquals(6, combineNode.getDegreeOfParallelism());
      assertEquals(8, reduceNode.getDegreeOfParallelism());
      assertEquals(8, sinkNode.getDegreeOfParallelism());
    }
    catch (Exception e) {
      System.err.println(e.getMessage());
      e.printStackTrace();
      fail(e.getClass().getSimpleName() + " in test: " + e.getMessage());
View Full Code Here

      OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(op);
     
      // get the original nodes
      SourcePlanNode sourceNode = resolver.getNode("source");
      SingleInputPlanNode reduceNode = resolver.getNode("reducer");
      SinkPlanNode sinkNode = resolver.getNode("sink");
     
      // get the key extractors and projectors
      SingleInputPlanNode keyExtractor = (SingleInputPlanNode) reduceNode.getInput().getSource();
      SingleInputPlanNode keyProjector = (SingleInputPlanNode) sinkNode.getInput().getSource();
     
      // check wiring
      assertEquals(sourceNode, keyExtractor.getInput().getSource());
      assertEquals(keyProjector, sinkNode.getInput().getSource());
     
      // check that both reduce and combiner have the same strategy
      assertEquals(DriverStrategy.SORTED_GROUP_REDUCE, reduceNode.getDriverStrategy());
     
      // check the keys
      assertEquals(new FieldList(0), reduceNode.getKeys(0));
      assertEquals(new FieldList(0), reduceNode.getInput().getLocalStrategyKeys());
     
      // check DOP
      assertEquals(6, sourceNode.getDegreeOfParallelism());
      assertEquals(6, keyExtractor.getDegreeOfParallelism());
     
      assertEquals(8, reduceNode.getDegreeOfParallelism());
      assertEquals(8, keyProjector.getDegreeOfParallelism());
      assertEquals(8, sinkNode.getDegreeOfParallelism());
    }
    catch (Exception e) {
      System.err.println(e.getMessage());
      e.printStackTrace();
      fail(e.getClass().getSimpleName() + " in test: " + e.getMessage());
View Full Code Here

      OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(op);
     
      // get the original nodes
      SourcePlanNode sourceNode = resolver.getNode("source");
      SingleInputPlanNode reduceNode = resolver.getNode("reducer");
      SinkPlanNode sinkNode = resolver.getNode("sink");
     
      // get the combiner
      SingleInputPlanNode combineNode = (SingleInputPlanNode) reduceNode.getInput().getSource();
     
      // get the key extractors and projectors
      SingleInputPlanNode keyExtractor = (SingleInputPlanNode) combineNode.getInput().getSource();
      SingleInputPlanNode keyProjector = (SingleInputPlanNode) sinkNode.getInput().getSource();
     
      // check wiring
      assertEquals(sourceNode, keyExtractor.getInput().getSource());
      assertEquals(keyProjector, sinkNode.getInput().getSource());
     
      // check that both reduce and combiner have the same strategy
      assertEquals(DriverStrategy.SORTED_GROUP_REDUCE, reduceNode.getDriverStrategy());
      assertEquals(DriverStrategy.SORTED_GROUP_COMBINE, combineNode.getDriverStrategy());
     
      // check the keys
      assertEquals(new FieldList(0), reduceNode.getKeys(0));
      assertEquals(new FieldList(0), combineNode.getKeys(0));
      assertEquals(new FieldList(0), combineNode.getKeys(1));
      assertEquals(new FieldList(0), reduceNode.getInput().getLocalStrategyKeys());
     
      // check DOP
      assertEquals(6, sourceNode.getDegreeOfParallelism());
      assertEquals(6, keyExtractor.getDegreeOfParallelism());
      assertEquals(6, combineNode.getDegreeOfParallelism());
     
      assertEquals(8, reduceNode.getDegreeOfParallelism());
      assertEquals(8, keyProjector.getDegreeOfParallelism());
      assertEquals(8, sinkNode.getDegreeOfParallelism());
    }
    catch (Exception e) {
      System.err.println(e.getMessage());
      e.printStackTrace();
      fail(e.getClass().getSimpleName() + " in test: " + e.getMessage());
View Full Code Here

      fail("The pact compiler is unable to compile this plan correctly.");
      return; // silence the compiler
    }
   
    OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
    SinkPlanNode sinkNode = resolver.getNode("Sink");
    SingleInputPlanNode reducer = resolver.getNode("Reduce");
   
    // verify the strategies
    Assert.assertEquals(ShipStrategyType.FORWARD, sinkNode.getInput().getShipStrategy());
    Assert.assertEquals(ShipStrategyType.PARTITION_HASH, reducer.getInput().getShipStrategy());
   
    Channel c = reducer.getInput();
    Assert.assertEquals(LocalStrategy.SORT, c.getLocalStrategy());
   
View Full Code Here

      fail("The pact compiler is unable to compile this plan correctly.");
      return; // silence the compiler
    }
   
    OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
    SinkPlanNode sinkNode = resolver.getNode("Sink");
    DualInputPlanNode coGroupNode = resolver.getNode("CoGroup");
   
    // verify the strategies
    Assert.assertEquals(ShipStrategyType.FORWARD, sinkNode.getInput().getShipStrategy());
    Assert.assertEquals(ShipStrategyType.PARTITION_HASH, coGroupNode.getInput1().getShipStrategy());
    Assert.assertEquals(ShipStrategyType.PARTITION_HASH, coGroupNode.getInput2().getShipStrategy());
   
    Channel c1 = coGroupNode.getInput1();
    Channel c2 = coGroupNode.getInput2();
View Full Code Here

   
    // check the optimized Plan
    // when reducer 1 distributes its data across the instances of map2, it needs to employ a local hash method,
    // because map2 has twice as many instances and key/value pairs with the same key need to be processed by the same
    // mapper respectively reducer
    SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
    SingleInputPlanNode red2Node = (SingleInputPlanNode) sinkNode.getPredecessor();
    SingleInputPlanNode map2Node = (SingleInputPlanNode) red2Node.getPredecessor();
   
    ShipStrategyType mapIn = map2Node.getInput().getShipStrategy();
    ShipStrategyType redIn = red2Node.getInput().getShipStrategy();
   
View Full Code Here

   
    // check the optimized Plan
    // when reducer 1 distributes its data across the instances of map2, it needs to employ a local hash method,
    // because map2 has twice as many instances and key/value pairs with the same key need to be processed by the same
    // mapper respectively reducer
    SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
    SingleInputPlanNode red2Node = (SingleInputPlanNode) sinkNode.getPredecessor();
    SingleInputPlanNode map2Node = (SingleInputPlanNode) red2Node.getPredecessor();
   
    ShipStrategyType mapIn = map2Node.getInput().getShipStrategy();
    ShipStrategyType reduceIn = red2Node.getInput().getShipStrategy();
   
View Full Code Here

TOP

Related Classes of org.apache.flink.compiler.plan.SinkPlanNode

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.