Package org.apache.flink.compiler.plan

Examples of org.apache.flink.compiler.plan.DualInputPlanNode


     
      // check the solution set join and the delta
      PlanNode ssDelta = iteration.getSolutionSetDeltaPlanNode();
      assertTrue(ssDelta instanceof DualInputPlanNode); // this is only true if the update functions preserves the partitioning
     
      DualInputPlanNode ssJoin = (DualInputPlanNode) ssDelta;
      assertEquals(DEFAULT_PARALLELISM, ssJoin.getDegreeOfParallelism());
      assertEquals(ShipStrategyType.PARTITION_HASH, ssJoin.getInput1().getShipStrategy());
      assertEquals(new FieldList(0), ssJoin.getInput1().getShipStrategyKeys());
     
      // check the workset set join
      DualInputPlanNode edgeJoin = (DualInputPlanNode) ssJoin.getInput1().getSource();
      assertEquals(DEFAULT_PARALLELISM, edgeJoin.getDegreeOfParallelism());
      assertEquals(ShipStrategyType.PARTITION_HASH, edgeJoin.getInput1().getShipStrategy());
      assertEquals(ShipStrategyType.FORWARD, edgeJoin.getInput2().getShipStrategy());
      assertTrue(edgeJoin.getInput1().getTempMode().isCached());
     
      assertEquals(new FieldList(0), edgeJoin.getInput1().getShipStrategyKeys());
     
      // check that the initial partitioning is pushed out of the loop
      assertEquals(ShipStrategyType.PARTITION_HASH, iteration.getInput1().getShipStrategy());
      assertEquals(ShipStrategyType.PARTITION_HASH, iteration.getInput2().getShipStrategy());
      assertEquals(new FieldList(0), iteration.getInput1().getShipStrategyKeys());
View Full Code Here


     
      // check the solution set join and the delta
      PlanNode ssDelta = iteration.getSolutionSetDeltaPlanNode();
      assertTrue(ssDelta instanceof DualInputPlanNode); // this is only true if the update functions preserves the partitioning
     
      DualInputPlanNode ssJoin = (DualInputPlanNode) ssDelta;
      assertEquals(DEFAULT_PARALLELISM, ssJoin.getDegreeOfParallelism());
      assertEquals(ShipStrategyType.PARTITION_HASH, ssJoin.getInput1().getShipStrategy());
      assertEquals(new FieldList(0), ssJoin.getInput1().getShipStrategyKeys());
     
      // check the workset set join
      DualInputPlanNode edgeJoin = (DualInputPlanNode) ssJoin.getInput1().getSource();
      assertEquals(DEFAULT_PARALLELISM, edgeJoin.getDegreeOfParallelism());
      assertEquals(ShipStrategyType.PARTITION_HASH, edgeJoin.getInput1().getShipStrategy());
      assertEquals(ShipStrategyType.FORWARD, edgeJoin.getInput2().getShipStrategy());
      assertTrue(edgeJoin.getInput1().getTempMode().isCached());
     
      assertEquals(new FieldList(0), edgeJoin.getInput1().getShipStrategyKeys());
     
      // check that the initial partitioning is pushed out of the loop
      assertEquals(ShipStrategyType.PARTITION_HASH, iteration.getInput1().getShipStrategy());
      assertEquals(ShipStrategyType.PARTITION_HASH, iteration.getInput2().getShipStrategy());
      assertEquals(new FieldList(0), iteration.getInput1().getShipStrategyKeys());
View Full Code Here

      else {
        g.addEdge(n.getPredecessor(), n);
      }
    }
    else if(visitable instanceof DualInputPlanNode) {
      DualInputPlanNode n = (DualInputPlanNode) visitable;
     
      if(n.getDriverStrategy().firstDam().equals(DamBehavior.FULL_DAM) || n.getInput1().getLocalStrategy().dams() || n.getInput1().getTempMode().breaksPipeline()) {
        g.addEdge(n, n.getInput1().getSource());
      }
      else {
        g.addEdge(n.getInput1().getSource(), n);
      }
     
      if(!n.getDriverStrategy().equals(DriverStrategy.NONE) && (n.getDriverStrategy().secondDam().equals(DamBehavior.FULL_DAM) || n.getInput2().getLocalStrategy().dams() || n.getInput2().getTempMode().breaksPipeline())) {
        g.addEdge(n, n.getInput2().getSource());
      }
      else {
        g.addEdge(n.getInput2().getSource(), n);
      }
    }
   
   
    // recursively fix iterations
View Full Code Here

            optNode.getPactContract().getName() + "'. Missing type information for field " + e.getFieldNumber());
        }
      }
    }
    else if (node instanceof DualInputPlanNode) {
      DualInputPlanNode dn = (DualInputPlanNode) node;
     
      // get the nodes current schema
      T schema1;
      T schema2;
      if (dn.postPassHelper1 == null) {
        schema1 = createEmptySchema();
        schema2 = createEmptySchema();
        dn.postPassHelper1 = schema1;
        dn.postPassHelper2 = schema2;
      } else {
        schema1 = (T) dn.postPassHelper1;
        schema2 = (T) dn.postPassHelper2;
      }

      schema1.increaseNumConnectionsThatContributed();
      schema2.increaseNumConnectionsThatContributed();
      TwoInputNode optNode = dn.getTwoInputNode();
     
      // add the parent schema to the schema
      if (propagateParentSchemaDown) {
        addSchemaToSchema(parentSchema, schema1, optNode, 0);
        addSchemaToSchema(parentSchema, schema2, optNode, 1);
      }
     
      // check whether all outgoing channels have not yet contributed. come back later if not.
      if (schema1.getNumConnectionsThatContributed() < dn.getOutgoingChannels().size()) {
        return;
      }
     
      // add the nodes local information
      try {
        getDualInputNodeSchema(dn, schema1, schema2);
      } catch (ConflictingFieldTypeInfoException e) {
        throw new CompilerPostPassException(getConflictingTypeErrorMessage(e, optNode.getPactContract().getName()));
      }
     
      // parameterize the node's driver strategy
      if (createUtilities) {
        if (dn.getDriverStrategy().getNumRequiredComparators() > 0) {
          // set the individual comparators
          try {
            dn.setComparator1(createComparator(dn.getKeysForInput1(), dn.getSortOrders(), schema1));
            dn.setComparator2(createComparator(dn.getKeysForInput2(), dn.getSortOrders(), schema2));
          } catch (MissingFieldTypeInfoException e) {
            throw new CompilerPostPassException("Could not set up runtime strategy for node '" +
                optNode.getPactContract().getName() + "'. Missing type information for field " + e.getFieldNumber());
          }
         
          // set the pair comparator
          try {
            dn.setPairComparator(createPairComparator(dn.getKeysForInput1(), dn.getKeysForInput2(),
              dn.getSortOrders(), schema1, schema2));
          } catch (MissingFieldTypeInfoException e) {
            throw new CompilerPostPassException("Could not set up runtime strategy for node '" +
                optNode.getPactContract().getName() + "'. Missing type information for field " + e.getFieldNumber());
          }
         
        }
      }
     
      // done, we can now propagate our info down
      try {
        propagateToChannel(schema1, dn.getInput1(), createUtilities);
      } catch (MissingFieldTypeInfoException e) {
        throw new CompilerPostPassException("Could not set up runtime strategy for the first input channel to node '"
          + optNode.getPactContract().getName() + "'. Missing type information for field " + e.getFieldNumber());
      }
      try {
        propagateToChannel(schema2, dn.getInput2(), createUtilities);
      } catch (MissingFieldTypeInfoException e) {
        throw new CompilerPostPassException("Could not set up runtime strategy for the second input channel to node '"
          + optNode.getPactContract().getName() + "'. Missing type information for field " + e.getFieldNumber());
      }
     
      // don't forget the broadcast inputs
      for (Channel c: dn.getBroadcastInputs()) {
        try {
          propagateToChannel(createEmptySchema(), c, createUtilities);
        } catch (MissingFieldTypeInfoException e) {
          throw new CompilerPostPassException("Could not set up runtime strategy for broadcast channel in node '" +
            optNode.getPactContract().getName() + "'. Missing type information for field " + e.getFieldNumber());
View Full Code Here

      }
     
      // switching build and probe side did not help -> pipeline breaker
      for(DeadlockVertex v : g.vertices) {
        if(v.getOriginal() instanceof DualInputPlanNode) {
          DualInputPlanNode n = (DualInputPlanNode) v.getOriginal();
         
          // what is the pipelined side? (other side should be a dam, otherwise operator could not be source of deadlock)
          if(!(n.getDriverStrategy().firstDam().equals(DamBehavior.FULL_DAM) || n.getInput1().getLocalStrategy().dams() || n.getInput1().getTempMode().breaksPipeline())
              && (n.getDriverStrategy().secondDam().equals(DamBehavior.FULL_DAM) || n.getInput2().getLocalStrategy().dams() || n.getInput2().getTempMode().breaksPipeline())) {
            n.getInput1().setTempMode(n.getInput1().getTempMode().makePipelineBreaker());
          }
          else if( !(n.getDriverStrategy().secondDam().equals(DamBehavior.FULL_DAM) || n.getInput2().getLocalStrategy().dams() || n.getInput2().getTempMode().breaksPipeline())
              && (n.getDriverStrategy().firstDam().equals(DamBehavior.FULL_DAM) || n.getInput1().getLocalStrategy().dams() || n.getInput1().getTempMode().breaksPipeline())) {
            n.getInput2().setTempMode(n.getInput2().getTempMode().makePipelineBreaker());
          }
         
          // Deadlock resolved?
          if(!hasDeadlock(sinks)) {
            break;
View Full Code Here

      for (Channel c: sn.getBroadcastInputs()) {
        traverseChannel(c);
      }
    }
    else if (node instanceof DualInputPlanNode) {
      DualInputPlanNode dn = (DualInputPlanNode) node;
     
      if (!(dn.getOptimizerNode().getPactContract() instanceof DualInputOperator)) {
        throw new RuntimeException("Wrong operator type found in post pass.");
      }
     
      DualInputOperator<?, ?, ?, ?> dualInputOperator = (DualInputOperator<?, ?, ?, ?>) dn.getOptimizerNode().getPactContract();
     
      // parameterize the node's driver strategy
      if (dn.getDriverStrategy().getNumRequiredComparators() > 0) {
        dn.setComparator1(createComparator(dualInputOperator.getOperatorInfo().getFirstInputType(), dn.getKeysForInput1(),
          getSortOrders(dn.getKeysForInput1(), dn.getSortOrders())));
        dn.setComparator2(createComparator(dualInputOperator.getOperatorInfo().getSecondInputType(), dn.getKeysForInput2(),
            getSortOrders(dn.getKeysForInput2(), dn.getSortOrders())));

        dn.setPairComparator(createPairComparator(dualInputOperator.getOperatorInfo().getFirstInputType(),
            dualInputOperator.getOperatorInfo().getSecondInputType()));
       
      }
           
      traverseChannel(dn.getInput1());
      traverseChannel(dn.getInput2());
     
      // don't forget the broadcast inputs
      for (Channel c: dn.getBroadcastInputs()) {
        traverseChannel(c);
      }
     
    }
    // catch the sources of the iterative step functions
View Full Code Here

       
       
        Plan p = env.createProgramPlan();
        OptimizedPlan op = compileNoStats(p);
        SinkPlanNode sink = op.getDataSinks().iterator().next();
        DualInputPlanNode mapper = (DualInputPlanNode) sink.getInput().getSource();
       
        assertTrue(mapper.getInput1().getTempMode().breaksPipeline());
      }
     
      {
        ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        env.setDegreeOfParallelism(64);
       
        DataSet<Long> initialSource = env.generateSequence(1, 10);
       
        Configuration conf= new Configuration();
        conf.setString(PactCompiler.HINT_LOCAL_STRATEGY, PactCompiler.HINT_LOCAL_STRATEGY_NESTEDLOOP_BLOCKED_OUTER_SECOND);
        initialSource
          .map(new IdentityMapper<Long>())
          .cross(initialSource).withParameters(conf)
          .print();
       
       
        Plan p = env.createProgramPlan();
        OptimizedPlan op = compileNoStats(p);
       
        SinkPlanNode sink = op.getDataSinks().iterator().next();
        DualInputPlanNode mapper = (DualInputPlanNode) sink.getInput().getSource();
       
        assertTrue(mapper.getInput2().getTempMode().breaksPipeline());
      }
     
      {
        ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        env.setDegreeOfParallelism(64);
       
        DataSet<Long> initialSource = env.generateSequence(1, 10);
       
        Configuration conf= new Configuration();
        conf.setString(PactCompiler.HINT_LOCAL_STRATEGY, PactCompiler.HINT_LOCAL_STRATEGY_NESTEDLOOP_STREAMED_OUTER_FIRST);
        initialSource
          .map(new IdentityMapper<Long>())
          .cross(initialSource).withParameters(conf)
          .print();
       
       
        Plan p = env.createProgramPlan();
        OptimizedPlan op = compileNoStats(p);
       
        SinkPlanNode sink = op.getDataSinks().iterator().next();
        DualInputPlanNode mapper = (DualInputPlanNode) sink.getInput().getSource();
       
        assertTrue(mapper.getInput1().getTempMode().breaksPipeline());
      }
     
      {
        ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        env.setDegreeOfParallelism(64);
       
        DataSet<Long> initialSource = env.generateSequence(1, 10);
       
        Configuration conf= new Configuration();
        conf.setString(PactCompiler.HINT_LOCAL_STRATEGY, PactCompiler.HINT_LOCAL_STRATEGY_NESTEDLOOP_STREAMED_OUTER_SECOND);
        initialSource
          .map(new IdentityMapper<Long>())
          .cross(initialSource).withParameters(conf)
          .print();
       
       
        Plan p = env.createProgramPlan();
        OptimizedPlan op = compileNoStats(p);
       
        SinkPlanNode sink = op.getDataSinks().iterator().next();
        DualInputPlanNode mapper = (DualInputPlanNode) sink.getInput().getSource();
       
        assertTrue(mapper.getInput2().getTempMode().breaksPipeline());
      }
    }
    catch (Exception e) {
      e.printStackTrace();
      fail(e.getMessage());
View Full Code Here

      fail("The pact compiler is unable to compile this plan correctly.");
      return; // silence the compiler
    }
   
    OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
    DualInputPlanNode joinWithInvariantNode = resolver.getNode(JOIN_WITH_INVARIANT_NAME);
    DualInputPlanNode joinWithSolutionSetNode = resolver.getNode(JOIN_WITH_SOLUTION_SET);
    SingleInputPlanNode worksetReducer = resolver.getNode(NEXT_WORKSET_REDUCER_NAME);
    SingleInputPlanNode deltaMapper = resolver.getNode(SOLUTION_DELTA_MAPPER_NAME);
   
    // iteration preserves partitioning in reducer, so the first partitioning is out of the loop,
    // the in-loop partitioning is before the final reducer
   
    // verify joinWithInvariant
    assertEquals(ShipStrategyType.FORWARD, joinWithInvariantNode.getInput1().getShipStrategy());
    assertEquals(ShipStrategyType.PARTITION_HASH, joinWithInvariantNode.getInput2().getShipStrategy());
    assertEquals(list0, joinWithInvariantNode.getKeysForInput1());
    assertEquals(list0, joinWithInvariantNode.getKeysForInput2());
   
    // verify joinWithSolutionSet
    assertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getInput1().getShipStrategy());
    assertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getInput2().getShipStrategy());
   
    // verify reducer
    assertEquals(ShipStrategyType.PARTITION_HASH, worksetReducer.getInput().getShipStrategy());
    assertEquals(list0, worksetReducer.getKeys(0));
   
View Full Code Here

      fail("The pact compiler is unable to compile this plan correctly.");
      return; // silence the compiler
    }
   
    OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
    DualInputPlanNode joinWithInvariantNode = resolver.getNode(JOIN_WITH_INVARIANT_NAME);
    DualInputPlanNode joinWithSolutionSetNode = resolver.getNode(JOIN_WITH_SOLUTION_SET);
    SingleInputPlanNode worksetReducer = resolver.getNode(NEXT_WORKSET_REDUCER_NAME);
   
    // iteration preserves partitioning in reducer, so the first partitioning is out of the loop,
    // the in-loop partitioning is before the final reducer
   
    // verify joinWithInvariant
    assertEquals(ShipStrategyType.FORWARD, joinWithInvariantNode.getInput1().getShipStrategy());
    assertEquals(ShipStrategyType.PARTITION_HASH, joinWithInvariantNode.getInput2().getShipStrategy());
    assertEquals(list0, joinWithInvariantNode.getKeysForInput1());
    assertEquals(list0, joinWithInvariantNode.getKeysForInput2());
   
    // verify joinWithSolutionSet
    assertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getInput1().getShipStrategy());
    assertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getInput2().getShipStrategy());
   
    // verify reducer
    assertEquals(ShipStrategyType.PARTITION_HASH, worksetReducer.getInput().getShipStrategy());
    assertEquals(list0, worksetReducer.getKeys(0));
   
   
    // verify solution delta
    assertEquals(2, joinWithSolutionSetNode.getOutgoingChannels().size());
    assertEquals(ShipStrategyType.PARTITION_HASH, joinWithSolutionSetNode.getOutgoingChannels().get(0).getShipStrategy());
    assertEquals(ShipStrategyType.PARTITION_HASH, joinWithSolutionSetNode.getOutgoingChannels().get(1).getShipStrategy());
   
    new NepheleJobGraphGenerator().compileJobGraph(oPlan);
  }
View Full Code Here

      fail("The pact compiler is unable to compile this plan correctly.");
      return; // silence the compiler
    }
   
    OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(oPlan);
    DualInputPlanNode joinWithInvariantNode = resolver.getNode(JOIN_WITH_INVARIANT_NAME);
    DualInputPlanNode joinWithSolutionSetNode = resolver.getNode(JOIN_WITH_SOLUTION_SET);
    SingleInputPlanNode worksetReducer = resolver.getNode(NEXT_WORKSET_REDUCER_NAME);
   
    // iteration preserves partitioning in reducer, so the first partitioning is out of the loop,
    // the in-loop partitioning is before the final reducer
   
    // verify joinWithInvariant
    assertEquals(ShipStrategyType.FORWARD, joinWithInvariantNode.getInput1().getShipStrategy());
    assertEquals(ShipStrategyType.PARTITION_HASH, joinWithInvariantNode.getInput2().getShipStrategy());
    assertEquals(list0, joinWithInvariantNode.getKeysForInput1());
    assertEquals(list0, joinWithInvariantNode.getKeysForInput2());
   
    // verify joinWithSolutionSet
    assertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getInput1().getShipStrategy());
    assertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getInput2().getShipStrategy());
   
    // verify reducer
    assertEquals(ShipStrategyType.FORWARD, worksetReducer.getInput().getShipStrategy());
    assertEquals(list0, worksetReducer.getKeys(0));
   
   
    // verify solution delta
    assertEquals(1, joinWithSolutionSetNode.getOutgoingChannels().size());
    assertEquals(ShipStrategyType.FORWARD, joinWithSolutionSetNode.getOutgoingChannels().get(0).getShipStrategy());
   
    new NepheleJobGraphGenerator().compileJobGraph(oPlan);
  }
View Full Code Here

TOP

Related Classes of org.apache.flink.compiler.plan.DualInputPlanNode

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.