Package org.apache.flink.compiler.plan

Examples of org.apache.flink.compiler.plan.Channel


    int numDynamicSenderTasksTotal = 0;
   

    // expand the channel to all the union channels, in case there is a union operator at its source
    while (allInChannels.hasNext()) {
      final Channel inConn = allInChannels.next();
     
      // sanity check the common serializer
      if (typeSerFact == null) {
        typeSerFact = inConn.getSerializer();
      } else if (!typeSerFact.equals(inConn.getSerializer())) {
        throw new CompilerException("Conflicting types in union operator.");
      }
     
      final PlanNode sourceNode = inConn.getSource();
      AbstractJobVertex sourceVertex = this.vertices.get(sourceNode);
      TaskConfig sourceVertexConfig;

      if (sourceVertex == null) {
        // this predecessor is chained to another task or an iteration
        final TaskInChain chainedTask;
        final IterationDescriptor iteration;
        if ((chainedTask = this.chainedTasks.get(sourceNode)) != null) {
          // push chained task
          if (chainedTask.getContainingVertex() == null) {
            throw new IllegalStateException("Bug: Chained task has not been assigned its containing vertex when connecting.");
          }
          sourceVertex = chainedTask.getContainingVertex();
          sourceVertexConfig = chainedTask.getTaskConfig();
        } else if ((iteration = this.iterations.get(sourceNode)) != null) {
          // predecessor is an iteration
          sourceVertex = iteration.getHeadTask();
          sourceVertexConfig = iteration.getHeadFinalResultConfig();
        } else {
          throw new CompilerException("Bug: Could not resolve source node for a channel.");
        }
      } else {
        // predecessor is its own vertex
        sourceVertexConfig = new TaskConfig(sourceVertex.getConfiguration());
      }
      DistributionPattern pattern = connectJobVertices(
        inConn, inputIndex, sourceVertex, sourceVertexConfig, targetVertex, targetVertexConfig, isBroadcast);
     
      // accounting on channels and senders
      numChannelsTotal++;
      if (inConn.isOnDynamicPath()) {
        numChannelsDynamicPath++;
        numDynamicSenderTasksTotal += getNumberOfSendersPerReceiver(pattern,
          sourceVertex.getParallelism(), targetVertex.getParallelism());
      }
    }
View Full Code Here


    final DriverStrategy ds = node.getDriverStrategy();
   
    // check, whether chaining is possible
    boolean chaining = false;
    {
      Channel inConn = node.getInput();
      PlanNode pred = inConn.getSource();
      chaining = ds.getPushChainDriverClass() != null &&
          !(pred instanceof NAryUnionPlanNode) &&  // first op after union is stand-alone, because union is merged
          !(pred instanceof BulkPartialSolutionPlanNode) &&  // partial solution merges anyways
          !(pred instanceof WorksetPlanNode) &&  // workset merges anyways
          !(pred instanceof IterationPlanNode) && // cannot chain with iteration heads currently
          inConn.getShipStrategy() == ShipStrategyType.FORWARD &&
          inConn.getLocalStrategy() == LocalStrategy.NONE &&
          pred.getOutgoingChannels().size() == 1 &&
          node.getDegreeOfParallelism() == pred.getDegreeOfParallelism() &&
          node.getBroadcastInputs().isEmpty();
     
      // cannot chain the nodes that produce the next workset or the next solution set, if they are not the
View Full Code Here

    // 5) There is no local strategy on the edge for the initial partial solution, as
    //    this translates to a local strategy that would only be executed in the first iteration
   
    final boolean merge;
    if (mergeIterationAuxTasks && pspn.getOutgoingChannels().size() == 1) {
      final Channel c = pspn.getOutgoingChannels().get(0);
      final PlanNode successor = c.getTarget();
      merge = c.getShipStrategy() == ShipStrategyType.FORWARD &&
          c.getLocalStrategy() == LocalStrategy.NONE &&
          c.getTempMode() == TempMode.NONE &&
          successor.getDegreeOfParallelism() == pspn.getDegreeOfParallelism() &&
          !(successor instanceof NAryUnionPlanNode) &&
          successor != iteration.getRootOfStepFunction() &&
          iteration.getInput().getLocalStrategy() == LocalStrategy.NONE;
    } else {
View Full Code Here

    // 5) There is no local strategy on the edge for the initial workset, as
    //    this translates to a local strategy that would only be executed in the first superstep
   
    final boolean merge;
    if (mergeIterationAuxTasks && wspn.getOutgoingChannels().size() == 1) {
      final Channel c = wspn.getOutgoingChannels().get(0);
      final PlanNode successor = c.getTarget();
      merge = c.getShipStrategy() == ShipStrategyType.FORWARD &&
          c.getLocalStrategy() == LocalStrategy.NONE &&
          c.getTempMode() == TempMode.NONE &&
          successor.getDegreeOfParallelism() == wspn.getDegreeOfParallelism() &&
          !(successor instanceof NAryUnionPlanNode) &&
          successor != iteration.getNextWorkSetPlanNode() &&
          iteration.getInitialWorksetInput().getLocalStrategy() == LocalStrategy.NONE;
    } else {
View Full Code Here

        if (report == FeedbackPropertiesMeetRequirementsReport.NO_PARTIAL_SOLUTION) {
          ; // depends only through broadcast variable on the partial solution
        }
        else if (report == FeedbackPropertiesMeetRequirementsReport.NOT_MET) {
          // attach a no-op node through which we create the properties of the original input
          Channel toNoOp = new Channel(candidate);
          globPropsReq.parameterizeChannel(toNoOp, false);
          locPropsReq.parameterizeChannel(toNoOp);
         
          UnaryOperatorNode rebuildPropertiesNode = new UnaryOperatorNode("Rebuild Partial Solution Properties", FieldList.EMPTY_LIST);
          rebuildPropertiesNode.setDegreeOfParallelism(candidate.getDegreeOfParallelism());
         
          SingleInputPlanNode rebuildPropertiesPlanNode = new SingleInputPlanNode(rebuildPropertiesNode, "Rebuild Partial Solution Properties", toNoOp, DriverStrategy.UNARY_NO_OP);
          rebuildPropertiesPlanNode.initProperties(toNoOp.getGlobalProperties(), toNoOp.getLocalProperties());
          estimator.costOperator(rebuildPropertiesPlanNode);
           
          GlobalProperties atEndGlobalModified = rebuildPropertiesPlanNode.getGlobalProperties();
          LocalProperties atEndLocalModified = rebuildPropertiesPlanNode.getLocalProperties();
           
View Full Code Here

      // verify the strategies
      Assert.assertEquals(ShipStrategyType.FORWARD, mapper.getInput().getShipStrategy());
      Assert.assertEquals(ShipStrategyType.PARTITION_HASH, reducer.getInput().getShipStrategy());
      Assert.assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
     
      Channel c = reducer.getInput();
      Assert.assertEquals(LocalStrategy.COMBININGSORT, c.getLocalStrategy());
      FieldList l = new FieldList(0);
      Assert.assertEquals(l, c.getShipStrategyKeys());
      Assert.assertEquals(l, c.getLocalStrategyKeys());
      Assert.assertTrue(Arrays.equals(c.getLocalStrategySortOrder(), reducer.getSortOrders(0)));
     
      // check the combiner
      SingleInputPlanNode combiner = (SingleInputPlanNode) reducer.getPredecessor();
      Assert.assertEquals(DriverStrategy.SORTED_GROUP_COMBINE, combiner.getDriverStrategy());
      Assert.assertEquals(l, combiner.getKeys(0));
View Full Code Here

        if (report == FeedbackPropertiesMeetRequirementsReport.NO_PARTIAL_SOLUTION) {
          ; // depends only through broadcast variable on the workset solution
        }
        else if (report == FeedbackPropertiesMeetRequirementsReport.NOT_MET) {
          // attach a no-op node through which we create the properties of the original input
          Channel toNoOp = new Channel(candidate);
          globPropsReqWorkset.parameterizeChannel(toNoOp, false);
          locPropsReqWorkset.parameterizeChannel(toNoOp);
         
          UnaryOperatorNode rebuildWorksetPropertiesNode = new UnaryOperatorNode("Rebuild Workset Properties", FieldList.EMPTY_LIST);
         
          rebuildWorksetPropertiesNode.setDegreeOfParallelism(candidate.getDegreeOfParallelism());
         
          SingleInputPlanNode rebuildWorksetPropertiesPlanNode = new SingleInputPlanNode(rebuildWorksetPropertiesNode, "Rebuild Workset Properties", toNoOp, DriverStrategy.UNARY_NO_OP);
          rebuildWorksetPropertiesPlanNode.initProperties(toNoOp.getGlobalProperties(), toNoOp.getLocalProperties());
          estimator.costOperator(rebuildWorksetPropertiesPlanNode);
           
          GlobalProperties atEndGlobalModified = rebuildWorksetPropertiesPlanNode.getGlobalProperties();
          LocalProperties atEndLocalModified = rebuildWorksetPropertiesPlanNode.getLocalProperties();
           
View Full Code Here

     
      Assert.assertEquals(ShipStrategyType.FORWARD, mapper.getInput().getShipStrategy());
      Assert.assertEquals(ShipStrategyType.PARTITION_RANGE, reducer.getInput().getShipStrategy());
      Assert.assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
     
      Channel c = reducer.getInput();
      Assert.assertEquals(LocalStrategy.COMBININGSORT, c.getLocalStrategy());
      FieldList l = new FieldList(0);
      Assert.assertEquals(l, c.getShipStrategyKeys());
      Assert.assertEquals(l, c.getLocalStrategyKeys());
     
      // check that the sort orders are descending
      Assert.assertFalse(c.getShipStrategySortOrder()[0]);
      Assert.assertFalse(c.getLocalStrategySortOrder()[0]);
     
      // check the combiner
      SingleInputPlanNode combiner = (SingleInputPlanNode) reducer.getPredecessor();
      Assert.assertEquals(DriverStrategy.SORTED_GROUP_COMBINE, combiner.getDriverStrategy());
      Assert.assertEquals(l, combiner.getKeys(0));
View Full Code Here

          continue;
        }
       
        for (RequestedGlobalProperties igps1: intGlobal1) {
          // create a candidate channel for the first input. mark it cached, if the connection says so
          final Channel c1 = new Channel(child1, this.input1.getMaterializationMode());
          if (this.input1.getShipStrategy() == null) {
            // free to choose the ship strategy
            igps1.parameterizeChannel(c1, dopChange1);
           
            // if the DOP changed, make sure that we cancel out properties, unless the
            // ship strategy preserves/establishes them even under changing DOPs
            if (dopChange1 && !c1.getShipStrategy().isNetworkStrategy()) {
              c1.getGlobalProperties().reset();
            }
          } else {
            // ship strategy fixed by compiler hint
            if (this.keys1 != null) {
              c1.setShipStrategy(this.input1.getShipStrategy(), this.keys1.toFieldList());
            } else {
              c1.setShipStrategy(this.input1.getShipStrategy());
            }
           
            if (dopChange1) {
              c1.adjustGlobalPropertiesForFullParallelismChange();
            }
          }
         
          for (RequestedGlobalProperties igps2: intGlobal2) {
            // create a candidate channel for the first input. mark it cached, if the connection says so
            final Channel c2 = new Channel(child2, this.input2.getMaterializationMode());
            if (this.input2.getShipStrategy() == null) {
              // free to choose the ship strategy
              igps2.parameterizeChannel(c2, dopChange2);
             
              // if the DOP changed, make sure that we cancel out properties, unless the
              // ship strategy preserves/establishes them even under changing DOPs
              if (dopChange2 && !c2.getShipStrategy().isNetworkStrategy()) {
                c2.getGlobalProperties().reset();
              }
            } else {
              // ship strategy fixed by compiler hint
              if (this.keys2 != null) {
                c2.setShipStrategy(this.input2.getShipStrategy(), this.keys2.toFieldList());
              } else {
                c2.setShipStrategy(this.input2.getShipStrategy());
              }
             
              if (dopChange2) {
                c2.adjustGlobalPropertiesForFullParallelismChange();
              }
            }
           
            /* ********************************************************************
             * NOTE: Depending on how we proceed with different partitionings,
             *       we might at some point need a compatibility check between
             *       the pairs of global properties.
             * *******************************************************************/
           
            for (GlobalPropertiesPair gpp : allGlobalPairs) {
              if (gpp.getProperties1().isMetBy(c1.getGlobalProperties()) &&
                gpp.getProperties2().isMetBy(c2.getGlobalProperties()) )
              {
                Channel c1Clone = c1.clone();
                c1Clone.setRequiredGlobalProps(gpp.getProperties1());
                c2.setRequiredGlobalProps(gpp.getProperties2());
               
                // we form a valid combination, so create the local candidates
                // for this
                addLocalCandidates(c1Clone, c2, broadcastPlanChannels, igps1, igps2, outputPlans, allLocalPairs, estimator);
View Full Code Here

  protected void addLocalCandidates(Channel template1, Channel template2, List<Set<? extends NamedChannel>> broadcastPlanChannels,
      RequestedGlobalProperties rgps1, RequestedGlobalProperties rgps2,
      List<PlanNode> target, LocalPropertiesPair[] validLocalCombinations, CostEstimator estimator)
  {
    for (RequestedLocalProperties ilp1 : this.input1.getInterestingProperties().getLocalProperties()) {
      final Channel in1 = template1.clone();
      ilp1.parameterizeChannel(in1);
     
      for (RequestedLocalProperties ilp2 : this.input2.getInterestingProperties().getLocalProperties()) {
        final Channel in2 = template2.clone();
        ilp2.parameterizeChannel(in2);
       
        for (OperatorDescriptorDual dps: getProperties()) {
          for (LocalPropertiesPair lpp : dps.getPossibleLocalProperties()) {
            if (lpp.getProperties1().isMetBy(in1.getLocalProperties()) &&
              lpp.getProperties2().isMetBy(in2.getLocalProperties()) )
            {
              // valid combination
              // for non trivial local properties, we need to check that they are co compatible
              // (such as when some sort order is requested, that both are the same sort order
              if (dps.areCoFulfilled(lpp.getProperties1(), lpp.getProperties2(),
                in1.getLocalProperties(), in2.getLocalProperties()))
              {
                // copy, because setting required properties and instantiation may
                // change the channels and should not affect prior candidates
                Channel in1Copy = in1.clone();
                in1Copy.setRequiredLocalProps(lpp.getProperties1());
               
                Channel in2Copy = in2.clone();
                in2Copy.setRequiredLocalProps(lpp.getProperties2());
               
                // all right, co compatible
                instantiate(dps, in1Copy, in2Copy, broadcastPlanChannels, target, estimator, rgps1, rgps2, ilp1, ilp2);
                break;
              } else {
View Full Code Here

TOP

Related Classes of org.apache.flink.compiler.plan.Channel

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.