Package org.apache.flink.compiler.plan

Examples of org.apache.flink.compiler.plan.Channel


    {
      return new SingleInputPlanNode(node, "Reduce ("+node.getPactContract().getName()+")", in, DriverStrategy.SORTED_REDUCE, this.keyList);
    }
    else {
      // non forward case. all local properties are killed anyways, so we can safely plug in a combiner
      Channel toCombiner = new Channel(in.getSource());
      toCombiner.setShipStrategy(ShipStrategyType.FORWARD);
     
      // create an input node for combine with same DOP as input node
      ReduceNode combinerNode = ((ReduceNode) node).getCombinerUtilityNode();
      combinerNode.setDegreeOfParallelism(in.getSource().getDegreeOfParallelism());

      SingleInputPlanNode combiner = new SingleInputPlanNode(combinerNode, "Combine ("+node.getPactContract().getName()+")", toCombiner, DriverStrategy.SORTED_PARTIAL_REDUCE, this.keyList);
      combiner.setCosts(new Costs(0, 0));
      combiner.initProperties(toCombiner.getGlobalProperties(), toCombiner.getLocalProperties());
     
      Channel toReducer = new Channel(combiner);
      toReducer.setShipStrategy(in.getShipStrategy(), in.getShipStrategyKeys(), in.getShipStrategySortOrder());
      toReducer.setLocalStrategy(LocalStrategy.SORT, in.getLocalStrategyKeys(), in.getLocalStrategySortOrder());
      return new SingleInputPlanNode(node, "Reduce("+node.getPactContract().getName()+")", toReducer, DriverStrategy.SORTED_REDUCE, this.keyList);
    }
  }
View Full Code Here


    InterestingProperties ips = this.input.getInterestingProperties();
    for (PlanNode p : subPlans) {
      for (RequestedGlobalProperties gp : ips.getGlobalProperties()) {
        for (RequestedLocalProperties lp : ips.getLocalProperties()) {
          Channel c = new Channel(p);
          gp.parameterizeChannel(c, dopChange);
          lp.parameterizeChannel(c);
          c.setRequiredLocalProps(lp);
          c.setRequiredGlobalProps(gp);
         
          // no need to check whether the created properties meet what we need in case
          // of ordering or global ordering, because the only interesting properties we have
          // are what we require
          outputPlans.add(new SinkPlanNode(this, "DataSink("+this.getPactContract().getName()+")" ,c));
View Full Code Here

        // output connection side
        if (inConns.hasNext() || inputNum > 0) {
          writer.print(", \"side\": \"" + (inputNum == 0 ? "first" : "second") + "\"");
        }
        // output shipping strategy and channel type
        final Channel channel = (inConn instanceof Channel) ? (Channel) inConn : null;
        final ShipStrategyType shipType = channel != null ? channel.getShipStrategy() :
            ((PactConnection) inConn).getShipStrategy();
         
        String shipStrategy = null;
        if (shipType != null) {
          switch (shipType) {
          case NONE:
            // nothing
            break;
          case FORWARD:
            shipStrategy = "Forward";
            break;
          case BROADCAST:
            shipStrategy = "Broadcast";
            break;
          case PARTITION_HASH:
            shipStrategy = "Hash Partition";
            break;
          case PARTITION_RANGE:
            shipStrategy = "Range Partition";
            break;
          case PARTITION_RANDOM:
            shipStrategy = "Redistribute";
            break;
          case PARTITION_FORCED_REBALANCE:
            shipStrategy = "Rebalance";
            break;
          default:
            throw new CompilerException("Unknown ship strategy '" + inConn.getShipStrategy().name()
              + "' in JSON generator.");
          }
        }
       
        if (channel != null && channel.getShipStrategyKeys() != null && channel.getShipStrategyKeys().size() > 0) {
          shipStrategy += " on " + (channel.getShipStrategySortOrder() == null ?
              channel.getShipStrategyKeys().toString() :
              Utils.createOrdering(channel.getShipStrategyKeys(), channel.getShipStrategySortOrder()).toString());
        }

        if (shipStrategy != null) {
          writer.print(", \"ship_strategy\": \"" + shipStrategy + "\"");
        }
       
        if (channel != null) {
          String localStrategy = null;
          switch (channel.getLocalStrategy()) {
          case NONE:
            break;
          case SORT:
            localStrategy = "Sort";
            break;
          case COMBININGSORT:
            localStrategy = "Sort (combining)";
            break;
          default:
            throw new CompilerException("Unknown local strategy " + channel.getLocalStrategy().name());
          }
         
          if (channel != null && channel.getLocalStrategyKeys() != null && channel.getLocalStrategyKeys().size() > 0) {
            localStrategy += " on " + (channel.getLocalStrategySortOrder() == null ?
                channel.getLocalStrategyKeys().toString() :
                Utils.createOrdering(channel.getLocalStrategyKeys(), channel.getLocalStrategySortOrder()).toString());
          }
         
          if (localStrategy != null) {
            writer.print(", \"local_strategy\": \"" + localStrategy + "\"");
          }
         
          if (channel != null && channel.getTempMode() != TempMode.NONE) {
            String tempMode = channel.getTempMode().toString();
            writer.print(", \"temp_mode\": \"" + tempMode + "\"");
          }
        }
       
        writer.print('}');
View Full Code Here

  @SuppressWarnings("unchecked")
  protected void traverse(PlanNode node, T parentSchema, boolean createUtilities) {
    // distinguish the node types
    if (node instanceof SinkPlanNode) {
      SinkPlanNode sn = (SinkPlanNode) node;
      Channel inchannel = sn.getInput();
     
      T schema = createEmptySchema();
      sn.postPassHelper = schema;
     
      // add the sinks information to the schema
View Full Code Here

    @Override
    public void postVisit(PlanNode visitable) {
     
      if (visitable instanceof BinaryUnionPlanNode) {
        final BinaryUnionPlanNode unionNode = (BinaryUnionPlanNode) visitable;
        final Channel in1 = unionNode.getInput1();
        final Channel in2 = unionNode.getInput2();
     
        PlanNode newUnionNode;

        List<Channel> inputs = new ArrayList<Channel>();
        collect(in1, inputs);
View Full Code Here

   
    // distinguish the node types
    if (node instanceof SinkPlanNode) {
      // descend to the input channel
      SinkPlanNode sn = (SinkPlanNode) node;
      Channel inchannel = sn.getInput();
      traverseChannel(inchannel);
    }
    else if (node instanceof SourcePlanNode) {
      TypeInformation<?> typeInfo = getTypeInfoFromSource((SourcePlanNode) node);
      ((SourcePlanNode) node).setSerializer(createSerializer(typeInfo));
View Full Code Here

      }
     
      totalCosts.addCosts(costs);
    }
   
    Channel firstInput = null;
    Channel secondInput = null;
    Costs driverCosts = new Costs();
    int costWeight = 1;
   
    // adjust with the cost weight factor
    if (n.isOnDynamicPath()) {
View Full Code Here

    // create all candidates
    for (PlanNode child : subPlans) {
      if (this.inConn.getShipStrategy() == null) {
        // pick the strategy ourselves
        for (RequestedGlobalProperties igps: intGlobal) {
          final Channel c = new Channel(child, this.inConn.getMaterializationMode());
          igps.parameterizeChannel(c, dopChange);
         
          // if the DOP changed, make sure that we cancel out properties, unless the
          // ship strategy preserves/establishes them even under changing DOPs
          if (dopChange && !c.getShipStrategy().isNetworkStrategy()) {
            c.getGlobalProperties().reset();
          }
         
          // check whether we meet any of the accepted properties
          // we may remove this check, when we do a check to not inherit
          // requested global properties that are incompatible with all possible
          // requested properties
          for (RequestedGlobalProperties rgps: allValidGlobals) {
            if (rgps.isMetBy(c.getGlobalProperties())) {
              c.setRequiredGlobalProps(rgps);
              addLocalCandidates(c, broadcastPlanChannels, igps, outputPlans, estimator);
              break;
            }
          }
        }
      } else {
        // hint fixed the strategy
        final Channel c = new Channel(child, this.inConn.getMaterializationMode());
        if (this.keys != null) {
          c.setShipStrategy(this.inConn.getShipStrategy(), this.keys.toFieldList());
        } else {
          c.setShipStrategy(this.inConn.getShipStrategy());
        }
       
        if (dopChange) {
          c.adjustGlobalPropertiesForFullParallelismChange();
        }

        // check whether we meet any of the accepted properties
        for (RequestedGlobalProperties rgps: allValidGlobals) {
          if (rgps.isMetBy(c.getGlobalProperties())) {
            addLocalCandidates(c, broadcastPlanChannels, rgps, outputPlans, estimator);
            break;
          }
        }
      }
View Full Code Here

 
  protected void addLocalCandidates(Channel template, List<Set<? extends NamedChannel>> broadcastPlanChannels, RequestedGlobalProperties rgps,
      List<PlanNode> target, CostEstimator estimator)
  {
    for (RequestedLocalProperties ilp : this.inConn.getInterestingProperties().getLocalProperties()) {
      final Channel in = template.clone();
      ilp.parameterizeChannel(in);
     
      // instantiate a candidate, if the instantiated local properties meet one possible local property set
      outer:
      for (OperatorDescriptorSingle dps: getPossibleProperties()) {
        for (RequestedLocalProperties ilps : dps.getPossibleLocalProperties()) {
          if (ilps.isMetBy(in.getLocalProperties())) {
            in.setRequiredLocalProps(ilps);
            instantiateCandidate(dps, in, broadcastPlanChannels, target, estimator, rgps, ilp);
            break outer;
          }
        }
      }
View Full Code Here

    if (in.getShipStrategy() == ShipStrategyType.FORWARD) {
      // locally connected, directly instantiate
      return new SingleInputPlanNode(node, "GroupReduce ("+node.getPactContract().getName()+")", in, DriverStrategy.ALL_GROUP_REDUCE);
    } else {
      // non forward case.plug in a combiner
      Channel toCombiner = new Channel(in.getSource());
      toCombiner.setShipStrategy(ShipStrategyType.FORWARD);
     
      // create an input node for combine with same DOP as input node
      GroupReduceNode combinerNode = ((GroupReduceNode) node).getCombinerUtilityNode();
      combinerNode.setDegreeOfParallelism(in.getSource().getDegreeOfParallelism());

      SingleInputPlanNode combiner = new SingleInputPlanNode(combinerNode, "Combine ("+node.getPactContract().getName()+")", toCombiner, DriverStrategy.ALL_GROUP_COMBINE);
      combiner.setCosts(new Costs(0, 0));
      combiner.initProperties(toCombiner.getGlobalProperties(), toCombiner.getLocalProperties());
     
      Channel toReducer = new Channel(combiner);
      toReducer.setShipStrategy(in.getShipStrategy(), in.getShipStrategyKeys(), in.getShipStrategySortOrder());
      toReducer.setLocalStrategy(in.getLocalStrategy(), in.getLocalStrategyKeys(), in.getLocalStrategySortOrder());
      return new SingleInputPlanNode(node, "GroupReduce ("+node.getPactContract().getName()+")", toReducer, DriverStrategy.ALL_GROUP_REDUCE);
    }
  }
View Full Code Here

TOP

Related Classes of org.apache.flink.compiler.plan.Channel

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.