Package org.apache.flink.runtime.operators.shipping

Examples of org.apache.flink.runtime.operators.shipping.ShipStrategyType


  @Override
  public void setInput(Map<Operator<?>, OptimizerNode> contractToNode) {
    // see if there is a hint that dictates which shipping strategy to use for BOTH inputs
    final Configuration conf = getPactContract().getParameters();
    ShipStrategyType preSet1 = null;
    ShipStrategyType preSet2 = null;
   
    String shipStrategy = conf.getString(PactCompiler.HINT_SHIP_STRATEGY, null);
    if (shipStrategy != null) {
      if (PactCompiler.HINT_SHIP_STRATEGY_FORWARD.equals(shipStrategy)) {
        preSet1 = preSet2 = ShipStrategyType.FORWARD;
View Full Code Here


        if (inConns.hasNext() || inputNum > 0) {
          writer.print(", \"side\": \"" + (inputNum == 0 ? "first" : "second") + "\"");
        }
        // output shipping strategy and channel type
        final Channel channel = (inConn instanceof Channel) ? (Channel) inConn : null;
        final ShipStrategyType shipType = channel != null ? channel.getShipStrategy() :
            ((PactConnection) inConn).getShipStrategy();
         
        String shipStrategy = null;
        if (shipType != null) {
          switch (shipType) {
View Full Code Here

  @Override
  public void setInput(Map<Operator<?>, OptimizerNode> contractToNode) throws CompilerException {
    // see if an internal hint dictates the strategy to use
    final Configuration conf = getPactContract().getParameters();
    final String shipStrategy = conf.getString(PactCompiler.HINT_SHIP_STRATEGY, null);
    final ShipStrategyType preSet;
   
    if (shipStrategy != null) {
      if (shipStrategy.equalsIgnoreCase(PactCompiler.HINT_SHIP_STRATEGY_REPARTITION_HASH)) {
        preSet = ShipStrategyType.PARTITION_HASH;
      } else if (shipStrategy.equalsIgnoreCase(PactCompiler.HINT_SHIP_STRATEGY_REPARTITION_RANGE)) {
View Full Code Here

    // verify reducer
    assertEquals(ShipStrategyType.PARTITION_HASH, worksetReducer.getInput().getShipStrategy());
    assertEquals(list0, worksetReducer.getKeys(0));
   
    // currently, the system may partition before or after the mapper
    ShipStrategyType ss1 = deltaMapper.getInput().getShipStrategy();
    ShipStrategyType ss2 = deltaMapper.getOutgoingChannels().get(0).getShipStrategy();
   
    assertTrue( (ss1 == ShipStrategyType.FORWARD && ss2 == ShipStrategyType.PARTITION_HASH) ||
          (ss2 == ShipStrategyType.FORWARD && ss1 == ShipStrategyType.PARTITION_HASH) );
   
    new NepheleJobGraphGenerator().compileJobGraph(oPlan);
View Full Code Here

      // verify reducer
      assertEquals(ShipStrategyType.PARTITION_HASH, worksetReducer.getInput().getShipStrategy());
      assertEquals(new FieldList(1, 2), worksetReducer.getKeys(0));
     
      // currently, the system may partition before or after the mapper
      ShipStrategyType ss1 = deltaMapper.getInput().getShipStrategy();
      ShipStrategyType ss2 = deltaMapper.getOutgoingChannels().get(0).getShipStrategy();
     
      assertTrue( (ss1 == ShipStrategyType.FORWARD && ss2 == ShipStrategyType.PARTITION_HASH) ||
            (ss2 == ShipStrategyType.FORWARD && ss1 == ShipStrategyType.PARTITION_HASH) );
   
      new NepheleJobGraphGenerator().compileJobGraph(oPlan);
View Full Code Here

    // mapper respectively reducer
    SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
    SingleInputPlanNode red2Node = (SingleInputPlanNode) sinkNode.getPredecessor();
    SingleInputPlanNode map2Node = (SingleInputPlanNode) red2Node.getPredecessor();
   
    ShipStrategyType mapIn = map2Node.getInput().getShipStrategy();
    ShipStrategyType redIn = red2Node.getInput().getShipStrategy();
   
    Assert.assertEquals("Invalid ship strategy for an operator.", ShipStrategyType.PARTITION_HASH, mapIn);
    Assert.assertEquals("Invalid ship strategy for an operator.", ShipStrategyType.FORWARD, redIn);
  }
View Full Code Here

    // mapper respectively reducer
    SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
    SingleInputPlanNode red2Node = (SingleInputPlanNode) sinkNode.getPredecessor();
    SingleInputPlanNode map2Node = (SingleInputPlanNode) red2Node.getPredecessor();
   
    ShipStrategyType mapIn = map2Node.getInput().getShipStrategy();
    ShipStrategyType reduceIn = red2Node.getInput().getShipStrategy();
   
    Assert.assertEquals("Invalid ship strategy for an operator.", ShipStrategyType.FORWARD, mapIn);
    Assert.assertEquals("Invalid ship strategy for an operator.", ShipStrategyType.PARTITION_HASH, reduceIn);
  }
View Full Code Here

    // mapper respectively reducer
    SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
    SingleInputPlanNode red2Node = (SingleInputPlanNode) sinkNode.getPredecessor();
    SingleInputPlanNode map2Node = (SingleInputPlanNode) red2Node.getPredecessor();
   
    ShipStrategyType mapIn = map2Node.getInput().getShipStrategy();
    ShipStrategyType reduceIn = red2Node.getInput().getShipStrategy();
   
    Assert.assertTrue("Invalid ship strategy for an operator.",
        (ShipStrategyType.PARTITION_RANDOM ==  mapIn && ShipStrategyType.PARTITION_HASH == reduceIn) ||
        (ShipStrategyType.PARTITION_HASH == mapIn && ShipStrategyType.FORWARD == reduceIn));
  }
View Full Code Here

      final List<RecordWriter<Record>> writers = new ArrayList<RecordWriter<Record>>(numOutputs);

      // create a writer for each output
      for (int i = 0; i < numOutputs; i++) {
        // create the OutputEmitter from output ship strategy
        final ShipStrategyType strategy = config.getOutputShipStrategy(i);
        final TypeComparatorFactory<?> compFact = config.getOutputComparator(i, cl);
        final RecordOutputEmitter oe;
        if (compFact == null) {
          oe = new RecordOutputEmitter(strategy);
        } else {
          @SuppressWarnings("unchecked")
          TypeComparator<Record> comparator = (TypeComparator<Record>) compFact.createComparator();
          if (!comparator.supportsCompareAgainstReference()) {
            throw new Exception("Incompatibe serializer-/comparator factories.");
          }
          final DataDistribution distribution = config.getOutputDataDistribution(i, cl);
          oe = new RecordOutputEmitter(strategy, comparator, distribution);
        }

        writers.add(new RecordWriter<Record>(task, oe));
      }
      if (eventualOutputs != null) {
        eventualOutputs.addAll(writers);
      }

      @SuppressWarnings("unchecked")
      final Collector<T> outColl = (Collector<T>) new RecordOutputCollector(writers);
      return outColl;
    }
    else {
      // generic case
      final List<RecordWriter<SerializationDelegate<T>>> writers = new ArrayList<RecordWriter<SerializationDelegate<T>>>(numOutputs);

      // create a writer for each output
      for (int i = 0; i < numOutputs; i++)
      {
        // create the OutputEmitter from output ship strategy
        final ShipStrategyType strategy = config.getOutputShipStrategy(i);
        final TypeComparatorFactory<T> compFactory = config.getOutputComparator(i, cl);
        final DataDistribution dataDist = config.getOutputDataDistribution(i, cl);

        final ChannelSelector<SerializationDelegate<T>> oe;
        if (compFactory == null) {
View Full Code Here

TOP

Related Classes of org.apache.flink.runtime.operators.shipping.ShipStrategyType

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.