Examples of WorksetIterationPlanNode


Examples of org.apache.flink.compiler.plan.WorksetIterationPlanNode

      // cannot chain the nodes that produce the next workset or the next solution set, if they are not the
      // in a tail
      if (this.currentIteration != null && this.currentIteration instanceof WorksetIterationPlanNode &&
          node.getOutgoingChannels().size() > 0)
      {
        WorksetIterationPlanNode wspn = (WorksetIterationPlanNode) this.currentIteration;
        if (wspn.getSolutionSetDeltaPlanNode() == pred || wspn.getNextWorkSetPlanNode() == pred) {
          chaining = false;
        }
      }
      // cannot chain the nodes that produce the next workset in a bulk iteration if a termination criterion follows
      if (this.currentIteration != null && this.currentIteration instanceof BulkIterationPlanNode)
      {
        BulkIterationPlanNode wspn = (BulkIterationPlanNode) this.currentIteration;
        if (node == wspn.getRootOfTerminationCriterion() && wspn.getRootOfStepFunction() == pred){
          chaining = false;
        }else if(node.getOutgoingChannels().size() > 0 &&(wspn.getRootOfStepFunction() == pred ||
            wspn.getRootOfTerminationCriterion() == pred)) {
          chaining = false;
        }
      }
    }
   
View Full Code Here

Examples of org.apache.flink.compiler.plan.WorksetIterationPlanNode

    return toReturn;
  }
 
  private AbstractJobVertex createWorksetIterationHead(WorksetPlanNode wspn) {
    // get the bulk iteration that corresponds to this partial solution node
    final WorksetIterationPlanNode iteration = wspn.getContainingIterationNode();
   
    // check whether we need an individual vertex for the partial solution, or whether we
    // attach ourselves to the vertex of the parent node. We can combine the head with a node of
    // the step function, if
    // 1) There is one parent that the partial solution connects to via a forward pattern and no
    //    local strategy
    // 2) DOP and the number of subtasks per instance does not change
    // 3) That successor is not a union
    // 4) That successor is not itself the last node of the step function
    // 5) There is no local strategy on the edge for the initial workset, as
    //    this translates to a local strategy that would only be executed in the first superstep
   
    final boolean merge;
    if (mergeIterationAuxTasks && wspn.getOutgoingChannels().size() == 1) {
      final Channel c = wspn.getOutgoingChannels().get(0);
      final PlanNode successor = c.getTarget();
      merge = c.getShipStrategy() == ShipStrategyType.FORWARD &&
          c.getLocalStrategy() == LocalStrategy.NONE &&
          c.getTempMode() == TempMode.NONE &&
          successor.getDegreeOfParallelism() == wspn.getDegreeOfParallelism() &&
          !(successor instanceof NAryUnionPlanNode) &&
          successor != iteration.getNextWorkSetPlanNode() &&
          iteration.getInitialWorksetInput().getLocalStrategy() == LocalStrategy.NONE;
    } else {
      merge = false;
    }
   
    // create or adopt the head vertex
    final AbstractJobVertex toReturn;
    final AbstractJobVertex headVertex;
    final TaskConfig headConfig;
    if (merge) {
      final PlanNode successor = wspn.getOutgoingChannels().get(0).getTarget();
      headVertex = (AbstractJobVertex) this.vertices.get(successor);
     
      if (headVertex == null) {
        throw new CompilerException(
          "Bug: Trying to merge solution set with its sucessor, but successor has not been created.");
      }
     
      // reset the vertex type to iteration head
      headVertex.setInvokableClass(IterationHeadPactTask.class);
      headConfig = new TaskConfig(headVertex.getConfiguration());
      toReturn = null;
    } else {
      // instantiate the head vertex and give it a no-op driver as the driver strategy.
      // everything else happens in the post visit, after the input (the initial partial solution)
      // is connected.
      headVertex = new AbstractJobVertex("IterationHead("+iteration.getNodeName()+")");
      headVertex.setInvokableClass(IterationHeadPactTask.class);
      headConfig = new TaskConfig(headVertex.getConfiguration());
      headConfig.setDriver(NoOpDriver.class);
      toReturn = headVertex;
    }
   
    headConfig.setSolutionSetUnmanaged(iteration.getIterationNode().getIterationContract().isSolutionSetUnManaged());
   
    // create the iteration descriptor and the iteration to it
    IterationDescriptor descr = this.iterations.get(iteration);
    if (descr == null) {
      throw new CompilerException("Bug: Iteration descriptor was not created at when translating the iteration node.");
View Full Code Here

Examples of org.apache.flink.compiler.plan.WorksetIterationPlanNode

            // mark that we materialize the input
            siSolutionDeltaCandidate.getInput().setTempMode(TempMode.PIPELINE_BREAKER);
            immediateDeltaUpdate = false;
          }
         
          WorksetIterationPlanNode wsNode = new WorksetIterationPlanNode(
            this, "WorksetIteration ("+this.getPactContract().getName()+")", solutionSetIn, worksetIn, sspn, wspn, worksetCandidate, solutionSetCandidate);
          wsNode.setImmediateSolutionSetUpdate(immediateDeltaUpdate);
          wsNode.initProperties(gp, lp);
          target.add(wsNode);
        }
      }
    }
  }
View Full Code Here

Examples of org.apache.flink.compiler.plan.WorksetIterationPlanNode

    }
   
    SourcePlanNode vertexSource = or.getNode(VERTEX_SOURCE);
    SourcePlanNode edgesSource = or.getNode(EDGES_SOURCE);
    SinkPlanNode sink = or.getNode(SINK);
    WorksetIterationPlanNode iter = or.getNode(ITERATION_NAME);
   
    DualInputPlanNode neighborsJoin = or.getNode(JOIN_NEIGHBORS_MATCH);
    SingleInputPlanNode minIdReducer = or.getNode(MIN_ID_REDUCER);
    SingleInputPlanNode minIdCombiner = (SingleInputPlanNode) minIdReducer.getPredecessor();
    DualInputPlanNode updatingMatch = or.getNode(UPDATE_ID_MATCH);
   
    // test all drivers
    Assert.assertEquals(DriverStrategy.NONE, sink.getDriverStrategy());
    Assert.assertEquals(DriverStrategy.NONE, vertexSource.getDriverStrategy());
    Assert.assertEquals(DriverStrategy.NONE, edgesSource.getDriverStrategy());
   
    Assert.assertEquals(DriverStrategy.HYBRIDHASH_BUILD_SECOND_CACHED, neighborsJoin.getDriverStrategy());
    Assert.assertTrue(!neighborsJoin.getInput1().getTempMode().isCached());
    Assert.assertTrue(!neighborsJoin.getInput2().getTempMode().isCached());
    Assert.assertEquals(set0, neighborsJoin.getKeysForInput1());
    Assert.assertEquals(set0, neighborsJoin.getKeysForInput2());
   
    Assert.assertEquals(DriverStrategy.HYBRIDHASH_BUILD_SECOND, updatingMatch.getDriverStrategy());
    Assert.assertEquals(set0, updatingMatch.getKeysForInput1());
    Assert.assertEquals(set0, updatingMatch.getKeysForInput2());
   
    // test all the shipping strategies
    Assert.assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
    Assert.assertEquals(ShipStrategyType.PARTITION_HASH, iter.getInitialSolutionSetInput().getShipStrategy());
    Assert.assertEquals(set0, iter.getInitialSolutionSetInput().getShipStrategyKeys());
    Assert.assertEquals(ShipStrategyType.PARTITION_HASH, iter.getInitialWorksetInput().getShipStrategy());
    Assert.assertEquals(set0, iter.getInitialWorksetInput().getShipStrategyKeys());
   
    Assert.assertEquals(ShipStrategyType.FORWARD, neighborsJoin.getInput1().getShipStrategy()); // workset
    Assert.assertEquals(ShipStrategyType.PARTITION_HASH, neighborsJoin.getInput2().getShipStrategy()); // edges
    Assert.assertEquals(set0, neighborsJoin.getInput2().getShipStrategyKeys());
   
    Assert.assertEquals(ShipStrategyType.PARTITION_HASH, minIdReducer.getInput().getShipStrategy());
    Assert.assertEquals(set0, minIdReducer.getInput().getShipStrategyKeys());
    Assert.assertEquals(ShipStrategyType.FORWARD, minIdCombiner.getInput().getShipStrategy());
   
    Assert.assertEquals(ShipStrategyType.FORWARD, updatingMatch.getInput1().getShipStrategy()); // min id
    Assert.assertEquals(ShipStrategyType.FORWARD, updatingMatch.getInput2().getShipStrategy()); // solution set
   
    // test all the local strategies
    Assert.assertEquals(LocalStrategy.NONE, sink.getInput().getLocalStrategy());
    Assert.assertEquals(LocalStrategy.NONE, iter.getInitialSolutionSetInput().getLocalStrategy());
    Assert.assertEquals(LocalStrategy.NONE, iter.getInitialWorksetInput().getLocalStrategy());
   
    Assert.assertEquals(LocalStrategy.NONE, neighborsJoin.getInput1().getLocalStrategy()); // workset
    Assert.assertEquals(LocalStrategy.NONE, neighborsJoin.getInput2().getLocalStrategy()); // edges
   
    Assert.assertEquals(LocalStrategy.COMBININGSORT, minIdReducer.getInput().getLocalStrategy());
    Assert.assertEquals(set0, minIdReducer.getInput().getLocalStrategyKeys());
    Assert.assertEquals(LocalStrategy.NONE, minIdCombiner.getInput().getLocalStrategy());
   
    Assert.assertEquals(LocalStrategy.NONE, updatingMatch.getInput1().getLocalStrategy()); // min id
    Assert.assertEquals(LocalStrategy.NONE, updatingMatch.getInput2().getLocalStrategy()); // solution set
   
    // check the dams
    Assert.assertTrue(TempMode.PIPELINE_BREAKER == iter.getInitialWorksetInput().getTempMode() ||
              LocalStrategy.SORT == iter.getInitialWorksetInput().getLocalStrategy());
   
    NepheleJobGraphGenerator jgg = new NepheleJobGraphGenerator();
    jgg.compileJobGraph(optPlan);
  }
View Full Code Here

Examples of org.apache.flink.compiler.plan.WorksetIterationPlanNode

    }
   
    SourcePlanNode vertexSource = or.getNode(VERTEX_SOURCE);
    SourcePlanNode edgesSource = or.getNode(EDGES_SOURCE);
    SinkPlanNode sink = or.getNode(SINK);
    WorksetIterationPlanNode iter = or.getNode(ITERATION_NAME);
   
    DualInputPlanNode neighborsJoin = or.getNode(JOIN_NEIGHBORS_MATCH);
    SingleInputPlanNode minIdReducer = or.getNode(MIN_ID_REDUCER);
    SingleInputPlanNode minIdCombiner = (SingleInputPlanNode) minIdReducer.getPredecessor();
    DualInputPlanNode updatingMatch = or.getNode(UPDATE_ID_MATCH);
   
    // test all drivers
    Assert.assertEquals(DriverStrategy.NONE, sink.getDriverStrategy());
    Assert.assertEquals(DriverStrategy.NONE, vertexSource.getDriverStrategy());
    Assert.assertEquals(DriverStrategy.NONE, edgesSource.getDriverStrategy());
   
    Assert.assertEquals(DriverStrategy.HYBRIDHASH_BUILD_SECOND_CACHED, neighborsJoin.getDriverStrategy());
    Assert.assertTrue(!neighborsJoin.getInput1().getTempMode().isCached());
    Assert.assertTrue(!neighborsJoin.getInput2().getTempMode().isCached());
    Assert.assertEquals(set0, neighborsJoin.getKeysForInput1());
    Assert.assertEquals(set0, neighborsJoin.getKeysForInput2());
   
    Assert.assertEquals(DriverStrategy.HYBRIDHASH_BUILD_FIRST, updatingMatch.getDriverStrategy());
    Assert.assertEquals(set0, updatingMatch.getKeysForInput1());
    Assert.assertEquals(set0, updatingMatch.getKeysForInput2());
   
    // test all the shipping strategies
    Assert.assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
    Assert.assertEquals(ShipStrategyType.PARTITION_HASH, iter.getInitialSolutionSetInput().getShipStrategy());
    Assert.assertEquals(set0, iter.getInitialSolutionSetInput().getShipStrategyKeys());
    Assert.assertEquals(ShipStrategyType.PARTITION_HASH, iter.getInitialWorksetInput().getShipStrategy());
    Assert.assertEquals(set0, iter.getInitialWorksetInput().getShipStrategyKeys());
   
    Assert.assertEquals(ShipStrategyType.FORWARD, neighborsJoin.getInput1().getShipStrategy()); // workset
    Assert.assertEquals(ShipStrategyType.PARTITION_HASH, neighborsJoin.getInput2().getShipStrategy()); // edges
    Assert.assertEquals(set0, neighborsJoin.getInput2().getShipStrategyKeys());
   
    Assert.assertEquals(ShipStrategyType.PARTITION_HASH, minIdReducer.getInput().getShipStrategy());
    Assert.assertEquals(set0, minIdReducer.getInput().getShipStrategyKeys());
    Assert.assertEquals(ShipStrategyType.FORWARD, minIdCombiner.getInput().getShipStrategy());
   
    Assert.assertEquals(ShipStrategyType.FORWARD, updatingMatch.getInput1().getShipStrategy()); // solution set
    Assert.assertEquals(ShipStrategyType.FORWARD, updatingMatch.getInput2().getShipStrategy()); // min id
   
    // test all the local strategies
    Assert.assertEquals(LocalStrategy.NONE, sink.getInput().getLocalStrategy());
    Assert.assertEquals(LocalStrategy.NONE, iter.getInitialSolutionSetInput().getLocalStrategy());
    Assert.assertEquals(LocalStrategy.NONE, iter.getInitialWorksetInput().getLocalStrategy());
   
    Assert.assertEquals(LocalStrategy.NONE, neighborsJoin.getInput1().getLocalStrategy()); // workset
    Assert.assertEquals(LocalStrategy.NONE, neighborsJoin.getInput2().getLocalStrategy()); // edges
   
    Assert.assertEquals(LocalStrategy.COMBININGSORT, minIdReducer.getInput().getLocalStrategy());
    Assert.assertEquals(set0, minIdReducer.getInput().getLocalStrategyKeys());
    Assert.assertEquals(LocalStrategy.NONE, minIdCombiner.getInput().getLocalStrategy());
   
    Assert.assertEquals(LocalStrategy.NONE, updatingMatch.getInput1().getLocalStrategy()); // min id
    Assert.assertEquals(LocalStrategy.NONE, updatingMatch.getInput2().getLocalStrategy()); // solution set
   
    // check the dams
    Assert.assertTrue(TempMode.PIPELINE_BREAKER == iter.getInitialWorksetInput().getTempMode() ||
              LocalStrategy.SORT == iter.getInitialWorksetInput().getLocalStrategy());
   
    NepheleJobGraphGenerator jgg = new NepheleJobGraphGenerator();
    jgg.compileJobGraph(optPlan);
  }
View Full Code Here

Examples of org.apache.flink.compiler.plan.WorksetIterationPlanNode

    }
   
    SourcePlanNode vertexSource = or.getNode(VERTEX_SOURCE);
    SourcePlanNode edgesSource = or.getNode(EDGES_SOURCE);
    SinkPlanNode sink = or.getNode(SINK);
    WorksetIterationPlanNode iter = or.getNode(ITERATION_NAME);
   
    DualInputPlanNode neighborsJoin = or.getNode(JOIN_NEIGHBORS_MATCH);
    DualInputPlanNode cogroup = or.getNode(MIN_ID_AND_UPDATE);
   
    // --------------------------------------------------------------------
    // Plan validation:
    //
    // We expect the plan to go with a sort-merge join, because the CoGroup
    // sorts and the join in the successive iteration can re-exploit the sorting.
    // --------------------------------------------------------------------
   
    // test all drivers
    Assert.assertEquals(DriverStrategy.NONE, sink.getDriverStrategy());
    Assert.assertEquals(DriverStrategy.NONE, vertexSource.getDriverStrategy());
    Assert.assertEquals(DriverStrategy.NONE, edgesSource.getDriverStrategy());
   
    Assert.assertEquals(DriverStrategy.MERGE, neighborsJoin.getDriverStrategy());
    Assert.assertEquals(set0, neighborsJoin.getKeysForInput1());
    Assert.assertEquals(set0, neighborsJoin.getKeysForInput2());
   
    Assert.assertEquals(DriverStrategy.CO_GROUP, cogroup.getDriverStrategy());
    Assert.assertEquals(set0, cogroup.getKeysForInput1());
    Assert.assertEquals(set0, cogroup.getKeysForInput2());
   
    // test all the shipping strategies
    Assert.assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
    Assert.assertEquals(ShipStrategyType.PARTITION_HASH, iter.getInitialSolutionSetInput().getShipStrategy());
    Assert.assertEquals(set0, iter.getInitialSolutionSetInput().getShipStrategyKeys());
    Assert.assertEquals(ShipStrategyType.PARTITION_HASH, iter.getInitialWorksetInput().getShipStrategy());
    Assert.assertEquals(set0, iter.getInitialWorksetInput().getShipStrategyKeys());
   
    Assert.assertEquals(ShipStrategyType.FORWARD, neighborsJoin.getInput1().getShipStrategy()); // workset
    Assert.assertEquals(ShipStrategyType.PARTITION_HASH, neighborsJoin.getInput2().getShipStrategy()); // edges
    Assert.assertEquals(set0, neighborsJoin.getInput2().getShipStrategyKeys());
    Assert.assertTrue(neighborsJoin.getInput2().getTempMode().isCached());
   
    Assert.assertEquals(ShipStrategyType.PARTITION_HASH, cogroup.getInput1().getShipStrategy()); // min id
    Assert.assertEquals(ShipStrategyType.FORWARD, cogroup.getInput2().getShipStrategy()); // solution set
   
    // test all the local strategies
    Assert.assertEquals(LocalStrategy.NONE, sink.getInput().getLocalStrategy());
    Assert.assertEquals(LocalStrategy.NONE, iter.getInitialSolutionSetInput().getLocalStrategy());
   
    // the sort for the neighbor join in the first iteration is pushed out of the loop
    Assert.assertEquals(LocalStrategy.SORT, iter.getInitialWorksetInput().getLocalStrategy());
    Assert.assertEquals(LocalStrategy.NONE, neighborsJoin.getInput1().getLocalStrategy()); // workset
    Assert.assertEquals(LocalStrategy.SORT, neighborsJoin.getInput2().getLocalStrategy()); // edges
   
    Assert.assertEquals(LocalStrategy.SORT, cogroup.getInput1().getLocalStrategy());
    Assert.assertEquals(LocalStrategy.NONE, cogroup.getInput2().getLocalStrategy()); // solution set
View Full Code Here

Examples of org.apache.flink.compiler.plan.WorksetIterationPlanNode

      SinkPlanNode sink = op.getDataSinks().iterator().next();
      assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
      assertEquals(DEFAULT_PARALLELISM, sink.getDegreeOfParallelism());
     
      // check the iteration
      WorksetIterationPlanNode iteration = (WorksetIterationPlanNode) sink.getInput().getSource();
      assertEquals(DEFAULT_PARALLELISM, iteration.getDegreeOfParallelism());
     
      // check the solution set join and the delta
      PlanNode ssDelta = iteration.getSolutionSetDeltaPlanNode();
      assertTrue(ssDelta instanceof DualInputPlanNode); // this is only true if the update functions preserves the partitioning
     
      DualInputPlanNode ssJoin = (DualInputPlanNode) ssDelta;
      assertEquals(DEFAULT_PARALLELISM, ssJoin.getDegreeOfParallelism());
      assertEquals(ShipStrategyType.PARTITION_HASH, ssJoin.getInput1().getShipStrategy());
      assertEquals(new FieldList(0), ssJoin.getInput1().getShipStrategyKeys());
     
      // check the workset set join
      DualInputPlanNode edgeJoin = (DualInputPlanNode) ssJoin.getInput1().getSource();
      assertEquals(DEFAULT_PARALLELISM, edgeJoin.getDegreeOfParallelism());
      assertEquals(ShipStrategyType.PARTITION_HASH, edgeJoin.getInput1().getShipStrategy());
      assertEquals(ShipStrategyType.FORWARD, edgeJoin.getInput2().getShipStrategy());
      assertTrue(edgeJoin.getInput1().getTempMode().isCached());
     
      assertEquals(new FieldList(0), edgeJoin.getInput1().getShipStrategyKeys());
     
      // check that the initial partitioning is pushed out of the loop
      assertEquals(ShipStrategyType.PARTITION_HASH, iteration.getInput1().getShipStrategy());
      assertEquals(ShipStrategyType.PARTITION_HASH, iteration.getInput2().getShipStrategy());
      assertEquals(new FieldList(0), iteration.getInput1().getShipStrategyKeys());
      assertEquals(new FieldList(0), iteration.getInput2().getShipStrategyKeys());
     
      // check that the initial workset sort is outside the loop
      assertEquals(LocalStrategy.SORT, iteration.getInput2().getLocalStrategy());
      assertEquals(new FieldList(0), iteration.getInput2().getLocalStrategyKeys());
    }
    catch (Exception e) {
      System.err.println(e.getMessage());
      e.printStackTrace();
      fail(e.getMessage());
View Full Code Here

Examples of org.apache.flink.compiler.plan.WorksetIterationPlanNode

    else if (visitable instanceof WorksetIterationPlanNode) {
     
      DeadlockPreventer dp = new DeadlockPreventer();
      List<PlanNode> planSinks = new ArrayList<PlanNode>(2);
     
      WorksetIterationPlanNode pspn = (WorksetIterationPlanNode) visitable;
      planSinks.add(pspn.getSolutionSetDeltaPlanNode());
      planSinks.add(pspn.getNextWorkSetPlanNode());
     
      dp.resolveDeadlocks(planSinks);
     
    }
  }
View Full Code Here

Examples of org.apache.flink.compiler.plan.WorksetIterationPlanNode

          + iterationNode.getPactContract().getName() + "'. Missing type information for key field " +
          e.getFieldNumber());
      }
    }
    else if (node instanceof WorksetIterationPlanNode) {
      WorksetIterationPlanNode iterationNode = (WorksetIterationPlanNode) node;
     
      // get the nodes current schema
      T schema;
      if (iterationNode.postPassHelper == null) {
        schema = createEmptySchema();
        iterationNode.postPassHelper = schema;
      } else {
        schema = (T) iterationNode.postPassHelper;
      }
      schema.increaseNumConnectionsThatContributed();
     
      // add the parent schema to the schema (which refers to the solution set schema)
      if (propagateParentSchemaDown) {
        addSchemaToSchema(parentSchema, schema, iterationNode.getPactContract().getName());
      }
     
      // check whether all outgoing channels have not yet contributed. come back later if not.
      if (schema.getNumConnectionsThatContributed() < iterationNode.getOutgoingChannels().size()) {
        return;
      }
      if (iterationNode.getNextWorkSetPlanNode() instanceof NAryUnionPlanNode) {
        throw new CompilerException("Optimizer cannot compile a workset iteration step function where the next workset is produced by a Union node.");
      }
      if (iterationNode.getSolutionSetDeltaPlanNode() instanceof NAryUnionPlanNode) {
        throw new CompilerException("Optimizer cannot compile a workset iteration step function where the solution set delta is produced by a Union node.");
      }
     
      // traverse the step function
      // pass an empty schema to the next workset and the parent schema to the solution set delta
      // these first traversals are schema only
      traverse(iterationNode.getNextWorkSetPlanNode(), createEmptySchema(), false);
      traverse(iterationNode.getSolutionSetDeltaPlanNode(), schema, false);
     
      T wss = (T) iterationNode.getWorksetPlanNode().postPassHelper;
      T sss = (T) iterationNode.getSolutionSetPlanNode().postPassHelper;
     
      if (wss == null) {
        throw new CompilerException("Error in Optimizer Post Pass: Workset schema is null after first traversal of the step function.");
      }
      if (sss == null) {
        throw new CompilerException("Error in Optimizer Post Pass: Solution set schema is null after first traversal of the step function.");
      }
     
      // make the second pass and instantiate the utilities
      traverse(iterationNode.getNextWorkSetPlanNode(), wss, createUtilities);
      traverse(iterationNode.getSolutionSetDeltaPlanNode(), sss, createUtilities);
     
      // add the types from the solution set schema to the iteration's own schema. since
      // the solution set input and the result must have the same schema, this acts as a sanity check.
      try {
        for (Map.Entry<Integer, X> entry : sss) {
          Integer pos = entry.getKey();
          schema.addType(pos, entry.getValue());
        }
      } catch (ConflictingFieldTypeInfoException e) {
        throw new CompilerPostPassException("Conflicting type information for field " + e.getFieldNumber()
          + " in node '" + iterationNode.getPactContract().getName() + "'. Contradicting types between the " +
          "result of the iteration and the solution set schema: " + e.getPreviousType() +
          " and " + e.getNewType() + ". Most probable cause: Invalid constant field annotations.");
      }
     
      // set the serializers and comparators
      if (createUtilities) {
        WorksetIterationNode optNode = iterationNode.getIterationNode();
        iterationNode.setWorksetSerializer(createSerializer(wss, iterationNode.getWorksetPlanNode()));
        iterationNode.setSolutionSetSerializer(createSerializer(sss, iterationNode.getSolutionSetPlanNode()));
        try {
          iterationNode.setSolutionSetComparator(createComparator(optNode.getSolutionSetKeyFields(), null, sss));
        } catch (MissingFieldTypeInfoException ex) {
          throw new CompilerPostPassException("Could not set up the solution set for workset iteration '" +
              optNode.getPactContract().getName() + "'. Missing type information for key field " + ex.getFieldNumber() + '.');
        }
      }
     
      // done, we can now propagate our info down
      try {
        propagateToChannel(schema, iterationNode.getInitialSolutionSetInput(), createUtilities);
        propagateToChannel(wss, iterationNode.getInitialWorksetInput(), createUtilities);
      } catch (MissingFieldTypeInfoException ex) {
        throw new CompilerPostPassException("Could not set up runtime strategy for input channel to node '"
          + iterationNode.getPactContract().getName() + "'. Missing type information for key field " +
          ex.getFieldNumber());
      }
    }
    else if (node instanceof SingleInputPlanNode) {
      SingleInputPlanNode sn = (SingleInputPlanNode) node;
View Full Code Here

Examples of org.apache.flink.compiler.plan.WorksetIterationPlanNode

      SinkPlanNode sink = op.getDataSinks().iterator().next();
      assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
      assertEquals(DEFAULT_PARALLELISM, sink.getDegreeOfParallelism());
     
      // check the iteration
      WorksetIterationPlanNode iteration = (WorksetIterationPlanNode) sink.getInput().getSource();
      assertEquals(DEFAULT_PARALLELISM, iteration.getDegreeOfParallelism());
     
      // check the solution set join and the delta
      PlanNode ssDelta = iteration.getSolutionSetDeltaPlanNode();
      assertTrue(ssDelta instanceof DualInputPlanNode); // this is only true if the update functions preserves the partitioning
     
      DualInputPlanNode ssJoin = (DualInputPlanNode) ssDelta;
      assertEquals(DEFAULT_PARALLELISM, ssJoin.getDegreeOfParallelism());
      assertEquals(ShipStrategyType.PARTITION_HASH, ssJoin.getInput1().getShipStrategy());
      assertEquals(new FieldList(0), ssJoin.getInput1().getShipStrategyKeys());
     
      // check the workset set join
      DualInputPlanNode edgeJoin = (DualInputPlanNode) ssJoin.getInput1().getSource();
      assertEquals(DEFAULT_PARALLELISM, edgeJoin.getDegreeOfParallelism());
      assertEquals(ShipStrategyType.PARTITION_HASH, edgeJoin.getInput1().getShipStrategy());
      assertEquals(ShipStrategyType.FORWARD, edgeJoin.getInput2().getShipStrategy());
      assertTrue(edgeJoin.getInput1().getTempMode().isCached());
     
      assertEquals(new FieldList(0), edgeJoin.getInput1().getShipStrategyKeys());
     
      // check that the initial partitioning is pushed out of the loop
      assertEquals(ShipStrategyType.PARTITION_HASH, iteration.getInput1().getShipStrategy());
      assertEquals(ShipStrategyType.PARTITION_HASH, iteration.getInput2().getShipStrategy());
      assertEquals(new FieldList(0), iteration.getInput1().getShipStrategyKeys());
      assertEquals(new FieldList(0), iteration.getInput2().getShipStrategyKeys());
    }
    catch (Exception e) {
      System.err.println(e.getMessage());
      e.printStackTrace();
      fail(e.getMessage());
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.