Examples of org.apache.flink.compiler.dag.WorksetIterationNode$SingleRootJoiner

org.apache.flink.compiler.dag.WorksetIterationNode
A node in the optimizer's program representation for a workset iteration.

          " and " + e.getNewType() + ". Most probable cause: Invalid constant field annotations.");
      }
      
      // set the serializers and comparators
      if (createUtilities) {
        WorksetIterationNode optNode = iterationNode.getIterationNode();
        iterationNode.setWorksetSerializer(createSerializer(wss, iterationNode.getWorksetPlanNode()));
        iterationNode.setSolutionSetSerializer(createSerializer(sss, iterationNode.getSolutionSetPlanNode()));
        try {
          iterationNode.setSolutionSetComparator(createComparator(optNode.getSolutionSetKeyFields(), null, sss));
        } catch (MissingFieldTypeInfoException ex) {
          throw new CompilerPostPassException("Could not set up the solution set for workset iteration '" + 
              optNode.getPactContract().getName() + "'. Missing type information for key field " + ex.getFieldNumber() + '.');
        }
      }
      
      // done, we can now propagate our info down
      try {
        propagateToChannel(schema, iterationNode.getInitialSolutionSetInput(), createUtilities);
        propagateToChannel(wss, iterationNode.getInitialWorksetInput(), createUtilities);
      } catch (MissingFieldTypeInfoException ex) {
        throw new CompilerPostPassException("Could not set up runtime strategy for input channel to node '"
          + iterationNode.getPactContract().getName() + "'. Missing type information for key field " + 
          ex.getFieldNumber());
      }
    }
    else if (node instanceof SingleInputPlanNode) {
      SingleInputPlanNode sn = (SingleInputPlanNode) node;
      
      // get the nodes current schema
      T schema;
      if (sn.postPassHelper == null) {
        schema = createEmptySchema();
        sn.postPassHelper = schema;
      } else {
        schema = (T) sn.postPassHelper;
      }
      schema.increaseNumConnectionsThatContributed();
      SingleInputNode optNode = sn.getSingleInputNode();
      
      // add the parent schema to the schema
      if (propagateParentSchemaDown) {
        addSchemaToSchema(parentSchema, schema, optNode, 0);
      }
      
      // check whether all outgoing channels have not yet contributed. come back later if not.
      if (schema.getNumConnectionsThatContributed() < sn.getOutgoingChannels().size()) {
        return;
      }
      
      // add the nodes local information
      try {
        getSingleInputNodeSchema(sn, schema);
      } catch (ConflictingFieldTypeInfoException e) {
        throw new CompilerPostPassException(getConflictingTypeErrorMessage(e, optNode.getPactContract().getName()));
      }
      
      if (createUtilities) {
        // parameterize the node's driver strategy
        for(int i=0;i<sn.getDriverStrategy().getNumRequiredComparators();i++) {
          try {
            sn.setComparator(createComparator(sn.getKeys(i), sn.getSortOrders(i), schema),i);
          } catch (MissingFieldTypeInfoException e) {
            throw new CompilerPostPassException("Could not set up runtime strategy for node '" + 
                optNode.getPactContract().getName() + "'. Missing type information for key field " +
                e.getFieldNumber());
          }
        }
      }
      
      // done, we can now propagate our info down
      try {
        propagateToChannel(schema, sn.getInput(), createUtilities);
      } catch (MissingFieldTypeInfoException e) {
        throw new CompilerPostPassException("Could not set up runtime strategy for input channel to node '" +
          optNode.getPactContract().getName() + "'. Missing type information for field " + e.getFieldNumber());
      }
      
      // don't forget the broadcast inputs
      for (Channel c: sn.getBroadcastInputs()) {
        try {
          propagateToChannel(createEmptySchema(), c, createUtilities);
        } catch (MissingFieldTypeInfoException e) {
          throw new CompilerPostPassException("Could not set up runtime strategy for broadcast channel in node '" +
            optNode.getPactContract().getName() + "'. Missing type information for field " + e.getFieldNumber());
        }
      }
    }
    else if (node instanceof DualInputPlanNode) {
      DualInputPlanNode dn = (DualInputPlanNode) node;
      
      // get the nodes current schema
      T schema1;
      T schema2;
      if (dn.postPassHelper1 == null) {
        schema1 = createEmptySchema();
        schema2 = createEmptySchema();
        dn.postPassHelper1 = schema1;
        dn.postPassHelper2 = schema2;
      } else {
        schema1 = (T) dn.postPassHelper1;
        schema2 = (T) dn.postPassHelper2;
      }


      schema1.increaseNumConnectionsThatContributed();
      schema2.increaseNumConnectionsThatContributed();
      TwoInputNode optNode = dn.getTwoInputNode();
      
      // add the parent schema to the schema
      if (propagateParentSchemaDown) {
        addSchemaToSchema(parentSchema, schema1, optNode, 0);
        addSchemaToSchema(parentSchema, schema2, optNode, 1);
      }
      
      // check whether all outgoing channels have not yet contributed. come back later if not.
      if (schema1.getNumConnectionsThatContributed() < dn.getOutgoingChannels().size()) {
        return;
      }
      
      // add the nodes local information
      try {
        getDualInputNodeSchema(dn, schema1, schema2);
      } catch (ConflictingFieldTypeInfoException e) {
        throw new CompilerPostPassException(getConflictingTypeErrorMessage(e, optNode.getPactContract().getName()));
      }
      
      // parameterize the node's driver strategy
      if (createUtilities) {
        if (dn.getDriverStrategy().getNumRequiredComparators() > 0) {
          // set the individual comparators
          try {
            dn.setComparator1(createComparator(dn.getKeysForInput1(), dn.getSortOrders(), schema1));
            dn.setComparator2(createComparator(dn.getKeysForInput2(), dn.getSortOrders(), schema2));
          } catch (MissingFieldTypeInfoException e) {
            throw new CompilerPostPassException("Could not set up runtime strategy for node '" + 
                optNode.getPactContract().getName() + "'. Missing type information for field " + e.getFieldNumber());
          }
          
          // set the pair comparator
          try {
            dn.setPairComparator(createPairComparator(dn.getKeysForInput1(), dn.getKeysForInput2(), 
              dn.getSortOrders(), schema1, schema2));
          } catch (MissingFieldTypeInfoException e) {
            throw new CompilerPostPassException("Could not set up runtime strategy for node '" + 
                optNode.getPactContract().getName() + "'. Missing type information for field " + e.getFieldNumber());
          }
          
        }
      }
      
      // done, we can now propagate our info down
      try {
        propagateToChannel(schema1, dn.getInput1(), createUtilities);
      } catch (MissingFieldTypeInfoException e) {
        throw new CompilerPostPassException("Could not set up runtime strategy for the first input channel to node '"
          + optNode.getPactContract().getName() + "'. Missing type information for field " + e.getFieldNumber());
      }
      try {
        propagateToChannel(schema2, dn.getInput2(), createUtilities);
      } catch (MissingFieldTypeInfoException e) {
        throw new CompilerPostPassException("Could not set up runtime strategy for the second input channel to node '"
          + optNode.getPactContract().getName() + "'. Missing type information for field " + e.getFieldNumber());
      }
      
      // don't forget the broadcast inputs
      for (Channel c: dn.getBroadcastInputs()) {
        try {
          propagateToChannel(createEmptySchema(), c, createUtilities);
        } catch (MissingFieldTypeInfoException e) {
          throw new CompilerPostPassException("Could not set up runtime strategy for broadcast channel in node '" +
            optNode.getPactContract().getName() + "'. Missing type information for field " + e.getFieldNumber());
        }
      }
    }
    else if (node instanceof NAryUnionPlanNode) {
      // only propagate the info down

View Full Code Here

      }
      else if (c instanceof BulkIterationBase) {
        n = new BulkIterationNode((BulkIterationBase<?>) c);
      }
      else if (c instanceof DeltaIterationBase) {
        n = new WorksetIterationNode((DeltaIterationBase<?, ?>) c);
      }
      else if (c instanceof Union){
        n = new BinaryUnionNode((Union<?>) c);
      }
      else if (c instanceof PartitionOperatorBase) {
        n = new PartitionNode((PartitionOperatorBase<?>) c);
      }
      else if (c instanceof PartialSolutionPlaceHolder) {
        if (this.parent == null) {
          throw new InvalidProgramException("It is currently not supported to create data sinks inside iterations.");
        }
        
        final PartialSolutionPlaceHolder<?> holder = (PartialSolutionPlaceHolder<?>) c;
        final BulkIterationBase<?> enclosingIteration = holder.getContainingBulkIteration();
        final BulkIterationNode containingIterationNode =
              (BulkIterationNode) this.parent.con2node.get(enclosingIteration);
        
        // catch this for the recursive translation of step functions
        BulkPartialSolutionNode p = new BulkPartialSolutionNode(holder, containingIterationNode);
        p.setDegreeOfParallelism(containingIterationNode.getDegreeOfParallelism());
        n = p;
      }
      else if (c instanceof WorksetPlaceHolder) {
        if (this.parent == null) {
          throw new InvalidProgramException("It is currently not supported to create data sinks inside iterations.");
        }
        
        final WorksetPlaceHolder<?> holder = (WorksetPlaceHolder<?>) c;
        final DeltaIterationBase<?, ?> enclosingIteration = holder.getContainingWorksetIteration();
        final WorksetIterationNode containingIterationNode =
              (WorksetIterationNode) this.parent.con2node.get(enclosingIteration);
        
        // catch this for the recursive translation of step functions
        WorksetNode p = new WorksetNode(holder, containingIterationNode);
        p.setDegreeOfParallelism(containingIterationNode.getDegreeOfParallelism());
        n = p;
      }
      else if (c instanceof SolutionSetPlaceHolder) {
        if (this.parent == null) {
          throw new InvalidProgramException("It is currently not supported to create data sinks inside iterations.");
        }
        
        final SolutionSetPlaceHolder<?> holder = (SolutionSetPlaceHolder<?>) c;
        final DeltaIterationBase<?, ?> enclosingIteration = holder.getContainingWorksetIteration();
        final WorksetIterationNode containingIterationNode =
              (WorksetIterationNode) this.parent.con2node.get(enclosingIteration);
        
        // catch this for the recursive translation of step functions
        SolutionSetNode p = new SolutionSetNode(holder, containingIterationNode);
        p.setDegreeOfParallelism(containingIterationNode.getDegreeOfParallelism());
        n = p;
      }
      else {
        throw new IllegalArgumentException("Unknown operator type: " + c);
      }

View Full Code Here

        if(terminationCriterion != null){
          terminationCriterion.accept(identifier);
        }
      }
      else if (n instanceof WorksetIterationNode) {
        final WorksetIterationNode iterNode = (WorksetIterationNode) n;
        final DeltaIterationBase<?, ?> iter = iterNode.getIterationContract();


        // we need to ensure that both the next-workset and the solution-set-delta depend on the workset. One check is for free
        // during the translation, we do the other check here as a pre-condition
        {
          WorksetFinder wsf = new WorksetFinder();
          iter.getNextWorkset().accept(wsf);
          if (!wsf.foundWorkset) {
            throw new CompilerException("In the given program, the next workset does not depend on the workset. This is a prerequisite in delta iterations.");
          }
        }
        
        // calculate the closure of the anonymous function
        HashMap<Operator<?>, OptimizerNode> closure = new HashMap<Operator<?>, OptimizerNode>(con2node);


        // first, recursively build the data flow for the step function
        final GraphCreatingVisitor recursiveCreator = new GraphCreatingVisitor(this, true, iterNode.getDegreeOfParallelism(), closure);
        
        // descend from the solution set delta. check that it depends on both the workset
        // and the solution set. If it does depend on both, this descend should create both nodes
        iter.getSolutionSetDelta().accept(recursiveCreator);
        
        final WorksetNode worksetNode = (WorksetNode) recursiveCreator.con2node.get(iter.getWorkset());
        
        if (worksetNode == null) {
          throw new CompilerException("In the given program, the solution set delta does not depend on the workset. This is a prerequisite in delta iterations.");
        }
        
        iter.getNextWorkset().accept(recursiveCreator);
        
        SolutionSetNode solutionSetNode = (SolutionSetNode) recursiveCreator.con2node.get(iter.getSolutionSet());
        
        if (solutionSetNode == null || solutionSetNode.getOutgoingConnections() == null || solutionSetNode.getOutgoingConnections().isEmpty()) {
          solutionSetNode = new SolutionSetNode((SolutionSetPlaceHolder<?>) iter.getSolutionSet(), iterNode);
        }
        else {
          for (PactConnection conn : solutionSetNode.getOutgoingConnections()) {
            OptimizerNode successor = conn.getTarget();
          
            if (successor.getClass() == MatchNode.class) {
              // find out which input to the match the solution set is
              MatchNode mn = (MatchNode) successor;
              if (mn.getFirstPredecessorNode() == solutionSetNode) {
                mn.makeJoinWithSolutionSet(0);
              } else if (mn.getSecondPredecessorNode() == solutionSetNode) {
                mn.makeJoinWithSolutionSet(1);
              } else {
                throw new CompilerException();
              }
            }
            else if (successor.getClass() == CoGroupNode.class) {
              CoGroupNode cg = (CoGroupNode) successor;
              if (cg.getFirstPredecessorNode() == solutionSetNode) {
                cg.makeCoGroupWithSolutionSet(0);
              } else if (cg.getSecondPredecessorNode() == solutionSetNode) {
                cg.makeCoGroupWithSolutionSet(1);
              } else {
                throw new CompilerException();
              }
            }
            else {
              throw new CompilerException("Error: The only operations allowed on the solution set are Join and CoGroup.");
            }
          }
        }
        
        final OptimizerNode nextWorksetNode = recursiveCreator.con2node.get(iter.getNextWorkset());
        final OptimizerNode solutionSetDeltaNode = recursiveCreator.con2node.get(iter.getSolutionSetDelta());
        
        // set the step function nodes to the iteration node
        iterNode.setPartialSolution(solutionSetNode, worksetNode);
        iterNode.setNextPartialSolution(solutionSetDeltaNode, nextWorksetNode);
        
        // go over the contained data flow and mark the dynamic path nodes
        StaticDynamicPathIdentifier pathIdentifier = new StaticDynamicPathIdentifier(iterNode.getCostWeight());
        nextWorksetNode.accept(pathIdentifier);
        iterNode.getSolutionSetDelta().accept(pathIdentifier);
      }
    }

View Full Code Here

TOP

Related Classes of org.apache.flink.compiler.dag.WorksetIterationNode$SingleRootJoiner

eu.stratosphere.types.NothingTypeInfo

org.apache.flink.api.common.operators.util.FieldList

org.apache.flink.api.common.typeinfo.NothingTypeInfo

org.apache.flink.compiler.dataproperties.GlobalProperties

org.apache.flink.compiler.dataproperties.InterestingProperties

org.apache.flink.compiler.dataproperties.LocalProperties

org.apache.flink.compiler.dataproperties.RequestedGlobalProperties

org.apache.flink.compiler.dataproperties.RequestedLocalProperties

org.apache.flink.compiler.operators.SolutionSetDeltaOperator

org.apache.flink.compiler.PactCompiler$GraphCreatingVisitor

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.