Package eu.stratosphere.compiler.plan

Examples of eu.stratosphere.compiler.plan.SingleInputPlanNode


     
      if (!validCombination) {
        continue;
      }
     
      final SingleInputPlanNode node = dps.instantiate(in, this);
      node.setBroadcastInputs(broadcastChannelsCombination);
     
      // compute how the strategy affects the properties
      GlobalProperties gProps = in.getGlobalProperties().clone();
      LocalProperties lProps = in.getLocalProperties().clone();
      gProps = dps.computeGlobalProperties(gProps);
      lProps = dps.computeLocalProperties(lProps);
     
      // filter by the user code field copies
      gProps = gProps.filterByNodesConstantSet(this, 0);
      lProps = lProps.filterByNodesConstantSet(this, 0);
     
      // apply
      node.initProperties(gProps, lProps);
      node.updatePropertiesWithUniqueSets(getUniqueFields());
      target.add(node);
    }
  }
View Full Code Here


        throw new CompilerException("Optimizer cannot compile an iteration step function where next partial solution is created by a Union node.");
      }
     
      // traverse the termination criterion for the first time. create schema only, no utilities. Needed in case of intermediate termination criterion
      if (iterationNode.getRootOfTerminationCriterion() != null) {
        SingleInputPlanNode addMapper = (SingleInputPlanNode) iterationNode.getRootOfTerminationCriterion();
        traverseChannel(addMapper.getInput());
      }

      BulkIterationBase<?> operator = (BulkIterationBase<?>) iterationNode.getPactContract();

      // set the serializer
      iterationNode.setSerializerForIterationChannel(createSerializer(operator.getOperatorInfo().getOutputType()));

      // done, we can now propagate our info down
      traverseChannel(iterationNode.getInput());
      traverse(iterationNode.getRootOfStepFunction());
    }
    else if (node instanceof WorksetIterationPlanNode) {
      WorksetIterationPlanNode iterationNode = (WorksetIterationPlanNode) node;
     
      if (iterationNode.getNextWorkSetPlanNode() instanceof NAryUnionPlanNode) {
        throw new CompilerException("Optimizer cannot compile a workset iteration step function where the next workset is produced by a Union node.");
      }
      if (iterationNode.getSolutionSetDeltaPlanNode() instanceof NAryUnionPlanNode) {
        throw new CompilerException("Optimizer cannot compile a workset iteration step function where the solution set delta is produced by a Union node.");
      }
     
      DeltaIterationBase<?, ?> operator = (DeltaIterationBase<?, ?>) iterationNode.getPactContract();
     
      // set the serializers and comparators for the workset iteration
      iterationNode.setSolutionSetSerializer(createSerializer(operator.getOperatorInfo().getFirstInputType()));
      iterationNode.setWorksetSerializer(createSerializer(operator.getOperatorInfo().getSecondInputType()));
      iterationNode.setSolutionSetComparator(createComparator(operator.getOperatorInfo().getFirstInputType(),
          iterationNode.getSolutionSetKeyFields(), getSortOrders(iterationNode.getSolutionSetKeyFields(), null)));
     
      // traverse the inputs
      traverseChannel(iterationNode.getInput1());
      traverseChannel(iterationNode.getInput2());
     
      // traverse the step function
      traverse(iterationNode.getSolutionSetDeltaPlanNode());
      traverse(iterationNode.getNextWorkSetPlanNode());
    }
    else if (node instanceof SingleInputPlanNode) {
      SingleInputPlanNode sn = (SingleInputPlanNode) node;
     
      if (!(sn.getOptimizerNode().getPactContract() instanceof SingleInputOperator)) {
       
        // Special case for delta iterations
        if(sn.getOptimizerNode().getPactContract() instanceof NoOpUnaryUdfOp) {
          traverseChannel(sn.getInput());
          return;
        } else {
          throw new RuntimeException("Wrong operator type found in post pass.");
        }
      }
     
      SingleInputOperator<?, ?, ?> singleInputOperator = (SingleInputOperator<?, ?, ?>) sn.getOptimizerNode().getPactContract();
     
      // parameterize the node's driver strategy
      if (sn.getDriverStrategy().requiresComparator()) {
        sn.setComparator(createComparator(singleInputOperator.getOperatorInfo().getInputType(), sn.getKeys(),
          getSortOrders(sn.getKeys(), sn.getSortOrders())));
      }
     
      // done, we can now propagate our info down
      traverseChannel(sn.getInput());
     
      // don't forget the broadcast inputs
      for (Channel c: sn.getBroadcastInputs()) {
        traverseChannel(c);
      }
    }
    else if (node instanceof DualInputPlanNode) {
      DualInputPlanNode dn = (DualInputPlanNode) node;
View Full Code Here

  @Override
  public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
    if (in.getShipStrategy() == ShipStrategyType.FORWARD) {
      // locally connected, directly instantiate
      return new SingleInputPlanNode(node, "GroupReduce ("+node.getPactContract().getName()+")", in, DriverStrategy.ALL_GROUP_REDUCE);
    } else {
      // non forward case.plug in a combiner
      Channel toCombiner = new Channel(in.getSource());
      toCombiner.setShipStrategy(ShipStrategyType.FORWARD);
     
      // create an input node for combine with same DOP as input node
      GroupReduceNode combinerNode = ((GroupReduceNode) node).getCombinerUtilityNode();
      combinerNode.setDegreeOfParallelism(in.getSource().getDegreeOfParallelism());
      combinerNode.setSubtasksPerInstance(in.getSource().getSubtasksPerInstance());
     
      SingleInputPlanNode combiner = new SingleInputPlanNode(combinerNode, "Combine ("+node.getPactContract().getName()+")", toCombiner, DriverStrategy.ALL_GROUP_COMBINE);
      combiner.setCosts(new Costs(0, 0));
      combiner.initProperties(toCombiner.getGlobalProperties(), toCombiner.getLocalProperties());
     
      Channel toReducer = new Channel(combiner);
      toReducer.setShipStrategy(in.getShipStrategy(), in.getShipStrategyKeys(), in.getShipStrategySortOrder());
      toReducer.setLocalStrategy(in.getLocalStrategy(), in.getLocalStrategyKeys(), in.getLocalStrategySortOrder());
      return new SingleInputPlanNode(node, "GroupReduce ("+node.getPactContract().getName()+")", toReducer, DriverStrategy.ALL_GROUP_REDUCE);
    }
  }
View Full Code Here

    return DriverStrategy.MAP;
  }

  @Override
  public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
    return new SingleInputPlanNode(node, "Map ("+node.getPactContract().getName()+")", in, DriverStrategy.MAP);
  }
View Full Code Here

  @Override
  public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
    if (in.getShipStrategy() == ShipStrategyType.FORWARD) {
      // locally connected, directly instantiate
      return new SingleInputPlanNode(node, "Reduce ("+node.getPactContract().getName()+")", in, DriverStrategy.ALL_REDUCE);
    } else {
      // non forward case.plug in a combiner
      Channel toCombiner = new Channel(in.getSource());
      toCombiner.setShipStrategy(ShipStrategyType.FORWARD);
     
      // create an input node for combine with same DOP as input node
      ReduceNode combinerNode = ((ReduceNode) node).getCombinerUtilityNode();
      combinerNode.setDegreeOfParallelism(in.getSource().getDegreeOfParallelism());
      combinerNode.setSubtasksPerInstance(in.getSource().getSubtasksPerInstance());
     
      SingleInputPlanNode combiner = new SingleInputPlanNode(combinerNode, "Combine ("+node.getPactContract().getName()+")", toCombiner, DriverStrategy.ALL_REDUCE);
      combiner.setCosts(new Costs(0, 0));
      combiner.initProperties(toCombiner.getGlobalProperties(), toCombiner.getLocalProperties());
     
      Channel toReducer = new Channel(combiner);
      toReducer.setShipStrategy(in.getShipStrategy(), in.getShipStrategyKeys(), in.getShipStrategySortOrder());
      toReducer.setLocalStrategy(in.getLocalStrategy(), in.getLocalStrategyKeys(), in.getLocalStrategySortOrder());
      return new SingleInputPlanNode(node, "Reduce("+node.getPactContract().getName()+")", toReducer, DriverStrategy.ALL_REDUCE);
    }
  }
View Full Code Here

        throw new CompilerException("Optimizer cannot compile an iteration step function where next partial solution is created by a Union node.");
      }
     
      // traverse the termination criterion for the first time. create schema only, no utilities. Needed in case of intermediate termination criterion
      if (iterationNode.getRootOfTerminationCriterion() != null) {
        SingleInputPlanNode addMapper = (SingleInputPlanNode) iterationNode.getRootOfTerminationCriterion();
        traverse(addMapper.getInput().getSource(), createEmptySchema(), false);
        try {
          addMapper.getInput().setSerializer(createSerializer(createEmptySchema()));
        } catch (MissingFieldTypeInfoException e) {
          throw new RuntimeException(e);
        }
      }
     
      // traverse the step function for the first time. create schema only, no utilities
      traverse(iterationNode.getRootOfStepFunction(), schema, false);
     
      T pss = (T) iterationNode.getPartialSolutionPlanNode().postPassHelper;
      if (pss == null) {
        throw new CompilerException("Error in Optimizer Post Pass: Partial solution schema is null after first traversal of the step function.");
      }
     
      // traverse the step function for the second time, taking the schema of the partial solution
      traverse(iterationNode.getRootOfStepFunction(), pss, createUtilities);
     
      if (iterationNode.getRootOfTerminationCriterion() != null) {
        SingleInputPlanNode addMapper = (SingleInputPlanNode) iterationNode.getRootOfTerminationCriterion();
        traverse(addMapper.getInput().getSource(), createEmptySchema(), createUtilities);
        try {
          addMapper.getInput().setSerializer(createSerializer(createEmptySchema()));
        } catch (MissingFieldTypeInfoException e) {
          throw new RuntimeException(e);
        }
      }
     
      // take the schema from the partial solution node and add its fields to the iteration result schema.
      // input and output schema need to be identical, so this is essentially a sanity check
      addSchemaToSchema(pss, schema, iterationNode.getPactContract().getName());
     
      // set the serializer
      if (createUtilities) {
        iterationNode.setSerializerForIterationChannel(createSerializer(pss, iterationNode.getPartialSolutionPlanNode()));
      }
     
      // done, we can now propagate our info down
      try {
        propagateToChannel(schema, iterationNode.getInput(), createUtilities);
      } catch (MissingFieldTypeInfoException e) {
        throw new CompilerPostPassException("Could not set up runtime strategy for input channel to node '"
          + iterationNode.getPactContract().getName() + "'. Missing type information for key field " +
          e.getFieldNumber());
      }
    }
    else if (node instanceof WorksetIterationPlanNode) {
      WorksetIterationPlanNode iterationNode = (WorksetIterationPlanNode) node;
     
      // get the nodes current schema
      T schema;
      if (iterationNode.postPassHelper == null) {
        schema = createEmptySchema();
        iterationNode.postPassHelper = schema;
      } else {
        schema = (T) iterationNode.postPassHelper;
      }
      schema.increaseNumConnectionsThatContributed();
     
      // add the parent schema to the schema (which refers to the solution set schema)
      if (propagateParentSchemaDown) {
        addSchemaToSchema(parentSchema, schema, iterationNode.getPactContract().getName());
      }
     
      // check whether all outgoing channels have not yet contributed. come back later if not.
      if (schema.getNumConnectionsThatContributed() < iterationNode.getOutgoingChannels().size()) {
        return;
      }
      if (iterationNode.getNextWorkSetPlanNode() instanceof NAryUnionPlanNode) {
        throw new CompilerException("Optimizer cannot compile a workset iteration step function where the next workset is produced by a Union node.");
      }
      if (iterationNode.getSolutionSetDeltaPlanNode() instanceof NAryUnionPlanNode) {
        throw new CompilerException("Optimizer cannot compile a workset iteration step function where the solution set delta is produced by a Union node.");
      }
     
      // traverse the step function
      // pass an empty schema to the next workset and the parent schema to the solution set delta
      // these first traversals are schema only
      traverse(iterationNode.getNextWorkSetPlanNode(), createEmptySchema(), false);
      traverse(iterationNode.getSolutionSetDeltaPlanNode(), schema, false);
     
      T wss = (T) iterationNode.getWorksetPlanNode().postPassHelper;
      T sss = (T) iterationNode.getSolutionSetPlanNode().postPassHelper;
     
      if (wss == null) {
        throw new CompilerException("Error in Optimizer Post Pass: Workset schema is null after first traversal of the step function.");
      }
      if (sss == null) {
        throw new CompilerException("Error in Optimizer Post Pass: Solution set schema is null after first traversal of the step function.");
      }
     
      // make the second pass and instantiate the utilities
      traverse(iterationNode.getNextWorkSetPlanNode(), wss, createUtilities);
      traverse(iterationNode.getSolutionSetDeltaPlanNode(), sss, createUtilities);
     
      // add the types from the solution set schema to the iteration's own schema. since
      // the solution set input and the result must have the same schema, this acts as a sanity check.
      try {
        for (Map.Entry<Integer, X> entry : sss) {
          Integer pos = entry.getKey();
          schema.addType(pos, entry.getValue());
        }
      } catch (ConflictingFieldTypeInfoException e) {
        throw new CompilerPostPassException("Conflicting type information for field " + e.getFieldNumber()
          + " in node '" + iterationNode.getPactContract().getName() + "'. Contradicting types between the " +
          "result of the iteration and the solution set schema: " + e.getPreviousType() +
          " and " + e.getNewType() + ". Most probable cause: Invalid constant field annotations.");
      }
     
      // set the serializers and comparators
      if (createUtilities) {
        WorksetIterationNode optNode = iterationNode.getIterationNode();
        iterationNode.setWorksetSerializer(createSerializer(wss, iterationNode.getWorksetPlanNode()));
        iterationNode.setSolutionSetSerializer(createSerializer(sss, iterationNode.getSolutionSetPlanNode()));
        try {
          iterationNode.setSolutionSetComparator(createComparator(optNode.getSolutionSetKeyFields(), null, sss));
        } catch (MissingFieldTypeInfoException ex) {
          throw new CompilerPostPassException("Could not set up the solution set for workset iteration '" +
              optNode.getPactContract().getName() + "'. Missing type information for key field " + ex.getFieldNumber() + '.');
        }
      }
     
      // done, we can now propagate our info down
      try {
        propagateToChannel(schema, iterationNode.getInitialSolutionSetInput(), createUtilities);
        propagateToChannel(wss, iterationNode.getInitialWorksetInput(), createUtilities);
      } catch (MissingFieldTypeInfoException ex) {
        throw new CompilerPostPassException("Could not set up runtime strategy for input channel to node '"
          + iterationNode.getPactContract().getName() + "'. Missing type information for key field " +
          ex.getFieldNumber());
      }
    }
    else if (node instanceof SingleInputPlanNode) {
      SingleInputPlanNode sn = (SingleInputPlanNode) node;
     
      // get the nodes current schema
      T schema;
      if (sn.postPassHelper == null) {
        schema = createEmptySchema();
        sn.postPassHelper = schema;
      } else {
        schema = (T) sn.postPassHelper;
      }
      schema.increaseNumConnectionsThatContributed();
      SingleInputNode optNode = sn.getSingleInputNode();
     
      // add the parent schema to the schema
      if (propagateParentSchemaDown) {
        addSchemaToSchema(parentSchema, schema, optNode, 0);
      }
     
      // check whether all outgoing channels have not yet contributed. come back later if not.
      if (schema.getNumConnectionsThatContributed() < sn.getOutgoingChannels().size()) {
        return;
      }
     
      // add the nodes local information
      try {
        getSingleInputNodeSchema(sn, schema);
      } catch (ConflictingFieldTypeInfoException e) {
        throw new CompilerPostPassException(getConflictingTypeErrorMessage(e, optNode.getPactContract().getName()));
      }
     
      if (createUtilities) {
        // parameterize the node's driver strategy
        if (sn.getDriverStrategy().requiresComparator()) {
          try {
            sn.setComparator(createComparator(sn.getKeys(), sn.getSortOrders(), schema));
          } catch (MissingFieldTypeInfoException e) {
            throw new CompilerPostPassException("Could not set up runtime strategy for node '" +
                optNode.getPactContract().getName() + "'. Missing type information for key field " +
                e.getFieldNumber());
          }
        }
      }
     
      // done, we can now propagate our info down
      try {
        propagateToChannel(schema, sn.getInput(), createUtilities);
      } catch (MissingFieldTypeInfoException e) {
        throw new CompilerPostPassException("Could not set up runtime strategy for input channel to node '" +
          optNode.getPactContract().getName() + "'. Missing type information for field " + e.getFieldNumber());
      }
     
      // don't forget the broadcast inputs
      for (Channel c: sn.getBroadcastInputs()) {
        try {
          propagateToChannel(createEmptySchema(), c, createUtilities);
        } catch (MissingFieldTypeInfoException e) {
          throw new CompilerPostPassException("Could not set up runtime strategy for broadcast channel in node '" +
            optNode.getPactContract().getName() + "'. Missing type information for field " + e.getFieldNumber());
View Full Code Here

    return DriverStrategy.SORTED_GROUP_REDUCE;
  }

  @Override
  public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
    return new SingleInputPlanNode(node, "Reduce("+node.getPactContract().getName()+")", in, DriverStrategy.SORTED_GROUP_REDUCE, this.keyList);
  }
View Full Code Here

      return;
    }
   
    // sanity check the solution set delta and cancel out the delta node, if it is not needed
    for (Iterator<PlanNode> deltaPlans = solutionSetDeltaCandidates.iterator(); deltaPlans.hasNext(); ) {
      SingleInputPlanNode candidate = (SingleInputPlanNode) deltaPlans.next();
      GlobalProperties gp = candidate.getGlobalProperties();
     
      if (gp.getPartitioning() != PartitioningProperty.HASH_PARTITIONED || gp.getPartitioningFields() == null ||
          !gp.getPartitioningFields().equals(this.solutionSetKeyFields))
      {
        throw new CompilerException("Bug: The solution set delta is not partitioned.");
      }
    }
   
    // 5) Create a candidate for the Iteration Node for every remaining plan of the step function.
   
    final GlobalProperties gp = new GlobalProperties();
    gp.setHashPartitioned(this.solutionSetKeyFields);
    gp.addUniqueFieldCombination(this.solutionSetKeyFields);
   
    final LocalProperties lp = new LocalProperties();
    lp.addUniqueFields(this.solutionSetKeyFields);
   
    // take all combinations of solution set delta and workset plans
    for (PlanNode solutionSetCandidate : solutionSetDeltaCandidates) {
      for (PlanNode worksetCandidate : worksetCandidates) {
        // check whether they have the same operator at their latest branching point
        if (this.singleRoot.areBranchCompatible(solutionSetCandidate, worksetCandidate)) {
         
          SingleInputPlanNode siSolutionDeltaCandidate = (SingleInputPlanNode) solutionSetCandidate;
          boolean immediateDeltaUpdate;
         
          // check whether we need a dedicated solution set delta operator, or whether we can update on the fly
          if (siSolutionDeltaCandidate.getInput().getShipStrategy() == ShipStrategyType.FORWARD && this.solutionDeltaImmediatelyAfterSolutionJoin) {
            // we do not need this extra node. we can make the predecessor the delta
            // sanity check the node and connection
            if (siSolutionDeltaCandidate.getDriverStrategy() != DriverStrategy.UNARY_NO_OP || siSolutionDeltaCandidate.getInput().getLocalStrategy() != LocalStrategy.NONE) {
              throw new CompilerException("Invalid Solution set delta node.");
            }
           
            solutionSetCandidate = siSolutionDeltaCandidate.getInput().getSource();
            immediateDeltaUpdate = true;
          } else {
            // was not partitioned, we need to keep this node.
            // mark that we materialize the input
            siSolutionDeltaCandidate.getInput().setTempMode(TempMode.PIPELINE_BREAKER);
            immediateDeltaUpdate = false;
          }
         
          WorksetIterationPlanNode wsNode = new WorksetIterationPlanNode(
            this, "WorksetIteration ("+this.getPactContract().getName()+")", solutionSetIn, worksetIn, sspn, wspn, worksetCandidate, solutionSetCandidate);
View Full Code Here

    // create in input node for combine with same DOP as input node
    GroupReduceNode combinerNode = new GroupReduceNode((GroupReduceOperatorBase<?, ?, ?>) node.getPactContract());
    combinerNode.setDegreeOfParallelism(in.getSource().getDegreeOfParallelism());
    combinerNode.setSubtasksPerInstance(in.getSource().getSubtasksPerInstance());
   
    return new SingleInputPlanNode(combinerNode, "Combine("+node.getPactContract().getName()+")", in, DriverStrategy.SORTED_GROUP_COMBINE, this.keyList);
  }
View Full Code Here

  }

  @Override
  public void postVisit(PlanNode visitable) {
    if(visitable instanceof SingleInputPlanNode) {
      SingleInputPlanNode n = (SingleInputPlanNode) visitable;
     
      if(n.getDriverStrategy().firstDam().equals(DamBehavior.FULL_DAM) || n.getInput().getLocalStrategy().dams() || n.getInput().getTempMode().breaksPipeline()) {
        g.addEdge(n, n.getPredecessor());
      }
      else {
        g.addEdge(n.getPredecessor(), n);
      }
    }
    else if(visitable instanceof DualInputPlanNode) {
      DualInputPlanNode n = (DualInputPlanNode) visitable;
     
      if(n.getDriverStrategy().firstDam().equals(DamBehavior.FULL_DAM) || n.getInput1().getLocalStrategy().dams() || n.getInput1().getTempMode().breaksPipeline()) {
        g.addEdge(n, n.getInput1().getSource());
      }
      else {
        g.addEdge(n.getInput1().getSource(), n);
      }
     
      if(!n.getDriverStrategy().equals(DriverStrategy.NONE) && (n.getDriverStrategy().secondDam().equals(DamBehavior.FULL_DAM) || n.getInput2().getLocalStrategy().dams() || n.getInput2().getTempMode().breaksPipeline())) {
        g.addEdge(n, n.getInput2().getSource());
      }
      else {
        g.addEdge(n.getInput2().getSource(), n);
      }
    }
   
   
    // recursively fix iterations
View Full Code Here

TOP

Related Classes of eu.stratosphere.compiler.plan.SingleInputPlanNode

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.