Package eu.stratosphere.compiler.dag

Source Code of eu.stratosphere.compiler.dag.DataSinkNode

/***********************************************************************************************************************
* Copyright (C) 2010-2013 by the Stratosphere project (http://stratosphere.eu)
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
*     http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
**********************************************************************************************************************/

package eu.stratosphere.compiler.dag;

import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;

import eu.stratosphere.api.common.distributions.DataDistribution;
import eu.stratosphere.api.common.operators.base.GenericDataSinkBase;
import eu.stratosphere.api.common.operators.Operator;
import eu.stratosphere.api.common.operators.Ordering;
import eu.stratosphere.compiler.CompilerException;
import eu.stratosphere.compiler.DataStatistics;
import eu.stratosphere.compiler.costs.CostEstimator;
import eu.stratosphere.compiler.dataproperties.InterestingProperties;
import eu.stratosphere.compiler.dataproperties.RequestedGlobalProperties;
import eu.stratosphere.compiler.dataproperties.RequestedLocalProperties;
import eu.stratosphere.compiler.plan.Channel;
import eu.stratosphere.compiler.plan.PlanNode;
import eu.stratosphere.compiler.plan.SinkPlanNode;
import eu.stratosphere.pact.runtime.task.util.LocalStrategy;
import eu.stratosphere.util.Visitor;

/**
* The Optimizer representation of a data sink.
*/
public class DataSinkNode extends OptimizerNode {
 
  protected PactConnection input;      // The input edge
 
  /**
   * Creates a new DataSinkNode for the given sink operator.
   *
   * @param sink The data sink contract object.
   */
  public DataSinkNode(GenericDataSinkBase<?> sink) {
    super(sink);
  }

  // --------------------------------------------------------------------------------------
 
  /**
   * Gets the input of the sink.
   *
   * @return The input connection.
   */
  public PactConnection getInputConnection() {
    return this.input;
  }
 
  /**
   *
   */
  public OptimizerNode getPredecessorNode() {
    if(this.input != null) {
      return input.getSource();
    } else {
      return null;
    }
  }

  /**
   * Gets the contract object for this data source node.
   *
   * @return The contract.
   */
  @Override
  public GenericDataSinkBase<?> getPactContract() {
    return (GenericDataSinkBase<?>) super.getPactContract();
  }

  @Override
  public String getName() {
    return "Data Sink";
  }

  @Override
  public boolean isMemoryConsumer() {
    return getPactContract().getPartitionOrdering() != null || getPactContract().getLocalOrder() != null;
  }

  @Override
  public List<PactConnection> getIncomingConnections() {
    return Collections.singletonList(this.input);
  }

  public List<PactConnection> getOutgoingConnections() {
    return Collections.emptyList();
  }

  @Override
  public void setInput(Map<Operator<?>, OptimizerNode> contractToNode) {
    Operator<?> children = getPactContract().getInput();

    final OptimizerNode pred;
    final PactConnection conn;
   
    pred = contractToNode.get(children);
    conn = new PactConnection(pred, this);
     
    // create the connection and add it
    this.input = conn;
    pred.addOutgoingConnection(conn);
  }

  /**
   * Computes the estimated outputs for the data sink. Since the sink does not modify anything, it simply
   * copies the output estimates from its direct predecessor.
   */
  @Override
  protected void computeOperatorSpecificDefaultEstimates(DataStatistics statistics) {
    this.estimatedNumRecords = getPredecessorNode().getEstimatedNumRecords();
    this.estimatedOutputSize = getPredecessorNode().getEstimatedOutputSize();
  }

  @Override
  public void computeInterestingPropertiesForInputs(CostEstimator estimator) {
    final InterestingProperties iProps = new InterestingProperties();
   
    {
      final Ordering partitioning = getPactContract().getPartitionOrdering();
      final DataDistribution dataDist = getPactContract().getDataDistribution();
      final RequestedGlobalProperties partitioningProps = new RequestedGlobalProperties();
      if (partitioning != null) {
        if(dataDist != null) {
          partitioningProps.setRangePartitioned(partitioning, dataDist);
        } else {
          partitioningProps.setRangePartitioned(partitioning);
        }
        iProps.addGlobalProperties(partitioningProps);
      }
      iProps.addGlobalProperties(partitioningProps);
    }
   
    {
      final Ordering localOrder = getPactContract().getLocalOrder();
      final RequestedLocalProperties orderProps = new RequestedLocalProperties();
      if (localOrder != null) {
        orderProps.setOrdering(localOrder);
      }
      iProps.addLocalProperties(orderProps);
    }
   
    this.input.setInterestingProperties(iProps);
  }
 
  // --------------------------------------------------------------------------------------------
  //                                     Branch Handling
  // --------------------------------------------------------------------------------------------

  @Override
  public void computeUnclosedBranchStack() {
    if (this.openBranches != null) {
      return;
    }

    // we need to track open branches even in the sinks, because they get "closed" when
    // we build a single "roor" for the data flow plan
    addClosedBranches(getPredecessorNode().closedBranchingNodes);
    this.openBranches = getPredecessorNode().getBranchesForParent(this.input);
  }
 
  @Override
  protected List<UnclosedBranchDescriptor> getBranchesForParent(PactConnection parent) {
    // return our own stack of open branches, because nothing is added
    return this.openBranches;
  }

  // --------------------------------------------------------------------------------------------
  //                                   Recursive Optimization
  // --------------------------------------------------------------------------------------------
 
  @Override
  public List<PlanNode> getAlternativePlans(CostEstimator estimator) {
    // check if we have a cached version
    if (this.cachedPlans != null) {
      return this.cachedPlans;
    }
   
    // calculate alternative sub-plans for predecessor
    List<? extends PlanNode> subPlans = getPredecessorNode().getAlternativePlans(estimator);
    List<PlanNode> outputPlans = new ArrayList<PlanNode>();
   
    final int dop = getDegreeOfParallelism();
    final int subPerInstance = getSubtasksPerInstance();
    final int inDop = getPredecessorNode().getDegreeOfParallelism();
    final int inSubPerInstance = getPredecessorNode().getSubtasksPerInstance();
    final int numInstances = dop / subPerInstance + (dop % subPerInstance == 0 ? 0 : 1);
    final int inNumInstances = inDop / inSubPerInstance + (inDop % inSubPerInstance == 0 ? 0 : 1);
   
    final boolean globalDopChange = numInstances != inNumInstances;
    final boolean localDopChange = numInstances == inNumInstances & subPerInstance != inSubPerInstance;
   
    InterestingProperties ips = this.input.getInterestingProperties();
    for (PlanNode p : subPlans) {
      for (RequestedGlobalProperties gp : ips.getGlobalProperties()) {
        for (RequestedLocalProperties lp : ips.getLocalProperties()) {
          Channel c = new Channel(p);
          gp.parameterizeChannel(c, globalDopChange, localDopChange);

          if (lp.isMetBy(c.getLocalPropertiesAfterShippingOnly())) {
            c.setLocalStrategy(LocalStrategy.NONE);
          } else {
            lp.parameterizeChannel(c);
          }
         
          // no need to check whether the created properties meet what we need in case
          // of ordering or global ordering, because the only interesting properties we have
          // are what we require
          outputPlans.add(new SinkPlanNode(this, "DataSink("+this.getPactContract().getName()+")" ,c));
        }
      }
    }
   
    // cost and prune the plans
    for (PlanNode node : outputPlans) {
      estimator.costOperator(node);
    }
    prunePlanAlternatives(outputPlans);

    this.cachedPlans = outputPlans;
    return outputPlans;
  }
 
  // --------------------------------------------------------------------------------------------
  //                                   Function Annotation Handling
  // --------------------------------------------------------------------------------------------
 
  public boolean isFieldConstant(int input, int fieldNumber) {
    return false;
  }
   
  // --------------------------------------------------------------------------------------------
  //                                     Miscellaneous
  // --------------------------------------------------------------------------------------------
 
  @Override
  public void accept(Visitor<OptimizerNode> visitor) {
    if (visitor.preVisit(this)) {
      if (getPredecessorNode() != null) {
        getPredecessorNode().accept(visitor);
      } else {
        throw new CompilerException();
      }
      visitor.postVisit(this);
    }
  }
}
TOP

Related Classes of eu.stratosphere.compiler.dag.DataSinkNode

TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.