Examples of eu.stratosphere.configuration.Configuration

eu.stratosphere.configuration.Configuration
Lightweight configuration object which can store key/value pairs. Configuration objects can be extracted from or integrated into the {@link GlobalConfiguration} object. They canbe transported via Nephele's IPC system to distribute configuration data at runtime. This class is thread-safe.

    if (cl == null) {
      throw new IOException("Cannot find class loader for vertex " + getID());
    }


    // Re-instantiate the configuration object with the correct class loader and read the configuration
    this.configuration = new Configuration(cl);
    this.configuration.read(in);


    // Read number of forward edges
    final int numForwardEdges = in.readInt();

View Full Code Here

    }
    else if (operation instanceof CrossOperatorBase.CrossWithLarge) {
      allowBCsecond = false;
    }
    
    Configuration conf = operation.getParameters();
    String localStrategy = conf.getString(PactCompiler.HINT_LOCAL_STRATEGY, null);
  
    if (localStrategy != null) {
      final OperatorDescriptorDual fixedDriverStrat;
      if (PactCompiler.HINT_LOCAL_STRATEGY_NESTEDLOOP_BLOCKED_OUTER_FIRST.equals(localStrategy)) {
        fixedDriverStrat = new CrossBlockOuterFirstDescriptor(allowBCfirst, allowBCsecond);

View Full Code Here

    headConfig.setOutputSerializer(recSerializer);
    headConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
    headConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
    
    // final output
    TaskConfig headFinalOutConfig = new TaskConfig(new Configuration());
    headFinalOutConfig.setOutputSerializer(recSerializer);
    headFinalOutConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
    headConfig.setIterationHeadFinalOutputConfig(headFinalOutConfig);
    
    // the sync

View Full Code Here

  }


  @Override
  public void openTask() throws Exception {
    // open the stub first
    final Configuration stubConfig = this.config.getStubParameters();
    RegularPactTask.openUserCode(this.combiner, stubConfig);


    // ----------------- Set up the asynchronous sorter -------------------------


    this.memManager = this.parent.getEnvironment().getMemoryManager();

View Full Code Here

   */
  public PactCompiler(DataStatistics stats, CostEstimator estimator, InetSocketAddress jobManagerConnection) {
    this.statistics = stats;
    this.costEstimator = estimator;


    Configuration config = GlobalConfiguration.getConfiguration();


    // determine the maximum number of instances to use
    this.maxMachines = -1;


    // determine the default parallelization degree
    this.defaultDegreeOfParallelism = config.getInteger(ConfigConstants.DEFAULT_PARALLELIZATION_DEGREE_KEY,
      ConfigConstants.DEFAULT_PARALLELIZATION_DEGREE);


    // determine the default intra-node parallelism
    int maxInNodePar = config.getInteger(ConfigConstants.PARALLELIZATION_MAX_INTRA_NODE_DEGREE_KEY,
      ConfigConstants.DEFAULT_MAX_INTRA_NODE_PARALLELIZATION_DEGREE);
    if (maxInNodePar == 0 || maxInNodePar < -1) {
      LOG.error("Invalid maximum degree of intra-node parallelism: " + maxInNodePar +
        ". Ignoring parameter.");
      maxInNodePar = ConfigConstants.DEFAULT_MAX_INTRA_NODE_PARALLELIZATION_DEGREE;
    }
    this.maxIntraNodeParallelism = maxInNodePar;


    // assign the connection to the job-manager
    if (jobManagerConnection != null) {
      this.jobManagerAddress = jobManagerConnection;
    } else {
      final String address = config.getString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, null);
      if (address == null) {
        throw new CompilerException(
          "Cannot find address to job manager's RPC service in the global configuration.");
      }

View Full Code Here

              getEnvironment().getJobID(), ioe);
      }
    }
    
    // obtain task configuration (including stub parameters)
    Configuration taskConf = getTaskConfiguration();
    taskConf.setClassLoader(this.userCodeClassLoader);
    this.config = new TaskConfig(taskConf);
    
    initInputFormat(this.userCodeClassLoader);
    
    try {

View Full Code Here

      
      final GenericCombine<E> combineStub = CombiningUnilateralSortMerger.this.combineStub;
      
      // now that we are actually spilling, take the combiner, and open it
      try {
        Configuration conf = CombiningUnilateralSortMerger.this.udfConfig; 
        combineStub.open(conf == null ? new Configuration() : conf);
      }
      catch (Throwable t) {
        throw new IOException("The user-defined combiner failed in its 'open()' method.", t);
      }

View Full Code Here

  }
  
  @Override
  protected List<OperatorDescriptorSingle> getPossibleProperties() {
    // see if an internal hint dictates the strategy to use
    final Configuration conf = getPactContract().getParameters();
    final String localStrategy = conf.getString(PactCompiler.HINT_LOCAL_STRATEGY, null);


    final boolean useCombiner;
    if (localStrategy != null) {
      if (PactCompiler.HINT_LOCAL_STRATEGY_SORT.equals(localStrategy)) {
        useCombiner = false;

View Full Code Here




  @Override
  public void setInput(Map<Operator<?>, OptimizerNode> contractToNode) {
    // see if there is a hint that dictates which shipping strategy to use for BOTH inputs
    final Configuration conf = getPactContract().getParameters();
    ShipStrategyType preSet1 = null;
    ShipStrategyType preSet2 = null;
    
    String shipStrategy = conf.getString(PactCompiler.HINT_SHIP_STRATEGY, null);
    if (shipStrategy != null) {
      if (PactCompiler.HINT_SHIP_STRATEGY_FORWARD.equals(shipStrategy)) {
        preSet1 = preSet2 = ShipStrategyType.FORWARD;
      } else if (PactCompiler.HINT_SHIP_STRATEGY_BROADCAST.equals(shipStrategy)) {
        preSet1 = preSet2 = ShipStrategyType.BROADCAST;
      } else if (PactCompiler.HINT_SHIP_STRATEGY_REPARTITION_HASH.equals(shipStrategy)) {
        preSet1 = preSet2 = ShipStrategyType.PARTITION_HASH;
      } else if (PactCompiler.HINT_SHIP_STRATEGY_REPARTITION_RANGE.equals(shipStrategy)) {
        preSet1 = preSet2 = ShipStrategyType.PARTITION_RANGE;
      } else if (shipStrategy.equalsIgnoreCase(PactCompiler.HINT_SHIP_STRATEGY_REPARTITION)) {
        preSet1 = preSet2 = ShipStrategyType.PARTITION_RANDOM;
      } else {
        throw new CompilerException("Unknown hint for shipping strategy: " + shipStrategy);
      }
    }


    // see if there is a hint that dictates which shipping strategy to use for the FIRST input
    shipStrategy = conf.getString(PactCompiler.HINT_SHIP_STRATEGY_FIRST_INPUT, null);
    if (shipStrategy != null) {
      if (PactCompiler.HINT_SHIP_STRATEGY_FORWARD.equals(shipStrategy)) {
        preSet1 = ShipStrategyType.FORWARD;
      } else if (PactCompiler.HINT_SHIP_STRATEGY_BROADCAST.equals(shipStrategy)) {
        preSet1 = ShipStrategyType.BROADCAST;
      } else if (PactCompiler.HINT_SHIP_STRATEGY_REPARTITION_HASH.equals(shipStrategy)) {
        preSet1 = ShipStrategyType.PARTITION_HASH;
      } else if (PactCompiler.HINT_SHIP_STRATEGY_REPARTITION_RANGE.equals(shipStrategy)) {
        preSet1 = ShipStrategyType.PARTITION_RANGE;
      } else if (shipStrategy.equalsIgnoreCase(PactCompiler.HINT_SHIP_STRATEGY_REPARTITION)) {
        preSet1 = ShipStrategyType.PARTITION_RANDOM;
      } else {
        throw new CompilerException("Unknown hint for shipping strategy of input one: " + shipStrategy);
      }
    }


    // see if there is a hint that dictates which shipping strategy to use for the SECOND input
    shipStrategy = conf.getString(PactCompiler.HINT_SHIP_STRATEGY_SECOND_INPUT, null);
    if (shipStrategy != null) {
      if (PactCompiler.HINT_SHIP_STRATEGY_FORWARD.equals(shipStrategy)) {
        preSet2 = ShipStrategyType.FORWARD;
      } else if (PactCompiler.HINT_SHIP_STRATEGY_BROADCAST.equals(shipStrategy)) {
        preSet2 = ShipStrategyType.BROADCAST;

View Full Code Here

      InputFormat<?, ?> format = null;
      String inFormatDescription = "<unknown>";
      
      try {
        format = getPactContract().getFormatWrapper().getUserCodeObject();
        Configuration config = getPactContract().getParameters();
        config.setClassLoader(format.getClass().getClassLoader());
        format.configure(config);
      }
      catch (Throwable t) {
        if (PactCompiler.LOG.isWarnEnabled()) {
          PactCompiler.LOG.warn("Could not instantiate InputFormat to obtain statistics."

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of eu.stratosphere.configuration.Configuration

eu.stratosphere.api.common.io.DelimitedInputFormatSamplingTest

eu.stratosphere.api.common.io.DelimitedInputFormatTest

eu.stratosphere.api.common.io.FileInputFormatTest

eu.stratosphere.api.common.io.FileOutputFormatTest

eu.stratosphere.api.common.io.FormatUtil

eu.stratosphere.api.common.io.GenericCsvInputFormatTest

eu.stratosphere.api.common.io.SequentialFormatTest

eu.stratosphere.api.java.io.CsvInputFormatTest

eu.stratosphere.api.java.io.TextInputFormatTest

eu.stratosphere.api.java.operators.OperatorTranslation

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.