Package org.apache.flink.configuration

Examples of org.apache.flink.configuration.Configuration

77
78
79
80
81
82
83
84
85
86
87
88
  public static Collection<Object[]> getConfigurations() throws FileNotFoundException, IOException {

    LinkedList<Configuration> tConfigs = new LinkedList<Configuration>();

    for(int i=1; i <= NUM_PROGRAMS; i++) {
      Configuration config = new Configuration();
      config.setInteger("ProgramId", i);
      tConfigs.add(config);
    }
   
    return toParameterList(tConfigs);
  }
View Full Code Here
60
61
62
63
64
65
66
67
68
69
70
   */
  public static <T, F extends FileInputFormat<T>> F openInput(
      Class<F> inputFormatClass, String path, Configuration configuration)
    throws IOException
  {
    configuration = configuration == null ? new Configuration() : configuration;

    Path normalizedPath = normalizePath(new Path(path));
    final F inputFormat = ReflectionUtil.newInstance(inputFormatClass);

    inputFormat.setFilePath(normalizedPath);
View Full Code Here
126
127
128
129
130
131
132
133
134
135
136
   * @throws IOException
   *         if an I/O error occurred while accessing the file or initializing the InputFormat.
   */
  public static <T, IS extends InputSplit, F extends InputFormat<T, IS>> F openInput(
      Class<F> inputFormatClass, Configuration configuration) throws IOException {
    configuration = configuration == null ? new Configuration() : configuration;

    final F inputFormat = ReflectionUtil.newInstance(inputFormatClass);
    inputFormat.configure(configuration);
    final IS[] splits = inputFormat.createInputSplits(1);
    inputFormat.open(splits[0]);
View Full Code Here
159
160
161
162
163
164
165
166
167
168
169
  {
    final F outputFormat = ReflectionUtil.newInstance(outputFormatClass);
    outputFormat.setOutputFilePath(new Path(path));
    outputFormat.setWriteMode(WriteMode.OVERWRITE);
 
    configuration = configuration == null ? new Configuration() : configuration;
   
    outputFormat.configure(configuration);
    outputFormat.open(0, 1);
    return outputFormat;
  }
View Full Code Here
102
103
104
105
106
107
108
109
110
111
112
    {
      TaskConfig taskConfig = new TaskConfig(pointsInput.getConfiguration());
      taskConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
      taskConfig.setOutputSerializer(serializer);
     
      TaskConfig chainedMapper = new TaskConfig(new Configuration());
      chainedMapper.setDriverStrategy(DriverStrategy.COLLECTOR_MAP);
      chainedMapper.setStubWrapper(new UserCodeObjectWrapper<PointBuilder>(new PointBuilder()));
      chainedMapper.addOutputShipStrategy(ShipStrategyType.FORWARD);
      chainedMapper.setOutputSerializer(serializer);
     
View Full Code Here
124
125
126
127
128
129
130
131
132
133
134
    {
      TaskConfig taskConfig = new TaskConfig(modelsInput.getConfiguration());
      taskConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
      taskConfig.setOutputSerializer(serializer);

      TaskConfig chainedMapper = new TaskConfig(new Configuration());
      chainedMapper.setDriverStrategy(DriverStrategy.COLLECTOR_MAP);
      chainedMapper.setStubWrapper(new UserCodeObjectWrapper<PointBuilder>(new PointBuilder()));
      chainedMapper.addOutputShipStrategy(ShipStrategyType.FORWARD);
      chainedMapper.setOutputSerializer(serializer);
     
View Full Code Here
173
174
175
176
177
178
179
180
181
182
183
    // output into iteration. broadcasting the centers
    headConfig.setOutputSerializer(serializer);
    headConfig.addOutputShipStrategy(ShipStrategyType.BROADCAST);
   
    // final output
    TaskConfig headFinalOutConfig = new TaskConfig(new Configuration());
    headFinalOutConfig.setOutputSerializer(serializer);
    headFinalOutConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
    headConfig.setIterationHeadFinalOutputConfig(headFinalOutConfig);
   
    // the sync
View Full Code Here
84
85
86
87
88
89
90
91
92
93
94
95
  public static Collection<Object[]> getConfigurations() throws FileNotFoundException, IOException {

    LinkedList<Configuration> tConfigs = new LinkedList<Configuration>();

    for(int i=1; i <= NUM_PROGRAMS; i++) {
      Configuration config = new Configuration();
      config.setInteger("ProgramId", i);
      tConfigs.add(config);
    }
   
    return toParameterList(tConfigs);
  }
View Full Code Here
389
390
391
392
393
394
395
396
397
398
399
400
          final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

          DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env)
              .map(new IdentityMapper<Tuple3<Integer, Long, String>>()).setParallelism(4);

          Configuration cfg = new Configuration();
          cfg.setString(PactCompiler.HINT_SHIP_STRATEGY, PactCompiler.HINT_SHIP_STRATEGY_REPARTITION);
          DataSet<Tuple2<Integer, String>> reduceDs = ds.reduceGroup(new Tuple3AllGroupReduceWithCombine())
              .withParameters(cfg);

          reduceDs.writeAsCsv(resultPath);
          env.execute();
View Full Code Here
57
58
59
60
61
62
63
64
65
66
67
68
    }
    else if (operation instanceof CrossOperatorBase.CrossWithLarge) {
      allowBCsecond = false;
    }
   
    Configuration conf = operation.getParameters();
    String localStrategy = conf.getString(PactCompiler.HINT_LOCAL_STRATEGY, null);
 
    if (localStrategy != null) {
      final OperatorDescriptorDual fixedDriverStrat;
      if (PactCompiler.HINT_LOCAL_STRATEGY_NESTEDLOOP_BLOCKED_OUTER_FIRST.equals(localStrategy)) {
        fixedDriverStrat = new CrossBlockOuterFirstDescriptor(allowBCfirst, allowBCsecond);
View Full Code Here
TOP

Related Classes of org.apache.flink.configuration.Configuration

  • org.apache.flink.api.avro.AvroExternalJarProgramITCase
  • org.apache.flink.api.common.io.BinaryInputFormatTest
  • org.apache.flink.api.common.io.DelimitedInputFormatSamplingTest
  • org.apache.flink.api.common.io.DelimitedInputFormatTest
  • org.apache.flink.api.common.io.FileInputFormatTest
  • org.apache.flink.api.common.io.FileOutputFormatTest
  • org.apache.flink.api.common.io.FormatUtil
  • org.apache.flink.api.common.io.GenericCsvInputFormatTest
  • org.apache.flink.api.common.io.SequentialFormatTest
  • org.apache.flink.api.io.avro.AvroRecordInputFormatTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.