Examples of org.apache.flink.api.java.ExecutionEnvironment

org.apache.flink.api.java.ExecutionEnvironment
The ExecutionEnviroment is the context in which a program is executed. A {@link LocalEnvironment} will cause execution in the current JVM, a{@link RemoteEnvironment} will cause execution on a remote setup.
The environment provides methods to control the job execution (such as setting the parallelism) and to interact with the outside world (data access).
Please note that the execution environment needs strong type information for the input and return types of all operations that are executed. This means that the environments needs to know that the return value of an operation is for example a Tuple of String and Integer. Because the Java compiler throws much of the generic type information away, most methods attempt to re- obtain that information using reflection. In certain cases, it may be necessary to manually supply that information to some of the methods. @see LocalEnvironment @see RemoteEnvironment

  }
  
  @Test
  public void testAllReduceWithCombiner() {
    try {
      ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
      env.setDegreeOfParallelism(8);
      
      DataSet<Long> data = env.generateSequence(1, 8000000).name("source");
      
      GroupReduceOperator<Long, Long> reduced = data.reduceGroup(new RichGroupReduceFunction<Long, Long>() {
        public void reduce(Iterable<Long> values, Collector<Long> out) {}
      }).name("reducer");
      
      reduced.setCombinable(true);
      reduced.print().name("sink");
      
      Plan p = env.createProgramPlan();
      OptimizedPlan op = compileNoStats(p);
      
      OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(op);
      
      // get the original nodes

View Full Code Here

  
  
  @Test
  public void testGroupedReduceWithFieldPositionKeyNonCombinable() {
    try {
      ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
      env.setDegreeOfParallelism(8);
      
      DataSet<Tuple2<String, Double>> data = env.readCsvFile("file:///will/never/be/read").types(String.class, Double.class)
        .name("source").setParallelism(6);
      
      data
        .groupBy(1)
        .reduceGroup(new RichGroupReduceFunction<Tuple2<String, Double>, Tuple2<String, Double>>() {
        public void reduce(Iterable<Tuple2<String, Double>> values, Collector<Tuple2<String, Double>> out) {}
      }).name("reducer")
      .print().name("sink");
      
      Plan p = env.createProgramPlan();
      OptimizedPlan op = compileNoStats(p);
      
      OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(op);
      
      // get the original nodes

View Full Code Here

  }
  
  @Test
  public void testGroupedReduceWithFieldPositionKeyCombinable() {
    try {
      ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
      env.setDegreeOfParallelism(8);
      
      DataSet<Tuple2<String, Double>> data = env.readCsvFile("file:///will/never/be/read").types(String.class, Double.class)
        .name("source").setParallelism(6);
      
      GroupReduceOperator<Tuple2<String, Double>, Tuple2<String, Double>> reduced = data
          .groupBy(1)
          .reduceGroup(new RichGroupReduceFunction<Tuple2<String, Double>, Tuple2<String, Double>>() {
        public void reduce(Iterable<Tuple2<String, Double>> values, Collector<Tuple2<String, Double>> out) {}
      }).name("reducer");
      
      reduced.setCombinable(true);
      reduced.print().name("sink");
      
      Plan p = env.createProgramPlan();
      OptimizedPlan op = compileNoStats(p);
      
      OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(op);
      
      // get the original nodes

View Full Code Here

  }
  
  @Test
  public void testGroupedReduceWithSelectorFunctionKeyNoncombinable() {
    try {
      ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
      env.setDegreeOfParallelism(8);
      
      DataSet<Tuple2<String, Double>> data = env.readCsvFile("file:///will/never/be/read").types(String.class, Double.class)
        .name("source").setParallelism(6);
      
      data
        .groupBy(new KeySelector<Tuple2<String,Double>, String>() { 
          public String getKey(Tuple2<String, Double> value) { return value.f0; }
        })
        .reduceGroup(new RichGroupReduceFunction<Tuple2<String, Double>, Tuple2<String, Double>>() {
        public void reduce(Iterable<Tuple2<String, Double>> values, Collector<Tuple2<String, Double>> out) {}
      }).name("reducer")
      .print().name("sink");
      
      Plan p = env.createProgramPlan();
      OptimizedPlan op = compileNoStats(p);
      
      OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(op);
      
      // get the original nodes

View Full Code Here

  }
  
  @Test
  public void testGroupedReduceWithSelectorFunctionKeyCombinable() {
    try {
      ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
      env.setDegreeOfParallelism(8);
      
      DataSet<Tuple2<String, Double>> data = env.readCsvFile("file:///will/never/be/read").types(String.class, Double.class)
        .name("source").setParallelism(6);
      
      GroupReduceOperator<Tuple2<String, Double>, Tuple2<String, Double>> reduced = data
        .groupBy(new KeySelector<Tuple2<String,Double>, String>() { 
          public String getKey(Tuple2<String, Double> value) { return value.f0; }
        })
        .reduceGroup(new RichGroupReduceFunction<Tuple2<String, Double>, Tuple2<String, Double>>() {
        public void reduce(Iterable<Tuple2<String, Double>> values, Collector<Tuple2<String, Double>> out) {}
      }).name("reducer");
      
      reduced.setCombinable(true);
      reduced.print().name("sink");
      
      Plan p = env.createProgramPlan();
      OptimizedPlan op = compileNoStats(p);
      
      OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(op);
      
      // get the original nodes

View Full Code Here

public class IterationsCompilerTest extends CompilerTestBase {


  @Test
  public void testSolutionSetDeltaDependsOnBroadcastVariable() {
    try {
      ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
      
      DataSet<Tuple2<Long, Long>> source =
            env.generateSequence(1, 1000).map(new DuplicateValueScalar<Long>());
      
      DataSet<Tuple2<Long, Long>> invariantInput =
          env.generateSequence(1, 1000).map(new DuplicateValueScalar<Long>());
      
      // iteration from here
      DeltaIteration<Tuple2<Long, Long>, Tuple2<Long, Long>> iter = source.iterateDelta(source, 1000, 1);
      
      DataSet<Tuple2<Long, Long>> result =
        invariantInput
          .map(new IdentityMapper<Tuple2<Long, Long>>()).withBroadcastSet(iter.getWorkset(), "bc data")
          .join(iter.getSolutionSet()).where(0).equalTo(1).projectFirst(1).projectSecond(1).types(Long.class, Long.class);
      
      iter.closeWith(result.map(new IdentityMapper<Tuple2<Long,Long>>()), result).print();
      
      OptimizedPlan p = compileNoStats(env.createProgramPlan());
      
      new PlanJSONDumpGenerator().getOptimizerPlanAsJSON(p);
    }
    catch (Exception e) {
      e.printStackTrace();

View Full Code Here

  private final String SUFFIX = "-suffixed";
  
  @Test
  public void testUnaryOp() {
    try {
      ExecutionEnvironment env = new CollectionEnvironment();
      
      DataSet<String> bcData = env.fromElements(SUFFIX);
      
      List<String> result = new ArrayList<String>();
      
      env.fromElements(TEST_DATA)
          .map(new SuffixAppender()).withBroadcastSet(bcData, BC_VAR_NAME)
          .output(new LocalCollectionOutputFormat<String>(result));
      
      env.execute();
      
      assertEquals(TEST_DATA.length, result.size());
      for (String s : result) {
        assertTrue(s.indexOf(SUFFIX) > 0);
      }

View Full Code Here

  }
  
  @Test
  public void testBinaryOp() {
    try {
      ExecutionEnvironment env = new CollectionEnvironment();
      
      DataSet<String> bcData = env.fromElements(SUFFIX);
      DataSet<String> inData = env.fromElements(TEST_DATA);
      
      List<String> result = new ArrayList<String>();
      
      inData.cross(inData).with(new SuffixCross()).withBroadcastSet(bcData, BC_VAR_NAME)
          .output(new LocalCollectionOutputFormat<String>(result));
      
      env.execute();
      
      assertEquals(TEST_DATA.length * TEST_DATA.length, result.size());
      for (String s : result) {
        assertTrue(s.indexOf(SUFFIX) == 2);
      }

View Full Code Here

  @Test
  public void testAccumulator() {
    try {
      final int NUM_ELEMENTS = 100;
      
      ExecutionEnvironment env = new CollectionEnvironment();
      
      env.generateSequence(1, NUM_ELEMENTS)
        .map(new CountingMapper())
        .output(new DiscardingOuputFormat<Long>());
      
      JobExecutionResult result = env.execute();
      
      assertTrue(result.getNetRuntime() >= 0);
      
      assertEquals(NUM_ELEMENTS, result.getAccumulatorResult(ACCUMULATOR_NAME));
    }

View Full Code Here

public class CollectionExecutionIterationTest implements java.io.Serializable {


  @Test
  public void testBulkIteration() {
    try {
      ExecutionEnvironment env = new CollectionEnvironment();
      
      IterativeDataSet<Integer> iteration = env.fromElements(1).iterate(10);
      
      DataSet<Integer> result = iteration.closeWith(iteration.map(new AddSuperstepNumberMapper()));
      
      List<Integer> collected = new ArrayList<Integer>();
      result.output(new LocalCollectionOutputFormat<Integer>(collected));
      
      env.execute();
      
      assertEquals(1, collected.size());
      assertEquals(56, collected.get(0).intValue());
    }
    catch (Exception e) {

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.flink.api.java.ExecutionEnvironment

org.apache.flink.api.avro.AvroOutputFormatTest

org.apache.flink.api.common.cache.DistributedCache.DistributedCacheEntry

org.apache.flink.api.common.operators.CollectionExecutionIterationTest

org.apache.flink.api.common.operators.CollectionExecutionWithBroadcastVariableTest

org.apache.flink.api.java.functions.SemanticPropertiesProjectionTest

org.apache.flink.api.java.functions.SemanticPropertiesTranslationTest

org.apache.flink.api.java.io.AvroInputFormatTypeExtractionTest

org.apache.flink.api.java.io.CsvReader

org.apache.flink.api.java.io.jdbc.example.JDBCExample

org.apache.flink.api.java.io.TextInputFormat

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.