Examples of org.apache.flink.api.java.ExecutionEnvironment

org.apache.flink.api.java.ExecutionEnvironment
The ExecutionEnviroment is the context in which a program is executed. A {@link LocalEnvironment} will cause execution in the current JVM, a{@link RemoteEnvironment} will cause execution on a remote setup.
The environment provides methods to control the job execution (such as setting the parallelism) and to interact with the outside world (data access).
Please note that the execution environment needs strong type information for the input and return types of all operations that are executed. This means that the environments needs to know that the return value of an operation is for example a Tuple of String and Integer. Because the Java compiler throws much of the generic type information away, most methods attempt to re- obtain that information using reflection. In certain cases, it may be necessary to manually supply that information to some of the methods. @see LocalEnvironment @see RemoteEnvironment

      switch(progId) {
      case 1: {
        /*
         * Union of 2 Same Data Sets
         */
        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        
        DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
        DataSet<Tuple3<Integer, Long, String>> unionDs = ds.union(CollectionDataSets.get3TupleDataSet(env));
        
        unionDs.writeAsCsv(resultPath);
        env.execute();
        
        // return expected result
        return FULL_TUPLE_3_STRING + FULL_TUPLE_3_STRING;
      }
      case 2: {
        /*
         * Union of 5 same Data Sets, with multiple unions
         */
        
        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        
        DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
        DataSet<Tuple3<Integer, Long, String>> unionDs = ds.union(CollectionDataSets.get3TupleDataSet(env))
            .union(CollectionDataSets.get3TupleDataSet(env))
            .union(CollectionDataSets.get3TupleDataSet(env))
            .union(CollectionDataSets.get3TupleDataSet(env));
        
        unionDs.writeAsCsv(resultPath);
        env.execute();
        
        // return expected result
        return FULL_TUPLE_3_STRING + FULL_TUPLE_3_STRING + FULL_TUPLE_3_STRING + FULL_TUPLE_3_STRING + FULL_TUPLE_3_STRING;
      }
      case 3: {
        /*
         * Test on union with empty dataset
         */
        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        
        // Don't know how to make an empty result in an other way than filtering it 
        DataSet<Tuple3<Integer, Long, String>> empty = CollectionDataSets.get3TupleDataSet(env).
            filter(new RichFilterFunction<Tuple3<Integer,Long,String>>() {
              private static final long serialVersionUID = 1L;


              @Override
              public boolean filter(Tuple3<Integer, Long, String> value) throws Exception {
                return false;
              }
            });
        
        DataSet<Tuple3<Integer, Long, String>> unionDs = CollectionDataSets.get3TupleDataSet(env)
          .union(empty);
      
        unionDs.writeAsCsv(resultPath);
        env.execute();
        
        // return expected result
        return FULL_TUPLE_3_STRING;        
      }
      default:

View Full Code Here

    compareResultsByLinesInMemory(WordCountData.COUNTS, resultPath);
  }
  
  @Override
  protected void testProgram() throws Exception {
    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();


    DataSet<String> text = env.readTextFile(textPath);


    DataSet<WC> counts = text
        .flatMap(new Tokenizer())
        .groupBy("word")
        .reduce(new ReduceFunction<WC>() {
          private static final long serialVersionUID = 1L;


          public WC reduce(WC value1, WC value2) {
            return new WC(value1.word, value1.count + value2.count);
          }
        });


    counts.writeAsText(resultPath);


    env.execute("WordCount with custom data types example");
  }

View Full Code Here

      case 1: {
        /*
         * Full Aggregate
         */
        
        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        
        DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
        DataSet<Tuple2<Integer, Long>> aggregateDs = ds
            .aggregate(Aggregations.SUM, 0)
            .and(Aggregations.MAX, 1)
            .project(0, 1).types(Integer.class, Long.class);
        
        aggregateDs.writeAsCsv(resultPath);
        env.execute();
        
        // return expected result
        return "231,6\n";
      }
      case 2: {
        /*
         * Grouped Aggregate
         */
        
        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        
        DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
        DataSet<Tuple2<Long, Integer>> aggregateDs = ds.groupBy(1)
            .aggregate(Aggregations.SUM, 0)
            .project(1, 0).types(Long.class, Integer.class);
        
        aggregateDs.writeAsCsv(resultPath);
        env.execute();
        
        // return expected result
        return "1,1\n" +
        "2,5\n" +
        "3,15\n" +
        "4,34\n" +
        "5,65\n" +
        "6,111\n";
      } 
      case 3: {
        /*
         * Nested Aggregate
         */
        
        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        
        DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
        DataSet<Tuple1<Integer>> aggregateDs = ds.groupBy(1)
            .aggregate(Aggregations.MIN, 0)
            .aggregate(Aggregations.MIN, 0)
            .project(0).types(Integer.class);
        
        aggregateDs.writeAsCsv(resultPath);
        env.execute();
        
        // return expected result
        return "1\n";
      }
      default:

View Full Code Here

        
        /*
         * check correctness of distinct on tuples with key field selector
         */
        
        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        
        DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.getSmall3TupleDataSet(env);
        DataSet<Tuple3<Integer, Long, String>> distinctDs = ds.union(ds).distinct(0, 1, 2);
        
        distinctDs.writeAsCsv(resultPath);
        env.execute();
        
        // return expected result
        return "1,1,Hi\n" +
            "2,2,Hello\n" +
            "3,2,Hello world\n";
      }
      case 2: {
        
        /*
         * check correctness of distinct on tuples with key field selector with not all fields selected
         */
        
        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        
        DataSet<Tuple5<Integer, Long,  Integer, String, Long>> ds = CollectionDataSets.getSmall5TupleDataSet(env);
        DataSet<Tuple1<Integer>> distinctDs = ds.union(ds).distinct(0).project(0).types(Integer.class);
        
        distinctDs.writeAsCsv(resultPath);
        env.execute();
        
        // return expected result
        return "1\n" +
            "2\n";
      }
      case 3: {
        
        /*
         * check correctness of distinct on tuples with key extractor
         */
        
        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        
        DataSet<Tuple5<Integer, Long,  Integer, String, Long>> ds = CollectionDataSets.getSmall5TupleDataSet(env);
        DataSet<Tuple1<Integer>> reduceDs = ds.union(ds)
            .distinct(new KeySelector<Tuple5<Integer, Long,  Integer, String, Long>, Integer>() {
                  private static final long serialVersionUID = 1L;
                  @Override
                  public Integer getKey(Tuple5<Integer, Long,  Integer, String, Long> in) {
                    return in.f0;
                  }
                }).project(0).types(Integer.class);
        
        reduceDs.writeAsCsv(resultPath);
        env.execute();
        
        // return expected result
        return "1\n" +
            "2\n";
                
      }
      case 4: {
        
        /*
         * check correctness of distinct on custom type with type extractor
         */
        
        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        
        DataSet<CustomType> ds = CollectionDataSets.getCustomTypeDataSet(env);
        DataSet<Tuple1<Integer>> reduceDs = ds
            .distinct(new KeySelector<CustomType, Integer>() {
                  private static final long serialVersionUID = 1L;
                  @Override
                  public Integer getKey(CustomType in) {
                    return in.myInt;
                  }
                })
            .map(new RichMapFunction<CustomType, Tuple1<Integer>>() {
              @Override
              public Tuple1<Integer> map(CustomType value) throws Exception {
                return new Tuple1<Integer>(value.myInt);
              }
            });
        
        reduceDs.writeAsCsv(resultPath);
        env.execute();
        
        // return expected result
        return "1\n" +
            "2\n" +
            "3\n" +
            "4\n" +
            "5\n" +
            "6\n";
        
      }
      case 5: {
        
        /*
         * check correctness of distinct on tuples
         */
        
        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        
        DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.getSmall3TupleDataSet(env);
        DataSet<Tuple3<Integer, Long, String>> distinctDs = ds.union(ds).distinct();
        
        distinctDs.writeAsCsv(resultPath);
        env.execute();
        
        // return expected result
        return "1,1,Hi\n" +
            "2,2,Hello\n" +
            "3,2,Hello world\n";
      }
      case 6: {
        
        /*
         * check correctness of distinct on custom type with tuple-returning type extractor
         */
        
        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        
        DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds = CollectionDataSets.get5TupleDataSet(env);
        DataSet<Tuple2<Integer, Long>> reduceDs = ds
            .distinct(new KeySelector<Tuple5<Integer, Long, Integer, String, Long>, Tuple2<Integer, Long>>() {
                  private static final long serialVersionUID = 1L;
                  @Override
                  public Tuple2<Integer,Long> getKey(Tuple5<Integer, Long, Integer, String, Long> t) {
                    return new Tuple2<Integer, Long>(t.f0, t.f4);
                  }
                })
            .project(0,4).types(Integer.class, Long.class);
        
        reduceDs.writeAsCsv(resultPath);
        env.execute();
        
        // return expected result
        return "1,1\n" +
            "2,1\n" +
            "2,2\n" +
            "3,2\n" +
            "3,3\n" +
            "4,1\n" +
            "4,2\n" +
            "5,1\n" +
            "5,2\n" +
            "5,3\n";
      }
      case 7: {
        
        /*
         * check correctness of distinct on tuples with field expressions
         */
        
        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        
        DataSet<Tuple5<Integer, Long,  Integer, String, Long>> ds = CollectionDataSets.getSmall5TupleDataSet(env);
        DataSet<Tuple1<Integer>> reduceDs = ds.union(ds)
            .distinct("f0").project(0).types(Integer.class);
        
        reduceDs.writeAsCsv(resultPath);
        env.execute();
        
        // return expected result
        return "1\n" +
            "2\n";
                
      }
      case 8: {
        
        /*
         * check correctness of distinct on Pojos
         */
        
        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        
        DataSet<POJO> ds = CollectionDataSets.getDuplicatePojoDataSet(env);
        DataSet<Integer> reduceDs = ds.distinct("nestedPojo.longNumber").map(new MapFunction<CollectionDataSets.POJO, Integer>() {
          @Override
          public Integer map(POJO value) throws Exception {
            return (int) value.nestedPojo.longNumber;
          }
        });
        
        reduceDs.writeAsText(resultPath);
        env.execute();
        
        // return expected result
        return "10000\n20000\n30000\n";
                
      }

View Full Code Here

  private List<Tuple2<Long, Long>> result = new ArrayList<Tuple2<Long, Long>>();
  
  @Override
  protected void testProgram() throws Exception {
    
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    
    DataSet<Tuple2<Long, Long>> input = env.generateSequence(1, 20).map(new Dupl());
        
    DeltaIteration<Tuple2<Long, Long>, Tuple2<Long, Long>> iter = input.iterateDelta(input, 20, 0);
    iter.closeWith(iter.getWorkset(), iter.getWorkset())
      .output(new LocalCollectionOutputFormat<Tuple2<Long, Long>>(result));
    
    env.execute();
  }

View Full Code Here

  }




  @Override
  protected void testProgram() throws Exception {
    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();


    DataSet<String> text = env.fromElements(WordCountData.TEXT);
    DataSet<Tuple2<String, Integer>> words = text.flatMap(new WordCount.Tokenizer());
    DataSet<Tuple2<String, Integer>> result = words.groupBy(0).aggregate(Aggregations.SUM, 1);


    result.output(new LocalCollectionOutputFormat<Tuple2<String, Integer>>(resultsCollected));
    env.execute("Word Count Collection");
  }

View Full Code Here

      case 1: {
        /*
         * Test all-rejecting filter.
         */
        
        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        
        DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
        DataSet<Tuple3<Integer, Long, String>> filterDs = ds.
            filter(new FilterFunction<Tuple3<Integer,Long,String>>() {
              private static final long serialVersionUID = 1L;


              @Override
              public boolean filter(Tuple3<Integer, Long, String> value) throws Exception {
                return false;
              }
            });
        
        filterDs.writeAsCsv(resultPath);
        env.execute();
        
        // return expected result
        return "\n";
      }
      case 2: {
        /*
         * Test all-passing filter.
         */
        
        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        
        DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
        DataSet<Tuple3<Integer, Long, String>> filterDs = ds.
            filter(new FilterFunction<Tuple3<Integer,Long,String>>() {
              private static final long serialVersionUID = 1L;


              @Override
              public boolean filter(Tuple3<Integer, Long, String> value) throws Exception {
                return true;
              }
            });
        filterDs.writeAsCsv(resultPath);
        env.execute();
        
        // return expected result
        return "1,1,Hi\n" +
            "2,2,Hello\n" +
            "3,2,Hello world\n" +
            "4,3,Hello world, how are you?\n" +
            "5,3,I am fine.\n" +
            "6,3,Luke Skywalker\n" +
            "7,4,Comment#1\n" +
            "8,4,Comment#2\n" +
            "9,4,Comment#3\n" +
            "10,4,Comment#4\n" +
            "11,5,Comment#5\n" +
            "12,5,Comment#6\n" +
            "13,5,Comment#7\n" +
            "14,5,Comment#8\n" +
            "15,5,Comment#9\n" +
            "16,6,Comment#10\n" +
            "17,6,Comment#11\n" +
            "18,6,Comment#12\n" +
            "19,6,Comment#13\n" +
            "20,6,Comment#14\n" +
            "21,6,Comment#15\n";
      }
      case 3: {
        /*
         * Test filter on String tuple field.
         */
          
        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        
        DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
        DataSet<Tuple3<Integer, Long, String>> filterDs = ds.
            filter(new FilterFunction<Tuple3<Integer,Long,String>>() {
              private static final long serialVersionUID = 1L;


              @Override
              public boolean filter(Tuple3<Integer, Long, String> value) throws Exception {
                return value.f2.contains("world");
              }
            });
        filterDs.writeAsCsv(resultPath);
        env.execute();
        
        // return expected result
        return "3,2,Hello world\n" +
            "4,3,Hello world, how are you?\n";
        
      }
      case 4: {
        /*
         * Test filter on Integer tuple field.
         */
          
        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        
        DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
        DataSet<Tuple3<Integer, Long, String>> filterDs = ds.
            filter(new FilterFunction<Tuple3<Integer,Long,String>>() {
              private static final long serialVersionUID = 1L;


              @Override
              public boolean filter(Tuple3<Integer, Long, String> value) throws Exception {
                return (value.f0 % 2) == 0;
              }
            });
        filterDs.writeAsCsv(resultPath);
        env.execute();
        
        // return expected result
        return "2,2,Hello\n" +
            "4,3,Hello world, how are you?\n" +
            "6,3,Luke Skywalker\n" +
            "8,4,Comment#2\n" +
            "10,4,Comment#4\n" +
            "12,5,Comment#6\n" +
            "14,5,Comment#8\n" +
            "16,6,Comment#10\n" +
            "18,6,Comment#12\n" +
            "20,6,Comment#14\n";
      }
      case 5: {
        /*
         * Test filter on basic type
         */
            
        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        
        DataSet<String> ds = CollectionDataSets.getStringDataSet(env);
        DataSet<String> filterDs = ds.
            filter(new FilterFunction<String>() {
              private static final long serialVersionUID = 1L;


              @Override
              public boolean filter(String value) throws Exception {
                return value.startsWith("H");
              }
            });
        filterDs.writeAsText(resultPath);
        env.execute();
        
        // return expected result
        return "Hi\n" +
             "Hello\n" + 
             "Hello world\n" +
             "Hello world, how are you?\n";
      }
      case 6: {
        /*
         * Test filter on custom type
         */
            
        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        
        DataSet<CustomType> ds = CollectionDataSets.getCustomTypeDataSet(env);
        DataSet<CustomType> filterDs = ds.
            filter(new FilterFunction<CustomType>() {
              private static final long serialVersionUID = 1L;


              @Override
              public boolean filter(CustomType value) throws Exception {
                return value.myString.contains("a");
              }
            });
        filterDs.writeAsText(resultPath);
        env.execute();
        
        // return expected result
        return "3,3,Hello world, how are you?\n" +
            "3,4,I am fine.\n" +
            "3,5,Luke Skywalker\n";
      }
      case 7: {
        /*
         * Test filter on String tuple field.
         */
          
        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        
        DataSet<Integer> ints = CollectionDataSets.getIntegerDataSet(env);
        
        DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
        DataSet<Tuple3<Integer, Long, String>> filterDs = ds.
            filter(new RichFilterFunction<Tuple3<Integer,Long,String>>() {
              private static final long serialVersionUID = 1L;


              int literal = -1;
              
              @Override
              public void open(Configuration config) {
                Collection<Integer> ints = this.getRuntimeContext().getBroadcastVariable("ints");
                for(int i: ints) {
                  literal = literal < i ? i : literal;
                }
              }
              
              @Override
              public boolean filter(Tuple3<Integer, Long, String> value) throws Exception {
                return value.f0 < literal;
              }
            }).withBroadcastSet(ints, "ints");
        filterDs.writeAsCsv(resultPath);
        env.execute();
        
        // return expected result
        return "1,1,Hi\n" +
            "2,2,Hello\n" +
            "3,2,Hello world\n" +
            "4,3,Hello world, how are you?\n";
      }
      case 8: {
        /*
         * Test filter with broadcast variables
         */
          
        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        
        DataSet<Integer> intDs = CollectionDataSets.getIntegerDataSet(env);
        
        DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
        DataSet<Tuple3<Integer, Long, String>> filterDs = ds.
            filter(new RichFilterFunction<Tuple3<Integer,Long,String>>() {
              private static final long serialVersionUID = 1L;
              private  int broadcastSum = 0;
              
              @Override
              public void open(Configuration config) {
                Collection<Integer> ints = this.getRuntimeContext().getBroadcastVariable("ints");
                for(Integer i : ints) {
                  broadcastSum += i;
                }
              }


              @Override
              public boolean filter(Tuple3<Integer, Long, String> value) throws Exception {
                return (value.f1 == (broadcastSum / 11));
              }
            }).withBroadcastSet(intDs, "ints");
        filterDs.writeAsCsv(resultPath);
        env.execute();
        
        // return expected result
        return "11,5,Comment#5\n" +
            "12,5,Comment#6\n" +
            "13,5,Comment#7\n" +

View Full Code Here

    compareResultsByLinesInMemory(WordCountData.COUNTS, resultPath);
  }
  
  @Override
  protected void testProgram() throws Exception {
    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    DataSet<String> text = env.readTextFile(textPath);


    DataSet<WC> counts = text
        .flatMap(new Tokenizer())
        .groupBy("complex.someTest")
        .reduce(new ReduceFunction<WC>() {
          private static final long serialVersionUID = 1L;
          public WC reduce(WC value1, WC value2) {
            return new WC(value1.complex.someTest, value1.count + value2.count);
          }
        });


    counts.writeAsText(resultPath);


    env.execute("WordCount with custom data types example");
  }

View Full Code Here

      case 1: {
        /*
         * Test identity map with basic type
         */
    
        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        
        DataSet<String> ds = CollectionDataSets.getStringDataSet(env);
        DataSet<String> identityMapDs = ds.
            map(new MapFunction<String, String>() {
              private static final long serialVersionUID = 1L;


              @Override
              public String map(String value) throws Exception {
                return value;
              }
            });
        
        identityMapDs.writeAsText(resultPath);
        env.execute();
        
        // return expected result
        return   "Hi\n" +
            "Hello\n" +
            "Hello world\n" +
            "Hello world, how are you?\n" +
            "I am fine.\n" +
            "Luke Skywalker\n" +
            "Random comment\n" +
            "LOL\n";
      }
      case 2: {
        /*
         * Test identity map with a tuple
         */
    
        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        
        DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
        DataSet<Tuple3<Integer, Long, String>> identityMapDs = ds.
            map(new MapFunction<Tuple3<Integer, Long, String>, Tuple3<Integer, Long, String>>() {
              private static final long serialVersionUID = 1L;


              @Override
              public Tuple3<Integer, Long, String> map(Tuple3<Integer, Long, String> value) 
                  throws Exception {
                return value;
              }
            });
        
        identityMapDs.writeAsCsv(resultPath);
        env.execute();
        
        // return expected result
        return  "1,1,Hi\n" +
            "2,2,Hello\n" +
            "3,2,Hello world\n" +
            "4,3,Hello world, how are you?\n" +
            "5,3,I am fine.\n" +
            "6,3,Luke Skywalker\n" +
            "7,4,Comment#1\n" +
            "8,4,Comment#2\n" +
            "9,4,Comment#3\n" +
            "10,4,Comment#4\n" +
            "11,5,Comment#5\n" +
            "12,5,Comment#6\n" +
            "13,5,Comment#7\n" +
            "14,5,Comment#8\n" +
            "15,5,Comment#9\n" +
            "16,6,Comment#10\n" +
            "17,6,Comment#11\n" +
            "18,6,Comment#12\n" +
            "19,6,Comment#13\n" +
            "20,6,Comment#14\n" +
            "21,6,Comment#15\n";
      }
      case 3: {
        /*
         * Test type conversion mapper (Custom -> Tuple)
         */
    
        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        
        DataSet<CustomType> ds = CollectionDataSets.getCustomTypeDataSet(env);
        DataSet<Tuple3<Integer, Long, String>> typeConversionMapDs = ds.
            map(new MapFunction<CustomType, Tuple3<Integer, Long, String>>() {
              private static final long serialVersionUID = 1L;
              private final Tuple3<Integer, Long, String> out = new Tuple3<Integer, Long, String>();
              
              @Override
              public Tuple3<Integer, Long, String> map(CustomType value) throws Exception {
                out.setField(value.myInt, 0);
                out.setField(value.myLong, 1);
                out.setField(value.myString, 2);
                return out;
              }
            });
        
        typeConversionMapDs.writeAsCsv(resultPath);
        env.execute();
        
        // return expected result
        return   "1,0,Hi\n" +
            "2,1,Hello\n" +
            "2,2,Hello world\n" +
            "3,3,Hello world, how are you?\n" +
            "3,4,I am fine.\n" +
            "3,5,Luke Skywalker\n" +
            "4,6,Comment#1\n" +
            "4,7,Comment#2\n" +
            "4,8,Comment#3\n" +
            "4,9,Comment#4\n" +
            "5,10,Comment#5\n" +
            "5,11,Comment#6\n" +
            "5,12,Comment#7\n" +
            "5,13,Comment#8\n" +
            "5,14,Comment#9\n" +
            "6,15,Comment#10\n" +
            "6,16,Comment#11\n" +
            "6,17,Comment#12\n" +
            "6,18,Comment#13\n" +
            "6,19,Comment#14\n" +
            "6,20,Comment#15\n";
      }
      case 4: {
        /*
         * Test type conversion mapper (Tuple -> Basic)
         */
    
        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        
        DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
        DataSet<String> typeConversionMapDs = ds.
            map(new MapFunction<Tuple3<Integer, Long, String>, String>() {
              private static final long serialVersionUID = 1L;
              
              @Override
              public String map(Tuple3<Integer, Long, String> value) throws Exception {
                return value.getField(2);
              }
            });
        
        typeConversionMapDs.writeAsText(resultPath);
        env.execute();
        
        // return expected result
        return   "Hi\n" + "Hello\n" + "Hello world\n" +
            "Hello world, how are you?\n" +
            "I am fine.\n" + "Luke Skywalker\n" +
            "Comment#1\n" +  "Comment#2\n" +
            "Comment#3\n" +  "Comment#4\n" +
            "Comment#5\n" +  "Comment#6\n" +
            "Comment#7\n" + "Comment#8\n" +
            "Comment#9\n" +  "Comment#10\n" +
            "Comment#11\n" + "Comment#12\n" +
            "Comment#13\n" + "Comment#14\n" +
            "Comment#15\n";
      }
      case 5: {
        /*
         * Test mapper on tuple - Increment Integer field, reorder second and third fields
         */
    
        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        
        DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
        DataSet<Tuple3<Integer, String, Long>> tupleMapDs = ds.
            map(new MapFunction<Tuple3<Integer, Long, String>, Tuple3<Integer, String, Long>>() {
              private static final long serialVersionUID = 1L;
              private final Tuple3<Integer, String, Long> out = new Tuple3<Integer, String, Long>();
              
              @Override
              public Tuple3<Integer, String, Long> map(Tuple3<Integer, Long, String> value) 
                  throws Exception {
                Integer incr = new Integer(value.f0.intValue() + 1);
                out.setFields(incr, value.f2, value.f1);
                return out;
              }
            });
        
        tupleMapDs.writeAsCsv(resultPath);
        env.execute();
        
        // return expected result
        return   "2,Hi,1\n" +
            "3,Hello,2\n" +
            "4,Hello world,2\n" +
            "5,Hello world, how are you?,3\n" +
            "6,I am fine.,3\n" +
            "7,Luke Skywalker,3\n" +
            "8,Comment#1,4\n" +
            "9,Comment#2,4\n" +
            "10,Comment#3,4\n" +
            "11,Comment#4,4\n" +
            "12,Comment#5,5\n" +
            "13,Comment#6,5\n" +
            "14,Comment#7,5\n" +
            "15,Comment#8,5\n" +
            "16,Comment#9,5\n" +
            "17,Comment#10,6\n" +
            "18,Comment#11,6\n" +
            "19,Comment#12,6\n" +
            "20,Comment#13,6\n" +
            "21,Comment#14,6\n" +
            "22,Comment#15,6\n";
      }
      case 6: {
        /*
         * Test mapper on Custom - lowercase myString
         */
    
        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        
        DataSet<CustomType> ds = CollectionDataSets.getCustomTypeDataSet(env);
        DataSet<CustomType> customMapDs = ds.
            map(new MapFunction<CustomType, CustomType>() {
              private static final long serialVersionUID = 1L;
              private final CustomType out = new CustomType(); 
              
              @Override
              public CustomType map(CustomType value) throws Exception {
                out.myInt = value.myInt;
                out.myLong = value.myLong;
                out.myString = value.myString.toLowerCase();
                return out;
              }
            });
        
        customMapDs.writeAsText(resultPath);
        env.execute();
        
        // return expected result
        return   "1,0,hi\n" +
            "2,1,hello\n" +
            "2,2,hello world\n" +
            "3,3,hello world, how are you?\n" +
            "3,4,i am fine.\n" +
            "3,5,luke skywalker\n" +
            "4,6,comment#1\n" +
            "4,7,comment#2\n" +
            "4,8,comment#3\n" +
            "4,9,comment#4\n" +
            "5,10,comment#5\n" +
            "5,11,comment#6\n" +
            "5,12,comment#7\n" +
            "5,13,comment#8\n" +
            "5,14,comment#9\n" +
            "6,15,comment#10\n" +
            "6,16,comment#11\n" +
            "6,17,comment#12\n" +
            "6,18,comment#13\n" +
            "6,19,comment#14\n" +
            "6,20,comment#15\n";
      }
      case 7: {
        /*
         * Test mapper if UDF returns input object - increment first field of a tuple
         */
    
        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        
        DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
        DataSet<Tuple3<Integer, Long, String>> inputObjMapDs = ds.
            map(new MapFunction<Tuple3<Integer, Long, String>, Tuple3<Integer, Long, String>>() {
              private static final long serialVersionUID = 1L;
              
              @Override
              public Tuple3<Integer, Long, String> map(Tuple3<Integer, Long, String> value) 
                  throws Exception {
                Integer incr = new Integer(value.f0.intValue() + 1);
                value.setField(incr, 0);
                return value;
              }
            });
        
        inputObjMapDs.writeAsCsv(resultPath);
        env.execute();
        
        // return expected result
        return   "2,1,Hi\n" +
            "3,2,Hello\n" +
            "4,2,Hello world\n" +
            "5,3,Hello world, how are you?\n" +
            "6,3,I am fine.\n" +
            "7,3,Luke Skywalker\n" +
            "8,4,Comment#1\n" +
            "9,4,Comment#2\n" +
            "10,4,Comment#3\n" +
            "11,4,Comment#4\n" +
            "12,5,Comment#5\n" +
            "13,5,Comment#6\n" +
            "14,5,Comment#7\n" +
            "15,5,Comment#8\n" +
            "16,5,Comment#9\n" +
            "17,6,Comment#10\n" +
            "18,6,Comment#11\n" +
            "19,6,Comment#12\n" +
            "20,6,Comment#13\n" +
            "21,6,Comment#14\n" +
            "22,6,Comment#15\n";
      }
      case 8: {
        /*
         * Test map with broadcast set 
         */
          
        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        
        DataSet<Integer> ints = CollectionDataSets.getIntegerDataSet(env);
        
        DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
        DataSet<Tuple3<Integer, Long, String>> bcMapDs = ds.
            map(new RichMapFunction<Tuple3<Integer,Long,String>, Tuple3<Integer,Long,String>>() {
              private static final long serialVersionUID = 1L;
              private final Tuple3<Integer, Long, String> out = new Tuple3<Integer, Long, String>();
              private Integer f2Replace = 0;
              
              @Override
              public void open(Configuration config) {
                Collection<Integer> ints = this.getRuntimeContext().getBroadcastVariable("ints");
                int sum = 0;
                for(Integer i : ints) {
                  sum += i;
                }
                f2Replace = sum;
              }
              
              @Override
              public Tuple3<Integer, Long, String> map(Tuple3<Integer, Long, String> value) 
                  throws Exception {
                out.setFields(f2Replace, value.f1, value.f2);
                return out;
              }
            }).withBroadcastSet(ints, "ints");
        bcMapDs.writeAsCsv(resultPath);
        env.execute();
        
        // return expected result
        return   "55,1,Hi\n" +
            "55,2,Hello\n" +
            "55,2,Hello world\n" +
            "55,3,Hello world, how are you?\n" +
            "55,3,I am fine.\n" +
            "55,3,Luke Skywalker\n" +
            "55,4,Comment#1\n" +
            "55,4,Comment#2\n" +
            "55,4,Comment#3\n" +
            "55,4,Comment#4\n" +
            "55,5,Comment#5\n" +
            "55,5,Comment#6\n" +
            "55,5,Comment#7\n" +
            "55,5,Comment#8\n" +
            "55,5,Comment#9\n" +
            "55,6,Comment#10\n" +
            "55,6,Comment#11\n" +
            "55,6,Comment#12\n" +
            "55,6,Comment#13\n" +
            "55,6,Comment#14\n" +
            "55,6,Comment#15\n";
      }
      case 9: {
        /*
         * Test passing configuration object.
         */
          
        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        
        DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.getSmall3TupleDataSet(env);
        Configuration conf = new Configuration();
        final String testKey = "testVariable";
        final int testValue = 666;
        conf.setInteger(testKey, testValue);
        DataSet<Tuple3<Integer, Long, String>> bcMapDs = ds.
            map(new RichMapFunction<Tuple3<Integer,Long,String>, Tuple3<Integer,Long,String>>() {
              private static final long serialVersionUID = 1L;
              
              @Override
              public void open(Configuration config) {
                int val = config.getInteger(testKey, -1);
                Assert.assertEquals(testValue, val);
              }
              
              @Override
              public Tuple3<Integer, Long, String> map(Tuple3<Integer, Long, String> value) {
                return value;
              }
            }).withParameters(conf);
        bcMapDs.writeAsCsv(resultPath);
        env.execute();
        
        // return expected result
        return   "1,1,Hi\n"
            + "2,2,Hello\n"
            + "3,2,Hello world";

View Full Code Here

    compareResultCollections(expected, result, new TupleComparator<Tuple2<String, Integer>>());
  }


  @Override
  protected void testProgram() throws Exception {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    
    DataSet<Tuple2<String, String>> data = env.fromCollection(input);
    
    data.mapPartition(new TestMapPartition()).output(new LocalCollectionOutputFormat<Tuple2<String,Integer>>(result));
    
    env.execute();
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.flink.api.java.ExecutionEnvironment

org.apache.flink.api.avro.AvroOutputFormatTest

org.apache.flink.api.common.cache.DistributedCache.DistributedCacheEntry

org.apache.flink.api.common.operators.CollectionExecutionIterationTest

org.apache.flink.api.common.operators.CollectionExecutionWithBroadcastVariableTest

org.apache.flink.api.java.functions.SemanticPropertiesProjectionTest

org.apache.flink.api.java.functions.SemanticPropertiesTranslationTest

org.apache.flink.api.java.io.AvroInputFormatTypeExtractionTest

org.apache.flink.api.java.io.CsvReader

org.apache.flink.api.java.io.jdbc.example.JDBCExample

org.apache.flink.api.java.io.TextInputFormat

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.