Examples of org.apache.flink.api.java.ExecutionEnvironment

org.apache.flink.api.java.ExecutionEnvironment
The ExecutionEnviroment is the context in which a program is executed. A {@link LocalEnvironment} will cause execution in the current JVM, a{@link RemoteEnvironment} will cause execution on a remote setup.
The environment provides methods to control the job execution (such as setting the parallelism) and to interact with the outside world (data access).
Please note that the execution environment needs strong type information for the input and return types of all operations that are executed. This means that the environments needs to know that the return value of an operation is for example a Tuple of String and Integer. Because the Java compiler throws much of the generic type information away, most methods attempt to re- obtain that information using reflection. In certain cases, it may be necessary to manually supply that information to some of the methods. @see LocalEnvironment @see RemoteEnvironment

    }
  }
  
  
  private DualInputPlanNode createPlanAndGetJoinNode(JoinHint hint) {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    
    DataSet<Long> i1 = env.generateSequence(1, 1000);
    DataSet<Long> i2 = env.generateSequence(1, 1000);
    
    i1.join(i2, hint).where(new IdentityKeySelector<Long>()).equalTo(new IdentityKeySelector<Long>()).print();
    
    Plan plan = env.createProgramPlan();
    
    // set statistics to the sources
    plan.accept(new Visitor<Operator<?>>() {
      @Override
      public boolean preVisit(Operator<?> visitable) {

View Full Code Here

    }
  }
  
  @Test
  public void testBCVariableClosure() {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    
    DataSet<String> input = env.readTextFile(IN_FILE).name("source1");
    
    DataSet<String> reduced = input
        .map(new IdentityMapper<String>())
        .reduceGroup(new Top1GroupReducer<String>());
    
    
    DataSet<String> initialSolution = input.map(new IdentityMapper<String>()).withBroadcastSet(reduced, "bc");
    
    
    IterativeDataSet<String> iteration = initialSolution.iterate(100);
    
    iteration.closeWith(iteration.map(new IdentityMapper<String>()).withBroadcastSet(reduced, "red"))
        .print();
    
    Plan plan = env.createProgramPlan();
    
    try{
      compileNoStats(plan);
    }catch(Exception e){
      e.printStackTrace();

View Full Code Here

    }
  }
  
  @Test
  public void testMultipleIterations() {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    
    DataSet<String> input = env.readTextFile(IN_FILE).name("source1");
    
    DataSet<String> reduced = input
        .map(new IdentityMapper<String>())
        .reduceGroup(new Top1GroupReducer<String>());
      
    IterativeDataSet<String> iteration1 = input.iterate(100);
    IterativeDataSet<String> iteration2 = input.iterate(20);
    IterativeDataSet<String> iteration3 = input.iterate(17);
    
    iteration1.closeWith(iteration1.map(new IdentityMapper<String>()).withBroadcastSet(reduced, "bc1")).print();
    iteration2.closeWith(iteration2.reduceGroup(new Top1GroupReducer<String>()).withBroadcastSet(reduced, "bc2")).print();
    iteration3.closeWith(iteration3.reduceGroup(new IdentityGroupReducer<String>()).withBroadcastSet(reduced, "bc3")).print();
    
    Plan plan = env.createProgramPlan();
    
    try{
      compileNoStats(plan);
    }catch(Exception e){
      e.printStackTrace();

View Full Code Here

    }
  }
  
  @Test
  public void testMultipleIterationsWithClosueBCVars() {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    
    DataSet<String> input = env.readTextFile(IN_FILE).name("source1");
      
    IterativeDataSet<String> iteration1 = input.iterate(100);
    IterativeDataSet<String> iteration2 = input.iterate(20);
    IterativeDataSet<String> iteration3 = input.iterate(17);
    
    
    iteration1.closeWith(iteration1.map(new IdentityMapper<String>())).print();
    iteration2.closeWith(iteration2.reduceGroup(new Top1GroupReducer<String>())).print();
    iteration3.closeWith(iteration3.reduceGroup(new IdentityGroupReducer<String>())).print();
    
    Plan plan = env.createProgramPlan();
    
    try{
      compileNoStats(plan);
    }catch(Exception e){
      e.printStackTrace();

View Full Code Here

public class ReduceCompilationTest extends CompilerTestBase implements java.io.Serializable {


  @Test
  public void testAllReduceNoCombiner() {
    try {
      ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
      env.setDegreeOfParallelism(8);
      
      DataSet<Double> data = env.fromElements(0.2, 0.3, 0.4, 0.5).name("source");
      
      data.reduce(new RichReduceFunction<Double>() {
        
        @Override
        public Double reduce(Double value1, Double value2){
          return value1 + value2;
        }
      }).name("reducer")
      .print().name("sink");
      
      Plan p = env.createProgramPlan();
      OptimizedPlan op = compileNoStats(p);
      
      OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(op);

View Full Code Here

  }
  
  @Test
  public void testAllReduceWithCombiner() {
    try {
      ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
      env.setDegreeOfParallelism(8);
      
      DataSet<Long> data = env.generateSequence(1, 8000000).name("source");
      
      data.reduce(new RichReduceFunction<Long>() {
        
        @Override
        public Long reduce(Long value1, Long value2){
          return value1 + value2;
        }
      }).name("reducer")
      .print().name("sink");
      
      Plan p = env.createProgramPlan();
      OptimizedPlan op = compileNoStats(p);
      
      OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(op);
      
      // get the original nodes

View Full Code Here

  }
  
  @Test
  public void testGroupedReduceWithFieldPositionKey() {
    try {
      ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
      env.setDegreeOfParallelism(8);
      
      DataSet<Tuple2<String, Double>> data = env.readCsvFile("file:///will/never/be/read").types(String.class, Double.class)
        .name("source").setParallelism(6);
      
      data
        .groupBy(1)
        .reduce(new RichReduceFunction<Tuple2<String,Double>>() {
        @Override
        public Tuple2<String, Double> reduce(Tuple2<String, Double> value1, Tuple2<String, Double> value2){
          return null;
        }
      }).name("reducer")
      .print().name("sink");
      
      Plan p = env.createProgramPlan();
      OptimizedPlan op = compileNoStats(p);
      
      OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(op);
      
      // get the original nodes

View Full Code Here

  }
  
  @Test
  public void testGroupedReduceWithSelectorFunctionKey() {
    try {
      ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
      env.setDegreeOfParallelism(8);
      
      DataSet<Tuple2<String, Double>> data = env.readCsvFile("file:///will/never/be/read").types(String.class, Double.class)
        .name("source").setParallelism(6);
      
      data
        .groupBy(new KeySelector<Tuple2<String,Double>, String>() { 
          public String getKey(Tuple2<String, Double> value) { return value.f0; }
        })
        .reduce(new RichReduceFunction<Tuple2<String,Double>>() {
        @Override
        public Tuple2<String, Double> reduce(Tuple2<String, Double> value1, Tuple2<String, Double> value2){
          return null;
        }
      }).name("reducer")
      .print().name("sink");
      
      Plan p = env.createProgramPlan();
      OptimizedPlan op = compileNoStats(p);
      
      OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(op);
      
      // get the original nodes

View Full Code Here

@SuppressWarnings("serial")
public class UnionReplacementTest extends CompilerTestBase {


  @Test
  public void testUnionReplacement(){
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    DataSet<String> input1 = env.fromElements("test1");
    DataSet<String> input2 = env.fromElements("test2");


    DataSet<String> union = input1.union(input2);


    union.print();
    union.print();


    Plan plan = env.createProgramPlan();
    try{
      OptimizedPlan oPlan = this.compileNoStats(plan);
      NepheleJobGraphGenerator jobGen = new NepheleJobGraphGenerator();
      jobGen.compileJobGraph(oPlan);
    }catch(CompilerException co){

View Full Code Here

public class GroupReduceCompilationTest extends CompilerTestBase implements java.io.Serializable {


  @Test
  public void testAllGroupReduceNoCombiner() {
    try {
      ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
      env.setDegreeOfParallelism(8);
      
      DataSet<Double> data = env.fromElements(0.2, 0.3, 0.4, 0.5).name("source");
      
      data.reduceGroup(new RichGroupReduceFunction<Double, Double>() {
        public void reduce(Iterable<Double> values, Collector<Double> out) {}
      }).name("reducer")
      .print().name("sink");
      
      Plan p = env.createProgramPlan();
      OptimizedPlan op = compileNoStats(p);
      
      OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(op);

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.flink.api.java.ExecutionEnvironment

org.apache.flink.api.avro.AvroOutputFormatTest

org.apache.flink.api.common.cache.DistributedCache.DistributedCacheEntry

org.apache.flink.api.common.operators.CollectionExecutionIterationTest

org.apache.flink.api.common.operators.CollectionExecutionWithBroadcastVariableTest

org.apache.flink.api.java.functions.SemanticPropertiesProjectionTest

org.apache.flink.api.java.functions.SemanticPropertiesTranslationTest

org.apache.flink.api.java.io.AvroInputFormatTypeExtractionTest

org.apache.flink.api.java.io.CsvReader

org.apache.flink.api.java.io.jdbc.example.JDBCExample

org.apache.flink.api.java.io.TextInputFormat

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.