Examples of org.apache.flink.compiler.PactCompiler.compile()

Class org.apache.flink.compiler.PactCompiler

Examples of org.apache.flink.compiler.PactCompiler.compile()

org.apache.flink.compiler.PactCompiler.compile()
Translates the given plan in to an OptimizedPlan, where all nodes have their local strategy assigned and all channels have a shipping strategy assigned. The compiler connects to the job manager to obtain information about the available instances and their memory and then chooses an instance type to schedule the execution on.
The compilation process itself goes through several phases:
1. Create an optimizer data flow representation of the program, assign parallelism and compute size estimates.
2. Compute interesting properties and auxiliary structures.
3. Enumerate plan alternatives. This cannot be done in the same step as the interesting property computation (as opposed to the Database approaches), because we support plans that are not trees.
@param program The program to be translated. @return The optimized plan. @throws CompilerException Thrown, if the plan is invalid or the optimizer encountered an inconsistent situation during the compilation process.

    Plan plan = new Plan(output);
    plan.setDefaultParallelism(DOP);


    // optimize and compile plan 
    PactCompiler pc = new PactCompiler(new DataStatistics());
    OptimizedPlan op = pc.compile(plan);
    
    // return job graph of failing job
    NepheleJobGraphGenerator jgg = new NepheleJobGraphGenerator();
    return jgg.compileJobGraph(op);
  }

View Full Code Here

    Plan plan = new Plan(output);
    plan.setDefaultParallelism(4);


    // optimize and compile plan
    PactCompiler pc = new PactCompiler(new DataStatistics());
    OptimizedPlan op = pc.compile(plan);


    // return job graph of working job
    NepheleJobGraphGenerator jgg = new NepheleJobGraphGenerator();
    return jgg.compileJobGraph(op);
  }

View Full Code Here

    
    Plan plan = new Plan(output);
    plan.setDefaultParallelism(DOP);


    PactCompiler pc = new PactCompiler(new DataStatistics());
    OptimizedPlan op = pc.compile(plan);


    NepheleJobGraphGenerator jgg = new NepheleJobGraphGenerator();
    return jgg.compileJobGraph(op);
  }

View Full Code Here

    }
  }


  private JobGraph getJobGraph(final Plan plan) throws Exception {
    final PactCompiler pc = new PactCompiler(new DataStatistics());
    final OptimizedPlan op = pc.compile(plan);
    final NepheleJobGraphGenerator jgg = new NepheleJobGraphGenerator();
    return jgg.compileJobGraph(op);
  }


  public void setTaskManagerNumSlots(int taskManagerNumSlots) { this.taskManagerNumSlots = taskManagerNumSlots; }

View Full Code Here

    testReducer.setInput(input);


    Plan plan = new Plan(output);


    PactCompiler pc = new PactCompiler(new DataStatistics());
    OptimizedPlan op = pc.compile(plan);


    NepheleJobGraphGenerator jgg = new NepheleJobGraphGenerator();
    return jgg.compileJobGraph(op);


  }

View Full Code Here

    
    private OptimizedPlan compileProgram(String jobName) {
      Plan p = createProgramPlan(jobName);
      
      PactCompiler pc = new PactCompiler(new DataStatistics());
      return pc.compile(p);
    }
    
    private void setAsContext() {
      initializeContextEnvironment(this);
    }

View Full Code Here

        shutDownAtEnd = false;
      }


      try {
        PactCompiler pc = new PactCompiler(new DataStatistics());
        OptimizedPlan op = pc.compile(plan);
        
        NepheleJobGraphGenerator jgg = new NepheleJobGraphGenerator();
        JobGraph jobGraph = jgg.compileJobGraph(op);
        
        JobClient jobClient = this.nephele.getJobClient(jobGraph);

View Full Code Here

   * @return JSON dump of the optimized plan.
   * @throws Exception
   */
  public String getOptimizerPlanAsJSON(Plan plan) throws Exception {
    PactCompiler pc = new PactCompiler(new DataStatistics());
    OptimizedPlan op = pc.compile(plan);
    PlanJSONDumpGenerator gen = new PlanJSONDumpGenerator();
  
    return gen.getOptimizerPlanAsJSON(op);
  }

View Full Code Here

  public static String optimizerPlanAsJSON(Plan plan) throws Exception {
    LocalExecutor exec = new LocalExecutor();
    try {
      exec.start();
      PactCompiler pc = new PactCompiler(new DataStatistics());
      OptimizedPlan op = pc.compile(plan);
      PlanJSONDumpGenerator gen = new PlanJSONDumpGenerator();


      return gen.getOptimizerPlanAsJSON(op);
    } finally {
      exec.stop();

View Full Code Here

    if (p == null) {
      Assert.fail("Error: Cannot obtain Pact plan. Did the thest forget to override either 'getPactPlan()' or 'getJobGraph()' ?");
    }
    
    PactCompiler pc = new PactCompiler(new DataStatistics());
    OptimizedPlan op = pc.compile(p);
    
    if (printPlan) {
      System.out.println(new PlanJSONDumpGenerator().getOptimizerPlanAsJSON(op)); 
    }

View Full Code Here

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.