Package org.apache.flink.runtime.jobgraph

Examples of org.apache.flink.runtime.jobgraph.JobGraph


      messageLoss = Double.parseDouble(args[13]);
    }

    int totalMemoryConsumption = 3*minorConsumer + matchMemory + coGroupSortMemory;

    JobGraph jobGraph = new JobGraph("CompensatableDanglingPageRank");
   
    // --------------- the inputs ---------------------

    // page rank input
    InputFormatVertex pageWithRankInput = JobGraphUtils.createInput(new CustomImprovedDanglingPageRankInputFormat(),
View Full Code Here


      messageLoss = Double.parseDouble(args[13]);
    }

    int totalMemoryConsumption = 3*minorConsumer + matchMemory + coGroupSortMemory;

    JobGraph jobGraph = new JobGraph("CompensatableDanglingPageRank");
   
    // --------------- the inputs ---------------------

    // page rank input
    InputFormatVertex pageWithRankInput = JobGraphUtils.createInput(new ImprovedDanglingPageRankInputFormat(),
View Full Code Here

      messageLoss = Double.parseDouble(args[13]);
    }

    int totalMemoryConsumption = 3*minorConsumer + 2*coGroupSortMemory + matchMemory;

    JobGraph jobGraph = new JobGraph("CompensatableDanglingPageRank");
   
    // --------------- the inputs ---------------------

    // page rank input
    InputFormatVertex pageWithRankInput = JobGraphUtils.createInput(new CustomImprovedDanglingPageRankInputFormat(),
View Full Code Here

    return getJobGraph(optPlan, prog.getAllLibraries());
  }
 
  private JobGraph getJobGraph(OptimizedPlan optPlan, List<File> jarFiles) {
    NepheleJobGraphGenerator gen = new NepheleJobGraphGenerator();
    JobGraph job = gen.compileJobGraph(optPlan);
   
    for (File jar : jarFiles) {
      job.addJar(new Path(jar.getAbsolutePath()));
    }
   
    return job;
  }
View Full Code Here

    return run(getOptimizedPlan(prog, parallelism), prog.getJarFiles(), wait);
  }
 

  public JobExecutionResult run(OptimizedPlan compiledPlan, List<File> libraries, boolean wait) throws ProgramInvocationException {
    JobGraph job = getJobGraph(compiledPlan, libraries);
    return run(job, wait);
  }
View Full Code Here

  private JobGraph createJobGraphV1(String pointsPath, String centersPath, String resultPath, int numSubTasks) {

    // -- init -------------------------------------------------------------------------------------------------
    final TypeSerializerFactory<?> serializer = RecordSerializerFactory.get();

    JobGraph jobGraph = new JobGraph("Distance Builder");

    // -- vertices ---------------------------------------------------------------------------------------------
    InputFormatVertex points = createPointsInput(jobGraph, pointsPath, numSubTasks, serializer);
    InputFormatVertex models = createModelsInput(jobGraph, centersPath, numSubTasks, serializer);
    AbstractJobVertex mapper = createMapper(jobGraph, numSubTasks, serializer);
View Full Code Here

      TaskConfig t = new TaskConfig(tic.getContainingVertex().getConfiguration());
      t.addChainedTask(tic.getChainedTask(), tic.getTaskConfig(), tic.getTaskName());
    }
   
    // create the jobgraph object
    JobGraph graph = new JobGraph(program.getJobName());
    graph.setNumberOfExecutionRetries(program.getOriginalPactPlan().getNumberOfExecutionRetries());
    graph.setAllowQueuedScheduling(false);
   
    // add vertices to the graph
    for (AbstractJobVertex vertex : this.vertices.values()) {
      graph.addVertex(vertex);
    }
   
    for (AbstractJobVertex vertex : this.auxVertices) {
      graph.addVertex(vertex);
      vertex.setSlotSharingGroup(sharingGroup);
    }
   
    // add registered cache file into job configuration
    for (Entry<String, DistributedCacheEntry> e : program.getOriginalPactPlan().getCachedFiles()) {
      DistributedCache.writeFileInfoToConfig(e.getKey(), e.getValue(), graph.getJobConfiguration());
    }
   
    // release all references again
    this.vertices = null;
    this.chainedTasks = null;
View Full Code Here

    // -- init -------------------------------------------------------------------------------------------------
    final TypeSerializerFactory<?> serializer = RecordSerializerFactory.get();
    @SuppressWarnings("unchecked")
    final TypeComparatorFactory<?> int0Comparator = new RecordComparatorFactory(new int[] { 0 }, new Class[] { IntValue.class });

    JobGraph jobGraph = new JobGraph("KMeans Iterative");

    // -- vertices ---------------------------------------------------------------------------------------------
    InputFormatVertex points = createPointsInput(jobGraph, pointsPath, numSubTasks, serializer);
    InputFormatVertex centers = createCentersInput(jobGraph, centersPath, numSubTasks, serializer);
   
View Full Code Here

    @SuppressWarnings("unchecked")
    final TypeComparatorFactory<?> comparator =
      new RecordComparatorFactory(new int[] { 0 }, new Class[] { LongValue.class }, new boolean[] { true });
    final TypePairComparatorFactory<?, ?> pairComparator = RecordPairComparatorFactory.get();

    JobGraph jobGraph = new JobGraph("Connected Components (Unified Tails)");

    // -- invariant vertices -----------------------------------------------------------------------------------
    InputFormatVertex vertices = createVerticesInput(jobGraph, verticesPath, numSubTasks, serializer, comparator);
    InputFormatVertex edges = createEdgesInput(jobGraph, edgesPath, numSubTasks, serializer, comparator);
    AbstractJobVertex head = createIterationHead(jobGraph, numSubTasks, serializer, comparator, pairComparator);
View Full Code Here

    @SuppressWarnings("unchecked")
    final TypeComparatorFactory<?> comparator =
      new RecordComparatorFactory(new int[] { 0 }, new Class[] { LongValue.class }, new boolean[] { true });
    final TypePairComparatorFactory<?, ?> pairComparator = RecordPairComparatorFactory.get();

    JobGraph jobGraph = new JobGraph("Connected Components (Unified Tails)");

    // input
    InputFormatVertex vertices = createVerticesInput(jobGraph, verticesPath, numSubTasks, serializer, comparator);
    InputFormatVertex edges = createEdgesInput(jobGraph, edgesPath, numSubTasks, serializer, comparator);
View Full Code Here

TOP

Related Classes of org.apache.flink.runtime.jobgraph.JobGraph

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.