Package org.apache.flink.runtime.jobgraph

Examples of org.apache.flink.runtime.jobgraph.JobGraph

443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
      receiver.setParallelism(3*NUM_TASKS);
     
      receiver.connectNewDataSetAsInput(sender1, DistributionPattern.POINTWISE);
      receiver.connectNewDataSetAsInput(sender2, DistributionPattern.BIPARTITE);
     
      final JobGraph jobGraph = new JobGraph("Bipartite Job", sender1, receiver, sender2);
     
      JobManager jm = startJobManager(6 * NUM_TASKS);
     
      final GlobalBufferPool bp = ((LocalInstanceManager) jm.getInstanceManager())
                .getTaskManagers()[0].getChannelManager().getGlobalBufferPool();
     
      try {
        JobSubmissionResult result = jm.submitJob(jobGraph);

        if (result.getReturnCode() != AbstractJobResult.ReturnCode.SUCCESS) {
          System.out.println(result.getDescription());
        }
        assertEquals(AbstractJobResult.ReturnCode.SUCCESS, result.getReturnCode());
       
        // monitor the execution
        ExecutionGraph eg = jm.getCurrentJobs().get(jobGraph.getJobID());
       
        if (eg != null) {
          eg.waitForJobEnd();
          assertEquals(JobStatus.FINISHED, eg.getState());
         
View Full Code Here
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
      sender.setParallelism(NUM_TASKS);
      receiver.setParallelism(NUM_TASKS);
     
      receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE);
     
      final JobGraph jobGraph = new JobGraph("Pointwise Job", sender, receiver);
     
      final JobManager jm = startJobManager(NUM_TASKS);
     
      final GlobalBufferPool bp = ((LocalInstanceManager) jm.getInstanceManager())
          .getTaskManagers()[0].getChannelManager().getGlobalBufferPool();
     
      try {
        assertEquals(NUM_TASKS, jm.getTotalNumberOfRegisteredSlots());
       
        JobSubmissionResult result = jm.submitJob(jobGraph);

        if (result.getReturnCode() != AbstractJobResult.ReturnCode.SUCCESS) {
          System.out.println(result.getDescription());
        }
        assertEquals(AbstractJobResult.ReturnCode.SUCCESS, result.getReturnCode());
       
        // monitor the execution
        ExecutionGraph eg = jm.getCurrentJobs().get(jobGraph.getJobID());
       
        if (eg != null) {
          eg.waitForJobEnd();
          assertEquals(JobStatus.FAILED, eg.getState());
         
View Full Code Here
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
      sender.setParallelism(NUM_TASKS);
      receiver.setParallelism(NUM_TASKS);
     
      receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE);
     
      final JobGraph jobGraph = new JobGraph("Pointwise Job", sender, receiver);
     
      final JobManager jm = startJobManager(NUM_TASKS);
     
      final GlobalBufferPool bp = ((LocalInstanceManager) jm.getInstanceManager())
          .getTaskManagers()[0].getChannelManager().getGlobalBufferPool();
     
      try {
        assertEquals(NUM_TASKS, jm.getTotalNumberOfRegisteredSlots());
       
        JobSubmissionResult result = jm.submitJob(jobGraph);

        if (result.getReturnCode() != AbstractJobResult.ReturnCode.SUCCESS) {
          System.out.println(result.getDescription());
        }
        assertEquals(AbstractJobResult.ReturnCode.SUCCESS, result.getReturnCode());
       
        // monitor the execution
        ExecutionGraph eg = jm.getCurrentJobs().get(jobGraph.getJobID());
       
        if (eg != null) {
          eg.waitForJobEnd();
          assertEquals(JobStatus.FAILED, eg.getState());
        }
View Full Code Here
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
      sender.setParallelism(NUM_TASKS);
      receiver.setParallelism(NUM_TASKS);
     
      receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE);
     
      final JobGraph jobGraph = new JobGraph("Pointwise Job", sender, receiver);
     
      final JobManager jm = startJobManager(2 * NUM_TASKS);
     
      final GlobalBufferPool bp = ((LocalInstanceManager) jm.getInstanceManager())
          .getTaskManagers()[0].getChannelManager().getGlobalBufferPool();
     
      try {
        JobSubmissionResult result = jm.submitJob(jobGraph);
       
        if (result.getReturnCode() != AbstractJobResult.ReturnCode.SUCCESS) {
          System.out.println(result.getDescription());
        }
        assertEquals(AbstractJobResult.ReturnCode.SUCCESS, result.getReturnCode());
       
        // monitor the execution
        ExecutionGraph eg = jm.getCurrentJobs().get(jobGraph.getJobID());
       
        if (eg != null) {
          eg.waitForJobEnd();
          assertEquals(JobStatus.FAILED, eg.getState());
         
View Full Code Here
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
      sender.setParallelism(NUM_TASKS);
      receiver.setParallelism(NUM_TASKS);
     
      receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE);
     
      final JobGraph jobGraph = new JobGraph("Pointwise Job", sender, receiver);
     
      final JobManager jm = startJobManager(NUM_TASKS);
     
      final GlobalBufferPool bp = ((LocalInstanceManager) jm.getInstanceManager())
          .getTaskManagers()[0].getChannelManager().getGlobalBufferPool();
     
      try {
        assertEquals(NUM_TASKS, jm.getTotalNumberOfRegisteredSlots());
       
        JobSubmissionResult result = jm.submitJob(jobGraph);

        if (result.getReturnCode() != AbstractJobResult.ReturnCode.SUCCESS) {
          System.out.println(result.getDescription());
        }
        assertEquals(AbstractJobResult.ReturnCode.SUCCESS, result.getReturnCode());
       
        // monitor the execution
        ExecutionGraph eg = jm.getCurrentJobs().get(jobGraph.getJobID());
       
        if (eg != null) {
          eg.waitForJobEnd();
          assertEquals(JobStatus.FAILED, eg.getState());
         
View Full Code Here
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
      sender.setParallelism(NUM_TASKS);
      receiver.setParallelism(NUM_TASKS);
     
      receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE);
     
      final JobGraph jobGraph = new JobGraph("Pointwise Job", sender, receiver);
     
      final JobManager jm = startJobManager(2*NUM_TASKS);
     
      final GlobalBufferPool bp = ((LocalInstanceManager) jm.getInstanceManager())
          .getTaskManagers()[0].getChannelManager().getGlobalBufferPool();
     
      try {
        assertEquals(2*NUM_TASKS, jm.getNumberOfSlotsAvailableToScheduler());
       
        JobSubmissionResult result = jm.submitJob(jobGraph);

        if (result.getReturnCode() != AbstractJobResult.ReturnCode.SUCCESS) {
          System.out.println(result.getDescription());
        }
        assertEquals(AbstractJobResult.ReturnCode.SUCCESS, result.getReturnCode());
       
        // monitor the execution
        ExecutionGraph eg = jm.getCurrentJobs().get(jobGraph.getJobID());
       
        if (eg != null) {
          eg.waitForJobEnd();
          assertEquals(JobStatus.FAILED, eg.getState());
         
View Full Code Here
206
207
208
209
210
211
212
213
214
215
216
    public JobExecutionResult execute(String jobName) throws Exception {
      try {
        OptimizedPlan op = compileProgram(jobName);
       
        NepheleJobGraphGenerator jgg = new NepheleJobGraphGenerator();
        JobGraph jobGraph = jgg.compileJobGraph(op);

        JobClient client = this.executor.getJobClient(jobGraph);
        client.setConsoleStreamForReporting(AbstractTestBase.getNullPrintStream());
        JobExecutionResult result = client.submitJobAndWait();
       
View Full Code Here
231
232
233
234
235
236
237
238
239
240
241
      try {
        PactCompiler pc = new PactCompiler(new DataStatistics());
        OptimizedPlan op = pc.compile(plan);
       
        NepheleJobGraphGenerator jgg = new NepheleJobGraphGenerator();
        JobGraph jobGraph = jgg.compileJobGraph(op);
       
        JobClient jobClient = this.nephele.getJobClient(jobGraph);
        JobExecutionResult result = jobClient.submitJobAndWait();
        return result;
      }
View Full Code Here
104
105
106
107
108
109
110
111
112
113
114
        e.printStackTrace();
        Assert.fail("Pre-submit work caused an error: " + e.getMessage());
      }
     
      // submit job
      JobGraph jobGraph = null;
      try {
        jobGraph = getJobGraph();
      }
      catch(Exception e) {
        System.err.println(e.getMessage());
View Full Code Here
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
public class JobGraphTest {

  @Test
  public void testSerialization() {
    try {
      JobGraph jg = new JobGraph("The graph");
     
      // add some configuration values
      {
        jg.getJobConfiguration().setString("some key", "some value");
        jg.getJobConfiguration().setDouble("Life of ", Math.PI);
      }
     
      // add some vertices
      {
        AbstractJobVertex source1 = new AbstractJobVertex("source1");
        AbstractJobVertex source2 = new AbstractJobVertex("source2");
        AbstractJobVertex target = new AbstractJobVertex("target");
        target.connectNewDataSetAsInput(source1, DistributionPattern.POINTWISE);
        target.connectNewDataSetAsInput(source2, DistributionPattern.BIPARTITE);
       
        jg.addVertex(source1);
        jg.addVertex(source2);
        jg.addVertex(target);
      }
     
      // de-/serialize and compare
      JobGraph copy = CommonTestUtils.createCopyWritable(jg);
     
      assertEquals(jg.getName(), copy.getName());
      assertEquals(jg.getJobID(), copy.getJobID());
      assertEquals(jg.getJobConfiguration(), copy.getJobConfiguration());
      assertEquals(jg.getNumberOfVertices(), copy.getNumberOfVertices());
     
      for (AbstractJobVertex vertex : copy.getVertices()) {
        AbstractJobVertex original = jg.findVertexByID(vertex.getID());
        assertNotNull(original);
        assertEquals(original.getName(), vertex.getName());
        assertEquals(original.getNumberOfInputs(), vertex.getNumberOfInputs());
        assertEquals(original.getNumberOfProducedIntermediateDataSets(), vertex.getNumberOfProducedIntermediateDataSets());
View Full Code Here
TOP

Related Classes of org.apache.flink.runtime.jobgraph.JobGraph

  • org.apache.flink.client.LocalExecutor
  • org.apache.flink.client.program.Client
  • org.apache.flink.client.web.JobSubmissionServlet
  • org.apache.flink.compiler.plantranslate.NepheleJobGraphGenerator
  • org.apache.flink.core.fs.FileSystem
  • org.apache.flink.core.fs.FSDataInputStream
  • org.apache.flink.core.fs.Path
  • org.apache.flink.core.memory.DataInputViewStream
  • org.apache.flink.core.memory.DataOutputViewStream
  • org.apache.flink.runtime.blob.BlobClient
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.