Package org.apache.flink.runtime.jobgraph

Examples of org.apache.flink.runtime.jobgraph.AbstractJobVertex

428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
  public void testTwoInputJob() {
   
    final int NUM_TASKS = 11;
   
    try {
      final AbstractJobVertex sender1 = new AbstractJobVertex("Sender1");
      final AbstractJobVertex sender2 = new AbstractJobVertex("Sender2");
      final AbstractJobVertex receiver = new AbstractJobVertex("Receiver");
     
      sender1.setInvokableClass(Sender.class);
      sender2.setInvokableClass(Sender.class);
      receiver.setInvokableClass(AgnosticBinaryReceiver.class);
     
      sender1.setParallelism(NUM_TASKS);
      sender2.setParallelism(2*NUM_TASKS);
      receiver.setParallelism(3*NUM_TASKS);
     
      receiver.connectNewDataSetAsInput(sender1, DistributionPattern.POINTWISE);
      receiver.connectNewDataSetAsInput(sender2, DistributionPattern.BIPARTITE);
     
      final JobGraph jobGraph = new JobGraph("Bipartite Job", sender1, receiver, sender2);
     
      JobManager jm = startJobManager(6 * NUM_TASKS);
     
 
View Full Code Here
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
  public void testJobFailingSender() {
   
    final int NUM_TASKS = 100;
   
    try {
      final AbstractJobVertex sender = new AbstractJobVertex("Sender");
      final AbstractJobVertex receiver = new AbstractJobVertex("Receiver");
     
      sender.setInvokableClass(ExceptionSender.class);
      receiver.setInvokableClass(Receiver.class);
     
      sender.setParallelism(NUM_TASKS);
      receiver.setParallelism(NUM_TASKS);
     
      receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE);
     
      final JobGraph jobGraph = new JobGraph("Pointwise Job", sender, receiver);
     
      final JobManager jm = startJobManager(NUM_TASKS);
     
View Full Code Here
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
  @Test
  public void testAllVertices() {
    try {
     
      AbstractJobVertex v1 = new AbstractJobVertex("v1");
      AbstractJobVertex v2 = new AbstractJobVertex("v2");
      AbstractJobVertex v3 = new AbstractJobVertex("v3");
      AbstractJobVertex v4 = new AbstractJobVertex("v4");
     
      v1.setParallelism(1);
      v2.setParallelism(7);
      v3.setParallelism(3);
      v4.setParallelism(2);
     
      ExecutionGraph eg = Mockito.mock(ExecutionGraph.class);
         
      ExecutionJobVertex ejv1 = new ExecutionJobVertex(eg, v1, 1);
      ExecutionJobVertex ejv2 = new ExecutionJobVertex(eg, v2, 1);
View Full Code Here
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
  public void testJobSometimesFailingSender() {
   
    final int NUM_TASKS = 100;
   
    try {
      final AbstractJobVertex sender = new AbstractJobVertex("Sender");
      final AbstractJobVertex receiver = new AbstractJobVertex("Receiver");
     
      sender.setInvokableClass(SometimesExceptionSender.class);
      receiver.setInvokableClass(Receiver.class);
     
      sender.setParallelism(NUM_TASKS);
      receiver.setParallelism(NUM_TASKS);
     
      receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE);
     
      final JobGraph jobGraph = new JobGraph("Pointwise Job", sender, receiver);
     
      final JobManager jm = startJobManager(NUM_TASKS);
     
View Full Code Here
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
  public void testJobFailingReceiver() {
   
    final int NUM_TASKS = 200;
   
    try {
      final AbstractJobVertex sender = new AbstractJobVertex("Sender");
      final AbstractJobVertex receiver = new AbstractJobVertex("Receiver");
     
      sender.setInvokableClass(Sender.class);
      receiver.setInvokableClass(ExceptionReceiver.class);
     
      sender.setParallelism(NUM_TASKS);
      receiver.setParallelism(NUM_TASKS);
     
      receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE);
     
      final JobGraph jobGraph = new JobGraph("Pointwise Job", sender, receiver);
     
      final JobManager jm = startJobManager(2 * NUM_TASKS);
     
 
View Full Code Here
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
  public void testJobFailingInstantiation() {
   
    final int NUM_TASKS = 200;
   
    try {
      final AbstractJobVertex sender = new AbstractJobVertex("Sender");
      final AbstractJobVertex receiver = new AbstractJobVertex("Receiver");
     
      sender.setInvokableClass(InstantiationErrorSender.class);
      receiver.setInvokableClass(Receiver.class);
     
      sender.setParallelism(NUM_TASKS);
      receiver.setParallelism(NUM_TASKS);
     
      receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE);
     
      final JobGraph jobGraph = new JobGraph("Pointwise Job", sender, receiver);
     
      final JobManager jm = startJobManager(NUM_TASKS);
     
View Full Code Here
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
  public void testJobFailingSomeInstantiations() {
   
    final int NUM_TASKS = 200;
   
    try {
      final AbstractJobVertex sender = new AbstractJobVertex("Sender");
      final AbstractJobVertex receiver = new AbstractJobVertex("Receiver");
     
      sender.setInvokableClass(SometimesInstantiationErrorSender.class);
      receiver.setInvokableClass(Receiver.class);
     
      sender.setParallelism(NUM_TASKS);
      receiver.setParallelism(NUM_TASKS);
     
      receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE);
     
      final JobGraph jobGraph = new JobGraph("Pointwise Job", sender, receiver);
     
      final JobManager jm = startJobManager(2*NUM_TASKS);
     
 
View Full Code Here
47
48
49
50
51
52
53
54
55
56
57
58
59
      Scheduler scheduler = new Scheduler();
      scheduler.newInstanceAvailable(instance);
     
      // The job:
     
      final AbstractJobVertex sender = new AbstractJobVertex("Task");
      sender.setInvokableClass(NoOpInvokable.class);
      sender.setParallelism(NUM_TASKS);
     
      final JobGraph jobGraph = new JobGraph("Pointwise Job", sender);
     
      ExecutionGraph eg = new ExecutionGraph(new JobID(), "test job", new Configuration());
      eg.setNumberOfRetriesLeft(0);
View Full Code Here
92
93
94
95
96
97
98
99
100
101
102
103
104
      Scheduler scheduler = new Scheduler();
      scheduler.newInstanceAvailable(instance);
     
      // The job:
     
      final AbstractJobVertex sender = new AbstractJobVertex("Task");
      sender.setInvokableClass(NoOpInvokable.class);
      sender.setParallelism(NUM_TASKS);
     
      final JobGraph jobGraph = new JobGraph("Pointwise Job", sender);
     
      ExecutionGraph eg = new ExecutionGraph(new JobID(), "test job", new Configuration());
      eg.setNumberOfRetriesLeft(1);
View Full Code Here
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
   * - v4 and v5 (connected) share slots
   */
  @Test
  public void testAssignSlotSharingGroup() {
    try {
      AbstractJobVertex v1 = new AbstractJobVertex("v1");
      AbstractJobVertex v2 = new AbstractJobVertex("v2");
      AbstractJobVertex v3 = new AbstractJobVertex("v3");
      AbstractJobVertex v4 = new AbstractJobVertex("v4");
      AbstractJobVertex v5 = new AbstractJobVertex("v5");
     
      v1.setParallelism(4);
      v2.setParallelism(5);
      v3.setParallelism(7);
      v4.setParallelism(1);
      v5.setParallelism(11);
     
      v2.connectNewDataSetAsInput(v1, DistributionPattern.POINTWISE);
      v5.connectNewDataSetAsInput(v4, DistributionPattern.POINTWISE);
     
      SlotSharingGroup jg1 = new SlotSharingGroup();
      v2.setSlotSharingGroup(jg1);
      v3.setSlotSharingGroup(jg1);
     
      SlotSharingGroup jg2 = new SlotSharingGroup();
      v4.setSlotSharingGroup(jg2);
      v5.setSlotSharingGroup(jg2);
     
      List<AbstractJobVertex> vertices = new ArrayList<AbstractJobVertex>(Arrays.asList(v1, v2, v3, v4, v5));
     
      ExecutionGraph eg = new ExecutionGraph(new JobID(), "test job", new Configuration());
      eg.attachJobGraph(vertices);
     
      // verify that the vertices are all in the same slot sharing group
      SlotSharingGroup group1 = null;
      SlotSharingGroup group2 = null;
     
      // verify that v1 tasks have no slot sharing group
      assertNull(eg.getJobVertex(v1.getID()).getSlotSharingGroup());
     
      // v2 and v3 are shared
      group1 = eg.getJobVertex(v2.getID()).getSlotSharingGroup();
      assertNotNull(group1);
      assertEquals(group1, eg.getJobVertex(v3.getID()).getSlotSharingGroup());
     
      assertEquals(2, group1.getJobVertexIds().size());
      assertTrue(group1.getJobVertexIds().contains(v2.getID()));
      assertTrue(group1.getJobVertexIds().contains(v3.getID()));
     
      // v4 and v5 are shared
      group2 = eg.getJobVertex(v4.getID()).getSlotSharingGroup();
      assertNotNull(group2);
      assertEquals(group2, eg.getJobVertex(v5.getID()).getSlotSharingGroup());
     
      assertEquals(2, group1.getJobVertexIds().size());
      assertTrue(group2.getJobVertexIds().contains(v4.getID()));
      assertTrue(group2.getJobVertexIds().contains(v5.getID()));
    }
    catch (Exception e) {
      e.printStackTrace();
      fail(e.getMessage());
    }
View Full Code Here
TOP

Related Classes of org.apache.flink.runtime.jobgraph.AbstractJobVertex

  • org.apache.flink.compiler.plantranslate.NepheleJobGraphGenerator
  • org.apache.flink.configuration.Configuration
  • org.apache.flink.runtime.executiongraph.AllVerticesIteratorTest
  • org.apache.flink.runtime.executiongraph.ExecutionGraphConstructionTest
  • org.apache.flink.runtime.executiongraph.ExecutionGraphDeploymentTest
  • org.apache.flink.runtime.executiongraph.ExecutionGraphRestartTest
  • org.apache.flink.runtime.executiongraph.ExecutionGraphTestUtils
  • org.apache.flink.runtime.executiongraph.ExecutionStateProgressTest
  • org.apache.flink.runtime.executiongraph.PointwisePatternTest
  • org.apache.flink.runtime.executiongraph.TaskManagerLossFailsTasksTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.