Package org.apache.flink.runtime.io.network.bufferprovider

Examples of org.apache.flink.runtime.io.network.bufferprovider.GlobalBufferPool

84
85
86
87
88
89
90
91
92
93
94
    this.channelLookupService = channelLookupService;
    this.connectionInfo = connectionInfo;

    try {
      this.globalBufferPool = new GlobalBufferPool(numNetworkBuffers, networkBufferSize);
    } catch (Throwable e) {
      throw new IOException("Failed to instantiate GlobalBufferPool.", e);
    }

    this.networkConnectionManager = networkConnectionManager;
View Full Code Here
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
     
      final JobGraph jobGraph = new JobGraph("Pointwise Job", sender, receiver);
     
      final JobManager jm = startJobManager(NUM_TASKS);
     
      final GlobalBufferPool bp = ((LocalInstanceManager) jm.getInstanceManager())
          .getTaskManagers()[0].getChannelManager().getGlobalBufferPool();
     
      try {
        JobSubmissionResult result = jm.submitJob(jobGraph);

        if (result.getReturnCode() != AbstractJobResult.ReturnCode.SUCCESS) {
          System.out.println(result.getDescription());
        }
        assertEquals(AbstractJobResult.ReturnCode.SUCCESS, result.getReturnCode());
       
        // monitor the execution
        ExecutionGraph eg = jm.getCurrentJobs().get(jobGraph.getJobID());
       
        if (eg != null) {
          eg.waitForJobEnd();
          assertEquals(JobStatus.FINISHED, eg.getState());
        }
        else {
          // already done, that was fast;
        }
       
        // make sure that in any case, the network buffers are all returned
        waitForTaskThreadsToBeTerminated();
        assertEquals(bp.numBuffers(), bp.numAvailableBuffers());
      }
      finally {
        jm.shutdown();
      }
    }
View Full Code Here
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
      final JobManager jm = startJobManager(2, NUM_TASKS / 2);
     
      final TaskManager tm1 = ((LocalInstanceManager) jm.getInstanceManager()).getTaskManagers()[0];
      final TaskManager tm2 = ((LocalInstanceManager) jm.getInstanceManager()).getTaskManagers()[1];
     
      final GlobalBufferPool bp1 = tm1.getChannelManager().getGlobalBufferPool();
      final GlobalBufferPool bp2 = tm2.getChannelManager().getGlobalBufferPool();
     
      try {
        JobSubmissionResult result = jm.submitJob(jobGraph);

        if (result.getReturnCode() != AbstractJobResult.ReturnCode.SUCCESS) {
          System.out.println(result.getDescription());
        }
        assertEquals(AbstractJobResult.ReturnCode.SUCCESS, result.getReturnCode());
       
        ExecutionGraph eg = jm.getCurrentJobs().get(jobGraph.getJobID());
       
        // wait until everyone has settled in
        long deadline = System.currentTimeMillis() + 2000;
        while (System.currentTimeMillis() < deadline) {
         
          boolean allrunning = true;
          for (ExecutionVertex v : eg.getJobVertex(receiver.getID()).getTaskVertices()) {
            if (v.getCurrentExecutionAttempt().getState() != ExecutionState.RUNNING) {
              allrunning = false;
              break;
            }
          }
         
          if (allrunning) {
            break;
          }
          Thread.sleep(200);
        }
       
        // kill one task manager
        TaskManager tm = ((LocalInstanceManager) jm.getInstanceManager()).getTaskManagers()[0];
        tm.shutdown();
       
        eg.waitForJobEnd();
       
        // make sure that in any case, the network buffers are all returned
        waitForTaskThreadsToBeTerminated();
        assertTrue(bp1.isDestroyed() || bp1.numBuffers() == bp1.numAvailableBuffers());
        assertTrue(bp2.isDestroyed() || bp2.numBuffers() == bp2.numAvailableBuffers());
      }
      finally {
        jm.shutdown();
      }
    }
View Full Code Here
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
     
      final JobGraph jobGraph = new JobGraph("Test Job", vertex);
     
      final JobManager jm = startJobManager(1);
     
      final GlobalBufferPool bp = ((LocalInstanceManager) jm.getInstanceManager())
          .getTaskManagers()[0].getChannelManager().getGlobalBufferPool();
     
      try {
       
        assertEquals(1, jm.getTotalNumberOfRegisteredSlots());
       
        JobSubmissionResult result = jm.submitJob(jobGraph);
        assertEquals(AbstractJobResult.ReturnCode.ERROR, result.getReturnCode());
       
        // monitor the execution
        ExecutionGraph eg = jm.getCurrentJobs().get(jobGraph.getJobID());
       
        if (eg != null) {
         
          long deadline = System.currentTimeMillis() + 60*1000;
          boolean success = false;
         
          while (System.currentTimeMillis() < deadline) {
            JobStatus state = eg.getState();
            if (state == JobStatus.FINISHED) {
              success = true;
              break;
            }
            else if (state == JobStatus.FAILED || state == JobStatus.CANCELED) {
              break;
            }
            else {
              Thread.sleep(200);
            }
          }
         
          assertTrue("The job did not finish successfully.", success);
         
//          assertEquals(0, eg.getRegisteredExecutions().size());
        }
        else {
          // already done, that was fast;
        }
       
       
       
        // make sure that in any case, the network buffers are all returned
        waitForTaskThreadsToBeTerminated();
        assertEquals(bp.numBuffers(), bp.numAvailableBuffers());
      }
      finally {
        jm.shutdown();
      }
    }
View Full Code Here
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
     
      final JobGraph jobGraph = new JobGraph("Test Job", vertex);
     
      final JobManager jm = startJobManager(NUM_TASKS);
     
      final GlobalBufferPool bp = ((LocalInstanceManager) jm.getInstanceManager())
          .getTaskManagers()[0].getChannelManager().getGlobalBufferPool();
     
      try {
       
        assertEquals(NUM_TASKS, jm.getTotalNumberOfRegisteredSlots());
       
        JobSubmissionResult result = jm.submitJob(jobGraph);
       
        if (result.getReturnCode() != AbstractJobResult.ReturnCode.SUCCESS) {
          System.out.println(result.getDescription());
        }
        assertEquals(AbstractJobResult.ReturnCode.SUCCESS, result.getReturnCode());
       
        // monitor the execution
        ExecutionGraph eg = jm.getCurrentJobs().get(jobGraph.getJobID());
       
        if (eg != null) {
         
          long deadline = System.currentTimeMillis() + 60*1000;
          boolean success = false;
         
          while (System.currentTimeMillis() < deadline) {
            JobStatus state = eg.getState();
            if (state == JobStatus.FINISHED) {
              success = true;
              break;
            }
            else if (state == JobStatus.FAILED || state == JobStatus.CANCELED) {
              break;
            }
            else {
              Thread.sleep(200);
            }
          }
         
          assertTrue("The job did not finish successfully.", success);
         
//          assertEquals(0, eg.getRegisteredExecutions().size());
        }
        else {
          // already done, that was fast;
        }
       
        // make sure that in any case, the network buffers are all returned
        waitForTaskThreadsToBeTerminated();
        assertEquals(bp.numBuffers(), bp.numAvailableBuffers());
      }
      finally {
        jm.shutdown();
      }
    }
View Full Code Here
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
      final JobGraph jobGraph = new JobGraph("Test Job", vertex);
      jobGraph.setAllowQueuedScheduling(true);
     
      final JobManager jm = startJobManager(10);
     
      final GlobalBufferPool bp = ((LocalInstanceManager) jm.getInstanceManager())
          .getTaskManagers()[0].getChannelManager().getGlobalBufferPool();
     
      try {
        JobSubmissionResult result = jm.submitJob(jobGraph);
       
        if (result.getReturnCode() != AbstractJobResult.ReturnCode.SUCCESS) {
          System.out.println(result.getDescription());
        }
        assertEquals(AbstractJobResult.ReturnCode.SUCCESS, result.getReturnCode());
       
        // monitor the execution
        ExecutionGraph eg = jm.getCurrentJobs().get(jobGraph.getJobID());
       
        if (eg != null) {
          eg.waitForJobEnd();
          assertEquals(JobStatus.FINISHED, eg.getState());
         
//          assertEquals(0, eg.getRegisteredExecutions().size());
        }
        else {
          // already done, that was fast;
        }
       
        // make sure that in any case, the network buffers are all returned
        waitForTaskThreadsToBeTerminated();
        assertEquals(bp.numBuffers(), bp.numAvailableBuffers());
      }
      finally {
        jm.shutdown();
      }
    }
View Full Code Here
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
     
      final JobGraph jobGraph = new JobGraph("Pointwise Job", sender, receiver);
     
      final JobManager jm = startJobManager(2 * NUM_TASKS);
     
      final GlobalBufferPool bp = ((LocalInstanceManager) jm.getInstanceManager())
          .getTaskManagers()[0].getChannelManager().getGlobalBufferPool();
     
      try {
        JobSubmissionResult result = jm.submitJob(jobGraph);

        if (result.getReturnCode() != AbstractJobResult.ReturnCode.SUCCESS) {
          System.out.println(result.getDescription());
        }
        assertEquals(AbstractJobResult.ReturnCode.SUCCESS, result.getReturnCode());
       
        // monitor the execution
        ExecutionGraph eg = jm.getCurrentJobs().get(jobGraph.getJobID());
       
        if (eg != null) {
          eg.waitForJobEnd();
          assertEquals(JobStatus.FINISHED, eg.getState());
         
//          assertEquals(0, eg.getRegisteredExecutions().size());
        }
        else {
          // already done, that was fast;
        }
       
        // make sure that in any case, the network buffers are all returned
        waitForTaskThreadsToBeTerminated();
        assertEquals(bp.numBuffers(), bp.numAvailableBuffers());
      }
      finally {
        jm.shutdown();
      }
    }
View Full Code Here
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
     
      final JobGraph jobGraph = new JobGraph("Bipartite Job", sender, receiver);
     
      final JobManager jm = startJobManager(2 * NUM_TASKS);
     
      final GlobalBufferPool bp = ((LocalInstanceManager) jm.getInstanceManager())
          .getTaskManagers()[0].getChannelManager().getGlobalBufferPool();
     
      try {
        JobSubmissionResult result = jm.submitJob(jobGraph);

        if (result.getReturnCode() != AbstractJobResult.ReturnCode.SUCCESS) {
          System.out.println(result.getDescription());
        }
        assertEquals(AbstractJobResult.ReturnCode.SUCCESS, result.getReturnCode());
       
        // monitor the execution
        ExecutionGraph eg = jm.getCurrentJobs().get(jobGraph.getJobID());
       
        if (eg != null) {
          eg.waitForJobEnd();
          assertEquals(JobStatus.FINISHED, eg.getState());
         
//          assertEquals(0, eg.getRegisteredExecutions().size());
        }
        else {
          // already done, that was fast;
        }
       
        // make sure that in any case, the network buffers are all returned
        waitForTaskThreadsToBeTerminated();
        assertEquals(bp.numBuffers(), bp.numAvailableBuffers());
      }
      finally {
        jm.shutdown();
      }
    }
View Full Code Here
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
     
      final JobGraph jobGraph = new JobGraph("Bipartite Job", sender1, receiver, sender2);
     
      final JobManager jm = startJobManager(6*NUM_TASKS);
     
      final GlobalBufferPool bp = ((LocalInstanceManager) jm.getInstanceManager())
          .getTaskManagers()[0].getChannelManager().getGlobalBufferPool();
     
      try {
        JobSubmissionResult result = jm.submitJob(jobGraph);

        if (result.getReturnCode() != AbstractJobResult.ReturnCode.SUCCESS) {
          System.out.println(result.getDescription());
        }
        assertEquals(AbstractJobResult.ReturnCode.SUCCESS, result.getReturnCode());
       
        // monitor the execution
        ExecutionGraph eg = jm.getCurrentJobs().get(jobGraph.getJobID());
       
        if (eg != null) {
          eg.waitForJobEnd();
          assertEquals(JobStatus.FAILED, eg.getState());
         
//          assertEquals(0, eg.getRegisteredExecutions().size());
        }
        else {
          // already done, that was fast;
        }
       
        // make sure that in any case, the network buffers are all returned
        waitForTaskThreadsToBeTerminated();
        assertEquals(bp.numBuffers(), bp.numAvailableBuffers());
      }
      finally {
        jm.shutdown();
      }
    }
View Full Code Here
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
     
      final JobGraph jobGraph = new JobGraph("Bipartite Job", sender1, receiver, sender2);
     
      JobManager jm = startJobManager(6 * NUM_TASKS);
     
      final GlobalBufferPool bp = ((LocalInstanceManager) jm.getInstanceManager())
                .getTaskManagers()[0].getChannelManager().getGlobalBufferPool();
     
      try {
        JobSubmissionResult result = jm.submitJob(jobGraph);

        if (result.getReturnCode() != AbstractJobResult.ReturnCode.SUCCESS) {
          System.out.println(result.getDescription());
        }
        assertEquals(AbstractJobResult.ReturnCode.SUCCESS, result.getReturnCode());
       
        // monitor the execution
        ExecutionGraph eg = jm.getCurrentJobs().get(jobGraph.getJobID());
       
        if (eg != null) {
          eg.waitForJobEnd();
          assertEquals(JobStatus.FINISHED, eg.getState());
         
//          assertEquals(0, eg.getRegisteredExecutions().size());
        }
        else {
          // already done, that was fast;
        }
       
        // make sure that in any case, the network buffers are all returned
        waitForTaskThreadsToBeTerminated();
        assertEquals(bp.numBuffers(), bp.numAvailableBuffers());
      }
      finally {
        jm.shutdown();
      }
    }
View Full Code Here
TOP

Related Classes of org.apache.flink.runtime.io.network.bufferprovider.GlobalBufferPool

  • org.apache.flink.core.memory.MemorySegment
  • org.apache.flink.runtime.io.network.ChannelManager
  • org.apache.flink.runtime.jobmanager.CoLocationConstraintITCase
  • org.apache.flink.runtime.jobmanager.JobManagerITCase
  • org.apache.flink.runtime.jobmanager.RecoveryITCase
  • org.apache.flink.runtime.jobmanager.SlotSharingITCase
  • org.apache.flink.runtime.jobmanager.TaskManagerFailsITCase
  • org.apache.flink.runtime.jobmanager.TaskManagerFailsWithSlotSharingITCase
  • org.apache.flink.runtime.taskmanager.TaskManagerTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.