Examples of org.apache.flink.runtime.jobmanager.JobManager

Package org.apache.flink.runtime.jobmanager

Examples of org.apache.flink.runtime.jobmanager.JobManager

org.apache.flink.runtime.jobmanager.JobManager
The JobManager is the master that coordinates the distributed execution. It receives jobs from clients, tracks the distributed execution.


public class TaskManagerTest {
  
  @Test
  public void testSetupTaskManager() {
    JobManager jobManager = null;
    TaskManager tm = null;
    try {
      jobManager = getJobManagerMockBase();
      tm = createTaskManager(jobManager);


      JobID jid = new JobID();
      JobVertexID vid = new JobVertexID();
      ExecutionAttemptID eid = new ExecutionAttemptID();
      
      TaskDeploymentDescriptor tdd = new TaskDeploymentDescriptor(jid, vid, eid, "TestTask", 2, 7,
          new Configuration(), new Configuration(), TestInvokableCorrect.class.getName(),
          Collections.<GateDeploymentDescriptor>emptyList(), 
          Collections.<GateDeploymentDescriptor>emptyList(),
          new ArrayList<BlobKey>(), 0);
      
      TaskOperationResult result = tm.submitTask(tdd);
      assertTrue(result.getDescription(), result.isSuccess());
      assertEquals(eid, result.getExecutionId());


      jobManager.shutdown();
      tm.shutdown();
    }
    catch (Exception e) {
      e.printStackTrace();
      fail(e.getMessage());
    }finally{
      if(jobManager != null){
        jobManager.shutdown();
      }


      if(tm != null){
        tm.shutdown();
      }

View Full Code Here

    }
  }
  
  @Test
  public void testJobSubmissionAndCanceling() {
    JobManager jobManager = null;
    TaskManager tm = null;
    try {
      jobManager = getJobManagerMockBase();
      tm = createTaskManager(jobManager);


      JobID jid1 = new JobID();
      JobID jid2 = new JobID();
      
      JobVertexID vid1 = new JobVertexID();
      JobVertexID vid2 = new JobVertexID();
      
      ExecutionAttemptID eid1 = new ExecutionAttemptID();
      ExecutionAttemptID eid2 = new ExecutionAttemptID();
      
      TaskDeploymentDescriptor tdd1 = new TaskDeploymentDescriptor(jid1, vid1, eid1, "TestTask1", 1, 5,
          new Configuration(), new Configuration(), TestInvokableBlockingCancelable.class.getName(),
          Collections.<GateDeploymentDescriptor>emptyList(), 
          Collections.<GateDeploymentDescriptor>emptyList(),
          new ArrayList<BlobKey>(), 0);
      
      TaskDeploymentDescriptor tdd2 = new TaskDeploymentDescriptor(jid2, vid2, eid2, "TestTask2", 2, 7,
          new Configuration(), new Configuration(), TestInvokableBlockingCancelable.class.getName(),
          Collections.<GateDeploymentDescriptor>emptyList(), 
          Collections.<GateDeploymentDescriptor>emptyList(),
          new ArrayList<BlobKey>(), 0);


      TaskOperationResult result1 = tm.submitTask(tdd1);
      TaskOperationResult result2 = tm.submitTask(tdd2);
      
      assertTrue(result1.getDescription(), result1.isSuccess());
      assertTrue(result2.getDescription(), result2.isSuccess());
      assertEquals(eid1, result1.getExecutionId());
      assertEquals(eid2, result2.getExecutionId());
      
      Map<ExecutionAttemptID, Task> tasks = tm.getAllRunningTasks();
      assertEquals(2, tasks.size());
      
      Task t1 = tasks.get(eid1);
      Task t2 = tasks.get(eid2);
      assertNotNull(t1);
      assertNotNull(t2);
      
      assertEquals(ExecutionState.RUNNING, t1.getExecutionState());
      assertEquals(ExecutionState.RUNNING, t2.getExecutionState());
      
      // cancel one task
      assertTrue(tm.cancelTask(eid1).isSuccess());
      t1.getEnvironment().getExecutingThread().join();
      assertEquals(ExecutionState.CANCELED, t1.getExecutionState());
      
      tasks = tm.getAllRunningTasks();
      assertEquals(1, tasks.size());
      
      // try to cancel a non existing task
      assertFalse(tm.cancelTask(eid1).isSuccess());
      
      // cancel the second task
      assertTrue(tm.cancelTask(eid2).isSuccess());
      t2.getEnvironment().getExecutingThread().join();
      assertEquals(ExecutionState.CANCELED, t2.getExecutionState());
      
      tasks = tm.getAllRunningTasks();
      assertEquals(0, tasks.size());
      
      assertNetworkResourcesReleased(tm);
    }
    catch (Exception e) {
      e.printStackTrace();
      fail(e.getMessage());
    }finally{
      if(jobManager != null){
        jobManager.shutdown();
      }


      if(tm != null){
        tm.shutdown();
      }

View Full Code Here

    }
  }
  
  @Test
  public void testGateChannelEdgeMismatch() {
    JobManager jobManager = null;
    TaskManager tm = null;
    try {
      jobManager = getJobManagerMockBase();
      tm = createTaskManager(jobManager);


      JobID jid = new JobID();;
      
      JobVertexID vid1 = new JobVertexID();
      JobVertexID vid2 = new JobVertexID();
      
      ExecutionAttemptID eid1 = new ExecutionAttemptID();
      ExecutionAttemptID eid2 = new ExecutionAttemptID();
      
      TaskDeploymentDescriptor tdd1 = new TaskDeploymentDescriptor(jid, vid1, eid1, "Sender", 0, 1,
          new Configuration(), new Configuration(), Sender.class.getName(),
          Collections.<GateDeploymentDescriptor>emptyList(),
          Collections.<GateDeploymentDescriptor>emptyList(),
          new ArrayList<BlobKey>(), 0);
      
      TaskDeploymentDescriptor tdd2 = new TaskDeploymentDescriptor(jid, vid2, eid2, "Receiver", 2, 7,
          new Configuration(), new Configuration(), Receiver.class.getName(),
          Collections.<GateDeploymentDescriptor>emptyList(),
          Collections.<GateDeploymentDescriptor>emptyList(),
          new ArrayList<BlobKey>(), 0);


      assertFalse(tm.submitTask(tdd1).isSuccess());
      assertFalse(tm.submitTask(tdd2).isSuccess());
      
      Map<ExecutionAttemptID, Task> tasks = tm.getAllRunningTasks();
      assertEquals(0, tasks.size());
      assertNetworkResourcesReleased(tm);
    }
    catch (Exception e) {
      e.printStackTrace();
      fail(e.getMessage());
    }finally{
      if(jobManager != null){
        jobManager.shutdown();
      }


      if(tm != null){
        tm.shutdown();
      }

View Full Code Here

    }
  }
  
  @Test
  public void testRunJobWithForwardChannel() {
    JobManager jobManager = null;
    TaskManager tm = null;


    try {
      JobID jid = new JobID();
      
      JobVertexID vid1 = new JobVertexID();
      JobVertexID vid2 = new JobVertexID();
      
      ExecutionAttemptID eid1 = new ExecutionAttemptID();
      ExecutionAttemptID eid2 = new ExecutionAttemptID();
      
      ChannelID senderId = new ChannelID();
      ChannelID receiverId = new ChannelID();
      
      jobManager = getJobManagerMockBase();
      when(jobManager.lookupConnectionInfo(Matchers.any(InstanceConnectionInfo.class), Matchers.eq(jid), Matchers.eq(senderId)))
        .thenReturn(ConnectionInfoLookupResponse.createReceiverFoundAndReady(receiverId));
      
      tm = createTaskManager(jobManager);
      
      ChannelDeploymentDescriptor cdd = new ChannelDeploymentDescriptor(senderId, receiverId);
      
      TaskDeploymentDescriptor tdd1 = new TaskDeploymentDescriptor(jid, vid1, eid1, "Sender", 0, 1,
          new Configuration(), new Configuration(), Sender.class.getName(),
          Collections.singletonList(new GateDeploymentDescriptor(Collections.singletonList(cdd))), 
          Collections.<GateDeploymentDescriptor>emptyList(),
          new ArrayList<BlobKey>(), 0);
      
      TaskDeploymentDescriptor tdd2 = new TaskDeploymentDescriptor(jid, vid2, eid2, "Receiver", 2, 7,
          new Configuration(), new Configuration(), Receiver.class.getName(),
          Collections.<GateDeploymentDescriptor>emptyList(),
          Collections.singletonList(new GateDeploymentDescriptor(Collections.singletonList(cdd))),
          new ArrayList<BlobKey>(), 0);


      // deploy sender before receiver, so the target is online when the sender requests the connection info
      TaskOperationResult result2 = tm.submitTask(tdd2);
      TaskOperationResult result1 = tm.submitTask(tdd1);


      assertTrue(result1.isSuccess());
      assertTrue(result2.isSuccess());
      assertEquals(eid1, result1.getExecutionId());
      assertEquals(eid2, result2.getExecutionId());
      
      Map<ExecutionAttemptID, Task> tasks = tm.getAllRunningTasks();
      
      Task t1 = tasks.get(eid1);
      Task t2 = tasks.get(eid2);
      
      // wait until the tasks are done. rare thread races may cause the tasks to be done before
      // we get to the check, so we need to guard the check
      if (t1 != null) {
        t1.getEnvironment().getExecutingThread().join();
        assertEquals(ExecutionState.FINISHED, t1.getExecutionState());
      }
      if (t2 != null) {
        t2.getEnvironment().getExecutingThread().join();
        assertEquals(ExecutionState.FINISHED, t2.getExecutionState());
      }
      
      tasks = tm.getAllRunningTasks();
      assertEquals(0, tasks.size());
      
      // make sure that the global buffer pool has all buffers back
      assertNetworkResourcesReleased(tm);
    }
    catch (Exception e) {
      e.printStackTrace();
      fail(e.getMessage());
    }
    finally {
      if (jobManager != null) {
        jobManager.shutdown();
      }


      if (tm != null) {
        tm.shutdown();
      }

View Full Code Here

    
    // this tests creates two tasks. the sender sends data, and fails to send the
    // state update back to the job manager
    // the second one blocks to be canceled


    JobManager jobManager = null;
    TaskManager tm = null;


    try {
      JobID jid = new JobID();
      
      JobVertexID vid1 = new JobVertexID();
      JobVertexID vid2 = new JobVertexID();
      
      ExecutionAttemptID eid1 = new ExecutionAttemptID();
      ExecutionAttemptID eid2 = new ExecutionAttemptID();
      
      ChannelID senderId = new ChannelID();
      ChannelID receiverId = new ChannelID();
      
      jobManager = getJobManagerMockBase();
      when(jobManager.updateTaskExecutionState(any(TaskExecutionState.class))).thenReturn(false);
      when(jobManager.lookupConnectionInfo(Matchers.any(InstanceConnectionInfo.class), Matchers.eq(jid), Matchers.eq(senderId)))
        .thenReturn(ConnectionInfoLookupResponse.createReceiverFoundAndReady(receiverId));
      
      tm = createTaskManager(jobManager);
      
      ChannelDeploymentDescriptor cdd = new ChannelDeploymentDescriptor(senderId, receiverId);
      
      TaskDeploymentDescriptor tdd1 = new TaskDeploymentDescriptor(jid, vid1, eid1, "Sender", 0, 1,
          new Configuration(), new Configuration(), Sender.class.getName(),
          Collections.singletonList(new GateDeploymentDescriptor(Collections.singletonList(cdd))), 
          Collections.<GateDeploymentDescriptor>emptyList(),
          new ArrayList<BlobKey>(), 0);
      
      TaskDeploymentDescriptor tdd2 = new TaskDeploymentDescriptor(jid, vid2, eid2, "Receiver", 2, 7,
          new Configuration(), new Configuration(), ReceiverBlocking.class.getName(),
          Collections.<GateDeploymentDescriptor>emptyList(),
          Collections.singletonList(new GateDeploymentDescriptor(Collections.singletonList(cdd))),
          new ArrayList<BlobKey>(), 0);
      
      // deploy sender before receiver, so the target is online when the sender requests the connection info
      TaskOperationResult result2 = tm.submitTask(tdd2);
      TaskOperationResult result1 = tm.submitTask(tdd1);
      
      assertTrue(result1.isSuccess());
      assertTrue(result2.isSuccess());
      
      Map<ExecutionAttemptID, Task> tasks = tm.getAllRunningTasks();
      
      Task t1 = tasks.get(eid1);
      Task t2 = tasks.get(eid2);
      
      // cancel task 2. task one should either fail, or be done
      tm.cancelTask(eid2);
      
      // wait until the task second task is canceled
      if (t2 != null) {
        t2.getEnvironment().getExecutingThread().join();
      }
      
      if (t1 != null) {
        if (t1.getExecutionState() == ExecutionState.RUNNING) {
          tm.cancelTask(eid1);
        }
        t1.getEnvironment().getExecutingThread().join();
      }
      
      // the task that failed to send the finished state 
      assertEquals(0, tasks.size());
      assertNetworkResourcesReleased(tm);
    }
    catch (Exception e) {
      e.printStackTrace();
      fail(e.getMessage());
    }finally{
      if(jobManager != null){
        jobManager.shutdown();
      }


      if(tm != null){
        tm.shutdown();
      }

View Full Code Here

  }
  
  // --------------------------------------------------------------------------------------------
  
  public static JobManager getJobManagerMockBase() throws Exception {
    JobManager jm = mock(JobManager.class);
    
    final InstanceID iid = new InstanceID();
    
    when(jm.registerTaskManager(Matchers.any(InstanceConnectionInfo.class), Matchers.any(HardwareDescription.class), Matchers.anyInt()))
      .thenReturn(iid);
    
    when(jm.sendHeartbeat(iid)).thenReturn(true);
    
    when(jm.updateTaskExecutionState(any(TaskExecutionState.class))).thenReturn(true);
    
    return jm;
  }

View Full Code Here

      cfg.addAll(additionalParams);
    }
    
    GlobalConfiguration.includeConfiguration(cfg);
    
    JobManager jm = new JobManager(ExecutionMode.LOCAL);
    
    // we need to wait until the taskmanager is registered
    // max time is 5 seconds
    long deadline = System.currentTimeMillis() + 5000;
    
    while (jm.getNumberOfSlotsAvailableToScheduler() < numTaskManagers * numSlotsPerTaskManager &&
        System.currentTimeMillis() < deadline)
    {
      Thread.sleep(10);
    }
    
    assertEquals(numTaskManagers * numSlotsPerTaskManager, jm.getNumberOfSlotsAvailableToScheduler());
    
    return jm;
  }

View Full Code Here

          t.join();
        }
      }


      // start the job manager
      jobManager = new JobManager(ExecutionMode.LOCAL);
  
      waitForJobManagerToBecomeReady(taskManagerNumSlots * numTaskManager);
    }
  }

View Full Code Here

      }
    }
  }
  
  public TaskManager[] getTaskManagers() {
    JobManager jm = this.jobManager;
    if (jm != null) {
      InstanceManager im = jm.getInstanceManager();
      if (im instanceof LocalInstanceManager) {
        return ((LocalInstanceManager) im).getTaskManagers();
      }
    }

View Full Code Here

TOP

Related Classes of org.apache.flink.runtime.jobmanager.JobManager

org.apache.commons.cli.CommandLine

org.apache.commons.cli.CommandLineParser

org.apache.commons.cli.GnuParser

org.apache.commons.cli.Option

org.apache.commons.cli.Options

org.apache.flink.client.minicluster.NepheleMiniCluster

org.apache.flink.configuration.Configuration

org.apache.flink.core.io.InputSplitAssigner

org.apache.flink.runtime.accumulators.AccumulatorEvent

org.apache.flink.runtime.blob.BlobServer

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.