Examples of org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.ApplicationState

Package org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore

Examples of org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.ApplicationState

org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.ApplicationState

    RMApp app2 = rm1.submitApp(200, "name", "user",
          new HashMap<ApplicationAccessType, String>(), false, "default", -1,
          null);


    // assert app1 info is saved
    ApplicationState appState = rmAppState.get(app1.getApplicationId());
    Assert.assertNotNull(appState);
    Assert.assertEquals(0, appState.getAttemptCount());
    Assert.assertEquals(appState.getApplicationSubmissionContext()
        .getApplicationId(), app1.getApplicationSubmissionContext()
        .getApplicationId());


    // Allocate the AM
    nm1.nodeHeartbeat(true);
    RMAppAttempt attempt = app1.getCurrentAppAttempt();
    ApplicationAttemptId attemptId1 = attempt.getAppAttemptId();
    rm1.waitForState(attemptId1, RMAppAttemptState.ALLOCATED);
    Assert.assertEquals(1, appState.getAttemptCount());
    ApplicationAttemptState attemptState = 
                                appState.getAttempt(attemptId1);
    Assert.assertNotNull(attemptState);
    Assert.assertEquals(BuilderUtils.newContainerId(attemptId1, 1), 
                        attemptState.getMasterContainer().getId());


    // Setting AMLivelinessMonitor interval to be 3 Secs.

View Full Code Here

    // submit an app with customized credential
    RMApp app = rm1.submitApp(200, "name", "user",
        new HashMap<ApplicationAccessType, String>(), false, "default", 1, ts);


    // assert app info is saved
    ApplicationState appState = rmAppState.get(app.getApplicationId());
    Assert.assertNotNull(appState);


    // assert delegation tokens exist in rm1 DelegationTokenRenewr
    Assert.assertEquals(tokenSet, rm1.getRMContext()
      .getDelegationTokenRenewer().getDelegationTokens());


    // assert delegation tokens are saved
    DataOutputBuffer dob = new DataOutputBuffer();
    ts.writeTokenStorageToStream(dob);
    ByteBuffer securityTokens =
        ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
    securityTokens.rewind();
    Assert.assertEquals(securityTokens, appState
      .getApplicationSubmissionContext().getAMContainerSpec()
      .getTokens());


    // start new RM
    MockRM rm2 = new TestSecurityMockRM(conf, memStore);

View Full Code Here

    RMApp app1 =
        rm1.submitApp(200, "name", "user",
          new HashMap<ApplicationAccessType, String>(), "default");


    // assert app info is saved
    ApplicationState appState = rmAppState.get(app1.getApplicationId());
    Assert.assertNotNull(appState);


    // Allocate the AM
    nm1.nodeHeartbeat(true);
    RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
    ApplicationAttemptId attemptId1 = attempt1.getAppAttemptId();
    rm1.waitForState(attemptId1, RMAppAttemptState.ALLOCATED);


    // assert attempt info is saved
    ApplicationAttemptState attemptState = appState.getAttempt(attemptId1);
    Assert.assertNotNull(attemptState);
    Assert.assertEquals(BuilderUtils.newContainerId(attemptId1, 1),
      attemptState.getMasterContainer().getId());


    // the appToken and clientTokenMasterKey that are generated when

View Full Code Here

    // submit an app with customized credential
    RMApp app = rm1.submitApp(200, "name", "user",
        new HashMap<ApplicationAccessType, String>(), false, "default", 1, ts);


    // assert app info is saved
    ApplicationState appState = rmAppState.get(app.getApplicationId());
    Assert.assertNotNull(appState);


    // assert all master keys are saved
    Set<DelegationKey> allKeysRM1 = rm1.getRMDTSecretManager().getAllMasterKeys();
    Assert.assertEquals(allKeysRM1, rmDTMasterKeyState);

View Full Code Here

    rm1.stop();


    // Assert app info is still saved even if stop is called with pending saving
    // request on dispatcher.
    for (RMApp app : appList) {
      ApplicationState appState = rmAppState.get(app.getApplicationId());
      Assert.assertNotNull(appState);
      Assert.assertEquals(0, appState.getAttemptCount());
      Assert.assertEquals(appState.getApplicationSubmissionContext()
        .getApplicationId(), app.getApplicationSubmissionContext()
        .getApplicationId());
    }
    Assert.assertTrue(rmAppState.size() == NUM_APPS);
  }

View Full Code Here

    }
  }
  
  @Override
  public void recover(RMState state) throws Exception{
    ApplicationState appState = state.getApplicationState().get(getApplicationId());
    this.recoveredFinalState = appState.getState();
    LOG.info("Recovering app: " + getApplicationId() + " with " + 
        + appState.getAttemptCount() + " attempts and final state = " + this.recoveredFinalState );
    this.diagnostics.append(appState.getDiagnostics());
    this.storedFinishTime = appState.getFinishTime();
    this.startTime = appState.getStartTime();


    for(int i=0; i<appState.getAttemptCount(); ++i) {
      // create attempt
      createNewAttempt(false);
      ((RMAppAttemptImpl)this.currentAttempt).recover(state);
    }
  }

View Full Code Here

      diags = getAppKilledDiagnostics();
      break;
    default:
      break;
    }
    ApplicationState appState =
        new ApplicationState(this.submitTime, this.startTime,
          this.submissionContext, this.user, stateToBeStored, diags,
          this.storedFinishTime);
    this.rmContext.getStateStore().updateApplicationState(appState);
  }

View Full Code Here

    }
  }


  @Override
  public void recover(RMState state) throws Exception {
    ApplicationState appState =
        state.getApplicationState().get(getAppAttemptId().getApplicationId());
    ApplicationAttemptState attemptState =
        appState.getAttempt(getAppAttemptId());
    assert attemptState != null;
    LOG.info("Recovering attempt: " + getAppAttemptId() + " with final state: "
        + attemptState.getState());
    diagnostics.append("Attempt recovered after RM restart");
    diagnostics.append(attemptState.getDiagnostics());

View Full Code Here

    store.setRMDispatcher(dispatcher);
    RMState state = store.loadState();
    Map<ApplicationId, ApplicationState> rmAppState =
        state.getApplicationState();


    ApplicationState appState = rmAppState.get(appId1);
    // app is loaded
    assertNotNull(appState);
    // app is loaded correctly
    assertEquals(submitTime, appState.getSubmitTime());
    assertEquals(startTime, appState.getStartTime());
    // submission context is loaded correctly
    assertEquals(appId1,
                 appState.getApplicationSubmissionContext().getApplicationId());
    ApplicationAttemptState attemptState = appState.getAttempt(attemptId1);
    // attempt1 is loaded correctly
    assertNotNull(attemptState);
    assertEquals(attemptId1, attemptState.getAttemptId());
    // attempt1 container is loaded correctly
    assertEquals(containerId1, attemptState.getMasterContainer().getId());
    // attempt1 applicationToken is loaded correctly
    HashSet<Token<?>> savedTokens = new HashSet<Token<?>>();
    savedTokens.addAll(attemptState.getAppAttemptCredentials().getAllTokens());
    assertEquals(attemptTokenSet1, savedTokens);
    // attempt1 client token master key is loaded correctly
    assertArrayEquals(clientTokenKey1.getEncoded(),
        attemptState.getAppAttemptCredentials()
        .getSecretKey(RMStateStore.AM_CLIENT_TOKEN_MASTER_KEY_NAME));


    attemptState = appState.getAttempt(attemptId2);
    // attempt2 is loaded correctly
    assertNotNull(attemptState);
    assertEquals(attemptId2, attemptState.getAttemptId());
    // attempt2 container is loaded correctly
    assertEquals(containerId2, attemptState.getMasterContainer().getId());
    // attempt2 applicationToken is loaded correctly
    savedTokens.clear();
    savedTokens.addAll(attemptState.getAppAttemptCredentials().getAllTokens());
    assertEquals(attemptTokenSet2, savedTokens);
    // attempt2 client token master key is loaded correctly
    assertArrayEquals(clientTokenKey2.getEncoded(),
        attemptState.getAppAttemptCredentials()
        .getSecretKey(RMStateStore.AM_CLIENT_TOKEN_MASTER_KEY_NAME));


    //******* update application/attempt state *******//
    ApplicationState appState2 =
        new ApplicationState(appState.submitTime, appState.startTime,
          appState.context, appState.user, RMAppState.FINISHED,
          "appDiagnostics", 1234);
    appState2.attempts.putAll(appState.attempts);
    store.updateApplicationState(appState2);


    ApplicationAttemptState oldAttemptState = attemptState;
    ApplicationAttemptState newAttemptState =
        new ApplicationAttemptState(oldAttemptState.getAttemptId(),
          oldAttemptState.getMasterContainer(),
          oldAttemptState.getAppAttemptCredentials(),
          oldAttemptState.getStartTime(), RMAppAttemptState.FINISHED,
          "myTrackingUrl", "attemptDiagnostics",
          FinalApplicationStatus.SUCCEEDED);
    store.updateApplicationAttemptState(newAttemptState);


    // test updating the state of an app/attempt whose initial state was not
    // saved.
    ApplicationId dummyAppId = ApplicationId.newInstance(1234, 10);
    ApplicationSubmissionContext dummyContext =
        new ApplicationSubmissionContextPBImpl();
    dummyContext.setApplicationId(dummyAppId);
    ApplicationState dummyApp =
        new ApplicationState(appState.submitTime, appState.startTime,
          dummyContext, appState.user, RMAppState.FINISHED, "appDiagnostics",
          1234);
    store.updateApplicationState(dummyApp);


    ApplicationAttemptId dummyAttemptId =
        ApplicationAttemptId.newInstance(dummyAppId, 6);
    ApplicationAttemptState dummyAttempt =
        new ApplicationAttemptState(dummyAttemptId,
          oldAttemptState.getMasterContainer(),
          oldAttemptState.getAppAttemptCredentials(),
          oldAttemptState.getStartTime(), RMAppAttemptState.FINISHED,
          "myTrackingUrl", "attemptDiagnostics",
          FinalApplicationStatus.SUCCEEDED);
    store.updateApplicationAttemptState(dummyAttempt);


    // let things settle down
    Thread.sleep(1000);
    store.close();


    // check updated application state.
    store = stateStoreHelper.getRMStateStore();
    store.setRMDispatcher(dispatcher);
    RMState newRMState = store.loadState();
    Map<ApplicationId, ApplicationState> newRMAppState =
        newRMState.getApplicationState();
    assertNotNull(newRMAppState.get(dummyApp.getAppId()));
    ApplicationState updatedAppState = newRMAppState.get(appId1);
    assertEquals(appState.getAppId(),updatedAppState.getAppId());
    assertEquals(appState.getSubmitTime(), updatedAppState.getSubmitTime());
    assertEquals(appState.getStartTime(), updatedAppState.getStartTime());
    assertEquals(appState.getUser(), updatedAppState.getUser());
    // new app state fields
    assertEquals( RMAppState.FINISHED, updatedAppState.getState());
    assertEquals("appDiagnostics", updatedAppState.getDiagnostics());
    assertEquals(1234, updatedAppState.getFinishTime());


    // check updated attempt state
    assertNotNull(newRMAppState.get(dummyApp.getAppId()).getAttempt(
      dummyAttemptId));
    ApplicationAttemptState updatedAttemptState =
        updatedAppState.getAttempt(newAttemptState.getAttemptId());
    assertEquals(oldAttemptState.getAttemptId(),
      updatedAttemptState.getAttemptId());
    assertEquals(containerId2, updatedAttemptState.getMasterContainer().getId());
    assertArrayEquals(clientTokenKey2.getEncoded(),
      updatedAttemptState.getAppAttemptCredentials().getSecretKey(

View Full Code Here

    }
  }


  @Override
  public void recover(RMState state) throws Exception {
    ApplicationState appState =
        state.getApplicationState().get(getAppAttemptId().getApplicationId());
    ApplicationAttemptState attemptState =
        appState.getAttempt(getAppAttemptId());
    assert attemptState != null;
    LOG.info("Recovering attempt: " + getAppAttemptId() + " with final state: "
        + attemptState.getState());
    diagnostics.append("Attempt recovered after RM restart");
    diagnostics.append(attemptState.getDiagnostics());

View Full Code Here

0 1 2 3 4 5 6 7 8

TOP

Related Classes of org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.ApplicationState

org.apache.hadoop.yarn.server.resourcemanager.applicationsmanager.TestAMRestart

org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStoreTestBase

org.apache.hadoop.yarn.server.resourcemanager.recovery.TestRMStateStore

org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl

org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl

org.apache.hadoop.yarn.server.resourcemanager.rmapp.TestRMAppTransitions

org.apache.hadoop.yarn.server.resourcemanager.TestRMRestart

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.