Examples of RMState

org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState

Examples of org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState

        try {
          rmStore.checkVersion();
          if (rmContext.isWorkPreservingRecoveryEnabled()) {
            rmContext.setEpoch(rmStore.getAndIncrementEpoch());
          }
          RMState state = rmStore.loadState();
          recover(state);
        } catch (Exception e) {
          // the Exception from loadState() needs to be handled for
          // HA and we need to give up master status if we got fenced
          LOG.error("Failed to load/recover state", e);

View Full Code Here

Examples of org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState

    assertAppState(RMAppState.KILLED, application);
  }
  
  @Test(timeout = 30000)
  public void testAppsRecoveringStates() throws Exception {
    RMState state = new RMState();
    Map<ApplicationId, ApplicationState> applicationState =
        state.getApplicationState();
    createRMStateForApplications(applicationState, RMAppState.FINISHED);
    createRMStateForApplications(applicationState, RMAppState.KILLED);
    createRMStateForApplications(applicationState, RMAppState.FAILED);
    for (ApplicationState appState : applicationState.values()) {
      testRecoverApplication(appState, state);

View Full Code Here

Examples of org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState

    modifyAppState();


    // load state
    store = stateStoreHelper.getRMStateStore();
    store.setRMDispatcher(dispatcher);
    RMState state = store.loadState();
    Map<ApplicationId, ApplicationState> rmAppState =
        state.getApplicationState();


    ApplicationState appState = rmAppState.get(appId1);
    // app is loaded
    assertNotNull(appState);
    // app is loaded correctly
    assertEquals(submitTime, appState.getSubmitTime());
    assertEquals(startTime, appState.getStartTime());
    // submission context is loaded correctly
    assertEquals(appId1,
                 appState.getApplicationSubmissionContext().getApplicationId());
    ApplicationAttemptState attemptState = appState.getAttempt(attemptId1);
    // attempt1 is loaded correctly
    assertNotNull(attemptState);
    assertEquals(attemptId1, attemptState.getAttemptId());
    assertEquals(-1000, attemptState.getAMContainerExitStatus());
    // attempt1 container is loaded correctly
    assertEquals(containerId1, attemptState.getMasterContainer().getId());
    // attempt1 applicationToken is loaded correctly
    HashSet<Token<?>> savedTokens = new HashSet<Token<?>>();
    savedTokens.addAll(attemptState.getAppAttemptCredentials().getAllTokens());
    assertEquals(attemptTokenSet1, savedTokens);
    // attempt1 client token master key is loaded correctly
    assertArrayEquals(clientTokenKey1.getEncoded(),
        attemptState.getAppAttemptCredentials()
        .getSecretKey(RMStateStore.AM_CLIENT_TOKEN_MASTER_KEY_NAME));


    attemptState = appState.getAttempt(attemptId2);
    // attempt2 is loaded correctly
    assertNotNull(attemptState);
    assertEquals(attemptId2, attemptState.getAttemptId());
    // attempt2 container is loaded correctly
    assertEquals(containerId2, attemptState.getMasterContainer().getId());
    // attempt2 applicationToken is loaded correctly
    savedTokens.clear();
    savedTokens.addAll(attemptState.getAppAttemptCredentials().getAllTokens());
    assertEquals(attemptTokenSet2, savedTokens);
    // attempt2 client token master key is loaded correctly
    assertArrayEquals(clientTokenKey2.getEncoded(),
        attemptState.getAppAttemptCredentials()
        .getSecretKey(RMStateStore.AM_CLIENT_TOKEN_MASTER_KEY_NAME));


    //******* update application/attempt state *******//
    ApplicationState appState2 =
        new ApplicationState(appState.submitTime, appState.startTime,
          appState.context, appState.user, RMAppState.FINISHED,
          "appDiagnostics", 1234);
    appState2.attempts.putAll(appState.attempts);
    store.updateApplicationState(appState2);


    ApplicationAttemptState oldAttemptState = attemptState;
    ApplicationAttemptState newAttemptState =
        new ApplicationAttemptState(oldAttemptState.getAttemptId(),
          oldAttemptState.getMasterContainer(),
          oldAttemptState.getAppAttemptCredentials(),
          oldAttemptState.getStartTime(), RMAppAttemptState.FINISHED,
          "myTrackingUrl", "attemptDiagnostics",
          FinalApplicationStatus.SUCCEEDED, 100);
    store.updateApplicationAttemptState(newAttemptState);


    // test updating the state of an app/attempt whose initial state was not
    // saved.
    ApplicationId dummyAppId = ApplicationId.newInstance(1234, 10);
    ApplicationSubmissionContext dummyContext =
        new ApplicationSubmissionContextPBImpl();
    dummyContext.setApplicationId(dummyAppId);
    ApplicationState dummyApp =
        new ApplicationState(appState.submitTime, appState.startTime,
          dummyContext, appState.user, RMAppState.FINISHED, "appDiagnostics",
          1234);
    store.updateApplicationState(dummyApp);


    ApplicationAttemptId dummyAttemptId =
        ApplicationAttemptId.newInstance(dummyAppId, 6);
    ApplicationAttemptState dummyAttempt =
        new ApplicationAttemptState(dummyAttemptId,
          oldAttemptState.getMasterContainer(),
          oldAttemptState.getAppAttemptCredentials(),
          oldAttemptState.getStartTime(), RMAppAttemptState.FINISHED,
          "myTrackingUrl", "attemptDiagnostics",
          FinalApplicationStatus.SUCCEEDED, 111);
    store.updateApplicationAttemptState(dummyAttempt);


    // let things settle down
    Thread.sleep(1000);
    store.close();


    // check updated application state.
    store = stateStoreHelper.getRMStateStore();
    store.setRMDispatcher(dispatcher);
    RMState newRMState = store.loadState();
    Map<ApplicationId, ApplicationState> newRMAppState =
        newRMState.getApplicationState();
    assertNotNull(newRMAppState.get(dummyApp.getAppId()));
    ApplicationState updatedAppState = newRMAppState.get(appId1);
    assertEquals(appState.getAppId(),updatedAppState.getAppId());
    assertEquals(appState.getSubmitTime(), updatedAppState.getSubmitTime());
    assertEquals(appState.getStartTime(), updatedAppState.getStartTime());

View Full Code Here

Examples of org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState

    // need events to move to further states.
    rmStore.start();


    if(recoveryEnabled) {
      try {
        RMState state = rmStore.loadState();
        recover(state);
      } catch (Exception e) {
        // the Exception from loadState() needs to be handled for 
        // HA and we need to give up master status if we got fenced
        LOG.error("Failed to load/recover state", e);

View Full Code Here

Examples of org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState

    conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
        YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);


    MemoryRMStateStore memStore = new MemoryRMStateStore();
    memStore.init(conf);
    RMState rmState = memStore.getState();
    Map<ApplicationId, ApplicationState> rmAppState = 
                                                  rmState.getApplicationState();
    
    
    // PHASE 1: create state in an RM
    
    // start RM

View Full Code Here

Examples of org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState

    conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
        YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);


    MemoryRMStateStore memStore = new MemoryRMStateStore();
    memStore.init(conf);
    RMState rmState = memStore.getState();


    Map<ApplicationId, ApplicationState> rmAppState =
        rmState.getApplicationState();  
    MockRM rm1 = new MockRM(conf, memStore);
    rm1.start();
    MockNM nm1 =
        new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService());
    nm1.registerNode();

View Full Code Here

Examples of org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState

        "kerberos");
    UserGroupInformation.setConfiguration(conf);


    MemoryRMStateStore memStore = new MemoryRMStateStore();
    memStore.init(conf);
    RMState rmState = memStore.getState();


    Map<ApplicationId, ApplicationState> rmAppState =
        rmState.getApplicationState();
    MockRM rm1 = new TestSecurityMockRM(conf, memStore);
    rm1.start();


    HashSet<Token<RMDelegationTokenIdentifier>> tokenSet =
        new HashSet<Token<RMDelegationTokenIdentifier>>();

View Full Code Here

Examples of org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState

      "kerberos");
    UserGroupInformation.setConfiguration(conf);


    MemoryRMStateStore memStore = new MemoryRMStateStore();
    memStore.init(conf);
    RMState rmState = memStore.getState();


    Map<ApplicationId, ApplicationState> rmAppState =
        rmState.getApplicationState();
    MockRM rm1 = new TestSecurityMockRM(conf, memStore);
    rm1.start();
    MockNM nm1 =
        new MockNM("0.0.0.0:4321", 15120, rm1.getResourceTrackerService());
    nm1.registerNode();

View Full Code Here

Examples of org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState

        "kerberos");
    conf.set(YarnConfiguration.RM_ADDRESS, "localhost:8032");
    UserGroupInformation.setConfiguration(conf);
    MemoryRMStateStore memStore = new MemoryRMStateStore();
    memStore.init(conf);
    RMState rmState = memStore.getState();


    Map<ApplicationId, ApplicationState> rmAppState =
        rmState.getApplicationState();
    Map<RMDelegationTokenIdentifier, Long> rmDTState =
        rmState.getRMDTSecretManagerState().getTokenState();
    Set<DelegationKey> rmDTMasterKeyState =
        rmState.getRMDTSecretManagerState().getMasterKeyState();


    MockRM rm1 = new TestSecurityMockRM(conf, memStore);


    rm1.start();


    // create an empty credential
    Credentials ts = new Credentials();


    // request a token and add into credential
    GetDelegationTokenRequest request1 =
        GetDelegationTokenRequest.newInstance("renewer1");
    UserGroupInformation.getCurrentUser().setAuthenticationMethod(
        AuthMethod.KERBEROS);
    GetDelegationTokenResponse response1 =
        rm1.getClientRMService().getDelegationToken(request1);
    org.apache.hadoop.yarn.api.records.Token delegationToken1 =
        response1.getRMDelegationToken();
    Token<RMDelegationTokenIdentifier> token1 =
        ConverterUtils.convertFromYarn(delegationToken1, rmAddr);
    RMDelegationTokenIdentifier dtId1 = token1.decodeIdentifier();


    HashSet<RMDelegationTokenIdentifier> tokenIdentSet =
        new HashSet<RMDelegationTokenIdentifier>();
    ts.addToken(token1.getService(), token1);
    tokenIdentSet.add(dtId1);


    // submit an app with customized credential
    RMApp app = rm1.submitApp(200, "name", "user",
        new HashMap<ApplicationAccessType, String>(), false, "default", 1, ts);


    // assert app info is saved
    ApplicationState appState = rmAppState.get(app.getApplicationId());
    Assert.assertNotNull(appState);


    // assert all master keys are saved
    Set<DelegationKey> allKeysRM1 = rm1.getRMDTSecretManager().getAllMasterKeys();
    Assert.assertEquals(allKeysRM1, rmDTMasterKeyState);


    // assert all tokens are saved
    Map<RMDelegationTokenIdentifier, Long> allTokensRM1 =
        rm1.getRMDTSecretManager().getAllTokens();
    Assert.assertEquals(tokenIdentSet, allTokensRM1.keySet());
    Assert.assertEquals(allTokensRM1, rmDTState);
    
    // assert sequence number is saved
    Assert.assertEquals(
      rm1.getRMDTSecretManager().getLatestDTSequenceNumber(),
      rmState.getRMDTSecretManagerState().getDTSequenceNumber());


    // request one more token
    GetDelegationTokenRequest request2 =
        GetDelegationTokenRequest.newInstance("renewer2");
    GetDelegationTokenResponse response2 =

View Full Code Here

Examples of org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState

    Thread.sleep(1000);
    store.close();


    // load state
    store = stateStoreHelper.getRMStateStore();
    RMState state = store.loadState();
    Map<ApplicationId, ApplicationState> rmAppState = state.getApplicationState();


    ApplicationState appState = rmAppState.get(appId1);
    // app is loaded
    assertNotNull(appState);
    // app is loaded correctly

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.