Examples of YarnRPC


Examples of org.apache.hadoop.yarn.ipc.YarnRPC

    this.jhsDTSecretManager = jhsDTSecretManager;
  }

  protected void serviceStart() throws Exception {
    Configuration conf = getConfig();
    YarnRPC rpc = YarnRPC.create(conf);
    initializeWebApp(conf);
    InetSocketAddress address = conf.getSocketAddr(
        JHAdminConfig.MR_HISTORY_ADDRESS,
        JHAdminConfig.DEFAULT_MR_HISTORY_ADDRESS,
        JHAdminConfig.DEFAULT_MR_HISTORY_PORT);

    server =
        rpc.getServer(HSClientProtocol.class, protocolHandler, address,
            conf, jhsDTSecretManager,
            conf.getInt(JHAdminConfig.MR_HISTORY_CLIENT_THREAD_COUNT,
                JHAdminConfig.DEFAULT_MR_HISTORY_CLIENT_THREAD_COUNT));

    // Enable service authorization?
View Full Code Here

Examples of org.apache.hadoop.yarn.ipc.YarnRPC

  }

  MRClientProtocol instantiateAMProxy(final InetSocketAddress serviceAddr)
      throws IOException {
    LOG.trace("Connecting to ApplicationMaster at: " + serviceAddr);
    YarnRPC rpc = YarnRPC.create(conf);
    MRClientProtocol proxy =
         (MRClientProtocol) rpc.getProxy(MRClientProtocol.class,
            serviceAddr, conf);
    LOG.trace("Connected to ApplicationMaster at: " + serviceAddr);
    return proxy;
  }
View Full Code Here

Examples of org.apache.hadoop.yarn.ipc.YarnRPC

    LOG.info("Running test for authenticated user");

    ResourceManager resourceManager = yarnCluster.getResourceManager();

    final YarnRPC yarnRPC = YarnRPC.create(conf);

    // Submit an application
    ApplicationId appID = resourceManager.getClientRMService()
        .getNewApplication(Records.newRecord(GetNewApplicationRequest.class))
        .getApplicationId();
    AMRMProtocol scheduler = submitAndRegisterApplication(resourceManager,
        yarnRPC, appID);

    // Now request a container.
    final Container allocatedContainer = requestAndGetContainer(scheduler,
        appID);

    // Now talk to the NM for launching the container.
    final ContainerId containerID = allocatedContainer.getId();
    UserGroupInformation authenticatedUser = UserGroupInformation
        .createRemoteUser(containerID.toString());
    ContainerToken containerToken = allocatedContainer.getContainerToken();
    Token<ContainerTokenIdentifier> token = new Token<ContainerTokenIdentifier>(
        containerToken.getIdentifier().array(), containerToken.getPassword()
            .array(), new Text(containerToken.getKind()), new Text(
            containerToken.getService()));
    authenticatedUser.addToken(token);
    authenticatedUser.doAs(new PrivilegedExceptionAction<Void>() {
      @Override
      public Void run() throws Exception {
        ContainerManager client = (ContainerManager) yarnRPC.getProxy(
            ContainerManager.class, NetUtils
                .createSocketAddr(allocatedContainer.getNodeId().toString()),
            conf);
        LOG.info("Going to make a legal stopContainer() request");
        StopContainerRequest request = recordFactory
View Full Code Here

Examples of org.apache.hadoop.yarn.ipc.YarnRPC

    LOG.info("Running test for malice user");

    ResourceManager resourceManager = yarnCluster.getResourceManager();

    final YarnRPC yarnRPC = YarnRPC.create(conf);

    // Submit an application
    ApplicationId appID = resourceManager.getClientRMService()
        .getNewApplication(Records.newRecord(GetNewApplicationRequest.class))
        .getApplicationId();
    AMRMProtocol scheduler = submitAndRegisterApplication(resourceManager,
        yarnRPC, appID);

    // Now request a container.
    final Container allocatedContainer = requestAndGetContainer(scheduler,
        appID);

    // Now talk to the NM for launching the container with modified resource
    final ContainerId containerID = allocatedContainer.getId();
    UserGroupInformation maliceUser = UserGroupInformation
        .createRemoteUser(containerID.toString());

    ContainerToken containerToken = allocatedContainer.getContainerToken();
    byte[] identifierBytes = containerToken.getIdentifier().array();

    DataInputBuffer di = new DataInputBuffer();
    di.reset(identifierBytes, identifierBytes.length);

    ContainerTokenIdentifier dummyIdentifier = new ContainerTokenIdentifier();
    dummyIdentifier.readFields(di);

    // Malice user modifies the resource amount
    Resource modifiedResource = BuilderUtils.newResource(2048);
    ContainerTokenIdentifier modifiedIdentifier =
        new ContainerTokenIdentifier(dummyIdentifier.getContainerID(),
          dummyIdentifier.getNmHostAddress(), "testUser", modifiedResource,
          Long.MAX_VALUE, dummyIdentifier.getMasterKeyId());
    Token<ContainerTokenIdentifier> modifiedToken = new Token<ContainerTokenIdentifier>(
        modifiedIdentifier.getBytes(), containerToken.getPassword().array(),
        new Text(containerToken.getKind()), new Text(containerToken
            .getService()));
    maliceUser.addToken(modifiedToken);
    maliceUser.doAs(new PrivilegedAction<Void>() {
      @Override
      public Void run() {
        ContainerManager client = (ContainerManager) yarnRPC.getProxy(
            ContainerManager.class, NetUtils
                .createSocketAddr(allocatedContainer.getNodeId().toString()),
            conf);

        LOG.info("Going to contact NM:  ilLegal request");
View Full Code Here

Examples of org.apache.hadoop.yarn.ipc.YarnRPC

    LOG.info("\n\nRunning test for malice user");

    ResourceManager resourceManager = yarnCluster.getResourceManager();

    final YarnRPC yarnRPC = YarnRPC.create(conf);

    // Submit an application
    final ApplicationId appID = resourceManager.getClientRMService()
        .getNewApplication(Records.newRecord(GetNewApplicationRequest.class))
        .getApplicationId();
    AMRMProtocol scheduler = submitAndRegisterApplication(resourceManager,
        yarnRPC, appID);

    // Now request a container.
    final Container allocatedContainer = requestAndGetContainer(scheduler,
        appID);

    // Now talk to the NM for launching the container with modified containerID
    final ContainerId containerID = allocatedContainer.getId();

    /////////// Test calls with illegal containerIDs and illegal Resources
    UserGroupInformation unauthorizedUser = UserGroupInformation
        .createRemoteUser(containerID.toString());
    ContainerToken containerToken = allocatedContainer.getContainerToken();

    byte[] identifierBytes = containerToken.getIdentifier().array();
    DataInputBuffer di = new DataInputBuffer();
    di.reset(identifierBytes, identifierBytes.length);
    final ContainerTokenIdentifier tokenId = new ContainerTokenIdentifier();
    tokenId.readFields(di);

    Token<ContainerTokenIdentifier> token = new Token<ContainerTokenIdentifier>(
        identifierBytes, containerToken.getPassword().array(), new Text(
            containerToken.getKind()), new Text(containerToken.getService()));

    unauthorizedUser.addToken(token);
    ContainerManager client =
        unauthorizedUser.doAs(new PrivilegedAction<ContainerManager>() {
      @Override
      public ContainerManager run() {
        ContainerManager client = (ContainerManager) yarnRPC.getProxy(
            ContainerManager.class, NetUtils
                .createSocketAddr(allocatedContainer.getNodeId().toString()),
            conf);

        LOG.info("Going to contact NM:  unauthorized request");

        callWithIllegalContainerID(client, tokenId);
        callWithIllegalResource(client, tokenId);
        callWithIllegalUserName(client, tokenId);

        return client;
      }
    });
   
    // ///////// End of testing for illegal containerIDs, illegal Resources and
    // illegal users

    /////////// Test calls with expired tokens
    RPC.stopProxy(client);
    unauthorizedUser = UserGroupInformation
        .createRemoteUser(containerID.toString());

    RMContainerTokenSecretManager containerTokenSecreteManager =
      resourceManager.getRMContainerTokenSecretManager();
    final ContainerTokenIdentifier newTokenId =
        new ContainerTokenIdentifier(tokenId.getContainerID(),
          tokenId.getNmHostAddress(), "testUser", tokenId.getResource(),
          System.currentTimeMillis() - 1,
          containerTokenSecreteManager.getCurrentKey().getKeyId());
    byte[] passowrd =
        containerTokenSecreteManager.createPassword(
            newTokenId);
    // Create a valid token by using the key from the RM.
    token = new Token<ContainerTokenIdentifier>(
        newTokenId.getBytes(), passowrd, new Text(
            containerToken.getKind()), new Text(containerToken.getService()));

    unauthorizedUser.addToken(token);
    unauthorizedUser.doAs(new PrivilegedAction<Void>() {
      @Override
      public Void run() {
        ContainerManager client = (ContainerManager) yarnRPC.getProxy(
            ContainerManager.class, NetUtils
                .createSocketAddr(allocatedContainer.getNodeId().toString()),
            conf);

        LOG.info("Going to contact NM with expired token");
View Full Code Here

Examples of org.apache.hadoop.yarn.ipc.YarnRPC

        && isSecurityEnabled();
  }

  protected ResourceTracker getRMClient() {
    Configuration conf = getConfig();
    YarnRPC rpc = YarnRPC.create(conf);
    return (ResourceTracker) rpc.getProxy(ResourceTracker.class, rmAddress,
        conf);
  }
View Full Code Here

Examples of org.apache.hadoop.yarn.ipc.YarnRPC

  }
 
  @Override
  public void start() {
    Configuration conf = getConfig();
    YarnRPC rpc = YarnRPC.create(conf);
    this.server =  
      rpc.getServer(ClientRMProtocol.class, this,
            clientBindAddress,
            conf, this.rmDTSecretManager,
            conf.getInt(YarnConfiguration.RM_CLIENT_THREAD_COUNT,
                YarnConfiguration.DEFAULT_RM_CLIENT_THREAD_COUNT));
   
View Full Code Here

Examples of org.apache.hadoop.yarn.ipc.YarnRPC

    this.protocolHandler = new MRClientProtocolHandler();
  }

  public void start() {
    Configuration conf = getConfig();
    YarnRPC rpc = YarnRPC.create(conf);
    InetSocketAddress address = NetUtils.createSocketAddr("0.0.0.0:0");
    InetAddress hostNameResolved = null;
    try {
      hostNameResolved = InetAddress.getLocalHost();
    } catch (UnknownHostException e) {
      throw new YarnException(e);
    }

    ClientToAMSecretManager secretManager = null;
    if (UserGroupInformation.isSecurityEnabled()) {
      secretManager = new ClientToAMSecretManager();
      String secretKeyStr =
          System
              .getenv(ApplicationConstants.APPLICATION_CLIENT_SECRET_ENV_NAME);
      byte[] bytes = Base64.decodeBase64(secretKeyStr);
      ClientTokenIdentifier identifier = new ClientTokenIdentifier(
          this.appContext.getApplicationID());
      secretManager.setMasterKey(identifier, bytes);
    }
    server =
        rpc.getServer(MRClientProtocol.class, protocolHandler, address,
            conf, secretManager,
            conf.getInt(MRJobConfig.MR_AM_JOB_CLIENT_THREAD_COUNT,
                MRJobConfig.DEFAULT_MR_AM_JOB_CLIENT_THREAD_COUNT));
   
    // Enable service authorization?
View Full Code Here

Examples of org.apache.hadoop.yarn.ipc.YarnRPC

    allocatorThread.setName("RMCommunicator Allocator");
    allocatorThread.start();
  }

  protected AMRMProtocol createSchedulerProxy() {
    final YarnRPC rpc = YarnRPC.create(getConfig());
    final Configuration conf = getConfig();
    final String serviceAddr = conf.get(
        YarnConfiguration.RM_SCHEDULER_ADDRESS,
        YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS);

    UserGroupInformation currentUser;
    try {
      currentUser = UserGroupInformation.getCurrentUser();
    } catch (IOException e) {
      throw new YarnException(e);
    }

    if (UserGroupInformation.isSecurityEnabled()) {
      String tokenURLEncodedStr = System.getenv().get(
          ApplicationConstants.APPLICATION_MASTER_TOKEN_ENV_NAME);
      if (LOG.isDebugEnabled()) {
        LOG.debug("AppMasterToken is " + tokenURLEncodedStr);
      }
      Token<? extends TokenIdentifier> token = new Token<TokenIdentifier>();

      try {
        token.decodeFromUrlString(tokenURLEncodedStr);
      } catch (IOException e) {
        throw new YarnException(e);
      }

      currentUser.addToken(token);
    }

    return currentUser.doAs(new PrivilegedAction<AMRMProtocol>() {
      @Override
      public AMRMProtocol run() {
        return (AMRMProtocol) rpc.getProxy(AMRMProtocol.class,
            NetUtils.createSocketAddr(serviceAddr), conf);
      }
    });
  }
View Full Code Here

Examples of org.apache.hadoop.yarn.ipc.YarnRPC

    this.jhsDTSecretManager = jhsDTSecretManager;
  }

  public void start() {
    Configuration conf = getConfig();
    YarnRPC rpc = YarnRPC.create(conf);
    initializeWebApp(conf);
    String serviceAddr = conf.get(JHAdminConfig.MR_HISTORY_ADDRESS,
        JHAdminConfig.DEFAULT_MR_HISTORY_ADDRESS);
    InetSocketAddress address = NetUtils.createSocketAddr(serviceAddr,
      JHAdminConfig.DEFAULT_MR_HISTORY_PORT,
      JHAdminConfig.DEFAULT_MR_HISTORY_ADDRESS);
    InetAddress hostNameResolved = null;
    try {
      hostNameResolved = InetAddress.getLocalHost();
      //address.getAddress().getLocalHost();
    } catch (UnknownHostException e) {
      throw new YarnException(e);
    }

    server =
        rpc.getServer(MRClientProtocol.class, protocolHandler, address,
            conf, jhsDTSecretManager,
            conf.getInt(JHAdminConfig.MR_HISTORY_CLIENT_THREAD_COUNT,
                JHAdminConfig.DEFAULT_MR_HISTORY_CLIENT_THREAD_COUNT));

    // Enable service authorization?
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.