Examples of org.apache.hadoop.mapreduce.v2.api.MRClientProtocol

Package org.apache.hadoop.mapreduce.v2.api

Examples of org.apache.hadoop.mapreduce.v2.api.MRClientProtocol

org.apache.hadoop.mapreduce.v2.api.MRClientProtocol

    conf.setInt(MRJobConfig.MR_CLIENT_MAX_RETRIES, 3);


    conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
    conf.setBoolean(MRJobConfig.JOB_AM_ACCESS_DISABLED,
        !isAMReachableFromClient);
    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
    when(historyServerProxy.getJobReport(any(GetJobReportRequest.class)))
        .thenReturn(getJobReportResponse());
    ResourceMgrDelegate rmDelegate = mock(ResourceMgrDelegate.class);
    try {
      when(rmDelegate.getApplicationReport(jobId.getAppId())).thenThrow(
          new java.lang.reflect.UndeclaredThrowableException(new IOException(

View Full Code Here

  private void testRMDownForJobStatusBeforeGetAMReport(Configuration conf,
      int noOfRetries) throws IOException {
    conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
    conf.setBoolean(MRJobConfig.JOB_AM_ACCESS_DISABLED,
        !isAMReachableFromClient);
    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
    ResourceMgrDelegate rmDelegate = mock(ResourceMgrDelegate.class);
    try {
      when(rmDelegate.getApplicationReport(jobId.getAppId())).thenThrow(
          new java.lang.reflect.UndeclaredThrowableException(new IOException(
              "Connection refuced1"))).thenThrow(

View Full Code Here

  
  
  private void testPbServerFactory() {
    InetSocketAddress addr = new InetSocketAddress(0);
    Configuration conf = new Configuration();
    MRClientProtocol instance = new MRClientProtocolTestImpl();
    Server server = null;
    try {
      server = 
        RpcServerFactoryPBImpl.get().getServer(
          MRClientProtocol.class, instance, addr, conf, null, 1);

View Full Code Here

  
  private void testPbClientFactory() {
    InetSocketAddress addr = new InetSocketAddress(0);
    System.err.println(addr.getHostName() + addr.getPort());
    Configuration conf = new Configuration();
    MRClientProtocol instance = new MRClientProtocolTestImpl();
    Server server = null;
    try {
      server = 
        RpcServerFactoryPBImpl.get().getServer(
            MRClientProtocol.class, instance, addr, conf, null, 1);
      server.start();
      System.err.println(server.getListenerAddress());
      System.err.println(NetUtils.getConnectAddress(server));


      MRClientProtocol client = null;
      try {
        client = (MRClientProtocol) RpcClientFactoryPBImpl.get().getClient(MRClientProtocol.class, 1, NetUtils.getConnectAddress(server), conf);
      } catch (YarnRuntimeException e) {
        e.printStackTrace();
        Assert.fail("Failed to crete client");

View Full Code Here

        new TaskAttemptStatusUpdateEvent(attempt.getID(), taskAttemptStatus));


    
    //verify that all object are fully populated by invoking RPCs.
    YarnRPC rpc = YarnRPC.create(conf);
    MRClientProtocol proxy =
      (MRClientProtocol) rpc.getProxy(MRClientProtocol.class,
          app.clientService.getBindAddress(), conf);
    GetCountersRequest gcRequest =
        recordFactory.newRecordInstance(GetCountersRequest.class);    
    gcRequest.setJobId(job.getID());
    Assert.assertNotNull("Counters is null",
        proxy.getCounters(gcRequest).getCounters());


    GetJobReportRequest gjrRequest =
        recordFactory.newRecordInstance(GetJobReportRequest.class);
    gjrRequest.setJobId(job.getID());
    JobReport jr = proxy.getJobReport(gjrRequest).getJobReport();
    verifyJobReport(jr);
    


    GetTaskAttemptCompletionEventsRequest gtaceRequest =
        recordFactory.newRecordInstance(GetTaskAttemptCompletionEventsRequest.class);
    gtaceRequest.setJobId(job.getID());
    gtaceRequest.setFromEventId(0);
    gtaceRequest.setMaxEvents(10);
    Assert.assertNotNull("TaskCompletionEvents is null", 
        proxy.getTaskAttemptCompletionEvents(gtaceRequest).getCompletionEventList());


    GetDiagnosticsRequest gdRequest =
        recordFactory.newRecordInstance(GetDiagnosticsRequest.class);
    gdRequest.setTaskAttemptId(attempt.getID());
    Assert.assertNotNull("Diagnostics is null", 
        proxy.getDiagnostics(gdRequest).getDiagnosticsList());


    GetTaskAttemptReportRequest gtarRequest =
        recordFactory.newRecordInstance(GetTaskAttemptReportRequest.class);
    gtarRequest.setTaskAttemptId(attempt.getID());
    TaskAttemptReport tar =
        proxy.getTaskAttemptReport(gtarRequest).getTaskAttemptReport();
    verifyTaskAttemptReport(tar);
    


    GetTaskReportRequest gtrRequest =
        recordFactory.newRecordInstance(GetTaskReportRequest.class);
    gtrRequest.setTaskId(task.getID());
    Assert.assertNotNull("TaskReport is null", 
        proxy.getTaskReport(gtrRequest).getTaskReport());


    GetTaskReportsRequest gtreportsRequest =
        recordFactory.newRecordInstance(GetTaskReportsRequest.class);
    gtreportsRequest.setJobId(job.getID());
    gtreportsRequest.setTaskType(TaskType.MAP);
    Assert.assertNotNull("TaskReports for map is null", 
        proxy.getTaskReports(gtreportsRequest).getTaskReportList());


    gtreportsRequest =
        recordFactory.newRecordInstance(GetTaskReportsRequest.class);
    gtreportsRequest.setJobId(job.getID());
    gtreportsRequest.setTaskType(TaskType.REDUCE);
    Assert.assertNotNull("TaskReports for reduce is null", 
        proxy.getTaskReports(gtreportsRequest).getTaskReportList());


    List<String> diag = proxy.getDiagnostics(gdRequest).getDiagnosticsList();
    Assert.assertEquals("Num diagnostics not correct", 1 , diag.size());
    Assert.assertEquals("Diag 1 not correct",
        diagnostic1, diag.get(0).toString());


    TaskReport taskReport = proxy.getTaskReport(gtrRequest).getTaskReport();
    Assert.assertEquals("Num diagnostics not correct", 1,
        taskReport.getDiagnosticsCount());


    //send the done signal to the task
    app.getContext().getEventHandler().handle(

View Full Code Here

            "viewonlyuser", new String[] {});
    Assert.assertTrue("viewonlyuser cannot view job",
        job.checkAccess(viewOnlyUser, JobACL.VIEW_JOB));
    Assert.assertFalse("viewonlyuser can modify job",
        job.checkAccess(viewOnlyUser, JobACL.MODIFY_JOB));
    MRClientProtocol client = viewOnlyUser.doAs(
        new PrivilegedExceptionAction<MRClientProtocol>() {
          @Override
          public MRClientProtocol run() throws Exception {
            YarnRPC rpc = YarnRPC.create(conf);
            return (MRClientProtocol) rpc.getProxy(MRClientProtocol.class,
                app.clientService.getBindAddress(), conf);
          }
        });


    KillJobRequest killJobRequest = recordFactory.newRecordInstance(
        KillJobRequest.class);
    killJobRequest.setJobId(app.getJobId());
    try {
      client.killJob(killJobRequest);
      fail("viewonlyuser killed job");
    } catch (AccessControlException e) {
      // pass
    }


    KillTaskRequest killTaskRequest = recordFactory.newRecordInstance(
        KillTaskRequest.class);
    killTaskRequest.setTaskId(task.getID());
    try {
      client.killTask(killTaskRequest);
      fail("viewonlyuser killed task");
    } catch (AccessControlException e) {
      // pass
    }


    KillTaskAttemptRequest killTaskAttemptRequest =
        recordFactory.newRecordInstance(KillTaskAttemptRequest.class);
    killTaskAttemptRequest.setTaskAttemptId(attempt.getID());
    try {
      client.killTaskAttempt(killTaskAttemptRequest);
      fail("viewonlyuser killed task attempt");
    } catch (AccessControlException e) {
      // pass
    }


    FailTaskAttemptRequest failTaskAttemptRequest =
        recordFactory.newRecordInstance(FailTaskAttemptRequest.class);
    failTaskAttemptRequest.setTaskAttemptId(attempt.getID());
    try {
      client.failTaskAttempt(failTaskAttemptRequest);
      fail("viewonlyuser killed task attempt");
    } catch (AccessControlException e) {
      // pass
    }
  }

View Full Code Here


  MRClientProtocol instantiateAMProxy(final InetSocketAddress serviceAddr)
      throws IOException {
    LOG.trace("Connecting to ApplicationMaster at: " + serviceAddr);
    YarnRPC rpc = YarnRPC.create(conf);
    MRClientProtocol proxy = 
         (MRClientProtocol) rpc.getProxy(MRClientProtocol.class,
            serviceAddr, conf);
    LOG.trace("Connected to ApplicationMaster at: " + serviceAddr);
    return proxy;
  }

View Full Code Here

  @Override
  public JobStatus submitJob(JobID jobId, String jobSubmitDir, Credentials ts)
  throws IOException, InterruptedException {
    
    /* check if we have a hsproxy, if not, no need */
    MRClientProtocol hsProxy = clientCache.getInitializedHSProxy();
    if (hsProxy != null) {
      // JobClient will set this flag if getDelegationToken is called, if so, get
      // the delegation tokens for the HistoryServer also.
      if (conf.getBoolean(JobClient.HS_DELEGATION_TOKEN_REQUIRED, 
          DEFAULT_HS_DELEGATION_TOKEN_REQUIRED)) {

View Full Code Here

    return Arrays.asList(data);
  }


  @Test
  public void testUnknownAppInRM() throws Exception {
    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
    when(historyServerProxy.getJobReport(getJobReportRequest())).thenReturn(
        getJobReportResponse());
    ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate(
        historyServerProxy, getRMDelegate());


    JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);

View Full Code Here

  }


  @Test
  public void testRemoteExceptionFromHistoryServer() throws Exception {


    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
    when(historyServerProxy.getJobReport(getJobReportRequest())).thenThrow(
        RPCUtil.getRemoteException("Job ID doesnot Exist"));


    ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);
    when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId()))
        .thenReturn(null);

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.mapreduce.v2.api.MRClientProtocol

org.apache.hadoop.conf.Configuration

org.apache.hadoop.fs.FileSystem

org.apache.hadoop.fs.FSDataInputStream

org.apache.hadoop.fs.Path

org.apache.hadoop.fs.permission.FsPermission

org.apache.hadoop.mapred.ClientServiceDelegate

org.apache.hadoop.mapred.JobConf

org.apache.hadoop.mapred.TaskAttemptListenerImpl

org.apache.hadoop.mapred.TestClientRedirect

org.apache.hadoop.mapred.TestClientRedirect$AMService

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.