Package org.apache.hadoop.mapreduce.v2.api

Examples of org.apache.hadoop.mapreduce.v2.api.MRClientProtocol

347
348
349
350
351
352
353
354
355
356
357
358
    conf.setInt(MRJobConfig.MR_CLIENT_MAX_RETRIES, 3);

    conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
    conf.setBoolean(MRJobConfig.JOB_AM_ACCESS_DISABLED,
        !isAMReachableFromClient);
    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
    when(historyServerProxy.getJobReport(any(GetJobReportRequest.class)))
        .thenReturn(getJobReportResponse());
    ResourceMgrDelegate rmDelegate = mock(ResourceMgrDelegate.class);
    try {
      when(rmDelegate.getApplicationReport(jobId.getAppId())).thenThrow(
          new java.lang.reflect.UndeclaredThrowableException(new IOException(
View Full Code Here
374
375
376
377
378
379
380
381
382
383
384
  private void testRMDownForJobStatusBeforeGetAMReport(Configuration conf,
      int noOfRetries) throws IOException {
    conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
    conf.setBoolean(MRJobConfig.JOB_AM_ACCESS_DISABLED,
        !isAMReachableFromClient);
    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
    ResourceMgrDelegate rmDelegate = mock(ResourceMgrDelegate.class);
    try {
      when(rmDelegate.getApplicationReport(jobId.getAppId())).thenThrow(
          new java.lang.reflect.UndeclaredThrowableException(new IOException(
              "Connection refuced1"))).thenThrow(
View Full Code Here
290
291
292
293
294
295
296
297
298
299
300
      GetDelegationTokenResponse getDtResponse =
          Records.newRecord(GetDelegationTokenResponse.class);
      getDtResponse.setDelegationToken(historyToken);

      // mock services
      MRClientProtocol mockHsProxy = mock(MRClientProtocol.class);
      doReturn(mockHsAddress).when(mockHsProxy).getConnectAddress();
      doReturn(getDtResponse).when(mockHsProxy).getDelegationToken(
          any(GetDelegationTokenRequest.class));

      ResourceMgrDelegate rmDelegate = mock(ResourceMgrDelegate.class);
View Full Code Here
349
350
351
352
353
354
355
356
357
358
359
360
    //Set the master principal in the config
    conf.set(YarnConfiguration.RM_PRINCIPAL,"foo@LOCAL");

    final String masterPrincipal = Master.getMasterPrincipal(conf);

    final MRClientProtocol hsProxy = mock(MRClientProtocol.class);
    when(hsProxy.getDelegationToken(any(GetDelegationTokenRequest.class))).thenAnswer(
        new Answer<GetDelegationTokenResponse>() {
          public GetDelegationTokenResponse answer(InvocationOnMock invocation) {
            GetDelegationTokenRequest request =
                (GetDelegationTokenRequest)invocation.getArguments()[0];
            // check that the renewer matches the cluster's RM principal
View Full Code Here
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
    final long initialInterval = 10000l;
    final long maxLifetime= 20000l;
    final long renewInterval = 10000l;

    JobHistoryServer jobHistoryServer = null;
    MRClientProtocol clientUsingDT = null;
    long tokenFetchTime;
    try {
      jobHistoryServer = new JobHistoryServer() {
        protected void doSecureLogin(Configuration conf) throws IOException {
          // no keytab based login
        };

        protected JHSDelegationTokenSecretManager createJHSSecretManager(
            Configuration conf) {
          return new JHSDelegationTokenSecretManager(initialInterval,
              maxLifetime, renewInterval, 3600000);
        }
      };
//      final JobHistoryServer jobHistoryServer = jhServer;
      jobHistoryServer.init(conf);
      jobHistoryServer.start();
      final MRClientProtocol hsService = jobHistoryServer.getClientService()
          .getClientHandler();

      // Fake the authentication-method
      UserGroupInformation loggedInUser = UserGroupInformation
          .createRemoteUser("testrenewer@APACHE.ORG");
View Full Code Here
263
264
265
266
267
268
269
270
271
272
273
      final InetSocketAddress hsAddress, String user, final Configuration conf) {
    UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user);
    ugi.addToken(ConverterUtils.convertFromYarn(token, hsAddress));

    final YarnRPC rpc = YarnRPC.create(conf);
    MRClientProtocol hsWithDT = ugi
        .doAs(new PrivilegedAction<MRClientProtocol>() {

          @Override
          public MRClientProtocol run() {
            return (MRClientProtocol) rpc.getProxy(HSClientProtocol.class,
View Full Code Here
85
86
87
88
89
90
91
92
93
94
95
96
    return Arrays.asList(data);
  }

  @Test
  public void testUnknownAppInRM() throws Exception {
    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
    when(historyServerProxy.getJobReport(getJobReportRequest())).thenReturn(
        getJobReportResponse());
    ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate(
        historyServerProxy, getRMDelegate());

    JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
View Full Code Here
98
99
100
101
102
103
104
105
106
107
108
109
  }

  @Test
  public void testRemoteExceptionFromHistoryServer() throws Exception {

    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
    when(historyServerProxy.getJobReport(getJobReportRequest())).thenThrow(
        RPCUtil.getRemoteException("Job ID doesnot Exist"));

    ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);
    when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId()))
        .thenReturn(null);
View Full Code Here
120
121
122
123
124
125
126
127
128
129
130
131
  }

  @Test
  public void testRetriesOnConnectionFailure() throws Exception {

    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
    when(historyServerProxy.getJobReport(getJobReportRequest())).thenThrow(
        new RuntimeException("1")).thenThrow(new RuntimeException("2"))      
        .thenReturn(getJobReportResponse());

    ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);
    when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId()))
View Full Code Here
161
162
163
164
165
166
167
168
169
170
171
172
    Assert.assertEquals(JobStatus.State.SUCCEEDED, jobStatus.getState());
  }
 
  @Test
  public void testJobReportFromHistoryServer() throws Exception {                                
    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);                          
    when(historyServerProxy.getJobReport(getJobReportRequest())).thenReturn(                     
        getJobReportResponseFromHistoryServer());                                                
    ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);                                    
    when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId()))                     
    .thenReturn(null);                                                                       
    ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate(                      
View Full Code Here
TOP

Related Classes of org.apache.hadoop.mapreduce.v2.api.MRClientProtocol

  • org.apache.hadoop.conf.Configuration
  • org.apache.hadoop.fs.FileSystem
  • org.apache.hadoop.fs.FSDataInputStream
  • org.apache.hadoop.fs.Path
  • org.apache.hadoop.fs.permission.FsPermission
  • org.apache.hadoop.mapred.ClientServiceDelegate
  • org.apache.hadoop.mapred.JobConf
  • org.apache.hadoop.mapred.TaskAttemptListenerImpl
  • org.apache.hadoop.mapred.TestClientRedirect
  • org.apache.hadoop.mapred.TestClientRedirect$AMService
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.