Package org.apache.hadoop.mapreduce.v2.api

Examples of org.apache.hadoop.mapreduce.v2.api.MRClientProtocol

249
250
251
252
253
254
255
256
257
258
259
260
  @Test
  public void testHistoryServerToken() throws Exception {
    final String masterPrincipal = Master.getMasterPrincipal(conf);

    final MRClientProtocol hsProxy = mock(MRClientProtocol.class);
    when(hsProxy.getDelegationToken(any(GetDelegationTokenRequest.class))).thenAnswer(
        new Answer<GetDelegationTokenResponse>() {
          public GetDelegationTokenResponse answer(InvocationOnMock invocation) {
            GetDelegationTokenRequest request =
                (GetDelegationTokenRequest)invocation.getArguments()[0];
            // check that the renewer matches the cluster's RM principal
View Full Code Here
261
262
263
264
265
266
267
268
269
270
271
  @Override
  public JobStatus submitJob(JobID jobId, String jobSubmitDir, Credentials ts)
  throws IOException, InterruptedException {
   
    /* check if we have a hsproxy, if not, no need */
    MRClientProtocol hsProxy = clientCache.getInitializedHSProxy();
    if (hsProxy != null) {
      // JobClient will set this flag if getDelegationToken is called, if so, get
      // the delegation tokens for the HistoryServer also.
      if (conf.getBoolean(JobClient.HS_DELEGATION_TOKEN_REQUIRED,
          DEFAULT_HS_DELEGATION_TOKEN_REQUIRED)) {
View Full Code Here
286
287
288
289
290
291
292
293
294
295
296
  MRClientProtocol instantiateAMProxy(final InetSocketAddress serviceAddr)
      throws IOException {
    LOG.trace("Connecting to ApplicationMaster at: " + serviceAddr);
    YarnRPC rpc = YarnRPC.create(conf);
    MRClientProtocol proxy =
         (MRClientProtocol) rpc.getProxy(MRClientProtocol.class,
            serviceAddr, conf);
    usingAMProxy.set(true);
    LOG.trace("Connected to ApplicationMaster at: " + serviceAddr);
    return proxy;
View Full Code Here
309
310
311
312
313
314
315
316
317
318
319
    maxClientRetry = this.conf.getInt(
        MRJobConfig.MR_CLIENT_MAX_RETRIES,
        MRJobConfig.DEFAULT_MR_CLIENT_MAX_RETRIES);
    IOException lastException = null;
    while (maxClientRetry > 0) {
      MRClientProtocol MRClientProxy = null;
      try {
        MRClientProxy = getProxy();
        return methodOb.invoke(MRClientProxy, args);
      } catch (InvocationTargetException e) {
        // Will not throw out YarnException anymore
View Full Code Here
84
85
86
87
88
89
90
91
92
93
94
95
    return Arrays.asList(data);
  }

  @Test
  public void testUnknownAppInRM() throws Exception {
    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
    when(historyServerProxy.getJobReport(getJobReportRequest())).thenReturn(
        getJobReportResponse());
    ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate(
        historyServerProxy, getRMDelegate());

    JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
View Full Code Here
97
98
99
100
101
102
103
104
105
106
107
108
  }

  @Test
  public void testRemoteExceptionFromHistoryServer() throws Exception {

    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
    when(historyServerProxy.getJobReport(getJobReportRequest())).thenThrow(
        new IOException("Job ID doesnot Exist"));

    ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);
    when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId()))
        .thenReturn(null);
View Full Code Here
120
121
122
123
124
125
126
127
128
129
130
131
  }

  @Test
  public void testRetriesOnConnectionFailure() throws Exception {

    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
    when(historyServerProxy.getJobReport(getJobReportRequest())).thenThrow(
        new RuntimeException("1")).thenThrow(new RuntimeException("2"))      
        .thenReturn(getJobReportResponse());

    ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);
    when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId()))
View Full Code Here
150
151
152
153
154
155
156
157
158
159
160
161
    when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId()))
      .thenReturn(getRunningApplicationReport("am1", 78));

    // throw exception in 1st, 2nd, 3rd and 4th call of getJobReport, and
    // succeed in the 5th call.
    final MRClientProtocol amProxy = mock(MRClientProtocol.class);
    when(amProxy.getJobReport(any(GetJobReportRequest.class)))
      .thenThrow(new RuntimeException("11"))
      .thenThrow(new RuntimeException("22"))
      .thenThrow(new RuntimeException("33"))
      .thenThrow(new RuntimeException("44")).thenReturn(getJobReportResponse());
    Configuration conf = new YarnConfiguration();
View Full Code Here
203
204
205
206
207
208
209
210
211
212
213
214
    Assert.assertEquals(JobStatus.State.SUCCEEDED, jobStatus.getState());
  }
 
  @Test
  public void testJobReportFromHistoryServer() throws Exception {                                
    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);                          
    when(historyServerProxy.getJobReport(getJobReportRequest())).thenReturn(                     
        getJobReportResponseFromHistoryServer());                                                
    ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);                                    
    when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId()))                     
    .thenReturn(null);                                                                       
    ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate(                      
View Full Code Here
222
223
224
225
226
227
228
229
230
231
232
233
    Assert.assertEquals(1.0f, jobStatus.getReduceProgress());                                    
  }
 
  @Test
  public void testCountersFromHistoryServer() throws Exception {                                
    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);                          
    when(historyServerProxy.getCounters(getCountersRequest())).thenReturn(                     
        getCountersResponseFromHistoryServer());
    ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);                                    
    when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId()))                     
        .thenReturn(null);                                                                       
    ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate(                      
View Full Code Here
TOP

Related Classes of org.apache.hadoop.mapreduce.v2.api.MRClientProtocol

  • org.apache.hadoop.conf.Configuration
  • org.apache.hadoop.fs.FileSystem
  • org.apache.hadoop.fs.FSDataInputStream
  • org.apache.hadoop.fs.Path
  • org.apache.hadoop.fs.permission.FsPermission
  • org.apache.hadoop.mapred.ClientServiceDelegate
  • org.apache.hadoop.mapred.JobConf
  • org.apache.hadoop.mapred.TaskAttemptListenerImpl
  • org.apache.hadoop.mapred.TestClientRedirect
  • org.apache.hadoop.mapred.TestClientRedirect$AMService
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.