Package org.apache.hadoop.yarn.util

Examples of org.apache.hadoop.yarn.util.SystemClock

120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
    dispatcher.start();

    TestingJobEventHandler jeh = new TestingJobEventHandler();
    dispatcher.register(JobEventType.class, jeh);

    SystemClock clock = new SystemClock();
    AppContext appContext = mock(AppContext.class);
    ApplicationAttemptId attemptid =
      ConverterUtils.toApplicationAttemptId("appattempt_1234567890000_0001_0");
    when(appContext.getApplicationID()).thenReturn(attemptid.getApplicationId());
    when(appContext.getApplicationAttemptId()).thenReturn(attemptid);
    when(appContext.getEventHandler()).thenReturn(
        dispatcher.getEventHandler());
    when(appContext.getClock()).thenReturn(clock);
    OutputCommitter committer = mock(OutputCommitter.class);
    TestingRMHeartbeatHandler rmhh =
        new TestingRMHeartbeatHandler();

    CommitterEventHandler ceh = new CommitterEventHandler(appContext,
        committer, rmhh);
    ceh.init(conf);
    ceh.start();

    // verify trying to commit when RM heartbeats are stale does not commit
    ceh.handle(new CommitterJobCommitEvent(null, null));
    long timeToWaitMs = 5000;
    while (rmhh.getNumCallbacks() != 1 && timeToWaitMs > 0) {
      Thread.sleep(10);
      timeToWaitMs -= 10;
    }
    Assert.assertEquals("committer did not register a heartbeat callback",
        1, rmhh.getNumCallbacks());
    verify(committer, never()).commitJob(any(JobContext.class));
    Assert.assertEquals("committer should not have committed",
        0, jeh.numCommitCompletedEvents);

    // set a fresh heartbeat and verify commit completes
    rmhh.setLastHeartbeatTime(clock.getTime());
    timeToWaitMs = 5000;
    while (jeh.numCommitCompletedEvents != 1 && timeToWaitMs > 0) {
      Thread.sleep(10);
      timeToWaitMs -= 10;
    }
View Full Code Here
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
    when(mockContext.hasSuccessfullyUnregistered()).thenReturn(true);
    JobImpl job = new JobImpl(jobId, Records
        .newRecord(ApplicationAttemptId.class), new Configuration(),
        mock(EventHandler.class),
        null, mock(JobTokenSecretManager.class), null,
        new SystemClock(), null,
        mrAppMetrics, null, true, null, 0, null, mockContext, null, null);
    job.handle(diagUpdateEvent);
    String diagnostics = job.getReport().getDiagnostics();
    Assert.assertNotNull(diagnostics);
    Assert.assertTrue(diagnostics.contains(diagMsg));

    job = new JobImpl(jobId, Records
        .newRecord(ApplicationAttemptId.class), new Configuration(),
        mock(EventHandler.class),
        null, mock(JobTokenSecretManager.class), null,
        new SystemClock(), null,
        mrAppMetrics, null, true, null, 0, null, mockContext, null, null);
    job.handle(new JobEvent(jobId, JobEventType.JOB_KILL));
    job.handle(diagUpdateEvent);
    diagnostics = job.getReport().getDiagnostics();
    Assert.assertNotNull(diagnostics);
View Full Code Here
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
                      job.getDiagnostics().toString().contains(EXCEPTIONMSG));
  }

  private static CommitterEventHandler createCommitterEventHandler(
      Dispatcher dispatcher, OutputCommitter committer) {
    final SystemClock clock = new SystemClock();
    AppContext appContext = mock(AppContext.class);
    when(appContext.getEventHandler()).thenReturn(
        dispatcher.getEventHandler());
    when(appContext.getClock()).thenReturn(clock);
    RMHeartbeatHandler heartbeatHandler = new RMHeartbeatHandler() {
      @Override
      public long getLastHeartbeatTime() {
        return clock.getTime();
      }
      @Override
      public void runOnNextHeartbeat(Runnable callback) {
        callback.run();
      }
View Full Code Here
946
947
948
949
950
951
952
953
954
955
956
    public StubbedJob(JobId jobId, ApplicationAttemptId applicationAttemptId,
        Configuration conf, EventHandler eventHandler, boolean newApiCommitter,
        String user, int numSplits, AppContext appContext) {
      super(jobId, applicationAttemptId, conf, eventHandler,
          null, new JobTokenSecretManager(), new Credentials(),
          new SystemClock(), Collections.<TaskId, TaskInfo> emptyMap(),
          MRAppMetrics.create(), null, newApiCommitter, user,
          System.currentTimeMillis(), null, appContext, null, null);

      initTransition = getInitTransition(numSplits);
      localFactory = stateMachineFactory.addTransition(JobStateInternal.NEW,
View Full Code Here
150
151
152
153
154
155
156
  }
 
  public MRApp(int maps, int reduces, boolean autoComplete, String testName,
      boolean cleanOnStart, String assignedQueue) {
    this(maps, reduces, autoComplete, testName, cleanOnStart, 1,
        new SystemClock(), assignedQueue);
  }
View Full Code Here
184
185
186
187
188
189
190
  }

  public MRApp(int maps, int reduces, boolean autoComplete, String testName,
      boolean cleanOnStart, int startCount) {
    this(maps, reduces, autoComplete, testName, cleanOnStart, startCount,
        new SystemClock(), null);
  }
View Full Code Here
190
191
192
193
194
195
196
  }

  public MRApp(int maps, int reduces, boolean autoComplete, String testName,
      boolean cleanOnStart, int startCount, boolean unregistered) {
    this(maps, reduces, autoComplete, testName, cleanOnStart, startCount,
        new SystemClock(), unregistered);
  }
View Full Code Here
211
212
213
214
215
216
217
  public MRApp(ApplicationAttemptId appAttemptId, ContainerId amContainerId,
      int maps, int reduces, boolean autoComplete, String testName,
      boolean cleanOnStart, int startCount, boolean unregistered) {
    this(appAttemptId, amContainerId, maps, reduces, autoComplete, testName,
        cleanOnStart, startCount, new SystemClock(), unregistered, null);
  }
View Full Code Here
218
219
220
221
222
223
224
  public MRApp(ApplicationAttemptId appAttemptId, ContainerId amContainerId,
      int maps, int reduces, boolean autoComplete, String testName,
      boolean cleanOnStart, int startCount) {
    this(appAttemptId, amContainerId, maps, reduces, autoComplete, testName,
        cleanOnStart, startCount, new SystemClock(), true, null);
  }
View Full Code Here
164
165
166
167
168
169
170
  }

  public MRApp(int maps, int reduces, boolean autoComplete, String testName,
      boolean cleanOnStart, int startCount) {
    this(maps, reduces, autoComplete, testName, cleanOnStart, startCount,
        new SystemClock());
  }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.yarn.util.SystemClock

  • org.apache.hadoop.mapred.TestTaskAttemptListenerImpl
  • org.apache.hadoop.mapreduce.v2.app.commit.TestCommitterEventHandler
  • org.apache.hadoop.mapreduce.v2.app.job.impl.TestJobImpl
  • org.apache.hadoop.mapreduce.v2.app.job.impl.TestJobImpl$StubbedJob
  • org.apache.hadoop.mapreduce.v2.app.job.impl.TestShuffleProvider
  • org.apache.hadoop.mapreduce.v2.app.job.impl.TestTaskAttempt
  • org.apache.hadoop.mapreduce.v2.app.job.impl.TestTaskAttemptContainerRequest
  • org.apache.hadoop.mapreduce.v2.app.job.impl.TestTaskImpl
  • org.apache.hadoop.mapreduce.v2.app.MRApp
  • org.apache.hadoop.mapreduce.v2.app.MRAppMaster
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.