Package org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event

Examples of org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceRequestEvent


   */
  @SuppressWarnings("unchecked") // dispatcher not typed
  private static class FetchResourceTransition extends ResourceTransition {
    @Override
    public void transition(LocalizedResource rsrc, ResourceEvent event) {
      ResourceRequestEvent req = (ResourceRequestEvent) event;
      LocalizerContext ctxt = req.getContext();
      ContainerId container = ctxt.getContainerId();
      rsrc.ref.add(container);
      rsrc.dispatcher.getEventHandler().handle(
          new LocalizerResourceRequestEvent(rsrc, req.getVisibility(), ctxt,
              req.getLocalResourceRequest().getPattern()));
    }
View Full Code Here


  private static class LocalizedResourceTransition
      extends ResourceTransition {
    @Override
    public void transition(LocalizedResource rsrc, ResourceEvent event) {
      // notify waiting containers
      ResourceRequestEvent reqEvent = (ResourceRequestEvent) event;
      ContainerId container = reqEvent.getContext().getContainerId();
      rsrc.ref.add(container);
      rsrc.dispatcher.getEventHandler().handle(
          new ContainerResourceLocalizedEvent(
            container, rsrc.rsrc, rsrc.localPath));
    }
View Full Code Here

    for (Map.Entry<LocalResourceVisibility, Collection<LocalResourceRequest>> e :
         rsrcs.entrySet()) {
      LocalResourcesTracker tracker = getLocalResourcesTracker(e.getKey(), c.getUser(),
          c.getContainerID().getApplicationAttemptId().getApplicationId());
      for (LocalResourceRequest req : e.getValue()) {
        tracker.handle(new ResourceRequestEvent(req, e.getKey(), ctxt));
      }
    }
  }
View Full Code Here

    for (Map.Entry<LocalResourceVisibility, Collection<LocalResourceRequest>> e :
         rsrcs.entrySet()) {
      LocalResourcesTracker tracker = getLocalResourcesTracker(e.getKey(), c.getUser(),
          c.getContainerID().getApplicationAttemptId().getApplicationId());
      for (LocalResourceRequest req : e.getValue()) {
        tracker.handle(new ResourceRequestEvent(req, e.getKey(), ctxt));
      }
    }
  }
View Full Code Here

      LocalResourcesTracker tracker =
          new LocalResourcesTrackerImpl(user, dispatcher, localrsrc, false,
            conf);

      ResourceEvent req11Event =
          new ResourceRequestEvent(req1, LocalResourceVisibility.PUBLIC, lc1);
      ResourceEvent req12Event =
          new ResourceRequestEvent(req1, LocalResourceVisibility.PUBLIC, lc2);
      ResourceEvent req21Event =
          new ResourceRequestEvent(req2, LocalResourceVisibility.PUBLIC, lc1);

      ResourceEvent rel11Event = new ResourceReleaseEvent(req1, cId1);
      ResourceEvent rel12Event = new ResourceReleaseEvent(req1, cId2);
      ResourceEvent rel21Event = new ResourceReleaseEvent(req2, cId1);
View Full Code Here

      ConcurrentMap<LocalResourceRequest, LocalizedResource> localrsrc = new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>();
      localrsrc.put(req1, lr1);
      LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user,
          dispatcher, localrsrc, false, conf);

      ResourceEvent req11Event = new ResourceRequestEvent(req1,
          LocalResourceVisibility.PUBLIC, lc1);

      ResourceEvent rel11Event = new ResourceReleaseEvent(req1, cId1);

      // Localize R1 for C1
View Full Code Here

      // same local resource.
      // Container 1 requesting local resource.
      ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
      LocalizerContext lc1 = new LocalizerContext(user, cId1, null);
      ResourceEvent reqEvent1 =
          new ResourceRequestEvent(lr, LocalResourceVisibility.PRIVATE, lc1);

      // No resource request is initially present in local cache
      Assert.assertEquals(0, localrsrc.size());

      // Container-1 requesting local resource.
      tracker.handle(reqEvent1);

      // New localized Resource should have been added to local resource map
      // and the requesting container will be added to its waiting queue.
      Assert.assertEquals(1, localrsrc.size());
      Assert.assertTrue(localrsrc.containsKey(lr));
      Assert.assertEquals(1, localrsrc.get(lr).getRefCount());
      Assert.assertTrue(localrsrc.get(lr).ref.contains(cId1));
      Assert.assertEquals(ResourceState.DOWNLOADING, localrsrc.get(lr)
        .getState());

      // Container 2 requesting the resource
      ContainerId cId2 = BuilderUtils.newContainerId(1, 1, 1, 2);
      LocalizerContext lc2 = new LocalizerContext(user, cId2, null);
      ResourceEvent reqEvent2 =
          new ResourceRequestEvent(lr, LocalResourceVisibility.PRIVATE, lc2);
      tracker.handle(reqEvent2);

      // Container 2 should have been added to the waiting queue of the local
      // resource
      Assert.assertEquals(2, localrsrc.get(lr).getRefCount());
      Assert.assertTrue(localrsrc.get(lr).ref.contains(cId2));

      // Failing resource localization
      ResourceEvent resourceFailedEvent = new ResourceFailedLocalizationEvent(
          lr,(new Exception("test").getMessage()));
     
      // Backing up the resource to track its state change as it will be
      // removed after the failed event.
      LocalizedResource localizedResource = localrsrc.get(lr);
     
      tracker.handle(resourceFailedEvent);

      // After receiving failed resource event; all waiting containers will be
      // notified with Container Resource Failed Event.
      Assert.assertEquals(0, localrsrc.size());
      verify(containerEventHandler, times(2)).handle(
        isA(ContainerResourceFailedEvent.class));
      Assert.assertEquals(ResourceState.FAILED, localizedResource.getState());

      // Container 1 trying to release the resource (This resource is already
      // deleted from the cache. This call should return silently without
      // exception.
      ResourceReleaseEvent relEvent1 = new ResourceReleaseEvent(lr, cId1);
      tracker.handle(relEvent1);

      // Container-3 now requests for the same resource. This request call
      // is coming prior to Container-2's release call.
      ContainerId cId3 = BuilderUtils.newContainerId(1, 1, 1, 3);
      LocalizerContext lc3 = new LocalizerContext(user, cId3, null);
      ResourceEvent reqEvent3 =
          new ResourceRequestEvent(lr, LocalResourceVisibility.PRIVATE, lc3);
      tracker.handle(reqEvent3);

      // Local resource cache now should have the requested resource and the
      // number of waiting containers should be 1.
      Assert.assertEquals(1, localrsrc.size());
View Full Code Here

      LocalResourceRequest lr1 = createLocalResourceRequest(user, 1, 1,
          LocalResourceVisibility.PUBLIC);
      LocalizerContext lc1 = new LocalizerContext(user, cId1, null);

      // Container 1 requests lr1 to be localized
      ResourceEvent reqEvent1 = new ResourceRequestEvent(lr1,
          LocalResourceVisibility.PUBLIC, lc1);
      tracker.handle(reqEvent1);

      // Simulate the process of localization of lr1
      Path hierarchicalPath1 = tracker.getPathForLocalization(lr1, localDir);
      // Simulate lr1 getting localized
      ResourceLocalizedEvent rle1 =
          new ResourceLocalizedEvent(lr1,
              new Path(hierarchicalPath1.toUri().toString() +
                  Path.SEPARATOR + "file1"), 120);
      tracker.handle(rle1);
      // Localization successful.

      LocalResourceRequest lr2 = createLocalResourceRequest(user, 3, 3,
          LocalResourceVisibility.PUBLIC);
      // Container 1 requests lr2 to be localized.
      ResourceEvent reqEvent2 =
          new ResourceRequestEvent(lr2, LocalResourceVisibility.PUBLIC, lc1);
      tracker.handle(reqEvent2);

      Path hierarchicalPath2 = tracker.getPathForLocalization(lr2, localDir);
      // localization failed.
      ResourceFailedLocalizationEvent rfe2 =
          new ResourceFailedLocalizationEvent(
              lr2, new Exception("Test").toString());
      tracker.handle(rfe2);

      /*
       * The path returned for two localization should be different because we
       * are limiting one file per sub-directory.
       */
      Assert.assertNotSame(hierarchicalPath1, hierarchicalPath2);

      LocalResourceRequest lr3 = createLocalResourceRequest(user, 2, 2,
          LocalResourceVisibility.PUBLIC);
      ResourceEvent reqEvent3 = new ResourceRequestEvent(lr3,
          LocalResourceVisibility.PUBLIC, lc1);
      tracker.handle(reqEvent3);
      Path hierarchicalPath3 = tracker.getPathForLocalization(lr3, localDir);
      // localization successful
      ResourceLocalizedEvent rle3 =
View Full Code Here

    for (Map.Entry<LocalResourceVisibility, Collection<LocalResourceRequest>> e :
         rsrcs.entrySet()) {
      LocalResourcesTracker tracker = getLocalResourcesTracker(e.getKey(), c.getUser(),
          c.getContainerID().getApplicationAttemptId().getApplicationId());
      for (LocalResourceRequest req : e.getValue()) {
        tracker.handle(new ResourceRequestEvent(req, e.getKey(), ctxt));
      }
    }
  }
View Full Code Here

      final LocalResourceVisibility vis0 = LocalResourceVisibility.PRIVATE;
      final LocalizerContext ctxt0 =
        new LocalizerContext("yak", container0, creds0);
      LocalResourceRequest rsrcA = new LocalResourceRequest(apiRsrc);
      LocalizedResource local = new LocalizedResource(rsrcA, dispatcher);
      local.handle(new ResourceRequestEvent(rsrcA, vis0, ctxt0));
      dispatcher.await();

      // Register C0, verify request event
      LocalizerEventMatcher matchesL0Req =
        new LocalizerEventMatcher(container0, creds0, vis0,
            LocalizerEventType.REQUEST_RESOURCE_LOCALIZATION);
      verify(localizerBus).handle(argThat(matchesL0Req));
      assertEquals(ResourceState.DOWNLOADING, local.getState());

      // Register C1, verify request event
      final Credentials creds1 = new Credentials();
      final ContainerId container1 = getMockContainer(1);
      final LocalizerContext ctxt1 =
        new LocalizerContext("yak", container1, creds1);
      final LocalResourceVisibility vis1 = LocalResourceVisibility.PUBLIC;
      local.handle(new ResourceRequestEvent(rsrcA, vis1, ctxt1));
      dispatcher.await();
      LocalizerEventMatcher matchesL1Req =
        new LocalizerEventMatcher(container1, creds1, vis1,
            LocalizerEventType.REQUEST_RESOURCE_LOCALIZATION);
      verify(localizerBus).handle(argThat(matchesL1Req));

      // Release C0 container localization, verify no notification
      local.handle(new ResourceReleaseEvent(rsrcA, container0));
      dispatcher.await();
      verify(containerBus, never()).handle(isA(ContainerEvent.class));
      assertEquals(ResourceState.DOWNLOADING, local.getState());

      // Release C1 container localization, verify no notification
      local.handle(new ResourceReleaseEvent(rsrcA, container1));
      dispatcher.await();
      verify(containerBus, never()).handle(isA(ContainerEvent.class));
      assertEquals(ResourceState.INIT, local.getState());

      // Register C2, C3
      final ContainerId container2 = getMockContainer(2);
      final LocalResourceVisibility vis2 = LocalResourceVisibility.PRIVATE;
      final Credentials creds2 = new Credentials();
      final LocalizerContext ctxt2 =
        new LocalizerContext("yak", container2, creds2);

      final ContainerId container3 = getMockContainer(3);
      final LocalResourceVisibility vis3 = LocalResourceVisibility.PRIVATE;
      final Credentials creds3 = new Credentials();
      final LocalizerContext ctxt3 =
        new LocalizerContext("yak", container3, creds3);

      local.handle(new ResourceRequestEvent(rsrcA, vis2, ctxt2));
      local.handle(new ResourceRequestEvent(rsrcA, vis3, ctxt3));
      dispatcher.await();
      LocalizerEventMatcher matchesL2Req =
        new LocalizerEventMatcher(container2, creds2, vis2,
            LocalizerEventType.REQUEST_RESOURCE_LOCALIZATION);
      verify(localizerBus).handle(argThat(matchesL2Req));
      LocalizerEventMatcher matchesL3Req =
        new LocalizerEventMatcher(container3, creds3, vis3,
            LocalizerEventType.REQUEST_RESOURCE_LOCALIZATION);
      verify(localizerBus).handle(argThat(matchesL3Req));

      // Successful localization. verify notification C2, C3
      Path locA = new Path("file:///cache/rsrcA");
      local.handle(new ResourceLocalizedEvent(rsrcA, locA, 10));
      dispatcher.await();
      ContainerEventMatcher matchesC2Localized =
        new ContainerEventMatcher(container2,
            ContainerEventType.RESOURCE_LOCALIZED);
      ContainerEventMatcher matchesC3Localized =
        new ContainerEventMatcher(container3,
            ContainerEventType.RESOURCE_LOCALIZED);
      verify(containerBus).handle(argThat(matchesC2Localized));
      verify(containerBus).handle(argThat(matchesC3Localized));
      assertEquals(ResourceState.LOCALIZED, local.getState());

      // Register C4, verify notification
      final ContainerId container4 = getMockContainer(4);
      final Credentials creds4 = new Credentials();
      final LocalizerContext ctxt4 =
        new LocalizerContext("yak", container4, creds4);
      final LocalResourceVisibility vis4 = LocalResourceVisibility.PRIVATE;
      local.handle(new ResourceRequestEvent(rsrcA, vis4, ctxt4));
      dispatcher.await();
      ContainerEventMatcher matchesC4Localized =
        new ContainerEventMatcher(container4,
            ContainerEventType.RESOURCE_LOCALIZED);
      verify(containerBus).handle(argThat(matchesC4Localized));
View Full Code Here

TOP

Related Classes of org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceRequestEvent

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.