Package org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event

Examples of org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerResourceRequestEvent

219
220
221
222
223
224
225
226
      ResourceRequestEvent req = (ResourceRequestEvent) event;
      LocalizerContext ctxt = req.getContext();
      ContainerId container = ctxt.getContainerId();
      rsrc.ref.add(container);
      rsrc.dispatcher.getEventHandler().handle(
          new LocalizerResourceRequestEvent(rsrc, req.getVisibility(), ctxt,
              req.getLocalResourceRequest().getPattern()));
    }
View Full Code Here
206
207
208
209
210
211
212
213
214
215
216
      this.idRef = idRef;
    }
    @Override
    public boolean matches(Object o) {
      if (!(o instanceof LocalizerResourceRequestEvent)) return false;
      LocalizerResourceRequestEvent evt = (LocalizerResourceRequestEvent) o;
      return idRef == evt.getContext().getContainerId()
          && type == evt.getType()
          && vis == evt.getVisibility()
          && creds == evt.getContext().getCredentials();
    }
View Full Code Here
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
    public void handle(LocalizerEvent event) {
      String locId = event.getLocalizerId();
      switch (event.getType()) {
      case REQUEST_RESOURCE_LOCALIZATION:
        // 0) find running localizer or start new thread
        LocalizerResourceRequestEvent req =
          (LocalizerResourceRequestEvent)event;
        switch (req.getVisibility()) {
        case PUBLIC:
          publicLocalizer.addResource(req);
          break;
        case PRIVATE:
        case APPLICATION:
          synchronized (privLocalizers) {
            LocalizerRunner localizer = privLocalizers.get(locId);
            if (null == localizer) {
              LOG.info("Created localizer for " + req.getLocalizerId());
              localizer = new LocalizerRunner(req.getContext(),
                  req.getLocalizerId());
              privLocalizers.put(locId, localizer);
              localizer.start();
            }
            // 1) propagate event
            localizer.addResource(req);
View Full Code Here
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
      try {
        // TODO shutdown, better error handling esp. DU
        while (!Thread.currentThread().isInterrupted()) {
          try {
            Future<Path> completed = queue.take();
            LocalizerResourceRequestEvent assoc = pending.remove(completed);
            try {
              Path local = completed.get();
              if (null == assoc) {
                LOG.error("Localized unkonwn resource to " + completed);
                // TODO delete
                return;
              }
              LocalResourceRequest key = assoc.getResource().getRequest();
              assoc.getResource().handle(
                  new ResourceLocalizedEvent(key,
                    local, FileUtil.getDU(new File(local.toUri()))));
              synchronized (attempts) {
                attempts.remove(key);
              }
            } catch (ExecutionException e) {
              LOG.info("Failed to download rsrc " + assoc.getResource(),
                  e.getCause());
              dispatcher.getEventHandler().handle(
                  new ContainerResourceFailedEvent(
                    assoc.getContext().getContainerId(),
                    assoc.getResource().getRequest(), e.getCause()));
              synchronized (attempts) {
                LocalResourceRequest req = assoc.getResource().getRequest();
                List<LocalizerResourceRequestEvent> reqs = attempts.get(req);
                if (null == reqs) {
                  LOG.error("Missing pending list for " + req);
                  return;
                }
View Full Code Here
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
     */
    private LocalResource findNextResource() {
      // TODO: Synchronization
      for (Iterator<LocalizerResourceRequestEvent> i = pending.iterator();
           i.hasNext();) {
        LocalizerResourceRequestEvent evt = i.next();
        LocalizedResource nRsrc = evt.getResource();
        if (ResourceState.LOCALIZED.equals(nRsrc.getState())) {
          i.remove();
          continue;
        }
        if (nRsrc.tryAcquire()) {
          LocalResourceRequest nextRsrc = nRsrc.getRequest();
          LocalResource next =
            recordFactory.newRecordInstance(LocalResource.class);
          next.setResource(
              ConverterUtils.getYarnUrlFromPath(nextRsrc.getPath()));
          next.setTimestamp(nextRsrc.getTimestamp());
          next.setType(nextRsrc.getType());
          next.setVisibility(evt.getVisibility());
          scheduled.put(nextRsrc, evt);
          return next;
        }
      }
      return null;
View Full Code Here
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
        try {
          req = new LocalResourceRequest(rsrc);
        } catch (URISyntaxException e) {
          // TODO fail? Already translated several times...
        }
        LocalizerResourceRequestEvent assoc = scheduled.get(req);
        if (assoc == null) {
          // internal error
          LOG.error("Unknown resource reported: " + req);
          continue;
        }
        switch (stat.getStatus()) {
          case FETCH_SUCCESS:
            // notify resource
            try {
              assoc.getResource().handle(
                  new ResourceLocalizedEvent(req,
                    ConverterUtils.getPathFromYarnURL(stat.getLocalPath()),
                    stat.getLocalSize()));
            } catch (URISyntaxException e) { }
            if (pending.isEmpty()) {
              // TODO: Synchronization
              response.setLocalizerAction(LocalizerAction.DIE);
              break;
            }
            response.setLocalizerAction(LocalizerAction.LIVE);
            LocalResource next = findNextResource();
            if (next != null) {
              response.addResource(next);
            }
            break;
          case FETCH_PENDING:
            response.setLocalizerAction(LocalizerAction.LIVE);
            break;
          case FETCH_FAILURE:
            LOG.info("DEBUG: FAILED " + req, stat.getException());
            assoc.getResource().unlock();
            response.setLocalizerAction(LocalizerAction.DIE);
            // TODO: Why is this event going directly to the container. Why not
            // the resource itself? What happens to the resource? Is it removed?
            dispatcher.getEventHandler().handle(
                new ContainerResourceFailedEvent(context.getContainerId(),
View Full Code Here
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
    public void handle(LocalizerEvent event) {
      String locId = event.getLocalizerId();
      switch (event.getType()) {
      case REQUEST_RESOURCE_LOCALIZATION:
        // 0) find running localizer or start new thread
        LocalizerResourceRequestEvent req =
          (LocalizerResourceRequestEvent)event;
        switch (req.getVisibility()) {
        case PUBLIC:
          publicLocalizer.addResource(req);
          break;
        case PRIVATE:
        case APPLICATION:
          synchronized (privLocalizers) {
            LocalizerRunner localizer = privLocalizers.get(locId);
            if (null == localizer) {
              LOG.info("Created localizer for " + locId);
              localizer = new LocalizerRunner(req.getContext(), locId);
              privLocalizers.put(locId, localizer);
              localizer.start();
            }
            // 1) propagate event
            localizer.addResource(req);
View Full Code Here
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
      try {
        // TODO shutdown, better error handling esp. DU
        while (!Thread.currentThread().isInterrupted()) {
          try {
            Future<Path> completed = queue.take();
            LocalizerResourceRequestEvent assoc = pending.remove(completed);
            try {
              Path local = completed.get();
              if (null == assoc) {
                LOG.error("Localized unkonwn resource to " + completed);
                // TODO delete
                return;
              }
              LocalResourceRequest key = assoc.getResource().getRequest();
              assoc.getResource().handle(
                  new ResourceLocalizedEvent(key,
                    local, FileUtil.getDU(new File(local.toUri()))));
              synchronized (attempts) {
                attempts.remove(key);
              }
            } catch (ExecutionException e) {
              LOG.info("Failed to download rsrc " + assoc.getResource(),
                  e.getCause());
              dispatcher.getEventHandler().handle(
                  new ContainerResourceFailedEvent(
                    assoc.getContext().getContainerId(),
                    assoc.getResource().getRequest(), e.getCause()));
              List<LocalizerResourceRequestEvent> reqs;
              synchronized (attempts) {
                LocalResourceRequest req = assoc.getResource().getRequest();
                reqs = attempts.get(req);
                if (null == reqs) {
                  LOG.error("Missing pending list for " + req);
                  return;
                }
View Full Code Here
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
     */
    private LocalResource findNextResource() {
      synchronized (pending) {
        for (Iterator<LocalizerResourceRequestEvent> i = pending.iterator();
            i.hasNext();) {
          LocalizerResourceRequestEvent evt = i.next();
          LocalizedResource nRsrc = evt.getResource();
          if (ResourceState.LOCALIZED.equals(nRsrc.getState())) {
            i.remove();
            continue;
          }
          if (nRsrc.tryAcquire()) {
            LocalResourceRequest nextRsrc = nRsrc.getRequest();
            LocalResource next =
              recordFactory.newRecordInstance(LocalResource.class);
            next.setResource(ConverterUtils.getYarnUrlFromPath(nextRsrc
              .getPath()));
            next.setTimestamp(nextRsrc.getTimestamp());
            next.setType(nextRsrc.getType());
            next.setVisibility(evt.getVisibility());
            next.setPattern(evt.getPattern());
            scheduled.put(nextRsrc, evt);
            return next;
          }
        }
        return null;
View Full Code Here
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
        try {
          req = new LocalResourceRequest(rsrc);
        } catch (URISyntaxException e) {
          // TODO fail? Already translated several times...
        }
        LocalizerResourceRequestEvent assoc = scheduled.get(req);
        if (assoc == null) {
          // internal error
          LOG.error("Unknown resource reported: " + req);
          continue;
        }
        switch (stat.getStatus()) {
          case FETCH_SUCCESS:
            // notify resource
            try {
              assoc.getResource().handle(
                  new ResourceLocalizedEvent(req,
                    ConverterUtils.getPathFromYarnURL(stat.getLocalPath()),
                    stat.getLocalSize()));
            } catch (URISyntaxException e) { }
            if (pending.isEmpty()) {
              // TODO: Synchronization
              response.setLocalizerAction(LocalizerAction.DIE);
              break;
            }
            response.setLocalizerAction(LocalizerAction.LIVE);
            LocalResource next = findNextResource();
            if (next != null) {
              response.addResource(next);
            }
            break;
          case FETCH_PENDING:
            response.setLocalizerAction(LocalizerAction.LIVE);
            break;
          case FETCH_FAILURE:
            LOG.info("DEBUG: FAILED " + req, stat.getException());
            assoc.getResource().unlock();
            response.setLocalizerAction(LocalizerAction.DIE);
            // TODO: Why is this event going directly to the container. Why not
            // the resource itself? What happens to the resource? Is it removed?
            dispatcher.getEventHandler().handle(
                new ContainerResourceFailedEvent(context.getContainerId(),
View Full Code Here
TOP

Related Classes of org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerResourceRequestEvent

  • org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.LocalizedResource$FetchResourceTransition
  • org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService$LocalizerRunner
  • org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService$LocalizerTracker
  • org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService$PublicLocalizer
  • org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.TestLocalizedResource$LocalizerEventMatcher
  • org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.TestResourceLocalizationService
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.