Examples of org.apache.hadoop.yarn.api.records.LocalResource

Package org.apache.hadoop.yarn.api.records

Examples of org.apache.hadoop.yarn.api.records.LocalResource

org.apache.hadoop.yarn.api.records.LocalResource

LocalResource represents a local resource required to run a container.

The NodeManager is responsible for localizing the resource prior to launching the container.

Applications can specify {@link LocalResourceType} and {@link LocalResourceVisibility}.
@see LocalResourceType @see LocalResourceVisibility @see ContainerLaunchContext @see ApplicationSubmissionContext @see ContainerManager#startContainer(org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest)

      long seed = r.nextLong();
      System.out.println("SEED: " + seed);
      r.setSeed(seed);
      
      // Send localization requests for one resource of each type.
      final LocalResource privResource = getPrivateMockedResource(r);
      final LocalResourceRequest privReq =
          new LocalResourceRequest(privResource);
      
      final LocalResource pubResource = getPublicMockedResource(r);
      final LocalResourceRequest pubReq = new LocalResourceRequest(pubResource);
      final LocalResource pubResource2 = getPublicMockedResource(r);
      final LocalResourceRequest pubReq2 =
          new LocalResourceRequest(pubResource2);
      
      final LocalResource appResource = getAppMockedResource(r);
      final LocalResourceRequest appReq = new LocalResourceRequest(appResource);
      
      Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req =
          new HashMap<LocalResourceVisibility, 
                      Collection<LocalResourceRequest>>();

View Full Code Here

      FSDataOutputStream out =
        new FSDataOutputStream(new DataOutputBuffer(), null);
      doReturn(out).when(spylfs).createInternal(isA(Path.class),
          isA(EnumSet.class), isA(FsPermission.class), anyInt(), anyShort(),
          anyLong(), isA(Progressable.class), anyInt(), anyBoolean());
      final LocalResource resource = getPrivateMockedResource(r);
      final LocalResourceRequest req = new LocalResourceRequest(resource);
      Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs =
        new HashMap<LocalResourceVisibility, 
                    Collection<LocalResourceRequest>>();
      rsrcs.put(LocalResourceVisibility.PRIVATE, Collections.singletonList(req));

View Full Code Here


  private static LocalResource getMockedResource(Random r, 
      LocalResourceVisibility vis) {
    String name = Long.toHexString(r.nextLong());
    URL url = getPath("/local/PRIVATE/" + name);
    LocalResource rsrc =
        BuilderUtils.newLocalResource(url, LocalResourceType.FILE, vis,
            r.nextInt(1024) + 1024L, r.nextInt(1024) + 2048L);
    return rsrc;
  }

View Full Code Here

      doLocalizeResources(false, skipRsrcCount);
    }
    
    public void failLocalizeSpecificResource(String rsrcKey)
        throws URISyntaxException {
      LocalResource rsrc = localResources.get(rsrcKey);
      LocalResourceRequest req = new LocalResourceRequest(rsrc);
      Exception e = new Exception("Fake localization error");
      c.handle(new ContainerResourceFailedEvent(c.getContainerID(), req, e));
      drainDispatcherEvents();
    }

View Full Code Here

      LocalResourceRequest key = request.getResource().getRequest();
      LOG.info("Downloading public rsrc:" + key);
      synchronized (attempts) {
        List<LocalizerResourceRequestEvent> sigh = attempts.get(key);
        if (null == sigh) {
          LocalResource resource = request.getResource().getRequest();
          try {
            Path publicDirDestPath = dirsHandler.getLocalPathForWrite(
                "." + Path.SEPARATOR + ContainerLocalizer.FILECACHE,
                ContainerLocalizer.getEstimatedSize(resource), true);
            pending.put(queue.submit(new FSDownload(

View Full Code Here

          i.remove();
          continue;
        }
        if (nRsrc.tryAcquire()) {
          LocalResourceRequest nextRsrc = nRsrc.getRequest();
          LocalResource next =
            recordFactory.newRecordInstance(LocalResource.class);
          next.setResource(
              ConverterUtils.getYarnUrlFromPath(nextRsrc.getPath()));
          next.setTimestamp(nextRsrc.getTimestamp());
          next.setType(nextRsrc.getType());
          next.setVisibility(evt.getVisibility());
          scheduled.put(nextRsrc, evt);
          return next;
        }
      }
      return null;

View Full Code Here

        recordFactory.newRecordInstance(LocalizerHeartbeatResponse.class);


      // The localizer has just spawned. Start giving it resources for
      // remote-fetching.
      if (remoteResourceStatuses.isEmpty()) {
        LocalResource next = findNextResource();
        if (next != null) {
          response.setLocalizerAction(LocalizerAction.LIVE);
          response.addResource(next);
        } else if (pending.isEmpty()) {
          // TODO: Synchronization
          response.setLocalizerAction(LocalizerAction.DIE);
        } else {
          response.setLocalizerAction(LocalizerAction.LIVE);
        }
        return response;
      }


      for (LocalResourceStatus stat : remoteResourceStatuses) {
        LocalResource rsrc = stat.getResource();
        LocalResourceRequest req = null;
        try {
          req = new LocalResourceRequest(rsrc);
        } catch (URISyntaxException e) {
          // TODO fail? Already translated several times...
        }
        LocalizerResourceRequestEvent assoc = scheduled.get(req);
        if (assoc == null) {
          // internal error
          LOG.error("Unknown resource reported: " + req);
          continue;
        }
        switch (stat.getStatus()) {
          case FETCH_SUCCESS:
            // notify resource
            try {
              assoc.getResource().handle(
                  new ResourceLocalizedEvent(req,
                    ConverterUtils.getPathFromYarnURL(stat.getLocalPath()),
                    stat.getLocalSize()));
            } catch (URISyntaxException e) { }
            if (pending.isEmpty()) {
              // TODO: Synchronization
              response.setLocalizerAction(LocalizerAction.DIE);
              break;
            }
            response.setLocalizerAction(LocalizerAction.LIVE);
            LocalResource next = findNextResource();
            if (next != null) {
              response.addResource(next);
            }
            break;
          case FETCH_PENDING:

View Full Code Here

    containerLaunchContext.setUser(this.user);


    URL resource_alpha =
        ConverterUtils.getYarnUrlFromPath(localFS
            .makeQualified(new Path(scriptFile.getAbsolutePath())));
    LocalResource rsrc_alpha =
        recordFactory.newRecordInstance(LocalResource.class);
    rsrc_alpha.setResource(resource_alpha);
    rsrc_alpha.setSize(-1);
    rsrc_alpha.setVisibility(LocalResourceVisibility.APPLICATION);
    rsrc_alpha.setType(LocalResourceType.FILE);
    rsrc_alpha.setTimestamp(scriptFile.lastModified());
    String destinationFile = "dest_file";
    Map<String, LocalResource> localResources = 
        new HashMap<String, LocalResource>();
    localResources.put(destinationFile, rsrc_alpha);
    containerLaunchContext.setLocalResources(localResources);

View Full Code Here

    doReturn(new FSDataInputStream(new FakeFSDataInputStream(appTokens))
        ).when(spylfs).open(tokenPath);


    // mock heartbeat responses from NM
    LocalizationProtocol nmProxy = mock(LocalizationProtocol.class);
    LocalResource rsrcA = getMockRsrc(r, LocalResourceVisibility.PRIVATE);
    LocalResource rsrcB = getMockRsrc(r, LocalResourceVisibility.PRIVATE);
    LocalResource rsrcC = getMockRsrc(r, LocalResourceVisibility.APPLICATION);
    LocalResource rsrcD = getMockRsrc(r, LocalResourceVisibility.PRIVATE);
    when(nmProxy.heartbeat(isA(LocalizerStatus.class)))
      .thenReturn(new MockLocalizerHeartbeatResponse(LocalizerAction.LIVE,
            Collections.singletonList(rsrcA)))
      .thenReturn(new MockLocalizerHeartbeatResponse(LocalizerAction.LIVE,
            Collections.singletonList(rsrcB)))
      .thenReturn(new MockLocalizerHeartbeatResponse(LocalizerAction.LIVE,
            Collections.singletonList(rsrcC)))
      .thenReturn(new MockLocalizerHeartbeatResponse(LocalizerAction.LIVE,
            Collections.singletonList(rsrcD)))
      .thenReturn(new MockLocalizerHeartbeatResponse(LocalizerAction.LIVE,
            Collections.<LocalResource>emptyList()))
      .thenReturn(new MockLocalizerHeartbeatResponse(LocalizerAction.DIE,
            null));
    doReturn(new FakeDownload(rsrcA.getResource().getFile(), true)).when(
        localizer).download(isA(LocalDirAllocator.class), eq(rsrcA),
        isA(UserGroupInformation.class));
    doReturn(new FakeDownload(rsrcB.getResource().getFile(), true)).when(
        localizer).download(isA(LocalDirAllocator.class), eq(rsrcB),
        isA(UserGroupInformation.class));
    doReturn(new FakeDownload(rsrcC.getResource().getFile(), true)).when(
        localizer).download(isA(LocalDirAllocator.class), eq(rsrcC),
        isA(UserGroupInformation.class));
    doReturn(new FakeDownload(rsrcD.getResource().getFile(), true)).when(
        localizer).download(isA(LocalDirAllocator.class), eq(rsrcD),
        isA(UserGroupInformation.class));
    doReturn(nmProxy).when(localizer).getProxy(nmAddr);
    doNothing().when(localizer).sleep(anyInt());

View Full Code Here

    return mockRF;
  }


  static LocalResource getMockRsrc(Random r,
      LocalResourceVisibility vis) {
    LocalResource rsrc = mock(LocalResource.class);


    String name = Long.toHexString(r.nextLong());
    URL uri = mock(org.apache.hadoop.yarn.api.records.URL.class);
    when(uri.getScheme()).thenReturn("file");
    when(uri.getHost()).thenReturn(null);
    when(uri.getFile()).thenReturn("/local/" + vis + "/" + name);


    when(rsrc.getResource()).thenReturn(uri);
    when(rsrc.getSize()).thenReturn(r.nextInt(1024) + 1024L);
    when(rsrc.getTimestamp()).thenReturn(r.nextInt(1024) + 2048L);
    when(rsrc.getType()).thenReturn(LocalResourceType.FILE);
    when(rsrc.getVisibility()).thenReturn(vis);


    return rsrc;
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.yarn.api.records.LocalResource

cascading.flow.tez.Hadoop2TezFlowStep

cascading.flow.tez.util.TezUtil

com.alibaba.jstorm.yarn.Util

com.cloudera.iterativereduce.Utils

com.cloudera.kitten.lua.LuaContainerLaunchParameters

com.hortonworks.simpleyarnapp.Client

com.yahoo.storm.yarn.Util

eu.stratosphere.yarn.ApplicationMaster

eu.stratosphere.yarn.Client

org.apache.flink.yarn.appMaster.ApplicationMaster

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.