Examples of org.apache.hadoop.yarn.api.records.LocalResource

Package org.apache.hadoop.yarn.api.records

Examples of org.apache.hadoop.yarn.api.records.LocalResource

org.apache.hadoop.yarn.api.records.LocalResource

LocalResource represents a local resource required to run a container.

The NodeManager is responsible for localizing the resource prior to launching the container.

Applications can specify {@link LocalResourceType} and {@link LocalResourceVisibility}.
@see LocalResourceType @see LocalResourceVisibility @see ContainerLaunchContext @see ApplicationSubmissionContext @see ContainerManager#startContainer(org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest)


      // The container for the eventual shell commands needs its own local resources too. 
      // In this scenario, if a shell script is specified, we need to have it copied 
      // and made available to the container. 
      if (!shellScriptPath.isEmpty()) {
        LocalResource shellRsrc = Records.newRecord(LocalResource.class);
        shellRsrc.setType(LocalResourceType.FILE);
        shellRsrc.setVisibility(LocalResourceVisibility.APPLICATION);
        try {
          shellRsrc.setResource(ConverterUtils.getYarnUrlFromURI(new URI(shellScriptPath)));
        } catch (URISyntaxException e) {
          LOG.error("Error when trying to use shell script path specified in env"
              + ", path=" + shellScriptPath);
          e.printStackTrace();


          // A failure scenario on bad input such as invalid shell script path 
          // We know we cannot continue launching the container 
          // so we should release it.
          // TODO
          numCompletedContainers.incrementAndGet();
          numFailedContainers.incrementAndGet();
          return;
        }
        shellRsrc.setTimestamp(shellScriptPathTimestamp);
        shellRsrc.setSize(shellScriptPathLen);
        localResources.put(ExecShellStringPath, shellRsrc);
      }
      ctx.setLocalResources(localResources);


      // Set the necessary command to execute on the allocated container

View Full Code Here

          : u.getFragment());
        if (name.isAbsolute()) {
          throw new IllegalArgumentException("Resource name must be relative");
        }
        String linkName = name.toUri().getPath();
        LocalResource orig = localResources.get(linkName);
        org.apache.hadoop.yarn.api.records.URL url = 
          ConverterUtils.getYarnUrlFromURI(p.toUri());
        if(orig != null && !orig.getResource().equals(url)) {
          LOG.warn(
              getResourceDescription(orig.getType()) + 
              toString(orig.getResource()) + " conflicts with " + 
              getResourceDescription(type) + toString(url) + 
              " This will be an error in Hadoop 2.0");
          continue;
        }
        localResources.put(

View Full Code Here

      // //////////// Set up JobJar to be localized properly on the remote NM.
      String jobJar = conf.get(MRJobConfig.JAR);
      if (jobJar != null) {
        Path remoteJobJar = (new Path(jobJar)).makeQualified(remoteFS
            .getUri(), remoteFS.getWorkingDirectory());
        LocalResource rc = createLocalResource(remoteFS, remoteJobJar,
            LocalResourceType.PATTERN, LocalResourceVisibility.APPLICATION);
        String pattern = conf.getPattern(JobContext.JAR_UNPACK_PATTERN, 
            JobConf.UNPACK_JAR_PATTERN_DEFAULT).pattern();
        rc.setPattern(pattern);
        localResources.put(MRJobConfig.JOB_JAR, rc);
        LOG.info("The job-jar file on the remote FS is "
            + remoteJobJar.toUri().toASCIIString());
      } else {
        // Job jar may be null. For e.g, for pipes, the job jar is the hadoop

View Full Code Here

      long seed = r.nextLong();
      System.out.println("SEED: " + seed);
      r.setSeed(seed);
      
      // Send localization requests for one resource of each type.
      final LocalResource privResource = getPrivateMockedResource(r);
      final LocalResourceRequest privReq =
          new LocalResourceRequest(privResource);
      
      final LocalResource pubResource = getPublicMockedResource(r);
      final LocalResourceRequest pubReq = new LocalResourceRequest(pubResource);
      final LocalResource pubResource2 = getPublicMockedResource(r);
      final LocalResourceRequest pubReq2 =
          new LocalResourceRequest(pubResource2);
      
      final LocalResource appResource = getAppMockedResource(r);
      final LocalResourceRequest appReq = new LocalResourceRequest(appResource);
      
      Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req =
          new HashMap<LocalResourceVisibility, 
                      Collection<LocalResourceRequest>>();

View Full Code Here

      FSDataOutputStream out =
        new FSDataOutputStream(new DataOutputBuffer(), null);
      doReturn(out).when(spylfs).createInternal(isA(Path.class),
          isA(EnumSet.class), isA(FsPermission.class), anyInt(), anyShort(),
          anyLong(), isA(Progressable.class), isA(ChecksumOpt.class), anyBoolean());
      final LocalResource resource = getPrivateMockedResource(r);
      final LocalResourceRequest req = new LocalResourceRequest(resource);
      Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs =
        new HashMap<LocalResourceVisibility, 
                    Collection<LocalResourceRequest>>();
      rsrcs.put(LocalResourceVisibility.PRIVATE, Collections.singletonList(req));

View Full Code Here


  private static LocalResource getMockedResource(Random r, 
      LocalResourceVisibility vis) {
    String name = Long.toHexString(r.nextLong());
    URL url = getPath("/local/PRIVATE/" + name);
    LocalResource rsrc =
        BuilderUtils.newLocalResource(url, LocalResourceType.FILE, vis,
            r.nextInt(1024) + 1024L, r.nextInt(1024) + 2048L);
    return rsrc;
  }

View Full Code Here

    return clientCache.getClient(jobId).getJobStatus(jobId);
  }


  private LocalResource createApplicationResource(FileContext fs, Path p, LocalResourceType type)
      throws IOException {
    LocalResource rsrc = recordFactory.newRecordInstance(LocalResource.class);
    FileStatus rsrcStat = fs.getFileStatus(p);
    rsrc.setResource(ConverterUtils.getYarnUrlFromPath(fs
        .getDefaultFileSystem().resolvePath(rsrcStat.getPath())));
    rsrc.setSize(rsrcStat.getLen());
    rsrc.setTimestamp(rsrcStat.getModificationTime());
    rsrc.setType(type);
    rsrc.setVisibility(LocalResourceVisibility.APPLICATION);
    return rsrc;
  }

View Full Code Here

    localResources.put(MRJobConfig.JOB_CONF_FILE,
        createApplicationResource(defaultFileContext,
            jobConfPath, LocalResourceType.FILE));
    if (jobConf.get(MRJobConfig.JAR) != null) {
      Path jobJarPath = new Path(jobConf.get(MRJobConfig.JAR));
      LocalResource rc = createApplicationResource(defaultFileContext,
          jobJarPath, 
          LocalResourceType.PATTERN);
      String pattern = conf.getPattern(JobContext.JAR_UNPACK_PATTERN, 
          JobConf.UNPACK_JAR_PATTERN_DEFAULT).pattern();
      rc.setPattern(pattern);
      localResources.put(MRJobConfig.JOB_JAR, rc);
    } else {
      // Job jar may be null. For e.g, for pipes, the job jar is the hadoop
      // mapreduce jar itself which is already on the classpath.
      LOG.info("Job jar is not present. "

View Full Code Here

      // //////////// Set up JobJar to be localized properly on the remote NM.
      String jobJar = conf.get(MRJobConfig.JAR);
      if (jobJar != null) {
        Path remoteJobJar = (new Path(jobJar)).makeQualified(remoteFS
            .getUri(), remoteFS.getWorkingDirectory());
        LocalResource rc = createLocalResource(remoteFS, remoteJobJar,
            LocalResourceType.PATTERN, LocalResourceVisibility.APPLICATION);
        String pattern = conf.getPattern(JobContext.JAR_UNPACK_PATTERN, 
            JobConf.UNPACK_JAR_PATTERN_DEFAULT).pattern();
        rc.setPattern(pattern);
        localResources.put(MRJobConfig.JOB_JAR, rc);
        LOG.info("The job-jar file on the remote FS is "
            + remoteJobJar.toUri().toASCIIString());
      } else {
        // Job jar may be null. For e.g, for pipes, the job jar is the hadoop

View Full Code Here

   * @param target the file to send to the remote container.
   */
  public static void addFileToResourceMap(Map<String, LocalResource>
    localResources, FileSystem fs, Path target)
    throws IOException {
    LocalResource resource = Records.newRecord(LocalResource.class);
    FileStatus destStatus = fs.getFileStatus(target);
    resource.setResource(ConverterUtils.getYarnUrlFromURI(target.toUri()));
    resource.setSize(destStatus.getLen());
    resource.setTimestamp(destStatus.getModificationTime());
    resource.setType(LocalResourceType.FILE); // use FILE, even for jars!
    resource.setVisibility(LocalResourceVisibility.APPLICATION);
    localResources.put(target.getName(), resource);
    LOG.info("Registered file in LocalResources: " + target.getName());
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.yarn.api.records.LocalResource

cascading.flow.tez.Hadoop2TezFlowStep

cascading.flow.tez.util.TezUtil

com.alibaba.jstorm.yarn.Util

com.cloudera.iterativereduce.Utils

com.cloudera.kitten.lua.LuaContainerLaunchParameters

com.hortonworks.simpleyarnapp.Client

com.yahoo.storm.yarn.Util

eu.stratosphere.yarn.ApplicationMaster

eu.stratosphere.yarn.Client

org.apache.flink.yarn.appMaster.ApplicationMaster

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.