Examples of org.apache.hadoop.yarn.api.records.LocalResourceVisibility

Package org.apache.hadoop.yarn.api.records

Examples of org.apache.hadoop.yarn.api.records.LocalResourceVisibility

org.apache.hadoop.yarn.api.records.LocalResourceVisibility

    TransferEntry entry = (TransferEntry) hdfsEntries.toArray()[0];
    String path = TestUtils.readField("path", entry);
    assertThat(path, is("/tmp/foo.jar"));
    LocalResourceType type = TestUtils.readField("type", entry);
    assertThat(type, is(LocalResourceType.FILE));
    LocalResourceVisibility visibility = TestUtils.readField("visibility", entry);
    assertThat(visibility, is(LocalResourceVisibility.PUBLIC));


    entry = (TransferEntry) hdfsEntries.toArray()[1];
    path = TestUtils.readField("path", entry);
    assertThat(path, is("/tmp/jee.jar"));

View Full Code Here

    TransferEntry entry = (TransferEntry) hdfsEntries.toArray()[0];
    String path = TestUtils.readField("path", entry);
    assertThat(path, is("/tmp/foo.jar"));
    LocalResourceType type = TestUtils.readField("type", entry);
    assertThat(type, is(LocalResourceType.FILE));
    LocalResourceVisibility visibility = TestUtils.readField("visibility", entry);
    assertThat(visibility, is(LocalResourceVisibility.APPLICATION));
  }

View Full Code Here

    private Path getPathForLocalization(LocalResource rsrc) throws IOException,
        URISyntaxException {
      String user = context.getUser();
      ApplicationId appId =
          context.getContainerId().getApplicationAttemptId().getApplicationId();
      LocalResourceVisibility vis = rsrc.getVisibility();
      LocalResourcesTracker tracker =
          getLocalResourcesTracker(vis, user, appId);
      String cacheDirectory = null;
      if (vis == LocalResourceVisibility.PRIVATE) {// PRIVATE Only
        cacheDirectory = getUserFileCachePath(user);

View Full Code Here

    LocalDirAllocator dirs =
      new LocalDirAllocator(TestFSDownload.class.getName());
    int[] sizes = new int[10];
    for (int i = 0; i < 10; ++i) {
      sizes[i] = rand.nextInt(512) + 512;
      LocalResourceVisibility vis = LocalResourceVisibility.PUBLIC;
      switch (i%3) {
      case 1:
        vis = LocalResourceVisibility.PRIVATE;
        break;
      case 2:

View Full Code Here

      new HashMap<LocalResource,Future<Path>>();
    ExecutorService exec = Executors.newSingleThreadExecutor();
    LocalDirAllocator dirs =
      new LocalDirAllocator(TestFSDownload.class.getName());
    for (int i = 0; i < 5; ++i) {
      LocalResourceVisibility vis = LocalResourceVisibility.PUBLIC;
      switch (rand.nextInt()%3) {
      case 1:
        vis = LocalResourceVisibility.PRIVATE;
        break;
      case 2:

View Full Code Here


      if (tezJarUris.length == 1 && (tezJarUris[0].endsWith(".tar.gz") || tezJarUris[0].endsWith(".tgz"))) {
        String fileName = tezJarUris[0];
        if (fileName.endsWith(".tar.gz") || fileName.endsWith(".tgz")) {
          FileStatus fStatus = getLRFileStatus(fileName, conf, false)[0];
          LocalResourceVisibility lrVisibility;
          if (checkAncestorPermissionsForAllUsers(conf, fileName, FsAction.EXECUTE) &&
              fStatus.getPermission().getOtherAction().implies(FsAction.READ)) {
            lrVisibility = LocalResourceVisibility.PUBLIC;
          } else {
            lrVisibility = LocalResourceVisibility.PRIVATE;
          }
          tezJarResources.put(TezConstants.TEZ_TAR_LR_NAME,
              LocalResource.newInstance(
                  ConverterUtils.getYarnUrlFromPath(fStatus.getPath()),
                  LocalResourceType.ARCHIVE,
                  lrVisibility,
                  fStatus.getLen(),
                  fStatus.getModificationTime()));
          tezJarPaths.add(fStatus.getPath());
        }
      } else { // Treat as non-archives - each entry being a directory
        for (String tezJarUri : tezJarUris) {
          boolean ancestorsHavePermission = checkAncestorPermissionsForAllUsers(conf, tezJarUri,
              FsAction.EXECUTE);
          FileStatus [] fileStatuses = getLRFileStatus(tezJarUri, conf, true);
          for (FileStatus fStatus : fileStatuses) {
            if (fStatus.isDirectory()) {
              // Skip directories - since tez.lib.uris is not recursive.
              continue;
            }
            LocalResourceVisibility lrVisibility;
            if (ancestorsHavePermission &&
                fStatus.getPermission().getOtherAction().implies(FsAction.READ)) {
              lrVisibility = LocalResourceVisibility.PUBLIC;
            } else {
              lrVisibility = LocalResourceVisibility.PRIVATE;

View Full Code Here

      new HashMap<LocalResource,Future<Path>>();
    ExecutorService exec = Executors.newSingleThreadExecutor();
    LocalDirAllocator dirs =
      new LocalDirAllocator(TestFSDownload.class.getName());
    int size = 512;
    LocalResourceVisibility vis = LocalResourceVisibility.PUBLIC;
    Path path = new Path(basedir, "test-file");
    LocalResource rsrc = createFile(files, path, size, rand, vis);
    rsrcVis.put(rsrc, vis);
    Path destPath = dirs.getLocalPathForWrite(
        basedir.toString(), size, conf);

View Full Code Here

    LocalDirAllocator dirs =
      new LocalDirAllocator(TestFSDownload.class.getName());
    int[] sizes = new int[10];
    for (int i = 0; i < 10; ++i) {
      sizes[i] = rand.nextInt(512) + 512;
      LocalResourceVisibility vis = LocalResourceVisibility.PRIVATE;
      if (i%2 == 1) {
        vis = LocalResourceVisibility.APPLICATION;
      }
      Path p = new Path(basedir, "" + i);
      LocalResource rsrc = createFile(files, p, sizes[i], rand, vis);

View Full Code Here

    ExecutorService exec = Executors.newSingleThreadExecutor();
    LocalDirAllocator dirs = new LocalDirAllocator(
        TestFSDownload.class.getName());


    int size = rand.nextInt(512) + 512;
    LocalResourceVisibility vis = LocalResourceVisibility.PRIVATE;


    Path p = new Path(basedir, "" + 1);
    LocalResource rsrc = createTarFile(files, p, size, rand, vis);
    Path destPath = dirs.getLocalPathForWrite(basedir.toString(), size, conf);
    destPath = new Path (destPath,

View Full Code Here

    ExecutorService exec = Executors.newSingleThreadExecutor();
    LocalDirAllocator dirs = new LocalDirAllocator(
        TestFSDownload.class.getName());


    int size = rand.nextInt(512) + 512;
    LocalResourceVisibility vis = LocalResourceVisibility.PRIVATE;


    Path p = new Path(basedir, "" + 1);
    LocalResource rsrcjar = createJarFile(files, p, size, rand, vis);
    rsrcjar.setType(LocalResourceType.PATTERN);
    Path destPathjar = dirs.getLocalPathForWrite(basedir.toString(), size, conf);

View Full Code Here

0 1 2 3 4 5

TOP

Related Classes of org.apache.hadoop.yarn.api.records.LocalResourceVisibility

com.alibaba.jstorm.yarn.JStormOnYarn

com.yahoo.storm.yarn.StormOnYarn

org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService$LocalizerRunner

org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.TestLocalizedResource

org.apache.hadoop.yarn.util.TestFSDownload

org.apache.tez.client.TezClientUtils

org.springframework.yarn.config.annotation.DefaultLocalresourcesAnnotationTests

org.springframework.yarn.config.LocalresourcesNamespaceTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.