Package org.apache.hadoop.yarn.api.records

Examples of org.apache.hadoop.yarn.api.records.LocalResourceVisibility


    ExecutorService exec = Executors.newSingleThreadExecutor();
    LocalDirAllocator dirs = new LocalDirAllocator(
        TestFSDownload.class.getName());

    int size = rand.nextInt(512) + 512;
    LocalResourceVisibility vis = LocalResourceVisibility.PRIVATE;

    Path p = new Path(basedir, "" + 1);
    LocalResource rsrczip = createZipFile(files, p, size, rand, vis);
    Path destPathjar = dirs.getLocalPathForWrite(basedir.toString(), size, conf);
    FSDownload fsdzip = new FSDownload(files,
View Full Code Here


      new HashMap<LocalResource,Future<Path>>();
    ExecutorService exec = Executors.newSingleThreadExecutor();
    LocalDirAllocator dirs =
      new LocalDirAllocator(TestFSDownload.class.getName());
    for (int i = 0; i < 5; ++i) {
      LocalResourceVisibility vis = LocalResourceVisibility.PRIVATE;
      if (i%2 == 1) {
        vis = LocalResourceVisibility.APPLICATION;
      }

      Path p = new Path(basedir, "dir" + i + ".jar");
View Full Code Here

      new HashMap<LocalResource,Future<Path>>();
    ExecutorService exec = Executors.newSingleThreadExecutor();
    LocalDirAllocator dirs =
      new LocalDirAllocator(TestFSDownload.class.getName());
    int size = 512;
    LocalResourceVisibility vis = LocalResourceVisibility.PUBLIC;
    Path path = new Path(basedir, "test-file");
    LocalResource rsrc = createFile(files, path, size, rand, vis);
    rsrcVis.put(rsrc, vis);
    Path destPath = dirs.getLocalPathForWrite(
        basedir.toString(), size, conf);
View Full Code Here

    LocalDirAllocator dirs =
      new LocalDirAllocator(TestFSDownload.class.getName());
    int[] sizes = new int[10];
    for (int i = 0; i < 10; ++i) {
      sizes[i] = rand.nextInt(512) + 512;
      LocalResourceVisibility vis = LocalResourceVisibility.PRIVATE;
      if (i%2 == 1) {
        vis = LocalResourceVisibility.APPLICATION;
      }
      Path p = new Path(basedir, "" + i);
      LocalResource rsrc = createFile(files, p, sizes[i], rand, vis);
View Full Code Here

    ExecutorService exec = Executors.newSingleThreadExecutor();
    LocalDirAllocator dirs = new LocalDirAllocator(
        TestFSDownload.class.getName());

    int size = rand.nextInt(512) + 512;
    LocalResourceVisibility vis = LocalResourceVisibility.PRIVATE;

    Path p = new Path(basedir, "" + 1);
    LocalResource rsrc = null;
    switch (fileType) {
    case TAR:
View Full Code Here

      new HashMap<LocalResource,Future<Path>>();
    ExecutorService exec = Executors.newSingleThreadExecutor();
    LocalDirAllocator dirs =
      new LocalDirAllocator(TestFSDownload.class.getName());
    for (int i = 0; i < 5; ++i) {
      LocalResourceVisibility vis = LocalResourceVisibility.PRIVATE;
      if (i%2 == 1) {
        vis = LocalResourceVisibility.APPLICATION;
      }

      Path p = new Path(basedir, "dir" + i + ".jar");
View Full Code Here

    destPath =
        new Path(destPath, Long.toString(uniqueNumberGenerator
            .incrementAndGet()));

    Path p = new Path(basedir, "dir" + 0 + ".jar");
    LocalResourceVisibility vis = LocalResourceVisibility.PRIVATE;
    LocalResource rsrc = createJar(files, p, vis);
    FSDownload fsd =
        new FSDownload(files, UserGroupInformation.getCurrentUser(), conf,
            destPath, rsrc);
    Future<Path> rPath = singleThreadedExec.submit(fsd);
View Full Code Here

            zip = new Path(storm_zip_location);
        } else {
            zip = new Path("/lib/storm/"+stormVersion+"/storm.zip");        
        }
        _stormConf.put("storm.zip.path", zip.makeQualified(fs).toUri().getPath());
        LocalResourceVisibility visibility = LocalResourceVisibility.PUBLIC;
        _stormConf.put("storm.zip.visibility", "PUBLIC");
        if (!Util.isPublic(fs, zip)) {
          visibility = LocalResourceVisibility.APPLICATION;
          _stormConf.put("storm.zip.visibility", "APPLICATION");
        }
View Full Code Here

      // mock resource
      LocalResource apiRsrc = createMockResource();

      final ContainerId container0 = getMockContainer(0);
      final Credentials creds0 = new Credentials();
      final LocalResourceVisibility vis0 = LocalResourceVisibility.PRIVATE;
      final LocalizerContext ctxt0 =
        new LocalizerContext("yak", container0, creds0);
      LocalResourceRequest rsrcA = new LocalResourceRequest(apiRsrc);
      LocalizedResource local = new LocalizedResource(rsrcA, dispatcher);
      local.handle(new ResourceRequestEvent(rsrcA, vis0, ctxt0));
      dispatcher.await();

      // Register C0, verify request event
      LocalizerEventMatcher matchesL0Req =
        new LocalizerEventMatcher(container0, creds0, vis0,
            LocalizerEventType.REQUEST_RESOURCE_LOCALIZATION);
      verify(localizerBus).handle(argThat(matchesL0Req));
      assertEquals(ResourceState.DOWNLOADING, local.getState());

      // Register C1, verify request event
      final Credentials creds1 = new Credentials();
      final ContainerId container1 = getMockContainer(1);
      final LocalizerContext ctxt1 =
        new LocalizerContext("yak", container1, creds1);
      final LocalResourceVisibility vis1 = LocalResourceVisibility.PUBLIC;
      local.handle(new ResourceRequestEvent(rsrcA, vis1, ctxt1));
      dispatcher.await();
      LocalizerEventMatcher matchesL1Req =
        new LocalizerEventMatcher(container1, creds1, vis1,
            LocalizerEventType.REQUEST_RESOURCE_LOCALIZATION);
      verify(localizerBus).handle(argThat(matchesL1Req));

      // Release C0 container localization, verify no notification
      local.handle(new ResourceReleaseEvent(rsrcA, container0));
      dispatcher.await();
      verify(containerBus, never()).handle(isA(ContainerEvent.class));
      assertEquals(ResourceState.DOWNLOADING, local.getState());

      // Release C1 container localization, verify no notification
      local.handle(new ResourceReleaseEvent(rsrcA, container1));
      dispatcher.await();
      verify(containerBus, never()).handle(isA(ContainerEvent.class));
      assertEquals(ResourceState.INIT, local.getState());

      // Register C2, C3
      final ContainerId container2 = getMockContainer(2);
      final LocalResourceVisibility vis2 = LocalResourceVisibility.PRIVATE;
      final Credentials creds2 = new Credentials();
      final LocalizerContext ctxt2 =
        new LocalizerContext("yak", container2, creds2);

      final ContainerId container3 = getMockContainer(3);
      final LocalResourceVisibility vis3 = LocalResourceVisibility.PRIVATE;
      final Credentials creds3 = new Credentials();
      final LocalizerContext ctxt3 =
        new LocalizerContext("yak", container3, creds3);

      local.handle(new ResourceRequestEvent(rsrcA, vis2, ctxt2));
      local.handle(new ResourceRequestEvent(rsrcA, vis3, ctxt3));
      dispatcher.await();
      LocalizerEventMatcher matchesL2Req =
        new LocalizerEventMatcher(container2, creds2, vis2,
            LocalizerEventType.REQUEST_RESOURCE_LOCALIZATION);
      verify(localizerBus).handle(argThat(matchesL2Req));
      LocalizerEventMatcher matchesL3Req =
        new LocalizerEventMatcher(container3, creds3, vis3,
            LocalizerEventType.REQUEST_RESOURCE_LOCALIZATION);
      verify(localizerBus).handle(argThat(matchesL3Req));

      // Successful localization. verify notification C2, C3
      Path locA = new Path("file:///cache/rsrcA");
      local.handle(new ResourceLocalizedEvent(rsrcA, locA, 10));
      dispatcher.await();
      ContainerEventMatcher matchesC2Localized =
        new ContainerEventMatcher(container2,
            ContainerEventType.RESOURCE_LOCALIZED);
      ContainerEventMatcher matchesC3Localized =
        new ContainerEventMatcher(container3,
            ContainerEventType.RESOURCE_LOCALIZED);
      verify(containerBus).handle(argThat(matchesC2Localized));
      verify(containerBus).handle(argThat(matchesC3Localized));
      assertEquals(ResourceState.LOCALIZED, local.getState());

      // Register C4, verify notification
      final ContainerId container4 = getMockContainer(4);
      final Credentials creds4 = new Credentials();
      final LocalizerContext ctxt4 =
        new LocalizerContext("yak", container4, creds4);
      final LocalResourceVisibility vis4 = LocalResourceVisibility.PRIVATE;
      local.handle(new ResourceRequestEvent(rsrcA, vis4, ctxt4));
      dispatcher.await();
      ContainerEventMatcher matchesC4Localized =
        new ContainerEventMatcher(container4,
            ContainerEventType.RESOURCE_LOCALIZED);
View Full Code Here

      new HashMap<LocalResource,Future<Path>>();
    ExecutorService exec = Executors.newSingleThreadExecutor();
    LocalDirAllocator dirs =
      new LocalDirAllocator(TestFSDownload.class.getName());
    int size = 512;
    LocalResourceVisibility vis = LocalResourceVisibility.PUBLIC;
    Path path = new Path(basedir, "test-file");
    LocalResource rsrc = createFile(files, path, size, rand, vis);
    rsrcVis.put(rsrc, vis);
    Path destPath = dirs.getLocalPathForWrite(
        basedir.toString(), size, conf);
View Full Code Here

TOP

Related Classes of org.apache.hadoop.yarn.api.records.LocalResourceVisibility

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.