Package org.apache.hadoop.yarn.server.nodemanager

Examples of org.apache.hadoop.yarn.server.nodemanager.DeletionService$FileDeletion


  }

  @Test(timeout=20000)
  @SuppressWarnings("unchecked")
  public void testStopAfterError() throws Exception {
    DeletionService delSrvc = mock(DeletionService.class);

    // get the AppLogAggregationImpl thread to crash
    LocalDirsHandlerService mockedDirSvc = mock(LocalDirsHandlerService.class);
    when(mockedDirSvc.getLogDirs()).thenThrow(new RuntimeException());
View Full Code Here


  }

  @Test
  @SuppressWarnings("unchecked")
  public void testLogAggregatorCleanup() throws Exception {
    DeletionService delSrvc = mock(DeletionService.class);

    // get the AppLogAggregationImpl thread to crash
    LocalDirsHandlerService mockedDirSvc = mock(LocalDirsHandlerService.class);

    DrainDispatcher dispatcher = createDispatcher();
View Full Code Here

public class TestNonAggregatingLogHandler {

  @Test
  @SuppressWarnings("unchecked")
  public void testLogDeletion() {
    DeletionService delService = mock(DeletionService.class);
    Configuration conf = new YarnConfiguration();
    String user = "testuser";

    File[] localLogDirs = new File[2];
    localLogDirs[0] =
View Full Code Here

  }

  @Test
  @SuppressWarnings("unchecked")
  public void testDelayedDelete() {
    DeletionService delService = mock(DeletionService.class);
    Configuration conf = new YarnConfiguration();
    String user = "testuser";

    File[] localLogDirs = new File[2];
    localLogDirs[0] =
View Full Code Here

      EventHandler<LocalizerEvent> containerEventHandler =
          mock(EventHandler.class);
      dispatcher.register(LocalizerEventType.class, localizerEventHandler);
      dispatcher.register(ContainerEventType.class, containerEventHandler);

      DeletionService mockDelService = mock(DeletionService.class);

      ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
      LocalizerContext lc1 = new LocalizerContext(user, cId1, null);
      ContainerId cId2 = BuilderUtils.newContainerId(1, 1, 1, 2);
      LocalizerContext lc2 = new LocalizerContext(user, cId2, null);
View Full Code Here

    List<Path> dirs = buildDirs(r, base, 20);
    createDirs(new Path("."), dirs);
    FakeDefaultContainerExecutor exec = new FakeDefaultContainerExecutor();
    Configuration conf = new Configuration();
    exec.setConf(conf);
    DeletionService del = new DeletionService(exec);
    del.init(conf);
    del.start();
    try {
      for (Path p : dirs) {
        del.delete((Long.parseLong(p.getName()) % 2) == 0 ? null : "dingo",
            p, null);
      }

      int msecToWait = 20 * 1000;
      for (Path p : dirs) {
        while (msecToWait > 0 && lfs.util().exists(p)) {
          Thread.sleep(100);
          msecToWait -= 100;
        }
        assertFalse(lfs.util().exists(p));
      }
    } finally {
      del.stop();
    }
  }
View Full Code Here

    createDirs(new Path("."), baseDirs);
    List<Path> content = buildDirs(r, new Path("."), 10);
    for (Path b : baseDirs) {
      createDirs(b, content);
    }
    DeletionService del =
      new DeletionService(new FakeDefaultContainerExecutor());
    del.init(new Configuration());
    del.start();
    try {
      for (Path p : content) {
        assertTrue(lfs.util().exists(new Path(baseDirs.get(0), p)));
        del.delete((Long.parseLong(p.getName()) % 2) == 0 ? null : "dingo",
            p, baseDirs.toArray(new Path[4]));
      }

      int msecToWait = 20 * 1000;
      for (Path p : baseDirs) {
        for (Path q : content) {
          Path fp = new Path(p, q);
          while (msecToWait > 0 && lfs.util().exists(fp)) {
            Thread.sleep(100);
            msecToWait -= 100;
          }
          assertFalse(lfs.util().exists(fp));
        }
      }
    } finally {
      del.stop();
    }
  }
View Full Code Here

    createDirs(new Path("."), dirs);
    FakeDefaultContainerExecutor exec = new FakeDefaultContainerExecutor();
    Configuration conf = new Configuration();
    conf.setInt(YarnConfiguration.DEBUG_NM_DELETE_DELAY_SEC, -1);
    exec.setConf(conf);
    DeletionService del = new DeletionService(exec);
    del.init(conf);
    del.start();
    try {
      for (Path p : dirs) {
        del.delete((Long.parseLong(p.getName()) % 2) == 0 ? null : "dingo", p,
            null);
      }
      int msecToWait = 20 * 1000;
      for (Path p : dirs) {
        while (msecToWait > 0 && lfs.util().exists(p)) {
          Thread.sleep(100);
          msecToWait -= 100;
        }
        assertTrue(lfs.util().exists(p));
      }
    } finally {
      del.stop();
    }
  }
View Full Code Here

    }
  }

  @Test
  public void testStopWithDelayedTasks() throws Exception {
    DeletionService del = new DeletionService(Mockito.mock(ContainerExecutor.class));
    Configuration conf = new YarnConfiguration();
    conf.setInt(YarnConfiguration.DEBUG_NM_DELETE_DELAY_SEC, 60);
    del.init(conf);
    del.start();
    try {
      del.delete("dingo", new Path("/does/not/exist"));
    } finally {
      del.stop();
    }
    assertTrue(del.isTerminated());
  }
View Full Code Here

    conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "077");
    AsyncDispatcher dispatcher = new AsyncDispatcher();
    dispatcher.init(new Configuration());

    ContainerExecutor exec = mock(ContainerExecutor.class);
    DeletionService delService = spy(new DeletionService(exec));
    delService.init(new Configuration());
    delService.start();

    AbstractFileSystem spylfs =
      spy(FileContext.getLocalFSFileContext().getDefaultFileSystem());
    FileContext lfs = FileContext.getFileContext(spylfs, conf);
    doNothing().when(spylfs).mkdir(
        isA(Path.class), isA(FsPermission.class), anyBoolean());

    List<Path> localDirs = new ArrayList<Path>();
    String[] sDirs = new String[4];
    for (int i = 0; i < 4; ++i) {
      localDirs.add(lfs.makeQualified(new Path(basedir, i + "")));
      sDirs[i] = localDirs.get(i).toString();
    }
    conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs);
    LocalDirsHandlerService diskhandler = new LocalDirsHandlerService();
    diskhandler.init(conf);

    ResourceLocalizationService locService =
      spy(new ResourceLocalizationService(dispatcher, exec, delService,
                                          diskhandler));
    doReturn(lfs)
      .when(locService).getLocalFileContext(isA(Configuration.class));
    try {
      dispatcher.start();

      // initialize ResourceLocalizationService
      locService.init(conf);

      final FsPermission defaultPerm = new FsPermission((short)0755);

      // verify directory creation
      for (Path p : localDirs) {
        p = new Path((new URI(p.toString())).getPath());
        Path usercache = new Path(p, ContainerLocalizer.USERCACHE);
        verify(spylfs)
          .mkdir(eq(usercache),
              eq(defaultPerm), eq(true));
        Path publicCache = new Path(p, ContainerLocalizer.FILECACHE);
        verify(spylfs)
          .mkdir(eq(publicCache),
              eq(defaultPerm), eq(true));
        Path nmPriv = new Path(p, ResourceLocalizationService.NM_PRIVATE_DIR);
        verify(spylfs).mkdir(eq(nmPriv),
            eq(ResourceLocalizationService.NM_PRIVATE_PERM), eq(true));
      }
    } finally {
      dispatcher.stop();
      delService.stop();
    }
  }
View Full Code Here

TOP

Related Classes of org.apache.hadoop.yarn.server.nodemanager.DeletionService$FileDeletion

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.