Examples of org.apache.hadoop.mapreduce.util.MRAsyncDiskService

Package org.apache.hadoop.mapreduce.util

Examples of org.apache.hadoop.mapreduce.util.MRAsyncDiskService

org.apache.hadoop.mapreduce.util.MRAsyncDiskService
This class is a container of multiple thread pools, each for a volume, so that we can schedule async disk operations easily. Examples of async disk operations are deletion of files. We can move the files to a "toBeDeleted" folder before asychronously deleting it, to make sure the caller can run it faster. Users should not write files into the "toBeDeleted" folder, otherwise the files can be gone any time we restart the MRAsyncDiskService. This class also contains all operations that will be performed by the thread pools.

    if (Thread.currentThread().isInterrupted()) {
      throw new InterruptedException();
    }
    
    // Same with 'localDir' except it's always on the local disk.
    asyncDiskService = new MRAsyncDiskService(FileSystem.getLocal(conf), conf.getLocalDirs());
    asyncDiskService.moveAndDeleteFromEachVolume(SUBDIR);


    // Initialize history DONE folder
    jobHistory.initDone(conf, fs);
    final String historyLogDir =

View Full Code Here

    if (Thread.currentThread().isInterrupted()) {
      throw new InterruptedException();
    }
    
    // Same with 'localDir' except it's always on the local disk.
    asyncDiskService = new MRAsyncDiskService(FileSystem.getLocal(conf), conf.getLocalDirs());
    asyncDiskService.moveAndDeleteFromEachVolume(SUBDIR);


    // Initialize history DONE folder
    jobHistory.initDone(conf, fs);
    final String historyLogDir =

View Full Code Here

    }
 
    // Check local disk, start async disk service, and clean up all 
    // local directories.
    checkLocalDirs(this.fConf.getLocalDirs());
    setAsyncDiskService(new MRAsyncDiskService(fConf));
    getAsyncDiskService().cleanupAllVolumes();


    // Clear out state tables
    this.tasks.clear();
    this.runningTasks = new LinkedHashMap<TaskAttemptID, TaskInProgress>();

View Full Code Here

  public void testMRAsyncDiskService() throws Throwable {
  
    FileSystem localFileSystem = FileSystem.getLocal(new Configuration());
    String[] vols = new String[]{TEST_ROOT_DIR + "/0",
        TEST_ROOT_DIR + "/1"};
    MRAsyncDiskService service = new MRAsyncDiskService(
        localFileSystem, vols);
    
    String a = "a";
    String b = "b";
    String c = "b/c";
    String d = "d";
    
    File fa = new File(vols[0], a);
    File fb = new File(vols[1], b);
    File fc = new File(vols[1], c);
    File fd = new File(vols[1], d);
    
    // Create the directories
    fa.mkdirs();
    fb.mkdirs();
    fc.mkdirs();
    fd.mkdirs();
    
    assertTrue(fa.exists());
    assertTrue(fb.exists());
    assertTrue(fc.exists());
    assertTrue(fd.exists());
    
    // Move and delete them
    service.moveAndDeleteRelativePath(vols[0], a);
    assertFalse(fa.exists());
    service.moveAndDeleteRelativePath(vols[1], b);
    assertFalse(fb.exists());
    assertFalse(fc.exists());
    
    // asyncDiskService is NOT able to delete files outside all volumes.
    IOException ee = null;
    try {
      service.moveAndDeleteAbsolutePath(TEST_ROOT_DIR + "/2");
    } catch (IOException e) {
      ee = e;
    }
    assertNotNull("asyncDiskService should not be able to delete files "
        + "outside all volumes", ee);
    // asyncDiskService is able to automatically find the file in one
    // of the volumes.
    assertTrue(service.moveAndDeleteAbsolutePath(vols[1] + Path.SEPARATOR_CHAR + d));
    
    // Make sure everything is cleaned up
    makeSureCleanedUp(vols, service);
  }

View Full Code Here

    tracker.setLocalFileSystem(tracker.systemFS);
    tracker.systemDirectory = new Path(TEST_ROOT_DIR.getAbsolutePath());


    tracker.runningTasks = new LinkedHashMap<TaskAttemptID, TaskInProgress>();
    tracker.runningJobs = new TreeMap<JobID, RunningJob>();
    tracker.setAsyncDiskService(new MRAsyncDiskService(trackerFConf));
    tracker.getAsyncDiskService().cleanupAllVolumes();


    // setup task controller
    taskController = createTaskController();
    taskController.setConf(trackerFConf);

View Full Code Here

    if (Thread.currentThread().isInterrupted()) {
      throw new InterruptedException();
    }
    
    // Same with 'localDir' except it's always on the local disk.
    asyncDiskService = new MRAsyncDiskService(FileSystem.getLocal(conf), conf.getLocalDirs());
    asyncDiskService.moveAndDeleteFromEachVolume(SUBDIR);


    // Initialize history DONE folder
    jobHistory.initDone(conf, fs);
    final String historyLogDir =

View Full Code Here

  @Test
  public void testMRAsyncDiskServiceMoveAndDeleteAllVolumes() throws Throwable {
    FileSystem localFileSystem = FileSystem.getLocal(new Configuration());
    String[] vols = new String[]{TEST_ROOT_DIR + "/0",
        TEST_ROOT_DIR + "/1"};
    MRAsyncDiskService service = new MRAsyncDiskService(
        localFileSystem, vols);


    String a = "a";
    String b = "b";
    String c = "b/c";
    String d = "d";
    
    File fa = new File(vols[0], a);
    File fb = new File(vols[1], b);
    File fc = new File(vols[1], c);
    File fd = new File(vols[1], d);
    
    // Create the directories
    fa.mkdirs();
    fb.mkdirs();
    fc.mkdirs();
    fd.mkdirs();


    assertTrue(fa.exists());
    assertTrue(fb.exists());
    assertTrue(fc.exists());
    assertTrue(fd.exists());
    
    // Delete all of them
    service.cleanupAllVolumes();
    
    assertFalse(fa.exists());
    assertFalse(fb.exists());
    assertFalse(fc.exists());
    assertFalse(fd.exists());

View Full Code Here

    assertTrue(fb.exists());
    assertTrue(fc.exists());
    assertTrue(fd.exists());
    
    // Create the asyncDiskService which will delete all contents inside SUBDIR
    MRAsyncDiskService service = new MRAsyncDiskService(
        localFileSystem, vols);
    
    // Make sure everything is cleaned up
    makeSureCleanedUp(vols, service);
  }

View Full Code Here

    // Put a file in one of the volumes to be cleared on startup.
    Path delDir = new Path(vols[0], MRAsyncDiskService.TOBEDELETED);
    localFileSystem.mkdirs(delDir);
    localFileSystem.create(new Path(delDir, "foo")).close();


    MRAsyncDiskService service = new MRAsyncDiskService(
        localFileSystem, vols);
    makeSureCleanedUp(vols, service);
  }

View Full Code Here

  public void testMRAsyncDiskService() throws Throwable {
  
    FileSystem localFileSystem = FileSystem.getLocal(new Configuration());
    String[] vols = new String[]{TEST_ROOT_DIR + "/0",
        TEST_ROOT_DIR + "/1"};
    MRAsyncDiskService service = new MRAsyncDiskService(
        localFileSystem, vols);
    
    String a = "a";
    String b = "b";
    String c = "b/c";
    String d = "d";
    
    File fa = new File(vols[0], a);
    File fb = new File(vols[1], b);
    File fc = new File(vols[1], c);
    File fd = new File(vols[1], d);
    
    // Create the directories
    fa.mkdirs();
    fb.mkdirs();
    fc.mkdirs();
    fd.mkdirs();
    
    assertTrue(fa.exists());
    assertTrue(fb.exists());
    assertTrue(fc.exists());
    assertTrue(fd.exists());
    
    // Move and delete them
    service.moveAndDeleteRelativePath(vols[0], a);
    assertFalse(fa.exists());
    service.moveAndDeleteRelativePath(vols[1], b);
    assertFalse(fb.exists());
    assertFalse(fc.exists());
    
    assertFalse(service.moveAndDeleteRelativePath(vols[1], "not_exists"));
    
    // asyncDiskService is NOT able to delete files outside all volumes.
    IOException ee = null;
    try {
      service.moveAndDeleteAbsolutePath(TEST_ROOT_DIR + "/2");
    } catch (IOException e) {
      ee = e;
    }
    assertNotNull("asyncDiskService should not be able to delete files "
        + "outside all volumes", ee);
    // asyncDiskService is able to automatically find the file in one
    // of the volumes.
    assertTrue(service.moveAndDeleteAbsolutePath(vols[1] + Path.SEPARATOR_CHAR + d));
    
    // Make sure everything is cleaned up
    makeSureCleanedUp(vols, service);
  }

View Full Code Here

0 1 2

TOP

Related Classes of org.apache.hadoop.mapreduce.util.MRAsyncDiskService

org.apache.hadoop.fs.Path

org.apache.hadoop.mapred.JobTracker

org.apache.hadoop.mapred.TaskTracker

org.apache.hadoop.mapred.TestTaskTrackerLocalization

org.apache.hadoop.mapred.UserLogCleaner

org.apache.hadoop.mapreduce.util.TestMRAsyncDiskService

org.apache.hadoop.util.AsyncDiskService

java.util.Date

java.io.IOException

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.