Examples of org.apache.hadoop.mapred.CleanupQueue.PathDeletionContext

Package org.apache.hadoop.mapred.CleanupQueue

Examples of org.apache.hadoop.mapred.CleanupQueue.PathDeletionContext

org.apache.hadoop.mapred.CleanupQueue.PathDeletionContext

      Path[] paths) {
    int i = 0;
    PathDeletionContext[] contexts = new PathDeletionContext[paths.length];


    for (Path p : paths) {
      contexts[i++] = new PathDeletionContext(fs, p.toUri().getPath());
    }
    return contexts;
  }

View Full Code Here

   * @param logPath
   * @throws IOException
   */
  private void deleteLogPath(String logPath) throws IOException {
    LOG.info("Deleting user log path " + logPath);
    PathDeletionContext context = new PathDeletionContext(localFs, logPath);
    cleanupQueue.addToQueue(context);
  }

View Full Code Here

          localFs.delete(localJobFile, true);
          localJobFile = null;
        }


        Path tempDir = jobtracker.getSystemDirectoryForJob(getJobID());
        new CleanupQueue().addToQueue(new PathDeletionContext(
            jobtracker.getFileSystem(), tempDir.toUri().getPath())); 
      } catch (IOException e) {
        LOG.warn("Error cleaning up "+profile.getJobID()+": "+e);
      }

View Full Code Here

        throw new IOException(e);
      }


      String queue = job.getProfile().getQueueName();
      if (!(queueManager.getQueues().contains(queue))) {
        new CleanupQueue().addToQueue(new PathDeletionContext(
            FileSystem.get(conf),
            getSystemDirectoryForJob(jobId).toUri().getPath()));
        job.fail();
        if (userFileForJob != null) {
          userFileForJob.delete();
        }
        throw new IOException("Queue \"" + queue + "\" does not exist");
      }


      // The task scheduler should validate the job configuration
      taskScheduler.checkJob(job);


      // check for access
      try {
        checkAccess(job, QueueManager.QueueOperation.SUBMIT_JOB);
      } catch (IOException ioe) {
        LOG.warn("Access denied for user " + job.getJobConf().getUser() +
            ". Ignoring job " + jobId, ioe);
        job.fail();
        if (userFileForJob != null) {
          userFileForJob.delete();
        }
        new CleanupQueue().addToQueue(new PathDeletionContext(
            FileSystem.get(conf),
            getSystemDirectoryForJob(jobId).toUri().getPath()));
        throw ioe;
      }


      // Check the job if it cannot run in the cluster because of invalid memory
      // requirements.
      try {
        checkMemoryRequirements(job);
      } catch (IOException ioe) {
        new CleanupQueue().addToQueue(new PathDeletionContext(
            FileSystem.get(conf),
            getSystemDirectoryForJob(jobId).toUri().getPath()));
        throw ioe;
      }

View Full Code Here

      Path[] paths) {
    int i = 0;
    PathDeletionContext[] contexts = new PathDeletionContext[paths.length];


    for (Path p : paths) {
      contexts[i++] = new PathDeletionContext(fs, p.toUri().getPath());
    }
    return contexts;
  }

View Full Code Here

      // JobClient always creates a new directory with job files
      // so we remove that directory to cleanup
      // Delete temp dfs dirs created if any, like in case of
      // speculative exn of reduces.
      Path tempDir = jobtracker.getSystemDirectoryForJob(getJobID());
      new CleanupQueue().addToQueue(new PathDeletionContext(
          FileSystem.get(conf), tempDir.toUri().getPath()));
    } catch (IOException e) {
      LOG.warn("Error cleaning up "+profile.getJobID()+": "+e);
    }

View Full Code Here

   * @throws IOException
   */
  void removeJobFiles(String user, JobID jobId) throws IOException {
    String userDir = getUserDir(user);
    String jobDir = getLocalJobDir(user, jobId.toString());
    PathDeletionContext jobCleanup = 
      new TaskController.DeletionContext(getTaskController(), false, user, 
                                         jobDir.substring(userDir.length()));
    directoryCleanupThread.addToQueue(jobCleanup);
    
    for (String str : localStorage.getDirs()) {
      Path ttPrivateJobDir = FileSystem.getLocal(fConf).makeQualified(
        new Path(str, TaskTracker.getPrivateDirForJob(user, jobId.toString())));
      PathDeletionContext ttPrivateJobCleanup =
        new CleanupQueue.PathDeletionContext(ttPrivateJobDir, fConf);
      directoryCleanupThread.addToQueue(ttPrivateJobCleanup);
    }
  }

View Full Code Here

        if (!removeOutputs) {
          taskDir = TaskTracker.getTaskWorkDir(user, jobId, taskId, cleanup);
        } else {
          taskDir = TaskTracker.getLocalTaskDir(user, jobId, taskId, cleanup);
        }
        PathDeletionContext item =
          new TaskController.DeletionContext(taskController, false, user,
                                             taskDir.substring(userDirLen));          
        directoryCleanupThread.addToQueue(item);
      }
    }

View Full Code Here

        if (!removeOutputs) {
          taskDir = TaskTracker.getTaskWorkDir(user, jobId, taskId, cleanup);
        } else {
          taskDir = TaskTracker.getLocalTaskDir(user, jobId, taskId, cleanup);
        }
        PathDeletionContext item =
          new TaskController.DeletionContext(taskController, false, user,
                                             taskDir.substring(userDirLen));          
        directoryCleanupThread.addToQueue(item);
      }
    }

View Full Code Here

   * @throws IOException
   */
  void removeJobFiles(String user, JobID jobId) throws IOException {
    String userDir = getUserDir(user);
    String jobDir = getLocalJobDir(user, jobId.toString());
    PathDeletionContext jobCleanup = 
      new TaskController.DeletionContext(getTaskController(), false, user, 
                                         jobDir.substring(userDir.length()));
    directoryCleanupThread.addToQueue(jobCleanup);
    
    for (String str : localStorage.getDirs()) {
      Path ttPrivateJobDir = FileSystem.getLocal(fConf).makeQualified(
        new Path(str, TaskTracker.getPrivateDirForJob(user, jobId.toString())));
      PathDeletionContext ttPrivateJobCleanup =
        new CleanupQueue.PathDeletionContext(ttPrivateJobDir, fConf);
      directoryCleanupThread.addToQueue(ttPrivateJobCleanup);
    }
  }

View Full Code Here

0 1 2 3 4 5

TOP

Related Classes of org.apache.hadoop.mapred.CleanupQueue.PathDeletionContext

org.apache.hadoop.mapred.JobInProgress

org.apache.hadoop.mapred.JobTracker

org.apache.hadoop.mapred.TaskTracker

org.apache.hadoop.mapred.TaskTracker$TaskInProgress

org.apache.hadoop.mapred.TestCleanupQueue

org.apache.hadoop.mapred.UserLogCleaner

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.