Package org.apache.hadoop.fs

Examples of org.apache.hadoop.fs.LocalDirAllocator$AllocatorPerContext


    }
    int partition = conf.getInt("mapred.task.partition", 0);
   
    // setup the local and user working directories
    FileSystem local = FileSystem.getLocal(conf);
    LocalDirAllocator lDirAlloc = new LocalDirAllocator("mapred.local.dir");

    File workDirName = TaskRunner.formWorkDir(lDirAlloc, taskId, false, conf);
    local.setWorkingDirectory(new Path(workDirName.toString()));
    FileSystem.get(conf).setWorkingDirectory(conf.getWorkingDirectory());
   
    // set up a classloader with the right classpath
    ClassLoader classLoader =
        makeClassLoader(conf, new File(workDirName.toString()));
    Thread.currentThread().setContextClassLoader(classLoader);
    conf.setClassLoader(classLoader);
   
    // split.dta file is used only by IsolationRunner. The file can now be in
    // any of the configured local disks, so use LocalDirAllocator to find out
    // where it is.
    Path localMetaSplit =
        new LocalDirAllocator("mapred.local.dir").getLocalPathToRead(
            TaskTracker.getLocalSplitFile(conf.getUser(), taskId.getJobID()
                .toString(), taskId.toString()), conf);
    DataInputStream splitFile = FileSystem.getLocal(conf).open(localMetaSplit);
    TaskSplitIndex splitIndex = new TaskSplitIndex();
    splitIndex.readFields(splitFile);
View Full Code Here


        shuffleMetrics.serverHandlerBusy();
        if(ClientTraceLog.isInfoEnabled())
          startTime = System.nanoTime();
        outStream = response.getOutputStream();
        JobConf conf = (JobConf) context.getAttribute("conf");
        LocalDirAllocator lDirAlloc =
          (LocalDirAllocator)context.getAttribute("localDirAllocator");
        FileSystem rfs = ((LocalFileSystem)
            context.getAttribute("local.file.system")).getRaw();

      String userName = null;
      String runAsUserName = null;
      synchronized (tracker.runningJobs) {
        RunningJob rjob = tracker.runningJobs.get(JobID.forName(jobId));
        if (rjob == null) {
          throw new IOException("Unknown job " + jobId + "!!");
        }
        userName = rjob.jobConf.getUser();
        runAsUserName = tracker.getTaskController().getRunAsUser(rjob.jobConf);
      }
      // Index file
      Path indexFileName =
          lDirAlloc.getLocalPathToRead(TaskTracker.getIntermediateOutputDir(
              userName, jobId, mapId)
              + "/file.out.index", conf);

      // Map-output file
      Path mapOutputFileName =
          lDirAlloc.getLocalPathToRead(TaskTracker.getIntermediateOutputDir(
              userName, jobId, mapId)
              + "/file.out", conf);

        /**
         * Read the index file to get the information about where
View Full Code Here

      this.trackerDistributerdCacheManager =
        new TrackerDistributedCacheManager(conf, new DefaultTaskController());
      this.taskDistributedCacheManager =
        trackerDistributerdCacheManager.newTaskDistributedCacheManager(conf);
      taskDistributedCacheManager.setup(
          new LocalDirAllocator("mapred.local.dir"),
          new File(systemJobDir.toString()),
          "archive", "archive");
     
      if (DistributedCache.getSymlink(conf)) {
        // This is not supported largely because,
View Full Code Here

      ServletContext context = getServletContext();
      int reduce = Integer.parseInt(reduceId);
      byte[] buffer = new byte[MAX_BYTES_TO_READ];
      OutputStream outStream = response.getOutputStream();
      JobConf conf = (JobConf) context.getAttribute("conf");
      LocalDirAllocator lDirAlloc =
        (LocalDirAllocator)context.getAttribute("localDirAllocator");
      FileSystem fileSys =
        (FileSystem) context.getAttribute("local.file.system");

      // Index file
      Path indexFileName = lDirAlloc.getLocalPathToRead(
                                            mapId+"/file.out.index", conf);
      FSDataInputStream indexIn = null;
        
      // Map-output file
      Path mapOutputFileName = lDirAlloc.getLocalPathToRead(
                                            mapId+"/file.out", conf);
      FSDataInputStream mapOutputIn = null;
       
      // true iff IOException was caused by attempt to access input
      boolean isInputException = true;
View Full Code Here

    int workerThreads = conf.getInt("tasktracker.http.threads", 40);
    server.setThreads(1, workerThreads);
    // let the jsp pages get to the task tracker, config, and other relevant
    // objects
    FileSystem local = FileSystem.getLocal(conf);
    this.localDirAllocator = new LocalDirAllocator("mapred.local.dir");
    server.setAttribute("task.tracker", this);
    server.setAttribute("local.file.system", local);
    server.setAttribute("conf", conf);
    server.setAttribute("log", LOG);
    server.setAttribute("localDirAllocator", localDirAllocator);
View Full Code Here

      }
    } else {
      this.conf = new JobConf(conf);
    }
    this.mapOutputFile.setConf(this.conf);
    this.lDirAlloc = new LocalDirAllocator("mapred.local.dir");
  }
View Full Code Here

                                      context.getAttribute("shuffleServerMetrics");
      try {
        shuffleMetrics.serverHandlerBusy();
        outStream = response.getOutputStream();
        JobConf conf = (JobConf) context.getAttribute("conf");
        LocalDirAllocator lDirAlloc =
          (LocalDirAllocator)context.getAttribute("localDirAllocator");
        FileSystem fileSys =
          (FileSystem) context.getAttribute("local.file.system");

        // Index file
        Path indexFileName = lDirAlloc.getLocalPathToRead(
            mapId+"/file.out.index", conf);
       
        // Map-output file
        Path mapOutputFileName = lDirAlloc.getLocalPathToRead(
            mapId+"/file.out", conf);

        /**
         * Read the index file to get the information about where
         * the map-output for the given reducer is available.
View Full Code Here

    this.shuffleServerMetrics = new ShuffleServerMetrics(fConf);
    server.setThreads(1, workerThreads);
    // let the jsp pages get to the task tracker, config, and other relevant
    // objects
    FileSystem local = FileSystem.getLocal(conf);
    this.localDirAllocator = new LocalDirAllocator("mapred.local.dir");
    server.setAttribute("task.tracker", this);
    server.setAttribute("local.file.system", local);
    server.setAttribute("conf", conf);
    server.setAttribute("log", LOG);
    server.setAttribute("localDirAllocator", localDirAllocator);
View Full Code Here

      this.conf = (JobConf) conf;
    } else {
      this.conf = new JobConf(conf);
    }
    this.mapOutputFile.setConf(this.conf);
    this.lDirAlloc = new LocalDirAllocator("mapred.local.dir");
    loadStaticResolutions(conf);
  }
View Full Code Here

      ArrayList<URL> urllist = new ArrayList<URL>();
     
      // add the jars and directories to the classpath
      String jar = conf.getJar();
      if (jar != null) {     
        LocalDirAllocator lDirAlloc =
                            new LocalDirAllocator("mapred.local.dir");
        File jobCacheDir = new File(lDirAlloc.getLocalPathToRead(
                                      TaskTracker.getJobCacheSubdir()
                                      + Path.SEPARATOR + getJobId()
                                      + Path.SEPARATOR 
                                      + "work", conf).toString());
View Full Code Here

TOP

Related Classes of org.apache.hadoop.fs.LocalDirAllocator$AllocatorPerContext

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.