Package org.apache.hadoop.fs

Examples of org.apache.hadoop.fs.LocalDirAllocator$AllocatorPerContext$PathIterator


                                      context.getAttribute("shuffleServerMetrics");
      try {
        shuffleMetrics.serverHandlerBusy();
        outStream = response.getOutputStream();
        JobConf conf = (JobConf) context.getAttribute("conf");
        LocalDirAllocator lDirAlloc =
          (LocalDirAllocator)context.getAttribute("localDirAllocator");
        FileSystem fileSys =
          (FileSystem) context.getAttribute("local.file.system");

        // Index file
        Path indexFileName = lDirAlloc.getLocalPathToRead(
            mapId+"/file.out.index", conf);
       
        // Map-output file
        Path mapOutputFileName = lDirAlloc.getLocalPathToRead(
            mapId+"/file.out", conf);

        /**
         * Read the index file to get the information about where
         * the map-output for the given reducer is available.
View Full Code Here


    this.shuffleServerMetrics = new ShuffleServerMetrics(conf);
    server.setThreads(1, workerThreads);
    // let the jsp pages get to the task tracker, config, and other relevant
    // objects
    FileSystem local = FileSystem.getLocal(conf);
    this.localDirAllocator = new LocalDirAllocator("mapred.local.dir");
    server.setAttribute("task.tracker", this);
    server.setAttribute("local.file.system", local);
    server.setAttribute("conf", conf);
    server.setAttribute("log", LOG);
    server.setAttribute("localDirAllocator", localDirAllocator);
View Full Code Here

  public CoronaJobTrackerRunner(
      TaskTracker.TaskInProgress tip, Task task, TaskTracker tracker,
      JobConf ttConf, CoronaSessionInfo info) throws IOException {
    super(tip, task, tracker, ttConf);
    this.coronaSessionInfo = info;
    LocalDirAllocator lDirAlloc = new LocalDirAllocator("mapred.local.dir");

    workDir = new File(lDirAlloc.getLocalPathForWrite(
        TaskTracker.getLocalTaskDir(
            task.getJobID().toString(),
            task.getTaskID().toString(),
            task.isTaskCleanupTask())
            + Path.SEPARATOR + MRConstants.WORKDIR,
View Full Code Here

        shuffleMetrics.serverHandlerBusy();
        if(ClientTraceLog.isInfoEnabled())
          startTime = System.nanoTime();
        outStream = response.getOutputStream();
        JobConf conf = (JobConf) context.getAttribute("conf");
        LocalDirAllocator lDirAlloc =
          (LocalDirAllocator)context.getAttribute("localDirAllocator");
        FileSystem rfs = ((LocalFileSystem)
            context.getAttribute("local.file.system")).getRaw();

        // Index file
        Path indexFileName = lDirAlloc.getLocalPathToRead(
            TaskTracker.getIntermediateOutputDir(jobId, mapId)
            + "/file.out.index", conf);

        // Map-output file
        Path mapOutputFileName = lDirAlloc.getLocalPathToRead(
            TaskTracker.getIntermediateOutputDir(jobId, mapId)
            + "/file.out", conf);

        /**
         * Read the index file to get the information about where
View Full Code Here

  protected void initNettyMapOutputHttpServer(JobConf conf) throws IOException {
    int nettyHttpPort = conf.getInt(NETTY_MAPOUTPUT_HTTP_PORT, 0);
    NettyMapOutputAttributes attributes = new NettyMapOutputAttributes(
      conf, this, FileSystem.getLocal(conf),
      new LocalDirAllocator("mapred.local.dir"), shuffleServerMetrics);
    nettyMapOutputServer = new NettyMapOutputHttpServer(nettyHttpPort);
    nettyMapOutputServer.init(conf);
    shuffleServerMetrics.setNettyWorkerThreadPool(
      nettyMapOutputServer.getWorkerThreadPool());
    HttpMapOutputPipelineFactory pipelineFactory =
View Full Code Here

    workerThreads = conf.getInt("tasktracker.http.threads", 40);
    server.setThreads(1, workerThreads);
    // let the jsp pages get to the task tracker, config, and other relevant
    // objects
    FileSystem local = FileSystem.getLocal(conf);
    this.localDirAllocator = new LocalDirAllocator("mapred.local.dir");
    server.setAttribute("task.tracker", this);
    server.setAttribute("local.file.system", local);
    server.setAttribute("conf", conf);
    server.setAttribute("log", LOG);
    server.setAttribute("localDirAllocator", localDirAllocator);
View Full Code Here

    lastMap.addListener(ChannelFutureListener.CLOSE);
  }

  protected ChannelFuture sendMapOutput(ChannelHandlerContext ctx, Channel ch,
      String jobId, String mapId, int reduce) throws IOException {
    LocalDirAllocator lDirAlloc = attributes.getLocalDirAllocator();
    FileSystem rfs = ((LocalFileSystem) attributes.getLocalFS()).getRaw();

    ShuffleServerMetrics shuffleMetrics = attributes.getShuffleServerMetrics();
    TaskTracker tracker = attributes.getTaskTracker();

    // Index file
    Path indexFileName = lDirAlloc.getLocalPathToRead(
        TaskTracker.getIntermediateOutputDir(jobId, mapId)
        + "/file.out.index", attributes.getJobConf());
    // Map-output file
    Path mapOutputFileName = lDirAlloc.getLocalPathToRead(
        TaskTracker.getIntermediateOutputDir(jobId, mapId)
        + "/file.out", attributes.getJobConf());

    /**
     * Read the index file to get the information about where
View Full Code Here

        // create the MergeStreams from the sorted map created in the constructor
        // and dump the final output to a file
        int numSegments = sortedSegmentSizes.size();
        int origFactor = factor;
        int passNo = 1;
        LocalDirAllocator lDirAlloc = new LocalDirAllocator("mapred.local.dir");
        do {
          // get the factor for this pass of merge
          factor = getPassFactor(passNo, numSegments);
          List<SegmentDescriptor> segmentsToMerge = new ArrayList<SegmentDescriptor>();
          int segmentsConsidered = 0;
          int numSegmentsToConsider = factor;
          while (true) {
            // extract the smallest 'factor' number of segment pointers from the
            // TreeMap. Call cleanup on the empty segments (no key/value data)
            SegmentDescriptor[] mStream = getSegmentDescriptors(numSegmentsToConsider);
            for (int i = 0; i < mStream.length; i++) {
              if (mStream[i].nextRawKey()) {
                segmentsToMerge.add(mStream[i]);
                segmentsConsidered++;
                // Count the fact that we read some bytes in calling nextRawKey()
                updateProgress(mStream[i].in.getPosition());
              } else {
                mStream[i].cleanup();
                numSegments--; // we ignore this segment for the merge
              }
            }
            // if we have the desired number of segments
            // or looked at all available segments, we break
            if (segmentsConsidered == factor || sortedSegmentSizes.size() == 0) {
              break;
            }
           
            numSegmentsToConsider = factor - segmentsConsidered;
          }
          // feed the streams to the priority queue
          initialize(segmentsToMerge.size());
          clear();
          for (int i = 0; i < segmentsToMerge.size(); i++) {
            put(segmentsToMerge.get(i));
          }
          // if we have lesser number of segments remaining, then just return the
          // iterator, else do another single level merge
          if (numSegments <= factor) {
            // calculate the length of the remaining segments. Required for
            // calculating the merge progress
            long totalBytes = 0;
            for (int i = 0; i < segmentsToMerge.size(); i++) {
              totalBytes += segmentsToMerge.get(i).segmentLength;
            }
            if (totalBytes != 0) // being paranoid
              progPerByte = 1.0f / (float) totalBytes;
            // reset factor to what it originally was
            factor = origFactor;
            return this;
          }
          // we want to spread the creation of temp files on multiple disks if
          // available under the space constraints
          long approxOutputSize = 0;
          for (SegmentDescriptor s : segmentsToMerge) {
            approxOutputSize += s.segmentLength + ChecksumFileSystem.getApproxChkSumLength(s.segmentLength);
          }
          Path tmpFilename = new Path(tmpDir, "intermediate").suffix("." + passNo);
         
          Path outputFile = lDirAlloc.getLocalPathForWrite(tmpFilename.toString(), approxOutputSize, conf);
          LOG.debug("writing intermediate results to " + outputFile);
          Writer writer = cloneFileAttributes(fs.makeQualified(segmentsToMerge.get(0).segmentPathName), fs.makeQualified(outputFile), null);
          writer.sync = null; // disable sync for temp files
          writeFile(this, writer);
          writer.close();
View Full Code Here

      // the base dir for an output dir
      baseDir = queryId.toString() + "/output" + "/" + executionBlockId.getId();

      // initialize LocalDirAllocator
      lDirAllocator = new LocalDirAllocator(ConfVars.WORKER_TEMPORAL_DIR.varname);

      baseDirPath = localFS.makeQualified(lDirAllocator.getLocalPathForWrite(baseDir, conf));
      LOG.info("TaskRunner basedir is created (" + baseDir +")");

      // Setup QueryEngine according to the query plan
View Full Code Here

      try {
        shuffleMetrics.serverHandlerBusy();
        outStream = response.getOutputStream();
        JobConf conf = (JobConf) context.getAttribute("conf");
        LocalDirAllocator lDirAlloc =
          (LocalDirAllocator)context.getAttribute("localDirAllocator");
        FileSystem fileSys =
          (FileSystem) context.getAttribute("local.file.system");

        // Index file
        Path indexFileName = lDirAlloc.getLocalPathToRead(
            TaskTracker.getJobCacheSubdir() + Path.SEPARATOR +
            jobId + Path.SEPARATOR +
            mapId + "/output" + "/file.out.index", conf);
       
        // Map-output file
        Path mapOutputFileName = lDirAlloc.getLocalPathToRead(
            TaskTracker.getJobCacheSubdir() + Path.SEPARATOR +
            jobId + Path.SEPARATOR +
            mapId + "/output" + "/file.out", conf);

        /**
 
View Full Code Here

TOP

Related Classes of org.apache.hadoop.fs.LocalDirAllocator$AllocatorPerContext$PathIterator

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.