Examples of FileChunk


Examples of org.apache.hadoop.io.FileChunk

          }
        }

        final FileStatus fStatus = localFS.getFileStatus(outputPath);
        // add to list of final disk outputs.
        onDiskMapOutputs.add(new FileChunk(outputPath, 0, fStatus.getLen(), false));

        LOG.info("Merged " + numMemDiskSegments + " segments, " +
                 inMemToDiskBytes + " bytes to disk to satisfy " +
                 "reduce memory limit");
        inMemToDiskBytes = 0;
View Full Code Here

Examples of org.apache.hadoop.io.FileChunk

          writer.close();
        }
      }

      // Note the output of the merge
      closeOnDiskFile(new FileChunk(outputPath, 0, outFileLen, false));
    }
View Full Code Here

Examples of org.apache.hadoop.io.FileChunk

        localFS.delete(outputPath, true);
        throw e;
      }

      final long outputLen = localFS.getFileStatus(outputPath).getLen();
      closeOnDiskFile(new FileChunk(outputPath, 0, outputLen, false));

      LOG.info(inputContext.getUniqueIdentifier() +
          " Finished merging " + inputs.size() +
          " map output files on disk of total-size " +
          approxOutputSize + "." +
View Full Code Here

Examples of org.apache.hadoop.io.FileChunk

    this.tmpOutputPath = tmpOutputPath;
    this.disk = null;

    if (type == Type.DISK || type == Type.DISK_DIRECT) {
      boolean preserve = (type == Type.DISK_DIRECT); // type disk are temp files.
      this.outputPath = new FileChunk(outputPath, offset, size, preserve);
    } else {
      this.outputPath = null;
    }

  }
View Full Code Here

Examples of org.apache.tajo.pullserver.retriever.FileChunk

    if (last || (endOffset == -1
        && comparator.compare(idxReader.getLastKey(), end) < 0)) {
      endOffset = data.length();
    }

    FileChunk chunk = new FileChunk(data, startOffset, endOffset - startOffset);
    LOG.info("Retrieve File Chunk: " + chunk);
    return chunk;
  }
View Full Code Here

Examples of org.apache.tajo.pullserver.retriever.FileChunk

        String startKey = params.get("start").get(0);
        String endKey = params.get("end").get(0);
        boolean last = params.get("final") != null;

        FileChunk chunk;
        try {
          chunk = getFileCunks(path, startKey, endKey, last);
        } catch (Throwable t) {
          LOG.error("ERROR Request: " + request.getUri(), t);
          sendError(ctx, "Cannot get file chunks to be sent", BAD_REQUEST);
          return;
        }
        if (chunk != null) {
          chunks.add(chunk);
        }

        // if a subquery requires a hash repartition
      } else if (repartitionType.equals("h")) {
        for (String ta : taskIds) {
          Path path = localFS.makeQualified(
              lDirAlloc.getLocalPathToRead(queryBaseDir + "/" + sid + "/" +
                  ta + "/output/" + partitionId, conf));
          File file = new File(path.toUri());
          FileChunk chunk = new FileChunk(file, 0, file.length());
          chunks.add(chunk);
        }
      } else {
        LOG.error("Unknown repartition type: " + repartitionType);
        return;
      }

      // Write the content.
      Channel ch = e.getChannel();
      if (chunks.size() == 0) {
        HttpResponse response = new DefaultHttpResponse(HTTP_1_1, NO_CONTENT);
        ch.write(response);
        if (!isKeepAlive(request)) {
          ch.close();
        }
      }  else {
        FileChunk[] file = chunks.toArray(new FileChunk[chunks.size()]);
        HttpResponse response = new DefaultHttpResponse(HTTP_1_1, OK);
        long totalSize = 0;
        for (FileChunk chunk : file) {
          totalSize += chunk.length();
        }
        setContentLength(response, totalSize);

        // Write the initial line and the header.
        ch.write(response);
View Full Code Here

Examples of org.apache.tajo.pullserver.retriever.FileChunk

    if (last || (endOffset == -1
        && comparator.compare(idxReader.getLastKey(), end) < 0)) {
      endOffset = data.length();
    }

    FileChunk chunk = new FileChunk(data, startOffset, endOffset - startOffset);
    LOG.info("Retrieve File Chunk: " + chunk);
    return chunk;
  }
View Full Code Here

Examples of org.apache.tajo.pullserver.retriever.FileChunk

        String startKey = params.get("start").get(0);
        String endKey = params.get("end").get(0);
        boolean last = params.get("final") != null;

        FileChunk chunk;
        try {
          chunk = getFileCunks(path, startKey, endKey, last);
        } catch (Throwable t) {
          LOG.error("ERROR Request: " + request.getUri(), t);
          sendError(ctx, "Cannot get file chunks to be sent", BAD_REQUEST);
          return;
        }
        if (chunk != null) {
          chunks.add(chunk);
        }

        // if a subquery requires a hash repartition
      } else if (repartitionType.equals("h")) {
        for (String ta : taskIds) {
          Path path = localFS.makeQualified(
              lDirAlloc.getLocalPathToRead(queryBaseDir + "/" + sid + "/" +
                  ta + "/output/" + partitionId, conf));
          File file = new File(path.toUri());
          FileChunk chunk = new FileChunk(file, 0, file.length());
          chunks.add(chunk);
        }
      } else {
        LOG.error("Unknown repartition type: " + repartitionType);
        return;
      }

      // Write the content.
      Channel ch = e.getChannel();
      if (chunks.size() == 0) {
        HttpResponse response = new DefaultHttpResponse(HTTP_1_1, NO_CONTENT);
        ch.write(response);
        if (!isKeepAlive(request)) {
          ch.close();
        }
      }  else {
        FileChunk[] file = chunks.toArray(new FileChunk[chunks.size()]);
        HttpResponse response = new DefaultHttpResponse(HTTP_1_1, OK);
        long totalSize = 0;
        for (FileChunk chunk : file) {
          totalSize += chunk.length();
        }
        setContentLength(response, totalSize);

        // Write the initial line and the header.
        ch.write(response);
View Full Code Here

Examples of org.apache.tajo.pullserver.retriever.FileChunk

    int sid = Integer.valueOf(params.get("sid").get(0));
    int partitionId = Integer.valueOf(params.get("p").get(0));
    for (String ta : taskIds) {

      File file = new File(base + "/" + sid + "/" + ta + "/output/" + partitionId);
      FileChunk chunk = new FileChunk(file, 0, file.length());
      chunks.add(chunk);
    }

    FileChunk[] file = chunks.toArray(new FileChunk[chunks.size()]);
//    try {
//      file = retriever.handle(ctx, request);
//    } catch (FileNotFoundException fnf) {
//      LOG.error(fnf);
//      sendError(ctx, NOT_FOUND);
//      return;
//    } catch (IllegalArgumentException iae) {
//      LOG.error(iae);
//      sendError(ctx, BAD_REQUEST);
//      return;
//    } catch (FileAccessForbiddenException fafe) {
//      LOG.error(fafe);
//      sendError(ctx, FORBIDDEN);
//      return;
//    } catch (IOException ioe) {
//      LOG.error(ioe);
//      sendError(ctx, INTERNAL_SERVER_ERROR);
//      return;
//    }

    // Write the content.
    Channel ch = e.getChannel();
    if (file == null) {
      HttpResponse response = new DefaultHttpResponse(HTTP_1_1, NO_CONTENT);
      ch.write(response);
      if (!isKeepAlive(request)) {
        ch.close();
      }
    }  else {
      HttpResponse response = new DefaultHttpResponse(HTTP_1_1, OK);
      long totalSize = 0;
      for (FileChunk chunk : file) {
        totalSize += chunk.length();
      }
      setContentLength(response, totalSize);

      // Write the initial line and the header.
      ch.write(response);
View Full Code Here

Examples of org.apache.tajo.pullserver.retriever.FileChunk

      endOffset = data.length();
    }

    idxReader.close();

    FileChunk chunk = new FileChunk(data, startOffset, endOffset - startOffset);
    LOG.info("Retrieve File Chunk: " + chunk);
    return chunk;
  }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.