Examples of org.apache.hadoop.fs.ContentSummary

org.apache.hadoop.fs.ContentSummary
Store the summary of a content (a directory or a file).

      throw new SemanticException("Failed to load indexHandler: " + index.getIndexHandlerClass(), e);
    }


    // check the size
    try {
      ContentSummary inputSummary = Utilities.getInputSummary(pctx.getContext(), task.getWork(), null);
      long inputSize = inputSummary.getLength();
      if (!indexHandler.checkQuerySize(inputSize, pctx.getConf())) {
        queryContext.setQueryTasks(null);
        return;
      }
    } catch (IOException e) {

View Full Code Here

    // map-reduce jobs will be run locally based on data size
    // first find out if any of the jobs needs to run non-locally
    boolean hasNonLocalJob = false;
    for (ExecDriver mrtask: mrtasks) {
      try {
        ContentSummary inputSummary = Utilities.getInputSummary
          (ctx, (MapredWork)mrtask.getWork(), p);
        int numReducers = getNumberOfReducers(mrtask.getWork(), conf);


        if (LOG.isDebugEnabled()) {
          LOG.debug("Task: " + mrtask.getId() + ", Summary: " +
                   inputSummary.getLength() + "," + inputSummary.getFileCount() + ","
                   + numReducers);
        }


        if(MapRedTask.isEligibleForLocalMode(conf, inputSummary, numReducers) != null) {
          hasNonLocalJob = true;

View Full Code Here

      final StreamingOutput streaming = getListingStream(np, fullpath);
      return Response.ok(streaming).type(MediaType.APPLICATION_JSON).build();
    }
    case GETCONTENTSUMMARY:
    {
      final ContentSummary contentsummary = np.getContentSummary(fullpath);
      final String js = JsonUtil.toJsonString(contentsummary);
      return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
    }
    case GETFILECHECKSUM:
    {

View Full Code Here

        // processed. 0 means disabled. I.e. blocking for the entire duration.
        ContentSummaryComputationContext cscc =


            new ContentSummaryComputationContext(this, getFSNamesystem(),
            contentCountLimit);
        ContentSummary cs = targetNode.computeAndConvertContentSummary(cscc);
        yieldCount += cscc.getYieldCount();
        return cs;
      }
    } finally {
      readUnlock();

View Full Code Here

  @Override
  public GetContentSummaryResponseProto getContentSummary(
      RpcController controller, GetContentSummaryRequestProto req)
      throws ServiceException {
    try {
      ContentSummary result = server.getContentSummary(req.getPath());
      return GetContentSummaryResponseProto.newBuilder()
          .setSummary(PBHelper.convert(result)).build();
    } catch (IOException e) {
      throw new ServiceException(e);
    }

View Full Code Here

        || spaceQuota == null) {
      return null;
    }


    try {
      return new ContentSummary(
          Long.parseLong(length),
          Long.parseLong(fileCount),
          Long.parseLong(directoryCount),
          Long.parseLong(quota),
          Long.parseLong(spaceConsumed),

View Full Code Here

  }


  @Override
  public ContentSummary getContentSummary(Path f) throws IOException {
    final String s = makeQualified(f).toUri().getPath();
    final ContentSummary cs = new ContentSummaryParser().getContentSummary(s);
    return cs != null? cs: super.getContentSummary(f);
  }

View Full Code Here

        build();
  }
  
  public static ContentSummary convert(ContentSummaryProto cs) {
    if (cs == null) return null;
    return new ContentSummary(
      cs.getLength(), cs.getFileCount(), cs.getDirectoryCount(), cs.getQuota(),
      cs.getSpaceConsumed(), cs.getSpaceQuota());
  }

View Full Code Here

          + ie.getLocalizedMessage());
      // expected
    }
    
    // check count update
    ContentSummary cBefore = dfs.getContentSummary(trgPath.getParent());
    
    // now concatenate
    dfs.concat(trgPath, files);
    
    // verify  count
    ContentSummary cAfter = dfs.getContentSummary(trgPath.getParent());
    assertEquals(cBefore.getFileCount(), cAfter.getFileCount()+files.length);
    
    // verify other stuff
    long totalLen = trgLen;
    long totalBlocks = trgBlocks;
    for(i=0; i<files.length; i++) {

View Full Code Here

   */
  public final ContentSummary computeAndConvertContentSummary(
      ContentSummaryComputationContext summary) {
    Content.Counts counts = computeContentSummary(summary).getCounts();
    final Quota.Counts q = getQuotaCounts();
    return new ContentSummary(counts.get(Content.LENGTH),
        counts.get(Content.FILE) + counts.get(Content.SYMLINK),
        counts.get(Content.DIRECTORY), q.get(Quota.NAMESPACE),
        counts.get(Content.DISKSPACE), q.get(Quota.DISKSPACE));
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.fs.ContentSummary

co.cask.cdap.internal.app.runtime.distributed.DistributedProgramRuntimeService$ClusterResourceReporter

com.cloudera.sqoop.TestTargetDir

com.hdfsTools.impl.checkImpl

com.linkedin.camus.etl.kafka.CamusJob

com.quantcast.qfs.hadoop.QuantcastFileSystem

com.taobao.zeus.jobs.sub.tool.OutputCheckJob

com.twitter.elephantbird.util.HdfsUtils

org.apache.accumulo.monitor.servlets.DefaultServlet

org.apache.accumulo.server.master.CoordinateRecoveryTask$RecoveryJob

org.apache.accumulo.server.monitor.servlets.DefaultServlet

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.