Examples of FetchResult


Examples of de.jungblut.crawl.FetchResult

      InputStream connection = getConnection(realUrl);
      String html = consumeStream(connection);

      final HashSet<String> set = extractOutlinks(html, realUrl);

      return new FetchResult(realUrl, set);
    } catch (ParserException pEx) {
      // ignore parser exceptions, they contain mostly garbage
    } catch (RuntimeException rEx) {
      rEx.printStackTrace();
    } catch (Exception e) {
View Full Code Here

Examples of freenet.client.FetchResult

  public FetchResult getCompletedRequestBlocking(final FreenetURI key) throws PersistenceDisabledException {
    ClientGet get = globalRebootClient.getCompletedRequest(key);
    if(get != null) {
      // FIXME race condition with free() - arrange refcounting for the data to prevent this
      return new FetchResult(new ClientMetadata(get.getMIMEType()), new NoFreeBucket(get.getBucket()));
    }

    FetchResult result = globalForeverClient.getRequestStatusCache().getShadowBucket(key, false);
    if(result != null) {
      return result;
    }

    class OutputWrapper {
      FetchResult result;
      boolean done;
    }

    final OutputWrapper ow = new OutputWrapper();

    core.clientContext.jobRunner.queue(new PersistentJob() {

      @Override
      public String toString() {
        return "FCP getCompletedRequestBlocking";
      }

      @Override
      public boolean run(ClientContext context) {
        FetchResult result = null;
        try {
          result = lookup(key, false, context, false, null);
        } finally {
          synchronized(ow) {
            ow.result = result;
View Full Code Here

Examples of freenet.client.FetchResult

        try {
          node.clientCore.clientContext.start(get);
        } catch (PersistenceDisabledException e) {
          // Impossible
        }
        FetchResult res = fw.waitForCompletion();
        return res.asBucket().getInputStream();
      } catch (FetchException e) {
        if ((e.getMode() == FetchExceptionMode.PERMANENT_REDIRECT) || (e.getMode() == FetchExceptionMode.TOO_MANY_PATH_COMPONENTS)) {
          uri = e.newURI;
          continue;
        }
View Full Code Here

Examples of freenet.client.FetchResult

      ((ClientGetter)parent).addKeyToBinaryBlob(block, context);
    Bucket data = extract(block, context);
    if(data == null) return; // failed
    context.uskManager.checkUSK(key.getURI(), fromStore, block.isMetadata());
    if(!block.isMetadata()) {
      onSuccess(new FetchResult(new ClientMetadata(null), data), context);
    } else {
      onFailure(new FetchException(FetchExceptionMode.INVALID_METADATA, "Metadata where expected data"), false, context);
    }
  }
View Full Code Here

Examples of freenet.client.FetchResult

            }
          }
        }
        if(needsFetch){
          //If we don't have the data, then we need to fetch it and block until it is available
          FetchResult result = fetch(key, maxSize, new RequestClient() {
            @Override
            public boolean persistent() {
              return false;
            }
            @Override
            public boolean realTimeFlag() {
              return true;
            } }, fctx);

          // Now, is it safe?

          data = result.asBucket();
          mimeType = result.getMimeType();
        }
      } else if(fe != null) throw fe;

      handleDownload(ctx, data, ctx.getBucketFactory(), mimeType, requestedMimeType, forceString, httprequest.isParameterSet("forcedownload"), "/", key, "&max-size="+maxSizeDownload, referer, true, ctx, core, fr != null, maybeCharset);
    } catch (FetchException e) {
View Full Code Here

Examples of freenet.client.FetchResult

        onSuccess(result, null);
        return true;
      } else if(fctx.overrideMIME != null && !fctx.overrideMIME.equals(result.getMimeType())) {
        // Change the MIME type.
        tracker.removeFetcher(this);
        onSuccess(new FetchResult(new ClientMetadata(fctx.overrideMIME), result.asBucket()), null);
        return true;
      }
    } else if(result.alreadyFiltered) {
      if(refilterPolicy == REFILTER_POLICY.RE_FETCH || !fctx.filterData) {
        // Can't use it.
        return false;
      } else if(fctx.filterData) {
        if(shouldAcceptCachedFilteredData(fctx, result)) {
          if(refilterPolicy == REFILTER_POLICY.ACCEPT_OLD) {
            tracker.removeFetcher(this);
            onSuccess(result, null);
            return true;
          } // else re-filter
        } else
          return false;
      } else {
        return false;
      }
    }
    data = result.asBucket();
    mimeType = result.getMimeType();
    if(mimeType == null || mimeType.equals("")) mimeType = DefaultMIMETypes.DEFAULT_MIME_TYPE;
    if(fctx.overrideMIME != null && !result.alreadyFiltered)
      mimeType = fctx.overrideMIME;
    else if(fctx.overrideMIME != null && !mimeType.equals(fctx.overrideMIME)) {
      // Doesn't work.
      return false;
    }
    String fullMimeType = mimeType;
    mimeType = ContentFilter.stripMIMEType(mimeType);
    FilterMIMEType type = ContentFilter.getMIMEType(mimeType);
    if(type == null || ((!type.safeToRead) && type.readFilter == null)) {
      UnknownContentTypeException e = new UnknownContentTypeException(mimeType);
      data.free();
      onFailure(new FetchException(e.getFetchErrorCode(), data.size(), e, mimeType), null);
      return true;
    } else if(type.safeToRead) {
      tracker.removeFetcher(this);
      onSuccess(new FetchResult(new ClientMetadata(mimeType), data), null);
      return true;
    } else {
      // Try to filter it.
      Bucket output = null;
      InputStream is = null;
      OutputStream os = null;
      try {
        output = context.tempBucketFactory.makeBucket(-1);
        is = data.getInputStream();
        os = output.getOutputStream();
        ContentFilter.filter(is, os, fullMimeType, uri.toURI("/"), null, null, fctx.charset, context.linkFilterExceptionProvider);
        is.close();
        is = null;
        os.close();
        os = null;
        // Since we are not re-using the data bucket, we can happily stay in the FProxyFetchTracker.
        this.onSuccess(new FetchResult(new ClientMetadata(fullMimeType), output), null);
        output = null;
        return true;
      } catch (IOException e) {
        Logger.normal(this, "Failed filtering coalesced data in fproxy");
        // Failed. :|
View Full Code Here

Examples of freenet.client.FetchResult

    } finally {
      Closer.close(output);
      Closer.close(pipeOut);
    }

    final FetchResult result = new FetchResult(clientMetadata, finalResult);
    context.uskManager.updateKnownGood(origUSK, state.getToken(), context);
    context.mainExecutor.execute(new PrioRunnable() {

      @Override
      public void run() {
View Full Code Here

Examples of freenet.client.FetchResult

    w.write(outsb.toString());
    w.flush();
                return false;
            }
            try {
        FetchResult result = client.fetch(uri);
        ClientMetadata cm = result.getMetadata();
                outsb.append("Content MIME type: ").append(cm.getMIMEType());
        Bucket data = result.asBucket();
        // FIXME limit it above
        if(data.size() > 32*1024) {
          System.err.println("Data is more than 32K: "+data.size());
          outsb.append("Data is more than 32K: ").append(data.size());
          outsb.append("\r\n");
          w.write(outsb.toString());
          w.flush();
          return false;
        }
        byte[] dataBytes = BucketTools.toByteArray(data);
        boolean evil = false;
        for(byte b: dataBytes) {
          // Look for escape codes
          if(b == '\n') continue;
          if(b == '\r') continue;
          if(b < 32) evil = true;
        }
        if(evil) {
          System.err.println("Data may contain escape codes which could cause the terminal to run arbitrary commands! Save it to a file if you must with GETFILE:");
          outsb.append("Data may contain escape codes which could cause the terminal to run arbitrary commands! Save it to a file if you must with GETFILE:");
          outsb.append("\r\n");
          w.write(outsb.toString());
          w.flush();
          return false;
        }
        outsb.append("Data:\r\n");
        outsb.append(new String(dataBytes, ENCODING));
      } catch (FetchException e) {
                outsb.append("Error: ").append(e.getMessage()).append("\r\n");
              if((e.getMode() == FetchExceptionMode.SPLITFILE_ERROR) && (e.errorCodes != null)) {
                outsb.append(e.errorCodes.toVerboseString());
              }
              if(e.newURI != null)
                    outsb.append("Permanent redirect: ").append(e.newURI).append("\r\n");
      }
        } else if(uline.startsWith("DUMP:")) {
              // Should have a key next
              String key = line.substring("DUMP:".length()).trim();
              Logger.normal(this, "Key: "+key);
              FreenetURI uri;
              try {
                  uri = new FreenetURI(key);
                  Logger.normal(this, "Key: "+uri);
              } catch (MalformedURLException e2) {
                  outsb.append("Malformed URI: ").append(key).append(" : ").append(e2);
      outsb.append("\r\n");
      w.write(outsb.toString());
      w.flush();
                  return false;
              }
              try {
                FetchContext context = client.getFetchContext();
              FetchWaiter fw = new FetchWaiter((RequestClient)client);
              ClientGetter get = new ClientGetter(fw, uri, context, RequestStarter.INTERACTIVE_PRIORITY_CLASS, null, null, null);
              get.setMetaSnoop(new DumperSnoopMetadata());
                get.start(n.clientCore.clientContext);
          FetchResult result = fw.waitForCompletion();
          ClientMetadata cm = result.getMetadata();
                  outsb.append("Content MIME type: ").append(cm.getMIMEType());
          Bucket data = result.asBucket();
          // FIXME limit it above
          if(data.size() > 32*1024) {
            System.err.println("Data is more than 32K: "+data.size());
            outsb.append("Data is more than 32K: ").append(data.size());
            outsb.append("\r\n");
            w.write(outsb.toString());
            w.flush();
            return false;
          }
          byte[] dataBytes = BucketTools.toByteArray(data);
          boolean evil = false;
          for(byte b: dataBytes) {
            // Look for escape codes
            if(b == '\n') continue;
            if(b == '\r') continue;
            if(b < 32) evil = true;
          }
          if(evil) {
            System.err.println("Data may contain escape codes which could cause the terminal to run arbitrary commands! Save it to a file if you must with GETFILE:");
            outsb.append("Data may contain escape codes which could cause the terminal to run arbitrary commands! Save it to a file if you must with GETFILE:");
            outsb.append("\r\n");
            w.write(outsb.toString());
            w.flush();
            return false;
          }
          outsb.append("Data:\r\n");
          outsb.append(new String(dataBytes, ENCODING));
        } catch (FetchException e) {
                  outsb.append("Error: ").append(e.getMessage()).append("\r\n");
                if((e.getMode() == FetchExceptionMode.SPLITFILE_ERROR) && (e.errorCodes != null)) {
                  outsb.append(e.errorCodes.toVerboseString());
                }
                if(e.newURI != null)
                      outsb.append("Permanent redirect: ").append(e.newURI).append("\r\n");
        }
        } else if(uline.startsWith("GETFILE:")) {
            // Should have a key next
            String key = line.substring("GETFILE:".length()).trim();
            Logger.normal(this, "Key: "+key);
            FreenetURI uri;
            try {
                uri = new FreenetURI(key);
            } catch (MalformedURLException e2) {
                outsb.append("Malformed URI: ").append(key).append(" : ").append(e2);
    outsb.append("\r\n");
    w.write(outsb.toString());
    w.flush();
                return false;
            }
            try {
              long startTime = System.currentTimeMillis();
        FetchResult result = client.fetch(uri);
        ClientMetadata cm = result.getMetadata();
                outsb.append("Content MIME type: ").append(cm.getMIMEType());
        Bucket data = result.asBucket();
                // Now calculate filename
                String fnam = uri.getDocName();
                fnam = sanitize(fnam);
                if(fnam.length() == 0) {
                    fnam = "freenet-download-"+HexUtil.bytesToHex(BucketTools.hash(data), 0, 10);
View Full Code Here

Examples of freenet.client.FetchResult

    OutputStream output = null;

    DecompressorThreadManager decompressorManager = null;
    ClientGetWorkerThread worker = null;
    Bucket finalResult = null;
    FetchResult result = null;

        long maxLen = -1;
        synchronized(this) {
            if(expectedSize > 0) {
                maxLen = expectedSize;
            }
        }
        if(ctx.filterData && maxLen >= 0) {
            maxLen = expectedSize * 2 + 1024;
        }
        if(maxLen == -1) {
            maxLen = Math.max(ctx.maxTempLength, ctx.maxOutputLength);
        }
       
    FetchException ex = null; // set on failure
    try {
      if(returnBucket == null) finalResult = context.getBucketFactory(persistent()).makeBucket(maxLen);
      else finalResult = returnBucket;
      if(logMINOR) Logger.minor(this, "Writing final data to "+finalResult+" return bucket is "+returnBucket);
      dataOutput .connect(dataInput);
      result = new FetchResult(clientMetadata, finalResult);

      // Decompress
      if(decompressors != null) {
        if(logMINOR) Logger.minor(this, "Decompressing...");
        decompressorManager =  new DecompressorThreadManager(dataInput, decompressors, maxLen);
        dataInput = decompressorManager.execute();
      }

      output = finalResult.getOutputStream();
      if(ctx.overrideMIME != null) mimeType = ctx.overrideMIME;
      worker = new ClientGetWorkerThread(new BufferedInputStream(dataInput), output, uri, mimeType, hashes, ctx.filterData, ctx.charset, ctx.prefetchHook, ctx.tagReplacer, context.linkFilterExceptionProvider);
      worker.start();
      try {
        streamGenerator.writeTo(dataOutput, context);
      } catch(IOException e) {
        //Check if the worker thread caught an exception
        worker.getError();
        //If not, throw the original error
        throw e;
      }

      // An error will propagate backwards, so wait for the worker first.
     
      if(logMINOR) Logger.minor(this, "Waiting for hashing, filtration, and writing to finish");
      worker.waitFinished();

      if(decompressorManager != null) {
        if(logMINOR) Logger.minor(this, "Waiting for decompression to finalize");
        decompressorManager.waitFinished();
      }

      if(worker.getClientMetadata() != null) {
        clientMetadata = worker.getClientMetadata();
        result = new FetchResult(clientMetadata, finalResult);
      }
      // These must be updated for ClientGet.
      synchronized(this) {
          this.expectedMIME = result.getMimeType();
          this.expectedSize = result.size();
      }
    } catch(UnsafeContentTypeException e) {
      Logger.normal(this, "Error filtering content: will not validate", e);
      ex = e.createFetchException(ctx.overrideMIME != null ? ctx.overrideMIME : expectedMIME, expectedSize);
      /*Not really the state's fault*/
    } catch(URISyntaxException e) {
      //Impossible
      Logger.error(this, "URISyntaxException converting a FreenetURI to a URI!: "+e, e);
      ex = new FetchException(FetchExceptionMode.INTERNAL_ERROR, e);
      /*Not really the state's fault*/
    } catch(CompressionOutputSizeException e) {
      Logger.error(this, "Caught "+e, e);
      ex = new FetchException(FetchExceptionMode.TOO_BIG, e);
    } catch (InsufficientDiskSpaceException e) {
        ex = new FetchException(FetchExceptionMode.NOT_ENOUGH_DISK_SPACE);
    } catch(IOException e) {
      Logger.error(this, "Caught "+e, e);
      ex = new FetchException(FetchExceptionMode.BUCKET_ERROR, e);
    } catch(FetchException e) {
      Logger.error(this, "Caught "+e, e);
      ex = e;
    } catch(Throwable t) {
      Logger.error(this, "Caught "+t, t);
      ex = new FetchException(FetchExceptionMode.INTERNAL_ERROR, t);
    } finally {
      Closer.close(dataInput);
      Closer.close(dataOutput);
      Closer.close(output);
    }
    if(ex != null) {
      onFailure(ex, state, context, true);
      if(finalResult != null && finalResult != returnBucket) {
        finalResult.free();
      }
      if(result != null) {
      Bucket data = result.asBucket();
      data.free();
      }
      return;
    }
    context.getJobRunner(persistent()).setCheckpointASAP();
View Full Code Here

Examples of freenet.client.FetchResult

        assert(completionFile != null);
        assert(!ctx.filterData);
        Logger.normal(this, "Succeeding via truncation from "+tempFile+" to "+completionFile);
        FetchException ex = null;
        RandomAccessFile raf = null;
        FetchResult result = null;
        try {
            raf = new RandomAccessFile(tempFile, "rw");
            if(raf.length() < length)
                throw new IOException("File is shorter than target length "+length);
            raf.setLength(length);
            InputStream is = new BufferedInputStream(new FileInputStream(raf.getFD()));
            // Check hashes...
           
            DecompressorThreadManager decompressorManager = null;
            ClientGetWorkerThread worker = null;

            worker = new ClientGetWorkerThread(is, new NullOutputStream(), uri, null, hashes, false, null, ctx.prefetchHook, ctx.tagReplacer, context.linkFilterExceptionProvider);
            worker.start();
           
            if(logMINOR) Logger.minor(this, "Waiting for hashing, filtration, and writing to finish");
            worker.waitFinished();
           
            is.close();
            is = null;
            raf = null; // FD is closed.
           
            // We are still here so it worked.
           
            if(!FileUtil.renameTo(tempFile, completionFile))
                throw new FetchException(FetchExceptionMode.BUCKET_ERROR, "Failed to rename from temp file "+tempFile);
           
            // Success!
           
            synchronized(this) {
                finished = true;
                currentState = null;
                expectedMIME = metadata.getMIMEType();
                expectedSize = length;
            }
           
            result = new FetchResult(metadata, returnBucket);
           
        } catch (IOException e) {
            Logger.error(this, "Failed while completing via truncation: "+e, e);
            ex = new FetchException(FetchExceptionMode.BUCKET_ERROR, e);
        } catch (URISyntaxException e) {
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.