Examples of URLFilters

com.digitalpebble.storm.crawler.filtering.URLFilters
Wrapper for the URLFilters defined in a JSON configuration
org.apache.nutch.net.URLFilters
Creates and caches {@link URLFilter} implementing plugins.

Examples of com.digitalpebble.storm.crawler.filtering.URLFilters

        String urlconfigfile = ConfUtils.getString(conf,
                "urlfilters.config.file", "urlfilters.json");


        if (urlconfigfile != null)
            try {
                filters = new URLFilters(urlconfigfile);
            } catch (IOException e) {
                LOG.error("Exception caught while loading the URLFilters");
            }


        ignoreOutsideHost = ConfUtils.getBoolean(conf,

View Full Code Here

Examples of org.apache.nutch.net.URLFilters

        InterruptedException {
      urlNormalizers = new URLNormalizers(context.getConfiguration(),
          URLNormalizers.SCOPE_INJECT);
      interval = context.getConfiguration().getInt("db.fetch.interval.default",
          2592000);
      filters = new URLFilters(context.getConfiguration());
      scfilters = new ScoringFilters(context.getConfiguration());
      scoreInjected = context.getConfiguration().getFloat("db.score.injected",
          1.0f);
      curTime = context.getConfiguration().getLong("injector.current.time",
          System.currentTimeMillis());

View Full Code Here

Examples of org.apache.nutch.net.URLFilters

  }


  @Override
  public void setup(Context context) {
    Configuration conf = context.getConfiguration();
    filters = new URLFilters(conf);
    curTime =
      conf.getLong(GeneratorJob.GENERATOR_CUR_TIME, System.currentTimeMillis());
    normalizers =
      new URLNormalizers(conf, URLNormalizers.SCOPE_GENERATE_HOST_COUNT);
    filter = conf.getBoolean(GeneratorJob.GENERATOR_FILTER, true);

View Full Code Here

Examples of org.apache.nutch.net.URLFilters

  public void configure(final JobConf job) {
    this.conf = job;
    this.segmentName = job.get(Nutch.SEGMENT_NAME_KEY);
    this.collectionName = job.get(ImportArcs.WAX_SUFFIX + ImportArcs.ARCCOLLECTION_KEY); // TODO MC
    this.urlNormalizers = new URLNormalizers(job, URLNormalizers.SCOPE_FETCHER); // TODO MC
    this.filters = new URLFilters(job); // TODO MC
  }

View Full Code Here

Examples of org.apache.nutch.net.URLFilters

      // Extract collection prefix from key to use later when adding
      // signature and link crawldatums.


      this.urlNormalizers =
        new URLNormalizers(job, URLNormalizers.SCOPE_OUTLINK);
      this.filters = new URLFilters(job);
      this.scfilters = new ScoringFilters(job);


      final float interval =
        job.getFloat("db.default.fetch.interval", 30f);
      final boolean ignoreExternalLinks =

View Full Code Here

Examples of org.apache.nutch.net.URLFilters

    this.nwIgnoreInternalLinks =
      job.getBoolean("db.ignore.internal.links", true);
      
    if (job.getBoolean(LinkDbFilter.URL_FILTERING, false))
    {
      this.nwUrlFilters = new URLFilters(job);
    }
    
    if (job.getBoolean(LinkDbFilter.URL_NORMALIZING, false))
    {
      this.nwUrlNormalizers =

View Full Code Here

Examples of org.apache.nutch.net.URLFilters

  
  public void setConf(Configuration conf) {
    super.setConf(conf);
    if (conf == null) return;
    if (conf.getBoolean("segment.merger.filter", false))
      filters = new URLFilters(conf);
    sliceSize = conf.getLong("segment.merger.slice", -1);
    if ((sliceSize > 0) && (LOG.isInfoEnabled())) {
      LOG.info("Slice size: " + sliceSize + " URLs.");
    }
  }

View Full Code Here

Examples of org.apache.nutch.net.URLFilters

    this.indexRedirects = job.getBoolean("wax.index.redirects", false);


    this.sha1 = job.getBoolean("wax.digest.sha1", false);


    this.urlNormalizers = new URLNormalizers(job, URLNormalizers.SCOPE_FETCHER);
    this.filters = new URLFilters(job);


    this.parseUtil = new ParseUtil(job);


    this.collectionName = job.get(ImportArcs.WAX_SUFFIX + ImportArcs.ARCCOLLECTION_KEY);

View Full Code Here

Examples of org.apache.nutch.net.URLFilters

    this.jobConf = job;
    urlFiltering = job.getBoolean(URL_FILTERING, false);
    urlNormalizers = job.getBoolean(URL_NORMALIZING, false);
        
    if (urlFiltering) {
      filters = new URLFilters(job);
    }
    if (urlNormalizers) {
      scope = job.get(URL_NORMALIZING_SCOPE, URLNormalizers.SCOPE_CRAWLDB);
      normalizers = new URLNormalizers(job, scope);
    }

View Full Code Here

Examples of org.apache.nutch.net.URLFilters

  public void configure(JobConf job) {
    maxAnchorLength = job.getInt("db.max.anchor.length", 100);
    maxInlinks = job.getInt("db.max.inlinks", 10000);
    ignoreInternalLinks = job.getBoolean("db.ignore.internal.links", true);
    if (job.getBoolean(LinkDbFilter.URL_FILTERING, false)) {
      urlFilters = new URLFilters(job);
    }
    if (job.getBoolean(LinkDbFilter.URL_NORMALIZING, false)) {
      urlNormalizers = new URLNormalizers(job, URLNormalizers.SCOPE_LINKDB);
    }
  }

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.