Package org.apache.nutch.net

Examples of org.apache.nutch.net.URLFilters

97
98
99
100
101
102
103
104
105
106
107
        InterruptedException {
      urlNormalizers = new URLNormalizers(context.getConfiguration(),
          URLNormalizers.SCOPE_INJECT);
      interval = context.getConfiguration().getInt("db.fetch.interval.default",
          2592000);
      filters = new URLFilters(context.getConfiguration());
      scfilters = new ScoringFilters(context.getConfiguration());
      scoreInjected = context.getConfiguration().getFloat("db.score.injected",
          1.0f);
      curTime = context.getConfiguration().getLong("injector.current.time",
          System.currentTimeMillis());
View Full Code Here
92
93
94
95
96
97
98
99
100
101
102
  }

  @Override
  public void setup(Context context) {
    Configuration conf = context.getConfiguration();
    filters = new URLFilters(conf);
    curTime =
      conf.getLong(GeneratorJob.GENERATOR_CUR_TIME, System.currentTimeMillis());
    normalizers =
      new URLNormalizers(conf, URLNormalizers.SCOPE_GENERATE_HOST_COUNT);
    filter = conf.getBoolean(GeneratorJob.GENERATOR_FILTER, true);
View Full Code Here
92
93
94
95
96
97
98
  public void configure(final JobConf job) {
    this.conf = job;
    this.segmentName = job.get(Nutch.SEGMENT_NAME_KEY);
    this.collectionName = job.get(ImportArcs.WAX_SUFFIX + ImportArcs.ARCCOLLECTION_KEY); // TODO MC
    this.urlNormalizers = new URLNormalizers(job, URLNormalizers.SCOPE_FETCHER); // TODO MC
    this.filters = new URLFilters(job); // TODO MC
  }
View Full Code Here
759
760
761
762
763
764
765
766
767
768
769
      // Extract collection prefix from key to use later when adding
      // signature and link crawldatums.

      this.urlNormalizers =
        new URLNormalizers(job, URLNormalizers.SCOPE_OUTLINK);
      this.filters = new URLFilters(job);
      this.scfilters = new ScoringFilters(job);

      final float interval =
        job.getFloat("db.default.fetch.interval", 30f);
      final boolean ignoreExternalLinks =
View Full Code Here
76
77
78
79
80
81
82
83
84
85
86
    this.nwIgnoreInternalLinks =
      job.getBoolean("db.ignore.internal.links", true);
     
    if (job.getBoolean(LinkDbFilter.URL_FILTERING, false))
    {
      this.nwUrlFilters = new URLFilters(job);
    }
   
    if (job.getBoolean(LinkDbFilter.URL_NORMALIZING, false))
    {
      this.nwUrlNormalizers =
View Full Code Here
285
286
287
288
289
290
291
292
293
294
295
 
  public void setConf(Configuration conf) {
    super.setConf(conf);
    if (conf == null) return;
    if (conf.getBoolean("segment.merger.filter", false))
      filters = new URLFilters(conf);
    sliceSize = conf.getLong("segment.merger.slice", -1);
    if ((sliceSize > 0) && (LOG.isInfoEnabled())) {
      LOG.info("Slice size: " + sliceSize + " URLs.");
    }
  }
View Full Code Here
252
253
254
255
256
257
258
259
260
261
262
    this.indexRedirects = job.getBoolean("wax.index.redirects", false);

    this.sha1 = job.getBoolean("wax.digest.sha1", false);

    this.urlNormalizers = new URLNormalizers(job, URLNormalizers.SCOPE_FETCHER);
    this.filters = new URLFilters(job);

    this.parseUtil = new ParseUtil(job);

    this.collectionName = job.get(ImportArcs.WAX_SUFFIX + ImportArcs.ARCCOLLECTION_KEY);
View Full Code Here
62
63
64
65
66
67
68
69
70
71
72
    this.jobConf = job;
    urlFiltering = job.getBoolean(URL_FILTERING, false);
    urlNormalizers = job.getBoolean(URL_NORMALIZING, false);
       
    if (urlFiltering) {
      filters = new URLFilters(job);
    }
    if (urlNormalizers) {
      scope = job.get(URL_NORMALIZING_SCOPE, URLNormalizers.SCOPE_CRAWLDB);
      normalizers = new URLNormalizers(job, scope);
    }     
View Full Code Here
96
97
98
99
100
101
102
103
104
105
106
  public void configure(JobConf job) {
    maxAnchorLength = job.getInt("db.max.anchor.length", 100);
    maxInlinks = job.getInt("db.max.inlinks", 10000);
    ignoreInternalLinks = job.getBoolean("db.ignore.internal.links", true);
    if (job.getBoolean(LinkDbFilter.URL_FILTERING, false)) {
      urlFilters = new URLFilters(job);
    }
    if (job.getBoolean(LinkDbFilter.URL_NORMALIZING, false)) {
      urlNormalizers = new URLNormalizers(job, URLNormalizers.SCOPE_LINKDB);
    }
  }
View Full Code Here
78
79
80
81
82
83
84
85
86
87
88
    public void configure(JobConf job) {
      super.configure(job);
      defaultInterval = job.getInt("db.fetch.interval.default", 0);
      scfilters = new ScoringFilters(job);
      if (job.getBoolean(FILTER_KEY, false)) {
        filters = new URLFilters(job);
      }
      if (job.getBoolean(NORMALIZE_KEY, false)) {
        normalizers = new URLNormalizers(job, URLNormalizers.SCOPE_INJECT);
      }
    }
View Full Code Here
TOP

Related Classes of org.apache.nutch.net.URLFilters

  • org.apache.nutch.crawl.CrawlDbFilter
  • org.apache.nutch.crawl.CrawlDbMerger$Merger
  • org.apache.nutch.crawl.Generator$Selector
  • org.apache.nutch.crawl.GeneratorMapper
  • org.apache.nutch.crawl.InjectorJob$UrlMapper
  • org.apache.nutch.crawl.LinkDb
  • org.apache.nutch.crawl.LinkDb$Merger
  • org.apache.nutch.crawl.LinkDbFilter
  • org.apache.nutch.fetcher.FetcherReducer$FetcherThread
  • org.apache.nutch.indexer.IndexerMapReduce
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.