Package org.apache.nutch.net

Examples of org.apache.nutch.net.URLFilters

120
121
122
123
124
125
126
127
128
129
130
  public void configure(JobConf job) {

    // set the url filters, scoring filters the parse util and the url
    // normalizers
    this.jobConf = job;
    this.urlFilters = new URLFilters(jobConf);
    this.scfilters = new ScoringFilters(jobConf);
    this.parseUtil = new ParseUtil(jobConf);
    this.normalizers = new URLNormalizers(jobConf, URLNormalizers.SCOPE_FETCHER);
    interval = jobConf.getInt("db.fetch.interval.default", 2592000);
  }
View Full Code Here
78
79
80
81
82
83
84
85
86
87
88
    @Override
    public void configure(JobConf job) {
      super.configure(job);
      scfilters = new ScoringFilters(job);
      if (job.getBoolean(FILTER_KEY, false)) {
        filters = new URLFilters(job);
      }
      if (job.getBoolean(NORMALIZE_KEY, false)) {
        normalizers = new URLNormalizers(job, URLNormalizers.SCOPE_INJECT);
      }
    }
View Full Code Here
120
121
122
123
124
125
126
127
128
129
130
  public void configure(JobConf job) {

    // set the url filters, scoring filters the parse util and the url
    // normalizers
    this.jobConf = job;
    this.urlFilters = new URLFilters(jobConf);
    this.scfilters = new ScoringFilters(jobConf);
    this.parseUtil = new ParseUtil(jobConf);
    this.normalizers = new URLNormalizers(jobConf, URLNormalizers.SCOPE_FETCHER);
    interval = jobConf.getInt("db.fetch.interval.default", 2592000);
  }
View Full Code Here
78
79
80
81
82
83
84
85
86
87
88
    @Override
    public void configure(JobConf job) {
      super.configure(job);
      scfilters = new ScoringFilters(job);
      if (job.getBoolean(FILTER_KEY, false)) {
        filters = new URLFilters(job);
      }
      if (job.getBoolean(NORMALIZE_KEY, false)) {
        normalizers = new URLNormalizers(job, URLNormalizers.SCOPE_INJECT);
      }
    }
View Full Code Here
59
60
61
62
63
64
65
66
67
68
69
 
  public void configure(JobConf job) {
    filter = job.getBoolean(URL_FILTERING, false);
    normalize = job.getBoolean(URL_NORMALIZING, false);
    if (filter) {
      filters = new URLFilters(job);
    }
    if (normalize) {
      scope = job.get(URL_NORMALIZING_SCOPE, URLNormalizers.SCOPE_LINKDB);
      normalizers = new URLNormalizers(job, scope);
    }
View Full Code Here
60
61
62
63
64
65
66
67
68
69
70
    urlFiltering = job.getBoolean(URL_FILTERING, false);
    urlNormalizers = job.getBoolean(URL_NORMALIZING, false);
    url404Purging = job.getBoolean(CrawlDb.CRAWLDB_PURGE_404, false);

    if (urlFiltering) {
      filters = new URLFilters(job);
    }
    if (urlNormalizers) {
      scope = job.get(URL_NORMALIZING_SCOPE, URLNormalizers.SCOPE_CRAWLDB);
      normalizers = new URLNormalizers(job, scope);
    }
View Full Code Here
63
64
65
66
67
68
69
70
71
72
73
 
  public void configure(JobConf job) {
    maxAnchorLength = job.getInt("db.max.anchor.length", 100);
    ignoreInternalLinks = job.getBoolean("db.ignore.internal.links", true);
    if (job.getBoolean(LinkDbFilter.URL_FILTERING, false)) {
      urlFilters = new URLFilters(job);
    }
    if (job.getBoolean(LinkDbFilter.URL_NORMALIZING, false)) {
      urlNormalizers = new URLNormalizers(job, URLNormalizers.SCOPE_LINKDB);
    }
  }
View Full Code Here
141
142
143
144
145
146
147
148
149
150
151
      if (maxCount==-1 && oldMaxPerHost!=-1){
        maxCount = oldMaxPerHost;
        byDomain = false;
      }
      if (GENERATOR_COUNT_VALUE_DOMAIN.equals(job.get(GENERATOR_COUNT_MODE))) byDomain = true;
      filters = new URLFilters(job);
      normalise = job.getBoolean(GENERATOR_NORMALISE, true);
      if (normalise) normalizers = new URLNormalizers(job,
          URLNormalizers.SCOPE_GENERATE_HOST_COUNT);
      scfilters = new ScoringFilters(job);
      partitioner.configure(job);
View Full Code Here
312
313
314
315
316
317
318
319
320
321
322
 
  public void setConf(Configuration conf) {
    super.setConf(conf);
    if (conf == null) return;
    if (conf.getBoolean("segment.merger.filter", false)) {
      filters = new URLFilters(conf);
      mergeFilters = new SegmentMergeFilters(conf);
    }
    if (conf.getBoolean("segment.merger.normalizer", false))
      normalizers = new URLNormalizers(conf, URLNormalizers.SCOPE_DEFAULT);
    sliceSize = conf.getLong("segment.merger.slice", -1);
View Full Code Here
168
169
170
171
172
173
174
175
176
   */
  public void setConf(Configuration conf) {
    this.conf = conf;
    this.parserFactory = new ParserFactory(conf);
    this.normalizers = new URLNormalizers(conf, URLNormalizers.SCOPE_OUTLINK);
    this.filters = new URLFilters(conf);
    this.defaultEncoding =
      conf.get("parser.character.encoding.default", "windows-1252");
  }
View Full Code Here
TOP

Related Classes of org.apache.nutch.net.URLFilters

  • org.apache.nutch.crawl.CrawlDbFilter
  • org.apache.nutch.crawl.CrawlDbMerger$Merger
  • org.apache.nutch.crawl.Generator$Selector
  • org.apache.nutch.crawl.GeneratorMapper
  • org.apache.nutch.crawl.InjectorJob$UrlMapper
  • org.apache.nutch.crawl.LinkDb
  • org.apache.nutch.crawl.LinkDb$Merger
  • org.apache.nutch.crawl.LinkDbFilter
  • org.apache.nutch.fetcher.FetcherReducer$FetcherThread
  • org.apache.nutch.indexer.IndexerMapReduce
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.