Package org.apache.nutch.net

Examples of org.apache.nutch.net.URLFilters

313
314
315
316
317
318
319
320
321
322
323
 
  public void setConf(Configuration conf) {
    super.setConf(conf);
    if (conf == null) return;
    if (conf.getBoolean("segment.merger.filter", false))
      filters = new URLFilters(conf);
    sliceSize = conf.getLong("segment.merger.slice", -1);
    if ((sliceSize > 0) && (LOG.isInfoEnabled())) {
      LOG.info("Slice size: " + sliceSize + " URLs.");
    }
  }
View Full Code Here
80
81
82
83
84
85
86
87
    public void configure(JobConf job) {
      curTime = job.getLong("crawl.gen.curTime", System.currentTimeMillis());
      limit = job.getLong("crawl.topN",Long.MAX_VALUE)/job.getNumReduceTasks();
      maxPerHost = job.getInt("generate.max.per.host", -1);
      byIP = job.getBoolean("generate.max.per.host.by.ip", false);
      filters = new URLFilters(job);
      scfilters = new ScoringFilters(job);
    }
View Full Code Here
52
53
54
55
56
57
58
59
   
    public void configure(JobConf job) {
      super.configure(job);
      _maxInlinks = job.getInt("db.max.inlinks", 10000);
      if (job.getBoolean("linkdb.merger.urlfilters", false)) {
        filters = new URLFilters(job);
      }
    }
View Full Code Here
58
59
60
61
62
63
64
    public void close() throws IOException {}

    public void configure(JobConf conf) {
      if (conf.getBoolean("crawldb.merger.urlfilters", false))
        filters = new URLFilters(conf);
    }
View Full Code Here
312
313
314
315
316
317
318
319
320
321
322
 
  public void setConf(Configuration conf) {
    super.setConf(conf);
    if (conf == null) return;
    if (conf.getBoolean("segment.merger.filter", false)) {
      filters = new URLFilters(conf);
      mergeFilters = new SegmentMergeFilters(conf);
    }
    if (conf.getBoolean("segment.merger.normalizer", false))
      normalizers = new URLNormalizers(conf, URLNormalizers.SCOPE_DEFAULT);
    sliceSize = conf.getLong("segment.merger.slice", -1);
View Full Code Here
168
169
170
171
172
173
174
175
176
   */
  public void setConf(Configuration conf) {
    this.conf = conf;
    this.parserFactory = new ParserFactory(conf);
    this.normalizers = new URLNormalizers(conf, URLNormalizers.SCOPE_OUTLINK);
    this.filters = new URLFilters(conf);
    this.defaultEncoding =
      conf.get("parser.character.encoding.default", "windows-1252");
  }
View Full Code Here
59
60
61
62
63
64
65
66
67
68
69
 
  public void configure(JobConf job) {
    filter = job.getBoolean(URL_FILTERING, false);
    normalize = job.getBoolean(URL_NORMALIZING, false);
    if (filter) {
      filters = new URLFilters(job);
    }
    if (normalize) {
      scope = job.get(URL_NORMALIZING_SCOPE, URLNormalizers.SCOPE_LINKDB);
      normalizers = new URLNormalizers(job, scope);
    }
View Full Code Here
56
57
58
59
60
61
62
63
64
65
66
  public void configure(JobConf job) {
    urlFiltering = job.getBoolean(URL_FILTERING, false);
    urlNormalizers = job.getBoolean(URL_NORMALIZING, false);
    if (urlFiltering) {
      filters = new URLFilters(job);
    }
    if (urlNormalizers) {
      scope = job.get(URL_NORMALIZING_SCOPE, URLNormalizers.SCOPE_CRAWLDB);
      normalizers = new URLNormalizers(job, scope);
    }
View Full Code Here
63
64
65
66
67
68
69
70
71
72
73
 
  public void configure(JobConf job) {
    maxAnchorLength = job.getInt("db.max.anchor.length", 100);
    ignoreInternalLinks = job.getBoolean("db.ignore.internal.links", true);
    if (job.getBoolean(LinkDbFilter.URL_FILTERING, false)) {
      urlFilters = new URLFilters(job);
    }
    if (job.getBoolean(LinkDbFilter.URL_NORMALIZING, false)) {
      urlNormalizers = new URLNormalizers(job, URLNormalizers.SCOPE_LINKDB);
    }
  }
View Full Code Here
140
141
142
143
144
145
146
147
148
149
150
      if (maxCount==-1 && oldMaxPerHost!=-1){
        maxCount = oldMaxPerHost;
        byDomain = false;
      }
      if (GENERATOR_COUNT_VALUE_DOMAIN.equals(job.get(GENERATOR_COUNT_MODE))) byDomain = true;
      filters = new URLFilters(job);
      normalise = job.getBoolean(GENERATOR_NORMALISE, true);
      if (normalise) normalizers = new URLNormalizers(job,
          URLNormalizers.SCOPE_GENERATE_HOST_COUNT);
      scfilters = new ScoringFilters(job);
      partitioner.configure(job);
View Full Code Here
TOP

Related Classes of org.apache.nutch.net.URLFilters

  • org.apache.nutch.crawl.CrawlDbFilter
  • org.apache.nutch.crawl.CrawlDbMerger$Merger
  • org.apache.nutch.crawl.Generator$Selector
  • org.apache.nutch.crawl.GeneratorMapper
  • org.apache.nutch.crawl.InjectorJob$UrlMapper
  • org.apache.nutch.crawl.LinkDb
  • org.apache.nutch.crawl.LinkDb$Merger
  • org.apache.nutch.crawl.LinkDbFilter
  • org.apache.nutch.fetcher.FetcherReducer$FetcherThread
  • org.apache.nutch.indexer.IndexerMapReduce
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.