Package org.apache.nutch.net

Examples of org.apache.nutch.net.URLFilters

61
62
63
64
65
66
67
68
69
70
71
  public void configure(JobConf job) {
    this.jobConf = job;
    urlFiltering = job.getBoolean(URL_FILTERING, false);
    urlNormalizers = job.getBoolean(URL_NORMALIZING, false);
    if (urlFiltering) {
      filters = new URLFilters(job);
    }
    if (urlNormalizers) {
      scope = job.get(URL_NORMALIZING_SCOPE, URLNormalizers.SCOPE_CRAWLDB);
      normalizers = new URLNormalizers(job, scope);
    }
View Full Code Here
95
96
97
98
99
100
101
102
103
104
105
  public void configure(JobConf job) {
    maxAnchorLength = job.getInt("db.max.anchor.length", 100);
    maxInlinks = job.getInt("db.max.inlinks", 10000);
    ignoreInternalLinks = job.getBoolean("db.ignore.internal.links", true);
    if (job.getBoolean(LinkDbFilter.URL_FILTERING, false)) {
      urlFilters = new URLFilters(job);
    }
    if (job.getBoolean(LinkDbFilter.URL_NORMALIZING, false)) {
      urlNormalizers = new URLNormalizers(job, URLNormalizers.SCOPE_LINKDB);
    }
  }
View Full Code Here
76
77
78
79
80
81
82
83
84
85
86
    @Override
    public void configure(JobConf job) {
      super.configure(job);
      scfilters = new ScoringFilters(job);
      if (job.getBoolean(FILTER_KEY, false)) {
        filters = new URLFilters(job);
      }
      if (job.getBoolean(NORMALIZE_KEY, false)) {
        normalizers = new URLNormalizers(job, URLNormalizers.SCOPE_INJECT);
      }
    }
View Full Code Here
303
304
305
306
307
308
309
310
311
312
313
 
  public void setConf(Configuration conf) {
    super.setConf(conf);
    if (conf == null) return;
    if (conf.getBoolean("segment.merger.filter", false))
      filters = new URLFilters(conf);
    if (conf.getBoolean("segment.merger.normalizer", false))
      normalizers = new URLNormalizers(conf, URLNormalizers.SCOPE_DEFAULT);
    sliceSize = conf.getLong("segment.merger.slice", -1);
    if ((sliceSize > 0) && (LOG.isInfoEnabled())) {
      LOG.info("Slice size: " + sliceSize + " URLs.");
View Full Code Here
105
106
107
108
109
110
111
112
113
114
115
    public void configure(JobConf job) {
      curTime = job.getLong(CRAWL_GEN_CUR_TIME, System.currentTimeMillis());
      limit = job.getLong(CRAWL_TOP_N,Long.MAX_VALUE)/job.getNumReduceTasks();
      maxPerHost = job.getInt(GENERATE_MAX_PER_HOST, -1);
      byIP = job.getBoolean(GENERATE_MAX_PER_HOST_BY_IP, false);
      filters = new URLFilters(job);
      normalizers = new URLNormalizers(job, URLNormalizers.SCOPE_GENERATE_HOST_COUNT);
      scfilters = new ScoringFilters(job);
      hostPartitioner.configure(job);
      filter = job.getBoolean(CRAWL_GENERATE_FILTER, true);
      genDelay = job.getLong(CRAWL_GEN_DELAY, 7L) * 3600L * 24L * 1000L;
 
View Full Code Here
91
92
93
94
95
96
97
98
    if (normalize) {
      urlNormalizers = new URLNormalizers(getConf(), URLNormalizers.SCOPE_INDEXER);
    }

    if (filter) {
      urlFilters = new URLFilters(getConf());
    }
  }
View Full Code Here
76
77
78
79
80
81
82
83
84
85
86
    @Override
    public void configure(JobConf job) {
      super.configure(job);
      scfilters = new ScoringFilters(job);
      if (job.getBoolean(FILTER_KEY, false)) {
        filters = new URLFilters(job);
      }
      if (job.getBoolean(NORMALIZE_KEY, false)) {
        normalizers = new URLNormalizers(job, URLNormalizers.SCOPE_INJECT);
      }
    }
View Full Code Here
311
312
313
314
315
316
317
318
319
320
321
 
  public void setConf(Configuration conf) {
    super.setConf(conf);
    if (conf == null) return;
    if (conf.getBoolean("segment.merger.filter", false))
      filters = new URLFilters(conf);
    if (conf.getBoolean("segment.merger.normalizer", false))
      normalizers = new URLNormalizers(conf, URLNormalizers.SCOPE_DEFAULT);
    sliceSize = conf.getLong("segment.merger.slice", -1);
    if ((sliceSize > 0) && (LOG.isInfoEnabled())) {
      LOG.info("Slice size: " + sliceSize + " URLs.");
View Full Code Here
61
62
63
64
65
66
67
68
69
70
71
 
  public void configure(JobConf job) {
    maxAnchorLength = job.getInt("db.max.anchor.length", 100);
    ignoreInternalLinks = job.getBoolean("db.ignore.internal.links", true);
    if (job.getBoolean(LinkDbFilter.URL_FILTERING, false)) {
      urlFilters = new URLFilters(job);
    }
    if (job.getBoolean(LinkDbFilter.URL_NORMALIZING, false)) {
      urlNormalizers = new URLNormalizers(job, URLNormalizers.SCOPE_LINKDB);
    }
  }
View Full Code Here
119
120
121
122
123
124
125
126
127
128
129
  public void configure(JobConf job) {

    // set the url filters, scoring filters the parse util and the url
    // normalizers
    this.jobConf = job;
    this.urlFilters = new URLFilters(jobConf);
    this.scfilters = new ScoringFilters(jobConf);
    this.parseUtil = new ParseUtil(jobConf);
    this.normalizers = new URLNormalizers(jobConf, URLNormalizers.SCOPE_FETCHER);
    interval = jobConf.getInt("db.fetch.interval.default", 2592000);
  }
View Full Code Here
TOP

Related Classes of org.apache.nutch.net.URLFilters

  • org.apache.nutch.crawl.CrawlDbFilter
  • org.apache.nutch.crawl.CrawlDbMerger$Merger
  • org.apache.nutch.crawl.Generator$Selector
  • org.apache.nutch.crawl.GeneratorMapper
  • org.apache.nutch.crawl.InjectorJob$UrlMapper
  • org.apache.nutch.crawl.LinkDb
  • org.apache.nutch.crawl.LinkDb$Merger
  • org.apache.nutch.crawl.LinkDbFilter
  • org.apache.nutch.fetcher.FetcherReducer$FetcherThread
  • org.apache.nutch.indexer.IndexerMapReduce
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.