Package org.apache.nutch.scoring

Examples of org.apache.nutch.scoring.ScoringFilters

160
161
162
163
164
165
166
  }

  public void configure(JobConf job) {
    setConf(job);
    this.filters = new IndexingFilters(getConf());
    this.scfilters = new ScoringFilters(getConf());
  }
View Full Code Here
52
53
54
55
56
57
58
59
    public void configure(JobConf job) {
      this.jobConf = job;
      urlNormalizer = new UrlNormalizerFactory(jobConf).getNormalizer();
      interval = jobConf.getFloat("db.default.fetch.interval", 30f);
      filters = new URLFilters(jobConf);
      scfilters = new ScoringFilters(jobConf);
      scoreInjected = jobConf.getFloat("db.score.injected", 1.0f);
    }
View Full Code Here
48
49
50
51
52
53
54
    super(conf);
  }

  public void configure(JobConf job) {
    setConf(job);
    this.scfilters = new ScoringFilters(job);
  }
View Full Code Here
81
82
83
84
85
86
87
      curTime = job.getLong("crawl.gen.curTime", System.currentTimeMillis());
      limit = job.getLong("crawl.topN",Long.MAX_VALUE)/job.getNumReduceTasks();
      maxPerHost = job.getInt("generate.max.per.host", -1);
      byIP = job.getBoolean("generate.max.per.host.by.ip", false);
      filters = new URLFilters(job);
      scfilters = new ScoringFilters(job);
    }
View Full Code Here
38
39
40
41
42
43
44
45
  private ScoringFilters scfilters = null;
  private float scoreInjected;

  public void configure(JobConf job) {
    retryMax = job.getInt("db.fetch.retry.max", 3);
    scfilters = new ScoringFilters(job);
    scoreInjected = job.getFloat("db.score.injected", 1.0f);
  }
View Full Code Here
52
53
54
55
56
57
58
59
60
61
62
  public RecordWriter getRecordWriter(FileSystem fs, JobConf job,
                                      String name, Progressable progress) throws IOException {

    this.urlNormalizer = new UrlNormalizerFactory(job).getNormalizer();
    this.filters = new URLFilters(job);
    this.scfilters = new ScoringFilters(job);
    final float interval = job.getFloat("db.default.fetch.interval", 30f);
    final boolean ignoreExternalLinks = job.getBoolean("db.ignore.external.links", false);
   
    Path text =
      new Path(new Path(job.getOutputPath(), ParseText.DIR_NAME), name);
View Full Code Here
545
546
547
548
549
550
551
552
553
554
555
    public FetcherThread(Configuration conf) {
      this.setDaemon(true);                       // don't hang JVM on exit
      this.setName("FetcherThread");              // use an informative name
      this.conf = conf;
      this.urlFilters = new URLFilters(conf);
      this.scfilters = new ScoringFilters(conf);
      this.parseUtil = new ParseUtil(conf);
      this.protocolFactory = new ProtocolFactory(conf);
      this.normalizers = new URLNormalizers(conf, URLNormalizers.SCOPE_FETCHER);
      this.maxCrawlDelay = conf.getInt("fetcher.max.crawl.delay", 30) * 1000;
      this.byIP = conf.getBoolean("fetcher.threads.per.host.by.ip", false);
View Full Code Here
57
58
59
60
61
62
63
  private ScoringFilters scfilters;

  public void configure(JobConf job) {
    setConf(job);
    this.filters = new IndexingFilters(getConf());
    this.scfilters = new ScoringFilters(getConf());
  }
View Full Code Here
103
104
105
106
107
108
109
110
111
112
    public FetcherThread(Configuration conf) {
      this.setDaemon(true);                       // don't hang JVM on exit
      this.setName("FetcherThread");              // use an informative name
      this.conf = conf;
      this.urlFilters = new URLFilters(conf);
      this.scfilters = new ScoringFilters(conf);
      this.parseUtil = new ParseUtil(conf);
      this.protocolFactory = new ProtocolFactory(conf);
      this.normalizers = new URLNormalizers(conf, URLNormalizers.SCOPE_FETCHER);
    }
View Full Code Here
83
84
85
86
87
88
89
90
91
92
93
  public RecordWriter<Text, Parse> getRecordWriter(FileSystem fs, JobConf job,
                                      String name, Progressable progress) throws IOException {

    this.filters = new URLFilters(job);
    this.normalizers = new URLNormalizers(job, URLNormalizers.SCOPE_OUTLINK);
    this.scfilters = new ScoringFilters(job);
    final int interval = job.getInt("db.fetch.interval.default", 2592000);
    final boolean ignoreExternalLinks = job.getBoolean("db.ignore.external.links", false);
    int maxOutlinksPerPage = job.getInt("db.max.outlinks.per.page", 100);
    final int maxOutlinks = (maxOutlinksPerPage < 0) ? Integer.MAX_VALUE
                                                     : maxOutlinksPerPage;
View Full Code Here
TOP

Related Classes of org.apache.nutch.scoring.ScoringFilters

  • org.apache.nutch.crawl.CrawlDbReducer
  • org.apache.nutch.crawl.DbUpdateMapper
  • org.apache.nutch.crawl.DbUpdateReducer
  • org.apache.nutch.crawl.DbUpdaterJob
  • org.apache.nutch.crawl.Generator$Selector
  • org.apache.nutch.crawl.GeneratorMapper
  • org.apache.nutch.crawl.Injector$InjectMapper
  • org.apache.nutch.crawl.InjectorJob$UrlMapper
  • org.apache.nutch.crawl.TestCrawlDbStates
  • org.apache.nutch.fetcher.Fetcher$FetcherThread
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.