Package org.apache.nutch.crawl.GeneratorJob

Examples of org.apache.nutch.crawl.GeneratorJob.SelectorEntry

85
86
87
88
89
90
91
92
    try {
      score = scoringFilters.generatorSortValue(url, page, score);
    } catch (ScoringFilterException e) {
      //ignore
    }
    SelectorEntry entry = new SelectorEntry(url, score);
    context.write(entry, page);
  }
View Full Code Here
159
160
161
162
163
164
165
166
167
168
169
   
    int numReduceTasks = 100;
   
    int partitionFromRef = refPartitioner.getPartition("http://www.example.org/", numReduceTasks);
    //init selector entry (score shouldn't matter)
    SelectorEntry selectorEntry = new SelectorEntry("http://www.example.org/", 1337);
    WebPage page = new WebPage();
    int partitionFromSig = sigPartitioner.getPartition(selectorEntry, page, numReduceTasks);
   
    assertEquals("partitions should be same",
        partitionFromRef, partitionFromSig);
View Full Code Here
165
166
167
168
169
170
171
172
173
174
175
   
    int numReduceTasks = 100;
   
    int partitionFromRef = refPartitioner.getPartition("http://www.example.org/", numReduceTasks);
    //init selector entry (score shouldn't matter)
    SelectorEntry selectorEntry = new SelectorEntry("http://www.example.org/", 1337);
    WebPage page = new WebPage();
    int partitionFromSig = sigPartitioner.getPartition(selectorEntry, page, numReduceTasks);
   
    assertEquals("partitions should be same",
        partitionFromRef, partitionFromSig);
View Full Code Here
TOP

Related Classes of org.apache.nutch.crawl.GeneratorJob.SelectorEntry

  • org.apache.nutch.crawl.GeneratorMapper
  • org.apache.nutch.crawl.TestURLPartitioner
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.