Package bixo.operations

Examples of bixo.operations.FixedScoreGenerator

525
526
527
528
529
530
531
532
533
534
535
       
        // This will limit us to one URL.
        final int maxUrls = 1;
        FetcherPolicy defaultPolicy = new FetcherPolicy();
        BaseFetcher fetcher = new FakeHttpFetcher(false, 1, defaultPolicy);
        BaseScoreGenerator scorer = new FixedScoreGenerator();
        BaseRobotsParser parser = new SimpleRobotRulesParser();
        BaseFetchJobPolicy fetchJobPolicy = new DefaultFetchJobPolicy(defaultPolicy.getMaxRequestsPerConnection(), maxUrls, BaseFetchJobPolicy.DEFAULT_CRAWL_DELAY);
        FetchPipe fetchPipe = new FetchPipe(pipe, scorer, fetcher, fetcher, parser, fetchJobPolicy, 1);

        // Create the output
View Full Code Here
90
91
92
93
94
95
96
97
98
99
100
            BasePath inputPath = platform.makePath(path.getFile());
            Tap in = platform.makeTap(platform.makeTextScheme(), inputPath);

            Pipe importPipe = new Each("url importer", new Fields("line"), new CreateUrlFunction());

            BaseScoreGenerator scorer = new FixedScoreGenerator();
            BaseFetcher fetcher = new FakeHttpFetcher(true, 10);
            FetchPipe fetchPipe = new FetchPipe(importPipe, scorer, fetcher, 1);

            // Create the output, which is a dual file sink tap.
            String output = "build/test/RunFakeFetchPipe/dual";
View Full Code Here
26
27
28
29
30
31
32
33
34
public class FixedScoreGeneratorTest {

    @Test
    public void testConstantScore() throws Exception {
        final double score = 50.0;
        FixedScoreGenerator fsg = new FixedScoreGenerator(score);
       
        Assert.assertEquals(score, fsg.generateScore("www.domain.com", "domain.com", "http://domain.com"));
    }
View Full Code Here
TOP

Related Classes of bixo.operations.FixedScoreGenerator

  • bixo.examples.crawl.DemoCrawlWorkflow
  • bixo.fetcher.FetcherTest
  • bixo.operations.FixedScoreGeneratorTest
  • bixo.pipes.AbstractFetchPipeTest
  • bixo.pipes.RunFakeFetchPipe
  • com.scaleunlimited.helpful.tools.AnalyzeEmail
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.