Package bixo.datum

Examples of bixo.datum.ScoredUrlDatum

67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
        BaseFetchJobPolicy policy = new DefaultFetchJobPolicy(maxUrlsPerSet, maxUrlsPerServer, BaseFetchJobPolicy.DEFAULT_CRAWL_DELAY);
       
        final int crawlDelay = 10000;
        policy.startFetchSet("groupingKey", crawlDelay);
       
        FetchSetInfo setInfo = policy.nextFetchSet(new ScoredUrlDatum("url1"));
        assertNotNull(setInfo);
        assertEquals(1, setInfo.getUrls().size());
        assertFalse(setInfo.isSkipping());
       
        assertNull(policy.nextFetchSet(new ScoredUrlDatum("url2")));
        assertNull(policy.nextFetchSet(new ScoredUrlDatum("url3")));
        assertNull(policy.nextFetchSet(new ScoredUrlDatum("url4")));
        assertNull(policy.nextFetchSet(new ScoredUrlDatum("url5")));

        setInfo = policy.endFetchSet();
        assertNotNull(setInfo);
        assertEquals(4, setInfo.getUrls().size());
        assertTrue(setInfo.isSkipping());
View Full Code Here
95
96
97
98
99
100
101
102
103
104
105
        final int crawlDelay = 10000;
        policy.startFetchSet("groupingKey", crawlDelay);
       
        long curSortKey = 0;
        for (int i = 0; i < maxUrlsPerServer; i++) {
            FetchSetInfo setInfo = policy.nextFetchSet(new ScoredUrlDatum("url" + i));
            assertNotNull(setInfo);
            assertTrue(setInfo.getSortKey() > curSortKey);
            curSortKey = setInfo.getSortKey();
        }
    }
View Full Code Here
TOP

Related Classes of bixo.datum.ScoredUrlDatum

  • bixo.config.DefaultFetchJobPolicyTest
  • bixo.fetcher.FetchTask
  • bixo.fetcher.http.SimpleHttpFetcherIntegrationTest
  • bixo.fetcher.SimpleHttpFetcherTest
  • bixo.operations.FilterAndScoreByUrlAndRobotsTest$MatchBlockedByRobotsKey
  • bixo.operations.MakeFetchSetsBuffer
  • bixo.operations.ProcessRobotsTask
  • bixo.operations.ResolveRedirectsTask
  • bixo.pipes.FetchPipe$MakeSkippedStatus
  • bixo.pipes.FetchPipe$SplitIntoSpecialAndRegularKeys
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.