Package bixo.fetcher

Examples of bixo.fetcher.FetchRequest

49
50
51
52
53
54
55
56
    }
   
    public FetchRequest getFetchRequest(long now, long crawlDelay, int maxUrls) {
        // Ignore crawlDelay, and always use our delay.
        long nextRequestTime = now + getCrawlDelay();
        FetchRequest result = new FetchRequest(Math.min(maxUrls, 1), nextRequestTime);
        return result;
    }
View Full Code Here
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
   
    public FetchRequest getFetchRequest(long now, long crawlDelay, int maxUrls) {
        // we want to fetch maxUrls in the remaining time, but the min delay might constrain us.
       
        if ((getCrawlDelay() == 0) || (maxUrls == 0)) {
            return new FetchRequest(Math.min(maxUrls, getMaxRequestsPerConnection()), now);
        }

        // Even if we're at the end of the crawl, we still want to do our calculation using our
        // default fetch interval, to avoid not crawling anything if we run over. We rely on
        // an external mechanism to do any pruning of remaining URLs, so that they get properly
        // aborted.
        long fetchInterval = Math.max(DEFAULT_FETCH_INTERVAL, getCrawlEndTime() - now);
       
        // Crawl delay must be between the min crawl delay and the default crawl delay.
        long customCrawlDelay = Math.max(getCrawlDelay(), Math.min(DEFAULT_CRAWL_DELAY, fetchInterval / maxUrls));
       
        // Figure out how many URLs we can get in 5 minutes,or the remaining time (whatever is less).
        int numUrls = Math.min((int)(Math.min(DEFAULT_FETCH_INTERVAL, fetchInterval) / customCrawlDelay), maxUrls);
        long nextFetchTime = now + (numUrls * customCrawlDelay);
        return new FetchRequest(numUrls, nextFetchTime);
    }
View Full Code Here
43
44
45
46
47
48
49
50
    }
   
    @Test
    public void testEmptyFetchList() {
        FakeUserFetcherPolicy policy = new FakeUserFetcherPolicy();
        FetchRequest request = policy.getFetchRequest(System.currentTimeMillis(), 30 * 1000L, 0);
        Assert.assertEquals(0, request.getNumUrls());
    }
View Full Code Here
60
61
62
63
64
65
66
67
68
69
70
71
72
       
        for (int i = 0; i < 100; i++) {
            long curTime = System.currentTimeMillis();
           
            // Pass in longer crawl delay, that we'll ignore
            FetchRequest request = policy.getFetchRequest(curTime, 100 * 1000L, 100);
            Assert.assertEquals(1, request.getNumUrls());
            int delayInSeconds = (int)(request.getNextRequestTime() - curTime)/1000;
           
            Assert.assertTrue("Delay #" + i + " must be at least 5 seconds: " + delayInSeconds, delayInSeconds >= 30);
            Assert.assertTrue(delayInSeconds <= 91);
            delayCounts[delayInSeconds - 30] += 1;
        }
View Full Code Here
32
33
34
35
36
37
38
39
40
41
        AdaptiveFetcherPolicy policy = new AdaptiveFetcherPolicy(now + (10 * 1000), 1000L);
       
        // Ask for more than we can get in the remaining time, so it has to use a 5 minutes
        // window.
        final long crawlDelay = 30 * 1000L;
        FetchRequest request = policy.getFetchRequest(now, crawlDelay, 1000);
       
        // No way we can get more than 300.
        Assert.assertEquals(300, request.getNumUrls());
    }
View Full Code Here
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
        AdaptiveFetcherPolicy policy = new AdaptiveFetcherPolicy(now + (600 * 1000), 1000L);

        // Ask for more than will fit in the duration assuming a 30 second delay, but less than what
        // would be constrained by the 1 second min delay.
        final long crawlDelay = 30 * 1000L;
        FetchRequest request = policy.getFetchRequest(now, crawlDelay, 100);
       
        // Our per-request rate is about 600/100 = 6 seconds. So in 5 minutes (the default
        // window) we should get back 50
        Assert.assertEquals(50, request.getNumUrls());
       
        // Our next fetch time should be around 5 minutes.
        long deltaFromTarget = Math.abs(request.getNextRequestTime() - (now + (5 * 60 * 1000L)));
        Assert.assertTrue(deltaFromTarget < 100);
    }
View Full Code Here
68
69
70
71
72
73
74
75
76
77
78
79
80
        AdaptiveFetcherPolicy policy = new AdaptiveFetcherPolicy(now - 1000, 10 * 1000L);
       
        // Ask for more than we can get in the remaining time, so it has to use a 5 minutes
        // window.
        final long crawlDelay = 30 * 1000L;
        FetchRequest request = policy.getFetchRequest(now, crawlDelay, 1000);
       
        Assert.assertEquals(30, request.getNumUrls());

        // Our next fetch time should be around 5 minutes.
        long deltaFromTarget = Math.abs(request.getNextRequestTime() - (now + (5 * 60 * 1000L)));
        Assert.assertTrue(deltaFromTarget < 100);
    }
View Full Code Here
TOP

Related Classes of bixo.fetcher.FetchRequest

  • bixo.config.AdaptiveFetcherPolicy
  • bixo.config.AdaptiveFetcherPolicyTest
  • bixo.config.FakeUserFetcherPolicy
  • bixo.config.FakeUserFetcherPolicyTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.