Package bixo.config.BaseFetchJobPolicy

Examples of bixo.config.BaseFetchJobPolicy.FetchSetInfo

91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
        PartitioningKey newKey = new PartitioningKey(key, _numReduceTasks);
       
        while (safeHasNext()) {
            ScoredUrlDatum scoredDatum = new ScoredUrlDatum(new TupleEntry(values.next()));
            FetchSetInfo setInfo = _policy.nextFetchSet(scoredDatum);
            if (setInfo != null) {
                FetchSetDatum result = makeFetchSetDatum(setInfo, newKey, safeHasNext());
                collector.add(BixoPlatform.clone(result.getTuple(), process));
            }
        }
       
        // See if we have another partially built datum to add.
        FetchSetInfo setInfo = _policy.endFetchSet();
        if (setInfo != null) {
            FetchSetDatum result = makeFetchSetDatum(setInfo, newKey, false);
            collector.add(BixoPlatform.clone(result.getTuple(), process));
        }
    }
View Full Code Here
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
        // Should be nothing yet.
        assertNull(policy.endFetchSet());
       
        assertNull(policy.nextFetchSet(new ScoredUrlDatum("url1")));
       
        FetchSetInfo setInfo = policy.nextFetchSet(new ScoredUrlDatum("url2"));
        assertNotNull(setInfo);
        assertEquals(2, setInfo.getUrls().size());
        assertEquals("url1", setInfo.getUrls().get(0).getUrl());
        assertEquals("url2", setInfo.getUrls().get(1).getUrl());
        assertEquals(crawlDelay * 2, setInfo.getFetchDelay());
       
        FetchSetInfo setInfo2 = policy.nextFetchSet(new ScoredUrlDatum("url3"));
        assertNotNull(setInfo2);
        assertEquals(1, setInfo.getUrls().size());
        assertEquals("url3", setInfo.getUrls().get(0).getUrl());
       
        assertNull(policy.endFetchSet());
View Full Code Here
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
        BaseFetchJobPolicy policy = new DefaultFetchJobPolicy(maxUrlsPerSet, maxUrlsPerServer, BaseFetchJobPolicy.DEFAULT_CRAWL_DELAY);
       
        final int crawlDelay = 10000;
        policy.startFetchSet("groupingKey", crawlDelay);
       
        FetchSetInfo setInfo = policy.nextFetchSet(new ScoredUrlDatum("url1"));
        assertNotNull(setInfo);
        assertEquals(1, setInfo.getUrls().size());
        assertFalse(setInfo.isSkipping());
       
        assertNull(policy.nextFetchSet(new ScoredUrlDatum("url2")));
        assertNull(policy.nextFetchSet(new ScoredUrlDatum("url3")));
        assertNull(policy.nextFetchSet(new ScoredUrlDatum("url4")));
        assertNull(policy.nextFetchSet(new ScoredUrlDatum("url5")));

        setInfo = policy.endFetchSet();
        assertNotNull(setInfo);
        assertEquals(4, setInfo.getUrls().size());
        assertTrue(setInfo.isSkipping());
    }
View Full Code Here
95
96
97
98
99
100
101
102
103
104
105
        final int crawlDelay = 10000;
        policy.startFetchSet("groupingKey", crawlDelay);
       
        long curSortKey = 0;
        for (int i = 0; i < maxUrlsPerServer; i++) {
            FetchSetInfo setInfo = policy.nextFetchSet(new ScoredUrlDatum("url" + i));
            assertNotNull(setInfo);
            assertTrue(setInfo.getSortKey() > curSortKey);
            curSortKey = setInfo.getSortKey();
        }
    }
View Full Code Here
TOP

Related Classes of bixo.config.BaseFetchJobPolicy.FetchSetInfo

  • bixo.config.DefaultFetchJobPolicyTest
  • bixo.operations.MakeFetchSetsBuffer
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.