Package us.codecraft.webmagic

Examples of us.codecraft.webmagic.Spider

18
19
20
21
22
23
24
25
26
        PageProcessor pageProcessor = mock(PageProcessor.class);
        Site site = mock(Site.class);
        when(pageProcessor.getSite()).thenReturn(site);
        when(site.getDomain()).thenReturn("codecraft.us");
        Worker worker = new Worker();
        Spider spider = Spider.create(pageProcessor);
        worker.addSpider(spider);
        assertThat(worker.getSpider("codecraft.us")).isEqualTo(spider);
    }
View Full Code Here
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
    @ExtractByUrl
    private String url;

    public static void main(String[] args) throws IOException, JMException {
        //Just for benchmark
        Spider thread = OOSpider.create(Site.me().addStartUrl("http://www.36kr.com/").setSleepTime(0), new PageModelPipeline() {
            @Override
            public void process(Object o, Task task) {

            }
        }, Kr36NewsModel.class).thread(20);
        thread.start();
        SpiderMonitor spiderMonitor = SpiderMonitor.instance();
        spiderMonitor.register(thread);
    }
View Full Code Here
76
77
78
79
80
81
82
83
84
    public Site getSite() {
        return site;
    }

    public static void main(String[] args) {
        Spider spider = Spider.create(new ZipCodePageProcessor()).scheduler(new PriorityScheduler()).addUrl("http://www.ip138.com/post/");

        spider.run();
    }
View Full Code Here
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
        return site;
    }

    public static void main(String[] args) {
        //single download
        Spider spider = Spider.create(new BaiduBaikePageProcessor()).thread(2);
        String urlTemplate = "http://baike.baidu.com/search/word?word=%s&pic=1&sug=1&enc=utf8";
        ResultItems resultItems = spider.<ResultItems>get(String.format(urlTemplate, "水力发电"));
        System.out.println(resultItems);

        //multidownload
        List<String> list = new ArrayList<String>();
        list.add(String.format(urlTemplate,"风力发电"));
        list.add(String.format(urlTemplate,"太阳能"));
        list.add(String.format(urlTemplate,"地热发电"));
        list.add(String.format(urlTemplate,"地热发电"));
        List<ResultItems> resultItemses = spider.<ResultItems>getAll(list);
        for (ResultItems resultItemse : resultItemses) {
            System.out.println(resultItemse.getAll());
        }
        spider.close();
    }
View Full Code Here
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
        ScriptProcessor pageProcessor = ScriptProcessorBuilder.custom()
                .language(params.getLanguage()).scriptFromFile(params.getScriptFileName()).thread(params.getThread()).build();
        pageProcessor.getSite().setSleepTime(params.getSleepTime());
        pageProcessor.getSite().setRetryTimes(3);
        pageProcessor.getSite().setAcceptStatCode(Sets.<Integer>newHashSet(200, 404,403, 500,502));
        Spider spider = Spider.create(pageProcessor).thread(params.getThread());
        spider.clearPipeline().addPipeline(new Pipeline() {
            @Override
            public void process(ResultItems resultItems, Task task) {

            }
        });
        if (params.getUrls() == null || params.getUrls().size() == 0) {
            System.err.println("Need at least one argument");
            System.out.println("Usage: java -jar webmagic.jar [-l language] -f script file [-t threadnum] [-s sleep time] url1 [url2 url3]");
            System.exit(-1);
        }
        for (String url : params.getUrls()) {
            spider.addUrl(url);
        }
        spider.run();
    }
View Full Code Here
32
33
34
35
36
37
38
39
40
        return site;

    }

    public static void main(String[] args) throws JMException {
        Spider spider = Spider.create(new OschinaBlogPageProcesser()).setScheduler(new QueueScheduler().setDuplicateRemover(new BloomFilterDuplicateRemover(2000)));
        SpiderMonitor.instance().register(spider);
        spider.run();
    }
View Full Code Here
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
*/
public class MonitorExample {

    public static void main(String[] args) throws Exception {

        Spider oschinaSpider = Spider.create(new OschinaBlogPageProcessor())
                .addUrl("http://my.oschina.net/flashsword/blog");
        Spider githubSpider = Spider.create(new GithubRepoPageProcessor())
                .addUrl("https://github.com/code4craft");

        SpiderMonitor.instance().register(oschinaSpider);
        SpiderMonitor.instance().register(githubSpider);
        oschinaSpider.start();
        githubSpider.start();
    }
View Full Code Here
18
19
20
21
22
23
24
25
26
27
28
29
30
            protected SpiderStatusMXBean getSpiderStatusMBean(Spider spider, MonitorSpiderListener monitorSpiderListener) {
                return new CustomSpiderStatus(spider, monitorSpiderListener);
            }
        };

        Spider oschinaSpider = Spider.create(new OschinaBlogPageProcessor())
                .addUrl("http://my.oschina.net/flashsword/blog").thread(2);
        Spider githubSpider = Spider.create(new GithubRepoPageProcessor())
                .addUrl("https://github.com/code4craft");

        spiderMonitor.register(oschinaSpider, githubSpider);

    }
View Full Code Here
TOP

Related Classes of us.codecraft.webmagic.Spider

  • us.codecraft.webmagic.downloader.HttpClientDownloader
  • us.codecraft.webmagic.example.MonitorExample
  • us.codecraft.webmagic.model.samples.Kr36NewsModel
  • us.codecraft.webmagic.monitor.SpiderMonitorTest
  • us.codecraft.webmagic.pipeline.CollectorPipeline
  • us.codecraft.webmagic.pipeline.ConsolePipeline
  • us.codecraft.webmagic.pipeline.ResultItemsCollectorPipeline
  • us.codecraft.webmagic.processor.example.BaiduBaikePageProcessor
  • us.codecraft.webmagic.samples.OschinaBlogPageProcesser
  • us.codecraft.webmagic.samples.scheduler.ZipCodePageProcessor
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.