Package us.codecraft.webmagic.pipeline

Examples of us.codecraft.webmagic.pipeline.ConsolePipeline

71
72
73
74
75
76
77
                '}';
    }

    public static void main(String[] args) {
        OOSpider.create(Site.me(), News163.class).addUrl("http://news.163.com/13/0802/05/958I1E330001124J_2.html")
                .scheduler(new RedisScheduler("localhost")).addPipeline(new MultiPagePipeline()).addPipeline(new ConsolePipeline()).run();
    }
View Full Code Here
279
280
281
282
283
284
285
286
287
288
289
    protected void initComponent() {
        if (downloader == null) {
            this.downloader = new HttpClientDownloader();
        }
        if (pipelines.isEmpty()) {
            pipelines.add(new ConsolePipeline());
        }
        downloader.setThread(threadNum);
        if (threadPool == null || threadPool.isShutdown()) {
            if (executorService != null && !executorService.isShutdown()) {
                threadPool = new CountableThreadPool(threadNum, executorService);
View Full Code Here
36
37
38
39
40
41
42
43
44
45
46
        init();
        String key = null;
        key = readKey(key);
        System.out.println("The demo started and will last 20 seconds...");
        //Start spider
        OOSpider.create(Site.me().addStartUrl(urlMap.get(key)), clazzMap.get(key)).pipeline(new MultiPagePipeline()).pipeline(new ConsolePipeline()).runAsync();

        try {
            Thread.sleep(20000);
        } catch (InterruptedException e) {
            e.printStackTrace();
View Full Code Here
TOP

Related Classes of us.codecraft.webmagic.pipeline.ConsolePipeline

  • us.codecraft.webmagic.main.QuickStarter
  • us.codecraft.webmagic.model.samples.News163
  • us.codecraft.webmagic.Spider
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.