Package us.codecraft.webmagic.scheduler

Examples of us.codecraft.webmagic.scheduler.FileCacheQueueScheduler

41
42
43
44
45
46
47
    private String url;

    public static void main(String[] args) {
        OOSpider.create(Site.me().addStartUrl("https://github.com/explore").setSleepTime(0).setRetryTimes(3),
                new JsonFilePageModelPipeline(), GithubRepo.class)
                .scheduler(new FileCacheQueueScheduler("/data/webmagic/cache/")).thread(15).run();
    }
View Full Code Here
39
40
41
42
43
44
45
46
47
48
49
        return site;
    }

    public static void main(String[] args) throws FileNotFoundException, UnsupportedEncodingException {
        Spider.create(new MamacnPageProcessor())
                .setScheduler(new FileCacheQueueScheduler("/data/webmagic/mamacn"))
                .addUrl("http://www.mama.cn/photo/t1-p1.html")
                .addPipeline(new OneFilePipeline("/data/webmagic/mamacn/data"))
                .thread(5)
                .run();
    }
View Full Code Here
29
30
31
32
33
34
35
36
37
38
//        Spider.me().pipeline(new FilePipeline()).scheduler(new FileCacheQueueScheduler(pageProcessor.getSite(),"/data/temp/webmagic/cache/")).
//                processor(pageProcessor).run();
        SimplePageProcessor pageProcessor2 = new SimplePageProcessor("http://www.diaoyuweng.com/home.php?mod=space&uid=88304&do=thread&view=me&type=thread&from=space", "http://www.diaoyuweng.com/thread-*-1-1.html");
        System.out.println(pageProcessor2.getSite().getCharset());
        pageProcessor2.getSite().setSleepTime(500);
        Spider.create(pageProcessor2).pipeline(new FilePipeline()).scheduler(new FileCacheQueueScheduler("/data/temp/webmagic/cache/")).
                run();


    }
View Full Code Here
27
28
29
30
31
32
33
34
        //Spider.me()是简化写法，其实就是new一个啦
        //Spider.pipeline()设定一个pipeline，支持链式调用
        //ConsolePipeline输出结果到控制台
        //FileCacheQueueSchedular保存url，支持断点续传，临时文件输出到/data/temp/webmagic/cache目录
        //Spider.run()执行
        Spider.create(sinaBlogProcessor).pipeline(new FilePipeline()).pipeline(pipeline).scheduler(new FileCacheQueueScheduler("/data/temp/webmagic/cache/")).
                run();
    }
View Full Code Here
TOP

Related Classes of us.codecraft.webmagic.scheduler.FileCacheQueueScheduler

  • us.codecraft.webmagic.model.samples.GithubRepo
  • us.codecraft.webmagic.processor.SinablogProcessorTest
  • us.codecraft.webmagic.Request
  • us.codecraft.webmagic.samples.MamacnPageProcessor
  • us.codecraft.webmagic.scheduler.component.DuplicateRemover
  • us.codecraft.webmagic.SpiderTest
  • java.util.concurrent.atomic.AtomicInteger
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.