Package us.codecraft.webmagic

Examples of us.codecraft.webmagic.Task

49
50
51
52
53
54
55
56
57
58
59
    }

    @Test
    public void testCycleTriedTimes() {
        HttpClientDownloader httpClientDownloader = new HttpClientDownloader();
        Task task = Site.me().setDomain("localhost").setCycleRetryTimes(5).toTask();
        Request request = new Request("http://localhost/404");
        Page page = httpClientDownloader.download(request, task);
        assertThat(page.getTargetRequests().size() > 0);
        assertThat((Integer) page.getTargetRequests().get(0).getExtra(Request.CYCLE_TRIED_TIMES)).isEqualTo(1);
        page = httpClientDownloader.download(page.getTargetRequests().get(0), task);
View Full Code Here
22
23
24
25
26
27
28
29
30
31
32
        resultItems = new ResultItems();
        resultItems.put("content", "webmagic 爬虫工具");
        Request request = new Request("http://www.baidu.com");
        resultItems.setRequest(request);

        task = new Task() {
            @Override
            public String getUUID() {
                return UUID.randomUUID().toString();
            }
View Full Code Here
20
21
22
23
24
25
26
27
28
29
30
  @Test
  public void test() {
    SeleniumDownloader seleniumDownloader = new SeleniumDownloader(chromeDriverPath);
    long time1 = System.currentTimeMillis();
    for (int i = 0; i < 100; i++) {
      Page page = seleniumDownloader.download(new Request("http://huaban.com/"), new Task() {
        @Override
        public String getUUID() {
          return "huaban.com";
        }
View Full Code Here
42
43
44
45
46
47
48
49
50
51
52
  @Test
  public void testBaiduWenku() {
    SeleniumDownloader seleniumDownloader = new SeleniumDownloader(chromeDriverPath);
        seleniumDownloader.setSleepTime(10000);
    long time1 = System.currentTimeMillis();
    Page page = seleniumDownloader.download(new Request("http://wenku.baidu.com/view/462933ff04a1b0717fd5ddc2.html"), new Task() {
      @Override
      public String getUUID() {
        return "huaban.com";
      }
View Full Code Here
20
21
22
23
24
25
26
27
28
29
30
    }

    @Ignore("environment depended")
    @Test
    public void test() {
        Task task = new Task() {
            @Override
            public String getUUID() {
                return "1";
            }
View Full Code Here
TOP

Related Classes of us.codecraft.webmagic.Task

  • us.codecraft.webmagic.downloader.HttpClientDownloaderTest
  • us.codecraft.webmagic.downloader.selenium.SeleniumDownloaderTest
  • us.codecraft.webmagic.pipeline.FilePipelineTest
  • us.codecraft.webmagic.scheduler.RedisSchedulerTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.