Package spiderman.plugin.util

Examples of spiderman.plugin.util.SpiderConfig

29
30
31
32
33
34
35
36
37
38
39
40
41
  public void destroy() {
  }
 
  public static void main(String[] args){
    PageFetcherImpl fetcher = new PageFetcherImpl();
    SpiderConfig config = new SpiderConfig();
    config.setCharset("utf-8");
    config.setPolitenessDelay(200);
    fetcher.setConfig(config);
    fetcher.init(null);
    try {
      String url = "http://www.livingsocial.com/cities/1964-klang-valley-kuala-lumpur/deals/638602-patong-bay-resotel-return-flight?append_ref_code=source_cities_show";
      FetchRequest req = new FetchRequest();
View Full Code Here
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
 
  public FetchResult fetch(Task task, FetchResult result) throws Exception {
    synchronized (site) {
      if (site.fetcher == null){
        PageFetcherImpl fetcher = new PageFetcherImpl();
        SpiderConfig config = new SpiderConfig();
        if (task.site.getCharset() != null && task.site.getCharset().trim().length() > 0)
          config.setCharset(task.site.getCharset());
        if (task.site.getUserAgent() != null && task.site.getUserAgent().trim().length() > 0)
          config.setUserAgentString(task.site.getUserAgent());
        if (task.site.getIncludeHttps() != null && task.site.getIncludeHttps().trim().length() > 0)
          config.setIncludeHttpsPages("1".equals(task.site.getIncludeHttps()) || "true".equals(task.site.getIncludeHttps()));
        String sdelay = task.site.getReqDelay();
        if (sdelay == null || sdelay.trim().length() == 0)
          sdelay = "200";
       
        int delay = CommonUtil.toSeconds(sdelay).intValue()*1000;
        if (delay < 0)
          delay = 200;
       
        config.setPolitenessDelay(delay);
        fetcher.setConfig(config);
       
        fetcher.init(site);
        site.fetcher = fetcher;
      }
View Full Code Here
TOP

Related Classes of spiderman.plugin.util.SpiderConfig

  • spiderman.plugin.impl.FetchPointImpl
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.