Package org.eweb4j.spiderman.task

Examples of org.eweb4j.spiderman.task.Task

95
96
97
98
99
100
101
102
103
104
105
   
    //记录已经访问过该url，下次不要重复访问它
    visitedUrls.add(nextUrl);
   
    //解析nextPage
    Task nextTask = new Task(nextUrl, task.url, task.site, 0);
    Model nextModel = new Model();
    List<Field> isAlsoParseInNextPageFields = target.getModel().getIsAlsoParseInNextPageFields();
    if (isAlsoParseInNextPageFields == null || isAlsoParseInNextPageFields.isEmpty())
      return ;
   
View Full Code Here
40
41
42
43
44
45
46
47
48
49
50
    if (this.site.db == null)
      return null;
   
    Collection<Task> validTasks = new ArrayList<Task>();
    for (String url : newUrls){
      Task newTask = new Task(url, task.url, site, 10);
      try {
        Target tgt = Util.isTargetUrl(newTask);
        boolean isFromSourceUrl = SourceUrlChecker.checkSourceUrl(site.getTargets().getSourceRules(), newTask.sourceUrl);
        //如果是目标url，但不是来自来源url，跳过
        if (tgt != null && !isFromSourceUrl){
View Full Code Here
150
151
152
153
154
155
156
157
158
159
160
    if (visitedUrls.contains(nextUrl)){
      return ;
    }
   
    //解析nextPage,找出里面的目标URL
    Task nextTask = new Task(nextUrl, task.url, task.site, 0);
   
    FetchRequest req = new FetchRequest();
    req.setUrl(nextUrl);
    FetchResult fr = task.site.fetcher.fetch(req);
    if (fr == null || fr.getPage() == null)
View Full Code Here
TOP

Related Classes of org.eweb4j.spiderman.task.Task

  • spiderman.plugin.impl.DigPointImpl
  • spiderman.plugin.impl.DupRemovalPointImpl
  • spiderman.plugin.impl.ParsePointImpl
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.