Package com.iqbon.spider.domain

Examples of com.iqbon.spider.domain.Record

31
32
33
34
35
36
37
38
39
    }
  }

  @Test
  public void testGetCrawlContentBySourceAndUrl() {
    Record record = controller.getCrawlContentBySourceAndUrl(
        "http://home.163.com/13/0119/10/8LJ06D77001048P8.html", sourceUrl);
    logger.info(ToStringBuilder.reflectionToString(record));
  }
View Full Code Here
37
38
39
40
41
42
43
44
    recordDao.insertRecordList(list);
  }

  @Test
  public void testQueryRecordByUrl() {
    Record r = recordDao.queryRecordByUrl(record.getUrl());
    logger.info(ToStringBuilder.reflectionToString(r));
  }
View Full Code Here
43
44
45
46
47
48
49
50
51
    logger.info(ToStringBuilder.reflectionToString(r));
  }

  @Test
  public void testUpdateContentById() {
    Record r = recordDao.queryRecordByUrl(record.getUrl());
    r.setContent("after test");
    recordDao.updateContentById(r.getId(), r.getContent());
  }
View Full Code Here
145
146
147
148
149
150
151
152
153
154
155
                  replace.getReplacement());
            }
          }
          String sourceId = source.getId().toString();
          String title = document.title();
          return new Record(title, url, recordContent, sourceId);
        }
      }
      EntityUtils.consume(entity);
      return null;
    } catch (Exception e) {
View Full Code Here
32
33
34
35
36
37
38
39
40
41
42
  public void run() {
    List<Record> records = new ArrayList<Record>();
    List<String> links = crawlService.getLinkFromSource(source);
    for (String url : links) {
      Record record = crawlService.getContentFromLink(url, source);
      if (record != null) {
        records.add(record);
      }
    }
    recordDao.insertRecordList(records);
View Full Code Here
37
38
39
40
41
42
43
44
45
  }

  @Test
  public void testGetContentFromLink() {
    Source source = sourceDao.querySourceByUrl(URL);
    Record record = crawlService.getContentFromLink(
        "http://home.163.com/13/0123/18/8LU3QD5700104JVC.html", source);
    logger.info(ToStringBuilder.reflectionToString(record));
  }
View Full Code Here
TOP

Related Classes of com.iqbon.spider.domain.Record

  • com.iqbon.spider.dao.RecordDaoTest
  • com.iqbon.spider.service.crawl.CrawlService
  • com.iqbon.spider.service.crawl.CrawlServiceTest
  • com.iqbon.spider.service.crawl.CrawlThreadJob
  • com.iqbon.spider.web.RecordControllerTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.