Examples of LinkExtractor

Examples of com.ontometrics.scraper.extraction.LinkExtractor

29
30
31
32
33
34
35
36
37
38
//  private static final String listingTableKeyword = "Opportunity Title";
//  private static final String linkPattern = "mode=VIEW";

  @Test
  public void canExtractLinksFromListingPage() {
    List<Link> foundLinks = new LinkExtractor().source(html().url(PagedListingTable.getUrl())).getLinks();

    log.info("found {} links: {}", foundLinks.size(), foundLinks);
    assertThat(foundLinks.size(), is(greaterThan(0)));
  }
View Full Code Here

Examples of com.ontometrics.scraper.extraction.LinkExtractor

40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
  @Test
    @Ignore
  public void canExtractLinksFromMultiplePagesThenFollowToDetailsPage() throws MalformedURLException {
    Iterator pageIterator = new Iterator() {

      List<Link> foundLinks = new LinkExtractor().source(html().url(PagedListingTable.getUrl())).getLinks();
      ListIterator<Link> foundLinksIterator = foundLinks.listIterator();

      @Override
      public URL next() {
        Link currentLink = (Link) foundLinksIterator.next();
        log.debug("current iterating page = {}", currentLink.getHref());

        String CurrentURLStr = PagedListingFolder.getPath() + "/" + currentLink.getHref();
        URL currentURL = TestUtil.getFileAsURL(CurrentURLStr);

        return currentURL;
      }

      @Override
      public boolean hasNext() {

        try {
          Link nextLink = foundLinks.get(foundLinksIterator.nextIndex());
          log.debug("next page to iterate = {}", nextLink.getHref());
        } catch (Exception e) {
          log.debug("Page iterator can't peek next link");
        }

        return foundLinksIterator.hasNext();
      }

      @Override
      public URL getBaseUrl() {
        // TODO Auto-generated method stub
        return PagedListingFolder.getUrl();
      }
    };
    HtmlExtractor htmlExtractor = html().url(PagedListingTable.getUrl());

    List<Record> records = new ListingDetailScraper().setConvertURLs(true)
        .iterator(pageIterator)
        .listing(new LinkExtractor().source(htmlExtractor))
        .details(new DefaultFieldExtractor().source(htmlExtractor))
        .getRecords();

    assertThat(records.size(), greaterThan(0));
View Full Code Here

Examples of org.apache.droids.parse.html.LinkExtractor

58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
    String charset = entity.getCharset();
    if (charset == null) {
      charset = "UTF-8";
    }
    EchoHandler data = new EchoHandler(charset);
    LinkExtractor extractor = new LinkExtractor(link, elements);
   
    TeeContentHandler parallelHandler = new TeeContentHandler(data, extractor);

    InputStream instream = entity.obtainContent();
    try {
      parser.parse(instream, parallelHandler, metadata);
     
      return new ParseImpl(data.toString(), extractor.getLinks());
    } catch (SAXException ex) {
      throw new DroidsException("Failure parsing document " + link.getId(), ex);
    } catch (TikaException ex) {
      throw new DroidsException("Failure parsing document " + link.getId(), ex);
    } finally {
View Full Code Here

Examples of penny.parser.LinkExtractor

32
33
34
35
36
37
38
39
40
41
    private List<String> srcLinks;

    public LinkParser(Download download) throws URISyntaxException {
        this.download = download;
        parsingModel = Model.getApplicationSettings().getParsingModel();
        linkExtractor = new LinkExtractor(download.getUrl().toURI(), this);
        linkExtractor.setLinkState(download.getLinkState());
        hrefLinks = new ArrayList<String>();
        srcLinks = new ArrayList<String>();
    }
View Full Code Here

Examples of penny.parser.LinkExtractor

70
71
72
73
74
75
76
77
78
79
            }
        }
    }

    public void reset() throws URISyntaxException {
        linkExtractor = new LinkExtractor(download.getUrl().toURI(), this);
        download.setLinkState(linkExtractor.getLinkState());
        download.clearHrefLinks();
        download.clearSrcLinks();
    }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.