Package edu.uci.ics.crawler4j.parser

Examples of edu.uci.ics.crawler4j.parser.HtmlParseData

110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
                final String pageURL = page.getWebURL().getURL();
                System.err.println( format("Processing page: [%s]", pageURL) );

                final ParseData parseData = page.getParseData();
                if (parseData instanceof HtmlParseData) {
                    final HtmlParseData htmlParseData = (HtmlParseData) parseData;
                    try {
                        synchronized (roverLock) {
                            Crawler.super.performExtraction(
                                    new StringDocumentSource(
                                            htmlParseData.getHtml(),
                                            pageURL

                                    )
                            );
                        }
View Full Code Here
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
  public void visit(Page page) {
    System.out.println("Visited: " + page.getWebURL().getURL());
    myCrawlStat.incProcessedPages();

    if (page.getParseData() instanceof HtmlParseData) {
      HtmlParseData parseData = (HtmlParseData) page.getParseData();
      List<WebURL> links = parseData.getOutgoingUrls();
      myCrawlStat.incTotalLinks(links.size());
      try {
        myCrawlStat.incTotalTextSize(parseData.getText().getBytes("UTF-8").length);
      } catch (UnsupportedEncodingException ignored) {
      }
    }
    // We dump this crawler statistics after processing every 50 pages
    if (myCrawlStat.getTotalProcessedPages() % 50 == 0) {
View Full Code Here
63
64
65
66
67
68
69
70
71
72
73
74
75
76
    System.out.println("Docid: " + docid);
    System.out.println("URL: " + url);
    System.out.println("Docid of parent page: " + parentDocid);

    if (page.getParseData() instanceof HtmlParseData) {
      HtmlParseData htmlParseData = (HtmlParseData) page.getParseData();
      String text = htmlParseData.getText();
      String html = htmlParseData.getHtml();
      List<WebURL> links = htmlParseData.getOutgoingUrls();

      System.out.println("Text length: " + text.length());
      System.out.println("Html length: " + html.length());
      System.out.println("Number of outgoing links: " + links.size());
    }
View Full Code Here
51
52
53
54
55
56
57
58
59
60
61
62
63
64
    System.out.println("Docid: " + docid);
    System.out.println("URL: " + url);
    System.out.println("Docid of parent page: " + parentDocid);

    if (page.getParseData() instanceof HtmlParseData) {
      HtmlParseData htmlParseData = (HtmlParseData) page.getParseData();
      String text = htmlParseData.getText();
      String html = htmlParseData.getHtml();
      List<WebURL> links = htmlParseData.getOutgoingUrls();

      System.out.println("Text length: " + text.length());
      System.out.println("Html length: " + html.length());
      System.out.println("Number of outgoing links: " + links.size());
    }
View Full Code Here
71
72
73
74
75
76
77
78
79
80
81
82
83
84
    Page page = download(url);
    if (page != null) {
      ParseData parseData = page.getParseData();
      if (parseData != null) {
        if (parseData instanceof HtmlParseData) {
          HtmlParseData htmlParseData = (HtmlParseData) parseData;
          System.out.println("Title: " + htmlParseData.getTitle());
          System.out.println("Text length: " + htmlParseData.getText().length());
          System.out.println("Html length: " + htmlParseData.getHtml().length());
        }
      } else {
        System.out.println("Couldn't parse the content of the page.");
      }
    } else {
View Full Code Here
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
      Page page = new Page(curURL);
      int docid = curURL.getDocid();
      if (fetchResult.fetchContent(page) && parser.parse(page, curURL.getURL())) {
        ParseData parseData = page.getParseData();
        if (parseData instanceof HtmlParseData) {
          HtmlParseData htmlParseData = (HtmlParseData) parseData;

          List<WebURL> toSchedule = new ArrayList<WebURL>();
          int maxCrawlDepth = myController.getConfig().getMaxDepthOfCrawling();
          for (WebURL webURL : htmlParseData.getOutgoingUrls()) {
            webURL.setParentDocid(docid);
            webURL.setParentUrl(curURL.getURL());
            int newdocid = docIdServer.getDocId(webURL.getURL());
            if (newdocid > 0) {
              // This is not the first time that this Url is
View Full Code Here
62
63
64
65
66
67
68
69
70
71
72
73
74
75
    System.out.println("Sub-domain: '" + subDomain + "'");
    System.out.println("Path: '" + path + "'");
    System.out.println("Parent page: " + parentUrl);

    if (page.getParseData() instanceof HtmlParseData) {
      HtmlParseData htmlParseData = (HtmlParseData) page.getParseData();
      String text = htmlParseData.getText();
      String html = htmlParseData.getHtml();
      List<WebURL> links = htmlParseData.getOutgoingUrls();

      System.out.println("Text length: " + text.length());
      System.out.println("Html length: " + html.length());
      System.out.println("Number of outgoing links: " + links.size());
    }
View Full Code Here
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
        return;
      }

      ParseData parseData = page.getParseData();
      if (parseData instanceof HtmlParseData) {
        HtmlParseData htmlParseData = (HtmlParseData) parseData;

        List<WebURL> toSchedule = new ArrayList<>();
        int maxCrawlDepth = myController.getConfig().getMaxDepthOfCrawling();
        for (WebURL webURL : htmlParseData.getOutgoingUrls()) {
          webURL.setParentDocid(docid);
          webURL.setParentUrl(curURL.getURL());
          int newdocid = docIdServer.getDocId(webURL.getURL());
          if (newdocid > 0) {
            // This is not the first time that this Url is
View Full Code Here
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
      Page page = new Page(curURL);
      int docid = curURL.getDocid();
      if (fetchResult.fetchContent(page) && parser.parse(page, curURL.getURL())) {
        ParseData parseData = page.getParseData();
        if (parseData instanceof HtmlParseData) {
          HtmlParseData htmlParseData = (HtmlParseData) parseData;

          List<WebURL> toSchedule = new ArrayList<WebURL>();
          int maxCrawlDepth = myController.getConfig().getMaxDepthOfCrawling();
          for (WebURL webURL : htmlParseData.getOutgoingUrls()) {
            webURL.setParentDocid(docid);
            webURL.setParentUrl(curURL.getURL());
            int newdocid = docIdServer.getDocId(webURL.getURL());
            if (newdocid > 0) {
              // This is not the first time that this Url is
View Full Code Here
TOP

Related Classes of edu.uci.ics.crawler4j.parser.HtmlParseData

  • edu.uci.ics.crawler4j.crawler.WebCrawler
  • edu.uci.ics.crawler4j.examples.basic.BasicCrawler
  • edu.uci.ics.crawler4j.examples.localdata.Downloader
  • edu.uci.ics.crawler4j.examples.localdata.LocalDataCollectorCrawler
  • edu.uci.ics.crawler4j.examples.multiple.BasicCrawler
  • edu.uci.ics.crawler4j.examples.shutdown.BasicCrawler
  • org.apache.any23.cli.Crawler
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.