Package edu.uci.ics.crawler4j.fetcher

Examples of edu.uci.ics.crawler4j.fetcher.PageFetcher

107
108
109
110
111
112
113
114
115
116
117
        try {
            crawlConfig = new CrawlConfig();
            crawlConfig.setCrawlStorageFolder( storageFolder.getAbsolutePath() );
            crawlConfig.setUserAgentString("Apache Any23 Web Crawler");
           
            final PageFetcher pageFetcher = new PageFetcher(crawlConfig);

            RobotstxtConfig robotstxtConfig = new RobotstxtConfig();
            final RobotstxtServer robotstxtServer = new RobotstxtServer(robotstxtConfig, pageFetcher);
           
            controller = new CrawlController(crawlConfig, pageFetcher, robotstxtServer);
View Full Code Here
40
41
42
43
44
45
46
47
48
49
50
    CrawlConfig config = new CrawlConfig();
    config.setCrawlStorageFolder(rootFolder);
    config.setMaxPagesToFetch(10);
    config.setPolitenessDelay(1000);

    PageFetcher pageFetcher = new PageFetcher(config);
    RobotstxtConfig robotstxtConfig = new RobotstxtConfig();
    RobotstxtServer robotstxtServer = new RobotstxtServer(robotstxtConfig, pageFetcher);
    CrawlController controller = new CrawlController(config, pageFetcher, robotstxtServer);

    controller.addSeed("http://www.ics.uci.edu/");
View Full Code Here
89
90
91
92
93
94
95
96
97
98
99
    config.setResumableCrawling(false);

    /*
     * Instantiate the controller for this crawl.
     */
    PageFetcher pageFetcher = new PageFetcher(config);
    RobotstxtConfig robotstxtConfig = new RobotstxtConfig();
    RobotstxtServer robotstxtServer = new RobotstxtServer(robotstxtConfig, pageFetcher);
    CrawlController controller = new CrawlController(config, pageFetcher, robotstxtServer);

    /*
 
View Full Code Here
59
60
61
62
63
64
65
66
67
68
69
70
    config2.setMaxPagesToFetch(100);

    /*
     * We will use different PageFetchers for the two crawlers.
     */
    PageFetcher pageFetcher1 = new PageFetcher(config1);
    PageFetcher pageFetcher2 = new PageFetcher(config2);

    /*
     * We will use the same RobotstxtServer for both of the crawlers.
     */
    RobotstxtConfig robotstxtConfig = new RobotstxtConfig();
View Full Code Here
56
57
58
59
60
61
62
63
64
65
66
     */
    config.setIncludeBinaryContentInCrawling(true);

    String[] crawlDomains = new String[] { "http://uci.edu/" };

    PageFetcher pageFetcher = new PageFetcher(config);
    RobotstxtConfig robotstxtConfig = new RobotstxtConfig();
    RobotstxtServer robotstxtServer = new RobotstxtServer(robotstxtConfig, pageFetcher);
    CrawlController controller = new CrawlController(config, pageFetcher, robotstxtServer);
    for (String domain : crawlDomains) {
      controller.addSeed(domain);
View Full Code Here
59
60
61
62
63
64
65
66
67
68
69
    config.setMaxPagesToFetch(-1);

    /*
     * Instantiate the controller for this crawl.
     */
    PageFetcher pageFetcher = new PageFetcher(config);
    RobotstxtConfig robotstxtConfig = new RobotstxtConfig();
    RobotstxtServer robotstxtServer = new RobotstxtServer(robotstxtConfig, pageFetcher);
    CrawlController controller = new CrawlController(config, pageFetcher, robotstxtServer);

    /*
 
View Full Code Here
38
39
40
41
42
43
44
  private PageFetcher pageFetcher;

  public Downloader() {
    CrawlConfig config = new CrawlConfig();
    parser = new Parser(config);
    pageFetcher = new PageFetcher(config);
  }
View Full Code Here
89
90
91
92
93
94
95
96
97
98
99
    config.setResumableCrawling(false);

    /*
     * Instantiate the controller for this crawl.
     */
    PageFetcher pageFetcher = new PageFetcher(config);
    RobotstxtConfig robotstxtConfig = new RobotstxtConfig();
    RobotstxtServer robotstxtServer = new RobotstxtServer(robotstxtConfig, pageFetcher);
    CrawlController controller = new CrawlController(config, pageFetcher, robotstxtServer);

    /*
 
View Full Code Here
TOP

Related Classes of edu.uci.ics.crawler4j.fetcher.PageFetcher

  • edu.uci.ics.crawler4j.examples.basic.BasicCrawlController
  • edu.uci.ics.crawler4j.examples.imagecrawler.ImageCrawlController
  • edu.uci.ics.crawler4j.examples.localdata.Downloader
  • edu.uci.ics.crawler4j.examples.localdata.LocalDataCollectorController
  • edu.uci.ics.crawler4j.examples.multiple.MultipleCrawlerController
  • edu.uci.ics.crawler4j.examples.shutdown.ControllerWithShutdown
  • edu.uci.ics.crawler4j.examples.statushandler.StatusHandlerCrawlController
  • org.apache.any23.plugin.crawler.SiteCrawler
  • org.apache.http.auth.AuthScope
  • org.apache.http.auth.UsernamePasswordCredentials
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.