Package websphinx

Examples of websphinx.RobotExclusion

81
82
83
84
85
86
87
     */
    public IterativeHTMLCrawler(String url_list_file, String html_dump_directory, String userAgent) {
        this.url_list_file = url_list_file;
        this.html_dump_directory = html_dump_directory;

        robot = new RobotExclusion(userAgent);
    }
View Full Code Here
98
99
100
101
102
103
104
105
106
107
108
        log.debug("URI list file: " + this.url_list_file);

        this.html_dump_directory = ce.getHTDocsDumpDirResolved();
        log.debug("HTDocs Dump Dir: " + this.html_dump_directory);

        robot = new RobotExclusion(ce.getUserAgent());

        String robots_file = ce.getRobotsFileResolved();
        log.debug("Robots File: " + robots_file);
        String robots_domain = ce.getRobotsDomain();
        if (robots_file != null && robots_domain != null) {
View Full Code Here
78
79
80
81
82
83
84
     */
    public IterativeHTMLCrawler(String url_list_file, String html_dump_directory, String userAgent) {
        this.url_list_file = url_list_file;
        this.html_dump_directory = html_dump_directory;

        robot = new RobotExclusion(userAgent);
    }
View Full Code Here
95
96
97
98
99
100
101
102
103
104
105
        log.debug("URI list file: " + this.url_list_file);

        this.html_dump_directory = ce.getHTDocsDumpDirResolved();
        log.debug("HTDocs Dump Dir: " + this.html_dump_directory);

        robot = new RobotExclusion(ce.getUserAgent());

        String robots_file = ce.getRobotsFileResolved();
        log.debug("Robots File: " + robots_file);
        String robots_domain = ce.getRobotsDomain();
        if (robots_file != null && robots_domain != null) {
View Full Code Here
120
121
122
123
124
125
126
     * @param _userAgent User-agent for robots.txt
     */
    public IterativeHTMLCrawler(String _uriList, String _htdocsDumpDir, String _userAgent) {
        this.uriList = _uriList;
        this.htdocsDumpDir = _htdocsDumpDir;
        this.robot = new RobotExclusion(_userAgent);
    }
View Full Code Here
140
141
142
143
144
145
146
147
148
149
150
          System.err.println("Cannot load crawler configuration! " + e.toString());
        } catch (final IOException e) {
          System.err.println("Cannot load crawler configuration! " + e.toString());
        }

        this.robot = new RobotExclusion(this.userAgent);

        if (this.robotsFile != null && this.robotsDomain != null) {
            log.debug(this.robotsFile + " " + this.robotsDomain);
            this.robot.addLocalEntries(this.robotsDomain, new File(this.robotsFile));
        }
View Full Code Here
96
97
98
99
100
101
102
     */
    public IterativeHTMLCrawler(String url_list_file, String html_dump_directory, String userAgent) {
        this.url_list_file = url_list_file;
        this.html_dump_directory = html_dump_directory;

        robot = new RobotExclusion(userAgent);
    }
View Full Code Here
86
87
88
89
90
91
92
     */
    public IterativeHTMLCrawler(String url_list_file, String html_dump_directory, String userAgent) {
        this.url_list_file = url_list_file;
        this.html_dump_directory = html_dump_directory;

        robot = new RobotExclusion(userAgent);
    }
View Full Code Here
83
84
85
86
87
88
89
     */
    public IterativeHTMLCrawler(String url_list_file, String html_dump_directory, String userAgent) {
        this.url_list_file = url_list_file;
        this.html_dump_directory = html_dump_directory;

        robot = new RobotExclusion(userAgent);
    }
View Full Code Here
98
99
100
101
102
103
104
105
106
107
108

        this.url_list_file = ce.resolvePath(ce.getURIList());
        this.html_dump_directory = ce.resolvePath(ce.getHTDocsDumpDir());

        robot = new RobotExclusion(ce.getUserAgent());

        String robots_file = ce.getRobotsFile();
        String robots_domain = ce.getRobotsDomain();
        if (robots_file != null && robots_domain != null) {
            log.debug(robots_file + " " + robots_domain);
View Full Code Here
TOP

Related Classes of websphinx.RobotExclusion

  • org.apache.lenya.search.crawler.IterativeHTMLCrawler
  • java.io.PushbackInputStream
  • java.net.URLConnection
  • java.io.BufferedInputStream
  • java.net.URL
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.