Examples of NoRobotClient

Examples of org.apache.droids.norobots.NoRobotClient

94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
    } catch (URISyntaxException ex) {
      log.error("Unable to determine base URI for " + uri);
      return false;
    }
   
    NoRobotClient nrc = new NoRobotClient(contentLoader, userAgent);
    try {
      nrc.parse(baseURI);
    } catch (NoRobotException ex) {
      log.error("Failure parsing robots.txt: " + ex.getMessage());
      return false;
    }
    boolean test = nrc.isUrlAllowed(uri);
    if (log.isInfoEnabled()) {
      log.info(uri + " is " + (test ? "allowed" : "denied"));
    }
    return test;
  }
View Full Code Here

Examples of org.apache.droids.norobots.NoRobotClient

94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
    } catch (URISyntaxException ex) {
      LOG.error("Unable to determine base URI for " + uri);
      return false;
    }
   
    NoRobotClient nrc = new NoRobotClient(contentLoader, userAgent);
    try {
      nrc.parse(baseURI);
    } catch (NoRobotException ex) {
      LOG.error("Failure parsing robots.txt: " + ex.getMessage());
      return false;
    }
    boolean test = nrc.isUrlAllowed(uri);
    if (LOG.isInfoEnabled()) {
      LOG.info(uri + " is " + (test ? "allowed" : "denied"));
    }
    return test;
  }
View Full Code Here

Examples of org.osjava.norbert.NoRobotClient

9
10
11
12
13
14
15
16
17
18
19
  protected NoRobotClient robot;

  public Robots(URL srcUrl, String robotsTxtContent, String userAgent, boolean wildcardsAllowed)
  {
    robot = new NoRobotClient(userAgent);
    robot.setWildcardsAllowed(wildcardsAllowed);
    try {
      if (srcUrl!=null && robotsTxtContent==null) {
        robot.parse(srcUrl);
      }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.