Package org.apache.droids.norobots

Examples of org.apache.droids.norobots.NoRobotClient

94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
    } catch (URISyntaxException ex) {
      log.error("Unable to determine base URI for " + uri);
      return false;
    }
   
    NoRobotClient nrc = new NoRobotClient(contentLoader, userAgent);
    try {
      nrc.parse(baseURI);
    } catch (NoRobotException ex) {
      log.error("Failure parsing robots.txt: " + ex.getMessage());
      return false;
    }
    boolean test = nrc.isUrlAllowed(uri);
    if (log.isInfoEnabled()) {
      log.info(uri + " is " + (test ? "allowed" : "denied"));
    }
    return test;
  }
View Full Code Here
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
    } catch (URISyntaxException ex) {
      LOG.error("Unable to determine base URI for " + uri);
      return false;
    }
   
    NoRobotClient nrc = new NoRobotClient(contentLoader, userAgent);
    try {
      nrc.parse(baseURI);
    } catch (NoRobotException ex) {
      LOG.error("Failure parsing robots.txt: " + ex.getMessage());
      return false;
    }
    boolean test = nrc.isUrlAllowed(uri);
    if (LOG.isInfoEnabled()) {
      LOG.info(uri + " is " + (test ? "allowed" : "denied"));
    }
    return test;
  }
View Full Code Here
TOP

Related Classes of org.apache.droids.norobots.NoRobotClient

  • org.apache.droids.protocol.http.HttpProtocol
  • java.io.InputStream
  • java.io.InputStreamReader
  • java.io.BufferedReader
  • java.net.URI
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.