protected NoRobotClient robot;
public Robots(URL srcUrl, String robotsTxtContent, String userAgent, boolean wildcardsAllowed)
{
robot = new NoRobotClient(userAgent);
robot.setWildcardsAllowed(wildcardsAllowed);
try {
if (srcUrl!=null && robotsTxtContent==null) {
robot.parse(srcUrl);
}