ignoreUrlFieldsNoSessionId = this.src.getUrlIgnoreFieldsNoSessionId();
startingUrls = this.src.getStartingUrls();
try {
boolean wildcardsAllowed = ("1".equals(config.getProperty("/crawler/param[@name='robots_wildcard_allowed']", "0")));
robots = new Robots(new URL(startingUrls.getUrlHome()), null, getUserAgent (config.getProperty("/crawler/param[@name='user_agent']", "CaBot"), this.src.getUserAgent()), wildcardsAllowed);
} catch (MalformedURLException e) {
e.printStackTrace();
robots = null;
} catch (Exception e2) {
e2.printStackTrace();