LOGGER.fine("Adding extra url("+firstUrlString+") for prev cached rules("+urlString+")");
rulesCache.put(firstUrlString, rules);
}
} else {
//long start = System.currentTimeMillis();;
Resource resource = null;
try {
PerfStats.timeStart(PerfStat.RobotsFetchTotal);
if (LOGGER.isLoggable(Level.FINE)) {
LOGGER.fine("ROBOT: NotCached - Downloading("+urlString+")");
}
tmpRules = new RobotRules();
resource = webCache.getCachedResource(new URL(urlString),
maxCacheMS,true);
//long elapsed = System.currentTimeMillis() - start;
//PerformanceLogger.noteElapsed("RobotRequest", elapsed, urlString);
if(resource.getStatusCode() != 200) {
LOGGER.info("ROBOT: NotAvailable("+urlString+")");
throw new LiveDocumentNotAvailableException(urlString);
}
tmpRules.parse(resource);
rulesCache.put(firstUrlString,tmpRules);
rules = tmpRules;
if (LOGGER.isLoggable(Level.FINE)) {
LOGGER.fine("ROBOT: Downloaded("+urlString+")");
}
} catch (LiveDocumentNotAvailableException e) {
LOGGER.info("ROBOT: LiveDocumentNotAvailableException("+urlString+")");
} catch (MalformedURLException e) {
// e.printStackTrace();
LOGGER.warning("ROBOT: MalformedURLException("+urlString+")");
return null;
} catch (IOException e) {
LOGGER.warning("ROBOT: IOException("+urlString+"):"+e.getLocalizedMessage());
return null;
} catch (LiveWebCacheUnavailableException e) {
LOGGER.severe("ROBOT: LiveWebCacheUnavailableException("+urlString+")");
if (filterGroup != null) {
filterGroup.setLiveWebGone();
}
return null;
} catch (LiveWebTimeoutException e) {
LOGGER.severe("ROBOT: LiveDocumentTimedOutException("+urlString+")");
if (filterGroup != null) {
filterGroup.setRobotTimedOut();
}
return null;
} finally {
if (resource != null) {
try {
resource.close();
} catch (IOException e) {
}
resource = null;
}