public void run() {
if (!spider.isRunning()) {
return;
}
HttpClient httpClient = new HttpClient(ClientConfiguration.getDefault().getDownloadAttempts(),
spider.httpTimeout * 1000, spider.jxlDocument.getUserAgent(),
ClientConfiguration.getDefault().getAcceptCharset());
httpClient.setReferrer(referrer);
try {
spider.fireRetrievalStarted(uri);
HttpResponse cached = spider.retrieveFromCache(uri);
long ifModifiedSince = 0;
if (cached != null) {
ifModifiedSince = cached.getDate();
} else {
if (Spider.isOffline()) {
logger.warning("Could not retrieve " + uri + " from HTTP cache.");
return;
}
}
HttpResponse response;
if (Spider.isOffline()) {
response = cached;
logger.info(response.getStatusCode() + " " + response.getStatusMessage() + ": " + uri);
} else {
response = httpClient.doGet(uri, ifModifiedSince, spider.cookieStore);
logger.info(response.getStatusCode() + " " + response.getStatusMessage() + ": " + uri);
if (response.getStatusCode() == 304) {
response = cached;
}
}