process.increment(FetchCounters.URLS_FAILED, 1);
status = new IOFetchException(item.getUrl(), new IOException(e));
} finally {
process.decrement(FetchCounters.URLS_FETCHING, 1);
Tuple tuple = result.getTuple();
tuple.add(status);
_fetchMgr.collect(tuple);
// Figure out how long it's been since the start of the request.
long fetchInterval = System.currentTimeMillis() - fetchStartTime;
// We want to avoid fetching faster than a max acceptable rate. Note that we always do
// this, even if there's not another page, so that this setting will have impact even
// if the next fetch set is ready right away.
if (fetchInterval < minPageFetchInterval) {
long delay = minPageFetchInterval - fetchInterval;
LOGGER.trace(String.format("FetchTask: sleeping for %dms", delay));
try {
Thread.sleep(delay);
} catch (InterruptedException e) {
LOGGER.warn("FetchTask interrupted!");
Thread.currentThread().interrupt();
continue;
}
}
}
}
// While we still have entries, we need to write them out to avoid losing them.
while (iter.hasNext()) {
ScoredUrlDatum item = iter.next();
FetchedDatum result = new FetchedDatum(item);
process.increment(FetchCounters.URLS_SKIPPED, 1);
AbortedFetchException status = new AbortedFetchException(item.getUrl(), AbortedFetchReason.INTERRUPTED);
Tuple tuple = result.getTuple();
tuple.add(status);
_fetchMgr.collect(tuple);
}
} catch (Throwable t) {
LOGGER.error("Exception while fetching", t);
} finally {