ThreadPoolExecutor executor = new ThreadPoolExecutor(config.minPoolSize(), config.maxPoolSize(), config
.keepAliveMilliseconds(), TimeUnit.MILLISECONDS, new DelayedBlockingQueue(config
.requestDelayMilliseconds()));
final ExecutorCounter counter = new ExecutorCounter();
try {
executor
.execute(new PageCrawlerExecutor(new Url(config.beginUrl(), 0), executor, counter, config
.downloader(), config.normalizer(), new DoesNotFollowVisitedUrlVisitor(config.beginUrl(),
visitor)));
while (counter.value() != 0) {
log.debug("executors that finished: " + executor.getCompletedTaskCount());
log.debug("Number of Executors alive: " + counter.value());
sleep();
}
} finally {
executor.shutdown();
}