fetchlistQueue = new CloseableQueue<FetchList>(1); // max one fetchlists in the queue
injectedFetchlistQueue = new CloseableQueue<FetchList>(); //TODO: put a limit, a large injectdb causes an OutOfMemoryError.
fetchdataQueue = new CloseableQueue<FetchData>(1); //TODO: analyze if the fetchdata should be written to disk.
cycleFinishedMonitor = new Object();
stopMonitor = new StopMonitor("stop");
urlFilter = new UrlFilter();
if (config.getBoolean("clustering.enable")) {
int port = PortUtil.getPort("clustering.rpc.crawler");
nodeListener = new NodeListener(port, config);
MonitorModule.addModuleListener(nodeListener, new CrawlerMonitoredNode(this));