webCrawler.setStopOnEmptyQueue(false);
return webCrawler;
}
private IHttpRequestEngine createRequestEngine(IScannerConfig config) {
final IHttpRequestEngineFactory factory = scanner.getHttpRequestEngineFactory();
final IHttpRequestEngineConfig requestEngineConfig = factory.createConfig();
if (config.getCookieList() != null && !config.getCookieList().isEmpty()) {
for (Cookie c: config.getCookieList()) {
cookieStore.addCookie(c);
}
}
if(config.getMaxRequestsPerSecond() > 0) {
requestEngineConfig.setRequestsPerMinute(config.getMaxRequestsPerSecond() * 60);
}
requestEngineConfig.setMaxConnections(config.getMaxConnections());
requestEngineConfig.setMaxConnectionsPerRoute(config.getMaxConnections());
requestEngineConfig.setMaximumResponseKilobytes(config.getMaxResponseKilobytes());
final IRequestOriginScanner requestOrigin = workspace.getRequestLog().getRequestOriginScanner(scanInstance);
final IHttpRequestEngine requestEngine = factory.createRequestEngine(IHttpRequestEngine.EngineConfigType.CONFIG_SCANNER, requestEngineConfig, requestOrigin);
HttpProtocolParams.setUserAgent(requestEngine.getHttpClient().getParams(), config.getUserAgent());
requestEngine.setCookieStore(cookieStore);
// REVISIT: consider moving authentication method to request engine config
IIdentity identity = config.getScanIdentity();
if (identity != null && identity.getAuthMethod() != null) {