listSize = limit;
RequestConfig requestConfig = RequestConfig.custom()
.setSocketTimeout(3000)
.setConnectTimeout(3000).build();
CloseableHttpAsyncClient httpclient = HttpAsyncClients.custom()
.setDefaultRequestConfig(requestConfig)
.build();
try {
httpclient.start();
double requestIndex = (double)limit/100;
int period = (int)Math.ceil(requestIndex);
final HttpGet[] requests = new HttpGet[period];
for (int i = 0; i < period; i++) {
requests[i] = new HttpGet(ARTICLELIST_URL_STRING + "/start:" + i*100);
}
final CountDownLatch latch = new CountDownLatch(requests.length);
// stringBuilder.append("<ol>");
for (final HttpGet request: requests) {
httpclient.execute(request, new FutureCallback<HttpResponse>() {
public void completed(final HttpResponse response) {
BasicResponseHandler handler = new BasicResponseHandler();
try {
extract(handler.handleResponse(response));
} catch (Exception e) {
System.out.println("ERROR: skipping");
}
latch.countDown();
System.out.println(request.getRequestLine() + "->" + response.getStatusLine());
}
public void failed(final Exception ex) {
latch.countDown();
System.out.println(request.getRequestLine() + "->" + ex);
}
public void cancelled() {
latch.countDown();
System.out.println(request.getRequestLine() + "cancelled");
}
});
}
latch.await();
System.out.println("Shutting down");
} finally {
try {
httpclient.close();
} catch(Exception e) {}
}
System.out.println("Done");
// stringBuilder.append("</ol>");
ObjectMapper jsonOutputMapper = new ObjectMapper();