if (it.hasNext()) {
// when parallel then aggregate on the fly
final AtomicBoolean running = new AtomicBoolean(true);
final AtomicBoolean allTasksSubmitted = new AtomicBoolean();
final CountDownLatch aggregationOnTheFlyDone = new CountDownLatch(1);
final AtomicException executionException = new AtomicException();
// issue task to execute in separate thread so it can aggregate on-the-fly
// while we submit new tasks, and those tasks complete concurrently
// this allows us to optimize work and reduce memory consumption
final AggregateOnTheFlyTask aggregateOnTheFlyTask = new AggregateOnTheFlyTask(result, original, total, completion, running,
aggregationOnTheFlyDone, allTasksSubmitted, executionException);
final AtomicBoolean aggregationTaskSubmitted = new AtomicBoolean();
LOG.trace("Starting to submit parallel tasks");
while (it.hasNext()) {
final ProcessorExchangePair pair = it.next();
final Exchange subExchange = pair.getExchange();
updateNewExchange(subExchange, total.intValue(), pairs, it);
completion.submit(new Callable<Exchange>() {
public Exchange call() throws Exception {
// only start the aggregation task when the task is being executed to avoid staring
// the aggregation task to early and pile up too many threads
if (aggregationTaskSubmitted.compareAndSet(false, true)) {
// but only submit the task once
aggregateExecutorService.submit(aggregateOnTheFlyTask);
}
if (!running.get()) {
// do not start processing the task if we are not running
return subExchange;
}
try {
doProcessParallel(pair);
} catch (Throwable e) {
subExchange.setException(e);
}
// Decide whether to continue with the multicast or not; similar logic to the Pipeline
Integer number = getExchangeIndex(subExchange);
boolean continueProcessing = PipelineHelper.continueProcessing(subExchange, "Parallel processing failed for number " + number, LOG);
if (stopOnException && !continueProcessing) {
// signal to stop running
running.set(false);
// throw caused exception
if (subExchange.getException() != null) {
// wrap in exception to explain where it failed
CamelExchangeException cause = new CamelExchangeException("Parallel processing failed for number " + number, subExchange, subExchange.getException());
subExchange.setException(cause);
}
}
LOG.trace("Parallel processing complete for exchange: {}", subExchange);
return subExchange;
}
});
total.incrementAndGet();
}
// signal all tasks has been submitted
LOG.trace("Signaling that all {} tasks has been submitted.", total.get());
allTasksSubmitted.set(true);
// its to hard to do parallel async routing so we let the caller thread be synchronously
// and have it pickup the replies and do the aggregation (eg we use a latch to wait)
// wait for aggregation to be done
LOG.debug("Waiting for on-the-fly aggregation to complete aggregating {} responses for exchangeId: {}", total.get(), original.getExchangeId());
aggregationOnTheFlyDone.await();
// did we fail for whatever reason, if so throw that caused exception
if (executionException.get() != null) {
if (LOG.isDebugEnabled()) {
LOG.debug("Parallel processing failed due {}", executionException.get().getMessage());
}
throw executionException.get();
}
}
// no everything is okay so we are done
LOG.debug("Done parallel processing {} exchanges", total);