return prefixes;
}
private Collection<SingularityS3Log> getS3Logs(Collection<String> prefixes) throws InterruptedException, ExecutionException, TimeoutException {
ListeningExecutorService es = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(Math.min(prefixes.size(), configuration.get().getMaxS3Threads()), new ThreadFactoryBuilder().setNameFormat("S3LogFetcher-%d").build()));
List<ListenableFuture<S3Object[]>> futures = Lists.newArrayListWithCapacity(prefixes.size());
for (final String s3Prefix : prefixes) {
futures.add(es.submit(new Callable<S3Object[]>() {
@Override
public S3Object[] call() throws Exception {
return s3.get().listObjects(configuration.get().getS3Bucket(), s3Prefix, null);
}
}));
}
final long start = System.currentTimeMillis();
List<S3Object[]> results = Futures.allAsList(futures).get(configuration.get().getWaitForS3ListSeconds(), TimeUnit.SECONDS);
List<S3Object> objects = Lists.newArrayListWithExpectedSize(results.size() * 2);
for (S3Object[] s3Objects : results) {
for (final S3Object s3Object : s3Objects) {
objects.add(s3Object);
}
}
LOG.trace("Got {} objects from S3 after {}", objects.size(), JavaUtils.duration(start));
List<ListenableFuture<SingularityS3Log>> logFutures = Lists.newArrayListWithCapacity(objects.size());
final Date expireAt = new Date(System.currentTimeMillis() + configuration.get().getExpireS3LinksAfterMillis());
for (final S3Object s3Object : objects) {
logFutures.add(es.submit(new Callable<SingularityS3Log>() {
@Override
public SingularityS3Log call() throws Exception {
String getUrl = s3.get().createSignedGetUrl(configuration.get().getS3Bucket(), s3Object.getKey(), expireAt);