/*
* JetS3t : Java S3 Toolkit
* Project hosted at http://bitbucket.org/jmurty/jets3t/
*
* Copyright 2006-2010 James Murty
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jets3t.service.multithread;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.net.URL;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.jets3t.service.Constants;
import org.jets3t.service.Jets3tProperties;
import org.jets3t.service.S3ObjectsChunk;
import org.jets3t.service.S3Service;
import org.jets3t.service.S3ServiceException;
import org.jets3t.service.ServiceException;
import org.jets3t.service.StorageObjectsChunk;
import org.jets3t.service.acl.AccessControlList;
import org.jets3t.service.io.BytesProgressWatcher;
import org.jets3t.service.io.InterruptableInputStream;
import org.jets3t.service.io.ProgressMonitoredInputStream;
import org.jets3t.service.io.TempFile;
import org.jets3t.service.model.S3Bucket;
import org.jets3t.service.model.S3Object;
import org.jets3t.service.model.S3Version;
import org.jets3t.service.security.ProviderCredentials;
import org.jets3t.service.utils.ServiceUtils;
import org.jets3t.service.utils.signedurl.SignedUrlAndObject;
import org.jets3t.service.utils.signedurl.SignedUrlHandler;
/**
* S3 service wrapper that performs multiple S3 requests at a time using multi-threading and an
* underlying thread-safe {@link S3Service} implementation.
* <p>
* This service is designed to be run in non-blocking threads that therefore communicates
* information about its progress by firing {@link ServiceEvent} events. It is the responsibility
* of applications using this service to correctly handle these events - see the JetS3t application
* {@link org.jets3t.apps.synchronize.Synchronize} for examples of how an application can use these
* events.
* </p>
* <p>
* For cases where the full power, and complexity, of the event notification mechanism is not required
* the simplified multi-threaded service {@link S3ServiceSimpleMulti} can be used.
* </p>
* <p>
* This class uses properties obtained through {@link Jets3tProperties}. For more information on
* these properties please refer to
* <a href="http://www.jets3t.org/toolkit/configuration.html">JetS3t Configuration</a>
* </p>
*
* @deprecated 0.8.0 use {@link org.jets3t.service.multi.ThreadedStorageService} instead.
*
* @author James Murty
*/
@Deprecated
public class S3ServiceMulti {
private static final Log log = LogFactory.getLog(S3ServiceMulti.class);
private S3Service s3Service = null;
private final boolean[] isShutdown = new boolean[] { false };
private final ArrayList<S3ServiceEventListener> serviceEventListeners =
new ArrayList<S3ServiceEventListener>();
private final long sleepTime;
/**
* Construct a multi-threaded service based on an S3Service and which sends event notifications
* to an event listening class. EVENT_IN_PROGRESS events are sent at the default time interval
* of 500ms.
*
* @param s3Service
* an S3Service implementation that will be used to perform S3 requests. This implementation
* <b>must</b> be thread-safe.
* @param listener
* the event listener which will handle event notifications.
*/
public S3ServiceMulti(S3Service s3Service, S3ServiceEventListener listener) {
this(s3Service, listener, 500);
}
/**
* Construct a multi-threaded service based on an S3Service and which sends event notifications
* to an event listening class, and which will send EVENT_IN_PROGRESS events at the specified
* time interval.
*
* @param s3Service
* an S3Service implementation that will be used to perform S3 requests. This implementation
* <b>must</b> be thread-safe.
* @param listener
* the event listener which will handle event notifications.
* @param threadSleepTimeMS
* how many milliseconds to wait before sending each EVENT_IN_PROGRESS notification event.
*/
public S3ServiceMulti(
S3Service s3Service, S3ServiceEventListener listener, long threadSleepTimeMS)
{
this.s3Service = s3Service;
addServiceEventListener(listener);
this.sleepTime = threadSleepTimeMS;
// Sanity-check the maximum thread and connection settings to ensure the maximum number
// of connections is at least equal to the largest of the maximum thread counts, and warn
// the use of potential problems.
int adminMaxThreadCount = this.s3Service.getJetS3tProperties()
.getIntProperty("s3service.admin-max-thread-count", 20);
int maxThreadCount = this.s3Service.getJetS3tProperties()
.getIntProperty("s3service.max-thread-count", 2);
int maxConnectionCount = this.s3Service.getJetS3tProperties()
.getIntProperty("httpclient.max-connections", 20);
if (maxConnectionCount < maxThreadCount) {
if (log.isWarnEnabled()) {
log.warn("Insufficient connections available (httpclient.max-connections="
+ maxConnectionCount + ") to run " + maxThreadCount
+ " simultaneous threads (s3service.max-thread-count) - please adjust JetS3t settings");
}
}
if (maxConnectionCount < adminMaxThreadCount) {
if (log.isWarnEnabled()) {
log.warn("Insufficient connections available (httpclient.max-connections="
+ maxConnectionCount + ") to run " + adminMaxThreadCount
+ " simultaneous admin threads (s3service.admin-max-thread-count) - please adjust JetS3t settings");
}
}
}
/**
* Make a best-possible effort to shutdown and clean up any resources used by this
* service such as HTTP connections, connection pools, threads etc. After calling
* this method the service instance will no longer be usable -- a new instance must
* be created to do more work.
*
* @throws S3ServiceException
*/
public void shutdown() throws S3ServiceException {
this.isShutdown[0] = true;
try {
this.getS3Service().shutdown();
} catch (ServiceException se) {
throw new S3ServiceException(se);
}
}
/**
* @return true if the {@link #shutdown()} method has been used to shut down and
* clean up this service. If this function returns true this service instance
* can no longer be used to do work.
*/
public boolean isShutdown() {
return this.isShutdown[0];
}
/**
* @return
* the underlying S3 service implementation.
*/
public S3Service getS3Service() {
return s3Service;
}
/**
* Adds a service event listener to the set of listeners that will be notified of events.
*
* @param listener
* an event listener to add to the event notification chain.
*/
public void addServiceEventListener(S3ServiceEventListener listener) {
if (listener != null) {
serviceEventListeners.add(listener);
}
}
/**
* Removes a service event listener from the set of listeners that will be notified of events.
*
* @param listener
* an event listener to remove from the event notification chain.
*/
public void removeServiceEventListener(S3ServiceEventListener listener) {
if (listener != null) {
serviceEventListeners.remove(listener);
}
}
/**
* Sends a service event to each of the listeners registered with this service.
* @param event
* the event to send to this service's registered event listeners.
*/
protected void fireServiceEvent(ServiceEvent event) {
if (serviceEventListeners.size() == 0) {
if (log.isWarnEnabled()) {
log.warn("S3ServiceMulti invoked without any S3ServiceEventListener objects, this is dangerous!");
}
}
Iterator<S3ServiceEventListener> listenerIter = serviceEventListeners.iterator();
while (listenerIter.hasNext()) {
S3ServiceEventListener listener = listenerIter.next();
if (event instanceof CreateObjectsEvent) {
listener.s3ServiceEventPerformed((CreateObjectsEvent) event);
} else if (event instanceof CopyObjectsEvent) {
listener.s3ServiceEventPerformed((CopyObjectsEvent) event);
} else if (event instanceof CreateBucketsEvent) {
listener.s3ServiceEventPerformed((CreateBucketsEvent) event);
} else if (event instanceof ListObjectsEvent) {
listener.s3ServiceEventPerformed((ListObjectsEvent) event);
} else if (event instanceof DeleteObjectsEvent) {
listener.s3ServiceEventPerformed((DeleteObjectsEvent) event);
} else if (event instanceof DeleteVersionedObjectsEvent) {
listener.s3ServiceEventPerformed((DeleteVersionedObjectsEvent) event);
} else if (event instanceof GetObjectsEvent) {
listener.s3ServiceEventPerformed((GetObjectsEvent) event);
} else if (event instanceof GetObjectHeadsEvent) {
listener.s3ServiceEventPerformed((GetObjectHeadsEvent) event);
} else if (event instanceof LookupACLEvent) {
listener.s3ServiceEventPerformed((LookupACLEvent) event);
} else if (event instanceof UpdateACLEvent) {
listener.s3ServiceEventPerformed((UpdateACLEvent) event);
} else if (event instanceof DownloadObjectsEvent) {
listener.s3ServiceEventPerformed((DownloadObjectsEvent) event);
} else {
throw new IllegalArgumentException("Listener not invoked for event class: " + event.getClass());
}
}
}
/**
* @return
* true if the underlying S3Service implementation is authenticated.
*/
public boolean isAuthenticatedConnection() {
return s3Service.isAuthenticatedConnection();
}
/**
* @return
* the AWS credentials in the underlying S3Service.
*/
public ProviderCredentials getAWSCredentials() {
return s3Service.getProviderCredentials();
}
/**
* Lists the objects in a bucket based on an array of prefix strings, and
* sends {@link ListObjectsEvent} notification events.
* The objects that match each prefix are listed in a separate background
* thread, potentially allowing you to list the contents of large buckets more
* quickly than if you had to list all the objects in sequence.
* <p>
* Objects in the bucket that do not match one of the prefixes will not be
* listed.
*
* @param bucketName
* the name of the bucket in which the objects are stored.
* @param prefixes
* an array of prefix strings. A separate listing thread will be run for
* each of these prefix strings, and the method will only complete once
* the entire object listing for each prefix has been obtained (unless the
* operation is cancelled, or an error occurs)
* @param delimiter
* an optional delimiter string to apply to each listing operation. This
* parameter should be null if you do not wish to apply a delimiter.
* @param maxListingLength
* the maximum number of objects to list in each iteration. This should be a
* value between 1 and 1000, where 1000 will be the best choice in almost all
* circumstances. Regardless of this value, all the objects in the bucket that
* match the criteria will be returned.
*
* <p>
* The maximum number of threads is controlled by the JetS3t configuration property
* <tt>s3service.admin-max-thread-count</tt>.
*
* @return
* true if all the threaded tasks completed successfully, false otherwise.
*/
public boolean listObjects(final String bucketName, final String[] prefixes,
final String delimiter, final long maxListingLength)
{
final Object uniqueOperationId = new Object(); // Special object used to identify this operation.
final boolean[] success = new boolean[] {true};
// Start all queries in the background.
ListObjectsRunnable[] runnables = new ListObjectsRunnable[prefixes.length];
for (int i = 0; i < runnables.length; i++) {
runnables[i] = new ListObjectsRunnable(bucketName, prefixes[i],
delimiter, maxListingLength, null);
}
// Wait for threads to finish, or be cancelled.
(new ThreadGroupManager(runnables, new ThreadWatcher(runnables.length),
this.s3Service.getJetS3tProperties(), true)
{
@Override
public void fireStartEvent(ThreadWatcher threadWatcher) {
fireServiceEvent(ListObjectsEvent.newStartedEvent(threadWatcher, uniqueOperationId));
}
@Override
public void fireProgressEvent(ThreadWatcher threadWatcher, List chunkList) {
fireServiceEvent(ListObjectsEvent.newInProgressEvent(threadWatcher, chunkList,
uniqueOperationId));
}
@Override
public void fireCancelEvent() {
success[0] = false;
fireServiceEvent(ListObjectsEvent.newCancelledEvent(uniqueOperationId));
}
@Override
public void fireCompletedEvent() {
fireServiceEvent(ListObjectsEvent.newCompletedEvent(uniqueOperationId));
}
@Override
public void fireErrorEvent(Throwable throwable) {
success[0] = false;
fireServiceEvent(ListObjectsEvent.newErrorEvent(throwable, uniqueOperationId));
}
@Override
public void fireIgnoredErrorsEvent(ThreadWatcher threadWatcher, Throwable[] ignoredErrors) {
success[0] = false;
fireServiceEvent(ListObjectsEvent.newIgnoredErrorsEvent(threadWatcher, ignoredErrors, uniqueOperationId));
}
}).run();
return success[0];
}
/**
* Creates multiple buckets, and sends {@link CreateBucketsEvent} notification events.
* <p>
* The maximum number of threads is controlled by the JetS3t configuration property
* <tt>s3service.admin-max-thread-count</tt>.
*
* @param buckets
* the buckets to create.
*
* @return
* true if all the threaded tasks completed successfully, false otherwise.
*/
public boolean createBuckets(final S3Bucket[] buckets) {
final List incompletedBucketList = new ArrayList();
final Object uniqueOperationId = new Object(); // Special object used to identify this operation.
final boolean[] success = new boolean[] {true};
// Start all queries in the background.
CreateBucketRunnable[] runnables = new CreateBucketRunnable[buckets.length];
for (int i = 0; i < runnables.length; i++) {
incompletedBucketList.add(buckets[i]);
runnables[i] = new CreateBucketRunnable(buckets[i]);
}
// Wait for threads to finish, or be cancelled.
(new ThreadGroupManager(runnables, new ThreadWatcher(runnables.length),
this.s3Service.getJetS3tProperties(), true)
{
@Override
public void fireStartEvent(ThreadWatcher threadWatcher) {
fireServiceEvent(CreateBucketsEvent.newStartedEvent(threadWatcher, uniqueOperationId));
}
@Override
public void fireProgressEvent(ThreadWatcher threadWatcher, List completedResults) {
incompletedBucketList.removeAll(completedResults);
S3Bucket[] completedBuckets = (S3Bucket[]) completedResults
.toArray(new S3Bucket[completedResults.size()]);
fireServiceEvent(CreateBucketsEvent.newInProgressEvent(threadWatcher, completedBuckets, uniqueOperationId));
}
@Override
public void fireCancelEvent() {
S3Bucket[] incompletedBuckets = (S3Bucket[]) incompletedBucketList
.toArray(new S3Bucket[incompletedBucketList.size()]);
success[0] = false;
fireServiceEvent(CreateBucketsEvent.newCancelledEvent(incompletedBuckets, uniqueOperationId));
}
@Override
public void fireCompletedEvent() {
fireServiceEvent(CreateBucketsEvent.newCompletedEvent(uniqueOperationId));
}
@Override
public void fireErrorEvent(Throwable throwable) {
success[0] = false;
fireServiceEvent(CreateBucketsEvent.newErrorEvent(throwable, uniqueOperationId));
}
@Override
public void fireIgnoredErrorsEvent(ThreadWatcher threadWatcher, Throwable[] ignoredErrors) {
success[0] = false;
fireServiceEvent(CreateBucketsEvent.newIgnoredErrorsEvent(threadWatcher, ignoredErrors, uniqueOperationId));
}
}).run();
return success[0];
}
/**
* Copies multiple objects within or between buckets, while sending
* {@link CopyObjectsEvent} notification events.
* <p>
* The maximum number of threads is controlled by the JetS3t configuration property
* <tt>s3service.admin-max-thread-count</tt>.
*
* @param sourceBucketName
* the name of the bucket containing the objects that will be copied.
* @param destinationBucketName
* the name of the bucket to which the objects will be copied. The destination
* bucket may be the same as the source bucket.
* @param sourceObjectKeys
* the key names of the objects that will be copied.
* @param destinationObjects
* objects that will be created by the copy operation. The AccessControlList
* setting of each object will determine the access permissions of the
* resultant object, and if the replaceMetadata flag is true the metadata
* items in each object will also be applied to the resultant object.
* @param replaceMetadata
* if true, the metadata items in the destination objects will be stored
* in S3 by using the REPLACE metadata copying option. If false, the metadata
* items will be copied unchanged from the original objects using the COPY
* metadata copying option.s
*
* @return
* true if all the threaded tasks completed successfully, false otherwise.
*/
public boolean copyObjects(final String sourceBucketName, final String destinationBucketName,
final String[] sourceObjectKeys, final S3Object[] destinationObjects, boolean replaceMetadata)
{
final List incompletedObjectsList = new ArrayList();
final Object uniqueOperationId = new Object(); // Special object used to identify this operation.
final boolean[] success = new boolean[] {true};
// Start all queries in the background.
CopyObjectRunnable[] runnables = new CopyObjectRunnable[sourceObjectKeys.length];
for (int i = 0; i < runnables.length; i++) {
incompletedObjectsList.add(destinationObjects[i]);
runnables[i] = new CopyObjectRunnable(sourceBucketName, destinationBucketName,
sourceObjectKeys[i], destinationObjects[i], replaceMetadata);
}
// Wait for threads to finish, or be cancelled.
(new ThreadGroupManager(runnables, new ThreadWatcher(runnables.length),
this.s3Service.getJetS3tProperties(), true)
{
@Override
public void fireStartEvent(ThreadWatcher threadWatcher) {
fireServiceEvent(CopyObjectsEvent.newStartedEvent(threadWatcher, uniqueOperationId));
}
@Override
public void fireProgressEvent(ThreadWatcher threadWatcher, List completedResults) {
incompletedObjectsList.removeAll(completedResults);
Map[] copyResults = (Map[]) completedResults
.toArray(new Map[completedResults.size()]);
fireServiceEvent(CopyObjectsEvent.newInProgressEvent(threadWatcher,
copyResults, uniqueOperationId));
}
@Override
public void fireCancelEvent() {
S3Object[] incompletedObjects = (S3Object[]) incompletedObjectsList
.toArray(new S3Object[incompletedObjectsList.size()]);
success[0] = false;
fireServiceEvent(CopyObjectsEvent.newCancelledEvent(incompletedObjects, uniqueOperationId));
}
@Override
public void fireCompletedEvent() {
fireServiceEvent(CopyObjectsEvent.newCompletedEvent(uniqueOperationId,
sourceObjectKeys, destinationObjects));
}
@Override
public void fireErrorEvent(Throwable throwable) {
success[0] = false;
fireServiceEvent(CopyObjectsEvent.newErrorEvent(throwable, uniqueOperationId));
}
@Override
public void fireIgnoredErrorsEvent(ThreadWatcher threadWatcher, Throwable[] ignoredErrors) {
success[0] = false;
fireServiceEvent(CopyObjectsEvent.newIgnoredErrorsEvent(threadWatcher, ignoredErrors, uniqueOperationId));
}
}).run();
return success[0];
}
/**
* Creates multiple objects in a bucket, and sends {@link CreateObjectsEvent} notification events.
* <p>
* The maximum number of threads is controlled by the JetS3t configuration property
* <tt>s3service.max-admin-thread-count</tt>.
*
* @param bucket
* the bucket to create the objects in
* @param objects
* the objects to create/upload.
*
* @return
* true if all the threaded tasks completed successfully, false otherwise.
*/
public boolean putObjects(final S3Bucket bucket, final S3Object[] objects) {
final List incompletedObjectsList = new ArrayList();
final List progressWatchers = new ArrayList();
final Object uniqueOperationId = new Object(); // Special object used to identify this operation.
final boolean[] success = new boolean[] {true};
// Start all queries in the background.
CreateObjectRunnable[] runnables = new CreateObjectRunnable[objects.length];
for (int i = 0; i < runnables.length; i++) {
incompletedObjectsList.add(objects[i]);
BytesProgressWatcher progressMonitor = new BytesProgressWatcher(objects[i].getContentLength());
runnables[i] = new CreateObjectRunnable(bucket, objects[i], progressMonitor);
progressWatchers.add(progressMonitor);
}
// Wait for threads to finish, or be cancelled.
ThreadWatcher threadWatcher = new ThreadWatcher(
(BytesProgressWatcher[]) progressWatchers.toArray(new BytesProgressWatcher[progressWatchers.size()]));
(new ThreadGroupManager(runnables, threadWatcher,
this.s3Service.getJetS3tProperties(), false)
{
@Override
public void fireStartEvent(ThreadWatcher threadWatcher) {
fireServiceEvent(CreateObjectsEvent.newStartedEvent(threadWatcher, uniqueOperationId));
}
@Override
public void fireProgressEvent(ThreadWatcher threadWatcher, List completedResults) {
incompletedObjectsList.removeAll(completedResults);
S3Object[] completedObjects = (S3Object[]) completedResults
.toArray(new S3Object[completedResults.size()]);
fireServiceEvent(CreateObjectsEvent.newInProgressEvent(threadWatcher,
completedObjects, uniqueOperationId));
}
@Override
public void fireCancelEvent() {
S3Object[] incompletedObjects = (S3Object[]) incompletedObjectsList
.toArray(new S3Object[incompletedObjectsList.size()]);
success[0] = false;
fireServiceEvent(CreateObjectsEvent.newCancelledEvent(incompletedObjects, uniqueOperationId));
}
@Override
public void fireCompletedEvent() {
fireServiceEvent(CreateObjectsEvent.newCompletedEvent(uniqueOperationId));
}
@Override
public void fireErrorEvent(Throwable throwable) {
success[0] = false;
fireServiceEvent(CreateObjectsEvent.newErrorEvent(throwable, uniqueOperationId));
}
@Override
public void fireIgnoredErrorsEvent(ThreadWatcher threadWatcher, Throwable[] ignoredErrors) {
success[0] = false;
fireServiceEvent(CreateObjectsEvent.newIgnoredErrorsEvent(threadWatcher, ignoredErrors, uniqueOperationId));
}
}).run();
return success[0];
}
/**
* Deletes multiple objects from a bucket, and sends {@link DeleteObjectsEvent} notification events.
* <p>
* The maximum number of threads is controlled by the JetS3t configuration property
* <tt>s3service.admin-max-thread-count</tt>.
*
* @param bucket
* the bucket containing the objects to be deleted
* @param objectKeys
* key names of objects to delete
*
* @return
* true if all the threaded tasks completed successfully, false otherwise.
*/
public boolean deleteObjects(final S3Bucket bucket, String[] objectKeys) {
S3Object objects[] = new S3Object[objectKeys.length];
for (int i = 0; i < objects.length; i++) {
objects[i] = new S3Object(objectKeys[i]);
}
return this.deleteObjects(bucket, objects);
}
/**
* Deletes multiple objects from a bucket, and sends {@link DeleteObjectsEvent} notification events.
* <p>
* The maximum number of threads is controlled by the JetS3t configuration property
* <tt>s3service.admin-max-thread-count</tt>.
*
* @param bucket
* the bucket containing the objects to be deleted
* @param objects
* the objects to delete
*
* @return
* true if all the threaded tasks completed successfully, false otherwise.
*/
public boolean deleteObjects(final S3Bucket bucket, final S3Object[] objects) {
final List objectsToDeleteList = new ArrayList();
final Object uniqueOperationId = new Object(); // Special object used to identify this operation.
final boolean[] success = new boolean[] {true};
// Start all queries in the background.
DeleteObjectRunnable[] runnables = new DeleteObjectRunnable[objects.length];
for (int i = 0; i < runnables.length; i++) {
objectsToDeleteList.add(objects[i]);
runnables[i] = new DeleteObjectRunnable(bucket, objects[i]);
}
// Wait for threads to finish, or be cancelled.
(new ThreadGroupManager(runnables, new ThreadWatcher(runnables.length),
this.s3Service.getJetS3tProperties(), true)
{
@Override
public void fireStartEvent(ThreadWatcher threadWatcher) {
fireServiceEvent(DeleteObjectsEvent.newStartedEvent(threadWatcher, uniqueOperationId));
}
@Override
public void fireProgressEvent(ThreadWatcher threadWatcher, List completedResults) {
objectsToDeleteList.removeAll(completedResults);
S3Object[] deletedObjects = (S3Object[]) completedResults
.toArray(new S3Object[completedResults.size()]);
fireServiceEvent(DeleteObjectsEvent.newInProgressEvent(threadWatcher, deletedObjects, uniqueOperationId));
}
@Override
public void fireCancelEvent() {
S3Object[] remainingObjects = (S3Object[]) objectsToDeleteList
.toArray(new S3Object[objectsToDeleteList.size()]);
success[0] = false;
fireServiceEvent(DeleteObjectsEvent.newCancelledEvent(remainingObjects, uniqueOperationId));
}
@Override
public void fireCompletedEvent() {
fireServiceEvent(DeleteObjectsEvent.newCompletedEvent(uniqueOperationId));
}
@Override
public void fireErrorEvent(Throwable throwable) {
success[0] = false;
fireServiceEvent(DeleteObjectsEvent.newErrorEvent(throwable, uniqueOperationId));
}
@Override
public void fireIgnoredErrorsEvent(ThreadWatcher threadWatcher, Throwable[] ignoredErrors) {
success[0] = false;
fireServiceEvent(DeleteObjectsEvent.newIgnoredErrorsEvent(threadWatcher, ignoredErrors, uniqueOperationId));
}
}).run();
return success[0];
}
/**
* Delete multiple object versions from a bucket in S3, and sends
* {@link DeleteVersionedObjectsEvent} notification events. This will delete only the specific
* version identified and will not affect any other Version or DeleteMarkers related to
* the object.
* <p>
* The maximum number of threads is controlled by the JetS3t configuration property
* <tt>s3service.admin-max-thread-count</tt>.
*
* @param versionIds
* the identifiers of the object versions that will be deleted.
* @param multiFactorSerialNumber
* the serial number for a multi-factor authentication device.
* @param multiFactorAuthCode
* a multi-factor authentication code generated by a device.
* @param bucketName
* the name of the versioned bucket containing the object to be deleted.
* @param objectKey
* the key representing the object in S3.
*
* @return
* true if all the threaded tasks completed successfully, false otherwise.
*/
public boolean deleteVersionsOfObjectWithMFA(final String[] versionIds,
String multiFactorSerialNumber, String multiFactorAuthCode,
String bucketName, String objectKey)
{
final List versionsToDeleteList = new ArrayList();
final Object uniqueOperationId = new Object(); // Special object used to identify this operation.
final boolean[] success = new boolean[] {true};
// Start all queries in the background.
DeleteVersionedObjectRunnable[] runnables =
new DeleteVersionedObjectRunnable[versionIds.length];
for (int i = 0; i < runnables.length; i++) {
versionsToDeleteList.add(new S3Version(objectKey, versionIds[i]));
runnables[i] = new DeleteVersionedObjectRunnable(
versionIds[i], multiFactorSerialNumber, multiFactorAuthCode,
bucketName, objectKey);
}
// Wait for threads to finish, or be cancelled.
(new ThreadGroupManager(runnables, new ThreadWatcher(runnables.length),
this.s3Service.getJetS3tProperties(), true)
{
@Override
public void fireStartEvent(ThreadWatcher threadWatcher) {
fireServiceEvent(DeleteVersionedObjectsEvent.newStartedEvent(
threadWatcher, uniqueOperationId));
}
@Override
public void fireProgressEvent(ThreadWatcher threadWatcher, List completedResults) {
versionsToDeleteList.removeAll(completedResults);
S3Version[] deletedVersions = (S3Version[]) completedResults
.toArray(new S3Version[completedResults.size()]);
fireServiceEvent(DeleteVersionedObjectsEvent.newInProgressEvent(
threadWatcher, deletedVersions, uniqueOperationId));
}
@Override
public void fireCancelEvent() {
S3Version[] remainingVersions = (S3Version[]) versionsToDeleteList
.toArray(new S3Version[versionsToDeleteList.size()]);
success[0] = false;
fireServiceEvent(DeleteVersionedObjectsEvent.newCancelledEvent(
remainingVersions, uniqueOperationId));
}
@Override
public void fireCompletedEvent() {
fireServiceEvent(DeleteVersionedObjectsEvent.newCompletedEvent(
uniqueOperationId));
}
@Override
public void fireErrorEvent(Throwable throwable) {
success[0] = false;
fireServiceEvent(DeleteVersionedObjectsEvent.newErrorEvent(
throwable, uniqueOperationId));
}
@Override
public void fireIgnoredErrorsEvent(ThreadWatcher threadWatcher,
Throwable[] ignoredErrors)
{
success[0] = false;
fireServiceEvent(DeleteVersionedObjectsEvent.newIgnoredErrorsEvent(
threadWatcher, ignoredErrors, uniqueOperationId));
}
}).run();
return success[0];
}
/**
* Delete multiple object versions from a bucket in S3, and sends
* {@link DeleteVersionedObjectsEvent} notification events. This will delete only the specific
* version identified and will not affect any other Version or DeleteMarkers related to
* the object.
* <p>
* The maximum number of threads is controlled by the JetS3t configuration property
* <tt>s3service.admin-max-thread-count</tt>.
*
* @param versionIds
* the identifiers of the object versions that will be deleted.
* @param bucketName
* the name of the versioned bucket containing the object to be deleted.
* @param objectKey
* the key representing the object in S3.
*
* @return
* true if all the threaded tasks completed successfully, false otherwise.
*/
public boolean deleteVersionsOfObject(final String[] versionIds,
String bucketName, String objectKey)
{
return deleteVersionsOfObjectWithMFA(versionIds, null, null, bucketName, objectKey);
}
/**
* Retrieves multiple objects (details and data) from a bucket, and sends
* {@link GetObjectsEvent} notification events.
*
* @param bucket
* the bucket containing the objects to retrieve.
* @param objects
* the objects to retrieve.
*
* @return
* true if all the threaded tasks completed successfully, false otherwise.
*/
public boolean getObjects(S3Bucket bucket, S3Object[] objects) {
String[] objectKeys = new String[objects.length];
for (int i = 0; i < objects.length; i++) {
objectKeys[i] = objects[i].getKey();
}
return getObjects(bucket, objectKeys);
}
/**
* Retrieves multiple objects (details and data) from a bucket, and sends
* {@link GetObjectsEvent} notification events.
* <p>
* The maximum number of threads is controlled by the JetS3t configuration property
* <tt>s3service.max-thread-count</tt>.
*
* @param bucket
* the bucket containing the objects to retrieve.
* @param objectKeys
* the key names of the objects to retrieve.
*
* @return
* true if all the threaded tasks completed successfully, false otherwise.
*/
public boolean getObjects(final S3Bucket bucket, final String[] objectKeys) {
final List pendingObjectKeysList = new ArrayList();
final Object uniqueOperationId = new Object(); // Special object used to identify this operation.
final boolean[] success = new boolean[] {true};
// Start all queries in the background.
GetObjectRunnable[] runnables = new GetObjectRunnable[objectKeys.length];
for (int i = 0; i < runnables.length; i++) {
pendingObjectKeysList.add(objectKeys[i]);
runnables[i] = new GetObjectRunnable(bucket, objectKeys[i], false);
}
// Wait for threads to finish, or be cancelled.
(new ThreadGroupManager(runnables, new ThreadWatcher(runnables.length),
this.s3Service.getJetS3tProperties(), false)
{
@Override
public void fireStartEvent(ThreadWatcher threadWatcher) {
fireServiceEvent(GetObjectsEvent.newStartedEvent(threadWatcher, uniqueOperationId));
}
@Override
public void fireProgressEvent(ThreadWatcher threadWatcher, List completedResults) {
S3Object[] completedObjects = (S3Object[]) completedResults
.toArray(new S3Object[completedResults.size()]);
for (int i = 0; i < completedObjects.length; i++) {
pendingObjectKeysList.remove(completedObjects[i].getKey());
}
fireServiceEvent(GetObjectsEvent.newInProgressEvent(threadWatcher, completedObjects, uniqueOperationId));
}
@Override
public void fireCancelEvent() {
List cancelledObjectsList = new ArrayList();
Iterator iter = pendingObjectKeysList.iterator();
while (iter.hasNext()) {
String key = (String) iter.next();
cancelledObjectsList.add(new S3Object(key));
}
S3Object[] cancelledObjects = (S3Object[]) cancelledObjectsList
.toArray(new S3Object[cancelledObjectsList.size()]);
success[0] = false;
fireServiceEvent(GetObjectsEvent.newCancelledEvent(cancelledObjects, uniqueOperationId));
}
@Override
public void fireCompletedEvent() {
fireServiceEvent(GetObjectsEvent.newCompletedEvent(uniqueOperationId));
}
@Override
public void fireErrorEvent(Throwable throwable) {
success[0] = false;
fireServiceEvent(GetObjectsEvent.newErrorEvent(throwable, uniqueOperationId));
}
@Override
public void fireIgnoredErrorsEvent(ThreadWatcher threadWatcher, Throwable[] ignoredErrors) {
success[0] = false;
fireServiceEvent(GetObjectsEvent.newIgnoredErrorsEvent(threadWatcher, ignoredErrors, uniqueOperationId));
}
}).run();
return success[0];
}
/**
* Retrieves details (but no data) about multiple objects from a bucket, and sends
* {@link GetObjectHeadsEvent} notification events.
*
* @param bucket
* the bucket containing the objects whose details will be retrieved.
* @param objects
* the objects with details to retrieve.
*
* @return
* true if all the threaded tasks completed successfully, false otherwise.
*/
public boolean getObjectsHeads(S3Bucket bucket, S3Object[] objects) {
String[] objectKeys = new String[objects.length];
for (int i = 0; i < objects.length; i++) {
objectKeys[i] = objects[i].getKey();
}
return getObjectsHeads(bucket, objectKeys);
}
/**
* Retrieves details (but no data) about multiple objects from a bucket, and sends
* {@link GetObjectHeadsEvent} notification events.
* <p>
* The maximum number of threads is controlled by the JetS3t configuration property
* <tt>s3service.admin-max-thread-count</tt>.
*
* @param bucket
* the bucket containing the objects whose details will be retrieved.
* @param objectKeys
* the key names of the objects with details to retrieve.
*
* @return
* true if all the threaded tasks completed successfully, false otherwise.
*/
public boolean getObjectsHeads(final S3Bucket bucket, final String[] objectKeys) {
final List pendingObjectKeysList = new ArrayList();
final Object uniqueOperationId = new Object(); // Special object used to identify this operation.
final boolean[] success = new boolean[] {true};
// Start all queries in the background.
GetObjectRunnable[] runnables = new GetObjectRunnable[objectKeys.length];
for (int i = 0; i < runnables.length; i++) {
pendingObjectKeysList.add(objectKeys[i]);
runnables[i] = new GetObjectRunnable(bucket, objectKeys[i], true);
}
// Wait for threads to finish, or be cancelled.
(new ThreadGroupManager(runnables, new ThreadWatcher(runnables.length),
this.s3Service.getJetS3tProperties(), true)
{
@Override
public void fireStartEvent(ThreadWatcher threadWatcher) {
fireServiceEvent(GetObjectHeadsEvent.newStartedEvent(threadWatcher, uniqueOperationId));
}
@Override
public void fireProgressEvent(ThreadWatcher threadWatcher, List completedResults) {
S3Object[] completedObjects = (S3Object[]) completedResults
.toArray(new S3Object[completedResults.size()]);
for (int i = 0; i < completedObjects.length; i++) {
pendingObjectKeysList.remove(completedObjects[i].getKey());
}
fireServiceEvent(GetObjectHeadsEvent.newInProgressEvent(threadWatcher, completedObjects, uniqueOperationId));
}
@Override
public void fireCancelEvent() {
List cancelledObjectsList = new ArrayList();
Iterator iter = pendingObjectKeysList.iterator();
while (iter.hasNext()) {
String key = (String) iter.next();
cancelledObjectsList.add(new S3Object(key));
}
S3Object[] cancelledObjects = (S3Object[]) cancelledObjectsList
.toArray(new S3Object[cancelledObjectsList.size()]);
success[0] = false;
fireServiceEvent(GetObjectHeadsEvent.newCancelledEvent(cancelledObjects, uniqueOperationId));
}
@Override
public void fireCompletedEvent() {
fireServiceEvent(GetObjectHeadsEvent.newCompletedEvent(uniqueOperationId));
}
@Override
public void fireErrorEvent(Throwable throwable) {
success[0] = false;
fireServiceEvent(GetObjectHeadsEvent.newErrorEvent(throwable, uniqueOperationId));
}
@Override
public void fireIgnoredErrorsEvent(ThreadWatcher threadWatcher, Throwable[] ignoredErrors) {
success[0] = false;
fireServiceEvent(GetObjectHeadsEvent.newIgnoredErrorsEvent(threadWatcher, ignoredErrors, uniqueOperationId));
}
}).run();
return success[0];
}
/**
* Retrieves Access Control List (ACL) information for multiple objects from a bucket, and sends
* {@link LookupACLEvent} notification events.
* <p>
* The maximum number of threads is controlled by the JetS3t configuration property
* <tt>s3service.admin-max-thread-count</tt>.
*
* @param bucket
* the bucket containing the objects
* @param objects
* the objects to retrieve ACL details for.
*
* @return
* true if all the threaded tasks completed successfully, false otherwise.
*/
public boolean getObjectACLs(final S3Bucket bucket, final S3Object[] objects) {
final List pendingObjectsList = new ArrayList();
final Object uniqueOperationId = new Object(); // Special object used to identify this operation.
final boolean[] success = new boolean[] {true};
// Start all queries in the background.
GetACLRunnable[] runnables = new GetACLRunnable[objects.length];
for (int i = 0; i < runnables.length; i++) {
pendingObjectsList.add(objects[i]);
runnables[i] = new GetACLRunnable(bucket, objects[i]);
}
// Wait for threads to finish, or be cancelled.
(new ThreadGroupManager(runnables, new ThreadWatcher(runnables.length),
this.s3Service.getJetS3tProperties(), true)
{
@Override
public void fireStartEvent(ThreadWatcher threadWatcher) {
fireServiceEvent(LookupACLEvent.newStartedEvent(threadWatcher, uniqueOperationId));
}
@Override
public void fireProgressEvent(ThreadWatcher threadWatcher, List completedResults) {
pendingObjectsList.removeAll(completedResults);
S3Object[] completedObjects = (S3Object[]) completedResults
.toArray(new S3Object[completedResults.size()]);
fireServiceEvent(LookupACLEvent.newInProgressEvent(threadWatcher, completedObjects, uniqueOperationId));
}
@Override
public void fireCancelEvent() {
S3Object[] cancelledObjects = (S3Object[]) pendingObjectsList
.toArray(new S3Object[pendingObjectsList.size()]);
success[0] = false;
fireServiceEvent(LookupACLEvent.newCancelledEvent(cancelledObjects, uniqueOperationId));
}
@Override
public void fireCompletedEvent() {
fireServiceEvent(LookupACLEvent.newCompletedEvent(uniqueOperationId));
}
@Override
public void fireErrorEvent(Throwable throwable) {
success[0] = false;
fireServiceEvent(LookupACLEvent.newErrorEvent(throwable, uniqueOperationId));
}
@Override
public void fireIgnoredErrorsEvent(ThreadWatcher threadWatcher, Throwable[] ignoredErrors) {
success[0] = false;
fireServiceEvent(LookupACLEvent.newIgnoredErrorsEvent(threadWatcher, ignoredErrors, uniqueOperationId));
}
}).run();
return success[0];
}
/**
* Updates/sets Access Control List (ACL) information for multiple objects in a bucket, and sends
* {@link UpdateACLEvent} notification events.
* <p>
* The maximum number of threads is controlled by the JetS3t configuration property
* <tt>s3service.admin-max-thread-count</tt>.
*
* @param bucket
* the bucket containing the objects
* @param objects
* the objects to update/set ACL details for.
*
* @return
* true if all the threaded tasks completed successfully, false otherwise.
*/
public boolean putACLs(final S3Bucket bucket, final S3Object[] objects) {
final List pendingObjectsList = new ArrayList();
final Object uniqueOperationId = new Object(); // Special object used to identify this operation.
final boolean[] success = new boolean[] {true};
// Start all queries in the background.
PutACLRunnable[] runnables = new PutACLRunnable[objects.length];
for (int i = 0; i < runnables.length; i++) {
pendingObjectsList.add(objects[i]);
runnables[i] = new PutACLRunnable(bucket, objects[i]);
}
// Wait for threads to finish, or be cancelled.
(new ThreadGroupManager(runnables, new ThreadWatcher(runnables.length),
this.s3Service.getJetS3tProperties(), true)
{
@Override
public void fireStartEvent(ThreadWatcher threadWatcher) {
fireServiceEvent(UpdateACLEvent.newStartedEvent(threadWatcher, uniqueOperationId));
}
@Override
public void fireProgressEvent(ThreadWatcher threadWatcher, List completedResults) {
pendingObjectsList.removeAll(completedResults);
S3Object[] completedObjects = (S3Object[]) completedResults
.toArray(new S3Object[completedResults.size()]);
fireServiceEvent(UpdateACLEvent.newInProgressEvent(threadWatcher, completedObjects, uniqueOperationId));
}
@Override
public void fireCancelEvent() {
S3Object[] cancelledObjects = (S3Object[]) pendingObjectsList
.toArray(new S3Object[pendingObjectsList.size()]);
success[0] = false;
fireServiceEvent(UpdateACLEvent.newCancelledEvent(cancelledObjects, uniqueOperationId));
}
@Override
public void fireCompletedEvent() {
fireServiceEvent(UpdateACLEvent.newCompletedEvent(uniqueOperationId));
}
@Override
public void fireErrorEvent(Throwable throwable) {
success[0] = false;
fireServiceEvent(UpdateACLEvent.newErrorEvent(throwable, uniqueOperationId));
}
@Override
public void fireIgnoredErrorsEvent(ThreadWatcher threadWatcher, Throwable[] ignoredErrors) {
success[0] = false;
fireServiceEvent(UpdateACLEvent.newIgnoredErrorsEvent(threadWatcher, ignoredErrors, uniqueOperationId));
}
}).run();
return success[0];
}
/**
* A convenience method to download multiple objects from S3 to pre-existing
* output streams, which is particularly useful for downloading objects to files.
* The S3 objects can be represented as S3Objects or as signed URLs in a
* {@link DownloadPackage} package. This method sends
* {@link DownloadObjectsEvent} notification events.
* <p>
* The maximum number of threads is controlled by the JetS3t configuration property
* <tt>s3service.max-thread-count</tt>.
* <p>
* If the JetS3t configuration property <tt>downloads.restoreLastModifiedDate</tt> is set
* to true, any files created by this method will have their last modified date set according
* to the value of the S3 object's {@link Constants#METADATA_JETS3T_LOCAL_FILE_DATE} metadata
* item.
*
* @param bucket
* the bucket containing the objects
* @param downloadPackages
* an array of download packages containing the object to be downloaded, and able to build
* an output stream where the object's contents will be written to.
*
* @return
* true if all the threaded tasks completed successfully, false otherwise.
* @throws S3ServiceException
*/
public boolean downloadObjects(final S3Bucket bucket,
final DownloadPackage[] downloadPackages) throws S3ServiceException
{
final List progressWatchers = new ArrayList();
final List incompleteObjectDownloadList = new ArrayList();
final Object uniqueOperationId = new Object(); // Special object used to identify this operation.
final boolean[] success = new boolean[] {true};
boolean restoreLastModifiedDate = this.s3Service.getJetS3tProperties()
.getBoolProperty("downloads.restoreLastModifiedDate", false);
// Start all queries in the background.
DownloadObjectRunnable[] runnables = new DownloadObjectRunnable[downloadPackages.length];
final S3Object[] objects = new S3Object[downloadPackages.length];
for (int i = 0; i < runnables.length; i++) {
if (downloadPackages[i].getObject() == null) {
// For signed URL downloads without corresponding object information, we create
// a surrogate S3Object containing nothing but the object's key name.
// This will allow the download to work, but total download size will not be known.
try {
URL url = new URL(downloadPackages[i].getSignedUrl());
objects[i] = ServiceUtils.buildObjectFromUrl(
url.getHost(), url.getPath(), s3Service.getEndpoint());
} catch (RuntimeException e) {
throw e;
} catch (Exception e) {
throw new S3ServiceException("Unable to determine S3 Object key name from signed URL: " +
downloadPackages[i].getSignedUrl());
}
} else {
objects[i] = downloadPackages[i].getObject();
}
BytesProgressWatcher progressMonitor = new BytesProgressWatcher(objects[i].getContentLength());
incompleteObjectDownloadList.add(objects[i]);
progressWatchers.add(progressMonitor);
if (downloadPackages[i].isSignedDownload()) {
runnables[i] = new DownloadObjectRunnable(
downloadPackages[i], progressMonitor, restoreLastModifiedDate);
} else {
runnables[i] = new DownloadObjectRunnable(bucket, objects[i].getKey(),
downloadPackages[i], progressMonitor, restoreLastModifiedDate);
}
}
// Wait for threads to finish, or be cancelled.
ThreadWatcher threadWatcher = new ThreadWatcher(
(BytesProgressWatcher[]) progressWatchers.toArray(new BytesProgressWatcher[progressWatchers.size()]));
(new ThreadGroupManager(runnables, threadWatcher,
this.s3Service.getJetS3tProperties(), false)
{
@Override
public void fireStartEvent(ThreadWatcher threadWatcher) {
fireServiceEvent(DownloadObjectsEvent.newStartedEvent(threadWatcher, uniqueOperationId));
}
@Override
public void fireProgressEvent(ThreadWatcher threadWatcher, List completedResults) {
incompleteObjectDownloadList.removeAll(completedResults);
S3Object[] completedObjects = (S3Object[]) completedResults
.toArray(new S3Object[completedResults.size()]);
fireServiceEvent(DownloadObjectsEvent.newInProgressEvent(threadWatcher, completedObjects, uniqueOperationId));
}
@Override
public void fireCancelEvent() {
S3Object[] incompleteObjects = (S3Object[]) incompleteObjectDownloadList
.toArray(new S3Object[incompleteObjectDownloadList.size()]);
success[0] = false;
fireServiceEvent(DownloadObjectsEvent.newCancelledEvent(incompleteObjects, uniqueOperationId));
}
@Override
public void fireCompletedEvent() {
fireServiceEvent(DownloadObjectsEvent.newCompletedEvent(uniqueOperationId));
}
@Override
public void fireErrorEvent(Throwable throwable) {
success[0] = false;
fireServiceEvent(DownloadObjectsEvent.newErrorEvent(throwable, uniqueOperationId));
}
@Override
public void fireIgnoredErrorsEvent(ThreadWatcher threadWatcher, Throwable[] ignoredErrors) {
success[0] = false;
fireServiceEvent(DownloadObjectsEvent.newIgnoredErrorsEvent(threadWatcher, ignoredErrors, uniqueOperationId));
}
}).run();
return success[0];
}
/**
* A convenience method to download multiple objects from S3 to pre-existing
* output streams, which is particularly useful for downloading objects to files.
* This method sends {@link DownloadObjectsEvent} notification events.
* <p>
* This method can only download S3 objects represented by {@link DownloadPackage}
* packages based on signed URL. To download objects when you don't have
* signed URLs, you must use the method
* {@link #downloadObjects(S3Bucket, DownloadPackage[])}
* <p>
* The maximum number of threads is controlled by the JetS3t configuration property
* <tt>s3service.max-thread-count</tt>.
* <p>
* If the JetS3t configuration property <tt>downloads.restoreLastModifiedDate</tt> is set
* to true, any files created by this method will have their last modified date set according
* to the value of the S3 object's {@link Constants#METADATA_JETS3T_LOCAL_FILE_DATE} metadata
* item.
*
* @param downloadPackages
* an array of download packages containing the object to be downloaded, represented
* with signed URL strings.
*
* @return
* true if all the threaded tasks completed successfully, false otherwise.
* @throws S3ServiceException
*/
public boolean downloadObjects(final DownloadPackage[] downloadPackages)
throws S3ServiceException
{
// Sanity check to ensure all packages are based on signed URLs
for (int i = 0; i < downloadPackages.length; i++) {
if (!downloadPackages[i].isSignedDownload()) {
throw new S3ServiceException(
"The downloadObjects(DownloadPackage[]) method may only be used with " +
"download packages based on signed URLs. Download package " + (i + 1) +
" of " + downloadPackages.length + " is not based on a signed URL");
}
}
return downloadObjects(null, downloadPackages);
}
/**
* Retrieves multiple objects (details and data) from a bucket using signed GET URLs corresponding
* to those objects.
* <p>
* Object retrieval using signed GET URLs can be performed without the underlying S3Service knowing
* the AWSCredentials for the target S3 account, however the underlying service must implement
* the {@link SignedUrlHandler} interface.
* <p>
* This method sends {@link GetObjectHeadsEvent} notification events.
* <p>
* The maximum number of threads is controlled by the JetS3t configuration property
* <tt>s3service.max-thread-count</tt>.
*
* @param signedGetURLs
* signed GET URL strings corresponding to the objects to be deleted.
*
* @throws IllegalStateException
* if the underlying S3Service does not implement {@link SignedUrlHandler}
*
* @return
* true if all the threaded tasks completed successfully, false otherwise.
*/
public boolean getObjects(final String[] signedGetURLs) throws MalformedURLException, UnsupportedEncodingException {
if (!(s3Service instanceof SignedUrlHandler)) {
throw new IllegalStateException("S3ServiceMutli's underlying S3Service must implement the"
+ "SignedUrlHandler interface to make the method getObjects(String[] signedGetURLs) available");
}
final List pendingObjectKeysList = new ArrayList();
final Object uniqueOperationId = new Object(); // Special object used to identify this operation.
final boolean[] success = new boolean[] {true};
// Start all queries in the background.
GetObjectRunnable[] runnables = new GetObjectRunnable[signedGetURLs.length];
for (int i = 0; i < runnables.length; i++) {
URL url = new URL(signedGetURLs[i]);
S3Object object = ServiceUtils.buildObjectFromUrl(
url.getHost(), url.getPath(), s3Service.getEndpoint());
pendingObjectKeysList.add(object);
runnables[i] = new GetObjectRunnable(signedGetURLs[i], false);
}
// Wait for threads to finish, or be cancelled.
(new ThreadGroupManager(runnables, new ThreadWatcher(runnables.length),
this.s3Service.getJetS3tProperties(), false)
{
@Override
public void fireStartEvent(ThreadWatcher threadWatcher) {
fireServiceEvent(GetObjectsEvent.newStartedEvent(threadWatcher, uniqueOperationId));
}
@Override
public void fireProgressEvent(ThreadWatcher threadWatcher, List completedResults) {
S3Object[] completedObjects = (S3Object[]) completedResults
.toArray(new S3Object[completedResults.size()]);
for (int i = 0; i < completedObjects.length; i++) {
pendingObjectKeysList.remove(completedObjects[i].getKey());
}
fireServiceEvent(GetObjectsEvent.newInProgressEvent(threadWatcher, completedObjects, uniqueOperationId));
}
@Override
public void fireCancelEvent() {
List cancelledObjectsList = new ArrayList();
Iterator iter = pendingObjectKeysList.iterator();
while (iter.hasNext()) {
String key = (String) iter.next();
cancelledObjectsList.add(new S3Object(key));
}
S3Object[] cancelledObjects = (S3Object[]) cancelledObjectsList
.toArray(new S3Object[cancelledObjectsList.size()]);
success[0] = false;
fireServiceEvent(GetObjectsEvent.newCancelledEvent(cancelledObjects, uniqueOperationId));
}
@Override
public void fireCompletedEvent() {
fireServiceEvent(GetObjectsEvent.newCompletedEvent(uniqueOperationId));
}
@Override
public void fireErrorEvent(Throwable throwable) {
success[0] = false;
fireServiceEvent(GetObjectsEvent.newErrorEvent(throwable, uniqueOperationId));
}
@Override
public void fireIgnoredErrorsEvent(ThreadWatcher threadWatcher, Throwable[] ignoredErrors) {
success[0] = false;
fireServiceEvent(GetObjectsEvent.newIgnoredErrorsEvent(threadWatcher, ignoredErrors, uniqueOperationId));
}
}).run();
return success[0];
}
/**
* Retrieves details (but no data) about multiple objects using signed HEAD URLs corresponding
* to those objects.
* <p>
* Detail retrieval using signed HEAD URLs can be performed without the underlying S3Service knowing
* the AWSCredentials for the target S3 account, however the underlying service must implement
* the {@link SignedUrlHandler} interface.
* <p>
* This method sends {@link GetObjectHeadsEvent} notification events.
* <p>
* The maximum number of threads is controlled by the JetS3t configuration property
* <tt>s3service.admin-max-thread-count</tt>.
*
* @param signedHeadURLs
* signed HEAD URL strings corresponding to the objects to be deleted.
*
* @throws IllegalStateException
* if the underlying S3Service does not implement {@link SignedUrlHandler}
*
* @return
* true if all the threaded tasks completed successfully, false otherwise.
*/
public boolean getObjectsHeads(final String[] signedHeadURLs) throws MalformedURLException, UnsupportedEncodingException {
if (!(s3Service instanceof SignedUrlHandler)) {
throw new IllegalStateException("S3ServiceMutli's underlying S3Service must implement the"
+ "SignedUrlHandler interface to make the method getObjectsHeads(String[] signedHeadURLs) available");
}
final List pendingObjectKeysList = new ArrayList();
final Object uniqueOperationId = new Object(); // Special object used to identify this operation.
final boolean[] success = new boolean[] {true};
// Start all queries in the background.
GetObjectRunnable[] runnables = new GetObjectRunnable[signedHeadURLs.length];
for (int i = 0; i < runnables.length; i++) {
URL url = new URL(signedHeadURLs[i]);
S3Object object = ServiceUtils.buildObjectFromUrl(
url.getHost(), url.getPath(), s3Service.getEndpoint());
pendingObjectKeysList.add(object);
runnables[i] = new GetObjectRunnable(signedHeadURLs[i], true);
}
// Wait for threads to finish, or be cancelled.
(new ThreadGroupManager(runnables, new ThreadWatcher(runnables.length),
this.s3Service.getJetS3tProperties(), true)
{
@Override
public void fireStartEvent(ThreadWatcher threadWatcher) {
fireServiceEvent(GetObjectHeadsEvent.newStartedEvent(threadWatcher, uniqueOperationId));
}
@Override
public void fireProgressEvent(ThreadWatcher threadWatcher, List completedResults) {
S3Object[] completedObjects = (S3Object[]) completedResults
.toArray(new S3Object[completedResults.size()]);
for (int i = 0; i < completedObjects.length; i++) {
pendingObjectKeysList.remove(completedObjects[i].getKey());
}
fireServiceEvent(GetObjectHeadsEvent.newInProgressEvent(threadWatcher, completedObjects, uniqueOperationId));
}
@Override
public void fireCancelEvent() {
List cancelledObjectsList = new ArrayList();
Iterator iter = pendingObjectKeysList.iterator();
while (iter.hasNext()) {
String key = (String) iter.next();
cancelledObjectsList.add(new S3Object(key));
}
S3Object[] cancelledObjects = (S3Object[]) cancelledObjectsList
.toArray(new S3Object[cancelledObjectsList.size()]);
success[0] = false;
fireServiceEvent(GetObjectHeadsEvent.newCancelledEvent(cancelledObjects, uniqueOperationId));
}
@Override
public void fireCompletedEvent() {
fireServiceEvent(GetObjectHeadsEvent.newCompletedEvent(uniqueOperationId));
}
@Override
public void fireErrorEvent(Throwable throwable) {
success[0] = false;
fireServiceEvent(GetObjectHeadsEvent.newErrorEvent(throwable, uniqueOperationId));
}
@Override
public void fireIgnoredErrorsEvent(ThreadWatcher threadWatcher, Throwable[] ignoredErrors) {
success[0] = false;
fireServiceEvent(GetObjectHeadsEvent.newIgnoredErrorsEvent(threadWatcher, ignoredErrors, uniqueOperationId));
}
}).run();
return success[0];
}
/**
* Updates/sets Access Control List (ACL) information for multiple objects in
* a bucket, and sends {@link UpdateACLEvent} notification events.
* The S3 objects are represented as signed URLs.
* <p>
* The maximum number of threads is controlled by the JetS3t configuration property
* <tt>s3service.admin-max-thread-count</tt>.
*
* @param signedURLs
* URL strings that are authenticated and signed to allow a PUT request to
* be performed for the referenced object.
* @param acl
* the access control list settings to apply to the objects.
*
* @return
* true if all the threaded tasks completed successfully, false otherwise.
*/
public boolean putObjectsACLs(final String[] signedURLs, final AccessControlList acl)
throws MalformedURLException, UnsupportedEncodingException
{
final List pendingObjectsList = new ArrayList();
final Object uniqueOperationId = new Object(); // Special object used to identify this operation.
final boolean[] success = new boolean[] {true};
// Start all queries in the background.
PutACLRunnable[] runnables = new PutACLRunnable[signedURLs.length];
for (int i = 0; i < runnables.length; i++) {
URL url = new URL(signedURLs[i]);
S3Object object = ServiceUtils.buildObjectFromUrl(
url.getHost(), url.getPath(), s3Service.getEndpoint());
pendingObjectsList.add(object);
runnables[i] = new PutACLRunnable(signedURLs[i], acl);
}
// Wait for threads to finish, or be cancelled.
(new ThreadGroupManager(runnables, new ThreadWatcher(runnables.length),
this.s3Service.getJetS3tProperties(), true)
{
@Override
public void fireStartEvent(ThreadWatcher threadWatcher) {
fireServiceEvent(UpdateACLEvent.newStartedEvent(threadWatcher, uniqueOperationId));
}
@Override
public void fireProgressEvent(ThreadWatcher threadWatcher, List completedResults) {
pendingObjectsList.removeAll(completedResults);
S3Object[] completedObjects = (S3Object[]) completedResults
.toArray(new S3Object[completedResults.size()]);
fireServiceEvent(UpdateACLEvent.newInProgressEvent(threadWatcher, completedObjects, uniqueOperationId));
}
@Override
public void fireCancelEvent() {
S3Object[] cancelledObjects = (S3Object[]) pendingObjectsList
.toArray(new S3Object[pendingObjectsList.size()]);
success[0] = false;
fireServiceEvent(UpdateACLEvent.newCancelledEvent(cancelledObjects, uniqueOperationId));
}
@Override
public void fireCompletedEvent() {
fireServiceEvent(UpdateACLEvent.newCompletedEvent(uniqueOperationId));
}
@Override
public void fireErrorEvent(Throwable throwable) {
success[0] = false;
fireServiceEvent(UpdateACLEvent.newErrorEvent(throwable, uniqueOperationId));
}
@Override
public void fireIgnoredErrorsEvent(ThreadWatcher threadWatcher, Throwable[] ignoredErrors) {
success[0] = false;
fireServiceEvent(UpdateACLEvent.newIgnoredErrorsEvent(threadWatcher, ignoredErrors, uniqueOperationId));
}
}).run();
return success[0];
}
/**
* Deletes multiple objects from a bucket using signed DELETE URLs corresponding to those objects.
* <p>
* Deletes using signed DELETE URLs can be performed without the underlying S3Service knowing
* the AWSCredentials for the target S3 account, however the underlying service must implement
* the {@link SignedUrlHandler} interface.
* <p>
* This method sends {@link DeleteObjectsEvent} notification events.
* <p>
* The maximum number of threads is controlled by the JetS3t configuration property
* <tt>s3service.admin-max-thread-count</tt>.
*
* @param signedDeleteUrls
* signed DELETE URL strings corresponding to the objects to be deleted.
*
* @throws IllegalStateException
* if the underlying S3Service does not implement {@link SignedUrlHandler}
*
* @return
* true if all the threaded tasks completed successfully, false otherwise.
*/
public boolean deleteObjects(final String[] signedDeleteUrls) throws MalformedURLException, UnsupportedEncodingException {
if (!(s3Service instanceof SignedUrlHandler)) {
throw new IllegalStateException("S3ServiceMutli's underlying S3Service must implement the"
+ "SignedUrlHandler interface to make the method deleteObjects(String[] signedDeleteURLs) available");
}
final List objectsToDeleteList = new ArrayList();
final Object uniqueOperationId = new Object(); // Special object used to identify this operation.
final boolean[] success = new boolean[] {true};
// Start all queries in the background.
DeleteObjectRunnable[] runnables = new DeleteObjectRunnable[signedDeleteUrls.length];
for (int i = 0; i < runnables.length; i++) {
URL url = new URL(signedDeleteUrls[i]);
S3Object object = ServiceUtils.buildObjectFromUrl(
url.getHost(), url.getPath(), s3Service.getEndpoint());
objectsToDeleteList.add(object);
runnables[i] = new DeleteObjectRunnable(signedDeleteUrls[i]);
}
// Wait for threads to finish, or be cancelled.
(new ThreadGroupManager(runnables, new ThreadWatcher(runnables.length),
this.s3Service.getJetS3tProperties(), true)
{
@Override
public void fireStartEvent(ThreadWatcher threadWatcher) {
fireServiceEvent(DeleteObjectsEvent.newStartedEvent(threadWatcher, uniqueOperationId));
}
@Override
public void fireProgressEvent(ThreadWatcher threadWatcher, List completedResults) {
objectsToDeleteList.removeAll(completedResults);
S3Object[] deletedObjects = (S3Object[]) completedResults
.toArray(new S3Object[completedResults.size()]);
fireServiceEvent(DeleteObjectsEvent.newInProgressEvent(threadWatcher, deletedObjects, uniqueOperationId));
}
@Override
public void fireCancelEvent() {
S3Object[] remainingObjects = (S3Object[]) objectsToDeleteList
.toArray(new S3Object[objectsToDeleteList.size()]);
success[0] = false;
fireServiceEvent(DeleteObjectsEvent.newCancelledEvent(remainingObjects, uniqueOperationId));
}
@Override
public void fireCompletedEvent() {
fireServiceEvent(DeleteObjectsEvent.newCompletedEvent(uniqueOperationId));
}
@Override
public void fireErrorEvent(Throwable throwable) {
success[0] = false;
fireServiceEvent(DeleteObjectsEvent.newErrorEvent(throwable, uniqueOperationId));
}
@Override
public void fireIgnoredErrorsEvent(ThreadWatcher threadWatcher, Throwable[] ignoredErrors) {
success[0] = false;
fireServiceEvent(DeleteObjectsEvent.newIgnoredErrorsEvent(threadWatcher, ignoredErrors, uniqueOperationId));
}
}).run();
return success[0];
}
/**
* Creates multiple objects in a bucket using a pre-signed PUT URL for each object.
* <p>
* Uploads using signed PUT URLs can be performed without the underlying S3Service knowing
* the AWSCredentials for the target S3 account, however the underlying service must implement
* the {@link SignedUrlHandler} interface.
* <p>
* This method sends {@link CreateObjectsEvent} notification events.
* <p>
* The maximum number of threads is controlled by the JetS3t configuration property
* <tt>s3service.max-thread-count</tt>.
*
* @param signedPutUrlAndObjects
* packages containing the S3Object to upload and the corresponding signed PUT URL.
*
* @throws IllegalStateException
* if the underlying S3Service does not implement {@link SignedUrlHandler}
*
* @return
* true if all the threaded tasks completed successfully, false otherwise.
*/
public boolean putObjects(final SignedUrlAndObject[] signedPutUrlAndObjects) {
if (!(s3Service instanceof SignedUrlHandler)) {
throw new IllegalStateException("S3ServiceMutli's underlying S3Service must implement the"
+ "SignedUrlHandler interface to make the method putObjects(SignedUrlAndObject[] signedPutUrlAndObjects) available");
}
final List progressWatchers = new ArrayList();
final List incompletedObjectsList = new ArrayList();
final Object uniqueOperationId = new Object(); // Special object used to identify this operation.
final boolean[] success = new boolean[] {true};
// Calculate total byte count being transferred.
S3Object objects[] = new S3Object[signedPutUrlAndObjects.length];
for (int i = 0; i < signedPutUrlAndObjects.length; i++) {
objects[i] = signedPutUrlAndObjects[i].getObject();
}
// Start all queries in the background.
SignedPutRunnable[] runnables = new SignedPutRunnable[signedPutUrlAndObjects.length];
for (int i = 0; i < runnables.length; i++) {
BytesProgressWatcher progressMonitor = new BytesProgressWatcher(objects[i].getContentLength());
progressWatchers.add(progressMonitor);
incompletedObjectsList.add(signedPutUrlAndObjects[i].getObject());
runnables[i] = new SignedPutRunnable(signedPutUrlAndObjects[i], progressMonitor);
}
// Wait for threads to finish, or be cancelled.
ThreadWatcher threadWatcher = new ThreadWatcher(
(BytesProgressWatcher[]) progressWatchers.toArray(new BytesProgressWatcher[progressWatchers.size()]));
(new ThreadGroupManager(runnables, threadWatcher,
this.s3Service.getJetS3tProperties(), false)
{
@Override
public void fireStartEvent(ThreadWatcher threadWatcher) {
fireServiceEvent(CreateObjectsEvent.newStartedEvent(threadWatcher, uniqueOperationId));
}
@Override
public void fireProgressEvent(ThreadWatcher threadWatcher, List completedResults) {
incompletedObjectsList.removeAll(completedResults);
S3Object[] completedObjects = (S3Object[]) completedResults
.toArray(new S3Object[completedResults.size()]);
fireServiceEvent(CreateObjectsEvent.newInProgressEvent(threadWatcher, completedObjects, uniqueOperationId));
}
@Override
public void fireCancelEvent() {
S3Object[] incompletedObjects = (S3Object[]) incompletedObjectsList
.toArray(new S3Object[incompletedObjectsList.size()]);
success[0] = false;
fireServiceEvent(CreateObjectsEvent.newCancelledEvent(incompletedObjects, uniqueOperationId));
}
@Override
public void fireCompletedEvent() {
fireServiceEvent(CreateObjectsEvent.newCompletedEvent(uniqueOperationId));
}
@Override
public void fireErrorEvent(Throwable throwable) {
success[0] = false;
fireServiceEvent(CreateObjectsEvent.newErrorEvent(throwable, uniqueOperationId));
}
@Override
public void fireIgnoredErrorsEvent(ThreadWatcher threadWatcher, Throwable[] ignoredErrors) {
success[0] = false;
fireServiceEvent(CreateObjectsEvent.newIgnoredErrorsEvent(threadWatcher, ignoredErrors, uniqueOperationId));
}
}).run();
return success[0];
}
/**
* Retrieves ACL information about multiple objects from a bucket using signed GET ACL URLs
* corresponding to those objects.
* The S3 objects are represented as signed URLs.
* <p>
* Object retrieval using signed GET URLs can be performed without the underlying S3Service knowing
* the AWSCredentials for the target S3 account, however the underlying service must implement
* the {@link SignedUrlHandler} interface.
* <p>
* This method sends {@link LookupACLEvent} notification events.
* <p>
* The maximum number of threads is controlled by the JetS3t configuration property
* <tt>s3service.max-thread-count</tt>.
*
* @param signedAclURLs
* signed GET URL strings corresponding to the objects to be queried.
*
* @throws IllegalStateException
* if the underlying S3Service does not implement {@link SignedUrlHandler}
*
* @return
* true if all the threaded tasks completed successfully, false otherwise.
*/
public boolean getObjectsACLs(final String[] signedAclURLs) throws MalformedURLException, UnsupportedEncodingException {
if (!(s3Service instanceof SignedUrlHandler)) {
throw new IllegalStateException("S3ServiceMutli's underlying S3Service must implement the"
+ "SignedUrlHandler interface to make the method getObjects(String[] signedGetURLs) available");
}
final List pendingObjectKeysList = new ArrayList();
final Object uniqueOperationId = new Object(); // Special object used to identify this operation.
final boolean[] success = new boolean[] {true};
// Start all queries in the background.
GetACLRunnable[] runnables = new GetACLRunnable[signedAclURLs.length];
for (int i = 0; i < runnables.length; i++) {
URL url = new URL(signedAclURLs[i]);
S3Object object = ServiceUtils.buildObjectFromUrl(
url.getHost(), url.getPath(),s3Service.getEndpoint());
pendingObjectKeysList.add(object);
runnables[i] = new GetACLRunnable(signedAclURLs[i]);
}
// Wait for threads to finish, or be cancelled.
(new ThreadGroupManager(runnables, new ThreadWatcher(runnables.length),
this.s3Service.getJetS3tProperties(), false)
{
@Override
public void fireStartEvent(ThreadWatcher threadWatcher) {
fireServiceEvent(LookupACLEvent.newStartedEvent(threadWatcher, uniqueOperationId));
}
@Override
public void fireProgressEvent(ThreadWatcher threadWatcher, List completedResults) {
S3Object[] completedObjects = (S3Object[]) completedResults
.toArray(new S3Object[completedResults.size()]);
for (int i = 0; i < completedObjects.length; i++) {
pendingObjectKeysList.remove(completedObjects[i].getKey());
}
fireServiceEvent(LookupACLEvent.newInProgressEvent(threadWatcher, completedObjects, uniqueOperationId));
}
@Override
public void fireCancelEvent() {
List cancelledObjectsList = new ArrayList();
Iterator iter = pendingObjectKeysList.iterator();
while (iter.hasNext()) {
cancelledObjectsList.add(iter.next());
}
S3Object[] cancelledObjects = (S3Object[]) cancelledObjectsList
.toArray(new S3Object[cancelledObjectsList.size()]);
success[0] = false;
fireServiceEvent(LookupACLEvent.newCancelledEvent(cancelledObjects, uniqueOperationId));
}
@Override
public void fireCompletedEvent() {
fireServiceEvent(LookupACLEvent.newCompletedEvent(uniqueOperationId));
}
@Override
public void fireErrorEvent(Throwable throwable) {
success[0] = false;
fireServiceEvent(LookupACLEvent.newErrorEvent(throwable, uniqueOperationId));
}
@Override
public void fireIgnoredErrorsEvent(ThreadWatcher threadWatcher, Throwable[] ignoredErrors) {
success[0] = false;
fireServiceEvent(LookupACLEvent.newIgnoredErrorsEvent(threadWatcher, ignoredErrors, uniqueOperationId));
}
}).run();
return success[0];
}
///////////////////////////////////////////////
// Private classes used by the methods above //
///////////////////////////////////////////////
/**
* All the operation threads used by this service extend this class, which provides common
* methods used to retrieve the result object from a completed thread (via {@link #getResult()}
* or force a thread to be interrupted (via {@link #forceInterrupt}.
*/
private abstract class AbstractRunnable implements Runnable {
public abstract Object getResult();
public abstract void forceInterruptCalled();
protected void forceInterrupt() {
forceInterruptCalled();
}
}
/**
* Thread for performing the update/set of Access Control List information for an object.
*/
private class PutACLRunnable extends AbstractRunnable {
private S3Bucket bucket = null;
private S3Object s3Object = null;
private String signedUrl = null;
private AccessControlList signedUrlAcl = null;
private Object result = null;
public PutACLRunnable(S3Bucket bucket, S3Object s3Object) {
this.bucket = bucket;
this.s3Object = s3Object;
}
public PutACLRunnable(String signedAclUrl, AccessControlList signedUrlAcl) {
this.signedUrl = signedAclUrl;
this.signedUrlAcl = signedUrlAcl;
this.bucket = null;
this.s3Object = null;
}
public void run() {
try {
if (signedUrl == null) {
if (s3Object == null) {
s3Service.putBucketAcl(bucket);
} else {
s3Service.putObjectAcl(bucket, s3Object);
}
result = s3Object;
} else {
SignedUrlHandler handler = s3Service;
handler.putObjectAclWithSignedUrl(signedUrl, signedUrlAcl);
URL url = new URL(signedUrl);
S3Object object = ServiceUtils.buildObjectFromUrl(
url.getHost(), url.getPath(), s3Service.getEndpoint());
object.setAcl(signedUrlAcl);
result = object;
}
} catch (RuntimeException e) {
result = e;
throw e;
} catch (Exception e) {
result = e;
}
}
@Override
public Object getResult() {
return result;
}
@Override
public void forceInterruptCalled() {
// This is an atomic operation, cannot interrupt. Ignore.
}
}
/**
* Thread for retrieving Access Control List information for an object.
*/
private class GetACLRunnable extends AbstractRunnable {
private S3Bucket bucket = null;
private S3Object object = null;
private String signedAclUrl = null;
private Object result = null;
public GetACLRunnable(S3Bucket bucket, S3Object object) {
this.bucket = bucket;
this.object = object;
}
public GetACLRunnable(String signedAclUrl) {
this.signedAclUrl = signedAclUrl;
this.bucket = null;
this.object = null;
}
public void run() {
try {
if (signedAclUrl == null) {
AccessControlList acl = s3Service.getObjectAcl(bucket, object.getKey());
object.setAcl(acl);
result = object;
} else {
SignedUrlHandler handler = s3Service;
AccessControlList acl = handler.getObjectAclWithSignedUrl(signedAclUrl);
URL url = new URL(signedAclUrl);
object = ServiceUtils.buildObjectFromUrl(
url.getHost(), url.getPath(), s3Service.getEndpoint());
object.setAcl(acl);
result = object;
}
} catch (RuntimeException e) {
result = e;
throw e;
} catch (Exception e) {
result = e;
}
}
@Override
public Object getResult() {
return result;
}
@Override
public void forceInterruptCalled() {
// This is an atomic operation, cannot interrupt. Ignore.
}
}
/**
* Thread for deleting an object.
*/
private class DeleteObjectRunnable extends AbstractRunnable {
private S3Bucket bucket = null;
private S3Object object = null;
private String signedDeleteUrl = null;
private Object result = null;
public DeleteObjectRunnable(S3Bucket bucket, S3Object object) {
this.signedDeleteUrl = null;
this.bucket = bucket;
this.object = object;
}
public DeleteObjectRunnable(String signedDeleteUrl) {
this.signedDeleteUrl = signedDeleteUrl;
this.bucket = null;
this.object = null;
}
public void run() {
try {
if (signedDeleteUrl == null) {
s3Service.deleteObject(bucket, object.getKey());
result = object;
} else {
SignedUrlHandler handler = s3Service;
handler.deleteObjectWithSignedUrl(signedDeleteUrl);
URL url = new URL(signedDeleteUrl);
result = ServiceUtils.buildObjectFromUrl(
url.getHost(), url.getPath(), s3Service.getEndpoint());
}
} catch (RuntimeException e) {
result = e;
throw e;
} catch (Exception e) {
result = e;
}
}
@Override
public Object getResult() {
return result;
}
@Override
public void forceInterruptCalled() {
// This is an atomic operation, cannot interrupt. Ignore.
}
}
/**
* Thread for deleting a versioned object.
*/
private class DeleteVersionedObjectRunnable extends AbstractRunnable {
private String versionId = null;
private String multiFactorSerialNumber = null;
private String multiFactorAuthCode = null;
private String bucketName = null;
private String objectKey = null;
private Object result = null;
public DeleteVersionedObjectRunnable(String versionId,
String multiFactorSerialNumber, String multiFactorAuthCode,
String bucketName, String objectKey)
{
this.versionId = versionId;
this.multiFactorSerialNumber = multiFactorSerialNumber;
this.multiFactorAuthCode = multiFactorAuthCode;
this.bucketName = bucketName;
this.objectKey = objectKey;
}
public void run() {
try {
s3Service.deleteVersionedObjectWithMFA(versionId,
multiFactorSerialNumber, multiFactorAuthCode,
bucketName, objectKey);
result = new S3Version(objectKey, versionId);
} catch (RuntimeException e) {
result = e;
throw e;
} catch (Exception e) {
result = e;
}
}
@Override
public Object getResult() {
return result;
}
@Override
public void forceInterruptCalled() {
// This is an atomic operation, cannot interrupt. Ignore.
}
}
/**
* Thread for creating a bucket.
*/
private class CreateBucketRunnable extends AbstractRunnable {
private S3Bucket bucket = null;
private Object result = null;
public CreateBucketRunnable(S3Bucket bucket) {
this.bucket = bucket;
}
public void run() {
try {
result = s3Service.createBucket(bucket);
} catch (S3ServiceException e) {
result = e;
}
}
@Override
public Object getResult() {
return result;
}
@Override
public void forceInterruptCalled() {
// This is an atomic operation, cannot interrupt. Ignore.
}
}
/**
* Thread for listing the objects in a bucket.
*/
private class ListObjectsRunnable extends AbstractRunnable {
private Object result = null;
private String bucketName = null;
private String prefix = null;
private String delimiter = null;
private long maxListingLength = 1000;
private String priorLastKey = null;
private boolean halted = false;
public ListObjectsRunnable(String bucketName, String prefix,
String delimiter, long maxListingLength, String priorLastKey)
{
this.bucketName = bucketName;
this.prefix = prefix;
this.delimiter = delimiter;
this.maxListingLength = maxListingLength;
this.priorLastKey = priorLastKey;
}
public void run() {
try {
List allObjects = new ArrayList();
List allCommonPrefixes = new ArrayList();
do {
StorageObjectsChunk chunk = s3Service.listObjectsChunked(
bucketName, prefix, delimiter, maxListingLength, priorLastKey);
priorLastKey = chunk.getPriorLastKey();
allObjects.addAll(Arrays.asList(chunk.getObjects()));
allCommonPrefixes.addAll(Arrays.asList(chunk.getCommonPrefixes()));
} while (!halted && priorLastKey != null);
result = new S3ObjectsChunk(
prefix, delimiter,
(S3Object[]) allObjects.toArray(new S3Object[allObjects.size()]),
(String[]) allCommonPrefixes.toArray(new String[allCommonPrefixes.size()]),
null);
} catch (ServiceException se) {
result = new S3ServiceException(se);
}
}
@Override
public Object getResult() {
return result;
}
@Override
public void forceInterruptCalled() {
halted = true;
}
}
/**
* Thread for creating/uploading an object. The upload of any object data is monitored with a
* {@link ProgressMonitoredInputStream} and can be can cancelled as the input stream is wrapped in
* an {@link InterruptableInputStream}.
*/
private class CreateObjectRunnable extends AbstractRunnable {
private S3Bucket bucket = null;
private S3Object s3Object = null;
private InterruptableInputStream interruptableInputStream = null;
private BytesProgressWatcher progressMonitor = null;
private Object result = null;
public CreateObjectRunnable(S3Bucket bucket, S3Object s3Object, BytesProgressWatcher progressMonitor) {
this.bucket = bucket;
this.s3Object = s3Object;
this.progressMonitor = progressMonitor;
}
public void run() {
try {
File underlyingFile = s3Object.getDataInputFile();
if (s3Object.getDataInputStream() != null) {
interruptableInputStream = new InterruptableInputStream(s3Object.getDataInputStream());
ProgressMonitoredInputStream pmInputStream = new ProgressMonitoredInputStream(
interruptableInputStream, progressMonitor);
s3Object.setDataInputStream(pmInputStream);
}
result = s3Service.putObject(bucket, s3Object);
if (underlyingFile instanceof TempFile) {
underlyingFile.delete();
}
} catch (ServiceException se) {
result = new S3ServiceException(se);
}
}
@Override
public Object getResult() {
return result;
}
@Override
public void forceInterruptCalled() {
if (interruptableInputStream != null) {
interruptableInputStream.interrupt();
}
}
}
/**
* Thread for copying an object.
*/
private class CopyObjectRunnable extends AbstractRunnable {
private String sourceBucketName = null;
private String destinationBucketName = null;
private String sourceObjectKey = null;
private S3Object destinationObject = null;
private boolean replaceMetadata = false;
private Object result = null;
public CopyObjectRunnable(String sourceBucketName, String destinationBucketName,
String sourceObjectKey, S3Object destinationObject, boolean replaceMetadata)
{
this.sourceBucketName = sourceBucketName;
this.destinationBucketName = destinationBucketName;
this.sourceObjectKey = sourceObjectKey;
this.destinationObject = destinationObject;
this.replaceMetadata = replaceMetadata;
}
public void run() {
try {
result = s3Service.copyObject(sourceBucketName, sourceObjectKey,
destinationBucketName, destinationObject, replaceMetadata);
} catch (ServiceException se) {
result = new S3ServiceException(se);
}
}
@Override
public Object getResult() {
return result;
}
@Override
public void forceInterruptCalled() {
// This is an atomic operation, cannot interrupt. Ignore.
}
}
/**
* Thread for retrieving an object.
*/
private class GetObjectRunnable extends AbstractRunnable {
private S3Bucket bucket = null;
private String objectKey = null;
private String signedGetOrHeadUrl = null;
private boolean headOnly = false;
private Object result = null;
public GetObjectRunnable(S3Bucket bucket, String objectKey, boolean headOnly) {
this.signedGetOrHeadUrl = null;
this.bucket = bucket;
this.objectKey = objectKey;
this.headOnly = headOnly;
}
public GetObjectRunnable(String signedGetOrHeadUrl, boolean headOnly) {
this.signedGetOrHeadUrl = signedGetOrHeadUrl;
this.bucket = null;
this.objectKey = null;
this.headOnly = headOnly;
}
public void run() {
try {
if (headOnly) {
if (signedGetOrHeadUrl == null) {
result = s3Service.getObjectDetails(bucket, objectKey);
} else {
SignedUrlHandler handler = s3Service;
result = handler.getObjectDetailsWithSignedUrl(signedGetOrHeadUrl);
}
} else {
if (signedGetOrHeadUrl == null) {
result = s3Service.getObject(bucket, objectKey);
} else {
SignedUrlHandler handler = s3Service;
result = handler.getObjectWithSignedUrl(signedGetOrHeadUrl);
}
}
} catch (ServiceException se) {
result = new S3ServiceException(se);
}
}
@Override
public Object getResult() {
return result;
}
@Override
public void forceInterruptCalled() {
// This is an atomic operation, cannot interrupt. Ignore.
}
}
/**
* Thread for downloading an object. The download of any object data is monitored with a
* {@link ProgressMonitoredInputStream} and can be can cancelled as the input stream is wrapped in
* an {@link InterruptableInputStream}.
*/
private class DownloadObjectRunnable extends AbstractRunnable {
private String objectKey = null;
private S3Bucket bucket = null;
private DownloadPackage downloadPackage = null;
private InterruptableInputStream interruptableInputStream = null;
private BytesProgressWatcher progressMonitor = null;
private boolean restoreLastModifiedDate = true;
private Object result = null;
public DownloadObjectRunnable(S3Bucket bucket, String objectKey, DownloadPackage downloadPackage,
BytesProgressWatcher progressMonitor, boolean restoreLastModifiedDate)
{
this.bucket = bucket;
this.objectKey = objectKey;
this.downloadPackage = downloadPackage;
this.progressMonitor = progressMonitor;
this.restoreLastModifiedDate = restoreLastModifiedDate;
}
public DownloadObjectRunnable(DownloadPackage downloadPackage, BytesProgressWatcher progressMonitor,
boolean restoreLastModifiedDate)
{
this.downloadPackage = downloadPackage;
this.progressMonitor = progressMonitor;
this.restoreLastModifiedDate = restoreLastModifiedDate;
}
public void run() {
BufferedInputStream bufferedInputStream = null;
BufferedOutputStream bufferedOutputStream = null;
S3Object object = null;
try {
if (!downloadPackage.isSignedDownload()) {
object = s3Service.getObject(bucket, objectKey);
} else {
SignedUrlHandler handler = s3Service;
object = handler.getObjectWithSignedUrl(downloadPackage.getSignedUrl());
}
// Replace the S3 object in the download package with the downloaded version to make metadata available.
downloadPackage.setObject(object);
// Setup monitoring of stream bytes transferred.
interruptableInputStream = new InterruptableInputStream(object.getDataInputStream());
bufferedInputStream = new BufferedInputStream(
new ProgressMonitoredInputStream(interruptableInputStream, progressMonitor));
bufferedOutputStream = new BufferedOutputStream(
downloadPackage.getOutputStream());
MessageDigest messageDigest = null;
try {
messageDigest = MessageDigest.getInstance("MD5");
} catch (NoSuchAlgorithmException e) {
if (log.isWarnEnabled()) {
log.warn("Unable to calculate MD5 hash of data received as algorithm is not available", e);
}
}
try {
byte[] buffer = new byte[1024];
int byteCount = -1;
while ((byteCount = bufferedInputStream.read(buffer)) != -1) {
bufferedOutputStream.write(buffer, 0, byteCount);
if (messageDigest != null) {
messageDigest.update(buffer, 0, byteCount);
}
}
// Check that actual bytes received match expected hash value
if (messageDigest != null) {
byte[] dataMD5Hash = messageDigest.digest();
String hexMD5OfDownloadedData = ServiceUtils.toHex(dataMD5Hash);
// Don't check MD5 hash against ETag if ETag doesn't look like an MD5 value
if (!ServiceUtils.isEtagAlsoAnMD5Hash(object.getETag())) {
// Use JetS3t's own MD5 hash metadata value for comparison, if it's available
if (!hexMD5OfDownloadedData.equals(object.getMd5HashAsHex())) {
if (log.isWarnEnabled()) {
log.warn("Unable to verify MD5 hash of downloaded data against"
+ " ETag returned by service because ETag value \""
+ object.getETag() + "\" is not an MD5 hash value"
+ ", for object key: " + object.getKey());
}
}
} else {
if (!hexMD5OfDownloadedData.equals(object.getETag())) {
throw new S3ServiceException("Mismatch between MD5 hash of downloaded data ("
+ hexMD5OfDownloadedData + ") and ETag returned by S3 ("
+ object.getETag() + ") for object key: "
+ object.getKey());
} else {
if (log.isDebugEnabled()) {
log.debug("Object download was automatically verified, the calculated MD5 hash "+
"value matched the ETag provided by S3: " + object.getKey());
}
}
}
}
} finally {
if (bufferedOutputStream != null) {
bufferedOutputStream.close();
}
if (bufferedInputStream != null) {
bufferedInputStream.close();
}
}
object.setDataInputStream(null);
object.setDataInputFile(downloadPackage.getDataFile());
// If data was downloaded to a file, set the file's Last Modified date
// to the original last modified date metadata stored with the object.
if (restoreLastModifiedDate && downloadPackage.getDataFile() != null) {
String metadataLocalFileDate = (String) object.getMetadata(
Constants.METADATA_JETS3T_LOCAL_FILE_DATE);
if (metadataLocalFileDate != null) {
if (log.isDebugEnabled()) {
log.debug("Restoring original Last Modified date for object '"
+ object.getKey() + "' to file '" + downloadPackage.getDataFile()
+ "': " + metadataLocalFileDate);
}
downloadPackage.getDataFile().setLastModified(
ServiceUtils.parseIso8601Date(metadataLocalFileDate).getTime());
}
}
result = object;
} catch (Throwable t) {
result = t;
} finally {
if (bufferedInputStream != null) {
try {
bufferedInputStream.close();
} catch (Exception e) {
if (log.isErrorEnabled()) {
log.error("Unable to close Object input stream", e);
}
}
}
if (bufferedOutputStream != null) {
try {
bufferedOutputStream.close();
} catch (Exception e) {
if (log.isErrorEnabled()) {
log.error("Unable to close download output stream", e);
}
}
}
}
}
@Override
public Object getResult() {
return result;
}
@Override
public void forceInterruptCalled() {
if (interruptableInputStream != null) {
interruptableInputStream.interrupt();
}
}
}
/**
* Thread for creating/uploading an object using a pre-signed PUT URL. The upload of any object
* data is monitored with a {@link ProgressMonitoredInputStream} and can be can cancelled as
* the input stream is wrapped in an {@link InterruptableInputStream}.
*/
private class SignedPutRunnable extends AbstractRunnable {
private SignedUrlAndObject signedUrlAndObject = null;
private InterruptableInputStream interruptableInputStream = null;
private BytesProgressWatcher progressMonitor = null;
private Object result = null;
public SignedPutRunnable(SignedUrlAndObject signedUrlAndObject, BytesProgressWatcher progressMonitor) {
this.signedUrlAndObject = signedUrlAndObject;
this.progressMonitor = progressMonitor;
}
public void run() {
try {
File underlyingFile = signedUrlAndObject.getObject().getDataInputFile();
if (signedUrlAndObject.getObject().getDataInputStream() != null) {
interruptableInputStream = new InterruptableInputStream(
signedUrlAndObject.getObject().getDataInputStream());
ProgressMonitoredInputStream pmInputStream = new ProgressMonitoredInputStream(
interruptableInputStream, progressMonitor);
signedUrlAndObject.getObject().setDataInputStream(pmInputStream);
}
SignedUrlHandler signedPutUploader = s3Service;
result = signedPutUploader.putObjectWithSignedUrl(
signedUrlAndObject.getSignedUrl(), signedUrlAndObject.getObject());
if (underlyingFile instanceof TempFile) {
underlyingFile.delete();
}
} catch (ServiceException se) {
result = new S3ServiceException(se);
} finally {
try {
signedUrlAndObject.getObject().closeDataInputStream();
} catch (IOException e) {
if (log.isErrorEnabled()) {
log.error("Unable to close Object's input stream", e);
}
}
}
}
@Override
public Object getResult() {
return result;
}
@Override
public void forceInterruptCalled() {
if (interruptableInputStream != null) {
interruptableInputStream.interrupt();
}
}
}
/**
* The thread group manager is responsible for starting, running and stopping the set of threads
* required to perform an S3 operation.
* <p>
* The manager starts all the threads, monitors their progress and stops threads when they are
* cancelled or an error occurs - all the while firing the appropriate {@link ServiceEvent} event
* notifications.
*/
private abstract class ThreadGroupManager {
private final Log log = LogFactory.getLog(ThreadGroupManager.class);
private int maxThreadCount = 1;
/**
* the set of runnable objects to execute.
*/
private AbstractRunnable[] runnables = null;
/**
* Thread objects that are currently running, where the index corresponds to the
* runnables index. Any AbstractThread runnable that is not started, or has completed,
* will have a null value in this array.
*/
private Thread[] threads = null;
private boolean ignoreExceptions = false;
/**
* set of flags indicating which runnable items have been started
*/
private boolean started[] = null;
/**
* set of flags indicating which threads have already had In Progress events fired on
* their behalf. These threads have finished running.
*/
private boolean alreadyFired[] = null;
private ThreadWatcher threadWatcher = null;
private long lastProgressEventFiredTime = 0;
public ThreadGroupManager(AbstractRunnable[] runnables,
ThreadWatcher threadWatcher, Jets3tProperties jets3tProperties,
boolean isAdminTask)
{
this.runnables = runnables;
this.threadWatcher = threadWatcher;
if (isAdminTask) {
this.maxThreadCount = jets3tProperties
.getIntProperty("s3service.admin-max-thread-count", 20);
} else {
this.maxThreadCount = jets3tProperties
.getIntProperty("s3service.max-thread-count", 2);
}
this.ignoreExceptions = jets3tProperties
.getBoolProperty("s3service.ignore-exceptions-in-multi", false);
this.threads = new Thread[runnables.length];
started = new boolean[runnables.length]; // All values initialized to false.
alreadyFired = new boolean[runnables.length]; // All values initialized to false.
}
/**
* Determine which threads, if any, have finished since the last time an In Progress event
* was fired.
*
* @return
* a list of the threads that finished since the last In Progress event was fired. This list may
* be empty.
*
* @throws Throwable
*/
private ResultsTuple getNewlyCompletedResults() throws Throwable
{
ArrayList completedResults = new ArrayList();
ArrayList errorResults = new ArrayList();
for (int i = 0; i < threads.length; i++) {
if (!alreadyFired[i] && started[i] && !threads[i].isAlive()) {
alreadyFired[i] = true;
if (log.isDebugEnabled()) {
log.debug("Thread " + (i+1) + " of " + threads.length
+ " has recently completed, releasing resources");
}
if (runnables[i].getResult() instanceof Throwable) {
Throwable throwable = (Throwable) runnables[i].getResult();
runnables[i] = null;
threads[i] = null;
if (ignoreExceptions) {
// Ignore exceptions
if (log.isWarnEnabled()) {
log.warn("Ignoring exception (property " +
"s3service.ignore-exceptions-in-multi is set to true)",
throwable);
}
errorResults.add(throwable);
} else {
throw throwable;
}
} else {
completedResults.add(runnables[i].getResult());
runnables[i] = null;
threads[i] = null;
}
}
}
Throwable[] ignoredErrors = new Throwable[] {};
if (errorResults.size() > 0) {
ignoredErrors = (Throwable[]) errorResults.toArray(new Throwable[errorResults.size()]);
}
return new ResultsTuple(completedResults, ignoredErrors);
}
/**
* Starts pending threads such that the total of running threads never exceeds the
* maximum count set in the jets3t property <i>s3service.max-thread-count</i>.
*
* @throws Throwable
*/
private void startPendingThreads()
throws Throwable
{
// Count active threads that are running (i.e. have been started but final event not fired)
int runningThreadCount = 0;
for (int i = 0; i < runnables.length; i++) {
if (started[i] && !alreadyFired[i]) {
runningThreadCount++;
}
}
// Start threads until we are running the maximum number allowed.
for (int i = 0; runningThreadCount < maxThreadCount && i < started.length; i++) {
if (!started[i]) {
threads[i] = new Thread(runnables[i]);
threads[i].start();
started[i] = true;
runningThreadCount++;
if (log.isDebugEnabled()) {
log.debug("Thread " + (i+1) + " of " + runnables.length + " has started");
}
}
}
}
/**
* @return
* the number of threads that have not finished running (sum of those currently running, and those awaiting start)
*/
private int getPendingThreadCount() {
int pendingThreadCount = 0;
for (int i = 0; i < runnables.length; i++) {
if (!alreadyFired[i]) {
pendingThreadCount++;
}
}
return pendingThreadCount;
}
/**
* Invokes the {@link AbstractRunnable#forceInterrupt} on all threads being managed.
*
*/
private void forceInterruptAllRunnables() {
if (log.isDebugEnabled()) {
log.debug("Setting force interrupt flag on all runnables");
}
for (int i = 0; i < runnables.length; i++) {
if (runnables[i] != null) {
runnables[i].forceInterrupt();
runnables[i] = null;
}
}
}
/**
* Runs and manages all the threads involved in an S3 multi-operation.
*
*/
public void run() {
if (log.isDebugEnabled()) {
log.debug("Started ThreadManager");
}
final boolean[] interrupted = new boolean[] { false };
/*
* Create a cancel event trigger, so all the managed threads can be cancelled if required.
*/
final CancelEventTrigger cancelEventTrigger = new CancelEventTrigger() {
private static final long serialVersionUID = 6328417466929608235L;
public void cancelTask(Object eventSource) {
if (log.isDebugEnabled()) {
log.debug("Cancel task invoked on ThreadManager");
}
// Flag that this ThreadManager class should shutdown.
interrupted[0] = true;
// Set force interrupt flag for all runnables.
forceInterruptAllRunnables();
}
};
// Actual thread management happens in the code block below.
try {
// Start some threads
startPendingThreads();
threadWatcher.updateThreadsCompletedCount(0, cancelEventTrigger);
fireStartEvent(threadWatcher);
// Loop while threads haven't been interrupted/cancelled, and at least one thread is
// still active (ie hasn't finished its work)
while (!interrupted[0] && getPendingThreadCount() > 0) {
try {
// Shut down threads if this service has been shutdown.
if (isShutdown[0]) {
throw new InterruptedException("S3ServiceMulti#shutdown method invoked");
}
Thread.sleep(100);
if (interrupted[0]) {
// Do nothing, we've been interrupted during sleep.
} else {
if (System.currentTimeMillis() - lastProgressEventFiredTime > sleepTime) {
// Fire progress event.
int completedThreads = runnables.length - getPendingThreadCount();
threadWatcher.updateThreadsCompletedCount(completedThreads, cancelEventTrigger);
ResultsTuple results = getNewlyCompletedResults();
lastProgressEventFiredTime = System.currentTimeMillis();
fireProgressEvent(threadWatcher, results.completedResults);
if (results.errorResults.length > 0) {
fireIgnoredErrorsEvent(threadWatcher, results.errorResults);
}
}
// Start more threads.
startPendingThreads();
}
} catch (InterruptedException e) {
interrupted[0] = true;
forceInterruptAllRunnables();
}
}
if (interrupted[0]) {
fireCancelEvent();
} else {
int completedThreads = runnables.length - getPendingThreadCount();
threadWatcher.updateThreadsCompletedCount(completedThreads, cancelEventTrigger);
ResultsTuple results = getNewlyCompletedResults();
fireProgressEvent(threadWatcher, results.completedResults);
if (results.completedResults.size() > 0) {
if (log.isDebugEnabled()) {
log.debug(results.completedResults.size() + " threads have recently completed");
}
}
if (results.errorResults.length > 0) {
fireIgnoredErrorsEvent(threadWatcher, results.errorResults);
}
fireCompletedEvent();
}
} catch (Throwable t) {
if (log.isErrorEnabled()) {
log.error("A thread failed with an exception. Firing ERROR event and cancelling all threads", t);
}
// Set force interrupt flag for all runnables.
forceInterruptAllRunnables();
fireErrorEvent(t);
}
}
public abstract void fireStartEvent(ThreadWatcher threadWatcher);
public abstract void fireProgressEvent(ThreadWatcher threadWatcher, List completedResults);
public abstract void fireCompletedEvent();
public abstract void fireCancelEvent();
public abstract void fireErrorEvent(Throwable t);
public abstract void fireIgnoredErrorsEvent(ThreadWatcher threadWatcher, Throwable[] ignoredErrors);
private class ResultsTuple {
public List completedResults = null;
public Throwable[] errorResults = null;
public ResultsTuple(List completedResults, Throwable[] errorResults) {
this.completedResults = completedResults;
this.errorResults = errorResults;
}
}
}
}