/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package com.cloud.utils;
import static com.amazonaws.Protocol.HTTP;
import static com.amazonaws.Protocol.HTTPS;
import static com.cloud.utils.StringUtils.join;
import static java.io.File.createTempFile;
import static java.lang.String.format;
import static java.lang.System.currentTimeMillis;
import static java.util.Arrays.asList;
import static java.util.Collections.emptyList;
import static java.util.Collections.singletonList;
import static java.util.Collections.unmodifiableList;
import static org.apache.commons.lang.ArrayUtils.isEmpty;
import static org.apache.commons.lang.StringUtils.isBlank;
import static org.apache.commons.lang.StringUtils.isNotBlank;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
import org.apache.commons.lang.ArrayUtils;
import org.apache.log4j.Logger;
import com.amazonaws.AmazonClientException;
import com.amazonaws.ClientConfiguration;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.model.Bucket;
import com.amazonaws.services.s3.model.GetObjectRequest;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.cloud.utils.exception.CloudRuntimeException;
public final class S3Utils {
private static final Logger LOGGER = Logger.getLogger(S3Utils.class);
public static final String SEPARATOR = "/";
private static final int MIN_BUCKET_NAME_LENGTH = 3;
private static final int MAX_BUCKET_NAME_LENGTH = 63;
private S3Utils() {
super();
}
private static AmazonS3 acquireClient(final ClientOptions clientOptions) {
final AWSCredentials credentials = new BasicAWSCredentials(
clientOptions.getAccessKey(), clientOptions.getSecretKey());
final ClientConfiguration configuration = new ClientConfiguration();
if (clientOptions.isHttps() != null) {
configuration.setProtocol(clientOptions.isHttps() == true ? HTTPS
: HTTP);
}
if (clientOptions.getConnectionTimeout() != null) {
configuration.setConnectionTimeout(clientOptions
.getConnectionTimeout());
}
if (clientOptions.getMaxErrorRetry() != null) {
configuration.setMaxErrorRetry(clientOptions.getMaxErrorRetry());
}
if (clientOptions.getSocketTimeout() != null) {
configuration.setSocketTimeout(clientOptions.getSocketTimeout());
}
if (LOGGER.isDebugEnabled()) {
LOGGER.debug(format(
"Creating S3 client with configuration: [protocol: %1$s, connectionTimeOut: "
+ "%2$s, maxErrorRetry: %3$s, socketTimeout: %4$s]",
configuration.getProtocol(),
configuration.getConnectionTimeout(),
configuration.getMaxErrorRetry(),
configuration.getSocketTimeout()));
}
final AmazonS3Client client = new AmazonS3Client(credentials,
configuration);
if (isNotBlank(clientOptions.getEndPoint())) {
if (LOGGER.isDebugEnabled()) {
LOGGER.debug(format(
"Setting the end point for S3 client %1$s to %2$s.",
client, clientOptions.getEndPoint()));
}
client.setEndpoint(clientOptions.getEndPoint());
}
return client;
}
public static void putFile(final ClientOptions clientOptions,
final File sourceFile, final String bucketName, final String key) {
assert clientOptions != null;
assert sourceFile != null;
assert !isBlank(bucketName);
assert !isBlank(key);
if (LOGGER.isDebugEnabled()) {
LOGGER.debug(format("Sending file %1$s as S3 object %2$s in "
+ "bucket %3$s", sourceFile.getName(), key, bucketName));
}
acquireClient(clientOptions).putObject(bucketName, key, sourceFile);
}
@SuppressWarnings("unchecked")
public static File getFile(final ClientOptions clientOptions,
final String bucketName, final String key,
final File targetDirectory, final FileNamingStrategy namingStrategy) {
assert clientOptions != null;
assert isNotBlank(bucketName);
assert isNotBlank(key);
assert targetDirectory != null && targetDirectory.isDirectory();
assert namingStrategy != null;
final AmazonS3 connection = acquireClient(clientOptions);
File tempFile = null;
try {
tempFile = createTempFile(
join(asList(targetDirectory.getName(), currentTimeMillis(),
"part"), "-"), "tmp", targetDirectory);
tempFile.deleteOnExit();
if (LOGGER.isDebugEnabled()) {
LOGGER.debug(format(
"Downloading object %1$s from bucket %2$s to temp file %3$s",
key, bucketName, tempFile.getName()));
}
connection.getObject(new GetObjectRequest(bucketName, key),
tempFile);
final File targetFile = new File(targetDirectory,
namingStrategy.determineFileName(key));
tempFile.renameTo(targetFile);
return targetFile;
} catch (FileNotFoundException e) {
throw new CloudRuntimeException(
format("Failed open file %1$s in order to get object %2$s from bucket %3$s.",
targetDirectory.getAbsoluteFile(), bucketName, key),
e);
} catch (IOException e) {
throw new CloudRuntimeException(
format("Unable to allocate temporary file in directory %1$s to download %2$s:%3$s from S3",
targetDirectory.getAbsolutePath(), bucketName, key),
e);
} finally {
if (tempFile != null) {
tempFile.delete();
}
}
}
public static List<File> getDirectory(final ClientOptions clientOptions,
final String bucketName, final String sourcePath,
final File targetDirectory, final FileNamingStrategy namingStrategy) {
assert clientOptions != null;
assert isNotBlank(bucketName);
assert isNotBlank(sourcePath);
assert targetDirectory != null;
final AmazonS3 connection = acquireClient(clientOptions);
// List the objects in the source directory on S3
final List<S3ObjectSummary> objectSummaries = listDirectory(bucketName,
sourcePath, connection);
final List<File> files = new ArrayList<File>();
for (final S3ObjectSummary objectSummary : objectSummaries) {
files.add(getFile(clientOptions, bucketName,
objectSummary.getKey(), targetDirectory, namingStrategy));
}
return unmodifiableList(files);
}
private static List<S3ObjectSummary> listDirectory(final String bucketName,
final String directory, final AmazonS3 client) {
final List<S3ObjectSummary> objects = client.listObjects(bucketName,
directory + SEPARATOR).getObjectSummaries();
if (objects == null) {
return emptyList();
}
return unmodifiableList(objects);
}
public static void putDirectory(final ClientOptions clientOptions,
final String bucketName, final File directory,
final FilenameFilter fileNameFilter,
final ObjectNamingStrategy namingStrategy) {
assert clientOptions != null;
assert isNotBlank(bucketName);
assert directory != null && directory.isDirectory();
assert fileNameFilter != null;
assert namingStrategy != null;
if (LOGGER.isDebugEnabled()) {
LOGGER.debug(format("Putting directory %1$s in S3 bucket %2$s.",
directory.getAbsolutePath(), bucketName));
}
// Determine the list of files to be sent using the passed filter ...
final File[] files = directory.listFiles(fileNameFilter);
if (LOGGER.isTraceEnabled()) {
LOGGER.trace(format("Putting files (%1$s) in S3 bucket %2$s.",
ArrayUtils.toString(files, "no files found"), bucketName));
}
// Skip spinning up an S3 connection when no files will be sent ...
if (isEmpty(files)) {
return;
}
final AmazonS3 client = acquireClient(clientOptions);
// Send the files to S3 using the passed ObjectNaming strategy to
// determine the key ...
for (final File file : files) {
final String key = namingStrategy.determineKey(file);
if (LOGGER.isDebugEnabled()) {
LOGGER.debug(format(
"Putting file %1$s into bucket %2$s with key %3$s.",
file.getAbsolutePath(), bucketName, key));
}
client.putObject(bucketName, key, file);
}
}
public static void deleteObject(final ClientOptions clientOptions,
final String bucketName, final String key) {
assert clientOptions != null;
assert isNotBlank(bucketName);
assert isNotBlank(key);
final AmazonS3 client = acquireClient(clientOptions);
client.deleteObject(bucketName, key);
}
public static void deleteDirectory(final ClientOptions clientOptions,
final String bucketName, final String directoryName) {
assert clientOptions != null;
assert isNotBlank(bucketName);
assert isNotBlank(directoryName);
final AmazonS3 client = acquireClient(clientOptions);
final List<S3ObjectSummary> objects = listDirectory(bucketName,
directoryName, client);
for (final S3ObjectSummary object : objects) {
client.deleteObject(bucketName, object.getKey());
}
client.deleteObject(bucketName, directoryName);
}
public static boolean canConnect(final ClientOptions clientOptions) {
try {
acquireClient(clientOptions);
return true;
} catch (AmazonClientException e) {
LOGGER.warn("Ignored Exception while checking connection options",
e);
return false;
}
}
public static boolean doesBucketExist(final ClientOptions clientOptions,
final String bucketName) {
assert clientOptions != null;
assert !isBlank(bucketName);
try {
final List<Bucket> buckets = acquireClient(clientOptions)
.listBuckets();
for (Bucket bucket : buckets) {
if (bucket.getName().equals(bucketName)) {
return true;
}
}
return false;
} catch (AmazonClientException e) {
LOGGER.warn("Ignored Exception while checking bucket existence", e);
return false;
}
}
public static boolean canReadWriteBucket(final ClientOptions clientOptions,
final String bucketName) {
assert clientOptions != null;
assert isNotBlank(bucketName);
try {
final AmazonS3 client = acquireClient(clientOptions);
final String fileContent = "testing put and delete";
final InputStream inputStream = new ByteArrayInputStream(
fileContent.getBytes());
final String key = UUID.randomUUID().toString() + ".txt";
final ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(fileContent.length());
client.putObject(bucketName, key, inputStream, metadata);
client.deleteObject(bucketName, key);
return true;
} catch (AmazonClientException e) {
return false;
}
}
public static List<String> checkClientOptions(ClientOptions clientOptions) {
assert clientOptions != null;
List<String> errorMessages = new ArrayList<String>();
errorMessages.addAll(checkRequiredField("access key",
clientOptions.getAccessKey()));
errorMessages.addAll(checkRequiredField("secret key",
clientOptions.getSecretKey()));
errorMessages.addAll(checkOptionalField("connection timeout",
clientOptions.getConnectionTimeout()));
errorMessages.addAll(checkOptionalField("socket timeout",
clientOptions.getSocketTimeout()));
errorMessages.addAll(checkOptionalField("max error retries",
clientOptions.getMaxErrorRetry()));
return unmodifiableList(errorMessages);
}
public static List<String> checkBucketName(final String bucketLabel,
final String bucket) {
assert isNotBlank(bucketLabel);
assert isNotBlank(bucket);
final List<String> errorMessages = new ArrayList<String>();
if (bucket.length() < MIN_BUCKET_NAME_LENGTH) {
errorMessages
.add(format(
"The length of %1$s "
+ "for the %2$s must have a length of at least %3$s "
+ "characters", bucket, bucketLabel,
MIN_BUCKET_NAME_LENGTH));
}
if (bucket.length() > MAX_BUCKET_NAME_LENGTH) {
errorMessages.add(format("The length of %1$s "
+ "for the %2$s must not have a length of at greater"
+ " than %3$s characters", bucket, bucketLabel,
MAX_BUCKET_NAME_LENGTH));
}
return unmodifiableList(errorMessages);
}
private static List<String> checkOptionalField(final String fieldName,
final Integer fieldValue) {
if (fieldValue != null && fieldValue < 0) {
return singletonList(format("The value of %1$s must "
+ "be greater than zero.", fieldName));
}
return emptyList();
}
private static List<String> checkRequiredField(String fieldName,
String fieldValue) {
if (isBlank(fieldValue)) {
return singletonList(format("A %1$s must be specified.", fieldName));
}
return emptyList();
}
public interface ClientOptions {
String getAccessKey();
String getSecretKey();
String getEndPoint();
Boolean isHttps();
Integer getConnectionTimeout();
Integer getMaxErrorRetry();
Integer getSocketTimeout();
}
public interface ObjectNamingStrategy {
String determineKey(File file);
}
public interface FileNamingStrategy {
String determineFileName(String key);
}
}