package com.techwarriors.simples3;
import java.io.File;
import java.text.DecimalFormat;
import org.apache.commons.lang.StringUtils;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.model.GetObjectRequest;
import com.amazonaws.services.s3.model.ListObjectsRequest;
import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.S3ObjectSummary;
public class Tool {
private static DecimalFormat FMT = new DecimalFormat("###,###,###,###,###") ;
private enum Commands {
put(3),
putdir(3),
get(3),
getdir(3),
ls(2) ;
private int requiredArgs ;
private Commands (int requiredArgs) {
this.requiredArgs = requiredArgs ;
}
public int getRequiredArgs() {
return requiredArgs;
}
}
public static void main(String[] args) {
boolean exitWithError = false ;
Commands comm = args.length > 0 ? Commands.valueOf(args[0]) : null ;
if (comm == null) {
showUsage() ;
exitWithError = true ;
} else {
if (args.length < comm.getRequiredArgs()) {
showUsage() ;
exitWithError = true ;
}
AmazonS3Client client = new AmazonS3Client() ;
try {
switch (comm) {
case get:
getOperation (client, args[1], args[2]) ;
break;
case getdir:
getDirOperation (client, args[1], args[2]) ;
break;
case put:
putOperation (client, args[1], args[2]) ;
break ;
case putdir:
putDirOperation (client, args[1], args[2]) ;
break ;
case ls:
lsOperation (client, args[1]) ;
break ;
}
} catch (Throwable e) {
e.printStackTrace() ;
exitWithError = true ;
} finally {
client.shutdown() ;
}
}
System.exit (exitWithError ? 1 : 0) ;
}
public static void putOperation (AmazonS3Client client, String source, String destination) {
File srcFile = new File(source) ;
if (srcFile.exists() && srcFile.isFile()) {
String [] destParts = StringUtils.split(destination, '/') ;
String bucketName = destParts[0] ;
String s3Key = null ;
if (destParts.length > 1) {
if (destination.endsWith("/"))
s3Key = StringUtils.join(destParts, '/', 1, destParts.length) + "/" + srcFile.getName();
else
s3Key = StringUtils.join(destParts, '/', 1, destParts.length) ;
} else {
s3Key = srcFile.getName() ;
}
System.out.println ("Attempting PUT - '" + srcFile + "' --> '" + bucketName + "/" + s3Key + "'");
long time = System.currentTimeMillis() ;
client.putObject(bucketName, s3Key, srcFile) ;
time = System.currentTimeMillis() - time ;
System.out.println ("Completed PUT - Transferred [" + FMT.format(srcFile.length()) + " bytes] in " + FMT.format (time) + " msecs");
}
}
public static void putDirOperation (AmazonS3Client client, String source, String destination) {
File srcFile = new File(source) ;
if (srcFile.exists() && srcFile.isDirectory()) {
String [] destParts = StringUtils.split(destination, '/') ;
String bucketName = destParts[0] ;
String s3Key = null ;
if (destParts.length > 1) {
if (destination.endsWith("/"))
s3Key = StringUtils.join(destParts, '/', 1, destParts.length) + "/" + srcFile.getName();
else
s3Key = StringUtils.join(destParts, '/', 1, destParts.length) ;
} else {
s3Key = srcFile.getName() ;
}
putDirOperation(client, srcFile, bucketName, s3Key) ;
}
}
public static void putDirOperation (AmazonS3Client client, File sourceDir, String s3Bucket, String s3Key) {
for (File f : sourceDir.listFiles()) {
if (f.isDirectory()) {
putDirOperation(client, f, s3Bucket, s3Key + "/" + f.getName()) ;
} else {
long time = System.currentTimeMillis() ;
client.putObject(s3Bucket, s3Key + "/" + f.getName(), f) ;
time = System.currentTimeMillis() - time ;
System.out.println ("Transferred '" + f.getAbsolutePath() + "' --> '" + s3Bucket + "/" + s3Key + "/" + f.getName() + "' [" + FMT.format(f.length()) + " bytes] in " + FMT.format (time) + " msecs");
}
}
}
public static void getOperation (AmazonS3Client client, String source, String destination) {
File destObj = new File(destination) ;
boolean explicitFilename = !destObj.isDirectory() ;
String [] srcParts = StringUtils.split(source, '/') ;
String bucketName = srcParts[0] ;
String s3Key = StringUtils.join(srcParts, '/', 1, srcParts.length) ;
File destFile = explicitFilename ? destObj : new File (destObj, srcParts[srcParts.length-1]) ;
System.out.println ("Attempting GET - '" + source + "' --> '" + destFile + "'");
long time = System.currentTimeMillis() ;
client.getObject (new GetObjectRequest (bucketName, s3Key), destFile) ;
time = System.currentTimeMillis() - time ;
System.out.println ("Completed GET - Transferred [" + FMT.format(destFile.length()) + " bytes] in " + FMT.format (time) + " msecs");
}
public static void getDirOperation (AmazonS3Client client, String source, String destination) {
File destObj = new File(destination) ;
if (!destObj.exists())
destObj.mkdirs() ;
if (destObj.isDirectory()) {
String [] srcParts = StringUtils.split(source, '/') ;
String bucketName = srcParts[0] ;
String s3Key = StringUtils.join(srcParts, '/', 1, srcParts.length) ;
String marker = null ;
do {
ListObjectsRequest lor = new ListObjectsRequest().withBucketName(bucketName).withPrefix(s3Key).withMarker(marker) ;
ObjectListing objectListing = client.listObjects (lor) ;
marker = objectListing.getNextMarker() ;
for (S3ObjectSummary s : objectListing.getObjectSummaries()) {
String fullKey = s.getKey().substring(s3Key.length()) ;
String[] keyParts = StringUtils.split(fullKey, '/') ;
File dir = (keyParts.length > 1) ? new File (destObj, StringUtils.join(keyParts, '/', 0, keyParts.length-1)) : destObj;
if (!dir.exists())
dir.mkdirs() ;
long time = System.currentTimeMillis() ;
File f = new File (dir, keyParts[keyParts.length-1]) ;
client.getObject (new GetObjectRequest (bucketName, s.getKey()), f) ;
time = System.currentTimeMillis() - time ;
System.out.println ("Transferred '" + bucketName + "/" + s.getKey() + "' --> '" + f.getAbsolutePath() + "' [" + FMT.format(s.getSize()) + " bytes] in " + FMT.format (time) + " msecs");
}
} while (!StringUtils.isEmpty(marker)) ;
}
}
public static void lsOperation (AmazonS3Client client, String prefix) {
String [] srcParts = StringUtils.split(prefix, '/') ;
String bucketName = srcParts[0] ;
String s3prefix = StringUtils.join(srcParts, '/', 1, srcParts.length) ;
String marker = null ;
do {
ListObjectsRequest lor = new ListObjectsRequest().withBucketName(bucketName).withPrefix(s3prefix).withMarker(marker) ;
ObjectListing objectListing = client.listObjects (lor) ;
marker = objectListing.getNextMarker() ;
for (S3ObjectSummary s : objectListing.getObjectSummaries())
System.out.println ("Object\t" + FMT.format(s.getSize()) + "\t" + s.getLastModified() + "\t" + s.getKey());
} while (!StringUtils.isEmpty(marker)) ;
}
public static void showUsage () {
System.out.println ("\nUsages:\n") ;
System.out.println ("java -jar xxx-executable.jar put {source-file} {s3-destination}");
System.out.println ("\n * Transfers a single local file to S3. If the destination ends in" +
"\n * '/', it is used as a prefix, otherwise it is used as the key.\n");
System.out.println ("java -jar xxx-executable.jar putdir {source-dir} {s3-destination}");
System.out.println ("\n * Transfers a directory and all its contents to S3. If the" +
"\n * destination ends in '/', it is used as a prefix, otherwise" +
"\n * it is used as the key.\n");
System.out.println ("java -jar xxx-executable.jar get {s3-source-key} {destination-dir-or-file}");
System.out.println ("\n * Transfers a single S3 object to local file. If the destination" +
"\n * is a directory, the object is transfered into it, otherwise it" +
"\n * is used as the filename.\n");
System.out.println ("java -jar xxx-executable.jar getdir {s3-source-prefix} {destination-dir}");
System.out.println ("\n * Transfers all S3 objects with the given prefix to local directory." +
"\n * If the destination directory does not exist, it is created.\n");
System.out.println ("java -jar xxx-executable.jar ls {s3-source-prefix}");
System.out.println ("\n * Lists all S3 objects with the given prefix to stdout.\n") ;
}
}