* JetS3t : Java S3 Toolkit

* Project hosted at http://bitbucket.org/jmurty/jets3t/

*

* Copyright 2006-2010 James Murty

*

* Licensed under the Apache License, Version 2.0 (the "License");

* you may not use this file except in compliance with the License.

* You may obtain a copy of the License at

*

*     http://www.apache.org/licenses/LICENSE-2.0

*

* Unless required by applicable law or agreed to in writing, software

* distributed under the License is distributed on an "AS IS" BASIS,

* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

* See the License for the specific language governing permissions and

* limitations under the License.

*/

* Runs generic functional tests that any storage service implementation should be

* able to perform.

* <p>

* Any test cases for specific StorageService implementations should extend this class as a

* starting point, then add test cases specific to that particular implementation.

*

* @author James Murty

*/

    protected static final String TARGET_SERVICE_S3 = "AmazonS3";

    protected static final String TARGET_SERVICE_GS = "GoogleStorage";

    protected String TEST_PROPERTIES_FILENAME = "test.properties";

    protected Properties testProperties = null;

    public BaseStorageServiceTests() throws Exception {

        // Load test properties

        InputStream propertiesIS =

            ClassLoader.getSystemResourceAsStream(TEST_PROPERTIES_FILENAME);

        if (propertiesIS == null) {

            throw new Exception("Unable to load test properties file from classpath: "

                + TEST_PROPERTIES_FILENAME);

        }

        this.testProperties = new Properties();

        this.testProperties.load(propertiesIS);

    }

    protected abstract ProviderCredentials getCredentials() throws Exception;

    protected abstract RestStorageService getStorageService(ProviderCredentials credentials) throws Exception;

    protected abstract String getTargetService();

    protected abstract AccessControlList buildAccessControlList();

    protected abstract StorageBucketLoggingStatus getBucketLoggingStatus(

        String targetBucketName, String logfilePrefix) throws Exception;

    /**

     * @param testName

     * @return unique per-account and per-test bucket name

     */

    protected String getBucketNameForTest(String testName) throws Exception {

        return

            "test-"

            + getCredentials().getAccessKey().toLowerCase()

            + "-"

            + testName.toLowerCase();

    }

    protected StorageBucket createBucketForTest(String testName) throws Exception {

        return this.createBucketForTest(testName, null);

    }

    protected StorageBucket createBucketForTest(String testName, String location) throws Exception {

        String bucketName = getBucketNameForTest(testName);

        StorageService service = getStorageService(getCredentials());

        if (service instanceof S3Service) {

            return ((S3Service)service).getOrCreateBucket(bucketName, location);

        } else {

            GSBucket bucket = new GSBucket(bucketName, location);

            return ((GoogleStorageService)service).createBucket(bucket);

        }

    }

    protected void deleteAllObjectsInBucket(String bucketName) {

        try {

            RestStorageService service = getStorageService(getCredentials());

            for (StorageObject o: service.listObjects(bucketName)) {

                service.deleteObject(bucketName, o.getKey());

            }

        } catch (Exception e) {

            // This shouldn't happen, but if it does don't ruin the test

            e.printStackTrace();

        }

    }

    protected void cleanupBucketForTest(String testName, boolean deleteAllObjects) {

        try {

            RestStorageService service = getStorageService(getCredentials());

            String bucketName = getBucketNameForTest(testName);

            if (deleteAllObjects) {

                deleteAllObjectsInBucket(bucketName);

            }

            service.deleteBucket(bucketName);

        } catch (Exception e) {

            // This shouldn't happen, but if it does don't ruin the test

            e.printStackTrace();

        }

    }

    protected void cleanupBucketForTest(String testName) {

        this.cleanupBucketForTest(testName, true);

    }

    /////////////////////////////

    // Actual tests start here //

    /////////////////////////////

    public void testListBuckets() throws Exception {

        // List without credentials

        try {

            getStorageService(null).listAllBuckets();

            fail("Bucket listing should fail without authentication");

        } catch (ServiceException e) {

        }

        // List with credentials

        getStorageService(getCredentials()).listAllBuckets();

        // Ensure newly-created bucket is listed

        String bucketName = createBucketForTest("testListBuckets").getName();

        try {

            StorageBucket[] buckets = getStorageService(getCredentials()).listAllBuckets();

            boolean found = false;

            for (StorageBucket bucket: buckets) {

                found = (bucket.getName().equals(bucketName)) || found;

            }

            assertTrue("Newly-created bucket was not listed", found);

        } finally {

            cleanupBucketForTest("testListBuckets");

        }

    }

    public void testBucketManagement() throws Exception {

        RestStorageService service = getStorageService(getCredentials());

        try {

            service.createBucket("");

            fail("Cannot create a bucket with empty name");

        } catch (ServiceException e) {

        }

        try {

            service.createBucket("test");

            fail("Cannot create a bucket with non-unique name");

        } catch (ServiceException e) {

        }

        String bucketName = createBucketForTest("testBucketManagement").getName();

        boolean bucketExists = service.isBucketAccessible(bucketName);

        assertTrue("Bucket should exist", bucketExists);

        try {

            service.deleteBucket((String) null);

            fail("Cannot delete a bucket with null name");

        } catch (ServiceException e) {

        }

        try {

            service.deleteBucket("");

            fail("Cannot delete a bucket with empty name");

        } catch (ServiceException e) {

        }

        try {

            service.deleteBucket("test");

            fail("Cannot delete a bucket you don't own");

        } catch (ServiceException e) {

        }

        // Ensure we can delete our bucket

        service.deleteBucket(bucketName);

    }

    public void testBucketStatusLookup() throws Exception {

        String bucketName = getBucketNameForTest("testBucketStatusLookup");

        RestStorageService service = getStorageService(getCredentials());

        try {

            // Non-existent bucket

            int status = service.checkBucketStatus(bucketName);

            assertEquals(S3Service.BUCKET_STATUS__DOES_NOT_EXIST, status);

            // Bucket is owned by someone else

            // NOTE: This test will fail if you actually own the "testing" bucket,

            // or if it is owned by someone else but has been made publicly readable.

            status = service.checkBucketStatus("testing");

            assertEquals(S3Service.BUCKET_STATUS__ALREADY_CLAIMED, status);

            service.createBucket(bucketName);

            // Bucket now exists and is owned by me.

            status = service.checkBucketStatus(bucketName);

            assertEquals(S3Service.BUCKET_STATUS__MY_BUCKET, status);

        } finally {

            // Clean up

            service.deleteBucket(bucketName);

        }

    }

    public void testBucketLocations() throws Exception {

        RestStorageService service = getStorageService(getCredentials());

        // Create bucket in default (null) location

        String bucketName = getBucketNameForTest("testBucketLocations-test1");

        try {

            service.createBucket(bucketName);

            assertEquals(StorageService.BUCKET_STATUS__MY_BUCKET, service.checkBucketStatus(bucketName));

            String bucketLocation = (service instanceof S3Service

                ? ((S3Service) service).getBucketLocation(bucketName)

                : ((GoogleStorageService) service).getBucketLocation(bucketName) );

            if (TARGET_SERVICE_S3.equals(getTargetService())) {

                assertEquals(null, bucketLocation); // For S3, the default/null location is literally null

            } else {

                assertEquals("US", bucketLocation); // For GS, "US" is default/null location

            }

        } finally {

            service.deleteBucket(bucketName);

        }

        // Create bucket in alternate location

        bucketName = getBucketNameForTest("testBucketLocations-test2");

        try {

            StorageBucket newBucket = null;

            String targetLocation = null;

            if (TARGET_SERVICE_S3.equals(getTargetService())) {

                targetLocation = S3Bucket.LOCATION_US_WEST;

                newBucket = new S3Bucket(bucketName, targetLocation);

            } else {

                targetLocation = GSBucket.LOCATION_EUROPE;

                newBucket = new GSBucket(bucketName, targetLocation);

            }

            service.createBucket(newBucket);

            assertEquals(StorageService.BUCKET_STATUS__MY_BUCKET, service.checkBucketStatus(bucketName));

            String bucketLocation = (service instanceof S3Service

                ? ((S3Service) service).getBucketLocation(bucketName)

                : ((GoogleStorageService) service).getBucketLocation(bucketName) );

            assertEquals(targetLocation, bucketLocation);

        } finally {

            service.deleteBucket(bucketName);

        }

    }

    public void testObjectManagement() throws Exception {

        String bucketName = createBucketForTest("testObjectManagement").getName();

        RestStorageService service = getStorageService(getCredentials());

        try {

            StorageObject object = new StorageObject("TestObject");

            try {

                service.putObject((String) null, null);

                fail("Cannot create an object without a valid bucket");

            } catch (ServiceException e) {

            }

            try {

                service.putObject((String) null, object);

                fail("Cannot create an object without a valid bucket");

            } catch (ServiceException e) {

            }

            try {

                service.putObject(bucketName, new StorageObject());

                fail("Cannot create an object without a valid object");

            } catch (ServiceException e) {

            }

            // Create basic object with no content type (use the default) and no data.

            StorageObject basicObject = service.putObject(bucketName, object);

            // Ensure Content-Type is set to binary by default

            // TODO: Google Storage bug: Content type returned on initial PUT is always "text/html"

            if (!TARGET_SERVICE_GS.equals(getTargetService())) {

                assertTrue("Unexpected default content type",

                    Mimetypes.MIMETYPE_OCTET_STREAM.equals(basicObject.getContentType()));

            }

            // Re-retrieve object to ensure it was correctly created.

            basicObject = service.getObject(bucketName, object.getKey());

            assertEquals("Unexpected content type",

                Mimetypes.MIMETYPE_OCTET_STREAM, basicObject.getContentType());

            assertEquals("Unexpected size for 'empty' object", 0, basicObject.getContentLength());

            basicObject.closeDataInputStream();

            // Make sure bucket cannot be removed while it has contents.

            try {

                service.deleteBucket(bucketName);

                fail("Should not be able to delete a bucket containing objects");

            } catch (ServiceException e) {

            }

            // Update/overwrite object with real data content and some metadata.

            String contentType = "text/plain";

            String objectData = "Just some rubbish text to include as data";

            String dataMd5HashAsHex = ServiceUtils.toHex(

                ServiceUtils.computeMD5Hash(objectData.getBytes()));

            HashMap<String, Object> metadata = new HashMap<String, Object>();

            metadata.put("creator", "testObjectManagement");

            metadata.put("purpose", "For testing purposes");

            object.replaceAllMetadata(metadata);

            object.setContentType(contentType);

            object.setDataInputStream(new ByteArrayInputStream(objectData.getBytes()));

            StorageObject dataObject = service.putObject(bucketName, object);

            // TODO: Google Storage bug: Content type returned on initial PUT is always "text/html"

            if (TARGET_SERVICE_GS.equals(getTargetService())) {

                dataObject = service.getObject(bucketName, object.getKey());

            }

            assertEquals("Unexpected content type", contentType, dataObject.getContentType());

            assertEquals("Mismatching MD5 hex hash", dataMd5HashAsHex, dataObject.getETag());

            // Retrieve data object to ensure it was correctly created, the server-side hash matches

            // what we expect, and we get our metadata back.

            dataObject = service.getObject(bucketName, object.getKey());

            assertEquals("Unexpected default content type", "text/plain", dataObject.getContentType());

            // TODO: Google Storage doesn't reliably return Content-Length in a GET

            if (!TARGET_SERVICE_GS.equals(getTargetService())) {

                assertEquals("Unexpected content-length for object",

                    objectData.length(), dataObject.getContentLength());

            }

            assertEquals("Mismatching hash", dataMd5HashAsHex, dataObject.getETag());

            // Check user's data are available in basic metadata map

            assertEquals("Missing creator metadata", "testObjectManagement",

                dataObject.getMetadata("creator"));

            assertEquals("Missing purpose metadata", "For testing purposes",

                dataObject.getMetadata("purpose"));

            // Check data are available in user metadata map

            assertEquals("Missing creator user metadata",

                "testObjectManagement", dataObject.getUserMetadataMap().get("creator"));

            assertEquals("Missing purpose user metadata",

                "For testing purposes", dataObject.getUserMetadataMap().get("purpose"));

            assertNotNull("Expected data input stream to be available", dataObject.getDataInputStream());

            // Check data are available in service metadata map

            assertNotNull(dataObject.getServiceMetadataMap().get("request-id"));

            // Ensure we can get the data from S3.

            StringBuffer sb = new StringBuffer();

            int b = -1;

            while ((b = dataObject.getDataInputStream().read()) != -1) {

                sb.append((char) b);

            }

            dataObject.closeDataInputStream();

            assertEquals("Mismatching data", objectData, sb.toString());

            // Retrieve only HEAD of data object (all metadata is available, but not the object content

            // data input stream)

            dataObject = service.getObjectDetails(bucketName, object.getKey());

            assertEquals("Unexpected default content type", "text/plain", dataObject.getContentType());

            assertEquals("Mismatching hash", dataMd5HashAsHex, dataObject.getETag());

            assertEquals("Missing creator metadata", "testObjectManagement",

                dataObject.getMetadata("creator"));

            assertEquals("Missing purpose metadata", "For testing purposes",

                dataObject.getMetadata("purpose"));

            assertNull("Expected data input stream to be unavailable", dataObject.getDataInputStream());

            assertEquals("Unexpected size for object", objectData.length(), dataObject.getContentLength());

            // Test object GET constraints.

            Calendar objectCreationTimeCal = Calendar.getInstance(TimeZone.getTimeZone("GMT"), Locale.US);

            objectCreationTimeCal.setTime(dataObject.getLastModifiedDate());

            Calendar yesterday = (Calendar) objectCreationTimeCal.clone();

            yesterday.add(Calendar.DAY_OF_YEAR, -1);

            Calendar tomorrow = (Calendar) objectCreationTimeCal.clone();

            tomorrow.add(Calendar.DAY_OF_YEAR, +2);

            // Precondition: Modified since yesterday

            service.getObjectDetails(bucketName, object.getKey(), yesterday, null, null, null);

            // Precondition: Mot modified since after creation date.

            try {

                service.getObjectDetails(bucketName, object.getKey(), objectCreationTimeCal, null, null, null);

                fail("Cannot have been modified since object was created");

            } catch (ServiceException e) { }

            // Precondition: Not modified since yesterday

            try {

                service.getObjectDetails(bucketName, object.getKey(), null, yesterday, null, null);

                fail("Cannot be unmodified since yesterday");

            } catch (ServiceException e) { }

            // Precondition: Not modified since tomorrow

            service.getObjectDetails(bucketName, object.getKey(), null, tomorrow, null, null);

            // Precondition: matches correct hash

            service.getObjectDetails(bucketName, object.getKey(), null, null, new String[] {dataMd5HashAsHex}, null);

            // Precondition: doesn't match incorrect hash

            try {

                service.getObjectDetails(bucketName, object.getKey(), null, null,

                    new String[] {"__" + dataMd5HashAsHex.substring(2)}, null);

                fail("Hash values should not match");

            } catch (ServiceException e) {

            }

            // Precondition: doesn't match correct hash

            try {

                service.getObjectDetails(bucketName, object.getKey(), null, null, null, new String[] {dataMd5HashAsHex});

                fail("Hash values should mis-match");

            } catch (ServiceException e) {

            }

            // Precondition: doesn't match incorrect hash

            service.getObjectDetails(bucketName, object.getKey(), null, null, null,

                new String[] {"__" + dataMd5HashAsHex.substring(2)});

            // Retrieve only a limited byte-range of the data, with a start and end.

            Long byteRangeStart = new Long(3);

            Long byteRangeEnd = new Long(12);

            dataObject = service.getObject(bucketName, object.getKey(), null, null, null, null, byteRangeStart, byteRangeEnd);

            String dataReceived = ServiceUtils.readInputStreamToString(

                dataObject.getDataInputStream(), Constants.DEFAULT_ENCODING);

            String dataExpected = objectData.substring(byteRangeStart.intValue(), byteRangeEnd.intValue() + 1);

            assertEquals("Mismatching data from range precondition", dataExpected, dataReceived);

            // Retrieve only a limited byte-range of the data, with a start range only.

            byteRangeStart = new Long(7);

            byteRangeEnd = null;

            dataObject = service.getObject(bucketName, object.getKey(), null, null, null, null, byteRangeStart, byteRangeEnd);

            dataReceived = ServiceUtils.readInputStreamToString(

                dataObject.getDataInputStream(), Constants.DEFAULT_ENCODING);

            dataExpected = objectData.substring(byteRangeStart.intValue());

            assertEquals("Mismatching data from range precondition", dataExpected, dataReceived);

            // Retrieve only a limited byte-range of the data, with an end range only.

            byteRangeStart = null;

            byteRangeEnd = new Long(13);

            dataObject = service.getObject(bucketName, object.getKey(), null, null, null, null, byteRangeStart, byteRangeEnd);

            dataReceived = ServiceUtils.readInputStreamToString(

                dataObject.getDataInputStream(), Constants.DEFAULT_ENCODING);

            dataExpected = objectData.substring(objectData.length() - byteRangeEnd.intValue());

            assertEquals("Mismatching data from range precondition", dataExpected, dataReceived);

            // Clean-up.

            service.deleteObject(bucketName, object.getKey());

            // Create object with tricky key.

            String trickyKey = "http://example.site.com/some/path/document name.html?param1=a@b#c$d&param2=(089)";

            StorageObject trickyObject = service.putObject(bucketName,

                new StorageObject(trickyKey, "Some test data"));

            assertEquals("Tricky key name mistmatch", trickyKey, trickyObject.getKey());

            // Make sure the tricky named object really exists with its full name.

            StorageObject[] objects = service.listObjects(bucketName);

            boolean trickyNamedObjectExists = false;

            for (int i = 0; !trickyNamedObjectExists && i < objects.length; i++) {

                if (trickyKey.equals(objects[i].getKey())) {

                    trickyNamedObjectExists = true;

                }

            }

            assertTrue("Tricky key name object does not exist with its full name", trickyNamedObjectExists);

            // Delete object with tricky key.

            service.deleteObject(bucketName, trickyObject.getKey());

        } finally {

            cleanupBucketForTest("testObjectManagement");

        }

    }

    public void testDirectoryPlaceholderObjects() throws Exception {

        String bucketName = createBucketForTest("testDirectoryPlaceholderObjects").getName();

        RestStorageService service = getStorageService(getCredentials());

        try {

            // Create new-style place-holder object (compatible with Amazon's AWS Console

            // and Panic's Transmit) -- note trailing slash

            StorageObject requestObject = new StorageObject("DirPlaceholderObject/");

            requestObject.setContentLength(0);

            requestObject.setContentType(Mimetypes.MIMETYPE_BINARY_OCTET_STREAM);

            assertTrue(requestObject.isDirectoryPlaceholder());

            service.putObject(bucketName, requestObject);

            StorageObject resultObject = service.getObjectDetails(bucketName, requestObject.getKey());

            assertTrue(resultObject.isDirectoryPlaceholder());

            // Create legacy-style place-holder object (compatible with objects stored using

            // JetS3t applications prior to version 0.8.0) -- note content type

            requestObject = new StorageObject("LegacyDirPlaceholderObject");

            requestObject.setContentLength(0);

            requestObject.setContentType(Mimetypes.MIMETYPE_JETS3T_DIRECTORY);

            assertTrue(requestObject.isDirectoryPlaceholder());

            service.putObject(bucketName, requestObject);

            resultObject = service.getObjectDetails(bucketName, requestObject.getKey());

            assertTrue(resultObject.isDirectoryPlaceholder());

            // Create place-holder object compatible with the S3 Organizer Firefox extension

            // -- note object name suffix.

            requestObject = new StorageObject("S3OrganizerDirPlaceholderObject_$folder$");

            requestObject.setContentLength(0);

            assertTrue(requestObject.isDirectoryPlaceholder());

            service.putObject(bucketName, requestObject);

            resultObject = service.getObjectDetails(bucketName, requestObject.getKey());

            assertTrue(resultObject.isDirectoryPlaceholder());

        } finally {

            cleanupBucketForTest("testDirectoryPlaceholderObjects");

        }

    }

    public void testCopyObjects() throws Exception {

        String sourceBucketName = createBucketForTest("testCopyObjects-source").getName();

        String targetBucketName = createBucketForTest("testCopyObjects-target").getName();

        RestStorageService service = getStorageService(getCredentials());

        try {

            // Create source objects to copy, with potentially troublesome names

            String[] objectNames = new String[] {

                "testing.txt",

                "test me.txt",

                "virtual-path/testing.txt",

                "vîrtüál-πå†h/tés†ing.txt"

            };

            for (int i = 0; i < objectNames.length; i++) {

                StorageObject object = new StorageObject(

                    objectNames[i], "A little data");

                object.addMetadata("object-offset", "" + i);

                service.putObject(sourceBucketName, object);

            }

            // Copy objects within bucket, retaining metadata

            String targetPath = "copies/";

            for (String objectName: objectNames) {

                StorageObject targetObject = new StorageObject(

                    targetPath + objectName);

                service.copyObject(sourceBucketName, objectName,

                    sourceBucketName, targetObject,

                    false // replaceMetadata

                    );

            }

            // Ensure objects are in target location and have the same metadata

            for (int i = 0; i < objectNames.length; i++) {

                StorageObject object = service.getObjectDetails(

                    sourceBucketName, targetPath + objectNames[i]);

                assertEquals("" + i, object.getMetadata("object-offset"));

            }

            // Copy object within bucket, replacing metadata

            StorageObject targetObject = new StorageObject(

                targetPath + objectNames[0]);

            targetObject.addMetadata("replaced-metadata", "booyah!");

            service.copyObject(sourceBucketName, objectNames[0],

                sourceBucketName, targetObject,

                true // replaceMetadata

                );

            StorageObject copiedObject = service.getObjectDetails(

                sourceBucketName, targetObject.getName());

            assertNull(copiedObject.getMetadata("object-offset"));

            assertEquals("booyah!", copiedObject.getMetadata("replaced-metadata"));

            // Copy objects between buckets

            List<Map<String, Object>> copyResults =

                new ArrayList<Map<String, Object>>();

            for (String objectName: objectNames) {

                targetObject = new StorageObject(objectName);

                Map<String, Object> copyResult = service.copyObject(

                    sourceBucketName, objectName,

                    targetBucketName, targetObject,

                    false // replaceMetadata

                    );

                copyResults.add(copyResult);

                copiedObject = service.getObjectDetails(

                    targetBucketName, targetObject.getName());

            }

            assertEquals(4, service.listObjects(targetBucketName).length);

            // Check results map from copy operation

            Map<String, Object> firstCopyResult = copyResults.get(0);

            assertEquals("\"5d11eb8a313fc3e205fef245b7be06c7\"",

                firstCopyResult.get("ETag"));

            assertEquals(sourceBucketName,

                firstCopyResult.get("X-JetS3t-SourceBucketName"));

            assertEquals(objectNames[0],

                firstCopyResult.get("X-JetS3t-SourceObjectKey"));

            assertEquals(targetBucketName,

                firstCopyResult.get("X-JetS3t-DestinationBucketName"));

            assertEquals(objectNames[0], // Note same source and target key

                firstCopyResult.get("X-JetS3t-DestinationObjectKey"));

            assertNull(

                firstCopyResult.get("X-JetS3t-VersionId"));

            assertNull(

                firstCopyResult.get("X-JetS3t-DestinationObjectStorageClass"));

            assertNull(

                firstCopyResult.get("X-JetS3t-DestinationObjectServerSideEncryptionAlgorithm"));

            // Rename convenience method

            int objectOffset = 3;

            targetObject = new StorageObject("my-new-name");

            service.renameObject(

                sourceBucketName, objectNames[objectOffset], targetObject);

            copiedObject = service.getObjectDetails(

                sourceBucketName, targetObject.getName());

            // Ensure we have a new object with the same metadata

            assertEquals("my-new-name", copiedObject.getKey());

            assertEquals("" + objectOffset, copiedObject.getMetadata("object-offset"));

            // Update metadata convenience method

            objectOffset = 2;

            targetObject = new StorageObject(objectNames[objectOffset]);

            targetObject.addMetadata("object-offset", "" + objectOffset); // Unchanged

            targetObject.addMetadata("was-i-updated", "yes!");

            service.updateObjectMetadata(sourceBucketName, targetObject);

            copiedObject = service.getObjectDetails(

                sourceBucketName, targetObject.getName());

            // Ensure we have the same object with updated metadata

            assertEquals("yes!", copiedObject.getUserMetadataMap().get("was-i-updated"));

            // Move object convenience method - retain metadata

            objectOffset = 0;

            targetObject = new StorageObject(objectNames[objectOffset]);

            service.moveObject(sourceBucketName, objectNames[objectOffset],

                targetBucketName, targetObject, false);

            try {

                service.getObjectDetails(

                    sourceBucketName, objectNames[objectOffset]);

                fail("Source object should be moved");

            } catch (ServiceException e) {

                // Expected

            }

            copiedObject = service.getObjectDetails(

                targetBucketName, targetObject.getName());

            assertEquals("" + objectOffset, copiedObject.getUserMetadataMap().get("object-offset"));

            // Move object convenience method - replace metadata

            objectOffset = 1;

            targetObject = new StorageObject(objectNames[objectOffset]);

            targetObject.addMetadata("was-i-moved-with-new-metadata", "yes!");

            service.moveObject(sourceBucketName, objectNames[objectOffset],

                targetBucketName, targetObject, true);

            try {

                service.getObjectDetails(

                    sourceBucketName, objectNames[objectOffset]);

                fail("Source object should be moved");

            } catch (ServiceException e) {

                // Expected

            }

            copiedObject = service.getObjectDetails(

                targetBucketName, targetObject.getName());

            assertNull(copiedObject.getMetadata("object-offset"));

            assertEquals("yes!", copiedObject.getMetadata("was-i-moved-with-new-metadata"));

        } finally {

            cleanupBucketForTest("testCopyObjects-source");

            cleanupBucketForTest("testCopyObjects-target");

        }

    }

    public void testUnicodeData() throws Exception {

        String bucketName = createBucketForTest("testUnicodeData").getName();

        RestStorageService service = getStorageService(getCredentials());

        try {

            // Unicode object name

            String unicodeText = "テストオブジェクト";

            StorageObject requestObject = new StorageObject("1." + unicodeText);

            service.putObject(bucketName, requestObject);

            StorageObject resultObject = service.getObjectDetails(bucketName, requestObject.getKey());

            assertEquals("1." + unicodeText, resultObject.getKey());

            // Unicode data content

            requestObject = new StorageObject("2." + unicodeText, unicodeText);

            service.putObject(bucketName, requestObject);

            resultObject = service.getObject(bucketName, requestObject.getKey());

            String data = ServiceUtils.readInputStreamToString(

                resultObject.getDataInputStream(), "UTF-8");

            assertEquals(unicodeText, data);

            // Unicode metadata values are not supported

            requestObject = new StorageObject("3." + unicodeText);

            requestObject.addMetadata("testing", unicodeText);

            try {

                service.putObject(bucketName, requestObject);

            } catch (ServiceException e) {

            }

            // Unicode metadata values can be encoded

            requestObject = new StorageObject("4." + unicodeText);

            requestObject.addMetadata("testing", URLEncoder.encode(unicodeText, "UTF-8"));

            service.putObject(bucketName, requestObject);

            resultObject = service.getObjectDetails(bucketName, requestObject.getKey());

            assertEquals(unicodeText, URLDecoder.decode(

                (String) resultObject.getMetadata("testing"), "UTF-8"));

            // Unicode metadata names are not possible with HTTP

            requestObject = new StorageObject("5." + unicodeText);

            requestObject.addMetadata(unicodeText, "value");

            try {

                service.putObject(bucketName, requestObject);

                fail("Illegal to use non-ASCII characters in HTTP headers");

            } catch (ServiceException e) {

            }

            // Unicode HTTP headers (via RFC 5987 encoding) -- not working...

            /*

            requestObject = new StorageObject("6." + unicodeText);

            requestObject.setContentDisposition(

                "attachment; filename*=UTF-8''" + RestUtils.encodeUrlString(unicodeText + ".txt"));

            service.putObject(bucketName, requestObject);

            resultObject = service.getObjectDetails(bucketName, requestObject.getKey());

            assertEquals(

                "attachment; filename=" + unicodeText + "", resultObject.getContentDisposition());

            */

        } finally {

            cleanupBucketForTest("testUnicodeData");

        }

    }

    public void testACLManagement() throws Exception {

        // Access public-readable third-party bucket: jets3t

        RestStorageService anonymousS3Service = getStorageService(null);

        boolean jets3tBucketAvailable = anonymousS3Service.isBucketAccessible("jets3t");

        assertTrue("Cannot find public jets3t bucket", jets3tBucketAvailable);

        RestStorageService service = getStorageService(getCredentials());

        // Use Google- or S3-specific URL endpoint to lookup objects, depending on the target service

        String linkUrlPrefix = null;

        if (TARGET_SERVICE_GS.equals(getTargetService())) {

            linkUrlPrefix = "https://commondatastorage.googleapis.com";

        } else {

            linkUrlPrefix = "https://s3.amazonaws.com";

        }

        // Use Google- or S3-specific ACL elements depending on which service class we're using

        GranteeInterface allUsersGrantee = null;

        if (service instanceof GoogleStorageService) {

            allUsersGrantee = new AllUsersGrantee();

        } else {

            allUsersGrantee = GroupGrantee.ALL_USERS;

        }

        StorageBucket bucket = createBucketForTest("testACLManagement");

        String bucketName = bucket.getName();

        StorageObject object = null;

        try {

            // Create private object (default permissions).

            String privateKey = "Private Object - " + System.currentTimeMillis();

            object = new StorageObject(privateKey, "Private object sample text");

            service.putObject(bucketName, object);

            URL url = new URL(linkUrlPrefix + "/" + bucketName + "/" + RestUtils.encodeUrlString(privateKey));

            assertEquals("Expected denied access (403) error", 403, ((HttpURLConnection) url

                .openConnection()).getResponseCode());

            // Get ACL details for private object so we can determine the account owner ID.

            AccessControlList objectACL = service.getObjectAcl(bucketName, privateKey);

            StorageOwner accountOwner = objectACL.getOwner();

            // Create a public object.

            String publicKey = "Public Object - " + System.currentTimeMillis();

            object = new StorageObject(publicKey, "Public object sample text");

            AccessControlList acl = buildAccessControlList();

            acl.setOwner(accountOwner);

            acl.grantPermission(allUsersGrantee, Permission.PERMISSION_READ);

            object.setAcl(acl);

            service.putObject(bucketName, object);

            url = new URL(linkUrlPrefix + "/" + bucketName + "/" + RestUtils.encodeUrlString(publicKey));

            assertEquals("Expected access (200)",

                    200, ((HttpURLConnection)url.openConnection()).getResponseCode());

            // Update ACL to make private object public.

            AccessControlList privateToPublicACL = service.getObjectAcl(bucketName, privateKey);

            privateToPublicACL.grantPermission(allUsersGrantee, Permission.PERMISSION_READ);

            object.setKey(privateKey);

            object.setAcl(privateToPublicACL);

            service.putObjectAcl(bucketName, object);

            url = new URL(linkUrlPrefix + "/" + bucketName + "/" + RestUtils.encodeUrlString(privateKey));

            assertEquals("Expected access (200)", 200, ((HttpURLConnection) url.openConnection())

                .getResponseCode());

            // Create a non-standard uncanned public object.

            String publicKey2 = "Public Object - " + System.currentTimeMillis();

            object = new StorageObject(publicKey2);

            object.setAcl(privateToPublicACL); // This ACL has ALL_USERS READ permission set above.

            service.putObject(bucketName, object);

            url = new URL(linkUrlPrefix + "/" + bucketName + "/" + RestUtils.encodeUrlString(publicKey2));

            assertEquals("Expected access (200)", 200, ((HttpURLConnection) url.openConnection())

                .getResponseCode());

            // Update ACL to make public object private.

            AccessControlList publicToPrivateACL = service.getObjectAcl(bucketName, publicKey);

            publicToPrivateACL.revokeAllPermissions(allUsersGrantee);

            object.setKey(publicKey);

            object.setAcl(publicToPrivateACL);

            service.putObjectAcl(bucketName, object);

            // TODO: Google Storage quirk: It may take some time for public object to become private again

            if (TARGET_SERVICE_GS.equals(getTargetService())) {

                // Confirm changes were applied on object's ACL, because we don't know

                // how long to wait until the object will really become private again.

                AccessControlList updatedAcl = service.getObjectAcl(bucketName, object.getKey());

                assertFalse(updatedAcl.hasGranteeAndPermission(

                    allUsersGrantee, Permission.PERMISSION_READ));

                assertEquals(0, updatedAcl.getPermissionsForGrantee(allUsersGrantee).size());

            } else {

                // In S3, objects are made private immediately.

                url = new URL(linkUrlPrefix + "/" + bucketName + "/" + RestUtils.encodeUrlString(publicKey));

                assertEquals("Expected denied access (403) error", 403, ((HttpURLConnection) url

                    .openConnection()).getResponseCode());

            }

            // Clean-up.

            service.deleteObject(bucketName, privateKey);

            service.deleteObject(bucketName, publicKey);

            service.deleteObject(bucketName, publicKey2);

        } finally {

            cleanupBucketForTest("testACLManagement");

        }

    }

    public void testACLManagementViaRestHeaders() throws Exception {

        RestStorageService service = getStorageService(getCredentials());

        StorageBucket bucket = createBucketForTest("testACLManagementViaRestHeaders");

        AccessControlList publicHeaderAcl = null;

        if (service instanceof GoogleStorageService) {

            publicHeaderAcl = GSAccessControlList.REST_CANNED_PUBLIC_READ;

        } else {

            publicHeaderAcl = AccessControlList.REST_CANNED_PUBLIC_READ;

        }

        try {

            // Try to create public object using HTTP header ACL settings.

            String publicKey = "PublicObject";

            StorageObject object = new StorageObject(publicKey);

            object.setAcl(publicHeaderAcl);

            object.setOwner(bucket.getOwner());

            try {

                service.putObject(bucket.getName(), object);

                URL url = new URL("https://" + service.getEndpoint()

                    + "/" + bucket.getName() + "/" + publicKey);

                assertEquals("Expected public access (200)",

                        200, ((HttpURLConnection)url.openConnection()).getResponseCode());

            } finally {

                service.deleteObject(bucket.getName(), object.getKey());

            }

        } finally {

            cleanupBucketForTest("testACLManagementViaRestHeaders");

        }

    }

    public void testObjectListing() throws Exception {

        RestStorageService service = getStorageService(getCredentials());

        StorageBucket bucket = createBucketForTest("testObjectListing");

        String bucketName = bucket.getName();

        try {

            // Represent a directory structure in S3.

            List<StorageObject> objectsList = new ArrayList<StorageObject>();

            objectsList.add(new StorageObject("dir1"));

            objectsList.add(new StorageObject("dir1/doc1Level1"));

            objectsList.add(new StorageObject("dir1/doc2level1"));

            objectsList.add(new StorageObject("dir1/dir1Level1"));

            objectsList.add(new StorageObject("dir1/dir1Level1/doc1Level2"));

            objectsList.add(new StorageObject("dir1/dir1Level1/dir1Level2"));

            objectsList.add(new StorageObject("dir1/dir1Level1/dir1Level2/doc1Level3"));

            // Create objects

            for (StorageObject object: objectsList) {

                service.putObject(bucketName, object);

            }

            StorageObject[] objects = null;

            // List all items in directory.

            objects = service.listObjects(bucketName);

            assertEquals("Incorrect number of objects in directory structure",

                objectsList.size(), objects.length);

            // Check all objects have bucket name property set

            for (StorageObject object: objects) {

                assertEquals(bucketName, object.getBucketName());

            }

            // List items in chunks of size 2, ensure we get a total of seven.

            int chunkedObjectsCount = 0;

            int chunkedIterationsCount = 0;

            String priorLastKey = null;

            do {

                StorageObjectsChunk chunk = service.listObjectsChunked(

                    bucketName, null, null, 2, priorLastKey);

                priorLastKey = chunk.getPriorLastKey();

                chunkedObjectsCount += chunk.getObjects().length;

                chunkedIterationsCount++;

            } while (priorLastKey != null);

            assertEquals("Chunked bucket listing retreived incorrect number of objects",

                objectsList.size(), chunkedObjectsCount);

            assertEquals("Chunked bucket listing ran for an unexpected number of iterations",

                (objectsList.size() + 1) / 2, chunkedIterationsCount);

            // List objects with a prefix and delimiter to check common prefixes.

            StorageObjectsChunk chunk = service.listObjectsChunked(

                bucketName, "dir1/", "/", 100, null);

            assertEquals("Chunked bucket listing with prefix and delimiter retreived incorrect number of objects",

                3, chunk.getObjects().length);

            assertEquals("Chunked bucket listing with prefix and delimiter retreived incorrect number of common prefixes",

                1, chunk.getCommonPrefixes().length);

            // List the same items with a prefix.

            objects = service.listObjects(bucketName, "dir1", null);

            assertEquals("Incorrect number of objects matching prefix", 7, objects.length);

            // List items up one directory with a prefix (will include dir1Level1)

            objects = service.listObjects(bucketName, "dir1/dir1Level1", null);

            assertEquals("Incorrect number of objects matching prefix", 4, objects.length);

            // List items up one directory with a prefix (will not include dir1Level1)

            objects = service.listObjects(bucketName, "dir1/dir1Level1/", null);

            assertEquals("Incorrect number of objects matching prefix", 3, objects.length);

            // Try a prefix matching no object keys.

            objects = service.listObjects(bucketName, "dir1-NonExistent", null);

            assertEquals("Expected no results", 0, objects.length);

            // Use delimiter with an partial prefix.

            objects = service.listObjects(bucketName, "dir", "/");

            assertEquals("Expected no results", 1, objects.length);

            // Use delimiter to find item dir1 only.

            objects = service.listObjects(bucketName, "dir1", "/");

            assertEquals("Incorrect number of objects matching prefix and delimiter", 1, objects.length);

            // Use delimiter to find items within dir1 only.

            objects = service.listObjects(bucketName, "dir1/", "/");

            assertEquals("Incorrect number of objects matching prefix and delimiter", 3, objects.length);

            // List items up one directory with prefix and delimiter (will include only dir1Level1)

            objects = service.listObjects(bucketName, "dir1/dir1Level1", "/");

            assertEquals("Incorrect number of objects matching prefix", 1, objects.length);

            // List items up one directory with prefix and delimiter (will include only contents of dir1Level1)

            objects = service.listObjects(bucketName, "dir1/dir1Level1/", "/");

            assertEquals("Incorrect number of objects matching prefix", 2, objects.length);

            // Clean up.

            for (StorageObject object: objectsList) {

                service.deleteObject(bucketName, object.getKey());

            }

        } finally {

            cleanupBucketForTest("testObjectListing");

        }

    }

    public void testHashVerifiedUploads() throws Exception {

        RestStorageService service = getStorageService(getCredentials());

        StorageBucket bucket = createBucketForTest("testHashVerifiedUploads");

        String bucketName = bucket.getName();

        try {

            // Create test object with an MD5 hash of the data.

            String dataString = "Text for MD5 hashing...";

            StorageObject object = new StorageObject("Testing MD5 Hashing", dataString);

            object.setContentType("text/plain");

            // Calculate hash data for object.

            byte[] md5Hash = ServiceUtils.computeMD5Hash(dataString.getBytes());

            // Ensure that using an invalid hash value fails.

            try {

                object.addMetadata("Content-MD5", "123");

                service.putObject(bucketName, object);

                fail("Should have failed due to invalid hash value");

            } catch (ServiceException e) {

                assertTrue("Expected error code indicating invalid md5 hash",

                    "InvalidDigest".equals(e.getErrorCode())  // S3 error code

                    || "BadDigest".equals(e.getErrorCode())   // GS error code

                    );

            }

            object = new StorageObject("Testing MD5 Hashing", dataString);

            // Ensure that using the wrong hash value fails.

            try {

                byte[] incorrectHash = new byte[md5Hash.length];

                System.arraycopy(md5Hash, 0, incorrectHash, 0, incorrectHash.length);

                incorrectHash[0] = incorrectHash[1];

                object.setMd5Hash(incorrectHash);

                service.putObject(bucketName, object);

                fail("Should have failed due to incorrect hash value");

            } catch (ServiceException e) {

                assertEquals("Expected error code indicating invalid md5 hash", "BadDigest", e.getErrorCode());

            }

            object = new StorageObject("Testing MD5 Hashing", dataString);

            // Ensure that correct hash value succeeds.

            object.setMd5Hash(md5Hash);

            StorageObject resultObject = service.putObject(bucketName, object);

            // Ensure the ETag result matches the hex-encoded MD5 hash.

            assertEquals("Hex-encoded MD5 hash should match ETag", resultObject.getETag(),

                ServiceUtils.toHex(md5Hash));

            // Ensure we can convert the hex-encoded ETag to Base64 that matches the Base64 md5 hash.

            String md5HashBase64 = ServiceUtils.toBase64(md5Hash);

            String eTagBase64 = ServiceUtils.toBase64(ServiceUtils.fromHex(resultObject.getETag()));

            assertEquals("Could not convert ETag and MD5 hash to matching Base64-encoded strings",

                md5HashBase64, eTagBase64);

            // Clean up.

            service.deleteObject(bucketName, object.getKey());

        } finally {

            cleanupBucketForTest("testHashVerifiedUploads");

        }

    }

    public void testIsObjectInBucket() throws Exception {

        RestStorageService service = getStorageService(getCredentials());

        StorageBucket bucket = createBucketForTest("testIsObjecInBucket");

        String bucketName = bucket.getName();

        try {

            service.putObject(bucketName, new StorageObject("does-exist"));

            assertTrue(service.isObjectInBucket(bucketName, "does-exist"));

            assertFalse(service.isObjectInBucket(bucketName, "does-not-exist"));

        } finally {

            cleanupBucketForTest("testIsObjecInBucket");

        }

    }

    public void testThreadedStorageService() throws Exception {

        RestStorageService service = getStorageService(getCredentials());

        StorageBucket bucket = createBucketForTest("testThreadedStorageService");

        String bucketName = bucket.getName();

        try {

            final int[] createObjectsEventCount = new int[] {0};

            final int[] getObjectHeadsEventCount = new int[] {0};

            final List<StorageObject> getObjectsList = new ArrayList<StorageObject>();

            final int[] deleteObjectsEventCount = new int[] {0};

            // Multi-threaded service with adaptor to count event occurrences.

            ThreadedStorageService threadedService = new ThreadedStorageService(

                service,

                new StorageServiceEventAdaptor() {

                    @Override

                    public void event(CreateObjectsEvent event) {

                        if (CreateObjectsEvent.EVENT_IN_PROGRESS == event.getEventCode()) {

                            createObjectsEventCount[0] += event.getCreatedObjects().length;

                        }

                    }

                    @Override

                    public void event(GetObjectHeadsEvent event) {

                        if (GetObjectHeadsEvent.EVENT_IN_PROGRESS == event.getEventCode()) {

                            getObjectHeadsEventCount[0] += event.getCompletedObjects().length;

                        }

                    }

                    @Override

                    public void event(GetObjectsEvent event) {

                        if (GetObjectsEvent.EVENT_IN_PROGRESS == event.getEventCode()) {

                            for (StorageObject object: event.getCompletedObjects()) {