Package org.jets3t.service

Examples of org.jets3t.service.S3Service


        PrintStream                     info,
        PrintStream                     err,
        PrintStream                     debug)
              throws ExecutionProblem
    {
        S3Service s3Service = null;
        try
        {
            s3Service = this.getService();

            String baseBucketName = this.args.getS3Bucket();
            String keyName = this.makeKey("", null);

            ArrayList files = new ArrayList();
            // first get all of this users objects
            S3Object[] usersVMs = s3Service.listObjects(baseBucketName, keyName, "", 1000);
            s3ObjToFileList(files, usersVMs, true, s3Service);
            S3Object[] VMs = s3Service.listObjects(baseBucketName, this.makeKey("", "common"), "", 1000);
            s3ObjToFileList(files, VMs, false, s3Service);

            return (FileListing[]) files.toArray(new FileListing[files.size()]);
        }
        catch(S3ServiceException s3ex)
View Full Code Here


        }
    }

    public void testS3WebsiteConfig() throws Exception {
        // Testing takes place in the us-west-1 location
        S3Service s3Service = (S3Service) getStorageService(getCredentials());
        StorageBucket bucket = createBucketForTest(
            "testS3WebsiteConfig",
            // Standard US Bucket location
            S3Bucket.LOCATION_US_WEST);
        String bucketName = bucket.getName();

        String s3WebsiteURL = "http://" + bucketName + "."
            // Website location must correspond to bucket location, in this case
            // the US Standard. For website endpoints see:
            // docs.amazonwebservices.com/AmazonS3/latest/dev/WebsiteEndpoints.html
            + "s3-website-us-west-1"
            + ".amazonaws.com";

        try {
            HttpClient httpClient = new DefaultHttpClient();
            HttpGet getMethod;

            // Check no existing website config
            try {
                s3Service.getWebsiteConfig(bucketName);
                fail("Unexpected website config for bucket " + bucketName);
            } catch (S3ServiceException e) {
                assertEquals(404, e.getResponseCode());
            }

            // Set index document
            s3Service.setWebsiteConfig(bucketName,
                new S3WebsiteConfig("index.html"));

            Thread.sleep(5000);

            // Confirm index document set
            S3WebsiteConfig config = s3Service.getWebsiteConfig(bucketName);
            assertTrue(config.isWebsiteConfigActive());
            assertEquals("index.html", config.getIndexDocumentSuffix());
            assertNull(config.getErrorDocumentKey());

            // Upload public index document
            S3Object indexObject = new S3Object("index.html", "index.html contents");
            indexObject.setAcl(AccessControlList.REST_CANNED_PUBLIC_READ);
            s3Service.putObject(bucketName, indexObject);

            // Confirm index document is served at explicit path
            getMethod = new HttpGet(s3WebsiteURL + "/index.html");
            HttpResponse response = httpClient.execute(getMethod);
            assertEquals(200, response.getStatusLine().getStatusCode());
            assertEquals("index.html contents", EntityUtils.toString(response.getEntity()));

            // Confirm index document is served at root path
            // (i.e. website config is effective)
            getMethod = new HttpGet(s3WebsiteURL + "/");
            response = httpClient.execute(getMethod);
            assertEquals(200, response.getStatusLine().getStatusCode());
            assertEquals("index.html contents", EntityUtils.toString(response.getEntity()));

            // Set index document and error document
            s3Service.setWebsiteConfig(bucketName,
                new S3WebsiteConfig("index.html", "error.html"));

            Thread.sleep(10000)// Config updates can take a long time... ugh!

            // Confirm index document and error document set
            config = s3Service.getWebsiteConfig(bucketName);
            assertTrue(config.isWebsiteConfigActive());
            assertEquals("index.html", config.getIndexDocumentSuffix());
            assertEquals("error.html", config.getErrorDocumentKey());

            // Upload public error document
            S3Object errorObject = new S3Object("error.html", "error.html contents");
            errorObject.setAcl(AccessControlList.REST_CANNED_PUBLIC_READ);
            s3Service.putObject(bucketName, errorObject);

            // Confirm error document served at explicit path
            getMethod = new HttpGet(s3WebsiteURL + "/error.html");
            response = httpClient.execute(getMethod);
            assertEquals(200, response.getStatusLine().getStatusCode());
            assertEquals("error.html contents", EntityUtils.toString(response.getEntity()));

            // Confirm error document served instead of 404 Not Found
            getMethod = new HttpGet(s3WebsiteURL + "/does-not-exist");
            response = httpClient.execute(getMethod);
            assertEquals(403, response.getStatusLine().getStatusCode()); // TODO: Why a 403?
            assertEquals("error.html contents", EntityUtils.toString(response.getEntity()));

            // Upload private document
            S3Object privateObject = new S3Object("private.html", "private.html contents");
            s3Service.putObject(bucketName, privateObject);

            // Confirm error document served instead for 403 Forbidden
            getMethod = new HttpGet(s3WebsiteURL + "/private.html");
            response = httpClient.execute(getMethod);
            assertEquals(403, response.getStatusLine().getStatusCode());

            // Delete website config
            s3Service.deleteWebsiteConfig(bucketName);

            Thread.sleep(5000);

            // Confirm website config deleted
            try {
                s3Service.getWebsiteConfig(bucketName);
                fail("Unexpected website config for bucket " + bucketName);
            } catch (S3ServiceException e) { }
        } finally {
            cleanupBucketForTest("testS3WebsiteConfig");
        }
View Full Code Here

        }
    }

    public void testNotificationConfig() throws Exception {
        // Testing takes place in the us-west-1 location
        S3Service s3Service = (S3Service) getStorageService(getCredentials());
        StorageBucket bucket = createBucketForTest("testNotificationConfig");
        String bucketName = bucket.getName();

        try {
            // Check no existing notification config
            NotificationConfig notificationConfig =
                s3Service.getNotificationConfig(bucketName);
            assertEquals(0, notificationConfig.getTopicConfigs().size());

            // Public SNS topic for testing
            String topicArn =
                "arn:aws:sns:us-east-1:916472402845:"
                + "JetS3t-Test-S3-Bucket-NotificationConfig";
            String event = NotificationConfig.EVENT_REDUCED_REDUNDANCY_LOST_OBJECT;

            // Set notification config
            notificationConfig = new NotificationConfig();
            notificationConfig.addTopicConfig(notificationConfig.new TopicConfig(topicArn, event));
            s3Service.setNotificationConfig(bucketName, notificationConfig);

            Thread.sleep(5000);

            // Get notification config
            notificationConfig = s3Service.getNotificationConfig(bucketName);
            assertEquals(1, notificationConfig.getTopicConfigs().size());
            TopicConfig topicConfig = notificationConfig.getTopicConfigs().get(0);
            assertEquals(topicArn, topicConfig.getTopic());
            assertEquals(event, topicConfig.getEvent());

            // Unset/clear notification config
            s3Service.unsetNotificationConfig(bucketName);

            Thread.sleep(5000);

            // Confirm notification config is no longer set
            notificationConfig = s3Service.getNotificationConfig(bucketName);
            assertEquals(0, notificationConfig.getTopicConfigs().size());
        } finally {
            cleanupBucketForTest("testNotificationConfig");
        }
    }
View Full Code Here

            cleanupBucketForTest("testNotificationConfig");
        }
    }

    public void testServerSideEncryption() throws Exception {
        S3Service s3Service = (S3Service) getStorageService(getCredentials());
        StorageBucket bucket = createBucketForTest("testServerSideEncryption");
        String bucketName = bucket.getName();

        try {
            // NONE server-side encryption variable == null
            assertEquals(S3Object.SERVER_SIDE_ENCRYPTION__NONE, null);

            // Create a normal object
            S3Object object = new S3Object("unencrypted-object", "Some data");
            object.setServerSideEncryptionAlgorithm(S3Object.SERVER_SIDE_ENCRYPTION__NONE);
            s3Service.putObject(bucketName, object);
            // Confirm object is not encrypted
            StorageObject objDetails = s3Service.getObjectDetails(bucketName, object.getKey());
            assertEquals(null, objDetails.getServerSideEncryptionAlgorithm());

            // Fail to create an encrypted object, due to invalid algorithm
            object = new S3Object("failed-encrypted-object", "Some data");
            object.setServerSideEncryptionAlgorithm("AES999");
            try {
                s3Service.putObject(bucketName, object);
                fail("Expected error about invalid server-side encryption algorithm");
            } catch (S3ServiceException e) {
                assertEquals("InvalidArgument", e.getErrorCode());
            }

            // Create an encrypted object, set explicitly
            object = new S3Object("encrypted-object", "Some data");
            object.setServerSideEncryptionAlgorithm(S3Object.SERVER_SIDE_ENCRYPTION__AES256);
            s3Service.putObject(bucketName, object);
            // Confirm object is encrypted
            objDetails = s3Service.getObjectDetails(bucketName, object.getKey());
            assertEquals(S3Object.SERVER_SIDE_ENCRYPTION__AES256, objDetails.getMetadata("server-side-encryption"));
            assertEquals(S3Object.SERVER_SIDE_ENCRYPTION__AES256, objDetails.getServerSideEncryptionAlgorithm());

            // Create an encrypted object, per default algorithm set in service properties
            Jets3tProperties properties = new Jets3tProperties();
            properties.setProperty("s3service.server-side-encryption",
                S3Object.SERVER_SIDE_ENCRYPTION__AES256);
            s3Service = (S3Service) getStorageService(getCredentials(), properties);
            object = new S3Object("encrypted-object-as-default", "Some data");
            s3Service.putObject(bucketName, object);
            // Confirm object is encrypted
            objDetails = s3Service.getObjectDetails(bucketName, object.getKey());
            assertEquals(S3Object.SERVER_SIDE_ENCRYPTION__AES256, objDetails.getMetadata("server-side-encryption"));
            assertEquals(S3Object.SERVER_SIDE_ENCRYPTION__AES256, objDetails.getServerSideEncryptionAlgorithm());

        } finally {
            cleanupBucketForTest("testServerSideEncryption");
View Full Code Here

            cleanupBucketForTest("testServerSideEncryption");
        }
    }

    public void testMultipleObjectDelete() throws Exception {
        S3Service s3Service = (S3Service) getStorageService(getCredentials());
        StorageBucket bucket = createBucketForTest("testMultipleObjectDelete");
        String bucketName = bucket.getName();

        try {
            // Can delete non-existent objects
            String[] keys = new String[] {
                "non-existent-1", "non-existent-2", "non-existent-3", "non-existent-4"};
            MultipleDeleteResult result = s3Service.deleteMultipleObjects(bucketName, keys);
            assertFalse(result.hasErrors());
            assertEquals(0, result.getErrorResults().size());
            assertEquals(4, result.getDeletedObjectResults().size());

            // Delete existing objects
            keys = new String[] {
                "existent-1", "existent-2", "existent-3", "existent-4"};
            for (String key: keys) {
                s3Service.putObject(bucketName, new S3Object(key, "Some data"));
            }
            result = s3Service.deleteMultipleObjects(bucketName, keys);
            assertFalse(result.hasErrors());
            assertEquals(4, result.getDeletedObjectResults().size());
            for (String key: keys) {
                assertFalse(s3Service.isObjectInBucket(bucketName, key));
            }

            // Quiet mode does not list deleted objects in result
            ObjectKeyAndVersion[] keyAndVersions = new ObjectKeyAndVersion[keys.length];
            int i = 0;
            for (String key: keys) {
                s3Service.putObject(bucketName, new S3Object(key, "Some data"));
                keyAndVersions[i++] = new ObjectKeyAndVersion(key);
            }
            result = s3Service.deleteMultipleObjects(bucketName, keyAndVersions, true);
            assertFalse(result.hasErrors());
            assertEquals(0, result.getDeletedObjectResults().size());
            for (String key: keys) {
                assertFalse(s3Service.isObjectInBucket(bucketName, key));
            }

        } finally {
            cleanupBucketForTest("testMultipleObjectDelete");
        }
View Full Code Here

            cleanupBucketForTest("testMultipleObjectDelete");
        }
    }

    public void testLifecycleConfiguration() throws Exception {
        S3Service s3Service = (S3Service) getStorageService(getCredentials());
        StorageBucket bucket = createBucketForTest("testBucketLifecycleConfiguration");
        String bucketName = bucket.getName();

        try {
            // No config rules initially for bucket
            LifecycleConfig config = s3Service.getLifecycleConfig(bucketName);
            assertNull(config);

            // Create Rule with Expiration by Days
            LifecycleConfig newConfig = new LifecycleConfig();
            Rule newRule = newConfig.newRule("rule-1", "/nothing", Boolean.TRUE);
            Expiration expiration = newRule.newExpiration();
            expiration.setDays(7);

            // Apply initial Rule
            s3Service.setLifecycleConfig(bucketName, newConfig);
            config = s3Service.getLifecycleConfig(bucketName);
            assertNotNull(config);
            assertEquals(1, config.getRules().size());
            Rule rule = config.getRules().get(0);
            assertEquals("rule-1", rule.getId());
            assertEquals("/nothing", rule.getPrefix());
            assertTrue(rule.getEnabled());
            assertEquals(new Integer(7), rule.getExpiration().getDays());
            assertNull(rule.getExpiration().getDate());
            assertNull(rule.getTransition());

            // Change Expiration to be by Date
            SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-ddZ");
            rule.getExpiration().setDate(sdf.parse("2020-01-01+0000"));
            s3Service.setLifecycleConfig(bucketName, config);
            config = s3Service.getLifecycleConfig(bucketName);
            rule = config.getRules().get(0);
            assertNull(rule.getExpiration().getDays());
            assertEquals(sdf.parse("2020-01-01+0000"), rule.getExpiration().getDate());

            // Add Transition by Date
            Transition transition = rule.newTransition()// Default's to GLACIER storage class
            transition.setDate(sdf.parse("2016-01-01+0000"));
            s3Service.setLifecycleConfig(bucketName, config);
            config = s3Service.getLifecycleConfig(bucketName);
            rule = config.getRules().get(0);
            assertEquals(sdf.parse("2016-01-01+0000"), rule.getTransition().getDate());
            assertNull(rule.getTransition().getDays());
            assertEquals(LifecycleConfig.STORAGE_CLASS_GLACIER,
                rule.getTransition().getStorageClass());

            // Change Transition to be by Days (Expiration must use same time-keeping mechanism)
            rule.getTransition().setDays(3);
            rule.getExpiration().setDays(7);
            s3Service.setLifecycleConfig(bucketName, config);
            config = s3Service.getLifecycleConfig(bucketName);
            rule = config.getRules().get(0);
            assertEquals(new Integer(3), rule.getTransition().getDays());
            assertEquals(new Integer(7), rule.getExpiration().getDays());

            // Add additional Rule
            Rule secondRule = config.newRule("second-rule", "/second", Boolean.FALSE);
            secondRule.newExpiration().setDays(100);
            s3Service.setLifecycleConfig(bucketName, config);
            config = s3Service.getLifecycleConfig(bucketName);
            assertEquals(2, config.getRules().size());
            rule = config.getRules().get(1);
            assertEquals("second-rule", rule.getId());
            assertEquals("/second", rule.getPrefix());
            assertFalse(rule.getEnabled());
            rule = config.getRules().get(1);
            assertEquals(new Integer(100), rule.getExpiration().getDays());
            assertNull(rule.getTransition());

            // Delete config
            s3Service.deleteLifecycleConfig(bucketName);
            config = s3Service.getLifecycleConfig(bucketName);
            assertNull(config);
        } finally {
            cleanupBucketForTest("testBucketLifecycleConfiguration");
        }
    }
View Full Code Here

         */
        final String bucketName = "jets3t";
        final String delimiter = "/";

        AWSCredentials awsCredentials = SamplesUtils.loadAWSCredentials();
        S3Service restService = new RestS3Service(awsCredentials);

        final List allObjects = Collections.synchronizedList(new ArrayList());
        final Exception s3ServiceExceptions[] = new Exception[1];

        /*
         * Identify top-level "subdirectory" names in a bucket by performing a
         * standard object listing with a delimiter string.
         */
        long startTime = System.currentTimeMillis();

        // Find all the objects and common prefixes at the top level.
        StorageObjectsChunk initialChunk = restService.listObjectsChunked(
            bucketName, null, delimiter, 1000, null, true);

        long totalElapsedTime = System.currentTimeMillis() - startTime;

        // We will use the common prefix strings, if any, to perform sub-listings
View Full Code Here

        } else {
            String userToken = args[4];
            String productToken = args[5];
            credentials = new AWSDevPayCredentials(accessKey, secretKey, userToken, productToken);
        }
        S3Service service = new RestS3Service(credentials);

        // Find all current multipart uploads
        List<MultipartUpload> multipartUploads =
            service.multipartListUploads(bucketName);

        // Identify only multipart uploads older than a certain date
        // (to try and avoid killing off an in-progress upload)
        long CUTOFF = System.currentTimeMillis() - (hoursAgo * 60 * 60 * 1000);
        List<MultipartUpload> oldMultipartUploads = new ArrayList<MultipartUpload>();
        for (MultipartUpload multipartUpload: multipartUploads) {
            if (multipartUpload.getInitiatedDate().getTime() < CUTOFF) {
                oldMultipartUploads.add(multipartUpload);
            }
        }
        System.out.println("Of " + multipartUploads.size() + " multipart upload(s) in "
            + bucketName + ", " + oldMultipartUploads.size() + " are older than "
            + hoursAgo + " hours ago");

        // If no candidates for deletion, no work to do
        if (oldMultipartUploads.size() < 1) {
            return;
        }

        // Prompt user to confirm deletion
        System.out.print("About to delete " + oldMultipartUploads.size()
            + " multipart uploads, is this OK? (y/n) ");
        BufferedReader br = new BufferedReader(new InputStreamReader(System.in));
        String response = br.readLine();
        if (!"y".equals(response.toLowerCase()) && !"yes".equals(response.toLowerCase())) {
            System.out.println("Aborting");
            return;
        }

        // Delete old multipart uploads
        for (MultipartUpload multipartUpload: oldMultipartUploads) {
            System.out.print("Deleting (aborting) " + multipartUpload + " ...");
            service.multipartAbortUpload(multipartUpload);
            System.out.println(" done.");
        }
    }
View Full Code Here

    };
  }

  private void downloadThrows(final S3Artifact s3Artifact, final Path downloadTo) throws Exception {
    final S3Service s3 = new RestS3Service(new AWSCredentials(configuration.getS3AccessKey(), configuration.getS3SecretKey()));

    long length = 0;

    if (s3Artifact.getFilesize().isPresent()) {
      length = s3Artifact.getFilesize().get();
    } else {
      StorageObject details = s3.getObjectDetails(s3Artifact.getS3Bucket(), s3Artifact.getS3ObjectKey());

      Preconditions.checkNotNull(details, "Couldn't find object at %s/%s", s3Artifact.getS3Bucket(), s3Artifact.getS3ObjectKey());

      length = details.getContentLength();
    }
View Full Code Here

    @Ensures("result != null")
    protected static GATKRunReport deserializeReport(final String downloaderAccessKey,
                                                     final String downloaderSecretKey,
                                                     final String bucketName,
                                                     final S3Object s3Object) throws Exception {
        final S3Service s3Service = initializeAWSService(downloaderAccessKey, downloaderSecretKey);

        // Retrieve the whole data object we created previously
        final S3Object objectComplete = s3Service.getObject(bucketName, s3Object.getName());

        // Read the data from the object's DataInputStream using a loop, and print it out.
        return deserializeReport(new GZIPInputStream(objectComplete.getDataInputStream()));
    }
View Full Code Here

TOP

Related Classes of org.jets3t.service.S3Service

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.