Package com.amazonaws.services.s3.internal.crypto

Examples of com.amazonaws.services.s3.internal.crypto.EncryptedUploadContext


     *
     * @return AmazonS3Client
     */
    AmazonS3 createS3Client() {
        AWSCredentials credentials = new BasicAWSCredentials(configuration.getAccessKey(), configuration.getSecretKey());
        AmazonS3 client = new AmazonS3Client(credentials);
        return client;
    }
View Full Code Here


  assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").get().getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS));

  Settings settings = internalCluster().getInstance(Settings.class);
  Settings bucket = settings.getByPrefix("repositories.s3.");
  AmazonS3 s3Client = internalCluster().getInstance(AwsS3Service.class).client(
    bucket.get("region", settings.get("repositories.s3.region")),
    bucket.get("access_key", settings.get("cloud.aws.access_key")),
    bucket.get("secret_key", settings.get("cloud.aws.secret_key")));

    String bucketName = bucket.get("bucket");
  logger.info("--> verify encryption for bucket [{}], prefix [{}]", bucketName, basePath);
  List<S3ObjectSummary> summaries = s3Client.listObjects(bucketName, basePath).getObjectSummaries();
  for (S3ObjectSummary summary : summaries) {
      assertThat(s3Client.getObjectMetadata(bucketName, summary.getKey()).getSSEAlgorithm(), equalTo("AES256"));
  }

  logger.info("--> delete some data");
  for (int i = 0; i < 50; i++) {
      client.prepareDelete("test-idx-1", "doc", Integer.toString(i)).get();
View Full Code Here

            String bucketName = bucket.get("bucket");

            // We check that settings has been set in elasticsearch.yml integration test file
            // as described in README
            assertThat("Your settings in elasticsearch.yml are incorrects. Check README file.", bucketName, notNullValue());
            AmazonS3 client = internalCluster().getInstance(AwsS3Service.class).client(region, accessKey, secretKey);
            try {
                ObjectListing prevListing = null;
                //From http://docs.amazonwebservices.com/AmazonS3/latest/dev/DeletingMultipleObjectsUsingJava.html
                //we can do at most 1K objects per delete
                //We don't know the bucket name until first object listing
                DeleteObjectsRequest multiObjectDeleteRequest = null;
                ArrayList<DeleteObjectsRequest.KeyVersion> keys = new ArrayList<DeleteObjectsRequest.KeyVersion>();
                while (true) {
                    ObjectListing list;
                    if (prevListing != null) {
                        list = client.listNextBatchOfObjects(prevListing);
                    } else {
                        list = client.listObjects(bucketName, basePath);
                        multiObjectDeleteRequest = new DeleteObjectsRequest(list.getBucketName());
                    }
                    for (S3ObjectSummary summary : list.getObjectSummaries()) {
                        keys.add(new DeleteObjectsRequest.KeyVersion(summary.getKey()));
                        //Every 500 objects batch the delete request
                        if (keys.size() > 500) {
                            multiObjectDeleteRequest.setKeys(keys);
                            client.deleteObjects(multiObjectDeleteRequest);
                            multiObjectDeleteRequest = new DeleteObjectsRequest(list.getBucketName());
                            keys.clear();
                        }
                    }
                    if (list.isTruncated()) {
                        prevListing = list;
                    } else {
                        break;
                    }
                }
                if (!keys.isEmpty()) {
                    multiObjectDeleteRequest.setKeys(keys);
                    client.deleteObjects(multiObjectDeleteRequest);
                }
            } catch (Throwable ex) {
                logger.warn("Failed to delete S3 repository [{}] in [{}]", ex, bucketName, region);
            }
        }
View Full Code Here

     *
     * @return AmazonS3Client
     */
    AmazonS3 createS3Client() {
        AWSCredentials credentials = new BasicAWSCredentials(configuration.getAccessKey(), configuration.getSecretKey());
        AmazonS3 client = new AmazonS3Client(credentials);
        return client;
    }
View Full Code Here

    }


    private synchronized AmazonS3 getClient(String endpoint, String account, String key) {
        Tuple<String, String> clientDescriptor = new Tuple<String, String>(endpoint, account);
        AmazonS3Client client = clients.get(clientDescriptor);
        if (client != null) {
            return client;
        }

        ClientConfiguration clientConfiguration = new ClientConfiguration();
        String protocol = componentSettings.get("protocol", "https").toLowerCase();
        protocol = componentSettings.get("s3.protocol", protocol).toLowerCase();
        if ("http".equals(protocol)) {
            clientConfiguration.setProtocol(Protocol.HTTP);
        } else if ("https".equals(protocol)) {
            clientConfiguration.setProtocol(Protocol.HTTPS);
        } else {
            throw new ElasticsearchIllegalArgumentException("No protocol supported [" + protocol + "], can either be [http] or [https]");
        }

        String proxyHost = componentSettings.get("proxy_host");
        if (proxyHost != null) {
            String portString = componentSettings.get("proxy_port", "80");
            Integer proxyPort;
            try {
                proxyPort = Integer.parseInt(portString, 10);
            } catch (NumberFormatException ex) {
                throw new ElasticsearchIllegalArgumentException("The configured proxy port value [" + portString + "] is invalid", ex);
            }
            clientConfiguration.withProxyHost(proxyHost).setProxyPort(proxyPort);
        }

        AWSCredentialsProvider credentials;

        if (account == null && key == null) {
            credentials = new AWSCredentialsProviderChain(
                    new EnvironmentVariableCredentialsProvider(),
                    new SystemPropertiesCredentialsProvider(),
                    new InstanceProfileCredentialsProvider()
            );
        } else {
            credentials = new AWSCredentialsProviderChain(
                    new StaticCredentialsProvider(new BasicAWSCredentials(account, key))
            );
        }
        client = new AmazonS3Client(credentials, clientConfiguration);

        if (endpoint != null) {
            client.setEndpoint(endpoint);
        }
        clients.put(clientDescriptor, client);
        return client;
    }
View Full Code Here

     *
     * @return AmazonS3Client
     */
    AmazonS3Client createS3Client() {
        AWSCredentials credentials = new BasicAWSCredentials(configuration.getAccessKey(), configuration.getSecretKey());
        AmazonS3Client client = new AmazonS3Client(credentials);
        if (configuration.getAmazonS3Endpoint() != null) {
            client.setEndpoint(configuration.getAmazonS3Endpoint());
        }
        return client;
    }
View Full Code Here

  public CompleteMultipartUploadResult completeMultipartUpload(
      CompleteMultipartUploadRequest completeMultipartUploadRequest)
      throws AmazonClientException, AmazonServiceException {

      String uploadId = completeMultipartUploadRequest.getUploadId();
      EncryptedUploadContext encryptedUploadContext = currentMultipartUploadSecretKeys.get(uploadId);

      if (encryptedUploadContext.hasFinalPartBeenSeen() == false) {
        throw new AmazonClientException("Unable to complete an encrypted multipart upload without being told which part was the last.  " +
            "Without knowing which part was the last, the encrypted data in Amazon S3 is incomplete and corrupt.");
      }

    CompleteMultipartUploadResult result = super.completeMultipartUpload(completeMultipartUploadRequest);

    // In InstructionFile mode, we want to write the instruction file only after the whole upload has completed correctly.
    if (cryptoConfig.getStorageMode() == CryptoStorageMode.InstructionFile) {
          Cipher symmetricCipher = EncryptionUtils.createSymmetricCipher(
              encryptedUploadContext.getEnvelopeEncryptionKey(),
              Cipher.ENCRYPT_MODE, cryptoConfig.getCryptoProvider(),
              encryptedUploadContext.getFirstInitializationVector());

          EncryptionMaterials encryptionMaterials = encryptionMaterialsProvider.getEncryptionMaterials();

          // Encrypt the envelope symmetric key
          byte[] encryptedEnvelopeSymmetricKey = EncryptionUtils.getEncryptedSymmetricKey(encryptedUploadContext.getEnvelopeEncryptionKey(), encryptionMaterials, cryptoConfig.getCryptoProvider());
      EncryptionInstruction instruction = new EncryptionInstruction(encryptionMaterials.getMaterialsDescription(), encryptedEnvelopeSymmetricKey, encryptedUploadContext.getEnvelopeEncryptionKey(), symmetricCipher);

          // Put the instruction file into S3
          super.putObject(EncryptionUtils.createInstructionPutRequest(encryptedUploadContext.getBucketName(), encryptedUploadContext.getKey(), instruction));
    }

    currentMultipartUploadSecretKeys.remove(uploadId);
    return result;
  }
View Full Code Here

          // Update the request's metadata to the updated metadata
          initiateMultipartUploadRequest.setObjectMetadata(metadata);
    }

    InitiateMultipartUploadResult result = super.initiateMultipartUpload(initiateMultipartUploadRequest);
    EncryptedUploadContext encryptedUploadContext = new EncryptedUploadContext(initiateMultipartUploadRequest.getBucketName(), initiateMultipartUploadRequest.getKey(), envelopeSymmetricKey);
    encryptedUploadContext.setNextInitializationVector(symmetricCipher.getIV());
    encryptedUploadContext.setFirstInitializationVector(symmetricCipher.getIV());
    currentMultipartUploadSecretKeys.put(result.getUploadId(), encryptedUploadContext);

    return result;
  }
View Full Code Here

          "of the cipher block size (" + JceEncryptionConstants.SYMMETRIC_CIPHER_BLOCK_SIZE + ") with the exception of the last part.  " +
            "Otherwise encryption adds extra padding that will corrupt the final object.");
    }

        // Generate the envelope symmetric key and initialize a cipher to encrypt the object's data
    EncryptedUploadContext encryptedUploadContext = currentMultipartUploadSecretKeys.get(uploadId);
      if (encryptedUploadContext == null) throw new AmazonClientException("No client-side information available on upload ID " + uploadId);

        SecretKey envelopeSymmetricKey = encryptedUploadContext.getEnvelopeEncryptionKey();
        byte[] iv = encryptedUploadContext.getNextInitializationVector();
        Cipher symmetricCipher = EncryptionUtils.createSymmetricCipher(envelopeSymmetricKey, Cipher.ENCRYPT_MODE, cryptoConfig.getCryptoProvider(), iv);

        // Create encrypted input stream
        InputStream encryptedInputStream = EncryptionUtils.getEncryptedInputStream(uploadPartRequest, symmetricCipher);
        uploadPartRequest.setInputStream(encryptedInputStream);

        // The last part of the multipart upload will contain extra padding from the encryption process, which
        // changes the
    if (uploadPartRequest.isLastPart()) {
      // We only change the size of the last part
      long cryptoContentLength = EncryptionUtils.calculateCryptoContentLength(symmetricCipher, uploadPartRequest);
      if (cryptoContentLength > 0) uploadPartRequest.setPartSize(cryptoContentLength);

      if (encryptedUploadContext.hasFinalPartBeenSeen()) {
        throw new AmazonClientException("This part was specified as the last part in a multipart upload, but a previous part was already marked as the last part.  " +
            "Only the last part of the upload should be marked as the last part, otherwise it will cause the encrypted data to be corrupted.");
      }

      encryptedUploadContext.setHasFinalPartBeenSeen(true);
    }

        // Treat all encryption requests as input stream upload requests, not as file upload requests.
        uploadPartRequest.setFile(null);
        uploadPartRequest.setFileOffset(0);

      UploadPartResult result = super.uploadPart(uploadPartRequest);

      if (encryptedInputStream instanceof ByteRangeCapturingInputStream) {
        ByteRangeCapturingInputStream bris = (ByteRangeCapturingInputStream)encryptedInputStream;
        encryptedUploadContext.setNextInitializationVector(bris.getBlock());
      } else {
        throw new AmazonClientException("Unable to access last block of encrypted data");
      }

    return result;
View Full Code Here

      throws AmazonClientException, AmazonServiceException {
     
      appendUserAgent(completeMultipartUploadRequest, USER_AGENT);

      String uploadId = completeMultipartUploadRequest.getUploadId();
      EncryptedUploadContext encryptedUploadContext = currentMultipartUploadSecretKeys.get(uploadId);

      if (encryptedUploadContext.hasFinalPartBeenSeen() == false) {
        throw new AmazonClientException("Unable to complete an encrypted multipart upload without being told which part was the last.  " +
            "Without knowing which part was the last, the encrypted data in Amazon S3 is incomplete and corrupt.");
      }

    CompleteMultipartUploadResult result = super.completeMultipartUpload(completeMultipartUploadRequest);

    // In InstructionFile mode, we want to write the instruction file only after the whole upload has completed correctly.
    if (cryptoConfig.getStorageMode() == CryptoStorageMode.InstructionFile) {
          Cipher symmetricCipher = EncryptionUtils.createSymmetricCipher(
              encryptedUploadContext.getEnvelopeEncryptionKey(),
              Cipher.ENCRYPT_MODE, cryptoConfig.getCryptoProvider(),
              encryptedUploadContext.getFirstInitializationVector());

          EncryptionMaterials encryptionMaterials = encryptionMaterialsProvider.getEncryptionMaterials();

          // Encrypt the envelope symmetric key
          byte[] encryptedEnvelopeSymmetricKey = EncryptionUtils.getEncryptedSymmetricKey(encryptedUploadContext.getEnvelopeEncryptionKey(), encryptionMaterials, cryptoConfig.getCryptoProvider());
      EncryptionInstruction instruction = new EncryptionInstruction(encryptionMaterials.getMaterialsDescription(), encryptedEnvelopeSymmetricKey, encryptedUploadContext.getEnvelopeEncryptionKey(), symmetricCipher);

          // Put the instruction file into S3
          super.putObject(EncryptionUtils.createInstructionPutRequest(encryptedUploadContext.getBucketName(), encryptedUploadContext.getKey(), instruction));
    }

    currentMultipartUploadSecretKeys.remove(uploadId);
    return result;
  }
View Full Code Here

TOP

Related Classes of com.amazonaws.services.s3.internal.crypto.EncryptedUploadContext

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.