Package com.amazonaws.services.s3.transfer

Examples of com.amazonaws.services.s3.transfer.TransferManager


        public PrestoS3OutputStream(AmazonS3 s3, TransferManagerConfiguration config, String host, String key, File tempFile)
                throws IOException
        {
            super(new BufferedOutputStream(new FileOutputStream(checkNotNull(tempFile, "tempFile is null"))));

            transferManager = new TransferManager(checkNotNull(s3, "s3 is null"));
            transferManager.setConfiguration(checkNotNull(config, "config is null"));

            this.host = checkNotNull(host, "host is null");
            this.key = checkNotNull(key, "key is null");
            this.tempFile = tempFile;
View Full Code Here


        if (LOGGER.isDebugEnabled()) {
            LOGGER.debug(format("Multipart sending file %1$s as S3 object %2$s in "
                    + "bucket %3$s", sourceFile.getName(), key, bucketName));
        }
        TransferManager tm = new TransferManager(S3Utils.acquireClient(clientOptions));
        Upload upload = tm.upload(bucketName, key, sourceFile);
        upload.waitForCompletion();
    }
View Full Code Here

        if (LOGGER.isDebugEnabled()) {
            LOGGER.debug(format("Multipart sending stream as S3 object %1$s in "
                    + "bucket %2$s", key, bucketName));
        }
        TransferManager tm = new TransferManager(S3Utils.acquireClient(clientOptions));
        Upload upload = tm.upload(bucketName, key, sourceStream, null);
        upload.waitForCompletion();
    }
View Full Code Here

        assert req != null;

        if (LOGGER.isDebugEnabled()) {
            LOGGER.debug("Multipart sending object to S3 using PutObjectRequest");
        }      
        TransferManager tm = new TransferManager(S3Utils.acquireClient(clientOptions));
        Upload upload = tm.upload(req);
        upload.waitForCompletion();

    }
View Full Code Here

     */
    public void useClient(AmazonS3Client client) {
        Region s3region = client.getRegion();

        synchronized (transferManagersByRegion) {
            TransferManager tm = transferManagersByRegion.remove(s3region);
            if (tm != null) {
                tm.shutdownNow();
            }
            clientsByRegion.put(s3region, client);
        }
    }
View Full Code Here

     *         instantiated automatically from any existing
     *         {@link AmazonS3Client},
     */
    public TransferManager getTransferManager(Region region) {
        synchronized (transferManagersByRegion) {
            TransferManager tm = transferManagersByRegion.get(region);
            if (tm == null) {
                tm = new TransferManager(getClient(region));
                transferManagersByRegion.put(region, tm);
            }
            return tm;
        }
    }
View Full Code Here

    super(credentialsProvider);
    init();
  }
 
  protected void init() {
    transferManager = new TransferManager(this);
    TransferManagerConfiguration configuration = new TransferManagerConfiguration();
    configuration.setMultipartUploadThreshold(100 * Constants.KB);
    transferManager.setConfiguration(configuration);
  }
 
View Full Code Here

        credentials = new PropertiesCredentials(S3TransferProgressSample.class
                .getResourceAsStream("AwsCredentials.properties"));

        // TransferManager manages a pool of threads, so we create a
        // single instance and share it throughout our application.
        tx = new TransferManager(credentials);
       
        bucketName = "s3-upload-sdk-sample-" + credentials.getAWSAccessKeyId().toLowerCase();
       
        new S3TransferProgressSample();
    }
View Full Code Here

        String bucketId = fileName.getBucketId();

        this.awsCredentials = awsCredentials;
        this.service = service;
        this.transferManager = new TransferManager(service);
        this.serverSideEncryption = S3FileSystemConfigBuilder
            .getInstance().getServerSideEncryption(fileSystemOptions);

        Region region = S3FileSystemConfigBuilder.getInstance().getRegion(
            fileSystemOptions);
View Full Code Here

  public void execute() throws MojoExecutionException, MojoFailureException {

    // Setup AWS S3 client
    AWSCredentials credentials = new BasicAWSCredentials(awsAccessKey, awsSecretKey);
    AmazonS3Client uploadClient = new AmazonS3Client(credentials);
    TransferManager transfers = new TransferManager(credentials);

    // Make sure key prefix does not start with a slash but has one at the
    // end
    if (keyPrefix.startsWith("/"))
      keyPrefix = keyPrefix.substring(1);
    if (!keyPrefix.endsWith("/"))
      keyPrefix = keyPrefix + "/";

    // Keep track of how much data has been transferred
    long totalBytesTransferred = 0L;
    int items = 0;
    Queue<Upload> uploads = new LinkedBlockingQueue<Upload>();

    try {
      // Check if S3 bucket exists
      getLog().debug("Checking whether bucket " + bucket + " exists");
      if (!uploadClient.doesBucketExist(bucket)) {
        getLog().error("Desired bucket '" + bucket + "' does not exist!");
        return;
      }

      getLog().debug("Collecting files to transfer from " + resources.getDirectory());
      List<File> res = getResources();
      for (File file : res) {
        // Make path of resource relative to resources directory
        String filename = file.getName();
        String extension = FilenameUtils.getExtension(filename);
        String path = file.getPath().substring(resources.getDirectory().length());
        String key = concat("/", keyPrefix, path).substring(1);

        // Delete old file version in bucket
        getLog().debug("Removing existing object at " + key);
        uploadClient.deleteObject(bucket, key);

        // Setup meta data
        ObjectMetadata meta = new ObjectMetadata();
        meta.setCacheControl("public, max-age=" + String.valueOf(valid * 3600));

        FileInputStream fis = null;
        GZIPOutputStream gzipos = null;
        final File fileToUpload;

        if (gzip && ("js".equals(extension) || "css".equals(extension))) {
          try {
            fis = new FileInputStream(file);
            File gzFile = File.createTempFile(file.getName(), null);
            gzipos = new GZIPOutputStream(new FileOutputStream(gzFile));
            IOUtils.copy(fis, gzipos);
            fileToUpload = gzFile;
            meta.setContentEncoding("gzip");
            if ("js".equals(extension))
              meta.setContentType("text/javascript");
            if ("css".equals(extension))
              meta.setContentType("text/css");
          } catch (FileNotFoundException e) {
            getLog().error(e);
            continue;
          } catch (IOException e) {
            getLog().error(e);
            continue;
          } finally {
            IOUtils.closeQuietly(fis);
            IOUtils.closeQuietly(gzipos);
          }
        } else {
          fileToUpload = file;
        }

        // Do a random check for existing errors before starting the next upload
        if (erroneousUpload != null)
          break;

        // Create put object request
        long bytesToTransfer = fileToUpload.length();
        totalBytesTransferred += bytesToTransfer;
        PutObjectRequest request = new PutObjectRequest(bucket, key, fileToUpload);
        request.setProgressListener(new UploadListener(credentials, bucket, key, bytesToTransfer));
        request.setMetadata(meta);

        // Schedule put object request
        getLog().info("Uploading " + key + " (" + FileUtils.byteCountToDisplaySize((int) bytesToTransfer) + ")");
        Upload upload = transfers.upload(request);
        uploads.add(upload);
        items ++;
      }
    } catch (AmazonServiceException e) {
      getLog().error("Uploading resources failed: " + e.getMessage());
View Full Code Here

TOP

Related Classes of com.amazonaws.services.s3.transfer.TransferManager

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.