public void upload(AbstractBackupPath path, InputStream in) throws BackupRestoreException
{
uploadCount.incrementAndGet();
AmazonS3 s3Client = getS3Client();
InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(config.getBackupPrefix(), path.getRemotePath());
InitiateMultipartUploadResult initResponse = s3Client.initiateMultipartUpload(initRequest);
DataPart part = new DataPart(config.getBackupPrefix(), path.getRemotePath(), initResponse.getUploadId());
List<PartETag> partETags = Lists.newArrayList();
long chunkSize = config.getBackupChunkSize();
if (path.getSize() > 0)
chunkSize = (path.getSize() / chunkSize >= MAX_CHUNKS) ? (path.getSize() / (MAX_CHUNKS - 1)) : chunkSize;
logger.info(String.format("Uploading to %s/%s with chunk size %d", config.getBackupPrefix(), path.getRemotePath(), chunkSize));
try
{
Iterator<byte[]> chunks = compress.compress(in, chunkSize);
// Upload parts.
int partNum = 0;
while (chunks.hasNext())
{
byte[] chunk = chunks.next();
rateLimiter.acquire(chunk.length);
DataPart dp = new DataPart(++partNum, chunk, config.getBackupPrefix(), path.getRemotePath(), initResponse.getUploadId());
S3PartUploader partUploader = new S3PartUploader(s3Client, dp, partETags);
executor.submit(partUploader);
bytesUploaded.addAndGet(chunk.length);
}
executor.sleepTillEmpty();