package de.kopis.glacier.commands;
/*
* #%L
* uploader
* $Id:$
* $HeadURL:$
* %%
* Copyright (C) 2012 Carsten Ringe
* Copyright (C) 2013 Deux Huit Huit
* %%
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public
* License along with this program. If not, see
* <http://www.gnu.org/licenses/gpl-3.0.html>.
* #L%
*/
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import joptsimple.OptionSet;
import com.amazonaws.AmazonClientException;
import com.amazonaws.AmazonServiceException;
import com.amazonaws.services.glacier.TreeHashGenerator;
import com.amazonaws.services.glacier.model.CompleteMultipartUploadRequest;
import com.amazonaws.services.glacier.model.CompleteMultipartUploadResult;
import com.amazonaws.services.glacier.model.InitiateMultipartUploadRequest;
import com.amazonaws.services.glacier.model.InitiateMultipartUploadResult;
import com.amazonaws.services.glacier.model.UploadMultipartPartRequest;
import com.amazonaws.services.glacier.model.UploadMultipartPartResult;
import com.amazonaws.services.s3.internal.InputSubstream;
import com.amazonaws.services.s3.internal.RepeatableFileInputStream;
import com.amazonaws.util.BinaryUtils;
import de.kopis.glacier.parsers.GlacierUploaderOptionParser;
import de.kopis.glacier.printers.HumanReadableSize;
public class UploadMultipartArchiveCommand extends AbstractCommand {
public UploadMultipartArchiveCommand(final URL endpoint, final File credentials) throws IOException {
super(endpoint, credentials);
}
// from:
// http://docs.amazonwebservices.com/amazonglacier/latest/dev/uploading-an-archive-mpu-using-java.html
public void upload(final String vaultName, final File uploadFile, final Long partSize) {
final String hPartSize = HumanReadableSize.parse(partSize);
final String hTotalSize = HumanReadableSize.parse(uploadFile.length());
log.info(String.format("Multipart uploading %s (%s) to vault %s with part size %s (%s).", uploadFile.getName(),
hTotalSize, vaultName, partSize, hPartSize));
try {
final String uploadId = this.initiateMultipartUpload(vaultName, partSize, uploadFile.getName());
final String checksum = this.uploadParts(uploadId, uploadFile, vaultName, partSize);
final CompleteMultipartUploadResult result = this.completeMultiPartUpload(uploadId, uploadFile, vaultName,
checksum);
log.info("Uploaded Archive ID: " + result.getArchiveId());
log.info("Local Checksum: " + checksum);
log.info("Remote Checksum: " + result.getChecksum());
if (checksum.equals(result.getChecksum())) {
log.info("Checksums are identical, upload succeeded.");
} else {
log.error("Checksums are different, upload failed.");
}
} catch (final IOException e) {
log.error("Something went wrong while multipart uploading " + uploadFile + "." + e.getLocalizedMessage(), e);
} catch (AmazonServiceException e) {
log.error("Something went wrong at Amazon while uploading " + uploadFile + "." + e.getLocalizedMessage(), e);
} catch (NoSuchAlgorithmException e) {
log.error("No such algorithm found " + e.getLocalizedMessage(), e);
} catch (AmazonClientException e) {
log.error("Something went wrong with the Amazon Client." + e.getLocalizedMessage(), e);
}
}
private String initiateMultipartUpload(final String vaultName, final Long partSize, final String fileName) {
// Initiate
InitiateMultipartUploadRequest request = new InitiateMultipartUploadRequest().withVaultName(vaultName)
.withArchiveDescription(fileName).withPartSize(partSize.toString());
InitiateMultipartUploadResult result = client.initiateMultipartUpload(request);
log.info("Upload ID (token): " + result.getUploadId());
return result.getUploadId();
}
/* This method contains a derivative of work from the following source:
*
* https://github.com/aws/aws-sdk-java/blob/master/src/main/java/com/amazonaws/services/glacier/transfer/ArchiveTransferManager.java?source=c
*
* from the 1.8.5 tag in the source tree.
*
* Copyright 2012-2014 Amazon Technologies, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at:
*
* http://aws.amazon.com/apache2.0
*/
private String uploadParts(String uploadId, File file, final String vaultName, final Long partSize)
throws AmazonServiceException, NoSuchAlgorithmException, AmazonClientException, IOException {
FileInputStream fileToUpload = null;
String checksum = "";
try {
long currentPosition = 0;
List<byte[]> binaryChecksums = new LinkedList<byte[]>();
fileToUpload = new FileInputStream(file);
int counter = 1;
int total = (int) Math.ceil(file.length() / (double) partSize);
while (currentPosition < file.length()) {
long length = partSize;
if (currentPosition + partSize > file.length()) {
length = file.length() - currentPosition;
}
Exception failedException = null;
boolean completed = false;
int tries = 0;
while (!completed && tries < 5) {
tries++;
InputStream inputSubStream = newInputSubstream(file, currentPosition, length);
inputSubStream.mark(-1);
checksum = TreeHashGenerator.calculateTreeHash(inputSubStream);
byte[] binaryChecksum = BinaryUtils.fromHex(checksum);
inputSubStream.reset();
String range = "bytes " + currentPosition + "-" + (currentPosition + length - 1) + "/*";
UploadMultipartPartRequest req = new UploadMultipartPartRequest().withChecksum(checksum)
.withBody(inputSubStream).withRange(range).withUploadId(uploadId).withVaultName(vaultName);
try {
UploadMultipartPartResult partResult = client.uploadMultipartPart(req);
log.info(String.format("Part %d/%d (%s) uploaded, checksum: %s", counter, total, range, partResult.getChecksum()));
completed = true;
binaryChecksums.add(binaryChecksum);
} catch (Exception e) {
failedException = e;
} finally {
if (inputSubStream != null) {
try {
inputSubStream.close();
} catch (IOException ex) {
log.debug("Ignore failure in closing the Closeable", ex);
}
}
}
}
if (!completed && failedException != null)
throw new AmazonClientException("Failed operation", failedException);
currentPosition += partSize;
++counter;
}
checksum = TreeHashGenerator.calculateTreeHash(binaryChecksums);
} finally {
if (fileToUpload != null) {
fileToUpload.close();
}
}
return checksum;
}
private InputSubstream newInputSubstream(File file, long startingPosition, long length) {
try {
return new InputSubstream(new RepeatableFileInputStream(file), startingPosition, length, true);
} catch (FileNotFoundException e) {
throw new AmazonClientException("Unable to find file '" + file.getAbsolutePath() + "'", e);
}
}
private CompleteMultipartUploadResult completeMultiPartUpload(String uploadId, File file, final String vaultName,
String checksum) throws NoSuchAlgorithmException, IOException {
CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest().withVaultName(vaultName)
.withUploadId(uploadId).withChecksum(checksum).withArchiveSize(String.valueOf(file.length()));
CompleteMultipartUploadResult compResult = client.completeMultipartUpload(compRequest);
return compResult;
}
@Override
public void exec(OptionSet options, GlacierUploaderOptionParser optionParser) {
final String vaultName = options.valueOf(optionParser.VAULT);
final List<File> optionsFiles = options.valuesOf(optionParser.MULTIPARTUPLOAD);
final Long partSize = options.valueOf(optionParser.PARTSIZE);
final List<String> nonOptions = options.nonOptionArguments();
final ArrayList<File> files = optionParser.mergeNonOptionsFiles(optionsFiles, nonOptions);
for (File uploadFile : files) {
this.upload(vaultName, uploadFile, partSize);
}
}
@Override
public boolean valid(OptionSet options, GlacierUploaderOptionParser optionParser) {
return options.has(optionParser.MULTIPARTUPLOAD) && options.hasArgument(optionParser.MULTIPARTUPLOAD);
}
}