response.getEntity().getContentLength()));
}
// Prepare exception with information about error response, whether or not it
// contains XML content, in case we decide not to retry.
ServiceException exception = null;
if (isXmlContentType(contentType)
&& response.getEntity() != null
&& response.getEntity().getContentLength() != 0)
{
// Prepare exception for XML-bearing error response
if(log.isDebugEnabled()) {
log.debug("Response '" + httpMethod.getURI().getRawPath()
+ "' - Received error response with XML message");
}
StringBuilder sb = new StringBuilder();
BufferedReader reader = null;
try {
reader = new BufferedReader(new InputStreamReader(
new HttpMethodReleaseInputStream(response)));
String line;
while((line = reader.readLine()) != null) {
sb.append(line).append("\n");
}
}
finally {
if(reader != null) {
reader.close();
}
}
// Prepare exception containing the XML message document.
exception = new ServiceException("Service Error Message.", sb.toString());
} else {
if(log.isDebugEnabled()) {
log.debug("Response '" + httpMethod.getURI().getRawPath()
+ "' - Received error response without XML content");
}
String responseText = null;
byte[] responseBody = null;
if(response.getEntity() != null) {
responseBody = EntityUtils.toByteArray(response.getEntity());
}
if(responseBody != null && responseBody.length > 0) {
responseText = new String(responseBody);
}
// Prepare exception containing the HTTP error fields.
HttpException httpException = new HttpException(
responseCode, response.getStatusLine().getReasonPhrase());
exception = new ServiceException(
"Request Error" + (responseText != null ? " [" + responseText + "]." : "."),
httpException);
}
exception.setResponseCode(responseCode);
exception.setResponseHeaders(RestUtils.convertHeadersToMap(response.getAllHeaders()));
// Consume and release connection.
EntityUtils.consume(response.getEntity());
/*
* For cases where the request may succeed if retried, count the number of attempts
* we have made to ensure we don't exceeded the max retry limit.
*/
// Sleep then retry on 5xx Internal Server errors.
if (responseCode >= 500) {
// Throws provided exception if we have exceeded the retry count
sleepOnInternalError(++internalErrorCount, exception);
}
// Retry after Temporary Redirect 307
else if(responseCode == 307) {
// Retry up to our limit; but at least once to support S3 bucket locations
if (redirectCount >= retryMaxCount
&& redirectCount > 0) // Ensure we have retried at least once...
{
throw exception;
}
// Set new URI from Location header
Header locationHeader = response.getFirstHeader("location");
URI newLocation = new URI(locationHeader.getValue());
// deal with implementations of HttpUriRequest
if(httpMethod instanceof HttpRequestBase) {
((HttpRequestBase) httpMethod).setURI(newLocation);
}
else if(httpMethod instanceof RequestWrapper) {
((RequestWrapper) httpMethod).setURI(newLocation);
}
skipNextAuthorizationCycle = true;
redirectCount++;
if(log.isDebugEnabled()) {
log.debug(
"Following Temporary Redirect (" + redirectCount + ") to: "
+ httpMethod.getURI().toString());
}
}
else if("RequestTimeout".equals(exception.getErrorCode())) {
if(requestTimeoutErrorCount >= retryMaxCount) {
throw exception;
}
requestTimeoutErrorCount++;
if(log.isWarnEnabled()) {
log.warn(
"Retrying connection that failed with RequestTimeout error ("
+ requestTimeoutErrorCount + ")");
}
}
else if("RequestTimeTooSkewed".equals(exception.getErrorCode())) {
if(requestTimeTooSkewedErrorCount >= retryMaxCount) {
throw exception;
}
requestTimeTooSkewedErrorCount++;
this.timeOffset = RestUtils.calculateTimeAdjustmentOffset(response);
if(log.isWarnEnabled()) {
log.warn("Adjusted time offset in response to RequestTimeTooSkewed error"
+ "(" + requestTimeTooSkewedErrorCount + ")."
+ "Local machine and service disagree on the time by approximately "
+ (this.timeOffset / 1000) + " seconds, please fix your system's time."
+ " Retrying connection.");
}
}
// Special handling for S3 object PUT failures causing NoSuchKey errors - Issue #85, #175
// Treat this as a special kind of internal server error.
else if(responseCode == 404
&& "PUT".equalsIgnoreCase(httpMethod.getMethod())
&& "NoSuchKey".equals(exception.getErrorCode())
// If PUT operation is trying to copy an existing source object, don't ignore 404
&& httpMethod.getFirstHeader(getRestHeaderPrefix() + "copy-source") == null) {
// Throws provided exception if we have exceeded the retry count
sleepOnInternalError(++internalErrorCount, exception);
if(log.isDebugEnabled()) {
log.debug("Ignoring NoSuchKey/404 error on PUT to: " + httpMethod.getURI().toString());
}
}
else if((responseCode == 403 || responseCode == 401) && this.isRecoverable403(httpMethod, exception)) {
// Retry up to our limit; but at least once
if (authFailureCount >= retryMaxCount
&& authFailureCount > 0) // Ensure we have retried at least once...
{
throw exception;
}
authFailureCount++;
if(log.isDebugEnabled()) {
log.debug("Retrying after 403 Forbidden");
}
}
// Special handling for requests to a region that requires AWS
// request signature version 4 when service isn't configured to
// use AWS4-HMAC-SHA256 signatures by default.
else if ("InvalidRequest".equals(exception.getErrorCode())
&& exception.getErrorMessage().contains("Please use AWS4-HMAC-SHA256"))
{
forceRequestSignatureVersion = "AWS4-HMAC-SHA256";
authFailureCount++;
if (log.isWarnEnabled()) {
log.warn(
"Retrying request with \"AWS4-HMAC-SHA256\" signing"
+ " mechanism: " + httpMethod);
}
}
// Special handling for requests signed using AWS request signature version
// 4 but sent to the wrong region due to an incorrect Host endpoint.
else if("AuthorizationHeaderMalformed".equals(exception.getErrorCode())) {
String expectedRegion = null;
try {
expectedRegion = exception.getXmlMessageAsBuilder()
.xpathFind("/Error/Region").getElement().getTextContent();
} catch(Exception ignored) {
// Throw original exception if we cannot parse expected
// Region out of error message, in which case this error
// was caused by something other than just wrong region.
throw exception;
}
// Cache correct region for this request's bucket name
this.regionEndpointCache.put(httpMethod, expectedRegion);
URI originalURI = httpMethod.getURI();
SignatureUtils.awsV4CorrectRequestHostForRegion(
httpMethod, expectedRegion);
authFailureCount++;
if(log.isWarnEnabled()) {
log.warn("Retrying request after automatic adjustment of"
+ " Host endpoint from " + "\"" + originalURI.getHost()
+ "\" to \"" + httpMethod.getURI().getHost()
+ "\" following request signing error"
+ " using AWS request signing version 4: " + httpMethod);
}
}
// If we haven't explicitly detected an retry-able error response type above,
// just throw the exception to abort the request.
else {
throw exception;
}
// Print warning message if a non-fatal error occurred (we only reach this
// point in the code if an exception isn't thrown above)
if(log.isWarnEnabled()) {
String requestDescription =
httpMethod.getMethod()
+ " '" + httpMethod.getURI().getPath()
+ (httpMethod.getURI().getQuery() != null
&& httpMethod.getURI().getQuery().length() > 0
? "?" + httpMethod.getURI().getQuery() : "")
+ "'"
+ " -- ResponseCode: " + responseCode
+ ", ResponseStatus: " + response.getStatusLine().getReasonPhrase()
+ ", Request Headers: [" + ServiceUtils.join(httpMethod.getAllHeaders(), ", ") + "]"
+ ", Response Headers: [" + ServiceUtils.join(response.getAllHeaders(), ", ") + "]";
requestDescription = requestDescription.replaceAll("[\\n\\r\\f]", ""); // Remove any newlines.
log.warn("Retrying request following error response: " + requestDescription);
}
}
}
// Top-level exception handler to deal with otherwise unhandled exceptions, or to augment
// a ServiceException thrown above with additional information.
catch(Exception t) {
if(log.isDebugEnabled()) {
String msg = "Rethrowing as a ServiceException error in performRequest: " + t;
if(t.getCause() != null) {
msg += ", with cause: " + t.getCause();
}
if(log.isTraceEnabled()) {
log.trace(msg, t);
}
else {
log.debug(msg);
}
}
if(log.isDebugEnabled() && !shuttingDown) {
log.debug("Releasing HttpClient connection after error: " + t.getMessage());
}
httpMethod.abort();
ServiceException serviceException;
if(t instanceof ServiceException) {
serviceException = (ServiceException) t;
}
else {
MxDelegate.getInstance().registerS3ServiceExceptionEvent();
serviceException = new ServiceException("Request Error: " + t.getMessage(), t);
}
// Add S3 request and host IDs from HTTP headers to exception, if they are available
// and have not already been populated by parsing an XML error response.
if(!serviceException.isParsedFromXmlMessage()
&& response != null
&& response.getFirstHeader(Constants.AMZ_REQUEST_ID_1) != null
&& response.getFirstHeader(Constants.AMZ_REQUEST_ID_2) != null) {
serviceException.setRequestAndHostIds(
response.getFirstHeader(Constants.AMZ_REQUEST_ID_1).getValue(),
response.getFirstHeader(Constants.AMZ_REQUEST_ID_2).getValue());
serviceException.setResponseHeaders(RestUtils.convertHeadersToMap(
response.getAllHeaders()));
}
if(response != null && response.getStatusLine() != null) {
// If no network connection is available, status info is not available
serviceException.setResponseCode(response.getStatusLine().getStatusCode());
serviceException.setResponseStatus(response.getStatusLine().getReasonPhrase());
}
if(httpMethod.getFirstHeader("Host") != null) {
serviceException.setRequestHost(
httpMethod.getFirstHeader("Host").getValue());
}
if(response != null && response.getFirstHeader("Date") != null) {
serviceException.setResponseDate(
response.getFirstHeader("Date").getValue());
}
throw serviceException;
}
return response;