/*
* Copyright 2010 Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License").
* You may not use this file except in compliance with the License.
* A copy of the License is located at
*
* http://aws.amazon.com/apache2.0
*
* or in the "license" file accompanying this file. This file is distributed
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing
* permissions and limitations under the License.
*/
package com.amazonaws.services.elasticmapreduce;
import org.w3c.dom.Node;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map.Entry;
import java.security.SignatureException;
import javax.xml.stream.XMLEventReader;
import com.amazonaws.*;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.QueryStringSigner;
import com.amazonaws.handlers.HandlerChainFactory;
import com.amazonaws.handlers.RequestHandler;
import com.amazonaws.http.DefaultResponseHandler;
import com.amazonaws.http.StaxResponseHandler;
import com.amazonaws.http.DefaultErrorResponseHandler;
import com.amazonaws.http.HttpClient;
import com.amazonaws.http.HttpMethodName;
import com.amazonaws.http.HttpRequest;
import com.amazonaws.transform.Unmarshaller;
import com.amazonaws.transform.StaxUnmarshallerContext;
import com.amazonaws.transform.VoidUnmarshaller;
import com.amazonaws.transform.StandardErrorUnmarshaller;
import com.amazonaws.services.elasticmapreduce.model.*;
import com.amazonaws.services.elasticmapreduce.model.transform.*;
/**
* Client for accessing AmazonElasticMapReduce. All service calls made
* using this client are blocking, and will not return until the service call
* completes.
* <p>
* <p>
* This is the Amazon Elastic MapReduce API Reference Guide. This guide
* is for programmers that need detailed information about the Amazon
* Elastic MapReduce SOAP and Query APIs.
* </p>
* <p>
* This document was last updated on May 5, 2010.
* </p>
*/
public class AmazonElasticMapReduceClient extends AmazonWebServiceClient implements AmazonElasticMapReduce {
/**
* The AWS credentials (access key ID and secret key) to use when
* authenticating with AWS services.
*/
private AWSCredentials awsCredentials;
/**
* List of exception unmarshallers for all AmazonElasticMapReduce exceptions.
*/
protected final List<Unmarshaller<AmazonServiceException, Node>> exceptionUnmarshallers;
/**
* Low level client for sending requests to AWS services.
*/
protected final HttpClient client;
/**
* Optional request handlers for additional request processing.
*/
private List<RequestHandler> requestHandlers = new ArrayList<RequestHandler>();
/**
* Constructs a new client to invoke service methods on
* AmazonElasticMapReduce using the specified AWS account credentials.
*
* <p>
* All service calls made using this new client object are blocking, and will not
* return until the service call completes.
*
* @param awsCredentials The AWS credentials (access key ID and secret key) to use
* when authenticating with AWS services.
*/
public AmazonElasticMapReduceClient(AWSCredentials awsCredentials) {
this(awsCredentials, new ClientConfiguration());
}
/**
* Constructs a new client to invoke service methods on
* AmazonElasticMapReduce using the specified AWS account credentials
* and client configuration options.
*
* <p>
* All service calls made using this new client object are blocking, and will not
* return until the service call completes.
*
* @param awsCredentials The AWS credentials (access key ID and secret key) to use
* when authenticating with AWS services.
* @param clientConfiguration The client configuration options controlling how this
* client connects to AmazonElasticMapReduce
* (ex: proxy settings, retry counts, etc.).
*/
public AmazonElasticMapReduceClient(AWSCredentials awsCredentials, ClientConfiguration clientConfiguration) {
super(clientConfiguration);
this.awsCredentials = awsCredentials;
exceptionUnmarshallers = new ArrayList<Unmarshaller<AmazonServiceException, Node>>();
exceptionUnmarshallers.add(new InternalServerErrorExceptionUnmarshaller());
exceptionUnmarshallers.add(new StandardErrorUnmarshaller());
setEndpoint("elasticmapreduce.amazonaws.com");
requestHandlers = new HandlerChainFactory().newRequestHandlerChain(
"/com/amazonaws/services/elasticmapreduce/request.handlers");
client = new HttpClient(clientConfiguration);
}
/**
* <p>
* AddJobFlowSteps adds new steps to a running job flow. The maximum
* number of steps in a job flow is 256.
* </p>
* <p>
* A step specifies the location of a JAR file stored either on the
* master node of the job flow or in Amazon S3. Each step is performed by
* the main function of the main class of the JAR file. The main class
* can be specified either in the manifest of the JAR or by using the
* MainFunction parameter of the step.
* </p>
* <p>
* SElastic MapReduce executes each step in the order listed. For a step
* to be considered complete, the main function must exit with a zero
* exit code and all Hadoop jobs started while the step was running must
* have completed and run successfully.
* </p>
* <p>
* You can only add steps to a job flow that is in one of the following
* states: STARTING, BOOTSTAPPING, RUNNING or WAITING.
* </p>
*
* @param addJobFlowStepsRequest Container for the necessary parameters
* to execute the AddJobFlowSteps service method on
* AmazonElasticMapReduce.
*
* @throws InternalServerErrorException
*
* @throws AmazonClientException
* If any internal errors are encountered inside the client while
* attempting to make the request or handle the response. For example
* if a network connection is not available.
* @throws AmazonServiceException
* If an error response is returned by AmazonElasticMapReduce indicating
* either a problem with the data in the request, or a server side issue.
*/
public void addJobFlowSteps(AddJobFlowStepsRequest addJobFlowStepsRequest)
throws AmazonServiceException, AmazonClientException {
Request<AddJobFlowStepsRequest> request = new AddJobFlowStepsRequestMarshaller().marshall(addJobFlowStepsRequest);
invoke(request, null);
}
/**
* <p>
* TerminateJobFlows shuts a list of job flows down. When a job flow is
* shut down, any step not yet completed is canceled and the EC2
* instances on which the job flow is running are stopped. Any log files
* not already saved are uploaded to Amazon S3 if a LogUri was specified
* when the job flow was created.
* </p>
*
* @param terminateJobFlowsRequest Container for the necessary parameters
* to execute the TerminateJobFlows service method on
* AmazonElasticMapReduce.
*
* @throws InternalServerErrorException
*
* @throws AmazonClientException
* If any internal errors are encountered inside the client while
* attempting to make the request or handle the response. For example
* if a network connection is not available.
* @throws AmazonServiceException
* If an error response is returned by AmazonElasticMapReduce indicating
* either a problem with the data in the request, or a server side issue.
*/
public void terminateJobFlows(TerminateJobFlowsRequest terminateJobFlowsRequest)
throws AmazonServiceException, AmazonClientException {
Request<TerminateJobFlowsRequest> request = new TerminateJobFlowsRequestMarshaller().marshall(terminateJobFlowsRequest);
invoke(request, null);
}
/**
* <p>
* DescribeJobFlows returns a list of job flows that match all of the
* supplied parameters. The parameters can include a list of job flow
* IDs, job flow states, and restrictions on job flow creation date and
* time.
* </p>
* <p>
* Regardless of supplied parameters, only job flows created within the
* last two months are returned.
* </p>
* <p>
* If no parameters are supplied, then job flows matching either the
* following criteria are returned:
* </p>
*
* <ul>
* <li>Job flows created and completed in the last two weeks.</li>
* <li> Job flows created within the last two months that are in one of
* the following states: <code>RUNNING</code> ,
*
* <code>WAITING</code> ,
*
* <code>SHUTTING_DOWN</code> ,
*
* <code>STARTING</code> .
* </li>
*
* </ul>
* <p>
* Amazon Elastic MapReduce can return a maximum of 512 job flow
* descriptions.
* </p>
*
* @param describeJobFlowsRequest Container for the necessary parameters
* to execute the DescribeJobFlows service method on
* AmazonElasticMapReduce.
*
* @return The response from the DescribeJobFlows service method, as
* returned by AmazonElasticMapReduce.
*
* @throws InternalServerErrorException
*
* @throws AmazonClientException
* If any internal errors are encountered inside the client while
* attempting to make the request or handle the response. For example
* if a network connection is not available.
* @throws AmazonServiceException
* If an error response is returned by AmazonElasticMapReduce indicating
* either a problem with the data in the request, or a server side issue.
*/
public DescribeJobFlowsResult describeJobFlows(DescribeJobFlowsRequest describeJobFlowsRequest)
throws AmazonServiceException, AmazonClientException {
Request<DescribeJobFlowsRequest> request = new DescribeJobFlowsRequestMarshaller().marshall(describeJobFlowsRequest);
return invoke(request, new DescribeJobFlowsResultStaxUnmarshaller());
}
/**
* <p>
* RunJobFlow creates and starts running a new job flow. The job flow
* will run the steps specified. Once the job flow completes, the EC2
* cluster is stopped and the HDFS partition is lost. To prevent loss of
* data, configure the last step of the job flow to store results in
* Amazon S3. If the JobFlowInstancesDetail : KeepJobFlowAliveWhenNoSteps
* parameter is set to TRUE, the job flow will transition to the WAITING
* state rather than shutting down once the steps have completed.
* </p>
* <p>
* A maximum of 256 steps are allowed in each job flow.
* </p>
* <p>
* For long running job flows, we recommended that you periodically store
* your results.
* </p>
*
* @param runJobFlowRequest Container for the necessary parameters to
* execute the RunJobFlow service method on AmazonElasticMapReduce.
*
* @return The response from the RunJobFlow service method, as returned
* by AmazonElasticMapReduce.
*
* @throws InternalServerErrorException
*
* @throws AmazonClientException
* If any internal errors are encountered inside the client while
* attempting to make the request or handle the response. For example
* if a network connection is not available.
* @throws AmazonServiceException
* If an error response is returned by AmazonElasticMapReduce indicating
* either a problem with the data in the request, or a server side issue.
*/
public RunJobFlowResult runJobFlow(RunJobFlowRequest runJobFlowRequest)
throws AmazonServiceException, AmazonClientException {
Request<RunJobFlowRequest> request = new RunJobFlowRequestMarshaller().marshall(runJobFlowRequest);
return invoke(request, new RunJobFlowResultStaxUnmarshaller());
}
/**
* <p>
* DescribeJobFlows returns a list of job flows that match all of the
* supplied parameters. The parameters can include a list of job flow
* IDs, job flow states, and restrictions on job flow creation date and
* time.
* </p>
* <p>
* Regardless of supplied parameters, only job flows created within the
* last two months are returned.
* </p>
* <p>
* If no parameters are supplied, then job flows matching either the
* following criteria are returned:
* </p>
*
* <ul>
* <li>Job flows created and completed in the last two weeks.</li>
* <li> Job flows created within the last two months that are in one of
* the following states: <code>RUNNING</code> ,
*
* <code>WAITING</code> ,
*
* <code>SHUTTING_DOWN</code> ,
*
* <code>STARTING</code> .
* </li>
*
* </ul>
* <p>
* Amazon Elastic MapReduce can return a maximum of 512 job flow
* descriptions.
* </p>
*
* @return The response from the DescribeJobFlows service method, as
* returned by AmazonElasticMapReduce.
*
* @throws InternalServerErrorException
*
* @throws AmazonClientException
* If any internal errors are encountered inside the client while
* attempting to make the request or handle the response. For example
* if a network connection is not available.
* @throws AmazonServiceException
* If an error response is returned by AmazonElasticMapReduce indicating
* either a problem with the data in the request, or a server side issue.
*/
public DescribeJobFlowsResult describeJobFlows() throws AmazonServiceException, AmazonClientException {
return describeJobFlows(new DescribeJobFlowsRequest());
}
private <X, Y extends AmazonWebServiceRequest> X invoke(Request<Y> request, Unmarshaller<X, StaxUnmarshallerContext> unmarshaller) {
request.setEndpoint(endpoint);
for (Entry<String, String> entry : request.getOriginalRequest().copyPrivateRequestParameters().entrySet()) {
request.addParameter(entry.getKey(), entry.getValue());
}
// Apply any additional service specific request handlers that need to be run
if (requestHandlers != null) {
for (RequestHandler requestHandler : requestHandlers) {
request = requestHandler.handleRequest(request);
}
}
QueryStringSigner<Y> signer = new QueryStringSigner<Y>(awsCredentials);
try {
signer.sign(request);
} catch (SignatureException e) {
throw new AmazonServiceException("Unable to sign request", e);
}
HttpRequest httpRequest = new HttpRequest(HttpMethodName.POST);
for (Entry<String, String> parameter : request.getParameters().entrySet()) {
httpRequest.addParameter(parameter.getKey(), parameter.getValue());
}
httpRequest.setServiceName(request.getServiceName());
httpRequest.setEndpoint(request.getEndpoint());
httpRequest.setResourcePath(request.getResourcePath());
StaxResponseHandler<X> responseHandler = new StaxResponseHandler<X>(unmarshaller);
DefaultErrorResponseHandler errorResponseHandler = new DefaultErrorResponseHandler(exceptionUnmarshallers);
return (X)client.execute(httpRequest, responseHandler, errorResponseHandler);
}
}