package org.xmlcml.ckan;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.net.URL;
import org.apache.commons.io.IOUtils;
import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.DefaultProxyRoutePlanner;
import org.apache.log4j.Logger;
/**
* Connection holds the connection details for this session
*
* @author Ross Jones <ross.jones@okfn.org>
* @version 1.7
* @since 2012-05-01
*/
public final class Connection {
public static final String APIKEY = "APIKEY";
private static final String HTTP_THEDATAHUB_ORG = "http://thedatahub.org";
public static final String X_CKAN_API_KEY = "X-CKAN-API-Key";
public static final String USER_AGENT = "User-Agent";
public static final String AUTHORIZATION = "Authorization";
private static final String HTTP = "http";
private static final String APPLICATION_JSON = "application/json";
private static final String HTTP_DATAHUB_IO = "http://datahub.io";
private static final String HTTP_PROXY_HOST = "http.proxyHost";
private static final String HTTP_PROXY_PORT = "http.proxyPort";
private static final Logger LOG = Logger.getLogger(Connection.class);
private String m_host;
private int m_port;
private String _apikey = null;
private int statusCode;
private HttpEntity httpEntity;
private HttpResponse httpResponse;
private CloseableHttpClient httpClient;
private HttpPost httpPost;
private HttpGet httpGet;
private URL currentUrl;
private HttpPost postRequest;
private String contentType;
public Connection() {
this(HTTP_DATAHUB_IO, 80);
}
public Connection( String host ) {
this( host, 80 );
}
public Connection( String host, int port ) {
setApiKey(System.getenv(APIKEY));
// Hack, since this.Post() does not follow redirects.
// http://stackoverflow.com/questions/7546849/httpclient-redirect-for-newbies
// http://stackoverflow.com/questions/5169468/handling-httpclient-redirects
// http://hc.apache.org/httpcomponents-client-ga/httpclient/apidocs/org/apache/http/client/RedirectStrategy.html
// http://hc.apache.org/httpcomponents-client-ga/httpclient/apidocs/index.html?overview-summary.html
if( HTTP_THEDATAHUB_ORG.equals(host) ) {
host = HTTP_DATAHUB_IO;
}
this.m_host = host;
this.m_port = port;
checkProperlyFormedURL();
this.contentType = "application/json";
}
private void checkProperlyFormedURL() {
try {
new URL( this.m_host + ":" + this.m_port + "/api");
} catch ( MalformedURLException mue ) {
throw new RuntimeException("malformed host/port in URL", mue);
}
}
public void setApiKey( String key ) {
this._apikey = key;
}
/**
* Makes a POST request
*
* Submits a POST HTTP request to the CKAN instance configured within
* the constructor, returning the entire contents of the response.
*
* @param path - The path to make the request to (appended to host and port info)
* @param data - The data to be posted to the URL
* @returns The String contents of the response.
*
* @throws A CKANException if the request fails
*/
protected String execPostRequest(String path, String data) throws CKANException {
URL url = createURLFromHostAndPort(path);
String body = execPostRequest(url, data);
return body;
}
/**
* Makes a GET request
*
*Experimental - CKAN may not like this
*
* Submits a GET HTTP request to the CKAN instance configured within
* the constructor, returning the entire contents of the response.
*
* @param path - The path to make the request to (appended to host and port info)
* @returns The String contents of the response.
*
* @throws A CKANException if the request fails
*/
protected String get(String path) throws CKANException {
URL url = createURLFromHostAndPort(path);
return execGetRequest(url);
}
/**
* Makes a POST request
*
* Submits a POST HTTP request to the CKAN instance configured within
* the constructor, returning the entire contents of the response.
*
* @param url - The URL to make the POST request to
* @returns The String contents of the response.
*
* @throws A CKANException if the request fails
*/
protected String execPostRequest(URL url) throws CKANException {
return execPostRequest(url, null);
}
/**
* Makes a POST request
*
* Submits a POST HTTP request to the CKAN instance configured within
* the constructor, returning the entire contents of the response.
*
* @param url - The URL to make the POST request to
* @param data - The data to be posted to the URL; may be null
* @param expect - the Expect field (e.g. 100-Continue); may be null
* @returns The String contents of the response.
*
* @throws A CKANException if the request fails
*/
protected String execPostRequest(URL url, String data) throws CKANException {
return execPostRequest(url, data, null);
}
/**
* Makes a POST request
*
* Submits a POST HTTP request to the CKAN instance configured within
* the constructor, returning the entire contents of the response.
*
* @param url - The URL to make the POST request to
* @param data - The data to be posted to the URL; may be null
* @param expect - the Expect field (e.g. 100-Continue); may be null
* @returns The String contents of the response.
*
* @throws A CKANException if the request fails
*/
protected String execPostRequest(URL url, String data, String expect) throws CKANException {
currentUrl = url;
String body = "";
httpClient = createHttpClientOrProxy();
try {
httpPost = createPostRequest(data, url);
// content-length is already present
if (contentType != null) {
httpPost.setHeader("Content-Type", contentType);
}
if (expect != null) {
httpPost.setHeader("Expect", expect);
}
body = executePost();
} catch( IOException ioe ) {
throw new RuntimeException("Cannot post/read", ioe);
} finally {
try {
httpClient.close();
} catch (IOException ioe) {
throw new RuntimeException("Cannout close client: ", ioe);
}
}
Header[] headers = httpResponse.getAllHeaders();
LOG.trace("headers: "+headers.length);
for (Header header : headers) {
LOG.debug("header: "+header);
}
return body;
}
/**
* Makes a GET request
*
* Submits a GET HTTP request to the CKAN instance configured within
* the constructor, returning the entire contents of the response.
*
* @param url - The URL to make the POST request to
* @returns The String contents of the response.
*
* @throws A CKANException if the request fails
*/
protected String execGetRequest(URL url) throws CKANException {
String body = "";
httpClient = Connection.createHttpClientOrProxy();
try {
httpGet = createGetRequest(url);
body = executeGet();
} catch( IOException ioe ) {
throw new RuntimeException("Cannot post/read", ioe);
} finally {
try {
httpClient.close();
} catch (IOException ioe) {
throw new RuntimeException("Cannout close client: ", ioe);
}
}
return body;
}
private String executePost() throws IOException, ClientProtocolException {
httpResponse = httpClient.execute(httpPost);
return processResponse();
}
private String executeGet() throws IOException, ClientProtocolException {
httpResponse = httpClient.execute(httpGet);
return processResponse();
}
private String processResponse() throws IOException {
String body = null;
httpEntity = httpResponse.getEntity();
statusCode = httpResponse.getStatusLine().getStatusCode();
LOG.info("response code " + statusCode);
if (statusCode == 100){
LOG.debug("100 continue ");
} else if (statusCode == 200){
body = createLinesFromEntity();
} else if (statusCode == 301 || statusCode == 302 || statusCode == 303 || statusCode == 307) {
LOG.debug("server redirect ");
redirect();
} else if (statusCode == 400) {
LOG.error("400 Bad request: "+currentUrl);
System.out.println("content: "+IOUtils.toString(httpEntity.getContent()));
} else if (statusCode == 404) {
LOG.error("NOT FOUND: "+currentUrl);
} else if (statusCode == 500) {
LOG.error("Internal Server Error "+currentUrl);
} else {
LOG.error("Untrapped HTTP code: "+statusCode);
}
return body;
}
private String createLinesFromEntity() throws IOException {
StringBuilder sb = new StringBuilder();
BufferedReader br = new BufferedReader(
new InputStreamReader((httpEntity.getContent())));
String line = "";
while ((line = br.readLine()) != null) {
sb.append(line);
}
return sb.toString();
}
private void redirect() {
// see http://hc.apache.org/httpclient-3.x/redirects.html
// Header locationHeader = method.getResponseHeader("location");
// if (locationHeader != null) {
// redirectLocation = locationHeader.getValue();
// postRequest.getMethod().getResponseHeader("location");
LOG.debug("Redirect NYI");
}
HttpPost createPostRequest(URL url) {
HttpPost httpPost = createPostRequest(null, url);
return httpPost;
}
HttpPost createPostRequest(String data, URL url) {
postRequest = new HttpPost(url.toString());
postRequest.setHeader( X_CKAN_API_KEY, this._apikey );
if (data != null) {
try {
StringEntity stringEntity = new StringEntity(data);
LOG.debug("stringEntity: "+ stringEntity.getContentLength());
postRequest.setEntity(stringEntity);
} catch (UnsupportedEncodingException e) {
throw new RuntimeException("bad encoding: ", e);
}
}
return postRequest;
}
HttpGet createGetRequest(URL url)
throws UnsupportedEncodingException {
HttpGet httpGet = new HttpGet(url.toString());
httpGet.setHeader( X_CKAN_API_KEY, this._apikey );
return httpGet;
}
public static CloseableHttpClient createHttpClientOrProxy() {
/**
* try:
* CloseableHttpClient client = HttpClients.createSystem();
* instead of the code below
*
* see http://stackoverflow.com/questions/20716909/how-do-i-replace-deprecated-httpclient-getparams-with-requestconfig
*/
CloseableHttpClient httpClient = HttpClientBuilder.create().build();
/*
* Set an HTTP proxy if it is specified in system properties.
*
* http://docs.oracle.com/javase/6/docs/technotes/guides/net/proxies.html
* http://hc.apache.org/httpcomponents-client-ga/httpclient/examples/org/apache/http/examples/client/ClientExecuteProxy.java
*/
if( isSet(System.getProperty(HTTP_PROXY_HOST)) ) {
LOG.warn("http.proxyHost = " + System.getProperty(HTTP_PROXY_HOST) );
LOG.warn("http.proxyPort = " + System.getProperty(HTTP_PROXY_PORT));
HttpClientBuilder hcBuilder = HttpClients.custom();
// Set HTTP proxy, if specified in system properties
if( isSet(System.getProperty(HTTP_PROXY_HOST)) ) {
int port = 80;
if( isSet(System.getProperty(HTTP_PROXY_PORT)) ) {
port = Integer.parseInt(System.getProperty(HTTP_PROXY_PORT));
}
HttpHost proxy = new HttpHost(System.getProperty(HTTP_PROXY_HOST), port, HTTP);
DefaultProxyRoutePlanner routePlanner = new DefaultProxyRoutePlanner(proxy);
hcBuilder.setRoutePlanner(routePlanner);
}
httpClient = hcBuilder.build();
}
return httpClient;
}
private URL createURLFromHostAndPort(String path) throws CKANException {
URL url = null;
try {
url = new URL( this.m_host + ":" + this.m_port + path);
LOG.warn("POSTing to " + url.toString());
} catch ( MalformedURLException mue ) {
throw new CKANException("malformed URL"+mue.getMessage());
}
return url;
}
private static boolean isSet(String string) {
return string != null && string.length() > 0;
}
public HttpResponse getHttpResponse() {
return httpResponse;
}
public int getStatusCode() {
return statusCode;
}
}