private void downloadSource(Source source) {
log.info("Spidering: "+source);
Model downloadModel = ModelFactory.createDefaultModel();
try {
URL url = new URL(source.getURIRef());
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
connection.setInstanceFollowRedirects(false);
connection.addRequestProperty(HeaderName.ACCEPT.toString(), "application/rdf+xml; q=1, application/xml; q=.9, text/xml; q=.8, */*; q=.1");
connection.setReadTimeout(3000);
connection.connect();
int responseCode = connection.getResponseCode();
if (responseCode != 200) {
log.info("Response code: "+connection.getResponseCode());
if ((responseCode >= 300) && (responseCode < 400)) {
String location = connection.getHeaderField(HeaderName.LOCATION.toString());
if (location != null) {
connection.disconnect();
downloadSource(new SourceImpl(location));
}
}
return;
}
downloadModel.read(connection.getInputStream(), source.getURIRef());
store.assertGraph(source, new FCAGraphImpl(downloadModel));
connection.disconnect();
} catch (MalformedURLException e) {
throw new RuntimeException(e);
} catch (IOException e) {
throw new RuntimeException(e);
} finally {