* Downloads the changeset files from the server and writes their contents
* to the output task.
*/
private void download() {
IntervalDownloaderConfiguration configuration;
TimestampTracker timestampTracker;
ChangesetFileNameFormatter fileNameFormatter;
Date currentTime;
Date maximumTime;
URL baseUrl;
int maxDownloadCount;
int downloadCount;
ArrayList<File> tmpFileList;
ArrayList<RunnableChangeSource> tasks;
ArrayList<TaskRunner> taskRunners;
boolean tasksSuccessful;
// Instantiate utility objects.
configuration = new IntervalDownloaderConfiguration(new File(workingDirectory, CONFIG_FILE));
timestampTracker = new TimestampTracker(
new File(workingDirectory, TSTAMP_FILE),
new File(workingDirectory, TSTAMP_NEW_FILE)
);
fileNameFormatter = new ChangesetFileNameFormatter(
configuration.getChangeFileBeginFormat(),
configuration.getChangeFileEndFormat()
);
// Create the base url.
try {
baseUrl = new URL(configuration.getBaseUrl());
} catch (MalformedURLException e) {
throw new OsmosisRuntimeException(
"Unable to convert URL string (" + configuration.getBaseUrl() + ") into a URL.", e);
}
tmpFileList = new ArrayList<File>();
// Load the current time from the timestamp tracking file.
currentTime = timestampTracker.getTime();
// Load the latest timestamp from the server.
maximumTime = getServerTimestamp(baseUrl);
// Process until all files have been retrieved from the server.
maxDownloadCount = configuration.getMaxDownloadCount();
downloadCount = 0;
while ((maxDownloadCount == 0 || downloadCount < maxDownloadCount) && currentTime.before(maximumTime)) {
Date nextTime;
String downloadFileName;
// Calculate the end of the next time interval.
nextTime = new Date(currentTime.getTime() + configuration.getIntervalLength());
// Generate the filename to be retrieved from the server.
downloadFileName = fileNameFormatter.generateFileName(currentTime, nextTime);
// Download the changeset from the server.
tmpFileList.add(downloadChangesetFile(downloadFileName, baseUrl));
// Move the current time to the next interval.
currentTime = nextTime;
// Increment the current download count.
downloadCount++;
}
// Generate a set of tasks for loading the change files and merge them
// into a single change stream.
tasks = new ArrayList<RunnableChangeSource>();
for (File tmpFile : tmpFileList) {
XmlChangeReader changeReader;
// Generate a change reader task for the current task.
changeReader = new XmlChangeReader(
tmpFile,
true,
CompressionMethod.GZip
);
// If tasks already exist, a change merge task must be used to merge
// existing output with this task output, otherwise this task can be
// added to the list directly.
if (tasks.size() > 0) {
ChangeMerger changeMerger;
// Create a new change merger merging the last task output with the current task.
changeMerger = new ChangeMerger(ConflictResolutionMethod.LatestSource, 10);
// Connect the inputs of this merger to the most recent change
// output and the new change output.
tasks.get(tasks.size() - 1).setChangeSink(changeMerger.getChangeSink(0));
changeReader.setChangeSink(changeMerger.getChangeSink(1));
tasks.add(changeReader);
tasks.add(changeMerger);
} else {
tasks.add(changeReader);
}
}
// We only need to execute sub-threads if tasks exist, otherwise we must
// notify the sink that we have completed.
if (tasks.size() > 0) {
// Connect the last task to the change sink.
tasks.get(tasks.size() - 1).setChangeSink(changeSink);
// Create task runners for each of the tasks to provide thread
// management.
taskRunners = new ArrayList<TaskRunner>(tasks.size());
for (int i = 0; i < tasks.size(); i++) {
taskRunners.add(new TaskRunner(tasks.get(i), "Thread-" + taskId + "-worker" + i));
}
// Launch all of the tasks.
for (int i = 0; i < taskRunners.size(); i++) {
TaskRunner taskRunner;
taskRunner = taskRunners.get(i);
LOG.fine("Launching changeset worker + " + i + " in a new thread.");
taskRunner.start();
}
// Wait for all the tasks to complete.
tasksSuccessful = true;
for (int i = 0; i < taskRunners.size(); i++) {
TaskRunner taskRunner;
taskRunner = taskRunners.get(i);
LOG.fine("Waiting for changeset worker " + i + " to complete.");
try {
taskRunner.join();
} catch (InterruptedException e) {
// We are already in an error condition so log and continue.
LOG.log(Level.WARNING, "The wait for task completion was interrupted.", e);
}
if (!taskRunner.isSuccessful()) {
LOG.log(Level.SEVERE, "Changeset worker " + i + " failed", taskRunner.getException());
tasksSuccessful = false;
}
}
} else {
changeSink.complete();
tasksSuccessful = true;
}
// Remove the temporary files.
for (File tmpFile : tmpFileList) {
if (!tmpFile.delete()) {
LOG.warning("Unable to delete file " + tmpFile.getName());
}
}
if (!tasksSuccessful) {
throw new OsmosisRuntimeException("One or more changeset workers failed.");
}
// Update the timestamp tracker.
timestampTracker.setTime(currentTime);
}