}
public void retrieveFiles(PropFilesInfo pfi, final DataFilesInfo dfi)
throws RetrievalMethodException {
FileRetrievalSystem dataFilesFRS = null;
try {
this.startPropFileDownload(pfi);
(dataFilesFRS = new FileRetrievalSystem(config, siteInfo))
.initialize();
dataFilesFRS.registerDownloadListener(linker);
File[] propFiles = null;
while ((propFiles = getCurrentlyDownloadedPropFiles(pfi)).length > 0
|| downloadingProps) {
for (File propFile : propFiles) {
try {
Parser parser = pfi.getParserForFile(propFile);
Class<RetrievalMethod> rmClass = config.getParserInfo()
.getRetrievalMethod(parser);
RetrievalMethod rm = null;
if ((rm = this.classToRmMap.get(rmClass)) == null) {
LOG.log(Level.INFO, "Creating '"
+ rmClass.getCanonicalName()
+ "' to download data files");
rm = PushPullObjectFactory
.createNewInstance(rmClass);
this.classToRmMap.put(rmClass, rm);
}
rm.processPropFile(dataFilesFRS, parser, propFile, dfi,
linker);
} catch (ParserException e) {
LOG.log(Level.SEVERE, "Failed to parse property file "
+ propFile + " : " + e.getMessage(), e);
linker.markAsFailed(propFile,
"Failed to parse property file " + propFile
+ " : " + e.getMessage());
} catch (Exception e) {
LOG.log(Level.SEVERE,
"Failed to finish downloading per property files "
+ propFile.getAbsolutePath() + " : "
+ e.getMessage(), e);
linker.markAsFailed(propFile,
"Error while downloading per property file "
+ propFile.getAbsolutePath() + " : "
+ e.getMessage());
}
}
dataFilesFRS.waitUntilAllCurrentDownloadsAreComplete();
for (File propFile : propFiles) {
try {
if (pfi.getLocalDir().equals(pfi.getOnSuccessDir())
|| pfi.getLocalDir().equals(pfi.getOnFailDir()))
alreadyProcessedPropFiles.add(propFile);
this.movePropsFileToFinalDestination(pfi, propFile,
linker.getErrorsAndEraseLinks(propFile));
} catch (Exception e) {
e.printStackTrace();
LOG.log(Level.SEVERE,
"Error occurred while writing errors to error dir for file '"
+ propFile + "' : " + e.getMessage());
}
}
}
} catch (Exception e) {
e.printStackTrace();
} finally {
if (dataFilesFRS != null)
dataFilesFRS.shutdown();
alreadyProcessedPropFiles.clear();
linker.clear();
}
}