throw e;
}
try {
//Dedup code, ironically enough partly duplicated in parse(), probably unnecessarily
DuplicateManager qr = _context.getDuplicateManager();
for(DocumentPojo doc: files)
{
try {
duplicateSources.clear();
if (null != doc.getSourceUrl()) {
boolean add = true;
// However still need to check for duplicates so can update entities correctly (+maintain _ids, etc)
// We only do this if the source URL changes (unless URL is taken from the object in which case all bets are off)
boolean sourceUrlUpdated = sourceUrlsGettingUpdated.contains(doc.getSourceUrl());
if (!doc.getHasDefaultUrl() || sourceUrlUpdated) { // src URL for a given URL
// (only if the the sourceUrl is not new...)
if (qr.isDuplicate_Url(doc.getUrl(), source, duplicateSources)) {
doc.setUpdateId(qr.getLastDuplicateId()); // (set _id to doc we're going to replace)
if (!sourceUrlUpdated && !_deleteExistingFilesBySourceKey) {
// Here update instead so we delete the old doc and add the new one
add = false;
docsToUpdate.add(doc);
}//TESTED
else {
// (else *still* don't add this to updates because we've added the source URL or source key to the delete list)
// (hence approximate create with the updateId...)
if (null != doc.getUpdateId()) {
doc.setCreated(new Date(doc.getUpdateId().getTime()));
}//TESTED
}//TESTED
}
//(note we don't get about duplicate sources in this case - just too complex+rare a case)
}//TESTED (src url changing, different src url, non-default URL)
// For composite files we (almost always) delete everything that already exists (via docsToRemove) and then add new docs
if (add) {
docsToAdd.add(doc);
}
//TESTED
}
else if (qr.isDuplicate_Url(doc.getUrl(), source, duplicateSources)) {
// Other files, if the file already exists then update it (essentially, delete/add)
doc.setUpdateId(qr.getLastDuplicateId()); // (set _id to doc we're going to replace)
docsToUpdate.add(doc);
}
else { // if duplicateSources is non-empty then this URL is a duplicate of one from a different source
if (!duplicateSources.isEmpty()) {
doc.setDuplicateFrom(duplicateSources.getFirst());