// main loop to keep requesting more objects until we're done
List<Element> records;
Set<String> errorSet = new HashSet<String>();
ListRecords listRecords = new ListRecords(oaiSource, fromDate, toDate, oaiSetId, descMDPrefix);
log.debug("Harvesting request parameters: listRecords " + oaiSource + " " + fromDate + " " + toDate + " " + oaiSetId + " " + descMDPrefix);
if (listRecords != null)
log.info("HTTP Request: " + listRecords.getRequestURL());
while (listRecords != null)
{
records = new ArrayList<Element>();
oaiResponse = db.build(listRecords.getDocument());
if (listRecords.getErrors() != null && listRecords.getErrors().getLength() > 0)
{
for (int i=0; i<listRecords.getErrors().getLength(); i++)
{
String errorCode = listRecords.getErrors().item(i).getAttributes().getNamedItem("code").getTextContent();
errorSet.add(errorCode);
}
if (errorSet.contains("noRecordsMatch"))
{
log.info("noRecordsMatch: OAI server did not contain any updates");
harvestRow.setHarvestResult(new Date(), "OAI server did not contain any updates");
harvestRow.setHarvestStatus(HarvestedCollection.STATUS_READY);
harvestRow.update();
return;
} else {
throw new HarvestingException(errorSet.toString());
}
}
else
{
root = oaiResponse.getRootElement();
records.addAll(root.getChild("ListRecords", OAI_NS).getChildren("record", OAI_NS));
}
// Process the obtained records
if (records != null && records.size()>0)
{
log.info("Found " + records.size() + " records to process");
for (Element record : records) {
// check for STOP interrupt from the scheduler
if (HarvestScheduler.interrupt == HarvestScheduler.HARVESTER_INTERRUPT_STOP)
throw new HarvestingException("Harvest process for " + targetCollection.getID() + " interrupted by stopping the scheduler.");
// check for timeout
if (expirationTime.before(new Date()))
throw new HarvestingException("runHarvest method timed out for collection " + targetCollection.getID());
processRecord(record,OREPrefix);
ourContext.commit();
}
}
// keep going if there are more records to process
resumptionToken = listRecords.getResumptionToken();
if (resumptionToken == null || resumptionToken.length() == 0) {
listRecords = null;
}
else {
listRecords = new ListRecords(oaiSource, resumptionToken);
}
targetCollection.update();
ourContext.commit();
}
}