}
// Some horrible logic to handle a DB cycle being >max records
int nNumRecordsToSkip = 0;
Integer distributionToken = null;
SourceHarvestStatusPojo lastHarvestInfo = source.getHarvestStatus();
// (this is never null when called from the harvester by construction)
if ((null != lastHarvestInfo) && (null != lastHarvestInfo.getHarvest_message())) { // either in delta query mode, or truncation mode of initial query
String harvestMessage = null;
if ((null == source.getDistributionTokens()) || source.getDistributionTokens().isEmpty()) {
harvestMessage = lastHarvestInfo.getHarvest_message();
}
else { // (currently only support one slice per harvester)
distributionToken = source.getDistributionTokens().iterator().next();
if (null != lastHarvestInfo.getDistributedStatus()) {
harvestMessage = lastHarvestInfo.getDistributedStatus().get(distributionToken.toString());
}
}//TODO (INF-2120): TOTEST
if (null != harvestMessage) {
harvestMessage = harvestMessage.replaceAll("\\[[0-9T:-]+\\]", "").trim();
if (harvestMessage.startsWith("query:")) { // Initial query
try {
int nEndOfNum = harvestMessage.indexOf('.', 6);
if (nEndOfNum > 0) {
nNumRecordsToSkip = Integer.parseInt(harvestMessage.substring(6, nEndOfNum));
}
}
catch (Exception e) {} // Do nothing, just default to 0
lastHarvestInfo = null; // (go through initial query branch below)
}
else { // Delta query or error!
if (harvestMessage.startsWith("deltaquery:")) { // Delta query, phew
try {
int nEndOfNum = harvestMessage.indexOf('.', 11);
if (nEndOfNum > 0) {
nNumRecordsToSkip = Integer.parseInt(harvestMessage.substring(11, nEndOfNum));
}
}
catch (Exception e) {} // Do nothing, just default to 0
}
}
}
}//TESTED ("query:" by eye, "deltaquery:" implicitly as cut and paste) //TOTEST with "." and with other fields added
// If the query has been performed before run delta and delete queries
if ((null != lastHarvestInfo) && (null != lastHarvestInfo.getHarvest_message()))
{
String deltaQuery = null;
String deleteQuery = null;
String deltaDate = null;
// Get the date last harvested from the harvest object and then convert the date
// to the proper format for the database being queried
if (source.getHarvestStatus().getHarvested() != null)
{
if (null != source.getHarvestStatus().getRealHarvested()) { // (this is more accurate)
deltaDate = getDatabaseDateString(dt, source.getHarvestStatus().getRealHarvested());
}
else {
deltaDate = getDatabaseDateString(dt, source.getHarvestStatus().getHarvested());
}
}
// Delta Query - get new data that has appeared in our source since the harvest last run
// Important Note: The query should be against a "last modified" field not publishedDate unless they are
// equal to ensure new records get added properly and not ignored by the harvester
if ((source.getDatabaseConfig().getDeltaQuery() != null) && !source.getDatabaseConfig().getDeltaQuery().isEmpty())
{
deltaQuery = source.getDatabaseConfig().getDeltaQuery();
// Replace '?' in the delta query with the date value retrieved via getHarvested()
if ((source.getHarvestStatus().getHarvested() != null) && (deltaQuery.contains("?")))
{
deltaQuery = deltaQuery.replace("?", "'" + deltaDate + "'").toString();
}
}
// Skip logic:
if (deltaQuery.contains("!SKIP!")) {
deltaQuery = deltaQuery.replace("!SKIP!", new Integer(nNumRecordsToSkip).toString());
}
else {
rdbms.setRecordsToSkip(nNumRecordsToSkip);
}
// Distribution logic, currently manual:
if ((null != distributionToken) && deltaQuery.contains("!TOKEN!")) {
deltaQuery = deltaQuery.replace("!TOKEN!", distributionToken.toString());
}//TODO (INF-2120): TOTEST
if ((null != source.getDistributionFactor()) && deltaQuery.contains("!NTOKENS!")) {
deltaQuery = deltaQuery.replace("!NTOKENS!", source.getDistributionFactor().toString());
}//TODO (INF-2120): TOTEST
// Delete docs from the index if they have been deleted from the source
// database. Designed to check a "deleted" table to get records that have been
// deleted to identify the records to remove form the index.
if ((source.getDatabaseConfig().getDeleteQuery() != null) && !source.getDatabaseConfig().getDeleteQuery().isEmpty())
{
deleteQuery = source.getDatabaseConfig().getDeleteQuery();
// Distribution logic, currently manual:
if ((null != distributionToken) && deleteQuery.contains("!TOKEN!")) {
deleteQuery = deltaQuery.replace("!TOKEN!", distributionToken.toString());
}//TODO (INF-2120): TOTEST
if ((null != source.getDistributionFactor()) && deleteQuery.contains("!NTOKENS!")) {
deleteQuery = deleteQuery.replace("!NTOKENS!", source.getDistributionFactor().toString());
}//TODO (INF-2120): TOTEST
// Replace '?' in the delete query with the date value retrieved via getHarvested()
if ((source.getHarvestStatus().getHarvested() != null) && (deleteQuery.contains("?")))
{
deleteQuery = deleteQuery.replace("?", "'" + deltaDate + "'").toString();
}
rdbms.setQuery(deleteQuery, deltaDate);
String errMessage = rdbms.executeQuery();
if (null == errMessage) {
deleteRecords(rdbms.getResultSet(), source);
}
else {
_context.getHarvestStatus().update(source, new Date(), HarvestEnum.error, "Error when harvesting DB: " + errMessage, false, true);
// Close the connection
try {
rdbms.closeConnection();
rdbms = null;
}
catch (Exception e) {
// Do nothing, we've already updated the harvest
}
return;
}//TOTEST (INF-2349)
}
// Set the rdbms query = deltaQuery
if ((deltaQuery != null) && (deltaDate != null))
{
rdbms.setQuery(deltaQuery, deltaDate);
}
else
{
rdbms.setQuery(source.getDatabaseConfig().getQuery());
}
}
else // Very first import (though might be in horrible initial query mode)
{
String query = source.getDatabaseConfig().getQuery();
if (query.contains("!SKIP!")) {
query = query.replace("!SKIP!", new Integer(nNumRecordsToSkip).toString());
}
else {
rdbms.setRecordsToSkip(nNumRecordsToSkip);
}
// Distribution logic, currently manual:
if ((null != distributionToken) && query.contains("!TOKEN!")) {
query = query.replace("!TOKEN!", distributionToken.toString());
}//TODO (INF-2120): TOTEST
if ((null != source.getDistributionFactor()) && query.contains("!NTOKENS!")) {
query = query.replace("!NTOKENS!", source.getDistributionFactor().toString());
}//TODO (INF-2120): TOTEST
// Set rdbs query = default source.database.query value
rdbms.setQuery(query);
}
// Execute the specified query
String sErr = rdbms.executeQuery();
if (null != sErr) {
_context.getHarvestStatus().update(source, new Date(), HarvestEnum.error, "Error when harvesting DB: " + sErr, false, true);
}
else {
// Build the list of docs using the result set from the query
try {
boolean bTruncated = addRecords(rdbms.getResultSet(), rdbms.getMetaData(), source);
// Update source document with harvest success message
String truncationMode = null;
if (bTruncated) {
source.setReachedMaxDocs();
if ((null != lastHarvestInfo) && (null != lastHarvestInfo.getHarvest_message())) {
// Was a delta query
truncationMode = new StringBuffer("deltaquery:").append(nNumRecordsToSkip + this.docsToAdd.size()).append('.').toString();
}
else { // Was an initial import
truncationMode = new StringBuffer("query:").append(nNumRecordsToSkip + this.docsToAdd.size()).append('.').toString();