double score = 100.0;
if (null != _asyncRequestsPerQuery) {
int added = 0;
BasicDBObject doc = null;
BasicDBList bsonArray = new BasicDBList();
PeekingIterator<FederatedRequest> it = Iterators.peekingIterator(_asyncRequestsPerQuery.iterator());
while (it.hasNext()) {
FederatedRequest request = it.next();
if (null == request.cachedDoc) { // no cached doc
try {
if (null == request.cachedResult) { // no cached value
if (null != request.importThread) {
// 1) wait for the thread to finish
if (null == request.endpointInfo.queryTimeout_secs) {
request.endpointInfo.queryTimeout_secs = 300;
}
for (int timer = 0; timer < request.endpointInfo.queryTimeout_secs; timer++) {
try {
request.importThread.join(1000L);
if (!request.importThread.isAlive()) {
break;
}
}//TESTED (by hand)
catch (Exception e) {
}
}
if (request.importThread.isAlive()) {
request.errorMessage = new RuntimeException("Script timed out");
}//TESTED (by hand)
// 2) Get the results
if (null != request.errorMessage) {
if (_testMode) {
throw new RuntimeException(request.errorMessage);
}
}
else if (null == request.scriptResult) {
if (_testMode) {
throw new RuntimeException("Script mode: no cached result found from: " + request.requestParameter);
}
}
else {
String url = buildScriptUrl(request.mergeKey, request.queryIndex);
if (_cacheMode) { // (don't cache python federated queries in test mode)
this.cacheApiResponse(url, request.scriptResult, request.endpointInfo);
}
bsonArray.add(request.scriptResult);
}
} // end script mode
else { // HTTP mode
Response endpointResponse = request.responseFuture.get();
request.asyncClient.close();
request.asyncClient = null;
String jsonStr = endpointResponse.getResponseBody();
String url = endpointResponse.getUri().toURL().toString();
Object bsonUnknownType = com.mongodb.util.JSON.parse(jsonStr);
BasicDBObject bson = null;
if (bsonUnknownType instanceof BasicDBObject) {
bson = (BasicDBObject) bsonUnknownType;
}
else if (bsonUnknownType instanceof BasicDBList) {
bson = new BasicDBObject("array", bsonUnknownType);
}
else if (bsonUnknownType instanceof String) {
bson = new BasicDBObject("value", bsonUnknownType);
}
//DEBUG
if (_DEBUG) _logger.debug("DEB: postQA1: " + url + ": " + jsonStr);
if (null != bson) {
MongoDbUtil.enforceTypeNamingPolicy(bson, 0);
this.cacheApiResponse(url, bson, request.endpointInfo);
bsonArray.add(bson);
}
}//(end script vs request method)
}//TESTED (3.1, 4.2)
else { // (just used cached value)
//DEBUG
if (_DEBUG) _logger.debug("DEB: postQA2: " + request.cachedResult.toString());
bsonArray.add((BasicDBObject)request.cachedResult.get(SimpleFederatedCache.cachedJson_));
}//TESTED (4.1, 4.3)
}
catch (Exception e) {
//DEBUG
if (null == request.subRequest) {
_logger.error("Error with script: " + e.getMessage());
if (_testMode) {
throw new RuntimeException("Error with script: " + e.getMessage(), e);
}
}
else {
_logger.error("Error with " + request.subRequest.endPointUrl + ": " + e.getMessage());
if (_testMode) {
throw new RuntimeException("Error with " + request.subRequest.endPointUrl + ": " + e.getMessage(), e);
}
}
}
if (!it.hasNext() || (request.mergeKey != it.peek().mergeKey)) { // deliberate ptr arithmetic
String url = buildScriptUrl(request.mergeKey, request.queryIndex);
//DEBUG
if (_DEBUG) _logger.debug("DEB: postQA3: " + url + ": " + bsonArray);
doc = createDocFromJson(bsonArray, url, request, request.endpointInfo);
}
} // (end if no cached doc)
else { // cached doc, bypass lots of processing because no merging and doc already built
doc = request.cachedDoc;
}//TESTED (by hand)
if (null != doc) {
// Cache the document unless already cached (or caching disabled)
if ((null == request.cachedDoc) && _cacheMode &&
((null == request.endpointInfo.cacheTime_days) || (request.endpointInfo.cacheTime_days >= 0)))
{
if (null != request.cachedDoc_expired) {
ObjectId updateId = request.cachedDoc_expired.getObjectId(DocumentPojo.updateId_);
if (null != updateId) {
doc.put(DocumentPojo.updateId_, updateId);
}
else {
doc.put(DocumentPojo.updateId_, request.cachedDoc_expired.getObjectId(DocumentPojo._id_));
}
BasicDBObject docUpdate = new BasicDBObject(DocumentPojo.url_, doc.getString(DocumentPojo.url_));
docUpdate.put(DocumentPojo.sourceKey_, doc.getString(DocumentPojo.sourceKey_));
DbManager.getDocument().getMetadata().remove(docUpdate);
//DEBUG
if (_DEBUG) _logger.debug("DEB: postQA4a: re-cached ... " + docUpdate.toString() + ": " + doc.getObjectId(DocumentPojo.updateId_));
}
//DEBUG
if (_DEBUG) _logger.debug("DEB: postQA4b: cached ... " + doc);
DbManager.getDocument().getMetadata().save(doc);
}//TESTED (by hand, 3 cases: cached not expired, cached expired first time, cached expired multiple times)
if (!grabbedScores) {
if (!docs.isEmpty()) {
BasicDBObject topDoc = docs.get(0);
aggregateSignif = topDoc.getDouble(DocumentPojo.aggregateSignif_, aggregateSignif);
queryRelevance = topDoc.getDouble(DocumentPojo.queryRelevance_, queryRelevance);
score = topDoc.getDouble(DocumentPojo.score_, score);
grabbedScores = true;
// OK would also like to grab the original matching entity
BasicDBList ents = (BasicDBList) topDoc.get(DocumentPojo.entities_);
if (null != ents) {
for (Object entObj: ents) {
BasicDBObject ent = (BasicDBObject)entObj;
String entIndex = ent.getString(EntityPojo.index_, "");
if (entIndex.equals(request.queryIndex)) {
ents = (BasicDBList) doc.get(DocumentPojo.entities_);
if (null != ents) {
ents.add(ent);
}
break;
}
}
}//TESTED (by hand)