if (null != _queryExtensions) {
queryId = new ObjectId();
queryExtensions = new ArrayList<IQueryExtension>(_queryExtensions.size());
for (Class<IQueryExtension> queryExtensionClass: _queryExtensions) {
// Don't catch any exceptions thrown here - let it bubble upwards
IQueryExtension queryExtension = queryExtensionClass.newInstance();
queryExtension.preQueryActivities(queryId, query, communityIdStrs);
queryExtensions.add(queryExtension);
}
}//TESTED (see test.QueryExtensionsTestCode)
// Built-in federated query engine ...
if (null != _federatedQueryCache) {
// 2 modes:
// 1) If srcInclude is true(default) then check each source vs the table
// 2) If srcInclude is false, or no sources specified, then check each community vs the table
// 1:
if ((null != query.input) && (null != query.input.sources) && ((null == query.input.srcInclude) || query.input.srcInclude))
{
for (String srcKey: query.input.sources) {
FederatedQueryInMemoryCache fedQueryCacheEl = _federatedQueryCache.get(srcKey);
if (null != fedQueryCacheEl) {
if (null == this._builtInFederatedQueryEngine) {
_builtInFederatedQueryEngine = new SimpleFederatedQueryEngine();
}
_builtInFederatedQueryEngine.addEndpoint(fedQueryCacheEl.source);
}
}
}//TESTED (//TESTED (http://localhost:8184/knowledge/document/query/53ab42a2e4b04bcfe2de4387?qt[0].entity=%22garyhart.com/externaldomain%22&output.docs.numReturn=10&input.sources=inf...federated.externaldomain.&input.srcInclude=true))
// 2:
else { //Get federated queries from communities
HashSet<String> excludeSrcs = null;
for (String commIdStr: communityIdStrs) {
FederatedQueryInMemoryCache fedQueryCacheEl = _federatedQueryCache.get(commIdStr);
if (null != fedQueryCacheEl) {
if ((null != query.input) && (null != query.input.sources)) { // (there are exclude sources)
if (null == excludeSrcs) {
excludeSrcs = new HashSet<String>(query.input.sources);
}
}//TESTED (http://localhost:8184/knowledge/document/query/53ab42a2e4b04bcfe2de4387?qt[0].entity=%22garyhart.com/externaldomain%22&output.docs.numReturn=10&input.sources=inf...federated.externaldomain.&input.srcInclude=false)
for (Map.Entry<String, SourceFederatedQueryConfigPojo> fedQueryKV: fedQueryCacheEl.sources.entrySet()) {
if ((null == excludeSrcs) || !excludeSrcs.contains(fedQueryKV.getKey())) {
if (null == this._builtInFederatedQueryEngine) {
_builtInFederatedQueryEngine = new SimpleFederatedQueryEngine();
}
_builtInFederatedQueryEngine.addEndpoint(fedQueryKV.getValue());
}
}
}
}//TESTED (by hand)
}
if (null != _builtInFederatedQueryEngine) {
_builtInFederatedQueryEngine.preQueryActivities(queryId, query, communityIdStrs);
}
}
// 0.6.2: the main query
if ((null != query.explain) && query.explain) { // (for diagnostic - will return lucene explanation)
searchSettings.setExplain(true);
}
SearchResponse queryResults = null;
// (_source can now be enabled, so this is necessary to avoid returning it)
searchSettings.addFields();
if ((null != query.raw) && (null != query.raw.query))
{
// (Can bypass all other settings)
searchSettings.setQuery(query.raw.query);
queryResults = indexMgr.doQuery(null, tempFilterInfo.parentFilterObj, searchSettings);
}//TESTED '{ "raw": { "match_all": {} } }'
else
{
// Where I can, use the source filter as part of the query so that
// facets will apply to query+filter, not just filter
queryObj = QueryBuilders.boolQuery().must(queryObj).must(QueryBuilders.constantScoreQuery(tempFilterInfo.parentFilterObj).boost(0.0F));
queryResults = indexMgr.doQuery(queryObj, null, searchSettings);
}//TESTED '{}' etc
long nLuceneTime = queryResults.getTookInMillis();
// 0.7] Lucene scores
long nProcTime = 0;
long nProcTime_tmp = System.currentTimeMillis();
StatisticsPojo stats = new StatisticsPojo();
stats.found = queryResults.getHits().getTotalHits();
stats.start = (long)nRecordsToSkip;
if (nRecordsToGet > 0) {
stats.setScore(queryResults.getHits(), (null != query.score.geoProx)||(null != query.score.timeProx), (null != query.explain) && query.explain);
}
//DEBUG
//System.out.println(new Gson().toJson(queryResults));
nProcTime += (System.currentTimeMillis() - nProcTime_tmp);
// 0.8] Get data from Mongo + handle scoring
//(timing)
long nMongoTime = System.currentTimeMillis();
List<BasicDBObject> docs = null;
//(aggregation)
LinkedList<BasicDBObject> lowAccuracyAggregatedEntities = null; // (always low accuracy)
LinkedList<BasicDBObject> standaloneEvents = null;
LinkedList<BasicDBObject> lowAccuracyAggregatedEvents = null;
LinkedList<BasicDBObject> lowAccuracyAggregatedFacts = null;
AggregationUtils.GeoContainer lowAccuracyAggregatedGeo = null;
AggregationUtils.GeoContainer extraAliasAggregatedGeo = null;
ScoringUtils scoreStats = null;
if (null != stats.getIds()) {
DBCursor docs0 = this.getDocIds(DbManager.getDocument().getMetadata(), stats.getIds(), nRecordsToGet, query.output, query.score);
nMongoTime = System.currentTimeMillis() - nMongoTime;
nProcTime_tmp = System.currentTimeMillis();
// Entity aggregation (CURRENTLY ALWAYS LOW AGGREGATION):
if ((null != query.output.aggregation) && (null != query.output.aggregation.entsNumReturn) && (query.output.aggregation.entsNumReturn > 0)) {
lowAccuracyAggregatedEntities = new LinkedList<BasicDBObject>();
}
// Standalone events:
if ((query.output.docs != null) && (query.output.docs.eventsTimeline != null) && query.output.docs.eventsTimeline) {
standaloneEvents = new LinkedList<BasicDBObject>();
}
// Low accuracy aggregations:
if ((null != manualEventsNumReturn) && (manualEventsNumReturn > 0)) {
lowAccuracyAggregatedEvents = new LinkedList<BasicDBObject>();
}
if ((null != manualFactsNumReturn) && (manualFactsNumReturn > 0)) {
lowAccuracyAggregatedFacts = new LinkedList<BasicDBObject>();
}
if ((null != manualGeoNumReturn) && (manualGeoNumReturn > 0)) {
lowAccuracyAggregatedGeo = new AggregationUtils.GeoContainer();
}
else if ((null != query.output.aggregation) && (null != query.output.aggregation.geoNumReturn) && (query.output.aggregation.geoNumReturn > 0))
{
// (only if not using low accuracy aggregation ... otherwise it all gets dumped in lowAccuracyAggregatedGeo)
extraAliasAggregatedGeo = new AggregationUtils.GeoContainer();
}
scoreStats = new ScoringUtils();
try {
boolean lockAcquired = true;
try {
lockAcquired = this.acquireConcurrentAccessLock();
} catch (InterruptedException e) {
//(that's fine just carry on)
lockAcquired = false;
}
if (!lockAcquired) {
rp.setResponse(new ResponseObject("Query", false, "Query engine busy, please try again later."));
return rp;
}
scoreStats.setAliasLookupTable(_aliasLookup);
docs = scoreStats.calcTFIDFAndFilter(DbManager.getDocument().getMetadata(),
docs0, query.score, query.output, stats, bLowAccuracyDecay,
nRecordsToSkip, nRecordsToOutput,
communityIdStrs,
tempFilterInfo.entityTypeFilterStrings, tempFilterInfo.assocVerbFilterStrings,
standaloneEvents,
lowAccuracyAggregatedEntities,
lowAccuracyAggregatedGeo, extraAliasAggregatedGeo,
lowAccuracyAggregatedEvents, lowAccuracyAggregatedFacts);
}
finally {
scoreStats.clearAsMuchMemoryAsPossible();
this.releaseConcurrentAccessLock();
}
nProcTime += (System.currentTimeMillis() - nProcTime_tmp);
}
else {
nMongoTime = 0;
}
//TESTED (all queries)
// 0.9] Output:
rp.setResponse(new ResponseObject("Query", true, querySummary.toString()));
// 0.9.1] Stats:
stats.resetArrays();
rp.setStats(stats); // (only actually uses the response pojo, but get rid of big fields anyway...)
// 0.9.2] Facets:
if (null != lowAccuracyAggregatedEntities) { // Entity aggregation
rp.setEntities(lowAccuracyAggregatedEntities);
}
if (null != standaloneEvents) {
rp.setEventsTimeline(standaloneEvents);
}
if (null != lowAccuracyAggregatedGeo) {
rp.setGeo(lowAccuracyAggregatedGeo.geotags, (int)lowAccuracyAggregatedGeo.maxCount, (int)lowAccuracyAggregatedGeo.minCount);
}
if (null != lowAccuracyAggregatedEvents) {
rp.setEvents(lowAccuracyAggregatedEvents);
}
if (null != lowAccuracyAggregatedFacts) {
rp.setFacts(lowAccuracyAggregatedFacts);
}
if ((null != query.output.aggregation) && (null != query.output.aggregation.raw)) {
rp.setFacets(queryResults.getFacets().facetsAsMap());
}
else if ((null != queryResults.getFacets()) && (null != queryResults.getFacets().getFacets())) { // "Logical" aggregation
if (0.0 == query.score.sigWeight) {
scoreStats = null; // (don't calculate event/fact aggregated significance if it's not wanted)
}
AggregationUtils.loadAggregationResults(rp, queryResults.getFacets().getFacets(), query.output.aggregation, scoreStats, _aliasLookup, tempFilterInfo.entityTypeFilterStrings, tempFilterInfo.assocVerbFilterStrings, extraAliasAggregatedGeo);
} // (end facets not overwritten)
scoreStats = null; // (now definitely never need scoreStats)
// 0.9.3] Documents
if (query.output.docs.enable) {
if ((null != docs) && (docs.size() > 0)) {
rp.setData(docs, (BasePojoApiMap<BasicDBObject>)null);
}
else { // (ensure there's always an empty list)
docs = new ArrayList<BasicDBObject>(0);
rp.setData(docs, (BasePojoApiMap<BasicDBObject>)null);
}
}
else { // (ensure there's always an empty list)
docs = new ArrayList<BasicDBObject>(0);
rp.setData(docs, (BasePojoApiMap<BasicDBObject>)null);
}
// 0.9.4] query extensions: post-query hook
if (null != queryExtensions) {
for (IQueryExtension queryExtension: queryExtensions) {
// Don't catch any exceptions thrown here - let it bubble upwards
queryExtension.postQueryActivities(queryId, docs, rp);
}
}//TESTED (see test.QueryExtensionsTestCode)
// (Built-in version)
if (null != _builtInFederatedQueryEngine) {