Package com.ikanow.infinit.e.api.knowledge.federated

Examples of com.ikanow.infinit.e.api.knowledge.federated.SimpleFederatedQueryEngine$FederatedHarvest


    fakeEndpoint.parentSource.setOwnedByAdmin(true);
    fakeEndpoint.parentSource.setTitle("fakeendpoint");
    fakeEndpoint.parentSource.setMediaType("Report");
    fakeEndpoint.titlePrefix = "fake endpoint: ";

    SimpleFederatedQueryEngine queryEngine = new SimpleFederatedQueryEngine();
   
    SimpleFederatedQueryEngine.TEST_MODE_ONLY = true;   
    SimpleFederatedCache.QUERY_FEDERATION_CACHE_CLEANSE_SIZE = 10; // (for 4.2 - override max size after which will intermittently cleanse cache)
   
    // CLEAR (TEST) CACHES:
    queryEngine.test_cacheClear(true, true, fakeEndpoint.parentSource.getKey());
   
    //TEST CASES:
    try {
     
      //(use this above the appropriate failing test to get debug info:)
      //queryEngine.setTestMode(true);     
     
      queryEngine.addEndpoint(fakeEndpoint);
      String[] communityIdStrs = new String[1];
      communityIdStrs[0] = "4c927585d591d31d7b37097a";
      AdvancedQueryPojo query = new AdvancedQueryPojo();
      AdvancedQueryPojo.QueryTermPojo qtText, qtDate, qtEntVal, qtEntIndex;   
     
      // 1.1] Query ignored because has any term other than entity/date
      query.qt = new ArrayList<AdvancedQueryPojo.QueryTermPojo>(1);
      qtText = new AdvancedQueryPojo.QueryTermPojo();
      qtText.etext="fail";
      query.qt.add(qtText);
      query.logic = null;
      queryEngine.preQueryActivities(new ObjectId(), query, communityIdStrs);
      queryEngine.test_CheckIfQueryLaunched("1.1a", false);
      queryEngine.test_queryClear(true);
      //(proper Test-Driven code would require all the types, but we know we're actually checking for anything except entity/date)
     
      // 1.2] Query ignored because date only
      query.qt = new ArrayList<AdvancedQueryPojo.QueryTermPojo>(1);
      qtDate = new AdvancedQueryPojo.QueryTermPojo();
      qtDate.time = new AdvancedQueryPojo.QueryTermPojo.TimeTermPojo();
      qtDate.time.max="now";
      qtDate.time.min="now-3d";
      query.qt.add(qtDate);
      query.logic = null;
      queryEngine.preQueryActivities(new ObjectId(), query, communityIdStrs);
      queryEngine.test_CheckIfQueryLaunched("1.2a", false);
      queryEngine.test_queryClear(true);   
     
      // 1.3] Query ignored because logic too complex
      // 1.3a) OR
      query.qt = new ArrayList<AdvancedQueryPojo.QueryTermPojo>(2);
      qtEntVal = new AdvancedQueryPojo.QueryTermPojo();
      qtEntVal.entityValue = "test1_3a";
      qtEntVal.entityType = "TestEntityIn";
      query.qt.add(qtEntVal);
      query.qt.add(qtDate);
      query.logic = "1 OR 2";
      queryEngine.preQueryActivities(new ObjectId(), query, communityIdStrs);
      queryEngine.test_CheckIfQueryLaunched("1.3a", false);
      queryEngine.test_queryClear(true);
      // 1.3b) NOT
      query.qt = new ArrayList<AdvancedQueryPojo.QueryTermPojo>(2);
      qtEntVal = new AdvancedQueryPojo.QueryTermPojo();
      qtEntVal.entityValue = "test1_3b";
      qtEntVal.entityType = "TestEntityIn";
      query.qt.add(qtEntVal);
      query.qt.add(qtDate);
      query.logic = "NOT (1 AND 2)";
      queryEngine.preQueryActivities(new ObjectId(), query, communityIdStrs);
      queryEngine.test_CheckIfQueryLaunched("1.3b", false);
      queryEngine.test_queryClear(true);
     
      // 1.4] Query ignored because multiple entities
      query.qt = new ArrayList<AdvancedQueryPojo.QueryTermPojo>(2);
      qtEntVal = new AdvancedQueryPojo.QueryTermPojo();
      qtEntVal.entityValue = "test1_4_1";
      qtEntVal.entityType = "TestEntityIn";
      qtEntIndex = new AdvancedQueryPojo.QueryTermPojo();
      qtEntIndex.entity = "test1_4_1/testentityin";
      query.qt.add(qtEntVal);
      query.qt.add(qtEntIndex);
      query.logic = null;
      queryEngine.preQueryActivities(new ObjectId(), query, communityIdStrs);
      queryEngine.test_CheckIfQueryLaunched("1.4a", false);
      queryEngine.test_queryClear(true);
     
      // 1.5] Single entity - not ignored (entity type and value)   
      query.qt = new ArrayList<AdvancedQueryPojo.QueryTermPojo>(1);
      qtEntVal = new AdvancedQueryPojo.QueryTermPojo();
      qtEntVal.entityValue = "test1_5a";
      qtEntVal.entityType = "TestEntityIn";
      query.qt.add(qtEntVal);
      query.logic = null;
      queryEngine.preQueryActivities(new ObjectId(), query, communityIdStrs);
      queryEngine.test_CheckIfQueryLaunched("1.5a", true);
      queryEngine.test_queryClear(true);
     
      // 1.6] Single entity and date - not ignored (entity index)
      // 1.6a) single date
      query.qt = new ArrayList<AdvancedQueryPojo.QueryTermPojo>(2);
      qtEntIndex = new AdvancedQueryPojo.QueryTermPojo();
      qtEntIndex.entity= "test1_6a/testentityin";
      query.qt.add(qtDate);
      query.qt.add(qtEntIndex);
      query.logic = "1 AND 2";
      queryEngine.preQueryActivities(new ObjectId(), query, communityIdStrs);
      queryEngine.test_CheckIfQueryLaunched("1.6a", true);
      queryEngine.test_queryClear(true);
      // 1.6b) multiple dates
      query.qt = new ArrayList<AdvancedQueryPojo.QueryTermPojo>(3);
      qtEntIndex = new AdvancedQueryPojo.QueryTermPojo();
      qtEntIndex.entity= "test1_6b/testentityin";
      query.qt.add(qtDate);
      query.qt.add(qtEntIndex);
      query.qt.add(qtDate);
      query.logic = "1 AND 2 AND 3";
      queryEngine.preQueryActivities(new ObjectId(), query, communityIdStrs);
      queryEngine.test_CheckIfQueryLaunched("1.6b", true);
      queryEngine.test_queryClear(true);
      // 1.6c) different orders
      query.qt = new ArrayList<AdvancedQueryPojo.QueryTermPojo>(2);
      qtEntVal = new AdvancedQueryPojo.QueryTermPojo();
      qtEntVal.entityValue = "test1_6c";
      qtEntVal.entityType = "TestEntityIn";
      query.qt.add(qtEntVal);
      query.qt.add(qtDate);
      query.logic = null;
      queryEngine.preQueryActivities(new ObjectId(), query, communityIdStrs);
      queryEngine.test_CheckIfQueryLaunched("1.6c", true);
      queryEngine.test_queryClear(true);
     
      // 2.1] Query ignored because no related entity types
      query.qt = new ArrayList<AdvancedQueryPojo.QueryTermPojo>(1);
      qtEntIndex = new AdvancedQueryPojo.QueryTermPojo();
      qtEntIndex.entity = "test2_1a/NOT_testentityin";
      query.qt.add(qtEntIndex);
      query.logic = null;
      queryEngine.preQueryActivities(new ObjectId(), query, communityIdStrs);
      queryEngine.test_CheckIfQueryLaunched("2.1a", false);
      queryEngine.test_queryClear(true);
     
      // 3.1] Test non-cached query (and that query result is added to the cache)
      query.qt = new ArrayList<AdvancedQueryPojo.QueryTermPojo>(1);
      qtEntVal = new AdvancedQueryPojo.QueryTermPojo();
      qtEntVal.entityValue = "test3_1";
      qtEntVal.entityType = "TestEntityIn";
      query.qt.add(qtEntVal);
      query.logic = null;
      ObjectId queryId = new ObjectId();
      queryEngine.preQueryActivities(queryId, query, communityIdStrs);
      //TODO: this deletes stuff (make it so it won't - should fallback that function to the clear code)
      //queryEngine.test_CheckIfQueryLaunched("3.1", true);
      ResponsePojo rp = new ResponsePojo();
      ArrayList<BasicDBObject> docs = new ArrayList<BasicDBObject>(1);
      BasicDBObject doc = new BasicDBObject();
      doc.put(DocumentPojo.aggregateSignif_, 115);
      doc.put(DocumentPojo.queryRelevance_, 105);
      doc.put(DocumentPojo.score_, 110);
      docs.add(doc);
      rp.setData(docs, (BasePojoApiMap<BasicDBObject>)null);
      queryEngine.postQueryActivities(queryId, docs, rp);
      queryEngine.test_CheckIfDocAdded("3.1", docs);
      // (don't clear from cache, next doc should return without making a request)
     
      // 3.2] Like 3.1 but with JsonPath
      // (clear cache here to ensure we don't just used the cached doc)
      queryEngine.test_cacheClear(true, true, fakeEndpoint.parentSource.getKey());
     
      query.qt = new ArrayList<AdvancedQueryPojo.QueryTermPojo>(1);
      qtEntVal = new AdvancedQueryPojo.QueryTermPojo();
      qtEntVal.entityValue = "test3_1";
      qtEntVal.entityType = "TestEntityIn";
      docConversionMap.remove("test:field");
      docConversionMap.remove("test:field2");
      docConversionMap.put("::field2","displayUrl");
      docConversionMap.put("::field", "TestEntityOut");
      query.qt.add(qtEntVal);
      query.logic = null;
      queryId = new ObjectId();
      queryEngine.preQueryActivities(queryId, query, communityIdStrs);
      //TODO: this deletes stuff (make it so it won't - should fallback that function to the clear code)
      //queryEngine.test_CheckIfQueryLaunched("3.1", true);
      rp = new ResponsePojo();
      docs = new ArrayList<BasicDBObject>(1);
      doc = new BasicDBObject();
      doc.put(DocumentPojo.aggregateSignif_, 115);
      doc.put(DocumentPojo.queryRelevance_, 105);
      doc.put(DocumentPojo.score_, 110);
      docs.add(doc);
      rp.setData(docs, (BasePojoApiMap<BasicDBObject>)null);
      queryEngine.postQueryActivities(queryId, docs, rp);
      queryEngine.test_CheckIfDocAdded("3.1", docs);
      // (don't clear from cache, next doc should return without making a request)
     
      // 4.1] Test cached query
      docs.remove(0);
      queryId = new ObjectId();
      queryEngine.preQueryActivities(queryId, query, communityIdStrs);
      queryEngine.test_CheckIfQueryLaunched("4.1", false); // (ie check no query added - because it was in the cache)
      queryEngine.postQueryActivities(queryId, docs, rp);
      queryEngine.test_CheckIfDocAdded("4.1", docs);
      // (don't clear from cache, next doc should check cache but remove expired value)
     
      // 4.2] Test expired then cached query
      // (currently need to clear the doc cache to make this work - see tests that need to be added properly, listed above)
      queryEngine.test_cacheClear(false, true, fakeEndpoint.parentSource.getKey());
      queryEngine.test_cacheExpire();
      docs.remove(0);
      queryId = new ObjectId();
      queryEngine.preQueryActivities(queryId, query, communityIdStrs);
      queryEngine.test_CheckIfQueryLaunched("4.2", true); // (ie check no query added - because it was in the cache)
      queryEngine.postQueryActivities(queryId, docs, rp);
      queryEngine.test_CheckIfDocAdded("4.2", docs);
      // (don't clear from cache, next doc should return without making a request)
     
      // 4.3] Test cached query - bulk remove
      // (currently need to clear the doc cache to make this work - see tests that need to be added properly, listed above)
      queryEngine.test_cacheClear(false, true, fakeEndpoint.parentSource.getKey());
      docs.remove(0);
      queryId = new ObjectId();
      queryEngine.test_cacheFill("4.3a", true, true);
      queryEngine.preQueryActivities(queryId, query, communityIdStrs);
      queryEngine.test_CheckIfQueryLaunched("4.3", false); // (ie check no query added - because it was in the cache)
      queryEngine.postQueryActivities(queryId, docs, rp);
      queryEngine.test_CheckIfDocAdded("4.3", docs);
      queryEngine.test_cacheFill("4.3b", false, false);
      queryEngine.test_queryClear(true);   
    }
    finally {
      System.out.println("All tests run, quitting in 5s.");
      Thread.sleep(5000);   
      testServer.stop();
View Full Code Here


      {
        for (String srcKey: query.input.sources) {
          FederatedQueryInMemoryCache fedQueryCacheEl = _federatedQueryCache.get(srcKey);
          if (null != fedQueryCacheEl) {
            if (null == this._builtInFederatedQueryEngine) {
              _builtInFederatedQueryEngine = new SimpleFederatedQueryEngine();
            }
            _builtInFederatedQueryEngine.addEndpoint(fedQueryCacheEl.source);
          }
        }
      }//TESTED (//TESTED (http://localhost:8184/knowledge/document/query/53ab42a2e4b04bcfe2de4387?qt[0].entity=%22garyhart.com/externaldomain%22&output.docs.numReturn=10&input.sources=inf...federated.externaldomain.&input.srcInclude=true))
     
      // 2:
      else { //Get federated queries from communities
        HashSet<String> excludeSrcs = null;
        for (String commIdStr: communityIdStrs) {
          FederatedQueryInMemoryCache fedQueryCacheEl = _federatedQueryCache.get(commIdStr);
          if (null != fedQueryCacheEl) {
           
            if ((null != query.input) && (null != query.input.sources)) { // (there are exclude sources)
              if (null == excludeSrcs) {
                excludeSrcs = new HashSet<String>(query.input.sources);
              }
            }//TESTED (http://localhost:8184/knowledge/document/query/53ab42a2e4b04bcfe2de4387?qt[0].entity=%22garyhart.com/externaldomain%22&output.docs.numReturn=10&input.sources=inf...federated.externaldomain.&input.srcInclude=false)
           
            for (Map.Entry<String, SourceFederatedQueryConfigPojo> fedQueryKV: fedQueryCacheEl.sources.entrySet()) {
              if ((null == excludeSrcs) || !excludeSrcs.contains(fedQueryKV.getKey())) {
                if (null == this._builtInFederatedQueryEngine) {
                  _builtInFederatedQueryEngine = new SimpleFederatedQueryEngine();
                }
                _builtInFederatedQueryEngine.addEndpoint(fedQueryKV.getValue());
              }
            }
          }
View Full Code Here

        if (null == testQuery) {
          rp.setResponse(new ResponseObject("Test Source",false,"source error: need to specifiy a valid IKANOW query to test federated queries, error: " + errMessage));     
          return rp;         
        }
        // OK if we're here then we can test the query
        SimpleFederatedQueryEngine testFederatedQuery = new SimpleFederatedQueryEngine();
        endpoint.parentSource = source;
        testFederatedQuery.addEndpoint(endpoint);
        ObjectId queryId = new ObjectId();
        String[] communityIdStrs = new String[source.getCommunityIds().size()];
        int i = 0;
        for (ObjectId commId: source.getCommunityIds()) {
          communityIdStrs[i] = commId.toString();
          i++;
        }
        testFederatedQuery.setTestMode(true);
        testFederatedQuery.preQueryActivities(queryId, testQuery, communityIdStrs);
        StatisticsPojo stats = new StatisticsPojo();
        stats.setSavedScores(0, 0);
        rp.setStats(stats);
        ArrayList<BasicDBObject> toAddTemp = new ArrayList<BasicDBObject>(1);
        testFederatedQuery.postQueryActivities(queryId, toAddTemp, rp);
        for (BasicDBObject docObj: toAddTemp) {
          DocumentPojo doc = DocumentPojo.fromDb(docObj, DocumentPojo.class);
          if (null != doc.getEntities()) {
            federatedQueryEnts += doc.getEntities().size();
          }
View Full Code Here

TOP

Related Classes of com.ikanow.infinit.e.api.knowledge.federated.SimpleFederatedQueryEngine$FederatedHarvest

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.