Package com.ikanow.infinit.e.data_model.control

Examples of com.ikanow.infinit.e.data_model.control.DocumentQueueControlPojo


     
      SharePojo savedQueryShare = SharePojo.fromDb(DbManager.getSocial().getShare().findOne(
          new BasicDBObject(SharePojo._id_, savedQuery._parentShare.get_id())), SharePojo.class);
     
      if (null != savedQueryShare) {
        DocumentQueueControlPojo toModify = DocumentQueueControlPojo.fromApi(savedQueryShare.getShare(), DocumentQueueControlPojo.class);
        List<BasicDBObject> docs = (List<BasicDBObject>) rp.getData();
        if ((null != docs) && !docs.isEmpty()) {
          if (null == toModify.getQueueList()) {
            toModify.setQueueList(new ArrayList<ObjectId>(docs.size()));
          }
          ObjectId ignoreBeforeId = toModify.getLastDocIdInserted();
          ObjectId maxDocId = toModify.getLastDocIdInserted();

          //DEBUG
          //System.out.println("before, num docs=" + toModify.getQueueList().size() + " adding " + docs.size() + " from " + ignoreBeforeId);
         
          // Some alerting preamble
          StringBuffer alertText = null;
          StringBuffer alertTitle = null;
          String rootUrl = new PropertiesManager().getURLRoot().replace("/api/", "");
          int maxDocsToAdd = 10; // (default)
          boolean alert = false;
          if ((null != toModify.getQueryInfo().getAlert()) && (null != toModify.getQueryInfo().getAlert().getEmailAddresses())
              && !toModify.getQueryInfo().getAlert().getEmailAddresses().isEmpty())
          {
            alert = true;
            alertText = new StringBuffer();
            if (null != toModify.getQueryInfo().getAlert().getMaxDocsToInclude()) {
              maxDocsToAdd = toModify.getQueryInfo().getAlert().getMaxDocsToInclude();
              if (maxDocsToAdd < 0) {
                maxDocsToAdd = Integer.MAX_VALUE;
              }
            }                       
            createAlertPreamble(alertText, toModify.getQueryInfo().getQuery(), savedQuery._parentShare.get_id(), rootUrl);
          }//TESTED
         
          // Add new docs...

          int numDocsAdded = 0;
          for (BasicDBObject doc: docs) {
            ObjectId docId = doc.getObjectId(DocumentPojo._id_);
            if (null != docId) {
              if (null != ignoreBeforeId) {
                if (docId.compareTo(ignoreBeforeId) <= 0) { // ie docId <= ignoreBeforeId
                  continue;
                }
              }//(end check if this doc has already been seen)             
             
              toModify.getQueueList().add(0, docId);

              //Alerting
              if (alert) {
                // (this fn checks if the max number of docs have been added):
                createAlertDocSummary(alertText, numDocsAdded, maxDocsToAdd, doc, rootUrl);
                numDocsAdded++;
              }
             
              if (null == maxDocId) {
                maxDocId = docId;
              }
              else if (maxDocId.compareTo(docId) < 0) { // ie maxDocId < docId
                maxDocId = docId;
              }
            }//TESTED (test5)
          }//(end loop over new docs)
         
          // More alerting
          if (alert && (numDocsAdded > 0)) {
            alertTitle = new StringBuffer("IKANOW: Queue \"").append(toModify.getQueueName()).append("\" has ").append(numDocsAdded).append(" new");
            if (numDocsAdded == 1) {
              alertTitle.append(" document.");
            }
            else {
              alertTitle.append(" documents.");             
            }
            // (terminate the doc list)
            if (maxDocsToAdd > 0) {
              alertText.append("</ol>");
              alertText.append("\n");
            }
           
            String to = (Arrays.toString(toModify.getQueryInfo().getAlert().getEmailAddresses().toArray()).replaceAll("[\\[\\]]", "")).replace(',', ';');
            try {
              new SendMail(null, to, alertTitle.toString(), alertText.toString()).send("text/html");
            }
            catch (Exception e) {
              //DEBUG
              //e.printStackTrace();
            }
          }//TESTED
         
          // Remove old docs...
         
          int maxDocs = query.output.docs.numReturn;
          if (null != toModify.getMaxDocs()) { // override
            maxDocs = toModify.getMaxDocs();
          }
         
          if (toModify.getQueueList().size() > maxDocs) {
            toModify.setQueueList(toModify.getQueueList().subList(0, maxDocs));
          }//TESTED (test2.2)

          //DEBUG
          //System.out.println("after, num docs=" + toModify.getQueueList().size() + " at " + maxDocId);
         
          // Update share info:
          toModify.setLastDocIdInserted(maxDocId);
         
          // We've modified the share so update it:
          savedQueryShare.setShare(toModify.toApi());
          savedQueryShare.setModified(new Date());
          DbManager.getSocial().getShare().save(savedQueryShare.toDb());
         
        }//(end found some docs)
       
View Full Code Here


      BasicDBObject query = new BasicDBObject(SharePojo.type_, DocumentQueueControlPojo.SavedQueryQueue);
      List<SharePojo> savedQueries = SharePojo.listFromDb(DbManager.getSocial().getShare().find(query), SharePojo.listType());
      if (null != savedQueries) {
        for (SharePojo savedQueryShare: savedQueries) {
          if (null != savedQueryShare.getShare()) {
            DocumentQueueControlPojo savedQuery = DocumentQueueControlPojo.fromApi(savedQueryShare.getShare(), DocumentQueueControlPojo.class);
           
            // Is this query well formed?
            if ((null != savedQuery.getQueryInfo()) &&
                ((null != savedQuery.getQueryInfo().getQuery()) || (null != savedQuery.getQueryInfo().getQueryId())))
            {
              Date now = new Date();
              long freqOffset;
              // Check if it's time to run the query
              if (savedQuery.getQueryInfo().getFrequency() == DocumentQueueControlPojo.SavedQueryInfo.DocQueueFrequency.Hourly)
              {
                freqOffset = 3600L*1000L;
                //(nothing to do here, just run whenever)
              }//TESTED (test3)
              else if (savedQuery.getQueryInfo().getFrequency() == DocumentQueueControlPojo.SavedQueryInfo.DocQueueFrequency.Daily)
              {
                if (null != savedQuery.getQueryInfo().getFrequencyOffset()) { // hour of day
                  freqOffset = 12L*3600L*1000L; // (already check vs hour of day so be more relaxed)
                  Calendar calendar = GregorianCalendar.getInstance();
                  calendar.setTime(now);

                  //DEBUG
                  //System.out.println("DAILY: " + calendar.get(Calendar.HOUR_OF_DAY) + " VS " + savedQuery.getQueryInfo().getFrequencyOffset()); 
                 
                  if (calendar.get(Calendar.HOUR_OF_DAY) != savedQuery.getQueryInfo().getFrequencyOffset()) {
                    continue;
                  }//TESTED (test4)
                }
                else {
                  freqOffset = 24L*3600L*1000L; // (just run every 24 hours)                  
                }
              }//TESTED (test4)
              else if (savedQuery.getQueryInfo().getFrequency() == DocumentQueueControlPojo.SavedQueryInfo.DocQueueFrequency.Weekly)
              {
                if (null != savedQuery.getQueryInfo().getFrequencyOffset()) { // day of week
                  freqOffset = 3L*24L*3600L*1000L; // (already check vs day of week so be more relaxed)
                  Calendar calendar = GregorianCalendar.getInstance();
                  calendar.setTime(now);

                  //DEBUG
                  //System.out.println("WEEKLY: " + calendar.get(Calendar.DAY_OF_WEEK) + " VS " + savedQuery.getQueryInfo().getFrequencyOffset());
                 
                  if (calendar.get(Calendar.DAY_OF_WEEK) != savedQuery.getQueryInfo().getFrequencyOffset()) {
                    continue;
                  }
                }
                else {
                  freqOffset = 7L*24L*3600L*1000L//(just run every 7 days)
                }
              }//TESTED (test5)
              else continue; // (no -supported- frequency, don't run)
             
              long nowTime = now.getTime();

              //DEBUG
              //System.out.println("Comparing: " + savedQuery.getQueryInfo().getLastRun() + " VS " + now + " @ " + freqOffset/1000L);
             
              if ((null == savedQuery.getQueryInfo().getLastRun()) ||
                  ((nowTime - savedQuery.getQueryInfo().getLastRun().getTime()) > freqOffset))
              {
                //(does nothing if the share already exists)
                DbManager.getCustom().getSavedQueryCache().insert(savedQueryShare.toDb());
                CommandResult cr = DbManager.getCustom().getSavedQueryCache().getDB().getLastError();
               
                if (null == cr.get("err")) { // if we've actually done something, update the main share table also
                  savedQuery.getQueryInfo().setLastRun(now);
                  savedQueryShare.setShare(savedQuery.toApi());
                  // (this will overwrite the existing version)
                  DbManager.getSocial().getShare().save(savedQueryShare.toDb());               
                }//TESTED (by hand with prints)
               
              }//TESTED (test3-5)
View Full Code Here

    }//(end acquired app lock)
  }//TESTED

  public static DocumentQueueControlPojo getSavedQueryToRun()
  {
    DocumentQueueControlPojo toReturn = null;
    try {
      SharePojo savedQueryShare = SharePojo.fromDb(DbManager.getCustom().getSavedQueryCache().findAndRemove(new BasicDBObject()), SharePojo.class);
      if (null == savedQueryShare) { // nothing to process
        return null;
      }//TESTED (test1)
      toReturn = DocumentQueueControlPojo.fromApi(savedQueryShare.getShare(), DocumentQueueControlPojo.class);
     
      // Get the user communities and append the query if possible
      Set<ObjectId> userAccess = AuthUtils.getCommunities(savedQueryShare.getOwner().get_id());
      if (userAccess.isEmpty()) {
        return toReturn; // (_parentShare is null so will be discarded)
      }
      if (null != toReturn.getQueryInfo().getQueryId()) {
        BasicDBObject queryQuery = new BasicDBObject(SharePojo._id_, toReturn.getQueryInfo().getQueryId());
        queryQuery.put(ShareCommunityPojo.shareQuery_id_, new BasicDBObject(DbManager.in_, userAccess));
        SharePojo shareContainingQuery = SharePojo.fromDb(DbManager.getSocial().getShare().findOne(queryQuery), SharePojo.class);

        if (null == shareContainingQuery) {
          return toReturn; // (_parentShare is null so will be discarded)
        }
        toReturn.getQueryInfo().setQuery(AdvancedQueryPojo.fromApi(shareContainingQuery.getShare(), AdvancedQueryPojo.class));
      }//TESTED (test6)
     
      if (null != toReturn.getQueryInfo().getQuery()) {
        // Check the communityIds...
        if (null != toReturn.getQueryInfo().getQuery().communityIds) {
          ArrayList<ObjectId> revisedCommunityList = new ArrayList<ObjectId>(toReturn.getQueryInfo().getQuery().communityIds.size());
          for (ObjectId commId: toReturn.getQueryInfo().getQuery().communityIds) {
            if (userAccess.contains(commId)) {
              revisedCommunityList.add(commId);
            }
          }//(end loop over unchecked communities)
          toReturn.getQueryInfo().getQuery().communityIds = revisedCommunityList;
        }
      }//TESTED (test1)
      toReturn._parentShare = savedQueryShare; // (if this is null then the subsequent processing will ignore this)
    }
    catch (Exception e) { // (this is some internal horror so log)
View Full Code Here

    // Spend at most 5 minutes per iteration
    long startTime = new Date().getTime();
    long iterationTime = startTime;
   
    DocumentQueueControlPojo savedQuery = CustomScheduleManager.getSavedQueryToRun();
    while (null != savedQuery) {
      CustomSavedQueryQueueLauncher.executeQuery(savedQuery);
     
      long timeTaken = new Date().getTime();
View Full Code Here

    queryObj.communityIds.add(new ObjectId("4e3706c48d26852237078005"));
    queryObj.score.numAnalyze = 20;
    queryObj.output.docs.numReturn = 10;
   
    // Build a template doc control pojo
    DocumentQueueControlPojo query = new DocumentQueueControlPojo();
    query.setQueueType(DocumentQueueControlPojo.DocQueueType.SavedQueryQueue);
    query.setQueueName("test" + testNum);
    query.setQueryInfo(new DocumentQueueControlPojo.SavedQueryInfo());
    query.getQueryInfo().setQuery(queryObj);

    if (4 == testNum) { // daily
      query.getQueryInfo().setFrequency(DocumentQueueControlPojo.SavedQueryInfo.DocQueueFrequency.Daily);   
      query.getQueryInfo().setFrequencyOffset(18);
      query.setMaxDocs(5);
    }
    else if (5 == testNum) { // weekly
      query.getQueryInfo().setFrequency(DocumentQueueControlPojo.SavedQueryInfo.DocQueueFrequency.Weekly);   
      query.getQueryInfo().setFrequencyOffset(6);
      query.setLastDocIdInserted(new ObjectId("509c04a81fda81052a25d660"));
    }
    else { // hourly
      query.getQueryInfo().setFrequency(DocumentQueueControlPojo.SavedQueryInfo.DocQueueFrequency.Hourly);   
      query.getQueryInfo().setFrequencyOffset(1);
    }   
    if (3 == testNum) {
      query.setLastDocIdInserted(new ObjectId("509c04a81fda81052a25d660"));
      query.getQueryInfo().getQuery().logic = "1";
      //3a: hourly
      query.getQueryInfo().setLastRun(new Date(new Date().getTime() - 4000000L));
      //3b: daily
      //query.getQueryInfo().setLastRun(new Date(new Date().getTime() - 27*3600*1000L));
      //query.getQueryInfo().setFrequency(DocQueueFrequency.Daily);
      //query.getQueryInfo().setFrequencyOffset(null);
      //3c: weekly
      //query.getQueryInfo().setLastRun(new Date(new Date().getTime() - 8*24*3600*1000L));
      //query.getQueryInfo().setFrequency(DocQueueFrequency.Weekly);
      //query.getQueryInfo().setFrequencyOffset(null);
    }   
    if (6 == testNum) {
      query.getQueryInfo().setQueryId(fixedShareId_2);
     
      SharePojo queryShare = new SharePojo();
      queryShare.set_id(fixedShareId_2);
      queryShare.setType("infinite-saved-query");
      queryShare.setTitle("test query " + testNum);
      queryShare.setDescription("test query " + testNum);
      queryShare.setShare(query.getQueryInfo().getQuery().toApi());   
      addSocial(queryShare);
      DbManager.getSocial().getShare().save(queryShare.toDb());

      query.getQueryInfo().setQuery(null);
    }
   
    // Build a template share
    SharePojo share = new SharePojo();
    if (testNum == 2) {
      share.set_id(fixedShareId_2);   
      query.setQueueList(new ArrayList<ObjectId>(5));
      for (int i = 0; i < 5; ++i) {
        query.getQueueList().add(new ObjectId());
      }
    }
    else {
      share.set_id(fixedShareId_1);
    }
    share.setType(DocumentQueueControlPojo.SavedQueryQueue);
    share.setTitle("test" + testNum);
    share.setDescription("test1" + testNum);
    share.setShare(query.toApi());   
    addSocial(share);
   
    DbManager.getSocial().getShare().save(share.toDb());
  }
View Full Code Here

          )
      {
        throw new RuntimeException("Can't find saved query, or is a custom job not a query, or authorization error");               
      }
      else { // share.share is a  DocumentQueueControlPojo
        DocumentQueueControlPojo queue = DocumentQueueControlPojo.fromApi(share.getShare(), DocumentQueueControlPojo.class);
        BasicDBObject docQuery1 = new BasicDBObject(DocumentPojo._id_, new BasicDBObject(DbManager.in_, queue.getQueueList()));
        BasicDBObject docQuery2 = new BasicDBObject(DocumentPojo.updateId_, new BasicDBObject(DbManager.in_, queue.getQueueList()));
        BasicDBObject docQuery = new BasicDBObject(DbManager.or_, Arrays.asList(docQuery1, docQuery2));
        DBCursor dbc = DbManager.getDocument().getMetadata().find(docQuery).limit(query.score.numAnalyze);
        ScoringUtils scoreStats = new ScoringUtils();
        List<BasicDBObject> docs = null;
        StatisticsPojo stats = new StatisticsPojo();
View Full Code Here

TOP

Related Classes of com.ikanow.infinit.e.data_model.control.DocumentQueueControlPojo

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.