Examples of gc.base.task.TaskStats

gc.base.task.TaskStats
Task statistics.

    this.setField(doc,FieldConstants.Task_IndexDate,new Date(System.currentTimeMillis()));
  }
  
  public void prepare(TaskContext context, XmlTypes xmlTypes, 
      SolrInputDocument doc, DocInfo info) throws Exception {
    TaskStats stats = context.getStats();
    String tn = context.getTaskName()+".prepare";
    
    // make the metadata document
    long t1 = System.currentTimeMillis();
    Document dom = DomUtil.makeDom(info.Xml_Metadata,true);
    long t2 = System.currentTimeMillis();
    stats.incrementTime(tn+".makeXmlDom",t2-t1);
    
    // interrogate the metadata document
    XmlInterrogator interrogator = new XmlInterrogator();
    XmlType xmlType = interrogator.interrogate(xmlTypes,dom);
    XsltReference xsltReference = xmlType.getToSolrXslt();
    XsltTemplate xsltTemplate = XsltTemplates.getCompiledTemplate(xsltReference.getSrc());
    info.MetadataType_Key = xmlType.getKey();
    info.MetadataType_Identifier = xmlType.getIdentifier();
    info.MetadataType_Indexables_Version = xsltReference.getVersion();
    long t3 = System.currentTimeMillis();
    stats.incrementTime(tn+".interrogateXmlType",t3-t2);
    
    // append fields at the top of the document
    this.append(context,doc,info,true,false);
    
    // transform the metadata document to a Solr document
    // (this appends fields from the transform)
    SolrDocSaxHandler saxHandler = new SolrDocSaxHandler(doc);
    DOMSource source = new DOMSource(dom);
    SAXResult result = new SAXResult(saxHandler);
    xsltTemplate.transform(source,result,null);
    long t4 = System.currentTimeMillis();
    stats.incrementTime(tn+".transformToSolrDoc",t4-t3);
    
    // append remaining fields
    this.append(context,doc,info,false,true);
    
  }

View Full Code Here

  public void query(TaskContext context, Connection con, 
      SqlQInfo info, SqlRowHandler handler) throws Exception {
    boolean closeCon = (con == null);
    PreparedStatement st = null;
    try {
      TaskStats stats = context.getStats();
      String tn = context.getTaskName()+".SqlQuery";
      long rowNum = 0;
      ensureTableName(context,info);
      String sql = makeSql(info);
      
      if (con == null) {
        /*
        long tc1 = System.currentTimeMillis();
        SqlConnectionBroker conBroker = new SqlConnectionBroker();
        con = conBroker.getConnection(context);
        long tc2 = System.currentTimeMillis();
        stats.incrementTime(tn+".getConnection",tc2-tc1);
        */
        throw new Exception("Null SQL connection.");
      }
      
      long t1 = System.currentTimeMillis();
      st = con.prepareStatement(sql);
      List<Object> bindings = info.getQueryBindings();
      if ((bindings != null) && (bindings.size() > 0)) {
        applyQueryBindings(st,bindings);
      }
      if (info.getMaxRows() > 0) st.setMaxRows(info.getMaxRows());
      if (info.getFetchSize() > 0) st.setFetchSize(info.getFetchSize());
      ResultSet rs = st.executeQuery();
      long t2 = System.currentTimeMillis();
      stats.incrementTime(tn+".executeQuery",t2-t1);
      while (rs.next()) {
        if (Thread.interrupted()) {
          throw new InterruptedException("Interrupted while iterating ResultSet.");
        }
        rowNum++;
        handler.handleSqlRow(context,con,rs,rowNum);
      }
      long t3 = System.currentTimeMillis();
      stats.incrementTime(tn+".iterate",t3-t2);
    } finally {
      try {if (st != null) st.close();} 
      catch (Exception ef) {ef.printStackTrace();}
      try {if (closeCon && (con != null)) con.close();} 
      catch (Exception ef) {ef.printStackTrace();}

View Full Code Here

  public void handleSqlRow(TaskContext context, Connection con, 
      ResultSet rs, long rowNum) throws Exception {
    try {
      this._handleSqlRow(context,con,rs,rowNum);
    } catch (Exception e) {
      TaskStats stats = context.getStats();
      String tn = context.getTaskName()+".sync";
      stats.incrementCount(tn+".exceptions");
      System.err.println(e.toString());
      //e.printStackTrace(System.err);
    }
  }

View Full Code Here

    }
  }
  
  private void _handleSqlRow(TaskContext context, Connection con, 
      ResultSet rs, long rowNum) throws Exception {
    TaskStats stats = context.getStats();
    String tn = context.getTaskName()+".sync";
    GptResource resource = new GptResource();
    resource.readFields(rs);


    String[] result = queryDoc(resource);
    String id = result[0];
    String fsMatched = result[1];
    
    if (this.approvedOnly) {
      String s = resource.approvalstatus;
      // handles an earlier problem with publishing "posted" docs
      if ((s != null) && s.equals("posted")) {
        fsMatched = null;
      }
    }
    
    if (fsMatched != null) {
      if ((okIds != null) && (okIds.size() <= this.maxIdsInMap)) {
        okIds.put(id,"");
      } else if (okIds != null) {
        okIds = null;
      }
      stats.incrementCount(tn+".noChange");
    } else {
      
      String s;
      boolean bContinue = true;
      if (bContinue && this.approvedOnly) {
        s = resource.approvalstatus;
        if (s == null) s = "";
        if (!s.equals("approved") && !s.equals("reviewed")) {
          stats.incrementCount(tn+".ignore.notApproved");
          bContinue = false;
        }
      }
      if (bContinue && this.emptyAclOnly) {
        s = resource.acl;
        if (s == null) s = "";
        if (s.trim().length() > 0) {
          stats.incrementCount(tn+".ignore.nonEmptyAcl");
          bContinue = false;
        }
      }
      if (bContinue && this.docsOnly) {
        if (resource.isHarvestingSite) {
          stats.incrementCount(tn+".ignore.harvestingSite");
          bContinue = false;
        }
      }
      
      GptResourceXml resourceXml = new GptResourceXml();
      if (bContinue) {
        resourceXml.querySqlDB(context,con,resource.docuuid);
        s = resourceXml.xml;
        if ((s == null) || (s.length() == 0)) {
          stats.incrementCount(tn+".ignore.noResourceXml");
          bContinue = false;
        }
      }
      
      if (bContinue) {
        if (id == null) {
          stats.incrementCount(tn+".insertRequired");
        } else {
          stats.incrementCount(tn+".updateRequired");
        }
        
        GptUser user = new GptUser();
        user.querySqlDB(context,con,resource.owner);
        
        GptResource parentSite = null;
        if (resource.isHarvestedDocument) {
          parentSite = new GptResource();
          parentSite.querySqlDB(context,con,resource.siteuuid);
        }
                
                GptCollections gptCollections = null;
                if (gptdb2SolrInstance.isGptIncludeCollections()) {
                  gptCollections = new GptCollections();
                  gptCollections.querySqlDB(context,con,resource.docuuid);
                }
  
        SolrInputDocument doc = makeDoc(id,resource,user,resourceXml,parentSite,gptCollections);
        //System.err.println(doc);
        updateDoc(doc);
        stats.incrementCount(tn+".solr.sent");
      }
      
      if (!bContinue && this.checkForDeletes) {
        if (id != null) {
          this.docPublisher.getUpdateServer().deleteById(id);
          stats.incrementCount(tn+".solr.sentForDelete");
        }
      }
    }
  }

View Full Code Here

  private void walkSolrDocs() throws SolrServerException, IOException {
    if (!checkForDeletes) return;
    if ((okIds == null) || (okIds.size() == 0)) return;
    
    TaskContext context = this.getContext();
    TaskStats stats = context.getStats();
    String tn = context.getTaskName()+".walkSolrDocs";
    stats.setString(tn,"...");
    
    String fl = FieldConstants.Id;
    String q = FieldConstants.Id_Table+":"+FieldConstants.Val_Id_Table_DocIndex;
    q += " AND "+FieldConstants.Sync_Type+":"+this.syncType;
    q += " AND "+FieldConstants.Sync_Foreign_InstanceId+":"+this.foreignInstanceId;
    ModifiableSolrParams params = new ModifiableSolrParams();
    params.set("q",q);
    params.set("fl",fl);
    stats.setString(tn+".q",q);
    
    boolean bContinue = true;
    long nDeepTotal = 0;
    long nFetched = 0;
    long nHits = 0;
    int nDocs = 0;
    long nStart = 0;
    int nRows = 1000;
    long nNextStart = 0;
    while (bContinue) {
      bContinue = false;
      params.set("start",""+nStart);
      params.set("rows",""+nRows);
      QueryResponse response = queryServer.query(params);
      SolrDocumentList docs = response.getResults();
      if (docs != null) {
        nHits = docs.getNumFound();
        nDocs = docs.size();
        nNextStart = nStart+nDocs;
        if ((nDocs > 0) && (nNextStart < nHits)) {
          bContinue = true;
        }
        for (int i=0;i<nDocs;i++) {
          SolrDocument doc = docs.get(i);
          String id = (String)doc.getFieldValue(FieldConstants.Id);
          nFetched++;
          stats.incrementCount(tn+".fetched");
          if (okIds.get(id) != null) {
            stats.incrementCount(tn+".idOk");
          } else {
            stats.incrementCount(tn+".idRequiresDelete");
            if ((delIds != null) && (delIds.size() <= this.maxIdsInMap)) {
              delIds.add(id);
            } else if (delIds != null) {
              delIds = null;
              bContinue = false;
              break;
            }
          }
          if ((nDeepTotal > 0) && (nFetched >= nDeepTotal)) {
            bContinue = false;
            break;
          }
        }
        nStart = nNextStart;
      }
    }
    
    if ((delIds != null) && (delIds.size() > 0)) {
      stats.incrementCount(context.getTaskName()+".solr.sentForDelete",delIds.size());
      this.docPublisher.getUpdateServer().deleteById(delIds);
    }
    
  }

View Full Code Here

TOP

Related Classes of gc.base.task.TaskStats

gc.base.sql.SqlQuery

gc.solr.publish.DocBuilder

gc.solr.task.Gptdb2SolrTask

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.