Examples of com.ikanow.infinit.e.data_model.store.config.source.SourcePojo$SourcePojoDeserializer

Package com.ikanow.infinit.e.data_model.store.config.source

Examples of com.ikanow.infinit.e.data_model.store.config.source.SourcePojo$SourcePojoDeserializer

com.ikanow.infinit.e.data_model.store.config.source.SourcePojo
Class used to establish the source information for a feed this defines the data necessary to create a feed in the system @author cmorgan

        _context.getHarvestStatus().logMessage(HarvestExceptionUtils.createExceptionMessage(e).toString(), true);
      }
    }
    else if (scriptLang.equalsIgnoreCase("javascript")) {
      try {
        SourcePojo src = f.getTempSource();
        intializeScriptEngine(src, src.getUnstructuredAnalysisConfig());


        // Setup input:
        if (null == flags) {
          flags = "t";
        }

View Full Code Here

      StringBuilder sb = new StringBuilder();
      String str;
      while ((str = in.readLine()) != null)
        sb.append(str);
      in.close();
      SourcePojo source = ApiManager.mapFromApi(sb.toString(), SourcePojo.class, new SourcePojoApiMap(null));
      
      harvester.setStandaloneMode(1, false);
      List<DocumentPojo> toAdd = new LinkedList<DocumentPojo>();
      List<DocumentPojo> toUpdate = new LinkedList<DocumentPojo>();
      List<DocumentPojo> toRemove = new LinkedList<DocumentPojo>();

View Full Code Here

      StringBuilder sb = new StringBuilder();
      String str;
      while ((str = in.readLine()) != null)
        sb.append(str);
      in.close();
      SourcePojo source = ApiManager.mapFromApi(sb.toString(), SourcePojo.class, new SourcePojoApiMap(null));
      
      harvester.setStandaloneMode(1, false);
      List<DocumentPojo> toAdd = new LinkedList<DocumentPojo>();
      List<DocumentPojo> toUpdate = new LinkedList<DocumentPojo>();
      List<DocumentPojo> toRemove = new LinkedList<DocumentPojo>();

View Full Code Here

      StringBuilder sb = new StringBuilder();
      String str;
      while ((str = in.readLine()) != null)
        sb.append(str);
      in.close();
      SourcePojo source = ApiManager.mapFromApi(sb.toString(), SourcePojo.class, new SourcePojoApiMap(null));
      
      List<DocumentPojo> toAdd = new LinkedList<DocumentPojo>();
      List<DocumentPojo> toUpdate = new LinkedList<DocumentPojo>();
      List<DocumentPojo> toRemove = new LinkedList<DocumentPojo>();
      harvester.harvestSource(source, toAdd, toUpdate, toRemove);

View Full Code Here

      StringBuilder sb = new StringBuilder();
      String str;
      while ((str = in.readLine()) != null)
        sb.append(str);
      in.close();
      SourcePojo source = ApiManager.mapFromApi(sb.toString(), SourcePojo.class, new SourcePojoApiMap(null));
      
      harvester.setStandaloneMode(1, false);
      List<DocumentPojo> toAdd = new LinkedList<DocumentPojo>();
      List<DocumentPojo> toUpdate = new LinkedList<DocumentPojo>();
      List<DocumentPojo> toRemove = new LinkedList<DocumentPojo>();

View Full Code Here

    List<DocumentPojo> toAdd = new LinkedList<DocumentPojo>();
    List<DocumentPojo> toUpdate = new LinkedList<DocumentPojo>();
    List<DocumentPojo> toRemove = new LinkedList<DocumentPojo>();
    
    BasicDBObject query = null;
    SourcePojo feedSource = null;
    
    // 1. Get documents from a "feed" source
    
    // 1.1 OPENCALAIS    
//    toAdd.clear();

View Full Code Here

    }
    DBCursor dbc = DbManager.getIngest().getSource().find(query);


    int nSrcFixCount = 0;
    while (dbc.hasNext()) {
      SourcePojo src = SourcePojo.fromDb(dbc.next(), SourcePojo.class);
      nSrcFixCount++;
      src.generateShah256Hash();
      DbManager.getIngest().getSource().update(new BasicDBObject(SourcePojo._id_, src.getId()), 
          new BasicDBObject(MongoDbManager.set_, new BasicDBObject(SourcePojo.shah256Hash_, src.getShah256Hash())));
    }
    if (nSrcFixCount > 0) {
      logger.info("Core.Server: Fixed " + nSrcFixCount + " missing source hash(es)");
    }    
  }//TESTED (by hand/eye)

View Full Code Here

          _logger.info("(Starting harvest thread for " + _sSourceType + ")");            
          }          
          while (!_sources.isEmpty()) {
            LinkedList<SourcePojo> sourceBatch = SourceUtils.getDistributedSourceList(_sources, _sSourceType, false);
            while (!sourceBatch.isEmpty()) {
              SourcePojo source = sourceBatch.pop();
          SourceHarvesterRunnable sourceRunner = new SourceHarvesterRunnable(source);
          
          boolean bSubmittedTask = false;
          for (int i = 0; (i < 5) && !bSubmittedTask; ++i) {
            try {
              bex.submitTask(sourceRunner, true);
              bSubmittedTask = true;
            } 
            catch (Exception e) { 
              try {
                  _logger.info("(Thread failure for " + _sSourceType + ", can probably recover)");            
                Thread.sleep(1000); // wait a second
                
              } catch (InterruptedException e1) { } 
            }
          }
          
          if (_bStopHarvest) { // Just need to update the status of all remaining source
              _logger.info("(Shutdown, cleaning up " + sourceBatch.size() + " queued sources for " + _sSourceType + ")");            
                for (SourcePojo sourceToDelete: sourceBatch) {
                  if ((null != source.getHarvestStatus()) && 
                      (null != source.getHarvestStatus().getHarvest_status()) &&
                        (HarvestEnum.in_progress != source.getHarvestStatus().getHarvest_status()))
                  {
                    // (If I can revert to old status)
                    SourceUtils.updateHarvestStatus(sourceToDelete, source.getHarvestStatus().getHarvest_status(), null, 0, null);
                  }
                  else {
                    SourceUtils.updateHarvestStatus(sourceToDelete, HarvestEnum.success, null, 0, null);              
                  }
                }

View Full Code Here

      
      // 1) Check authentication
      
      String sourceKey = null;
      ObjectId sourceId = null;
      SourcePojo source = null;
      BasicDBObject query = new BasicDBObject(SourcePojo.communityIds_, 
                            new BasicDBObject(DbManager.in_, communityIds));
      try {
        sourceId = new ObjectId(sourceKeyOrId);
        query.put(SourcePojo._id_, sourceId);
        source = SourcePojo.fromDb(
            MongoDbManager.getIngest().getSource().findOne(query), 
            SourcePojo.class);
      }
      catch (Exception e) {
        // it's a job name
        sourceKey = sourceKeyOrId;
        query.put(SourcePojo.key_, sourceKey);
        source = SourcePojo.fromDb(
            MongoDbManager.getIngest().getSource().findOne(query), 
            SourcePojo.class);
      } 
      if (null == source) {
        throw new RuntimeException("Authentication failure or no matching source");
      }
      sourceKey = source.getKey();
      sourceId = source.getId();
      
      DBCollection cacheCollection = DbManager.getDocument().getMetadata();
      
      // 2) Do we already have this cache?

View Full Code Here

      else // if ( returnRawData )
      {
        //check if the harvest type is file, return the file instead
        //if file is db return the json
        //get source
        SourcePojo source = getSourceFromKey(dp.getSourceKey());
        if ( source.getExtractType().equals( "File" ))
        {
          //get file from harvester
          String fileURL = dp.getUrl();
          if ( dp.getSourceUrl() != null )
            fileURL = dp.getSourceUrl();

View Full Code Here

0 1 2 3

TOP

Related Classes of com.ikanow.infinit.e.data_model.store.config.source.SourcePojo$SourcePojoDeserializer

com.google.gson.Gson

com.google.gson.JsonElement

com.ikanow.infinit.e.api.config.source.SourceHandler

com.ikanow.infinit.e.api.knowledge.DocumentHandler

com.ikanow.infinit.e.api.test.FederatedQueryTestCode

com.ikanow.infinit.e.core.execute_harvest.HarvestThenProcessController$SourceTypeHarvesterRunnable

com.ikanow.infinit.e.core.utils.SourceUtils

com.ikanow.infinit.e.data_model.api.config.SourcePojoApiMap$SourcePojoDeserializer

com.ikanow.infinit.e.data_model.custom.InfiniteFileInputFormat

com.ikanow.infinit.e.data_model.custom.InfiniteFileInputReader

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.