Package com.ikanow.infinit.e.data_model.index

Examples of com.ikanow.infinit.e.data_model.index.ElasticSearchManager


      StringBuffer sb = new StringBuffer();
      if (null == lessThanId) {// slower version, be slightly more thorough...
        sb.append(DocumentPojoIndexMap.globalDocumentIndexCollection_).append(",");
      }
      sb.append(DocumentPojoIndexMap.manyGeoDocumentIndexCollection_).append(",docs_").append(communityId).append('/').append(DocumentPojoIndexMap.documentType_);
      ElasticSearchManager indexManager = IndexManager.getIndex(sb.toString());
      BaseQueryBuilder soloOrCombinedQuery = QueryBuilders.termQuery(DocumentPojo.sourceKey_, sourceKey);
      if (null != lessThanId) {
        //(_id isn't indexed - _uid is and == _type + "#" + _id)
        soloOrCombinedQuery = QueryBuilders.boolQuery().must(soloOrCombinedQuery).
                    must(QueryBuilders.rangeQuery("_uid").lte("document_index#" + lessThanId.toString()));
       
      }//TESTED
      indexManager.doDeleteByQuery(soloOrCombinedQuery);           
     
      CommandResult result = DbManager.getDocument().getLastError("metadata");
      return result.getLong("n", 0);
     
    } catch (Exception e) {
View Full Code Here


   * @param docs
   */
  public void addToSearch(List<DocumentPojo> docs)
  {   
    String sSavedIndex = null;
    ElasticSearchManager indexManager = null;
    LinkedList<DocumentPojo> tmpDocs = new LinkedList<DocumentPojo>();
    int nTmpDocs = 0;
    for ( DocumentPojo doc : docs )
    {     
      String sThisDocIndex = doc.getIndex();
View Full Code Here

   */

  public void removeFromSearch(List<DocumentPojo> docs)
  {
    String sIndex = null;
    ElasticSearchManager indexManager = null;
    LinkedList<String> tmpDocs = new LinkedList<String>();
    int nTmpDocs = 0;
    for ( DocumentPojo doc : docs )
    { 
      if (null == doc.getId()) { // Normally this will be sourceUrls, eg files pointing to many docs
View Full Code Here

   
    if ((null == job.appendResults) || !job.appendResults) {
     
      // delete/re-create index, fill with data, swap the aliases, delete the old index
     
      ElasticSearchManager customIndex = createNewIndex(job, true);
      transferData(customIndex, job, newIdThreshold);
      swapAliases(customIndex, job.communityIds, false);
      // remove old index     
      String indexToRemove = new StringBuilder("custom_").append(job.outputCollection).toString();     
      if (IndexManager.pingIndex(indexToRemove)) { // delete if it exists...
        IndexManager.getIndex(indexToRemove).deleteMe();
      }

    }
    else { // append mode
      //TODO: note that if swapping from non-append to append mode, then might run into...
      // ...shards-too-small issue: will live with this for now (preferred option is to use aliases)
     
      // create index (and add aliases), fill with data, remove aged-out data
      ElasticSearchManager customIndex = getExistingIndex(job);
      if (null == customIndex) { // easy case, create index
        customIndex = createNewIndex(job, false);
        swapAliases(customIndex, job.communityIds, false);
      }
      transferData(customIndex, job, newIdThreshold);
View Full Code Here

      numShards++; // (somewhat arbitrary equation, 1 shard -> 5 was pretty much the justification...)
    }   
   
    Builder localSettingsEvent = ImmutableSettings.settingsBuilder();
    localSettingsEvent.put("number_of_shards", numShards).put("number_of_replicas", 0);
    ElasticSearchManager customIndex = IndexManager.createIndex(indexToBuild, "custom", false, null, null, localSettingsEvent);
      // (don't need to set the mapping - that happens automatically because I've registered a template)
   
    return customIndex;   
  }//TESTED (basic functionality)
View Full Code Here

          localSettingsGaz.putArray("index.analysis.analyzer.suggestAnalyzer.filter", "standard","lowercase");         
        }
     
        //event feature
        String eventGazMapping = new Gson().toJson(new AssociationFeaturePojoIndexMap.Mapping(), AssociationFeaturePojoIndexMap.Mapping.class)
        ElasticSearchManager eventIndex = IndexManager.createIndex(AssociationFeaturePojoIndexMap.indexName_, null, false, null, eventGazMapping, localSettingsEvent);
        if (null == eventIndex) { // (if has been previously referenced in this process space)
          eventIndex = IndexManager.getIndex(AssociationFeaturePojoIndexMap.indexName_);
        }
        eventIndex.createAlias(AssociationFeaturePojoIndexMap.indexCollectionName_);
        if (bDeleteEventFeature) {
          eventIndex.deleteMe();
          eventIndex = IndexManager.createIndex(AssociationFeaturePojoIndexMap.indexName_, null, false, null, eventGazMapping, localSettingsEvent);
        }
        //entity feature
        String gazMapping = new Gson().toJson(new EntityFeaturePojoIndexMap.Mapping(), EntityFeaturePojoIndexMap.Mapping.class)
        ElasticSearchManager entityIndex = IndexManager.createIndex(EntityFeaturePojoIndexMap.indexName_, null, false, null, gazMapping, localSettingsGaz);
        if (null == entityIndex) { // (if has been previously referenced in this process space)
          entityIndex = IndexManager.getIndex(EntityFeaturePojoIndexMap.indexName_);
        }
        entityIndex.createAlias(EntityFeaturePojoIndexMap.indexCollectionName_);
        if (bDeleteEntityFeature) {
          entityIndex.deleteMe();
          entityIndex = IndexManager.createIndex(EntityFeaturePojoIndexMap.indexName_, null, false, null, gazMapping, localSettingsGaz);
        }
      }
     
      //DOCS - much more complicated than anything else

      boolean bPingMainIndexFailed = !ElasticSearchManager.pingIndex(DocumentPojoIndexMap.globalDocumentIndex_);
        // (ie if main doc index doesn't exist then always rebuild all indexes)
     
      if (bPingMainIndexFailed) { // extra level of robustness... sleep for a minute then double check the index is really missing...
        try { Thread.sleep(60000); } catch (Exception e) {}
        bPingMainIndexFailed = !ElasticSearchManager.pingIndex(DocumentPojoIndexMap.globalDocumentIndex_);
      }
      bRebuildDocsIndex |= bPingMainIndexFailed;
     
      // check the main index has the "collection" alias - if not then rebuild everything

      if (!bPingMainIndexFailed && (null == _aliasInfo)) {
        ElasticSearchManager docIndex = ElasticSearchManager.getIndex(DocumentPojoIndexMap.globalDocumentIndex_);
        ClusterStateResponse clusterState = docIndex.getRawClient().admin().cluster().state(new ClusterStateRequest()).actionGet();
        _aliasInfo = CrossVersionImmutableMapOfImmutableMaps.getAliases(clusterState.getState().getMetaData());
        if (!_aliasInfo.containsKey(DocumentPojoIndexMap.globalDocumentIndexCollection_)) {
          bRebuildDocsIndex = true;
        }
      } //TESTED
View Full Code Here

        if (languageNormalization) {
          localSettingsGroupIndex.put("index.analysis.analyzer.default.tokenizer","standard");
          localSettingsGroupIndex.putArray("index.analysis.analyzer.default.filter", "icu_normalizer","icu_folding","standard","lowercase","stop");
        }//TESTED

        ElasticSearchManager docIndex = IndexManager.createIndex(sGroupIndex, DocumentPojoIndexMap.documentType_, false, null, docMapping, localSettingsGroupIndex);
        if (null == docIndex) { // index has already been referenced, hence createIndex returns null
          docIndex = IndexManager.getIndex(sGroupIndex);
        }
        if (bClearIndex) {
          docIndex.deleteMe();
          docIndex = IndexManager.createIndex(sGroupIndex, DocumentPojoIndexMap.documentType_, false, null, docMapping, localSettingsGroupIndex);
        }
        if (null != docIndex) {
          try {
            docIndex.pingIndex(); // (wait until it's created itself)
          }
          catch (Exception e) {} // (just make sure this doesn't die horribly)
        }
        else {
          docIndex = IndexManager.getIndex(sGroupIndex);
        }
        if (null != docIndex) { // should always be true
          docIndex.createAlias(sAliasIndex);
          docIndex.closeIndex();
        }
      }
      else if (!bParentsOnly) { // A sub-index of a parent      
       
        parentCommunityId = getRootCommunity(parentCommunityId);
       
        if (null != parentCommunityId) {
          String parentCommunityIdStr = parentCommunityId.toString();
         
          String sParentGroupIndex = new StringBuffer("doc_").append(new ObjectId(parentCommunityIdStr).toString()).toString();
          ElasticSearchManager docIndex = IndexManager.getIndex(sParentGroupIndex);
         
          //DEBUG (alias corruption)
//          if (null == _aliasInfo) {
//            ClusterStateResponse clusterState = docIndex.getRawClient().admin().cluster().state(new ClusterStateRequest()).actionGet();
//            _aliasInfo = CrossVersionImmutableMapOfImmutableMaps.getAliases(clusterState.getState().getMetaData());
//          }
//          else {
//            if (_aliasInfo.containsKey(sGroupIndex)) { // has no aliases, we're not good
//              return;
//            }
//            else {
//              //DEBUG
//              System.out.println("Alias " + sGroupIndex + " has no aliases (but should)");           
//              ElasticSearchManager docIndex2 = IndexManager.getIndex(sGroupIndex);
//              docIndex2.deleteMe();
//            }
//          }
       
          docIndex.createAlias(sGroupIndex); // for indexing
            // (this is going to be tricky when the functionality is fully implemented
            //  because it will need to handle the parent index splitting)
          docIndex.createAlias(sAliasIndex); // for queries
          docIndex.closeIndex();
          // (do nothing on delete - that will be handled at the parent index level)
        }
      }
      //TESTED (parents, children, and personal + docs_ aliases)
    }
    else { // (Personal group)
      // Just create the dummy index, no different to getting it in practice
      Builder localSettingsGroupIndex = ImmutableSettings.settingsBuilder();
      localSettingsGroupIndex.put("number_of_shards", 1).put("number_of_replicas", 0); // (ie guaranteed to be local to each ES node) 
      ElasticSearchManager dummyGroupIndex = IndexManager.createIndex(DocumentPojoIndexMap.dummyDocumentIndex_, DocumentPojoIndexMap.documentType_, false, null, docMapping, localSettingsGroupIndex);
      if (null == dummyGroupIndex) {
        dummyGroupIndex = IndexManager.getIndex(DocumentPojoIndexMap.dummyDocumentIndex_);
      }     
     
      // Just create an alias, so that queries work arbitrarily:
      dummyGroupIndex.createAlias(sGroupIndex); // (at some point we should delete the sGroupIndex alias, but leave it in for bw compatibility for now)
      dummyGroupIndex.createAlias(sAliasIndex); // (never index dummy indices so only need query index)
      // (do nothing on delete since don't have any docs in here anyway)
    }
  }
View Full Code Here

        sAliasIndex = nameOrCommunityIdStr.replaceAll("doc(?:ument)?_", "docs_");
      }
      //TESTED
    }   
    if (bPersonalGroup) {
      ElasticSearchManager dummyGroupIndex = IndexManager.getIndex(DocumentPojoIndexMap.dummyDocumentIndex_);
      dummyGroupIndex.removeAlias(sAliasIndex);
      dummyGroupIndex.removeAlias(sGroupIndex);
    }
    else if (null != parentCommunityId) {
     
      parentCommunityId = getRootCommunity(parentCommunityId);
      if (null != parentCommunityId) {
        String sParentGroupIndex = new StringBuffer("doc_").append(parentCommunityId.toString()).toString();
        ElasticSearchManager docIndex = IndexManager.getIndex(sParentGroupIndex);
        docIndex.removeAlias(sGroupIndex);
        docIndex.removeAlias(sAliasIndex);
        docIndex.closeIndex();
      }   
    }
    else {
      ElasticSearchManager docIndex = IndexManager.getIndex(sGroupIndex);
      docIndex.deleteMe();     
    }
    //TESTED (parent, children, and personal)
   
    // Also need to delete any records indexes:
    // It's a bit more complex because we're not exactly sure which indexes exist:
   
    if (null != communityId) {
      ElasticSearchManager indexMgr = ElasticSearchManager.getIndex(DocumentPojoIndexMap.globalDocumentIndex_);
        // (just something that's guaranteed to exist)

      String stashedIndex = "recs_" + communityId.toString();
      String liveIndicesPrefix = "recs_t_" + communityId.toString();
     
      ClusterStateResponse clusterState = indexMgr.getRawClient().admin().cluster().state(new ClusterStateRequest()).actionGet();
      String indices[] = clusterState.getState().getMetaData().getConcreteAllOpenIndices();
      for (String index: indices) {
        if (index.startsWith(stashedIndex) || index.startsWith(liveIndicesPrefix)) {
          ElasticSearchManager.getIndex(index).deleteMe();
        }
View Full Code Here

    if (_diagnosticMode) {
      System.out.println("EntityAggregationUtils.synchronizeEntityFeature, synchronize: " + new StringBuffer(entityFeature.getIndex()).append(':').append(communityId).toString() + " = " +
          IndexManager.mapToIndex(entityFeature, new EntityFeaturePojoIndexMap()));
    }
    else {
      ElasticSearchManager esm = IndexManager.getIndex(EntityFeaturePojoIndexMap.indexName_);       
      esm.addDocument(entityFeature, new EntityFeaturePojoIndexMap(), null, true);
        //(_id is set by the index map to index:communityId)
    }
  }//TESTED (by eye, mostly cut-and-paste from test Beta)
View Full Code Here

    if (_diagnosticMode) {
      System.out.println("EventAggregationUtils.synchronizeEventFeature, synchronize: " + new StringBuffer(eventFeature.getIndex()).append(':').append(communityId).toString() + " = " +
          IndexManager.mapToIndex(eventFeature, new AssociationFeaturePojoIndexMap()));
    }
    else {
      ElasticSearchManager esm = IndexManager.getIndex(AssociationFeaturePojoIndexMap.indexName_);       
      esm.addDocument(eventFeature, new AssociationFeaturePojoIndexMap(), null, true);
    }
  }//TESTED
View Full Code Here

TOP

Related Classes of com.ikanow.infinit.e.data_model.index.ElasticSearchManager

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.