Examples of org.apache.lucene.index.TermEnum

org.apache.lucene.index.TermEnum
Abstract class for enumerating terms.
Term enumerations are always ordered by Term.compareTo(). Each term in the enumeration is greater than all that precede it.

    r.close();
    d.close();
  }


  public void termEnumTest(IndexReader r) throws Exception {
    TermEnum terms = r.terms();


    assertNull(terms.term());
    assertFalse(terms.next());


  }

View Full Code Here

      if (fields == null || fields.contains(fieldName)) {
        getTermsByFieldAndText().put(fieldName, new HashMap<String, InstantiatedTerm>(5000));
      }
    }
    List<InstantiatedTerm> terms = new ArrayList<InstantiatedTerm>(5000 * getTermsByFieldAndText().size());
    TermEnum termEnum = sourceIndexReader.terms();
    while (termEnum.next()) {
      if (fields == null || fields.contains(termEnum.term().field())) { // todo skipto if not using field
        InstantiatedTerm instantiatedTerm = new InstantiatedTerm(termEnum.term().field(), termEnum.term().text());
        getTermsByFieldAndText().get(termEnum.term().field()).put(termEnum.term().text(), instantiatedTerm);
        instantiatedTerm.setTermIndex(terms.size());
        terms.add(instantiatedTerm);
        instantiatedTerm.setAssociatedDocuments(new InstantiatedTermDocumentInformation[termEnum.docFreq()]);
      }
    }
    termEnum.close();
    orderedTerms = terms.toArray(new InstantiatedTerm[terms.size()]);


    // create term-document informations
    for (InstantiatedTerm term : orderedTerms) {
      TermPositions termPositions = sourceIndexReader.termPositions(term.getTerm());

View Full Code Here

        LinkedList termsWithPrefix = new LinkedList();
        IndexReader ir = IndexReader.open(indexStore, true);


        // this TermEnum gives "piccadilly", "pie" and "pizza".
        String prefix = "pi";
        TermEnum te = ir.terms(new Term("body", prefix));
        do {
            if (te.term().text().startsWith(prefix))
            {
                termsWithPrefix.add(te.term());
            }
        } while (te.next());


        query1.add((Term[])termsWithPrefix.toArray(new Term[0]));
        assertEquals("body:\"blueberry (piccadilly pie pizza)\"", query1.toString());
        query2.add((Term[])termsWithPrefix.toArray(new Term[0]));
        assertEquals("body:\"strawberry (piccadilly pie pizza)\"", query2.toString());


        ScoreDoc[] result;
        result = searcher.search(query1, null, 1000).scoreDocs;
        assertEquals(2, result.length);
        result = searcher.search(query2, null, 1000).scoreDocs;
        assertEquals(0, result.length);


        // search for "blue* pizza":
        MultiPhraseQuery query3 = new MultiPhraseQuery();
        termsWithPrefix.clear();
        prefix = "blue";
        te = ir.terms(new Term("body", prefix));
        do {
            if (te.term().text().startsWith(prefix))
            {
                termsWithPrefix.add(te.term());
            }
        } while (te.next());
        query3.add((Term[])termsWithPrefix.toArray(new Term[0]));
        query3.add(new Term("body", "pizza"));


        result = searcher.search(query3, null, 1000).scoreDocs;
        assertEquals(2, result.length); // blueberry pizza, bluebird pizza

View Full Code Here

   * @throws IOException
   */
  public int addStopWords(IndexReader reader, String fieldName, int maxDocFreq) throws IOException {
    HashSet<String> stopWords = new HashSet<String>();
    String internedFieldName = StringHelper.intern(fieldName);
    TermEnum te = reader.terms(new Term(fieldName));
    Term term = te.term();
    while (term != null) {
      if (term.field() != internedFieldName) {
        break;
      }
      if (te.docFreq() > maxDocFreq) {
        stopWords.add(term.text());
      }
      if (!te.next()) {
        break;
      }
      term = te.term();
    }
    stopWordsPerField.put(fieldName, stopWords);
    
    /* if the stopwords for a field are changed,
     * then saved streams for that field are erased.

View Full Code Here

            }


            for (Iterator it = startTerms.iterator(); it.hasNext(); ) {
                Term startTerm = (Term) it.next();


                TermEnum terms = reader.terms(startTerm);
                try {
                    TermDocs docs = reader.termDocs();
                    try {
                        do {
                            Term term = terms.term();
                            if (term != null
                                    && term.field() == testField
                                    && term.text().startsWith(namePrefix)) {
                                if (checkLower) {
                                    int compare = termCompare(term.text(), lowerTerm.text(), propNameLength);
                                    if (compare > 0 || compare == 0 && inclusive) {
                                        // do not check lower term anymore if no
                                        // transformation is done on the term enum
                                        checkLower = transform == TRANSFORM_NONE ? false : true;
                                    } else {
                                        // continue with next term
                                        continue;
                                    }
                                }
                                if (upperTerm != null) {
                                    int compare = termCompare(term.text(), upperTerm.text(), propNameLength);
                                    // if beyond the upper term, or is exclusive and
                                    // this is equal to the upper term
                                    if ((compare > 0) || (!inclusive && compare == 0)) {
                                        // only break out if no transformation
                                        // was done on the term from the enum
                                        if (transform == TRANSFORM_NONE) {
                                            break;
                                        } else {
                                            // because of the transformation
                                            // it is possible that the next
                                            // term will be included again if
                                            // we still enumerate on the same
                                            // property name
                                            if (term.text().startsWith(namePrefix)) {
                                                continue;
                                            } else {
                                                break;
                                            }
                                        }
                                    }
                                }


                                docs.seek(terms);
                                while (docs.next()) {
                                    hits.set(docs.doc());
                                }
                            } else {
                                break;
                            }
                        } while(terms.next());
                    } finally {
                        docs.close();
                    }
                } finally {
                    terms.close();
                }
            }


            hitsCalculated = true;
            // put to cache

View Full Code Here

         */
        private void calculateHits() throws IOException {
            if (hitsCalculated) {
                return;
            }
            TermEnum terms = new WildcardTermEnum(reader, field, propName, pattern, transform);
            try {
                // use unpositioned TermDocs
                TermDocs docs = reader.termDocs();
                try {
                    while (terms.term() != null) {
                        docs.seek(terms);
                        while (docs.next()) {
                            hits.set(docs.doc());
                        }
                        if (!terms.next()) {
                            break;
                        }
                    }
                } finally {
                    docs.close();
                }
            } finally {
                terms.close();
            }
            hitsCalculated = true;
            // put to cache
            synchronized (resultMap) {
                resultMap.put(cacheKey, hits);

View Full Code Here

     *                     the search index.
     */
    private void calculateDocFilter() throws IOException {
        docFilter = new BitSet(reader.maxDoc());
        // we match all terms
        TermEnum terms = reader.terms(new Term(FieldNames.PROPERTIES, field));
        try {
            TermDocs docs = reader.termDocs();
            try {
                while (terms.term() != null
                        && terms.term().field() == FieldNames.PROPERTIES
                        && terms.term().text().startsWith(field)) {
                    docs.seek(terms);
                    while (docs.next()) {
                        docFilter.set(docs.doc());
                    }
                    terms.next();
                }
            } finally {
                docs.close();
            }
        } finally {
            terms.close();
        }
    }

View Full Code Here

    
    FieldType ftleft=schema.getFieldType(fieldLeft);
    
    String prefixLeft=TrieField.getMainValuePrefix(ftleft);
    Term tiLeft=new Term(fieldLeft, prefixLeft==null?"":prefixLeft);
    TermEnum teLeft = null;
    TermDocs tdleft=null;
    if(this.leftreader!=null)
    {
      this.joinRevert=new int[this.leftreader.maxDoc()+1];
      tdleft=this.leftreader.termDocs();
      teLeft=this.leftreader.terms(tiLeft);
    }else{
      this.joinRevert=new int[readerleft.getReader().maxDoc()+1];
      teLeft=readerleft.getReader().terms(tiLeft);
      tdleft=readerleft.getReader().termDocs();
    }
    
    for(int i=0;i<this.joinRevert.length;i++)
    {
      this.joinRevert[i]=-1;
    }
    


    
    FieldType ftright =readerright.getSchema().getFieldType(fieldRigth);
    String prefixRight=TrieField.getMainValuePrefix(ftright);
    Term tiRight=new Term(fieldRigth, prefixRight==null?"":prefixRight);
    
    TermEnum teRight = readerright.getReader().terms(tiRight.createTerm(prefixRight==null?"":prefixRight));
    TermDocs tdRight=readerright.getReader().termDocs();
    this.join=new int[readerright.getReader().maxDoc()+1][];
    for(int i=0;i<this.join.length;i++)
    {
      this.join[i]=null;
    }
    
    int[] docs = new int[1000];
    int[] freqs = new int[1000];
    
    
    int debugline=0;
    
    HashMap<Integer,HashSet<Integer>> join_tmp=new HashMap<Integer,HashSet<Integer>>();
//    HashMap<Integer,Integer> joinRevert_tmp=new HashMap<Integer,Integer>();
    for (;;) {
      Term tleft= teLeft.term();
      Term tRight=teRight.term();
      
      
      if (tleft == null||tRight==null) {
        LOG.info("###termbreak###"+String.valueOf(tleft)+">>>>"+String.valueOf(tRight)+","+fieldLeft+","+fieldRigth);
        break;
      }
      if((!tleft.field().equals(fieldLeft))||(!tRight.field().equals(fieldRigth)))
      {
        LOG.info("###termbreak fieldchange###"+String.valueOf(tleft)+">>>>"+String.valueOf(tRight)+","+fieldLeft+","+fieldRigth);
        break;
      }
      
      String tvleft=ftleft.indexedToReadable(tleft.text());
      String tvRight=ftright.indexedToReadable(tRight.text());




      if(tvleft.equals(tvRight))
      {
        if(debugline++<10)
        {
          LOG.info("###termok###"+String.valueOf(tvleft)+">>>>"+String.valueOf(tvRight)+","+fieldLeft+","+fieldRigth);
        }


        if(tvleft!=null&&!tvleft.trim().isEmpty())
        {
          IntArr LeftArr=this.getListArr(teLeft, tdleft, docs, freqs,Integer.MAX_VALUE);
          IntArr RightArr=this.getListArr(teRight, tdRight, docs, freqs,1);
  
          for (Integer docid:RightArr.list) {
            HashSet<Integer> list=join_tmp.get(docid);
            if(list==null)
            {
              list=new HashSet<Integer>();
              join_tmp.put(docid, list);
            }
            for(int jid:LeftArr.list)
            {
  //            joinRevert_tmp.put(jid, docid);
              this.joinRevert[jid]=docid;
              list.add(jid);
            }
          }
        }else{
          LOG.info("###empty###"+String.valueOf(tvleft)+">>>>"+String.valueOf(tvRight)+","+fieldLeft+","+fieldRigth);
        }
        
        
        teLeft.next();
        teRight.next();
      }else if(tvleft.compareTo(tvRight)>0)
      {
        teRight.next();
      }else{
        teLeft.next();
      }
    }
    
    teLeft.close();
    teRight.close();
        
    for(Entry<Integer, HashSet<Integer>> e:join_tmp.entrySet())
    {
      this.join[e.getKey()]=IntArr.parse(e.getValue()).list;
    }

View Full Code Here

    
    indexInfo.add("numDocs", reader.numDocs());
    indexInfo.add("maxDoc", reader.maxDoc());
    
    if( countTerms ) {
      TermEnum te = null;
      try{
        te = reader.terms();
        int numTerms = 0;
        while (te.next()) {
          numTerms++;
        }
        indexInfo.add("numTerms", numTerms );
      }
      finally{
        if( te != null ) te.close();
      }
    }


    indexInfo.add("version", reader.getVersion());  // TODO? Is this different then: IndexReader.getCurrentVersion( dir )?
    indexInfo.add("segmentCount", reader.getSequentialSubReaders().length);

View Full Code Here


  private static Map<String,TopTermQueue> getTopTerms( IndexReader reader, Set<String> fields, int numTerms, Set<String> junkWords ) throws Exception 
  {
    Map<String,TopTermQueue> info = new HashMap<String, TopTermQueue>();
    
    TermEnum terms = null;
    try{
      terms = reader.terms();    
      while (terms.next()) {
        String field = terms.term().field();
        String t = terms.term().text();
  
        // Compute distinct terms for every field
        TopTermQueue tiq = info.get( field );
        if( tiq == null ) {
          tiq = new TopTermQueue( numTerms+1 );
          info.put( field, tiq );
        }
        tiq.distinctTerms++;
        tiq.histogram.add( terms.docFreq() );  // add the term to the histogram
        
        // Only save the distinct terms for fields we worry about
        if (fields != null && fields.size() > 0) {
          if( !fields.contains( field ) ) {
            continue;
          }
        }
        if( junkWords != null && junkWords.contains( t ) ) {
          continue;
        }
        
        if( terms.docFreq() > tiq.minFreq ) {
          tiq.add(new TopTermQueue.TermInfo(terms.term(), terms.docFreq()));
            if (tiq.size() > numTerms) { // if tiq full
            tiq.pop(); // remove lowest in tiq
            tiq.minFreq = ((TopTermQueue.TermInfo)tiq.top()).docFreq; // reset minFreq
          }
        }
      }
    }
    finally {
      if( terms != null ) terms.close();
    }
    return info;
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.lucene.index.TermEnum

com.browseengine.bobo.api.BoboIndexReader

com.browseengine.bobo.facets.data.FacetDataCache

com.browseengine.bobo.facets.data.MultiValueFacetDataCache

com.browseengine.bobo.facets.impl.CompactMultiValueFacetHandler

com.greplin.lucene.filter.PrefixFilter

com.greplin.lucene.query.PhrasePrefixQuery

com.tamingtext.tagging.LuceneTagExtractor

er.indexing.ERIndex

it.unibz.instasearch.indexing.Searcher

lucli.LuceneMethods

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.