Package org.apache.lucene.index

Examples of org.apache.lucene.index.SlowCompositeReaderWrapper


  public void testRandomIndex() throws IOException {
    Directory dir = newDirectory();
    RandomIndexWriter w = new RandomIndexWriter(random(), dir);
    createRandomIndex(atLeast(50), w, random().nextLong());
    DirectoryReader reader = w.getReader();
    SlowCompositeReaderWrapper wrapper = new SlowCompositeReaderWrapper(reader);
    String field = "body";
    Terms terms = wrapper.terms(field);
    PriorityQueue<TermAndFreq> lowFreqQueue = new PriorityQueue<CommonTermsQueryTest.TermAndFreq>(
        5) {
     
      @Override
      protected boolean lessThan(TermAndFreq a, TermAndFreq b) {
        return a.freq > b.freq;
      }
     
    };
    PriorityQueue<TermAndFreq> highFreqQueue = new PriorityQueue<CommonTermsQueryTest.TermAndFreq>(
        5) {
     
      @Override
      protected boolean lessThan(TermAndFreq a, TermAndFreq b) {
        return a.freq < b.freq;
      }
     
    };
    try {
      TermsEnum iterator = terms.iterator(null);
      while (iterator.next() != null) {
        if (highFreqQueue.size() < 5) {
          highFreqQueue.add(new TermAndFreq(
              BytesRef.deepCopyOf(iterator.term()), iterator.docFreq()));
          lowFreqQueue.add(new TermAndFreq(
              BytesRef.deepCopyOf(iterator.term()), iterator.docFreq()));
        } else {
          if (highFreqQueue.top().freq < iterator.docFreq()) {
            highFreqQueue.top().freq = iterator.docFreq();
            highFreqQueue.top().term = BytesRef.deepCopyOf(iterator.term());
            highFreqQueue.updateTop();
          }
         
          if (lowFreqQueue.top().freq > iterator.docFreq()) {
            lowFreqQueue.top().freq = iterator.docFreq();
            lowFreqQueue.top().term = BytesRef.deepCopyOf(iterator.term());
            lowFreqQueue.updateTop();
          }
        }
      }
      int lowFreq = lowFreqQueue.top().freq;
      int highFreq = highFreqQueue.top().freq;
      assumeTrue("unlucky index", highFreq - 1 > lowFreq);
      List<TermAndFreq> highTerms = queueToList(highFreqQueue);
      List<TermAndFreq> lowTerms = queueToList(lowFreqQueue);
     
      IndexSearcher searcher = new IndexSearcher(reader);
      Occur lowFreqOccur = randomOccur(random());
      BooleanQuery verifyQuery = new BooleanQuery();
      CommonTermsQuery cq = new CommonTermsQuery(randomOccur(random()),
          lowFreqOccur, highFreq - 1, random().nextBoolean());
      for (TermAndFreq termAndFreq : lowTerms) {
        cq.add(new Term(field, termAndFreq.term));
        verifyQuery.add(new BooleanClause(new TermQuery(new Term(field,
            termAndFreq.term)), lowFreqOccur));
      }
      for (TermAndFreq termAndFreq : highTerms) {
        cq.add(new Term(field, termAndFreq.term));
      }
     
      TopDocs cqSearch = searcher.search(cq, reader.maxDoc());
     
      TopDocs verifySearch = searcher.search(verifyQuery, reader.maxDoc());
      assertEquals(verifySearch.totalHits, cqSearch.totalHits);
      Set<Integer> hits = new HashSet<Integer>();
      for (ScoreDoc doc : verifySearch.scoreDocs) {
        hits.add(doc.doc);
      }
     
      for (ScoreDoc doc : cqSearch.scoreDocs) {
        assertTrue(hits.remove(doc.doc));
      }
     
      assertTrue(hits.isEmpty());
     
      /*
       *  need to force merge here since QueryUtils adds checks based
       *  on leave readers which have different statistics than the top
       *  level reader if we have more than one segment. This could
       *  result in a different query / results.
       */
      w.forceMerge(1);
      DirectoryReader reader2 = w.getReader();
      QueryUtils.check(random(), cq, newSearcher(reader2));
      reader2.close();
    } finally {
      reader.close();
      wrapper.close();
      w.close();
      dir.close();
    }
   
  }
View Full Code Here


      final DirectoryReader r = w.getReader();
      w.close();

      // NOTE: intentional but temporary field cache insanity!
      final FieldCache.Ints docIdToFieldId = FieldCache.DEFAULT.getInts(new SlowCompositeReaderWrapper(r), "id", false);
      final int[] fieldIdToDocID = new int[numDocs];
      for (int i = 0; i < numDocs; i++) {
        int fieldId = docIdToFieldId.get(i);
        fieldIdToDocID[fieldId] = i;
      }
View Full Code Here

      final DirectoryReader r = w.getReader();
      w.close();

      // NOTE: intentional but temporary field cache insanity!
      final FieldCache.Ints docIDToID = FieldCache.DEFAULT.getInts(new SlowCompositeReaderWrapper(r), "id", false);
      DirectoryReader rBlocks = null;
      Directory dirBlocks = null;

      try {
        final IndexSearcher s = newSearcher(r);
        if (VERBOSE) {
          System.out.println("\nTEST: searcher=" + s);
        }

        if (SlowCompositeReaderWrapper.class.isAssignableFrom(s.getIndexReader().getClass())) {
          canUseIDV = false;
        } else {
          canUseIDV = !preFlex;
        }
        final ShardState shards = new ShardState(s);

        for(int contentID=0;contentID<3;contentID++) {
          final ScoreDoc[] hits = s.search(new TermQuery(new Term("content", "real"+contentID)), numDocs).scoreDocs;
          for(ScoreDoc hit : hits) {
            final GroupDoc gd = groupDocs[docIDToID.get(hit.doc)];
            assertTrue(gd.score == 0.0);
            gd.score = hit.score;
            assertEquals(gd.id, docIDToID.get(hit.doc));
          }
        }

        for(GroupDoc gd : groupDocs) {
          assertTrue(gd.score != 0.0);
        }

        // Build 2nd index, where docs are added in blocks by
        // group, so we can use single pass collector
        dirBlocks = newDirectory();
        rBlocks = getDocBlockReader(dirBlocks, groupDocs);
        final Filter lastDocInBlock = new CachingWrapperFilter(new QueryWrapperFilter(new TermQuery(new Term("groupend", "x"))));
        final FieldCache.Ints docIDToIDBlocks = FieldCache.DEFAULT.getInts(new SlowCompositeReaderWrapper(rBlocks), "id", false);

        final IndexSearcher sBlocks = newSearcher(rBlocks);
        final ShardState shardsBlocks = new ShardState(sBlocks);

        // ReaderBlocks only increases maxDoc() vs reader, which
View Full Code Here

    ramdir.close();   
  }

  private void duellReaders(CompositeReader other, AtomicReader memIndexReader)
      throws IOException {
    AtomicReader competitor = new SlowCompositeReaderWrapper(other);
    Fields memFields = memIndexReader.fields();
    for (String field : competitor.fields()) {
      Terms memTerms = memFields.terms(field);
      Terms iwTerms = memIndexReader.terms(field);
      if (iwTerms == null) {
        assertNull(memTerms);
      } else {
        NumericDocValues normValues = competitor.getNormValues(field);
        NumericDocValues memNormValues = memIndexReader.getNormValues(field);
        if (normValues != null) {
          // mem idx always computes norms on the fly
          assertNotNull(memNormValues);
          assertEquals(normValues.get(0), memNormValues.get(0));
View Full Code Here

      Document doc = new Document();
      int term = i * 10; //terms are units of 10;
      doc.add(newStringField(fieldName, "" + term, Field.Store.YES));
      w.addDocument(doc);
    }
    IndexReader reader = new SlowCompositeReaderWrapper(w.getReader());
    assertTrue(reader.getContext() instanceof AtomicReaderContext);
    AtomicReaderContext context = (AtomicReaderContext) reader.getContext();
    w.close();

    List<Term> terms = new ArrayList<Term>();
    terms.add(new Term(fieldName, "19"));
    FixedBitSet bits = (FixedBitSet) termsFilter(random().nextBoolean(), terms).getDocIdSet(context, context.reader().getLiveDocs());
    assertNull("Must match nothing", bits);

    terms.add(new Term(fieldName, "20"));
    bits = (FixedBitSet) termsFilter(random().nextBoolean(), terms).getDocIdSet(context, context.reader().getLiveDocs());
    assertEquals("Must match 1", 1, bits.cardinality());

    terms.add(new Term(fieldName, "10"));
    bits = (FixedBitSet) termsFilter(random().nextBoolean(), terms).getDocIdSet(context, context.reader().getLiveDocs());
    assertEquals("Must match 2", 2, bits.cardinality());

    terms.add(new Term(fieldName, "00"));
    bits = (FixedBitSet) termsFilter(random().nextBoolean(), terms).getDocIdSet(context, context.reader().getLiveDocs());
    assertEquals("Must match 2", 2, bits.cardinality());

    reader.close();
    rd.close();
  }
View Full Code Here

    dir.close();
  }


  protected void checkCorrectClassification(Classifier<BytesRef> classifier, Analyzer analyzer) throws Exception {
    SlowCompositeReaderWrapper compositeReaderWrapper = null;
    try {
      populateIndex(analyzer);
      compositeReaderWrapper = new SlowCompositeReaderWrapper(indexWriter.getReader());
      classifier.train(compositeReaderWrapper, textFieldName, classFieldName, analyzer);
      String newText = "Much is made of what the likes of Facebook, Google and Apple know about users. Truth is, Amazon may know more.";
      ClassificationResult<BytesRef> classificationResult = classifier.assignClass(newText);
      assertNotNull(classificationResult.getAssignedClass());
      assertEquals(new BytesRef("technology"), classificationResult.getAssignedClass());
      assertTrue(classificationResult.getScore() > 0);
    } finally {
      if (compositeReaderWrapper != null)
        compositeReaderWrapper.close();
    }
  }
View Full Code Here

  }
 
  public void testBasics() throws Exception {
    // sanity check of norms writer
    // TODO: generalize
    AtomicReader slow = new SlowCompositeReaderWrapper(reader);
    NumericDocValues fooNorms = slow.getNormValues("foo");
    NumericDocValues barNorms = slow.getNormValues("bar");
    for (int i = 0; i < slow.maxDoc(); i++) {
      assertFalse(fooNorms.get(i) == barNorms.get(i));
    }
   
    // sanity check of searching
    TopDocs foodocs = searcher.search(new TermQuery(new Term("foo", "brown")), 10);
View Full Code Here

    // We need this to create thread-safe MultiSortedSetDV
    // per collector:
    if (reader instanceof AtomicReader) {
      topReader = (AtomicReader) reader;
    } else {
      topReader = new SlowCompositeReaderWrapper((CompositeReader) reader);
    }
    SortedSetDocValues dv = topReader.getSortedSetDocValues(field);
    if (dv == null) {
      throw new IllegalArgumentException("field \"" + field + "\" was not indexed with SortedSetDocValues");
    }
View Full Code Here

      final DirectoryReader r = w.getReader();
      w.close();

      // NOTE: intentional but temporary field cache insanity!
      final FieldCache.Ints docIdToFieldId = FieldCache.DEFAULT.getInts(new SlowCompositeReaderWrapper(r), "id", false);
      final int[] fieldIdToDocID = new int[numDocs];
      for (int i = 0; i < numDocs; i++) {
        int fieldId = docIdToFieldId.get(i);
        fieldIdToDocID[fieldId] = i;
      }
View Full Code Here

    addDoc(writer, "admin guest", "010", "20040101", "Y");
    addDoc(writer, "guest", "020", "20040101", "Y");
    addDoc(writer, "guest", "020", "20050101", "Y");
    addDoc(writer, "admin", "020", "20050101", "Maybe");
    addDoc(writer, "admin guest", "030", "20050101", "N");
    reader = new SlowCompositeReaderWrapper(writer.getReader());
    writer.close();
  }
View Full Code Here

TOP

Related Classes of org.apache.lucene.index.SlowCompositeReaderWrapper

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.