Examples of org.apache.lucene.pruning.PruningReader

org.apache.lucene.pruning.PruningReader
This class produces a subset of the input index, by removing some postings data according to rules implemented in a {@link TermPruningPolicy}, and optionally it can also remove stored fields of documents according to rules implemented in a {@link StorePruningPolicy}.

  public void testTfPruning() throws Exception {
    RAMDirectory targetDir = new RAMDirectory();
    //IndexReader in = IndexReader.open(sourceDir, true); 
    IndexReader in = IndexReader.open(sourceDir);
    TFTermPruningPolicy tfp = new TFTermPruningPolicy(in, null, null, 2);
    PruningReader tfr = new PruningReader(in, null, tfp);
    // verify
    assertTD(tfr, new Term("body", "one"), new int[]{1, 2, 3, 4});
    assertTD(tfr, new Term("body", "two"), new int[]{1, 2});
    assertTD(tfr, new Term("body", "three"), new int[]{1, 3});
    assertTD(tfr, new Term("test", "one"), new int[]{4});
    assertTDCount(tfr, new Term("body", "four"), 0);
    assertTDCount(tfr, new Term("test", "four"), 0);
    // verify new reader
    IndexWriter iw = new IndexWriter(targetDir, new WhitespaceAnalyzer(), /*MaxFieldLength.LIMITED*/ true);
    iw.addIndexes(new IndexReader[]{tfr});
    iw.close();
    //IndexReader ir = IndexReader.open(targetDir, true); 
    IndexReader ir = IndexReader.open(targetDir);
    assertTD(ir, new Term("body", "one"), new int[]{1, 2, 3, 4});
    assertTD(ir, new Term("body", "two"), new int[]{1, 2});
    assertTD(ir, new Term("body", "three"), new int[]{1, 3});
    assertTD(ir, new Term("test", "one"), new int[]{4});
    tfr.close();
    ir.close();
  }

View Full Code Here

    Map<String, Integer> thresholds = new HashMap<String, Integer>();
    thresholds.put("test", 3);
    //IndexReader in = IndexReader.open(sourceDir, true); 
    IndexReader in = IndexReader.open(sourceDir);
    TFTermPruningPolicy tfp = new TFTermPruningPolicy(in, null, thresholds, 2);
    PruningReader tfr = new PruningReader(in, null, tfp);
    assertTDCount(tfr, new Term("test", "one"), 0);
    assertTDCount(tfr, new Term("test", "two"), 0);
    assertTD(tfr, new Term("test", "three"), new int[]{4});
    assertTDCount(tfr, new Term("test", "four"), 0);
  }

View Full Code Here

    removeFields.put("test", PruningPolicy.DEL_POSTINGS | PruningPolicy.DEL_STORED);
    //IndexReader in = IndexReader.open(sourceDir, true); 
    IndexReader in = IndexReader.open(sourceDir);
    TFTermPruningPolicy tfp = new TFTermPruningPolicy(in, removeFields, null, 2);
    StorePruningPolicy stp = new StorePruningPolicy(in, removeFields);
    PruningReader tfr = new PruningReader(in, stp, tfp);
    Document doc = tfr.document(4);
    // removed stored values?
    assertNull(doc.get("test"));
    // removed postings ?
    TermEnum te = tfr.terms();
    while (te.next()) {
      assertFalse("test".equals(te.term().field()));
    }
    // but vectors should be present !
    TermFreqVector tv = tfr.getTermFreqVector(4, "test");
    assertNotNull(tv);
    assertEquals(4, tv.getTerms().length); // term "four" not deleted yet from TermEnum
    // verify new reader
    IndexWriter iw = new IndexWriter(targetDir, new WhitespaceAnalyzer(), /*MaxFieldLength.LIMITED*/ true); 
    iw.addIndexes(new IndexReader[]{tfr});

View Full Code Here

TOP

Related Classes of org.apache.lucene.pruning.PruningReader

org.apache.lucene.index.SegmentTermPositionVector

org.apache.lucene.index.Term

org.apache.lucene.pruning.TestPruningReader

java.util.Collection

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.