Examples of org.apache.lucene.pruning.TFTermPruningPolicy

org.apache.lucene.pruning.TFTermPruningPolicy
This class produces a subset of the input index, by removing postings data for those terms where their in-document frequency is below a specified threshold. The net effect of this processing is a much smaller index that for many types of queries returns nearly identical top-N results as compared with the original index, but with increased performance.
See the comments in {@link CarmelTermPruningPolicy} for more details. Thisimplementation uses simple term frequency thresholds to remove all postings from documents where a given term occurs rarely (i.e. its TF in a document is smaller than the threshold).
Threshold values in this method are expressed as absolute term frequencies.


  public void testTfPruning() throws Exception {
    RAMDirectory targetDir = new RAMDirectory();
    //IndexReader in = IndexReader.open(sourceDir, true); 
    IndexReader in = IndexReader.open(sourceDir);
    TFTermPruningPolicy tfp = new TFTermPruningPolicy(in, null, null, 2);
    PruningReader tfr = new PruningReader(in, null, tfp);
    // verify
    assertTD(tfr, new Term("body", "one"), new int[]{1, 2, 3, 4});
    assertTD(tfr, new Term("body", "two"), new int[]{1, 2});
    assertTD(tfr, new Term("body", "three"), new int[]{1, 3});

View Full Code Here

  public void testThresholds() throws Exception {
    Map<String, Integer> thresholds = new HashMap<String, Integer>();
    thresholds.put("test", 3);
    //IndexReader in = IndexReader.open(sourceDir, true); 
    IndexReader in = IndexReader.open(sourceDir);
    TFTermPruningPolicy tfp = new TFTermPruningPolicy(in, null, thresholds, 2);
    PruningReader tfr = new PruningReader(in, null, tfp);
    assertTDCount(tfr, new Term("test", "one"), 0);
    assertTDCount(tfr, new Term("test", "two"), 0);
    assertTD(tfr, new Term("test", "three"), new int[]{4});
    assertTDCount(tfr, new Term("test", "four"), 0);

View Full Code Here

    RAMDirectory targetDir = new RAMDirectory();
    Map<String, Integer> removeFields = new HashMap<String, Integer>();
    removeFields.put("test", PruningPolicy.DEL_POSTINGS | PruningPolicy.DEL_STORED);
    //IndexReader in = IndexReader.open(sourceDir, true); 
    IndexReader in = IndexReader.open(sourceDir);
    TFTermPruningPolicy tfp = new TFTermPruningPolicy(in, removeFields, null, 2);
    StorePruningPolicy stp = new StorePruningPolicy(in, removeFields);
    PruningReader tfr = new PruningReader(in, stp, tfp);
    Document doc = tfr.document(4);
    // removed stored values?
    assertNull(doc.get("test"));

View Full Code Here

TOP

Related Classes of org.apache.lucene.pruning.TFTermPruningPolicy

org.apache.lucene.index.TermDocs

org.apache.lucene.pruning.TestPruningReader

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.