Examples of org.apache.lucene.facet.taxonomy.TaxonomyWriter

Package org.apache.lucene.facet.taxonomy

Examples of org.apache.lucene.facet.taxonomy.TaxonomyWriter

org.apache.lucene.facet.taxonomy.TaxonomyWriter
TaxonomyWriter is the interface which the faceted-search library uses to dynamically build the taxonomy at indexing time.
Notes about concurrent access to the taxonomy:
An implementation must allow multiple readers and a single writer to be active concurrently. Readers follow so-called "point in time" semantics, i.e., a reader object will only see taxonomy entries which were available at the time it was created. What the writer writes is only available to (new) readers after the writer's commit() is called.
Faceted search keeps two indices - namely Lucene's main index, and this taxonomy index. When one or more readers are active concurrently with the writer, care must be taken to avoid an inconsistency between the state of these two indices: When writing to the indices, the taxonomy must always be committed to disk *before* the main index, because the main index refers to categories listed in the taxonomy. Such control can best be achieved by turning off the main index's "autocommit" feature, and explicitly calling commit() for both indices (first for the taxonomy, then for the main index). In old versions of Lucene (2.2 or earlier), when autocommit could not be turned off, a more complicated solution needs to be used. E.g., use some sort of (possibly inter-process) locking to ensure that a reader is being opened only right after both indices have been flushed (and before anything else is written to them). @lucene.experimental

    
    // index some documents
    Directory indexDir = newDirectory();
    Directory taxoDir = newDirectory();
    IndexWriter indexWriter = new IndexWriter(indexDir, newIndexWriterConfig(TEST_VERSION_CURRENT, null).setMaxBufferedDocs(2));
    TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
    FacetFields facetFields = new FacetFields(taxoWriter, indexingParams);
    int ndocs = atLeast(random, 10);
    for (int i = 0; i < ndocs; i++) {
      Document doc = new Document();
      int numCategories = random.nextInt(numDimensions) + 1;

View Full Code Here

    // 4. Segment w/ categories, but only some results
    
    IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
    conf.setMergePolicy(NoMergePolicy.COMPOUND_FILES); // prevent merges, so we can control the index segments
    IndexWriter indexWriter = new IndexWriter(indexDir, conf);
    TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);


    Map<String,OrdinalPolicy> policies = new HashMap<String,CategoryListParams.OrdinalPolicy>();
    policies.put(CP_B.components[0], OrdinalPolicy.ALL_PARENTS);
    policies.put(CP_C.components[0], OrdinalPolicy.NO_PARENTS);
    policies.put(CP_D.components[0], OrdinalPolicy.NO_PARENTS);

View Full Code Here

      @Override
      public CategoryListIterator createCategoryListIterator(int partition) throws IOException {
        return new AssertingCategoryListIterator(super.createCategoryListIterator(partition));
      }
    });
    TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
    FacetFields facetFields = new FacetFields(taxoWriter, fip);
    indexTwoDocs(indexWriter, facetFields, false); // 1st segment, no content, with categories
    indexTwoDocs(indexWriter, null, true);         // 2nd segment, with content, no categories
    indexTwoDocs(indexWriter, facetFields, true);  // 3rd segment ok
    indexTwoDocs(indexWriter, null, false);        // 4th segment, no content, or categories

View Full Code Here

          assertEquals("field", name);
          return sim;
        }
      });
    RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc);
    TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir, IndexWriterConfig.OpenMode.CREATE);
    FacetFields facetFields = new FacetFields(taxoWriter);      


    Document doc = new Document();
    doc.add(newTextField("field", "text", Field.Store.NO));
    facetFields.addFields(doc, Collections.singletonList(new CategoryPath("a/path", '/')));
    writer.addDocument(doc);
    writer.close();
    taxoWriter.close();
    dir.close();
    taxoDir.close();
  }

View Full Code Here

    Random r = random();
    RandomIndexWriter writer = new RandomIndexWriter(r, dir, 
        newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(r, MockTokenizer.KEYWORD, false)));
    
    taxoDir = newDirectory();
    TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
    
    for (int i = 0; i < 100; i++) {
      ArrayList<CategoryPath> paths = new ArrayList<CategoryPath>();
      Document doc = new Document();
      if (i % 2 == 0) { // 50
        doc.add(new TextField("content", "foo", Field.Store.NO));
      }
      if (i % 3 == 0) { // 33
        doc.add(new TextField("content", "bar", Field.Store.NO));
      }
      if (i % 4 == 0) { // 25
        if (r.nextBoolean()) {
          paths.add(new CategoryPath("a/1", '/'));
        } else {
          paths.add(new CategoryPath("a/2", '/'));
        }
      }
      if (i % 5 == 0) { // 20
        paths.add(new CategoryPath("b"));
      }
      FacetFields facetFields = new FacetFields(taxoWriter);
      if (paths.size() > 0) {
        facetFields.addFields(doc, paths);
      }
      writer.addDocument(doc);
    }
    
    taxoWriter.close();
    reader = writer.getReader();
    writer.close();
    
    taxo = new DirectoryTaxonomyReader(taxoDir);
  }

View Full Code Here

  @Test
  public void testSumScoreAggregator() throws Exception {
    Directory indexDir = newDirectory();
    Directory taxoDir = newDirectory();


    TaxonomyWriter taxonomyWriter = new DirectoryTaxonomyWriter(taxoDir);
    IndexWriter iw = new IndexWriter(indexDir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())));


    FacetFields facetFields = new FacetFields(taxonomyWriter);
    for(int i = atLeast(30); i > 0; --i) {
      Document doc = new Document();
      if (random().nextBoolean()) { // don't match all documents
        doc.add(new StringField("f", "v", Store.NO));
      }
      facetFields.addFields(doc, Collections.singletonList(new CategoryPath("a")));
      iw.addDocument(doc);
    }
    
    taxonomyWriter.close();
    iw.close();
    
    DirectoryReader r = DirectoryReader.open(indexDir);
    DirectoryTaxonomyReader taxo = new DirectoryTaxonomyReader(taxoDir);

View Full Code Here

  @Test
  public void testMultiCountingLists() throws Exception {
    Directory indexDir = newDirectory();
    Directory taxoDir = newDirectory();
    
    TaxonomyWriter taxonomyWriter = new DirectoryTaxonomyWriter(taxoDir);
    IndexWriter iw = new IndexWriter(indexDir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())));
    FacetIndexingParams fip = new PerDimensionIndexingParams(Collections.singletonMap(new CategoryPath("b"), new CategoryListParams("$b")));
    
    FacetFields facetFields = new FacetFields(taxonomyWriter, fip);
    for(int i = atLeast(30); i > 0; --i) {
      Document doc = new Document();
      doc.add(new StringField("f", "v", Store.NO));
      List<CategoryPath> cats = new ArrayList<CategoryPath>();
      cats.add(new CategoryPath("a"));
      cats.add(new CategoryPath("b"));
      facetFields.addFields(doc, cats);
      iw.addDocument(doc);
    }
    
    taxonomyWriter.close();
    iw.close();
    
    DirectoryReader r = DirectoryReader.open(indexDir);
    DirectoryTaxonomyReader taxo = new DirectoryTaxonomyReader(taxoDir);

View Full Code Here

  @Test
  public void testCountAndSumScore() throws Exception {
    Directory indexDir = newDirectory();
    Directory taxoDir = newDirectory();
    
    TaxonomyWriter taxonomyWriter = new DirectoryTaxonomyWriter(taxoDir);
    IndexWriter iw = new IndexWriter(indexDir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())));
    FacetIndexingParams fip = new PerDimensionIndexingParams(Collections.singletonMap(new CategoryPath("b"), new CategoryListParams("$b")));
    
    FacetFields facetFields = new FacetFields(taxonomyWriter, fip);
    for(int i = atLeast(30); i > 0; --i) {
      Document doc = new Document();
      doc.add(new StringField("f", "v", Store.NO));
      List<CategoryPath> cats = new ArrayList<CategoryPath>();
      cats.add(new CategoryPath("a"));
      cats.add(new CategoryPath("b"));
      facetFields.addFields(doc, cats);
      iw.addDocument(doc);
    }
    
    taxonomyWriter.close();
    iw.close();
    
    DirectoryReader r = DirectoryReader.open(indexDir);
    DirectoryTaxonomyReader taxo = new DirectoryTaxonomyReader(taxoDir);

View Full Code Here

  public void testCountRoot() throws Exception {
    // LUCENE-4882: FacetsAccumulator threw NPE if a FacetRequest was defined on CP.EMPTY
    Directory indexDir = newDirectory();
    Directory taxoDir = newDirectory();
    
    TaxonomyWriter taxonomyWriter = new DirectoryTaxonomyWriter(taxoDir);
    IndexWriter iw = new IndexWriter(indexDir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())));
    
    FacetFields facetFields = new FacetFields(taxonomyWriter);
    for(int i = atLeast(30); i > 0; --i) {
      Document doc = new Document();
      facetFields.addFields(doc, Arrays.asList(new CategoryPath("a"), new CategoryPath("b")));
      iw.addDocument(doc);
    }
    
    taxonomyWriter.close();
    iw.close();
    
    DirectoryReader r = DirectoryReader.open(indexDir);
    DirectoryTaxonomyReader taxo = new DirectoryTaxonomyReader(taxoDir);

View Full Code Here

  public void testGetFacetResultsTwice() throws Exception {
    // LUCENE-4893: counts were multiplied as many times as getFacetResults was called.
    Directory indexDir = newDirectory();
    Directory taxoDir = newDirectory();
    
    TaxonomyWriter taxonomyWriter = new DirectoryTaxonomyWriter(taxoDir);
    IndexWriter iw = new IndexWriter(indexDir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())));
    
    FacetFields facetFields = new FacetFields(taxonomyWriter);
    Document doc = new Document();
    facetFields.addFields(doc, Arrays.asList(new CategoryPath("a/1", '/'), new CategoryPath("b/1", '/')));
    iw.addDocument(doc);
    taxonomyWriter.close();
    iw.close();
    
    DirectoryReader r = DirectoryReader.open(indexDir);
    DirectoryTaxonomyReader taxo = new DirectoryTaxonomyReader(taxoDir);

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.lucene.facet.taxonomy.TaxonomyWriter

com.gentics.cr.lucene.indexer.index.CRLuceneIndexJob

org.apache.lucene.benchmark.byTask.tasks.CommitTaxonomyIndexTask

org.apache.lucene.facet.associations.AssociationsFacetRequestTest

org.apache.lucene.facet.complements.TestTotalFacetCounts

org.apache.lucene.facet.complements.TestTotalFacetCountsCache

org.apache.lucene.facet.enhancements.TwoEnhancementsTest

org.apache.lucene.facet.example.association.AssociationIndexer

org.apache.lucene.facet.example.simple.SimpleIndexer

org.apache.lucene.facet.FacetTestBase

org.apache.lucene.facet.index.categorypolicy.OrdinalPolicyTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.