Examples of org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter$OrdinalMap

Package org.apache.lucene.facet.taxonomy.directory

Examples of org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter$OrdinalMap

org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter
Mapping from old ordinal to new ordinals, used when merging indexes wit separate taxonomies.
addToTaxonomies() merges one or more taxonomies into the given taxonomy (this). An OrdinalMap is filled for each of the added taxonomies, containing the new ordinal (in the merged taxonomy) of each of the categories in the old taxonomy.
There exist two implementations of OrdinalMap: MemoryOrdinalMap and DiskOrdinalMap. As their names suggest, the former keeps the map in memory and the latter in a temporary disk file. Because these maps will later be needed one by one (to remap the counting lists), not all at the same time, it is recommended to put the first taxonomy's map in memory, and all the rest on disk (later to be automatically read into memory one by one, when needed).

        pair.searchDir = newDirectory();
        pair.taxoDir = newDirectory();
      }
      
      RandomIndexWriter iw = new RandomIndexWriter(random(), pair.searchDir, getIndexWriterConfig(getAnalyzer()));
      TaxonomyWriter taxo = new DirectoryTaxonomyWriter(pair.taxoDir, OpenMode.CREATE);
      
      populateIndex(iw, taxo, fip);
      
      // commit changes (taxonomy prior to search index for consistency)
      taxo.commit();
      iw.commit();
      taxo.close();
      iw.close();
      
      dirsPerPartitionSize.put(Integer.valueOf(partitionSize), pair);
    }

View Full Code Here

  }


  private void index100Docs(Directory indexDir, Directory taxoDir, FacetIndexingParams fip) throws IOException {
    IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, null);
    IndexWriter w = new IndexWriter(indexDir, iwc);
    TaxonomyWriter tw = new DirectoryTaxonomyWriter(taxoDir);
    
    FacetFields facetFields = new FacetFields(tw, fip);
    for (int i = 0; i < 100; i++) {
      Document doc = new Document();
      CategoryPath cp = new CategoryPath("root",Integer.toString(i / 10), Integer.toString(i));

View Full Code Here

   *  faceting, with DrillSideways and taxonomy. */
  public void testMixedRangeAndNonRangeTaxonomy() throws Exception {
    Directory d = newDirectory();
    RandomIndexWriter w = new RandomIndexWriter(random(), d);
    Directory td = newDirectory();
    DirectoryTaxonomyWriter tw = new DirectoryTaxonomyWriter(td, IndexWriterConfig.OpenMode.CREATE);
    FacetFields ff = new FacetFields(tw);


    for (long l = 0; l < 100; l++) {
      Document doc = new Document();
      // For computing range facet counts:
      doc.add(new NumericDocValuesField("field", l));
      // For drill down by numeric range:
      doc.add(new LongField("field", l, Field.Store.NO));


      CategoryPath cp;
      if ((l&3) == 0) {
        cp = new CategoryPath("dim", "a");
      } else {
        cp = new CategoryPath("dim", "b");
      }
      ff.addFields(doc, Collections.singletonList(cp));
      w.addDocument(doc);
    }


    final IndexReader r = w.getReader();
    w.close();


    final TaxonomyReader tr = new DirectoryTaxonomyReader(tw);
    tw.close();


    IndexSearcher s = newSearcher(r);


    final CountFacetRequest countRequest = new CountFacetRequest(new CategoryPath("dim"), 2);
    final RangeFacetRequest<LongRange> rangeRequest = new RangeFacetRequest<LongRange>("field",

View Full Code Here


  private static final String[] CATEGORIES = new String[] { "a/b", "c/d", "a/e", "a/d", "c/g", "c/z", "b/a", "1/2", "b/c" };


  private void index(Directory indexDir, Directory taxoDir) throws IOException {
    IndexWriter indexWriter = new IndexWriter(indexDir, newIndexWriterConfig(TEST_VERSION_CURRENT, null));
    TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
    FacetFields facetFields = new FacetFields(taxoWriter);
    
    for (String cat : CATEGORIES) {
      Document doc = new Document();
      facetFields.addFields(doc, Collections.singletonList(new CategoryPath(cat, '/')));

View Full Code Here

    Directory indexDir = newDirectory();
    Directory taxoDir = newDirectory();


    // Create our index/taxonomy writers
    IndexWriter indexWriter = new IndexWriter(indexDir, newIndexWriterConfig(TEST_VERSION_CURRENT, null));
    TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
    FacetIndexingParams iParams = FacetIndexingParams.DEFAULT;


    // Add a facet to the index
    addFacets(iParams, indexWriter, taxoWriter, "a", "b");


    // Commit Changes
    indexWriter.commit();
    taxoWriter.commit();


    // Open readers
    DirectoryReader indexReader = DirectoryReader.open(indexDir);
    TaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoDir);

View Full Code Here

    Directory indexDir = newDirectory();
    Directory taxoDir = newDirectory();
    
    // Create our index/taxonomy writers
    IndexWriter indexWriter = new IndexWriter(indexDir, newIndexWriterConfig(TEST_VERSION_CURRENT, null));
    TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
    FacetIndexingParams iParams = new FacetIndexingParams() {
      @Override
      public int getPartitionSize() {
        return 2;
      }
    };
    // Add a facet to the index
    addFacets(iParams, indexWriter, taxoWriter, "a", "b");
    // Commit Changes
    indexWriter.commit();
    taxoWriter.commit();


    DirectoryReader indexReader = DirectoryReader.open(indexDir);
    TaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoDir);


    // Create TFC and write cache to disk
    File outputFile = _TestUtil.createTempFile("test", "tmp", TEMP_DIR);
    TFC.store(outputFile, indexReader, taxoReader, iParams);
    
    // Make the taxonomy grow without touching the index
    for (int i = 0; i < 10; i++) {
      taxoWriter.addCategory(new CategoryPath("foo", Integer.toString(i)));
    }
    taxoWriter.commit();
    TaxonomyReader newTaxoReader = TaxonomyReader.openIfChanged(taxoReader);
    assertNotNull(newTaxoReader);
    taxoReader.close();
    taxoReader = newTaxoReader;

View Full Code Here

    SlowRAMDirectory taxoDir = new SlowRAMDirectory(-1, null);


    // Write index using 'normal' directories
    IndexWriter w = new IndexWriter(indexDir, new IndexWriterConfig(
        TEST_VERSION_CURRENT, new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)));
    DirectoryTaxonomyWriter tw = new DirectoryTaxonomyWriter(taxoDir);
    FacetIndexingParams iParams = FacetIndexingParams.DEFAULT;
    // Add documents and facets
    for (int i = 0; i < 1000; i++) {
      addFacets(iParams, w, tw, "facet", Integer.toString(i));
    }
    w.close();
    tw.close();


    indexDir.setSleepMillis(1);
    taxoDir.setSleepMillis(1);


    IndexReader r = DirectoryReader.open(indexDir);

View Full Code Here

    Directory taxoDir1 = newDirectory(), taxoDir2 = newDirectory();
    
    // Create our index/taxonomy writers
    IndexWriter indexWriter1 = new IndexWriter(indexDir1, newIndexWriterConfig(TEST_VERSION_CURRENT, null));
    IndexWriter indexWriter2 = new IndexWriter(indexDir2, newIndexWriterConfig(TEST_VERSION_CURRENT, null));
    TaxonomyWriter taxoWriter1 = new DirectoryTaxonomyWriter(taxoDir1);
    TaxonomyWriter taxoWriter2 = new DirectoryTaxonomyWriter(taxoDir2);
    FacetIndexingParams iParams = FacetIndexingParams.DEFAULT;


    // Add a facet to the index
    addFacets(iParams, indexWriter1, taxoWriter1, "a", "b");
    addFacets(iParams, indexWriter1, taxoWriter1, "d", "e");
    // Commit Changes
    indexWriter1.commit();
    indexWriter2.commit();
    taxoWriter1.commit();
    taxoWriter2.commit();


    // Open two readers
    DirectoryReader indexReader1 = DirectoryReader.open(indexDir1);
    DirectoryReader indexReader2 = DirectoryReader.open(indexDir2);
    TaxonomyReader taxoReader1 = new DirectoryTaxonomyReader(taxoDir1);

View Full Code Here

    taxoDir = newDirectory();
    // preparations - index, taxonomy, content
    RandomIndexWriter writer = new RandomIndexWriter(random(), dir, newIndexWriterConfig(TEST_VERSION_CURRENT, 
        new MockAnalyzer(random(), MockTokenizer.KEYWORD, false)));
    
    TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
    
    AssociationsFacetFields assocFacetFields = new AssociationsFacetFields(taxoWriter);
    
    // index documents, 50% have only 'b' and all have 'a'
    for (int i = 0; i < 110; i++) {
      Document doc = new Document();
      CategoryAssociationsContainer associations = new CategoryAssociationsContainer();
      // every 11th document is added empty, this used to cause the association
      // aggregators to go into an infinite loop
      if (i % 11 != 0) {
        associations.setAssociation(aint, new CategoryIntAssociation(2));
        associations.setAssociation(afloat, new CategoryFloatAssociation(0.5f));
        if (i % 2 == 0) { // 50
          associations.setAssociation(bint, new CategoryIntAssociation(3));
          associations.setAssociation(bfloat, new CategoryFloatAssociation(0.2f));
        }
      }
      assocFacetFields.addFields(doc, associations);
      writer.addDocument(doc);
    }
    
    taxoWriter.close();
    reader = writer.getReader();
    writer.close();
  }

View Full Code Here

    replicator.publish(createRevision(1));
    client.updateNow();
    
    // recreate index and taxonomy
    Directory newTaxo = newDirectory();
    new DirectoryTaxonomyWriter(newTaxo).close();
    publishTaxoWriter.replaceTaxonomy(newTaxo);
    publishIndexWriter.deleteAll();
    replicator.publish(createRevision(2));
    
    client.updateNow();

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter$OrdinalMap

org.apache.lucene.analysis.KeywordAnalyzer

org.apache.lucene.benchmark.byTask.tasks.CreateTaxonomyIndexTask

org.apache.lucene.benchmark.byTask.tasks.OpenTaxonomyIndexTask

org.apache.lucene.demo.facet.AssociationsFacetsExample

org.apache.lucene.demo.facet.ExpressionAggregationFacetsExample

org.apache.lucene.demo.facet.MultiCategoryListsFacetsExample

org.apache.lucene.demo.facet.SimpleFacetsExample

org.apache.lucene.document.Document

org.apache.lucene.document.Field

org.apache.lucene.document.FieldType

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.