Package org.apache.lucene.analysis.standard

Examples of org.apache.lucene.analysis.standard.StandardAnalyzer


  }

  // LUCENE-1404
  public void testNPESpanQuery() throws Throwable {
    final Directory dir = new MockRAMDirectory();
    final IndexWriter writer = new IndexWriter(dir, new StandardAnalyzer(org.apache.lucene.util.Version.LUCENE_CURRENT, Collections.emptySet()), IndexWriter.MaxFieldLength.LIMITED);

    // Add documents
    addDoc(writer, "1", "the big dogs went running to the market");
    addDoc(writer, "2", "the cat chased the mouse, then the cat ate the mouse quickly");
   
View Full Code Here


        super.setUp();
        super.setUp();

        // create test index
        mDirectory = new RAMDirectory();
        final IndexWriter writer = new IndexWriter(mDirectory, new StandardAnalyzer(org.apache.lucene.util.Version.LUCENE_CURRENT), true, IndexWriter.MaxFieldLength.LIMITED);
        addDocument(writer, "1", "I think it should work.");
        addDocument(writer, "2", "I think it should work.");
        addDocument(writer, "3", "I think it should work.");
        addDocument(writer, "4", "I think it should work.");
        writer.close();
View Full Code Here

    @Override
    protected void setUp() throws Exception {
        super.setUp();

        // create test index
        final IndexWriter writer = new IndexWriter(mDirectory, new StandardAnalyzer(org.apache.lucene.util.Version.LUCENE_CURRENT), false, IndexWriter.MaxFieldLength.LIMITED);
        addDocument(writer, "A", "Should we, could we, would we?");
        addDocument(writer, "B", "It should.  Should it?");
        addDocument(writer, "C", "It shouldn't.");
        addDocument(writer, "D", "Should we, should we, should we.");
        writer.close();
View Full Code Here

  // create an index for testing
  private Directory getIndex()
  throws IOException {
          RAMDirectory indexStore = new RAMDirectory ();
          IndexWriter writer = new IndexWriter (indexStore, new StandardAnalyzer(org.apache.lucene.util.Version.LUCENE_CURRENT), true, IndexWriter.MaxFieldLength.LIMITED);
          RandomGen random = new RandomGen(newRandom());
          for (int i=0; i<INDEX_SIZE; ++i) { // don't decrease; if to low the problem doesn't show up
          Document doc = new Document();
              if((i%5)!=0) { // some documents must not have an entry in the first sort field
                  doc.add (new Field("publicationDate_", random.getLuceneDate(), Field.Store.YES, Field.Index.NOT_ANALYZED));
View Full Code Here

    FSDirectory storeDirectory;
    storeDirectory = new MMapDirectory(new File(storePathname), null);

    // plan to add a set of useful stopwords, consider changing some of the
    // interior filters.
    StandardAnalyzer analyzer = new StandardAnalyzer(org.apache.lucene.util.Version.LUCENE_CURRENT, Collections.emptySet());
    // TODO: something about lock timeouts and leftover locks.
    IndexWriter writer = new IndexWriter(storeDirectory, analyzer, true, IndexWriter.MaxFieldLength.LIMITED);
    IndexSearcher searcher = new IndexSearcher(storeDirectory, true);
   
    for(int dx = 0; dx < 1000; dx ++) {
View Full Code Here

            Field.Store.YES, Field.Index.ANALYZED));
        lDoc3.add(new Field("id", "doc3", Field.Store.YES, Field.Index.NOT_ANALYZED));
        lDoc3.add(new Field("handle", "1", Field.Store.YES, Field.Index.NOT_ANALYZED));

        // creating an index writer for the first index
        IndexWriter writerA = new IndexWriter(indexStoreA, new StandardAnalyzer(org.apache.lucene.util.Version.LUCENE_CURRENT), true, IndexWriter.MaxFieldLength.LIMITED);
        // creating an index writer for the second index, but writing nothing
        IndexWriter writerB = new IndexWriter(indexStoreB, new StandardAnalyzer(org.apache.lucene.util.Version.LUCENE_CURRENT), true, IndexWriter.MaxFieldLength.LIMITED);

        //--------------------------------------------------------------------
        // scenario 1
        //--------------------------------------------------------------------

        // writing the documents to the first index
        writerA.addDocument(lDoc);
        writerA.addDocument(lDoc2);
        writerA.addDocument(lDoc3);
        writerA.optimize();
        writerA.close();

        // closing the second index
        writerB.close();

        // creating the query
        QueryParser parser = new QueryParser(Version.LUCENE_CURRENT, "fulltext", new StandardAnalyzer(org.apache.lucene.util.Version.LUCENE_CURRENT));
        Query query = parser.parse("handle:1");

        // building the searchables
        Searcher[] searchers = new Searcher[2];
        // VITAL STEP:adding the searcher for the empty index first, before the searcher for the populated index
        searchers[0] = new IndexSearcher(indexStoreB, true);
        searchers[1] = new IndexSearcher(indexStoreA, true);
        // creating the multiSearcher
        Searcher mSearcher = getMultiSearcherInstance(searchers);
        // performing the search
        ScoreDoc[] hits = mSearcher.search(query, null, 1000).scoreDocs;

        assertEquals(3, hits.length);

        // iterating over the hit documents
        for (int i = 0; i < hits.length; i++) {
          mSearcher.doc(hits[i].doc);
        }
        mSearcher.close();


        //--------------------------------------------------------------------
        // scenario 2
        //--------------------------------------------------------------------

        // adding one document to the empty index
        writerB = new IndexWriter(indexStoreB, new StandardAnalyzer(org.apache.lucene.util.Version.LUCENE_CURRENT), false, IndexWriter.MaxFieldLength.LIMITED);
        writerB.addDocument(lDoc);
        writerB.optimize();
        writerB.close();

        // building the searchables
        Searcher[] searchers2 = new Searcher[2];
        // VITAL STEP:adding the searcher for the empty index first, before the searcher for the populated index
        searchers2[0] = new IndexSearcher(indexStoreB, true);
        searchers2[1] = new IndexSearcher(indexStoreA, true);
        // creating the mulitSearcher
        MultiSearcher mSearcher2 = getMultiSearcherInstance(searchers2);
        // performing the same search
        ScoreDoc[] hits2 = mSearcher2.search(query, null, 1000).scoreDocs;

        assertEquals(4, hits2.length);

        // iterating over the hit documents
        for (int i = 0; i < hits2.length; i++) {
          // no exception should happen at this point
          mSearcher2.doc(hits2[i].doc);
        }

        // test the subSearcher() method:
        Query subSearcherQuery = parser.parse("id:doc1");
        hits2 = mSearcher2.search(subSearcherQuery, null, 1000).scoreDocs;
        assertEquals(2, hits2.length);
        assertEquals(0, mSearcher2.subSearcher(hits2[0].doc));   // hit from searchers2[0]
        assertEquals(1, mSearcher2.subSearcher(hits2[1].doc));   // hit from searchers2[1]
        subSearcherQuery = parser.parse("id:doc2");
        hits2 = mSearcher2.search(subSearcherQuery, null, 1000).scoreDocs;
        assertEquals(1, hits2.length);
        assertEquals(1, mSearcher2.subSearcher(hits2[0].doc));   // hit from searchers2[1]
        mSearcher2.close();

        //--------------------------------------------------------------------
        // scenario 3
        //--------------------------------------------------------------------

        // deleting the document just added, this will cause a different exception to take place
        Term term = new Term("id", "doc1");
        IndexReader readerB = IndexReader.open(indexStoreB, false);
        readerB.deleteDocuments(term);
        readerB.close();

        // optimizing the index with the writer
        writerB = new IndexWriter(indexStoreB, new StandardAnalyzer(org.apache.lucene.util.Version.LUCENE_CURRENT), false, IndexWriter.MaxFieldLength.LIMITED);
        writerB.optimize();
        writerB.close();

        // building the searchables
        Searcher[] searchers3 = new Searcher[2];
View Full Code Here

  @Override
  protected void setUp() throws Exception {
    super.setUp();
    similarityOne = new SimilarityOne();
    anlzr = new StandardAnalyzer(org.apache.lucene.util.Version.LUCENE_CURRENT);
  }
View Full Code Here

    // two documents:
    Directory dir1 = getDir1();

    // one document only:
    Directory dir2 = new MockRAMDirectory();
    IndexWriter w2 = new IndexWriter(dir2, new StandardAnalyzer(org.apache.lucene.util.Version.LUCENE_CURRENT), true, IndexWriter.MaxFieldLength.LIMITED);
    Document d3 = new Document();
    d3.add(new Field("f3", "v1", Field.Store.YES, Field.Index.ANALYZED));
    w2.addDocument(d3);
    w2.close();
   
View Full Code Here

  public void testIsOptimized() throws IOException {
    Directory dir1 = getDir1();
    Directory dir2 = getDir2();
   
    // add another document to ensure that the indexes are not optimized
    IndexWriter modifier = new IndexWriter(dir1, new StandardAnalyzer(org.apache.lucene.util.Version.LUCENE_CURRENT), IndexWriter.MaxFieldLength.LIMITED);
    Document d = new Document();
    d.add(new Field("f1", "v1", Field.Store.YES, Field.Index.ANALYZED));
    modifier.addDocument(d);
    modifier.close();
   
    modifier = new IndexWriter(dir2, new StandardAnalyzer(org.apache.lucene.util.Version.LUCENE_CURRENT), IndexWriter.MaxFieldLength.LIMITED);
    d = new Document();
    d.add(new Field("f2", "v2", Field.Store.YES, Field.Index.ANALYZED));
    modifier.addDocument(d);
    modifier.close();

   
    ParallelReader pr = new ParallelReader();
    pr.add(IndexReader.open(dir1, false));
    pr.add(IndexReader.open(dir2, false));
    assertFalse(pr.isOptimized());
    pr.close();
   
    modifier = new IndexWriter(dir1, new StandardAnalyzer(org.apache.lucene.util.Version.LUCENE_CURRENT), IndexWriter.MaxFieldLength.LIMITED);
    modifier.optimize();
    modifier.close();
   
    pr = new ParallelReader();
    pr.add(IndexReader.open(dir1, false));
    pr.add(IndexReader.open(dir2, false));
    // just one of the two indexes are optimized
    assertFalse(pr.isOptimized());
    pr.close();

   
    modifier = new IndexWriter(dir2, new StandardAnalyzer(org.apache.lucene.util.Version.LUCENE_CURRENT), IndexWriter.MaxFieldLength.LIMITED);
    modifier.optimize();
    modifier.close();
   
    pr = new ParallelReader();
    pr.add(IndexReader.open(dir1, false));
View Full Code Here

  }

  // Fields 1-4 indexed together:
  private Searcher single() throws IOException {
    Directory dir = new MockRAMDirectory();
    IndexWriter w = new IndexWriter(dir, new StandardAnalyzer(org.apache.lucene.util.Version.LUCENE_CURRENT), true, IndexWriter.MaxFieldLength.LIMITED);
    Document d1 = new Document();
    d1.add(new Field("f1", "v1", Field.Store.YES, Field.Index.ANALYZED));
    d1.add(new Field("f2", "v1", Field.Store.YES, Field.Index.ANALYZED));
    d1.add(new Field("f3", "v1", Field.Store.YES, Field.Index.ANALYZED));
    d1.add(new Field("f4", "v1", Field.Store.YES, Field.Index.ANALYZED));
View Full Code Here

TOP

Related Classes of org.apache.lucene.analysis.standard.StandardAnalyzer

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.