Package org.apache.lucene.analysis

Examples of org.apache.lucene.analysis.MockAnalyzer


    final double SECONDS = 2.0;

    Directory dir = newDirectory();
    ExpirationTimeDeletionPolicy policy = new ExpirationTimeDeletionPolicy(dir, SECONDS);
    IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT,
        new MockAnalyzer(random))
        .setIndexDeletionPolicy(policy);
    MergePolicy mp = conf.getMergePolicy();
    if (mp instanceof LogMergePolicy) {
      setUseCompoundFile(mp, true);
    }
    IndexWriter writer = new IndexWriter(dir, conf);
    writer.close();

    final int ITER = 9;

    long lastDeleteTime = 0;
    for(int i=0;i<ITER;i++) {
      // Record last time when writer performed deletes of
      // past commits
      lastDeleteTime = System.currentTimeMillis();
      conf = newIndexWriterConfig(TEST_VERSION_CURRENT,
          new MockAnalyzer(random)).setOpenMode(
          OpenMode.APPEND).setIndexDeletionPolicy(policy);
      mp = conf.getMergePolicy();
      if (mp instanceof LogMergePolicy) {
        setUseCompoundFile(mp, true);
      }
View Full Code Here


      Directory dir = newDirectory();
      policy.dir = dir;

      IndexWriterConfig conf = newIndexWriterConfig(
          TEST_VERSION_CURRENT, new MockAnalyzer(random))
          .setIndexDeletionPolicy(policy).setMaxBufferedDocs(10)
          .setMergeScheduler(new SerialMergeScheduler());
      MergePolicy mp = conf.getMergePolicy();
      if (mp instanceof LogMergePolicy) {
        setUseCompoundFile(mp, useCompoundFile);
      }
      IndexWriter writer = new IndexWriter(dir, conf);
      for(int i=0;i<107;i++) {
        addDoc(writer);
      }
      writer.close();

      final boolean isOptimized;
      {
        IndexReader r = IndexReader.open(dir);
        isOptimized = r.isOptimized();
        r.close();
      }
      if (!isOptimized) {
        conf = newIndexWriterConfig(TEST_VERSION_CURRENT,
                                    new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setOpenMode(
                                                                                              OpenMode.APPEND).setIndexDeletionPolicy(policy);
        mp = conf.getMergePolicy();
        if (mp instanceof LogMergePolicy) {
          setUseCompoundFile(mp, true);
        }
        if (VERBOSE) {
          System.out.println("TEST: open writer for optimize");
        }
        writer = new IndexWriter(dir, conf);
        writer.setInfoStream(VERBOSE ? System.out : null);
        writer.optimize();
        writer.close();
      }
      assertEquals(isOptimized ? 0:1, policy.numOnInit);

      // If we are not auto committing then there should
      // be exactly 2 commits (one per close above):
      assertEquals(1 + (isOptimized ? 0:1), policy.numOnCommit);

      // Test listCommits
      Collection<IndexCommit> commits = IndexReader.listCommits(dir);
      // 2 from closing writer
      assertEquals(1 + (isOptimized ? 0:1), commits.size());

      // Make sure we can open a reader on each commit:
      for (final IndexCommit commit : commits) {
        IndexReader r = IndexReader.open(commit, null, false);
        r.close();
      }

      // Simplistic check: just verify all segments_N's still
      // exist, and, I can open a reader on each:
      dir.deleteFile(IndexFileNames.SEGMENTS_GEN);
      long gen = SegmentInfos.getCurrentSegmentGeneration(dir);
      while(gen > 0) {
        IndexReader reader = IndexReader.open(dir, true);
        reader.close();
        dir.deleteFile(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
        gen--;

        if (gen > 0) {
          // Now that we've removed a commit point, which
          // should have orphan'd at least one index file.
          // Open & close a writer and assert that it
          // actually removed something:
          int preCount = dir.listAll().length;
          writer = new IndexWriter(dir, newIndexWriterConfig(
              TEST_VERSION_CURRENT,
              new MockAnalyzer(random)).setOpenMode(
              OpenMode.APPEND).setIndexDeletionPolicy(policy));
          writer.close();
          int postCount = dir.listAll().length;
          assertTrue(postCount < preCount);
        }
View Full Code Here

    if (VERBOSE) {
      System.out.println("TEST: setUp");
    }
    // prepare a small index with just a few documents. 
    dir = newDirectory();
    anlzr = new MockAnalyzer(random);
    IndexWriterConfig iwc = newIndexWriterConfig( TEST_VERSION_CURRENT, anlzr).setMergePolicy(newLogMergePolicy());
    RandomIndexWriter iw = new RandomIndexWriter(random, dir, iwc);
    if (doMultiSegment) {
      iw.w.setMaxBufferedDocs(_TestUtil.nextInt(random, 2, 7));
    }
View Full Code Here

    final LineFileDocs docs = new LineFileDocs(random);
    final File tempDir = _TestUtil.getTempDir("nrtopenfiles");
    final MockDirectoryWrapper dir = newFSDirectory(tempDir);
    dir.setCheckIndexOnClose(false); // don't double-checkIndex, we do it ourselves.
    final IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random));
    conf.setMergedSegmentWarmer(new IndexWriter.IndexReaderWarmer() {
      @Override
      public void warm(IndexReader reader) throws IOException {
        if (VERBOSE) {
          System.out.println("TEST: now warm merged reader=" + reader);
View Full Code Here

    Directory dir = newDirectory();
    policy.dir = dir;

    IndexWriter writer = new IndexWriter(
        dir,
        newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
            setIndexDeletionPolicy(policy).
            setMaxBufferedDocs(2).
            setMergePolicy(newLogMergePolicy(10))
    );
    for(int i=0;i<10;i++) {
      addDoc(writer);
      if ((1+i)%2 == 0)
        writer.commit();
    }
    writer.close();

    Collection<IndexCommit> commits = IndexReader.listCommits(dir);
    assertEquals(5, commits.size());
    IndexCommit lastCommit = null;
    for (final IndexCommit commit : commits) {
      if (lastCommit == null || commit.getGeneration() > lastCommit.getGeneration())
        lastCommit = commit;
    }
    assertTrue(lastCommit != null);

    // Now add 1 doc and optimize
    writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT,
        new MockAnalyzer(random)).setIndexDeletionPolicy(policy));
    addDoc(writer);
    assertEquals(11, writer.numDocs());
    writer.optimize();
    writer.close();

    assertEquals(6, IndexReader.listCommits(dir).size());

    // Now open writer on the commit just before optimize:
    writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))
        .setIndexDeletionPolicy(policy).setIndexCommit(lastCommit));
    assertEquals(10, writer.numDocs());

    // Should undo our rollback:
    writer.rollback();

    IndexReader r = IndexReader.open(dir, true);
    // Still optimized, still 11 docs
    assertTrue(r.isOptimized());
    assertEquals(11, r.numDocs());
    r.close();

    writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))
        .setIndexDeletionPolicy(policy).setIndexCommit(lastCommit));
    assertEquals(10, writer.numDocs());
    // Commits the rollback:
    writer.close();

    // Now 8 because we made another commit
    assertEquals(7, IndexReader.listCommits(dir).size());
   
    r = IndexReader.open(dir, true);
    // Not optimized because we rolled it back, and now only
    // 10 docs
    assertTrue(!r.isOptimized());
    assertEquals(10, r.numDocs());
    r.close();

    // Reoptimize
    writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setIndexDeletionPolicy(policy));
    writer.optimize();
    writer.close();

    r = IndexReader.open(dir, true);
    assertTrue(r.isOptimized());
    assertEquals(10, r.numDocs());
    r.close();

    // Now open writer on the commit just before optimize,
    // but this time keeping only the last commit:
    writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setIndexCommit(lastCommit));
    assertEquals(10, writer.numDocs());
   
    // Reader still sees optimized index, because writer
    // opened on the prior commit has not yet committed:
    r = IndexReader.open(dir, true);
View Full Code Here

      KeepNoneOnInitDeletionPolicy policy = new KeepNoneOnInitDeletionPolicy();

      Directory dir = newDirectory();

      IndexWriterConfig conf = newIndexWriterConfig(
          TEST_VERSION_CURRENT, new MockAnalyzer(random))
          .setOpenMode(OpenMode.CREATE).setIndexDeletionPolicy(policy)
          .setMaxBufferedDocs(10);
      MergePolicy mp = conf.getMergePolicy();
      if (mp instanceof LogMergePolicy) {
        setUseCompoundFile(mp, useCompoundFile);
      }
      IndexWriter writer = new IndexWriter(dir, conf);
      for(int i=0;i<107;i++) {
        addDoc(writer);
      }
      writer.close();

      conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))
          .setOpenMode(OpenMode.APPEND).setIndexDeletionPolicy(policy);
      mp = conf.getMergePolicy();
      if (mp instanceof LogMergePolicy) {
        setUseCompoundFile(mp, true);
      }
View Full Code Here

      KeepLastNDeletionPolicy policy = new KeepLastNDeletionPolicy(N);

      for(int j=0;j<N+1;j++) {
        IndexWriterConfig conf = newIndexWriterConfig(
            TEST_VERSION_CURRENT, new MockAnalyzer(random))
            .setOpenMode(OpenMode.CREATE).setIndexDeletionPolicy(policy)
            .setMaxBufferedDocs(10);
        MergePolicy mp = conf.getMergePolicy();
        if (mp instanceof LogMergePolicy) {
          setUseCompoundFile(mp, useCompoundFile);
View Full Code Here

      KeepLastNDeletionPolicy policy = new KeepLastNDeletionPolicy(N);

      Directory dir = newDirectory();
      IndexWriterConfig conf = newIndexWriterConfig(
          TEST_VERSION_CURRENT, new MockAnalyzer(random))
        .setOpenMode(OpenMode.CREATE).setIndexDeletionPolicy(policy).setMergePolicy(newLogMergePolicy());
      MergePolicy mp = conf.getMergePolicy();
      if (mp instanceof LogMergePolicy) {
        setUseCompoundFile(mp, useCompoundFile);
      }
      IndexWriter writer = new IndexWriter(dir, conf);
      writer.close();
      Term searchTerm = new Term("content", "aaa");       
      Query query = new TermQuery(searchTerm);

      for(int i=0;i<N+1;i++) {
        if (VERBOSE) {
          System.out.println("\nTEST: cycle i=" + i);
        }
        conf = newIndexWriterConfig(
            TEST_VERSION_CURRENT, new MockAnalyzer(random))
          .setOpenMode(OpenMode.APPEND).setIndexDeletionPolicy(policy).setMergePolicy(newLogMergePolicy());
        mp = conf.getMergePolicy();
        if (mp instanceof LogMergePolicy) {
          setUseCompoundFile(mp, useCompoundFile);
        }
        writer = new IndexWriter(dir, conf);
        writer.setInfoStream(VERBOSE ? System.out : null);
        for(int j=0;j<17;j++) {
          addDoc(writer);
        }
        // this is a commit
        if (VERBOSE) {
          System.out.println("TEST: close writer");
        }
        writer.close();
        IndexReader reader = IndexReader.open(dir, policy, false);
        reader.deleteDocument(3*i+1);
        reader.setNorm(4*i+1, "content", 2.0F);
        IndexSearcher searcher = newSearcher(reader);
        ScoreDoc[] hits = searcher.search(query, null, 1000).scoreDocs;
        assertEquals(16*(1+i), hits.length);
        // this is a commit
        if (VERBOSE) {
          System.out.println("TEST: close reader numOnCommit=" + policy.numOnCommit);
        }
        reader.close();
        searcher.close();
      }
      conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))
          .setOpenMode(OpenMode.APPEND).setIndexDeletionPolicy(policy);
      mp = conf.getMergePolicy();
      if (mp instanceof LogMergePolicy) {
        setUseCompoundFile(mp, useCompoundFile);
      }
View Full Code Here

      KeepLastNDeletionPolicy policy = new KeepLastNDeletionPolicy(N);

      Directory dir = newDirectory();
      IndexWriterConfig conf = newIndexWriterConfig(
          TEST_VERSION_CURRENT, new MockAnalyzer(random))
          .setOpenMode(OpenMode.CREATE).setIndexDeletionPolicy(policy)
          .setMaxBufferedDocs(10);
      MergePolicy mp = conf.getMergePolicy();
      if (mp instanceof LogMergePolicy) {
        setUseCompoundFile(mp, useCompoundFile);
      }
      IndexWriter writer = new IndexWriter(dir, conf);
      writer.close();
      Term searchTerm = new Term("content", "aaa");       
      Query query = new TermQuery(searchTerm);

      for(int i=0;i<N+1;i++) {

        conf = newIndexWriterConfig(
            TEST_VERSION_CURRENT, new MockAnalyzer(random))
            .setOpenMode(OpenMode.APPEND).setIndexDeletionPolicy(policy)
            .setMaxBufferedDocs(10);
        mp = conf.getMergePolicy();
        if (mp instanceof LogMergePolicy) {
          setUseCompoundFile(mp, useCompoundFile);
        }
        writer = new IndexWriter(dir, conf);
        for(int j=0;j<17;j++) {
          addDoc(writer);
        }
        // this is a commit
        writer.close();
        IndexReader reader = IndexReader.open(dir, policy, false);
        reader.deleteDocument(3);
        reader.setNorm(5, "content", 2.0F);
        IndexSearcher searcher = newSearcher(reader);
        ScoreDoc[] hits = searcher.search(query, null, 1000).scoreDocs;
        assertEquals(16, hits.length);
        // this is a commit
        reader.close();
        searcher.close();

        writer = new IndexWriter(dir, newIndexWriterConfig(
            TEST_VERSION_CURRENT, new MockAnalyzer(random))
            .setOpenMode(OpenMode.CREATE).setIndexDeletionPolicy(policy));
        // This will not commit: there are no changes
        // pending because we opened for "create":
        writer.close();
      }
View Full Code Here

  @Override
  public void setUp() throws Exception {
    super.setUp();
    similarityOne = new SimilarityOne();
    anlzr = new MockAnalyzer(random);
  }
View Full Code Here

TOP

Related Classes of org.apache.lucene.analysis.MockAnalyzer

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.