Package org.apache.lucene.analysis

Examples of org.apache.lucene.analysis.MockAnalyzer


    // Default separator is INFORMATION SEPARATOR TWO
    // (0x1e), so no input token is allowed to contain it
    Iterable<Input> keys = shuffle(
        new Input("foo bar baz", 50)
    );
    FreeTextSuggester sug = new FreeTextSuggester(new MockAnalyzer(random()));
    sug.build(new InputArrayIterator(keys));

    try {
      sug.lookup("foo\u001eb", 10);
      fail("did not hit expected exception");
View Full Code Here


  @Ignore
  public void testWiki() throws Exception {
    final LineFileDocs lfd = new LineFileDocs(null, "/lucenedata/enwiki/enwiki-20120502-lines-1k.txt", false);
    // Skip header:
    lfd.nextDoc();
    FreeTextSuggester sug = new FreeTextSuggester(new MockAnalyzer(random()));
    sug.build(new InputIterator() {

        private int count;

        @Override
View Full Code Here

  public void testUnigrams() throws Exception {
    Iterable<Input> keys = shuffle(
        new Input("foo bar baz blah boo foo bar foo bee", 50)
    );

    Analyzer a = new MockAnalyzer(random());
    FreeTextSuggester sug = new FreeTextSuggester(a, a, 1, (byte) 0x20);
    sug.build(new InputArrayIterator(keys));
    // Sorts first by count, descending, second by term, ascending
    assertEquals("bar/0.22 baz/0.11 bee/0.11 blah/0.11 boo/0.11",
                 toString(sug.lookup("b", 10)));
View Full Code Here

  // Make sure the last token is not duplicated
  public void testNoDupsAcrossGrams() throws Exception {
    Iterable<Input> keys = shuffle(
        new Input("foo bar bar bar bar", 50)
    );
    Analyzer a = new MockAnalyzer(random());
    FreeTextSuggester sug = new FreeTextSuggester(a, a, 2, (byte) 0x20);
    sug.build(new InputArrayIterator(keys));
    assertEquals("foo bar/1.00",
                 toString(sug.lookup("foo b", 10)));
  }
View Full Code Here

  // Lookup of just empty string produces unicode only matches:
  public void testEmptyString() throws Exception {
    Iterable<Input> keys = shuffle(
        new Input("foo bar bar bar bar", 50)
    );
    Analyzer a = new MockAnalyzer(random());
    FreeTextSuggester sug = new FreeTextSuggester(a, a, 2, (byte) 0x20);
    sug.build(new InputArrayIterator(keys));
    try {
      sug.lookup("", 10);
      fail("did not hit exception");
View Full Code Here

        terms[seen.size()] = token;
        seen.add(token);
      }
    }

    Analyzer a = new MockAnalyzer(random());

    int numDocs = atLeast(10);
    long totTokens = 0;
    final String[][] docs = new String[numDocs][];
    for(int i=0;i<numDocs;i++) {
View Full Code Here

        ((MockDirectoryWrapper) directory).setAssertNoUnrefencedFilesOnClose(false);
      }

      IndexWriter writer = new IndexWriter(
          directory,
          newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())).
              setOpenMode(OpenMode.CREATE).
              setMaxBufferedDocs(-1).
              setMergePolicy(newLogMergePolicy(10))
      );

      SegmentCommitInfo si1 = indexDoc(writer, "test.txt");
      printSegment(out, si1);

      SegmentCommitInfo si2 = indexDoc(writer, "test2.txt");
      printSegment(out, si2);
      writer.close();

      SegmentCommitInfo siMerge = merge(directory, si1, si2, "_merge", false);
      printSegment(out, siMerge);

      SegmentCommitInfo siMerge2 = merge(directory, si1, si2, "_merge2", false);
      printSegment(out, siMerge2);

      SegmentCommitInfo siMerge3 = merge(directory, siMerge, siMerge2, "_merge3", false);
      printSegment(out, siMerge3);
     
      directory.close();
      out.close();
      sw.close();

      String multiFileOutput = sw.getBuffer().toString();
      //System.out.println(multiFileOutput);

      sw = new StringWriter();
      out = new PrintWriter(sw, true);

      directory = newFSDirectory(indexDir, null);

      if (directory instanceof MockDirectoryWrapper) {
        // We create unreferenced files (we don't even write
        // a segments file):
        ((MockDirectoryWrapper) directory).setAssertNoUnrefencedFilesOnClose(false);
      }

      writer = new IndexWriter(
          directory,
          newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())).
              setOpenMode(OpenMode.CREATE).
              setMaxBufferedDocs(-1).
              setMergePolicy(newLogMergePolicy(10))
      );
View Full Code Here

public class TestTieredMergePolicy extends LuceneTestCase {

  public void testForceMergeDeletes() throws Exception {
    Directory dir = newDirectory();
    IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
    TieredMergePolicy tmp = newTieredMergePolicy();
    conf.setMergePolicy(tmp);
    conf.setMaxBufferedDocs(4);
    tmp.setMaxMergeAtOnce(100);
    tmp.setSegmentsPerTier(100);
View Full Code Here

    for(int iter=0;iter<num;iter++) {
      if (VERBOSE) {
        System.out.println("TEST: iter=" + iter);
      }
      Directory dir = newDirectory();
      IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
      conf.setMergeScheduler(new SerialMergeScheduler());
      TieredMergePolicy tmp = newTieredMergePolicy();
      conf.setMergePolicy(tmp);
      conf.setMaxBufferedDocs(2);
      tmp.setMaxMergeAtOnce(3);
View Full Code Here

    }
  }

  public void testForceMergeDeletesMaxSegSize() throws Exception {
    final Directory dir = newDirectory();
    final IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
    final TieredMergePolicy tmp = new TieredMergePolicy();
    tmp.setMaxMergedSegmentMB(0.01);
    tmp.setForceMergeDeletesPctAllowed(0.0);
    conf.setMergePolicy(tmp);
View Full Code Here

TOP

Related Classes of org.apache.lucene.analysis.MockAnalyzer

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.