Examples of org.apache.lucene.analysis.MockTokenFilter

org.apache.lucene.analysis.MockTokenFilter
A tokenfilter for testing that removes terms accepted by a DFA.
- Union a list of singletons to act like a stopfilter.
- Use the complement to act like a keepwordfilter
- Use a regex like .{12,} to act like a lengthfilter

    Directory dir = newDirectory();
    Analyzer a = new Analyzer() {
      @Override
      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
        Tokenizer tokenizer = new MockTokenizer(reader);
        TokenStream stream = new MockTokenFilter(tokenizer, MockTokenFilter.ENGLISH_STOPSET);
        return new TokenStreamComponents(tokenizer, stream);
      }
    };
    RandomIndexWriter iw = new RandomIndexWriter(random(), dir, a);
    Document doc = new Document();

View Full Code Here

    final Automaton secondSet = BasicAutomata.makeString("foobar");
    Analyzer a = new Analyzer() {
      @Override
      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
        Tokenizer tokenizer = new MockTokenizer(reader);
        TokenStream stream = new MockTokenFilter(tokenizer, MockTokenFilter.ENGLISH_STOPSET);
        stream = new MockTokenFilter(stream, new CharacterRunAutomaton(secondSet));
        return new TokenStreamComponents(tokenizer, stream);
      }
    };
    RandomIndexWriter iw = new RandomIndexWriter(random(), dir, a);
    Document doc = new Document();

View Full Code Here

    Analyzer a = new Analyzer() {
        @Override
        public TokenStreamComponents createComponents(String fieldName, Reader reader) {
          MockTokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, true, 100);
          tokenizer.setEnableChecks(true);
          MockTokenFilter filt = new MockTokenFilter(tokenizer, MockTokenFilter.EMPTY_STOPSET);
          return new TokenStreamComponents(tokenizer, filt);
        }
      };
    IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, a);
    iwc.setCodec(TestUtil.alwaysPostingsFormat(new IDVersionPostingsFormat()));

View Full Code Here

TOP

Related Classes of org.apache.lucene.analysis.MockTokenFilter

org.apache.lucene.codecs.idversion.TestIDVersionPostingsFormat

org.apache.lucene.index.TestIndexWriter

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.