Package org.apache.lucene.analysis

Examples of org.apache.lucene.analysis.MockTokenizer

44
45
46
47
48
49
50
51
  }

  public void testIgnoreCase() throws IOException {
    StringReader reader = new StringReader("Now is The Time");
    Set<Object> stopWords = new HashSet<Object>(Arrays.asList( "is", "the", "Time" ));
    TokenStream stream = new StopFilter(TEST_VERSION_CURRENT, new MockTokenizer(reader, MockTokenizer.WHITESPACE, false), stopWords, true);
    assertTokenStreamContents(stream, new String[] { "Now" });
  }
View Full Code Here
52
53
54
55
56
57
58
59
  public void testStopFilt() throws IOException {
    StringReader reader = new StringReader("Now is The Time");
    String[] stopWords = new String[] { "is", "the", "Time" };
    Set<Object> stopSet = StopFilter.makeStopSet(TEST_VERSION_CURRENT, stopWords);
    TokenStream stream = new StopFilter(TEST_VERSION_CURRENT, new MockTokenizer(reader, MockTokenizer.WHITESPACE, false), stopSet);
    assertTokenStreamContents(stream, new String[] { "Now", "The" });
  }
View Full Code Here
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
    String stopWords[] = a.toArray(new String[0]);
    for (int i=0; i<a.size(); i++) log("Stop: "+stopWords[i]);
    Set<Object> stopSet = StopFilter.makeStopSet(TEST_VERSION_CURRENT, stopWords);
    // with increments
    StringReader reader = new StringReader(sb.toString());
    StopFilter stpf = new StopFilter(Version.LUCENE_24, new MockTokenizer(reader, MockTokenizer.WHITESPACE, false), stopSet);
    doTestStopPositons(stpf,true);
    // without increments
    reader = new StringReader(sb.toString());
    stpf = new StopFilter(TEST_VERSION_CURRENT, new MockTokenizer(reader, MockTokenizer.WHITESPACE, false), stopSet);
    doTestStopPositons(stpf,false);
    // with increments, concatenating two stop filters
    ArrayList<String> a0 = new ArrayList<String>();
    ArrayList<String> a1 = new ArrayList<String>();
    for (int i=0; i<a.size(); i++) {
      if (i%2==0) {
        a0.add(a.get(i));
      } else {
        a1.add(a.get(i));
      }
    }
    String stopWords0[] =  a0.toArray(new String[0]);
    for (int i=0; i<a0.size(); i++) log("Stop0: "+stopWords0[i]);
    String stopWords1[] =  a1.toArray(new String[0]);
    for (int i=0; i<a1.size(); i++) log("Stop1: "+stopWords1[i]);
    Set<Object> stopSet0 = StopFilter.makeStopSet(TEST_VERSION_CURRENT, stopWords0);
    Set<Object> stopSet1 = StopFilter.makeStopSet(TEST_VERSION_CURRENT, stopWords1);
    reader = new StringReader(sb.toString());
    StopFilter stpf0 = new StopFilter(TEST_VERSION_CURRENT, new MockTokenizer(reader, MockTokenizer.WHITESPACE, false), stopSet0); // first part of the set
    stpf0.setEnablePositionIncrements(true);
    StopFilter stpf01 = new StopFilter(TEST_VERSION_CURRENT, stpf0, stopSet1); // two stop filters concatenated!
    doTestStopPositons(stpf01,true);
  }
View Full Code Here
328
329
330
331
332
333
334
335
336
337
338
339
    w.addDocument(doc);

    Analyzer analyzer = new Analyzer() {
      @Override
      public TokenStream tokenStream(String fieldName, Reader reader) {
        MockTokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
        tokenizer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases.
        return new CrashingFilter(fieldName, tokenizer);
      }
    };

    Document crashDoc = new Document();
View Full Code Here
399
400
401
402
403
404
405
406
407
408
409
410
    Directory dir = newDirectory();
    IndexWriterConfig conf = newIndexWriterConfig( TEST_VERSION_CURRENT, new Analyzer() {

      @Override
      public TokenStream tokenStream(String fieldName, Reader reader) {
        MockTokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
        tokenizer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases.
        return new TokenFilter(tokenizer) {
          private int count = 0;

          @Override
          public boolean incrementToken() throws IOException {
View Full Code Here
529
530
531
532
533
534
535
536
537
538
539
540
  public void testDocumentsWriterExceptions() throws IOException {
    Analyzer analyzer = new Analyzer() {
      @Override
      public TokenStream tokenStream(String fieldName, Reader reader) {
        MockTokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
        tokenizer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases.
        return new CrashingFilter(fieldName, tokenizer);
      }
    };

    for(int i=0;i<2;i++) {
View Full Code Here
631
632
633
634
635
636
637
638
639
640
641
642
  public void testDocumentsWriterExceptionThreads() throws Exception {
    Analyzer analyzer = new Analyzer() {
      @Override
      public TokenStream tokenStream(String fieldName, Reader reader) {
        MockTokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
        tokenizer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases.
        return new CrashingFilter(fieldName, tokenizer);
      }
    };

    final int NUM_THREAD = 3;
View Full Code Here
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
      doc.add(newField("id", docCount+"", Field.Index.NOT_ANALYZED));
      doc.add(newField("content", "silly content " + docCount, Field.Index.ANALYZED));
      if (docCount == 4) {
        Field f = newField("crash", "", Field.Index.ANALYZED);
        doc.add(f);
        MockTokenizer tokenizer = new MockTokenizer(new StringReader("crash me on the 4th token"), MockTokenizer.WHITESPACE, false);
        tokenizer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases.
        f.setTokenStream(new CrashingFilter("crash", tokenizer));
      }
    }
    try {
      w.addDocuments(docs);
View Full Code Here
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
      doc.add(newField("id", docCount+"", Field.Index.NOT_ANALYZED));
      doc.add(newField("content", "silly content " + docCount, Field.Index.ANALYZED));
      if (docCount == crashAt) {
        Field f = newField("crash", "", Field.Index.ANALYZED);
        doc.add(f);
        MockTokenizer tokenizer = new MockTokenizer(new StringReader("crash me on the 4th token"), MockTokenizer.WHITESPACE, false);
        tokenizer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases.
        f.setTokenStream(new CrashingFilter("crash", tokenizer));
      }
    }

    try {
View Full Code Here
51
52
53
54
55
56
57
58
59
  }
 
  public void testWithKeywordAttribute() throws IOException {
    CharArraySet set = new CharArraySet(TEST_VERSION_CURRENT, 1, true);
    set.add("yourselves");
    Tokenizer tokenizer = new MockTokenizer(new StringReader("yourselves yours"), MockTokenizer.WHITESPACE, false);
    TokenStream filter = new PorterStemFilter(new KeywordMarkerFilter(tokenizer, set));  
    assertTokenStreamContents(filter, new String[] {"yourselves", "your"});
  }
View Full Code Here
TOP

Related Classes of org.apache.lucene.analysis.MockTokenizer

  • org.apache.lucene.analysis.bg.TestBulgarianStemmer
  • org.apache.lucene.analysis.charfilter.TestHTMLStripCharFilterFactory
  • org.apache.lucene.analysis.charfilter.TestMappingCharFilter
  • org.apache.lucene.analysis.cjk.TestCJKAnalyzer
  • org.apache.lucene.analysis.cn.smart.TestSmartChineseAnalyzer
  • org.apache.lucene.analysis.compound.TestCompoundWordTokenFilter
  • org.apache.lucene.analysis.compound.TestHyphenationCompoundWordTokenFilterFactory
  • org.apache.lucene.analysis.core.TestStopFilter
  • org.apache.lucene.analysis.de.TestGermanStemFilter
  • org.apache.lucene.analysis.en.TestPorterStemFilter
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.