Package org.apache.lucene.analysis.standard

Examples of org.apache.lucene.analysis.standard.UAX29URLEmailTokenizer
552
553
554
555
556
557
558
559
560
561
562
  @Deprecated
  public void testVersion40() throws Exception {
    Analyzer a = new Analyzer() {
      @Override
      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
        Tokenizer tokenizer = new UAX29URLEmailTokenizer(Version.LUCENE_40, reader);
        return new TokenStreamComponents(tokenizer);
      }
    };
    // U+061C is a new combining mark in 6.3, found using "[[\p{WB:Format}\p{WB:Extend}]&[^\p{Age:6.2}]]"
    // on the online UnicodeSet utility: <http://unicode.org/cldr/utility/list-unicodeset.jsp>
View Full Code Here
49
50
51
52
53
54
55
56
57
    maxTokenLength = getInt("maxTokenLength",
                            StandardAnalyzer.DEFAULT_MAX_TOKEN_LENGTH);
  }

  public UAX29URLEmailTokenizer create(Reader input) {
    UAX29URLEmailTokenizer tokenizer = new UAX29URLEmailTokenizer(input);
    tokenizer.setMaxTokenLength(maxTokenLength);
    return tokenizer;
  }
View Full Code Here
172
173
174
175
176
177
178
179
180
181
182
            @Override public String name() {
                return "uax_url_email";
            }

            @Override public Tokenizer create(Reader reader) {
                return new UAX29URLEmailTokenizer(reader);
            }
        }));

        tokenizerFactories.put("path_hierarchy", new PreBuiltTokenizerFactoryFactory(new TokenizerFactory() {
            @Override public String name() {
View Full Code Here
41
42
43
44
45
46
47
48
49
        super(index, indexSettings, name, settings);
        maxTokenLength = settings.getAsInt("max_token_length", StandardAnalyzer.DEFAULT_MAX_TOKEN_LENGTH);
    }

    @Override public Tokenizer create(Reader reader) {
        UAX29URLEmailTokenizer tokenizer = new UAX29URLEmailTokenizer(reader);
        tokenizer.setMaxTokenLength(maxTokenLength);
        return tokenizer;
    }
View Full Code Here
38
39
40
41
42
43
44
45
    char whitespace[] = new char[4094];
    Arrays.fill(whitespace, ' ');
    sb.append(whitespace);
    sb.append("testing 1234");
    String input = sb.toString();
    UAX29URLEmailTokenizer tokenizer = new UAX29URLEmailTokenizer(TEST_VERSION_CURRENT, new StringReader(input));
    BaseTokenStreamTestCase.assertTokenStreamContents(tokenizer, new String[] { "testing", "1234" });
  }
View Full Code Here
428
429
430
431
432
433
434
435
436
437
438
    Analyzer a = new ReusableAnalyzerBase() {
      @Override
      protected TokenStreamComponents createComponents
        (String fieldName, Reader reader) {

        Tokenizer tokenizer = new UAX29URLEmailTokenizer(reader);
        return new TokenStreamComponents(tokenizer);
      }
    };
    checkOneTerm(a, "ざ", "さ"); // hiragana Bug
    checkOneTerm(a, "ザ", "ザ"); // katakana Works
View Full Code Here
48
49
50
51
52
53
54
55
56
                            StandardAnalyzer.DEFAULT_MAX_TOKEN_LENGTH);
  }

  @Override
  public UAX29URLEmailTokenizer create(Reader input) {
    UAX29URLEmailTokenizer tokenizer = new UAX29URLEmailTokenizer(luceneMatchVersion, input);
    tokenizer.setMaxTokenLength(maxTokenLength);
    return tokenizer;
  }
View Full Code Here
45
46
47
48
49
50
51
52
    char whitespace[] = new char[4094];
    Arrays.fill(whitespace, ' ');
    sb.append(whitespace);
    sb.append("testing 1234");
    String input = sb.toString();
    UAX29URLEmailTokenizer tokenizer = new UAX29URLEmailTokenizer(TEST_VERSION_CURRENT, newAttributeFactory(), new StringReader(input));
    BaseTokenStreamTestCase.assertTokenStreamContents(tokenizer, new String[] { "testing", "1234" });
  }
View Full Code Here
508
509
510
511
512
513
514
515
516
517
518
    Analyzer a = new Analyzer() {
      @Override
      protected TokenStreamComponents createComponents
        (String fieldName, Reader reader) {

        Tokenizer tokenizer = new UAX29URLEmailTokenizer(Version.LUCENE_3_1, reader);
        return new TokenStreamComponents(tokenizer);
      }
    };
    checkOneTerm(a, "ざ", "さ"); // hiragana Bug
    checkOneTerm(a, "ザ", "ザ"); // katakana Works
View Full Code Here
525
526
527
528
529
530
531
532
533
534
535
  @Deprecated
  public void testMailtoBackwards()  throws Exception {
    Analyzer a = new Analyzer() {
      @Override
      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
        Tokenizer tokenizer = new UAX29URLEmailTokenizer(Version.LUCENE_3_4, reader);
        return new TokenStreamComponents(tokenizer);
      }
    };
    assertAnalyzesTo(a, "mailto:test@example.org",
        new String[] { "mailto:test", "example.org" });
View Full Code Here
TOP

Related Classes of org.apache.lucene.analysis.standard.UAX29URLEmailTokenizer

  • org.apache.lucene.analysis.core.TestUAX29URLEmailTokenizer
  • org.apache.lucene.analysis.standard.std31.UAX29URLEmailTokenizerImpl31
  • org.apache.lucene.analysis.standard.std34.UAX29URLEmailTokenizerImpl34
  • org.apache.lucene.analysis.standard.std36.UAX29URLEmailTokenizerImpl36
  • org.apache.lucene.analysis.standard.std40.UAX29URLEmailTokenizerImpl40
  • org.apache.lucene.analysis.standard.UAX29URLEmailTokenizerFactory
  • org.apache.lucene.analysis.TestUAX29URLEmailTokenizer
  • org.apache.solr.analysis.UAX29URLEmailTokenizerFactory
  • org.elasticsearch.index.analysis.UAX29URLEmailTokenizerFactory
  • org.elasticsearch.indices.analysis.IndicesAnalysisService
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.