Package org.apache.lucene.analysis.compound.hyphenation

Examples of org.apache.lucene.analysis.compound.hyphenation.HyphenationTree

66
67
68
69
70
71
72
73
74
75
76
  public void testHyphenationCompoundWordsDELongestMatch() throws Exception {
    CharArraySet dict = makeDictionary("basketball", "basket", "ball", "kurv");

    InputSource is = new InputSource(getClass().getResource("da_UTF8.xml").toExternalForm());
    HyphenationTree hyphenator = HyphenationCompoundWordTokenFilter
        .getHyphenationTree(is);

    // the word basket will not be added due to the longest match option
    HyphenationCompoundWordTokenFilter tf = new HyphenationCompoundWordTokenFilter(TEST_VERSION_CURRENT,
        new MockTokenizer(new StringReader("basketballkurv"), MockTokenizer.WHITESPACE, false),
View Full Code Here
88
89
90
91
92
93
94
95
96
97
98
   * With hyphenation-only, you can get a lot of nonsense tokens.
   * This can be controlled with the min/max subword size.
   */
  public void testHyphenationOnly() throws Exception {
    InputSource is = new InputSource(getClass().getResource("da_UTF8.xml").toExternalForm());
    HyphenationTree hyphenator = HyphenationCompoundWordTokenFilter
        .getHyphenationTree(is);
   
    HyphenationCompoundWordTokenFilter tf = new HyphenationCompoundWordTokenFilter(
        TEST_VERSION_CURRENT,
        new MockTokenizer(new StringReader("basketballkurv"), MockTokenizer.WHITESPACE, false),
View Full Code Here
352
353
354
355
356
357
358
359
360
361
362
      }
    };
    checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER);
   
    InputSource is = new InputSource(getClass().getResource("da_UTF8.xml").toExternalForm());
    final HyphenationTree hyphenator = HyphenationCompoundWordTokenFilter.getHyphenationTree(is);
    Analyzer b = new Analyzer() {

      @Override
      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
        Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
View Full Code Here
378
379
380
381
382
383
384
385
386
387
388
      }
    };
    checkOneTermReuse(a, "", "");
   
    InputSource is = new InputSource(getClass().getResource("da_UTF8.xml").toExternalForm());
    final HyphenationTree hyphenator = HyphenationCompoundWordTokenFilter.getHyphenationTree(is);
    Analyzer b = new Analyzer() {

      @Override
      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
        Tokenizer tokenizer = new KeywordTokenizer(reader);
View Full Code Here
158
159
160
161
162
163
164
165
166
   * @return An object representing the hyphenation patterns
   * @throws IOException If there is a low-level I/O error.
   */
  public static HyphenationTree getHyphenationTree(InputSource hyphenationSource)
      throws IOException {
    HyphenationTree tree = new HyphenationTree();
    tree.loadPatterns(hyphenationSource);
    return tree;
  }
View Full Code Here
63
64
65
66
67
68
69
70
71
72
73
    if (reader == null) {
      // we gracefully die if we have no reader
      return;
    }

    HyphenationTree hyphenator = HyphenationCompoundWordTokenFilter
        .getHyphenationTree(reader);

    HyphenationCompoundWordTokenFilter tf = new HyphenationCompoundWordTokenFilter(
        new WhitespaceTokenizer(new StringReader(
            "Rindfleischüberwachungsgesetz Drahtschere abba")), hyphenator,
View Full Code Here
89
90
91
92
93
94
95
96
97
98
99
    if (reader == null) {
      // we gracefully die if we have no reader
      return;
    }

    HyphenationTree hyphenator = HyphenationCompoundWordTokenFilter
        .getHyphenationTree(reader);

    HyphenationCompoundWordTokenFilter tf = new HyphenationCompoundWordTokenFilter(
        new WhitespaceTokenizer(new StringReader(
            "Rindfleischüberwachungsgesetz")), hyphenator, dict,
View Full Code Here
137
138
139
140
141
142
143
144
145
146
147
   * @return An object representing the hyphenation patterns
   * @throws Exception
   */
  public static HyphenationTree getHyphenationTree(Reader hyphenationReader)
      throws Exception {
    HyphenationTree tree = new HyphenationTree();

    tree.loadPatterns(new InputSource(hyphenationReader));

    return tree;
  }
View Full Code Here
140
141
142
143
144
145
146
147
148
149
150
   * @return An object representing the hyphenation patterns
   * @throws Exception
   */
  public static HyphenationTree getHyphenationTree(Reader hyphenationReader)
      throws Exception {
    HyphenationTree tree = new HyphenationTree();

    tree.loadPatterns(new InputSource(hyphenationReader));

    return tree;
  }
View Full Code Here
64
65
66
67
68
69
70
71
72
73
74
    if (reader == null) {
      // we gracefully die if we have no reader
      return;
    }

    HyphenationTree hyphenator = HyphenationCompoundWordTokenFilter
        .getHyphenationTree(reader);

    HyphenationCompoundWordTokenFilter tf = new HyphenationCompoundWordTokenFilter(
        new WhitespaceTokenizer(new StringReader(
            "Rindfleischüberwachungsgesetz Drahtschere abba")), hyphenator,
View Full Code Here
TOP

Related Classes of org.apache.lucene.analysis.compound.hyphenation.HyphenationTree

  • org.apache.lucene.analysis.compound.HyphenationCompoundWordTokenFilter
  • org.apache.lucene.analysis.compound.TestCompoundWordTokenFilter
  • org.xml.sax.InputSource
  • java.util.HashMap
  • java.util.ArrayList
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.