Package org.apache.lucene.analysis

Examples of org.apache.lucene.analysis.TokenFilter

40
41
42
43
44
45
46
47
48
49
50
    String test = "Plop, juste pour voir l'embrouille avec O'brian. M'enfin.";
    Tokenizer tokenizer = new StandardTokenizer(new StringReader(test));
    Set articles = new HashSet();
    articles.add("l");
    articles.add("M");
    TokenFilter filter = new ElisionFilter(tokenizer, articles);
    List tas = filtre(filter);
    assertEquals("embrouille", tas.get(4));
    assertEquals("O'brian", tas.get(6));
    assertEquals("enfin", tas.get(7));
  }
View Full Code Here
67
68
69
70
71
72
73
74
75
76
77
    if (log == null)
      throw new IllegalArgumentException("logStream must not be null");

    return new Analyzer() {
      public TokenStream tokenStream(final String fieldName, Reader reader) {
        return new TokenFilter(child.tokenStream(fieldName, reader)) {
          private int position = -1;
         
          public Token next(final Token reusableToken) throws IOException {
            assert reusableToken != null;
            Token nextToken = input.next(reusableToken); // from filter super class
View Full Code Here
116
117
118
119
120
121
122
123
124
125
126
    if (maxTokens == Integer.MAX_VALUE)
      return child; // no need to wrap
 
    return new Analyzer() {
      public TokenStream tokenStream(String fieldName, Reader reader) {
        return new TokenFilter(child.tokenStream(fieldName, reader)) {
          private int todo = maxTokens;
         
          public Token next(final Token reusableToken) throws IOException {
            assert reusableToken != null;
            return --todo >= 0 ? input.next(reusableToken) : null;
View Full Code Here
236
237
238
239
240
241
242
243
244
245
246
      public TokenStream tokenStream(String fieldName, Reader reader) {
        final ArrayList tokens = (ArrayList) cache.get(fieldName);
        if (tokens == null) { // not yet cached
          final ArrayList tokens2 = new ArrayList();
          TokenStream tokenStream = new TokenFilter(child.tokenStream(fieldName, reader)) {

            public Token next(final Token reusableToken) throws IOException {
              assert reusableToken != null;
              Token nextToken = input.next(reusableToken); // from filter super class
              if (nextToken != null) tokens2.add(nextToken.clone());
View Full Code Here
141
142
143
144
145
146
147
148
149
150
151
  public void testTokenReuse() throws IOException {
    Analyzer analyzer = new Analyzer() {
      @Override
      public TokenStream tokenStream(String fieldName, Reader reader) {
        return new TokenFilter(new WhitespaceTokenizer(reader)) {
          boolean first=true;
          AttributeSource.State state;

          @Override
          public boolean incrementToken() throws IOException {
View Full Code Here
71
72
73
74
75
76
77
78
79
80
81
    if (log == null)
      throw new IllegalArgumentException("logStream must not be null");

    return new Analyzer() {
      public TokenStream tokenStream(final String fieldName, Reader reader) {
        return new TokenFilter(child.tokenStream(fieldName, reader)) {
          private int position = -1;
          private TermAttribute termAtt = (TermAttribute) addAttribute(TermAttribute.class);
          private PositionIncrementAttribute posIncrAtt = (PositionIncrementAttribute) addAttribute(PositionIncrementAttribute.class);
          private OffsetAttribute offsetAtt = (OffsetAttribute) addAttribute(OffsetAttribute.class);
          private TypeAttribute typeAtt = (TypeAttribute) addAttribute(TypeAttribute.class);
View Full Code Here
123
124
125
126
127
128
129
130
131
132
133
    if (maxTokens == Integer.MAX_VALUE)
      return child; // no need to wrap
 
    return new Analyzer() {
      public TokenStream tokenStream(String fieldName, Reader reader) {
        return new TokenFilter(child.tokenStream(fieldName, reader)) {
          private int todo = maxTokens;
         
          public boolean incrementToken() throws IOException {
            return --todo >= 0 ? input.incrementToken() : false;
          }
View Full Code Here
242
243
244
245
246
247
248
249
250
251
252
      public TokenStream tokenStream(String fieldName, Reader reader) {
        final ArrayList tokens = (ArrayList) cache.get(fieldName);
        if (tokens == null) { // not yet cached
          final ArrayList tokens2 = new ArrayList();
          TokenStream tokenStream = new TokenFilter(child.tokenStream(fieldName, reader)) {

            public boolean incrementToken() throws IOException {
              boolean hasNext = input.incrementToken();
              if (hasNext) tokens2.add(captureState());
              return hasNext;
View Full Code Here
137
138
139
140
141
142
143
144
145
146
147
  }

  public void testTokenReuse() throws IOException {
    Analyzer analyzer = new Analyzer() {
      public TokenStream tokenStream(String fieldName, Reader reader) {
        return new TokenFilter(new WhitespaceTokenizer(reader)) {
          boolean first=true;
          AttributeSource.State state;

          public boolean incrementToken() throws IOException {
            if (state != null) {
View Full Code Here
39
40
41
42
43
44
45
46
47
48
49
    String test = "Plop, juste pour voir l'embrouille avec O'brian. M'enfin.";
    Tokenizer tokenizer = new StandardTokenizer(new StringReader(test));
    Set articles = new HashSet();
    articles.add("l");
    articles.add("M");
    TokenFilter filter = new ElisionFilter(tokenizer, articles);
    List tas = filtre(filter);
    assertEquals("embrouille", tas.get(4));
    assertEquals("O'brian", tas.get(6));
    assertEquals("enfin", tas.get(7));
  }
View Full Code Here
TOP

Related Classes of org.apache.lucene.analysis.TokenFilter

  • com.enigmastation.extractors.impl.LuceneStemmingWordLister
  • gov.nysenate.openleg.lucene.OpenLegislationAnalyzer
  • org.apache.lucene.analysis.cjk.TestCJKAnalyzer
  • org.apache.lucene.analysis.cn.smart.TestSmartChineseAnalyzer
  • org.apache.lucene.analysis.compound.TestCompoundWordTokenFilter
  • org.apache.lucene.analysis.core.TestBugInSomething
  • org.apache.lucene.analysis.core.TestRandomChains$MockRandomAnalyzer
  • org.apache.lucene.analysis.core.TestStopFilter
  • org.apache.lucene.analysis.fr.TestElision
  • org.apache.lucene.analysis.hi.TestHindiNormalizer
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.