Package org.apache.lucene.analysis

Examples of org.apache.lucene.analysis.TokenStream$TokenWrapperAttributeFactory

43
44
45
46
47
48
49
50
51
52
   * @see org.apache.lucene.analysis.KeywordAnalyzer#tokenStream(java.lang.String,
   * java.io.Reader)
   */
  @Override
  public TokenStream tokenStream(String fieldName, Reader reader) {
    TokenStream result = new WhitespaceTokenizer(reader);
    result = new LowerCaseFilter(result);

    return result;
  }
View Full Code Here
57
58
59
60
61
62
63
64
65
66
67
   */
  @Override
  public TokenStream tokenStream(String fieldName, Reader reader) {

    /* Split at delim Char */
    TokenStream result = new CharTokenizer(reader) {

      @Override
      protected boolean isTokenChar(char c) {
        return c != fDelim;
      }
View Full Code Here
59
60
61
62
63
64
65
66
67
68
     * @see org.apache.lucene.analysis.KeywordAnalyzer#tokenStream(java.lang.String,
     * java.io.Reader)
     */
    @Override
    public TokenStream tokenStream(String fieldName, Reader reader) {
      TokenStream result = super.tokenStream(fieldName, reader);
      result = new LowerCaseFilter(result);

      return result;
    }
View Full Code Here
285
286
287
288
289
290
291
292
293
294
295
296
297
  private BooleanClause createAllNewsFieldsClause(ISearchCondition condition, boolean matchAllConditions) throws IOException {
    BooleanQuery allFieldsQuery = new BooleanQuery();
    String value = String.valueOf(condition.getValue());

    LowercaseWhitespaceAnalyzer analyzer = new LowercaseWhitespaceAnalyzer();
    TokenStream tokenStream = analyzer.tokenStream(String.valueOf(IEntity.ALL_FIELDS), new StringReader(value));
    Token token = null;
    while ((token = tokenStream.next()) != null) {
      String termText = token.termText();

      /* Contained in Title */
      WildcardQuery titleQuery = new WildcardQuery(new Term(String.valueOf(INews.TITLE), termText));
      allFieldsQuery.add(new BooleanClause(titleQuery, Occur.SHOULD));
View Full Code Here
457
458
459
460
461
462
463
464
465
466
467
468
469
        /* Fuzzy Query */
      case SIMILIAR_TO: {
        BooleanQuery similarityQuery = new BooleanQuery();

        LowercaseWhitespaceAnalyzer analyzer = new LowercaseWhitespaceAnalyzer();
        TokenStream tokenStream = analyzer.tokenStream(String.valueOf(IEntity.ALL_FIELDS), new StringReader(value));
        Token token = null;
        while ((token = tokenStream.next()) != null) {
          Term term = new Term(fieldname, token.termText());
          similarityQuery.add(new BooleanClause(new FuzzyQuery(term), Occur.MUST));
        }

        return similarityQuery;
View Full Code Here
621
622
623
624
625
626
627
628
629
630
631
632
633
        /* Fuzzy Query */
      case SIMILIAR_TO: {
        BooleanQuery similarityQuery = new BooleanQuery();

        LowercaseWhitespaceAnalyzer similarAnalyzer = new LowercaseWhitespaceAnalyzer();
        TokenStream tokenStream = similarAnalyzer.tokenStream(String.valueOf(IEntity.ALL_FIELDS), new StringReader(value));
        Token token = null;
        while ((token = tokenStream.next()) != null) {
          String termText = new String(token.termBuffer(), 0, token.termLength());
          Term term = new Term(fieldname, termText);
          similarityQuery.add(new BooleanClause(new FuzzyQuery(term), Occur.MUST));
        }

View Full Code Here
112
113
114
115
116
117
118
119
120
121
     * org.apache.lucene.analysis.KeywordAnalyzer#tokenStream(java.lang.String,
     * java.io.Reader)
     */
    @Override
    public TokenStream tokenStream(String fieldName, Reader reader) {
      TokenStream result = super.tokenStream(fieldName, reader);
      result = new LowerCaseFilter(result);

      return result;
    }
View Full Code Here
177
178
179
180
181
182
183
184
185
186
187
    PatternAnalyzer analyzer = new PatternAnalyzer(pattern, toLowerCase, stopWords);
    return analyzer.tokenStream("", text);
  }
 
  private TokenStream luceneTokenStream(String text, boolean letters, boolean toLowerCase, Set stopWords) {
    TokenStream stream;
    if (letters)
      stream = new LetterTokenizer(new StringReader(text));
    else
      stream = new WhitespaceTokenizer(new StringReader(text));
    if (toLowerCase)  stream = new LowerCaseFilter(stream);
View Full Code Here
78
79
80
81
82
83
84
85
86
87
88
        // The HTML escaping forces us to first fragment with internal placeholders...
        Highlighter highlighter = new Highlighter(new SimpleHTMLFormatter(INTERNAL_BEGIN_HIT, INTERNAL_END_HIT), new QueryScorer(query));
        highlighter.setTextFragmenter(fragmenter);
        try {
            // Use the same analyzer as the indexer!
            TokenStream tokenStream = new StandardAnalyzer().tokenStream(null, new StringReader(indexedText));

            String unescapedFragements =
                    highlighter.getBestFragments(tokenStream, indexedText, numOfFragments, getFragmentSeparator());

            String escapedFragments = WikiUtil.escapeHtml(WikiUtil.removeMacros(unescapedFragements), false, false);
View Full Code Here
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
    private Query buildPhraseQuery(String fieldName, String terms) {
        try {
            PhraseQuery query = new PhraseQuery();
            query.setSlop(0);

            TokenStream includeStream =
                    new StandardAnalyzer().tokenStream(null, new StringReader(escape(terms).toLowerCase()));

            while (true) {
                Token t = includeStream.next();
                if (t == null) break;
                query.add( new Term(fieldName, t.termText()) );
            }

            return query.getTerms().length > 0 ? query : null;
View Full Code Here
TOP

Related Classes of org.apache.lucene.analysis.TokenStream$TokenWrapperAttributeFactory

  • com.code972.elasticsearch.rest.action.RestHebrewAnalyzerCheckWordAction
  • com.gentics.cr.lucene.analysis.CustomPatternAnalyzerTest
  • com.github.pmerienne.trident.ml.preprocessing.EnglishTokenizer
  • com.o19s.RegexPathHierarchyTokenizerTest
  • com.stimulus.archiva.search.FileNameAnalyzer
  • com.stimulus.archiva.search.FileNameAnalyzer$LowercaseDelimiterAnalyzer
  • ivory.core.tokenize.LuceneAnalyzer
  • ivory.core.tokenize.LuceneArabicAnalyzer
  • ivory.core.tokenize.LuceneTokenizer
  • org.apache.jackrabbit.core.query.lucene.SearchIndex
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.