Package org.apache.lucene.analysis.tokenattributes

Examples of org.apache.lucene.analysis.tokenattributes.CharTermAttribute.buffer()

53
54
55
56
57
58
59
60
61
62
63
        Analyzer a = new HebrewQueryLightAnalyzer();
        TokenStream ts = a.tokenStream("foo", word);
        ts.reset();
        while (ts.incrementToken()) {
            CharTermAttribute cta = ts.getAttribute(CharTermAttribute.class);
            ret.add(new String(cta.buffer(), 0, cta.length()));
        }
        ts.close();
        a.close();
        return ret;
    }
View Full Code Here
55
56
57
58
59
60
61
62
63
64
65
                .addAttribute(CharTermAttribute.class);
        StringTuple document = new StringTuple();
        stream.reset();
        while (stream.incrementToken()) {
            if (termAtt.length() > 0) {
                document.add(new String(termAtt.buffer(), 0, termAtt.length()));
            }
        }
        context.write(key, document);
    }
View Full Code Here
789
790
791
792
793
794
795
796
797
798
799
      int tokenCount = 0;
      // for every token
      while (ts.incrementToken())
      {
         CharTermAttribute term = ts.getAttribute(CharTermAttribute.class);
         String word = new String(term.buffer(), 0, term.length());
         tokenCount++;
         if (tokenCount > maxNumTokensParsed)
         {
            break;
         }
View Full Code Here
245
246
247
248
249
250
251
252
253
254
255
      {
         while (ts.incrementToken())
         {
            OffsetAttribute offset = ts.getAttribute(OffsetAttribute.class);
            CharTermAttribute term = ts.getAttribute(CharTermAttribute.class);
            String termText = new String(term.buffer(), 0, term.length());
            TermVectorOffsetInfo[] info = termMap.get(termText);
            if (info == null)
            {
               info = new TermVectorOffsetInfo[1];
            }
View Full Code Here
373
374
375
376
377
378
379
380
381
382
383
         {
            String word;
            while (ts.incrementToken())
            {

               word = new String(term.buffer(), 0, term.length());
               //            while ((t = ts.next()) != null)
               //            {
               String origWord = statement.substring(offset.startOffset(), offset.endOffset());
               if (positionIncrement.getPositionIncrement() > 0)
               {
View Full Code Here
54
55
56
57
58
59
60
61
62
63
64
    final TokenStream stream = analyzer.tokenStream( field, new StringReader( text ) );
    try {
      CharTermAttribute term = stream.addAttribute( CharTermAttribute.class );
      stream.reset();
      while ( stream.incrementToken() ) {
        String s = new String( term.buffer(), 0, term.length() );
        tokenList.add( s );
      }
      stream.end();
    }
    finally {
View Full Code Here
73
74
75
76
77
78
79
80
81
82
83
    try {
      CharTermAttribute term = stream.addAttribute( CharTermAttribute.class );
      stream.reset();
      while ( stream.incrementToken() ) {
        Token token = new Token();
        token.copyBuffer( term.buffer(), 0, term.length() );
        tokenList.add( token );
      }
      stream.end();
    }
    finally {
View Full Code Here
93
94
95
96
97
98
99
100
101
102
103
    try {
      CharTermAttribute attribute = stream.addAttribute( CharTermAttribute.class );
      stream.reset();
      while ( stream.incrementToken() ) {
        if ( attribute.length() > 0 ) {
          String term = new String( attribute.buffer(), 0, attribute.length() );
          terms.add( term );
        }
      }
      stream.end();
    }
View Full Code Here
113
114
115
116
117
118
119
120
121
122
123
        if ( termsAtSamePosition == null ) {
          termsAtSamePosition = new ArrayList<Term>();
          termsPerPosition.put( position, termsAtSamePosition );
        }

        String termString = new String( termAttribute.buffer(), 0, termAttribute.length() );
        termsAtSamePosition.add( new Term( fieldName, termString ) );
        if ( termsAtSamePosition.size() > 1 ) {
          isMultiPhrase = true;
        }
      }
View Full Code Here
76
77
78
79
80
81
82
83
84
85
86
    try {
      CharTermAttribute attribute = stream.addAttribute( CharTermAttribute.class );
      stream.reset();
      while ( stream.incrementToken() ) {
        if ( attribute.length() > 0 ) {
          String term = new String( attribute.buffer(), 0, attribute.length() );
          terms.add( term );
        }
      }
      stream.end();
    }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.