Package org.apache.lucene.analysis.tokenattributes

Examples of org.apache.lucene.analysis.tokenattributes.CharTermAttribute.buffer()

76
77
78
79
80
81
82
83
84
85
86
    try {
      CharTermAttribute attribute = stream.addAttribute( CharTermAttribute.class );
      stream.reset();
      while ( stream.incrementToken() ) {
        if ( attribute.length() > 0 ) {
          String term = new String( attribute.buffer(), 0, attribute.length() );
          terms.add( term );
        }
      }
      stream.end();
    }
View Full Code Here
96
97
98
99
100
101
102
103
104
105
106
        if ( termsAtSamePosition == null ) {
          termsAtSamePosition = new ArrayList<Term>();
          termsPerPosition.put( position, termsAtSamePosition );
        }

        String termString = new String( termAttribute.buffer(), 0, termAttribute.length() );
        termsAtSamePosition.add( new Term( fieldName, termString ) );
        if ( termsAtSamePosition.size() > 1 ) {
          isMultiPhrase = true;
        }
      }
View Full Code Here
129
130
131
132
133
134
135
136
137
138
139
        int end = reuse.offset + reuse.length;
        if (reuse.length > 0) {
          reuse.chars[end++] = SynonymMap.WORD_SEPARATOR;
          reuse.length++;
        }
        System.arraycopy(termAtt.buffer(), 0, reuse.chars, end, length);
        reuse.length += length;
      }
      ts.end();
      ts.close();
      if (reuse.length == 0) {
View Full Code Here
47
48
49
50
51
52
53
54
55
56
57
                CharTermAttribute termAtt = tokenizer
                        .getAttribute(CharTermAttribute.class);
                OffsetAttribute offsetAtt = tokenizer
                        .getAttribute(OffsetAttribute.class);
                if (tokenizer.incrementToken()) {
                    String word = new String(termAtt.buffer(), 0,
                            termAtt.length());
                    int start = offsetAtt.startOffset();
                    int end = offsetAtt.endOffset();
                    if (prevWord == null) {
                        prevWord = word;
View Full Code Here
88
89
90
91
92
93
94
95
96
97
98
                new StringReader(input));

        CharTermAttribute termAtt = tokenizer
                .getAttribute(CharTermAttribute.class);
        while (tokenizer.incrementToken()) {
            String word = new String(termAtt.buffer(), 0, termAtt.length());
            sb.append(word).append("|");
        }
        System.out.println(input + " => " + sb.toString());
    }
View Full Code Here
95
96
97
98
99
100
101
102
103
104
105
                .getAttribute(CharTermAttribute.class);
        OffsetAttribute offsetAtt = tokenizer
                .getAttribute(OffsetAttribute.class);
        String i = "";
        while (tokenizer.incrementToken()) {
            String word = new String(termAtt.buffer(), 0, termAtt.length());
            // int s = offsetAtt.startOffset();
            // int e = offsetAtt.endOffset();
            i = word;
        }
        if (print) {
View Full Code Here
59
60
61
62
63
64
65
66
67
68
69
                            .getAttribute(CharTermAttribute.class);
                    OffsetAttribute offsetAtt = tokenizer
                            .getAttribute(OffsetAttribute.class);
                    String w = "";
                    while (tokenizer.incrementToken()) {
                        String word = new String(termAtt.buffer(), 0, termAtt.length());
                        // int s = offsetAtt.startOffset();
                        // int e = offsetAtt.endOffset();
                        w = word;
                    }
                    long segs = System.currentTimeMillis() - start;
View Full Code Here
40
41
42
43
44
45
46
47
48
49
50
    reusableToken.clear();
    if(termAtt != null) {
      //lucene 3.0
      //reusableToken.setTermBuffer(termAtt.termBuffer(), 0, termAtt.termLength());
      //lucene 3.1
      reusableToken.copyBuffer(termAtt.buffer(), 0, termAtt.length());
    }
    if(offsetAtt != null) {
      reusableToken.setStartOffset(offsetAtt.startOffset());
      reusableToken.setEndOffset(offsetAtt.endOffset());
    }
View Full Code Here
112
113
114
115
116
117
118
119
120
121
122
        if ( termsAtSamePosition == null ) {
          termsAtSamePosition = new ArrayList<Term>();
          termsPerPosition.put( position, termsAtSamePosition );
        }

        String termString = new String( termAttribute.buffer(), 0, termAttribute.length() );
        termsAtSamePosition.add( new Term( fieldName, termString ) );
        if ( termsAtSamePosition.size() > 1 ) {
          isMultiPhrase = true;
        }
      }
View Full Code Here
126
127
128
129
130
131
132
133
134
135
136
        ts.reset();

        Set<String> terms = new HashSet<>();
        while (ts.incrementToken()) {
            CharTermAttribute att = ts.getAttribute(CharTermAttribute.class);
            terms.add(new String(att.buffer(), 0, att.length()));
            //System.out.println(new String(att.buffer(), 0, att.length()));
        }
    }

    @SuppressWarnings("StatementWithEmptyBody")
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.