Package org.apache.lucene.analysis

Examples of org.apache.lucene.analysis.Token.buffer()

136
137
138
139
140
141
142
143
144
145
146
        PositionIncrementAttribute newPosIncAtt = newTok.addAttribute(PositionIncrementAttribute.class);

        OffsetAttribute lastOffsetAtt = lastTok.addAttribute(OffsetAttribute.class);

        newOffsetAtt.setOffset(newOffsetAtt.startOffset(), lastOffsetAtt.endOffset());
        newTermAtt.copyBuffer(repTok.buffer(), 0, repTok.length());
        repPos += repTok.getPositionIncrement();
        if (i==0) repPos=origPos;  // make position of first token equal to original

        // if necessary, insert original tokens and adjust position increment
        while (origTok != null && origPos <= repPos) {
View Full Code Here
569
570
571
572
573
574
575
576
577
578
579
        }

        if (hasFreqInfo) {
          isCorrectlySpelled = isCorrectlySpelled && spellingResult.getTokenFrequency(inputToken) > 0;
        }
        result.add(new String(inputToken.buffer(), 0, inputToken.length()), suggestionList);
      }
    }
    if (hasFreqInfo) {
      result.add("correctlySpelled", isCorrectlySpelled);
    } else if(extendedResults && suggestions.size() == 0) { // if the word is misspelled, its added to suggestions with freqinfo
View Full Code Here
122
123
124
125
126
127
128
129
130
131
132
  public final boolean incrementToken() throws IOException {
    clearAttributes();
    Token token = nextToken(reusableToken);
    if(token != null) {
      termAtt.copyBuffer(token.buffer(), 0, token.length());
      offsetAtt.setOffset(token.startOffset(), token.endOffset());
      typeAtt.setType(token.type());
      return true;
    } else {
      end();
View Full Code Here
65
66
67
68
69
70
71
72
73
74
75
    if(nextToken != null &&
        (Word.TYPE_LETTER_OR_DIGIT.equalsIgnoreCase(nextToken.type())
          || Word.TYPE_DIGIT_OR_LETTER.equalsIgnoreCase(nextToken.type()))
        ) {
      final char[] buffer = nextToken.buffer();
      final int length = nextToken.length();
      byte lastType = (byte) Character.getType(buffer[0]);  //与上次的字符是否同类
      int termBufferOffset = 0;
      int termBufferLength = 0;
      for(int i=0;i<length;i++) {
View Full Code Here
136
137
138
139
140
141
142
143
144
145
146
        PositionIncrementAttribute newPosIncAtt = newTok.addAttribute(PositionIncrementAttribute.class);

        OffsetAttribute lastOffsetAtt = lastTok.addAttribute(OffsetAttribute.class);

        newOffsetAtt.setOffset(newOffsetAtt.startOffset(), lastOffsetAtt.endOffset());
        newTermAtt.copyBuffer(repTok.buffer(), 0, repTok.length());
        repPos += repTok.getPositionIncrement();
        if (i==0) repPos=origPos;  // make position of first token equal to original

        // if necessary, insert original tokens and adjust position increment
        while (origTok != null && origPos <= repPos) {
View Full Code Here
36
37
38
39
40
41
42
43
44
45
46
    public AB_Q_Stream(TokenStream input) {super(input);}
    @Override
    protected Token process(Token t) throws IOException {
      if ("A".equals(new String(t.buffer(), 0, t.length()))) {
        Token t2 = read();
        if (t2!=null && "B".equals(new String(t2.buffer(), 0, t2.length()))) t.setEmpty().append("Q");
        if (t2!=null) pushBack(t2);
      }
      return t;
    }
  }
View Full Code Here
135
136
137
138
139
140
141
142
143
144
145
        PositionIncrementAttribute newPosIncAtt = newTok.addAttribute(PositionIncrementAttribute.class);

        OffsetAttribute lastOffsetAtt = lastTok.addAttribute(OffsetAttribute.class);

        newOffsetAtt.setOffset(newOffsetAtt.startOffset(), lastOffsetAtt.endOffset());
        newTermAtt.copyBuffer(repTok.buffer(), 0, repTok.length());
        repPos += repTok.getPositionIncrement();
        if (i==0) repPos=origPos;  // make position of first token equal to original

        // if necessary, insert original tokens and adjust position increment
        while (origTok != null && origPos <= repPos) {
View Full Code Here
568
569
570
571
572
573
574
575
576
577
578
        }

        if (hasFreqInfo) {
          isCorrectlySpelled = isCorrectlySpelled && spellingResult.getTokenFrequency(inputToken) > 0;
        }
        result.add(new String(inputToken.buffer(), 0, inputToken.length()), suggestionList);
      }
    }
    if (hasFreqInfo) {
      result.add("correctlySpelled", isCorrectlySpelled);
    } else if(extendedResults && suggestions.size() == 0) { // if the word is misspelled, its added to suggestions with freqinfo
View Full Code Here
138
139
140
141
142
143
144
145
146
147
148
                PositionIncrementAttribute newPosIncAtt = newTok.addAttribute(PositionIncrementAttribute.class);

                OffsetAttribute lastOffsetAtt = lastTok.addAttribute(OffsetAttribute.class);

                newOffsetAtt.setOffset(newOffsetAtt.startOffset(), lastOffsetAtt.endOffset());
                newTermAtt.copyBuffer(repTok.buffer(), 0, repTok.length());
                repPos += repTok.getPositionIncrement();
                if (i == 0) repPos = origPos;  // make position of first token equal to original

                // if necessary, insert original tokens and adjust position increment
                while (origTok != null && origPos <= repPos) {
View Full Code Here
56
57
58
59
60
61
62
63
64
65
66
    @Override
    public final boolean incrementToken() {
      clearAttributes();
      if (index < testToken.length) {
        Token t = testToken[index++];
        termAtt.copyBuffer(t.buffer(), 0, t.length());
        offsetAtt.setOffset(t.startOffset(), t.endOffset());
        posIncrAtt.setPositionIncrement(t.getPositionIncrement());
        typeAtt.setType(TypeAttribute.DEFAULT_TYPE);
        return true;
      } else {
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.