Package org.apache.lucene.analysis

Examples of org.apache.lucene.analysis.Token.endOffset()

60
61
62
63
64
65
66
67
68
69
70
      if (increment > 0)
      { position = position + increment;
        System.out.println();
        System.out.print(position + ": ");
      }
      System.out.print("\t [" + token.termText() + ": " + token.type() + "] " + token.startOffset() + ":" + token.endOffset());
     
     } //*-- end of for
    System.out.println("");
  }

View Full Code Here
85
86
87
88
89
90
91
92
93
94
95
   Token ctoken = input.next();  if (ctoken == null) return (null);
   String ctext = ctoken.termText();
  
   //*-- check for an unigram entity
   String etype = ehash.get(ctext);
   if (etype != null) stoken = new Token(etype, ctoken.startOffset(), ctoken.endOffset(), "<ENTITY>");
  
   //*-- check for a bigram entity
   if (ptoken != null)   
   { String bigram = ptoken.termText() + " " + ctext;
     etype = ehash.get(bigram);
View Full Code Here
92
93
94
95
96
97
98
99
100
101
102
   //*-- check for a bigram entity
   if (ptoken != null)   
   { String bigram = ptoken.termText() + " " + ctext;
     etype = ehash.get(bigram);
     if (etype != null)
      stoken = new Token(etype, ptoken.startOffset(), ctoken.endOffset(), "<ENTITY>");
   }
  
   if (number.matcher(ctext).matches())
     stoken = new Token("enumber", ctoken.startOffset(), ctoken.endOffset(), "<ENTITY>");
  
View Full Code Here
96
97
98
99
100
101
102
103
104
105
106
     if (etype != null)
      stoken = new Token(etype, ptoken.startOffset(), ctoken.endOffset(), "<ENTITY>");
   }
  
   if (number.matcher(ctext).matches())
     stoken = new Token("enumber", ctoken.startOffset(), ctoken.endOffset(), "<ENTITY>");
  
   ptoken = ctoken;
   return (ctoken);
  }
}
View Full Code Here
205
206
207
208
209
210
211
212
213
214
215
      for (; i < size; i++) {
        Token t1 = (Token) tokens1.get(i);
        Token t2 = (Token) tokens2.get(i);
        if (!(t1.termText().equals(t2.termText()))) throw new IllegalStateException("termText");
        if (t1.startOffset() != t2.startOffset()) throw new IllegalStateException("startOffset");
        if (t1.endOffset() != t2.endOffset()) throw new IllegalStateException("endOffset");
        if (!(t1.type().equals(t2.type()))) throw new IllegalStateException("type");
      }
      if (tokens1.size() != tokens2.size())   throw new IllegalStateException("size1=" + tokens1.size() + ", size2=" + tokens2.size());
    }
View Full Code Here
53
54
55
56
57
58
59
60
61
62
63
  public void collect(String word, int offset, int end) {
    Token c = candidate != null ? candidate : last;
    if (c == null) {
      candidate = new Token(word, offset, end);
    } else if (offset == c.startOffset()) {
      if (end > c.endOffset()) {
        candidate = new Token(word, offset, end);
      }
    } else if (offset > c.startOffset()) {
      if (candidate != null) {
        select(candidate);
View Full Code Here
60
61
62
63
64
65
66
67
68
69
70
      }
    } else if (offset > c.startOffset()) {
      if (candidate != null) {
        select(candidate);
      }
      if (end > c.endOffset()) {
        candidate = new Token(word, offset, end);
      } else {
        candidate = null;
      }
    } else if (end >= c.endOffset()) {
View Full Code Here
65
66
67
68
69
70
71
72
73
74
75
      if (end > c.endOffset()) {
        candidate = new Token(word, offset, end);
      } else {
        candidate = null;
      }
    } else if (end >= c.endOffset()) {
      if (last != null && last.startOffset() >= offset
          && last.endOffset() <= end) {
        for (Iterator iter = tokens.iterator(); iter.hasNext();) {
          last = (Token) iter.next();
          if (last.startOffset() >= offset && last.endOffset() <= end) {
View Full Code Here
123
124
125
126
127
128
129
130
131
132
133
  public boolean incrementToken() throws IOException {
    clearAttributes();
    Token token = nextToken(reusableToken);
    if(token != null) {
      termAtt.setTermBuffer(token.termBuffer(), 0, token.termLength());
      offsetAtt.setOffset(token.startOffset(), token.endOffset());
      typeAtt.setType(token.type());
      return true;
    } else {
      end();
      return false;
View Full Code Here
173
174
175
176
177
178
179
180
181
182
183
          return false;
        }
        Token token = tokens[currentToken++];
        clearAttributes();
        termAtt.setEmpty().append(token);
        offsetAtt.setOffset(token.startOffset(), token.endOffset());
        posincAtt
            .setPositionIncrement(currentToken <= 1
                || tokens[currentToken - 1].startOffset() > tokens[currentToken - 2]
                    .startOffset() ? 1 : 0);
        return true;
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.