Examples of termLength()

Examples of org.apache.lucene.analysis.Token.termLength()

68
69
70
71
72
73
74
75
76
77
78
  private void assertTokEq(List<Token> a, List<Token> b, boolean checkOff) {
    int pos=0;
    for (Iterator iter = a.iterator(); iter.hasNext();) {
      Token tok = (Token)iter.next();
      pos += tok.getPositionIncrement();
      if (!tokAt(b, new String(tok.termBuffer(), 0, tok.termLength()), pos
              , checkOff ? tok.startOffset() : -1
              , checkOff ? tok.endOffset() : -1
              ))
      {
        fail(a + "!=" + b);
View Full Code Here

Examples of org.apache.lucene.analysis.Token.termLength()

83
84
85
86
87
88
89
90
91
92
93
  public boolean tokAt(List<Token> lst, String val, int tokPos, int startOff, int endOff) {
    int pos=0;
    for (Iterator iter = lst.iterator(); iter.hasNext();) {
      Token tok = (Token)iter.next();
      pos += tok.getPositionIncrement();
      if (pos==tokPos && new String(tok.termBuffer(), 0, tok.termLength()).equals(val)
          && (startOff==-1 || tok.startOffset()==startOff)
          && (endOff  ==-1 || tok.endOffset()  ==endOff  )
           )
      {
        return true;
View Full Code Here

Examples of org.apache.lucene.analysis.Token.termLength()

35
36
37
38
39
40
41
42
43
44
45
  public static class AB_Q_Stream extends BufferedTokenStream {
    public AB_Q_Stream(TokenStream input) {super(input);}
    protected Token process(Token t) throws IOException {
      if ("A".equals(new String(t.termBuffer(), 0, t.termLength()))) {
        Token t2 = read();
        if (t2!=null && "B".equals(new String(t2.termBuffer(), 0, t2.termLength()))) t.setTermText("Q");
        if (t2!=null) pushBack(t2);
      }
      return t;
    }
  }
View Full Code Here

Examples of org.apache.lucene.analysis.Token.termLength()

27
28
29
30
31
32
33
34
35
36
37
  public void testSize4FalseInject() throws Exception {
    TokenStream stream = new IterTokenStream("international");
    TokenStream filter = new DoubleMetaphoneFilter(stream, 4, false);

    Token token = filter.next(new Token());
    assertEquals(4, token.termLength());
    assertEquals("ANTR", new String(token.termBuffer(), 0, token.termLength()));

    assertNull(filter.next(new Token()));
  }
View Full Code Here

Examples of org.apache.lucene.analysis.Token.termLength()

28
29
30
31
32
33
34
35
36
37
38
    TokenStream stream = new IterTokenStream("international");
    TokenStream filter = new DoubleMetaphoneFilter(stream, 4, false);

    Token token = filter.next(new Token());
    assertEquals(4, token.termLength());
    assertEquals("ANTR", new String(token.termBuffer(), 0, token.termLength()));

    assertNull(filter.next(new Token()));
  }

  public void testSize4TrueInject() throws Exception {
View Full Code Here

Examples of org.apache.lucene.analysis.Token.termLength()

38
39
40
41
42
43
44
45
46
47
48
  public void testSize4TrueInject() throws Exception {
    TokenStream stream = new IterTokenStream("international");
    TokenStream filter = new DoubleMetaphoneFilter(stream, 4, true);

    Token token = filter.next(new Token());
    assertEquals(13, token.termLength());
    assertEquals("international", new String(token.termBuffer(), 0, token
        .termLength()));

    token = filter.next(new Token());
    assertEquals(4, token.termLength());
View Full Code Here

Examples of org.apache.lucene.analysis.tokenattributes.TermAttribute.termLength()

32
33
34
35
36
37
38
39
40
41
42
    TermAttribute termAtt = (TermAttribute) result
        .addAttribute(TermAttribute.class);
    StringBuilder buf = new StringBuilder();
    try {
      while (result.incrementToken()) {
        String word = new String(termAtt.termBuffer(), 0, termAtt
            .termLength());
        buf.append(filter.encode(word)).append(" ");
       
      }
    } catch (IOException e) {
View Full Code Here

Examples of org.apache.lucene.analysis.tokenattributes.TermAttribute.termLength()

29
30
31
32
33
34
35
36
37
38
39
    TermAttribute termAtt = ts.addAttribute(TermAttribute.class);

    Vector v1 = new RandomAccessSparseVector(100);                  
    while (ts.incrementToken()) {
      char[] termBuffer = termAtt.termBuffer();
      int termLen = termAtt.termLength();
      String w = new String(termBuffer, 0, termLen);                
      encoder.addToVector(w, 1, v1);                                
    }
    System.out.printf("%s\n", new SequentialAccessSparseVector(v1));
  }
View Full Code Here

Examples of org.apache.lucene.analysis.tokenattributes.TermAttribute.termLength()

35
36
37
38
39
40
41
42
43
44
45
   
    TermAttribute termAtt = (TermAttribute) result.addAttribute(TermAttribute.class);
    StringBuilder buf = new StringBuilder();
    try {
      while (result.incrementToken()) {
        if (termAtt.termLength() < 3) continue;
        String word = new String(termAtt.termBuffer(), 0, termAtt.termLength());
        Matcher m = alphabets.matcher(word);
       
        if (m.matches()) {
          buf.append(word).append(" ");
View Full Code Here

Examples of org.apache.lucene.analysis.tokenattributes.TermAttribute.termLength()

36
37
38
39
40
41
42
43
44
45
46
    TermAttribute termAtt = (TermAttribute) result.addAttribute(TermAttribute.class);
    StringBuilder buf = new StringBuilder();
    try {
      while (result.incrementToken()) {
        if (termAtt.termLength() < 3) continue;
        String word = new String(termAtt.termBuffer(), 0, termAtt.termLength());
        Matcher m = alphabets.matcher(word);
       
        if (m.matches()) {
          buf.append(word).append(" ");
        }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.