Package org.apache.lucene.analysis

Examples of org.apache.lucene.analysis.Token.termText()

84
85
86
87
88
89
90
91
92
93
94
        TokenStream ts = new StandardAnalyzer().tokenStream("",new HTMLParser(file).getReader());

        Token token = null;

        while ((token = ts.next()) != null) {
            System.out.println("ReTokenizeFile.reTokenize(File): " + token.termText() + " " +
                token.startOffset() + " " + token.endOffset() + " " + token.type());
        }

        return file.getAbsolutePath();
    }
View Full Code Here
41
42
43
44
45
46
47
48
49
50
51
        Token t = input.next();

        if (t == null)
            return null;

        String txt = t.termText();

        char[] chArray = txt.toCharArray();
        for (int i = 0; i < chArray.length; i++)
        {
            chArray[i] = RussianCharsets.toLowerCase(chArray[i], charset);
View Full Code Here
50
51
52
53
54
55
56
57
58
59
60
      Term term = null;
      while (token != null)
      {
        if (term == null)
        {
          term = new Term(fieldName, token.termText());
        } else
        {
//           create from previous to save fieldName.intern overhead
          term = term.createTerm(token.termText());
        }
View Full Code Here
54
55
56
57
58
59
60
61
62
63
64
        {
          term = new Term(fieldName, token.termText());
        } else
        {
//           create from previous to save fieldName.intern overhead
          term = term.createTerm(token.termText());
        }
        tf.addTerm(term);
        token = ts.next();
      }
    }
View Full Code Here
38
39
40
41
42
43
44
45
46
47
48
      ArrayList clausesList=new ArrayList();
      TokenStream ts=analyzer.tokenStream(fieldName,new StringReader(value));
      Token token=ts.next();
      while(token!=null)
      {
          SpanTermQuery stq=new SpanTermQuery(new Term(fieldName,token.termText()));
          clausesList.add(stq);
        token=ts.next();       
      }
      SpanOrQuery soq=new SpanOrQuery((SpanQuery[]) clausesList.toArray(new SpanQuery[clausesList.size()]));
      soq.setBoost(DOMUtils.getAttribute(e,"boost",1.0f));
View Full Code Here
84
85
86
87
88
89
90
91
92
93
94
            assertEquals(
                "Unicode",
                token.termText(),
                sampleToken == null
                ? null
                : sampleToken.termText());
        }

        inWords.close();
        sampleUnicode.close();
    }
View Full Code Here
121
122
123
124
125
126
127
128
129
130
131
            assertEquals(
                "KOI8",
                token.termText(),
                sampleToken == null
                ? null
                : sampleToken.termText());

        }

        inWordsKOI8.close();
        sampleKOI8.close();
View Full Code Here
158
159
160
161
162
163
164
165
166
167
168
            assertEquals(
                "1251",
                token.termText(),
                sampleToken == null
                ? null
                : sampleToken.termText());

        }

        inWords1251.close();
        sample1251.close();
View Full Code Here
126
127
128
129
130
131
132
133
134
                                new StringReader("one two three four five"));

    while (true) {
      Token token = ts.next();
      if (token == null) break;
      assertEquals(token.termText(), 1, token.getPositionIncrement());
    }
  }
}
View Full Code Here
69
70
71
72
73
74
75
76
77
78
    StandardTokenizer tokenStream = new StandardTokenizer(new StringReader(input));
    GermanStemFilter filter = new GermanStemFilter(tokenStream);
    Token t = filter.next();
    if (t == null)
      fail();
    assertEquals(expected, t.termText());
    filter.close();
  }

}
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.