ArrayList<Token> result = new ArrayList<>();
TokenStream ts = analyzer.tokenStream("full", new StringReader(text));
CharTermAttribute term = ts.addAttribute(CharTermAttribute.class);
OffsetAttribute offset = ts.addAttribute(OffsetAttribute.class);
while (ts.incrementToken()) {
Token t = new Token(term.buffer(), 0, term.length(), offset.startOffset(), offset.endOffset());
result.add(t);
}
return result.toArray(new Token[result.size()]);
}