@Override
protected TokenStreamComponents createComponents(String fileName,
Reader reader) {
if (expansionType.equals(ExpansionType.URI)) {
final KeywordTokenizer src = new KeywordTokenizer(reader);
TokenStream tok = new SKOSURIFilter(src, skosEngine,
new StandardAnalyzer(matchVersion), types);
tok = new LowerCaseFilter(matchVersion, tok);
return new TokenStreamComponents(src, tok);
} else {
final StandardTokenizer src = new StandardTokenizer(matchVersion, reader);
src.setMaxTokenLength(maxTokenLength);
TokenStream tok = new StandardFilter(matchVersion, src);
// prior to this we get the classic behavior, standardfilter does it for
// us.
tok = new SKOSLabelFilter(tok, skosEngine, new StandardAnalyzer(
matchVersion), bufferSize, types);
tok = new LowerCaseFilter(matchVersion, tok);
tok = new StopFilter(matchVersion, tok, stopwords);
tok = new RemoveDuplicatesTokenFilter(tok);
return new TokenStreamComponents(src, tok) {
@Override
protected void setReader(final Reader reader) throws IOException {
src.setMaxTokenLength(maxTokenLength);
super.setReader(reader);
}
};
}
}