if(field==INDEXFIELD.TERM)
{
try
{
return new Tokenizer(new AccentRule(), new ApostropheRule(), new CapitalizationRule(),new HyphenRule(), new PunctuationRule(), new WhitespaceRule(),new NumberRule(),new SpecialCharRule(),new StopWordsRule());
}
catch (TokenizerException e)
{ // TODO Auto-generated catch block
e.printStackTrace();
}
}
else if(field==INDEXFIELD.AUTHOR)
{
try
{
return new Tokenizer(new AccentRule(), new ApostropheRule(), new CapitalizationRule(),new HyphenRule(), new PunctuationRule(),new SpecialCharRule());
}
catch (TokenizerException e)
{ // TODO Auto-generated catch block
e.printStackTrace();
}
}
else if(field==INDEXFIELD.CATEGORY)
{
try
{
return new Tokenizer(new AccentRule(), new ApostropheRule(), new CapitalizationRule(),new HyphenRule(), new WhitespaceRule(),new NumberRule(),new SpecialCharRule(),new PunctuationRule(),new StopWordsRule());
}
catch (TokenizerException e)
{ // TODO Auto-generated catch block
e.printStackTrace();
}
}
else if(field==INDEXFIELD.LINK)
{
try
{
return new Tokenizer(new CapitalizationRule(),new WhitespaceRule(),new SpecialCharRule(),new NumberRule(), new StopWordsRule(),new PunctuationRule() );
}
catch (TokenizerException e)
{ // TODO Auto-generated catch block
e.printStackTrace();