tokens.add(tokenFactory("earth", 0, 1f, 5, 10, ShingleMatrixFilter.TokenPositioner.newRow));
tokens.add(tokenFactory("tellus", 0, 1f, 5, 10, ShingleMatrixFilter.TokenPositioner.newRow));
tls = new TokenListStream(tokens);
ts = new PrefixAndSuffixAwareTokenFilter(new SingleTokenTokenStream(tokenFactory("^", 1, 100f, 0, 0)), tls, new SingleTokenTokenStream(tokenFactory("$", 1, 50f, 0, 0)));
tls = new CachingTokenFilter(ts);
// bi-grams, position incrememnt, weight, start offset, end offset
ts = new ShingleMatrixFilter(tls, 2, 2, new Character('_'), false);