TokenOffsetPayloadTokenFilter nptf = new TokenOffsetPayloadTokenFilter(new WhitespaceTokenizer(new StringReader(test)));
Token tok = new Token();
int count = 0;
while ((tok = nptf.next(tok)) != null){
assertTrue("tok is null and it shouldn't be", tok != null);
Payload pay = tok.getPayload();
assertTrue("pay is null and it shouldn't be", pay != null);
byte [] data = pay.getData();
int start = PayloadHelper.decodeInt(data, 0);
assertTrue(start + " does not equal: " + tok.startOffset(), start == tok.startOffset());
int end = PayloadHelper.decodeInt(data, 4);
assertTrue(end + " does not equal: " + tok.endOffset(), end == tok.endOffset());
count++;