// A byte set for word fields tracking
final BitSet fieldIndices = new BitSet(context.allFields.name.length);
// A stack for pushing information about the term's documents.
final IntStack wordDocuments = new IntStack();
if (documentIndexesArray[tokenImagesOrder[0]] >= 0)
{
wordDocuments.push(documentIndexesArray[tokenImagesOrder[0]]);
}
// Go through the ordered token images
for (int i = 0; i < tokenImagesOrder.length - 1; i++)
{
final char [] image = tokenImages[tokenImagesOrder[i]];
final char [] nextImage = tokenImages[tokenImagesOrder[i + 1]];
final int tokenType = tokenTypesArray[tokenImagesOrder[i]];
final int documentIndex = documentIndexesArray[tokenImagesOrder[i + 1]];
// Reached the end of non-null tokens?
if (image == null)
{
break;
}
// Check if we want to index this token at all
if (isNotIndexed(tokenType))
{
variantStartIndex = i + 1;
maxTfVariantIndex = tokenImagesOrder[i + 1];
resetForNewTokenImage(documentIndexesArray, tokenImagesOrder,
fieldIndices, wordDocuments, i);
continue;
}
fieldIndices.set(tokensFieldIndex[tokenImagesOrder[i]]);
// Now check if image case is changing
final boolean sameCase = CharArrayComparators.FAST_CHAR_ARRAY_COMPARATOR
.compare(image, nextImage) == 0;
if (sameCase)
{
// Case has not changed, just increase counters
tf++;
totalTf++;
wordDocuments.push(documentIndex);
continue;
}
// Case (or even token image) has changed. Update most frequent case
// variant
if (maxTf < tf)
{
maxTf = tf;
maxTfVariantIndex = tokenImagesOrder[i];
tf = 1;
}
final boolean sameImage = CharArrayComparators.CASE_INSENSITIVE_CHAR_ARRAY_COMPARATOR
.compare(image, nextImage) == 0;
// Check if token image has changed
if (sameImage)
{
totalTf++;
wordDocuments.push(documentIndex);
}
else
{
// The image has changed completely.
// Before we start processing the new image, we need to
// see if we want to store the previous image, and if so
// we need add some data about it to the arrays
// wordDocuments.size() may contain duplicate entries from the same document,
// but this check is faster than deduping, so we do it first.
if (wordDocuments.size() >= dfThreshold)
{
// Flatten the list of documents this term occurred in.
final int [] sparseEncoding = SparseArray.toSparseEncoding(wordDocuments);
final int df = (sparseEncoding.length >> 1);
if (df >= dfThreshold)