* Tokenize xoo files (separator is whitespace) for duplication detection
*/
public class XooTokenizerSensor implements Sensor {
private void computeTokens(InputFile inputFile, SensorContext context) {
DuplicationTokenBuilder tokenBuilder = context.duplicationTokenBuilder(inputFile);
File ioFile = inputFile.file();
int lineId = 0;
try {
for (String line : FileUtils.readLines(ioFile)) {
lineId++;
for (String token : Splitter.on(" ").split(line)) {
tokenBuilder.addToken(lineId, token);
}
}
tokenBuilder.done();
} catch (IOException e) {
throw new IllegalStateException("Unable to read file " + ioFile, e);
}
}