System.out.println("building tokeninfo dict...");
TokenInfoDictionaryBuilder tokenInfoBuilder = new TokenInfoDictionaryBuilder(format, encoding, normalizeEntries, addUnnormalizedEntries, dictionaryFilter);
TokenInfoDictionary tokenInfoDictionary = tokenInfoBuilder.build(inputDirname);
System.out.print(" building double array trie...");
DoubleArrayTrie trie = DoubleArrayTrieBuilder.build(tokenInfoBuilder.entrySet(), compactTrie);
trie.write(outputDirname);
System.out.println(" done");
System.out.print(" processing target map...");
for (Entry<Integer, String> entry : tokenInfoBuilder.entrySet()) {
int tokenInfoId = entry.getKey();
String surfaceForm = entry.getValue();
int doubleArrayId = trie.lookup(surfaceForm);
assert doubleArrayId > 0;
tokenInfoDictionary.addMapping(doubleArrayId, tokenInfoId);
}
tokenInfoDictionary.write(outputDirname);