SymbolTable vocab = new Vocabulary();
Corpus corpus = suffixArray.getCorpus();
NGramLanguageModel largeLM = new LMGrammarJAVA(
vocab,
lmOrder,
largeArpaLM,
JoshuaConfiguration.use_left_equivalent_state,
JoshuaConfiguration.use_right_equivalent_state);
NGramLanguageModel testLM = new LMGrammarJAVA(
vocab,
lmOrder,
testArpaLM,
JoshuaConfiguration.use_left_equivalent_state,
JoshuaConfiguration.use_right_equivalent_state);
this.weights = new float[corpus.getNumSentences()];
for (int i=0, n=corpus.getNumSentences(); i<n; i++) {
Phrase sentence = corpus.getSentence(i);
int[] words = sentence.getWordIDs();
double largeProbLM = largeLM.ngramLogProbability(words);
double testProbLM = testLM.ngramLogProbability(words);
double ratio = testProbLM - largeProbLM;
this.weights[i] = (float) ratio;
}
}