/* This file is part of the Joshua Machine Translation System.
*
* Joshua is free software; you can redistribute it and/or modify it
* under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation; either version 2.1 of the License, or
* (at your option) any later version.
*
* This library is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
* or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
* License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this library; if not, write to the Free Software Foundation,
* Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*/
package joshua.zmert;
import java.io.File;
import java.io.FileNotFoundException;
import java.util.Scanner;
import joshua.zmert.BLEU;
import joshua.zmert.EvaluationMetric;
import org.testng.Assert;
import org.testng.annotations.Parameters;
import org.testng.annotations.Test;
/**
* Unit tests for BLEU class.
*
* @author Lane Schwartz
* @version $LastChangedDate: 2009-05-04 09:56:46 -0500 (Mon, 04 May 2009) $
*/
public class BLEUTest {
@Test
public void metricName() {
// Setup the EvaluationMetric class
EvaluationMetric.set_numSentences(0);
EvaluationMetric.set_refsPerSen(1);
EvaluationMetric.set_refSentences(null);
BLEU bleu = new BLEU();
Assert.assertEquals(bleu.get_metricName(), "BLEU");
}
@Test
public void defaultConstructor() {
// Setup the EvaluationMetric class
EvaluationMetric.set_numSentences(0);
EvaluationMetric.set_refsPerSen(1);
EvaluationMetric.set_refSentences(null);
BLEU bleu = new BLEU();
// Default constructor should use a maximum n-gram length of 4
Assert.assertEquals(bleu.maxGramLength, 4);
// Default constructor should use the closest reference
Assert.assertEquals(bleu.effLengthMethod, BLEU.EffectiveLengthMethod.CLOSEST);
}
@Test
public void simpleTest() {
String ref = "this is the fourth chromosome whose sequence has been completed to date . it comprises more than 87 million pairs of dna .";
String test = "this is the fourth chromosome to be fully sequenced up till now and it comprises of over 87 million pairs of deoxyribonucleic acid ( dna ) .";
// refSentences[i][r] stores the r'th reference of the i'th sentence
String[][] refSentences = new String[1][1];
refSentences[0][0] = ref;
EvaluationMetric.set_numSentences(1);
EvaluationMetric.set_refsPerSen(1);
EvaluationMetric.set_refSentences(refSentences);
BLEU bleu = new BLEU();
// testSentences[i] stores the candidate translation for the i'th sentence
String[] testSentences = new String[1];
testSentences[0] = test;
try {
// Check BLEU score matches
double actualScore = bleu.score(testSentences);
double expectedScore = 0.2513;
double acceptableScoreDelta = 0.00001f;
Assert.assertEquals(actualScore, expectedScore, acceptableScoreDelta);
// Check sufficient statistics match
int[] actualSS = bleu.suffStats(testSentences);
int[] expectedSS = {14,27,8,26,5,25,3,24,27,23};
Assert.assertEquals(actualSS[0], expectedSS[0], 0); // 1-gram matches
Assert.assertEquals(actualSS[1], expectedSS[1], 0); // 1-gram total
Assert.assertEquals(actualSS[2], expectedSS[2], 0); // 2-gram matches
Assert.assertEquals(actualSS[3], expectedSS[3], 0); // 2-gram total
Assert.assertEquals(actualSS[4], expectedSS[4], 0); // 3-gram matches
Assert.assertEquals(actualSS[5], expectedSS[5], 0); // 3-gram total
Assert.assertEquals(actualSS[6], expectedSS[6], 0); // 4-gram matches
Assert.assertEquals(actualSS[7], expectedSS[7], 0); // 4-gram total
Assert.assertEquals(actualSS[8], expectedSS[8], 0); // candidate length
Assert.assertEquals(actualSS[9], expectedSS[9], 0); // reference length
} catch (Exception e) {
Assert.fail();
}
}
@Parameters({"referenceFile","testFile"})
@Test
public void fileTest(String referenceFile, String testFile) throws FileNotFoundException {
//TODO You can now read in the files, and do something useful with them.
Scanner refScanner = new Scanner(new File(referenceFile));
while (refScanner.hasNextLine()) {
String refLine = refScanner.nextLine();
}
}
}