package rougeN;
import static org.junit.Assert.assertEquals;
import interfaces.IRougeSummaryModel;
import java.io.File;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import org.junit.Before;
import org.junit.Test;
import rouge.RougeN;
import common.RougeSummaryModel;
import common.ScoreType;
public class RougeNTest
{
private RougeN rouge = null;
private RougeSummaryModel testSummary1;
private RougeSummaryModel refSummary_1_1;
private RougeSummaryModel refSummary_1_2;
private RougeSummaryModel testSummary2;
private RougeSummaryModel refSummary_2_1;
private RougeSummaryModel refSummary_2_2;
private HashSet<IRougeSummaryModel> refSummaries2;
private HashSet<IRougeSummaryModel> refSummaries1;
@Before
public void setUp() throws Exception
{
// init document
testSummary1 = new RougeSummaryModel(new File("peer"));
testSummary1.setTitle("Weighing 8 pounds and standing 13 inches tall, Oscar was created by Metro-Goldwyn-Mayer studios art director Cedric Gibbons, who went on to win 11 of the trophies.")
.addSentence("Oscar, manufactured by the R.S. Owens Co., Chicago, is made of Britannia metal, copper plate, nickel plate and gold plate.")
.addSentence("According to the Academy of Motion Pictures Arts and Sciences, the only engraving mistake was in 1938 when the best actor trophy given to Spencer Tracy for \"Boy\"s Town\" read: \"Best Actor: Dick Tracy.\"")
.addSentence("\"Ben-Hur\" in 1959 was the most-awarded film with 11, and Walt Disney was the most-awarded person with 32.");
// init gs
refSummary_1_1 = new RougeSummaryModel(new File("model.1"));
refSummary_1_1.setTitle("The Oscar, created 60 years ago by MGM art director Cedric Gibbons, weighs 8 pounds and stands 13 inches tall.")
.addSentence("It is made of Britannia metal and plated with copper, nickel, and gold. From 1942-44 it was made of plaster. ")
.addSentence("It is manufactured by Chicago's R.S. Owens Co. 1,816 have been produced so far.")
.addSentence("The only engraving mistake was \"Dick Tracy\" instead of \"Spencer Tracy\" in 1938.")
.addSentence("The Academy of Motion Picture Arts and Sciences reserves the first right to buy an Oscar from anyone.")
.addSentence("\"All About Eve\" is the most-nominated film (14), Ben Hur the most awarded (11), and Walt Disney the most- awarded person (32).");
refSummary_1_2 = new RougeSummaryModel(new File("model.2"));
refSummary_1_2.setTitle("MGM studios art director Cedric Gibbons created Oscar, the Academy of Motion Pictures Arts and Sciences famous award, 60 years ago.")
.addSentence("Manufactured by the R.S. Owens Co., the gold plated metal statue weighs 8 pounds and stands 13 inches tall.")
.addSentence("According to Academy spokesman Bob Werden, the Academy holds all rights on the statue and reserves the right to buy back an Oscar before it is pawned.")
.addSentence("\"All About Eve\" (1950) is the most-nominated film (14).")
.addSentence("\"Ben-Hur\" (1959) is the most-awarded film (11).")
.addSentence("Walt Disney was the most-awarded person (32).");
refSummaries1 = new HashSet<IRougeSummaryModel>();
refSummaries1.add(refSummary_1_1);
refSummaries1.add(refSummary_1_2);
testSummary2 = new RougeSummaryModel(new File("peer"));
testSummary2.setTitle("I was really skating well this week.")
.addSentence("But I tried.")
.addSentence("Combined with a fourth in the compulsory figures and a third-place finish in the short program earlier in the week, the performance put Manley in second place.")
.addSentence("By contrast, Manley had the sellout crowd at the Olympic Saddledome enraptured.")
.addSentence("\"Thomas\" bronze was the third figure skating medal here for the United States.")
.addSentence("In addition to the three figure skating medals, the U.S. team had three speed-skating medals: one each gold, silver and bronze.")
.addSentence("She won the gold in the 500 and the bronze in the 1,000 meters.");
// init gs
refSummary_2_1 = new RougeSummaryModel(new File("model.1"));
refSummary_2_1.setTitle("The United States' Debi Thomas' dream of Olympic gold crumbled Saturday as East Germany's Katarina Witt won her second straight Olympic championship and Canadian Elizabeth Manley took home the silver.")
.addSentence("Thomas won the bronze despite three faulty landings.")
.addSentence("Thomas, of San Jose, Calif., the first black to win a U.S. figure skating crown and the 1986 world champion, skated poorly Saturday after doing well earlier in the games.")
.addSentence("Brian Boitano of the U.S. won the men's figure skating crown.")
.addSentence("In the pairs competition, a bronze went to a U.S. team.")
.addSentence("The U.S. also captured three speed-skating medals, a gold, silver and bronze.");
refSummary_2_2 = new RougeSummaryModel(new File("model.2"));
refSummary_2_2.setTitle("Debi Thomas, the US hope for a figure skating gold medal in the 1988 Winter Olympics, was disappointed when the Gold went to Germany's Katarina Witt and the Silver to Canadian Elizabeth Manley.")
.addSentence("Manley was loudly cheered by fellow Canadians throughout her performance.")
.addSentence("Thomas, who was a US and 1986 world champion, was awarded the Bronze medal.")
.addSentence("Other US figure skating medal winners Brian Boitano with the Gold and Jill Watson and Peter Oppegard with the Bronze in pairs.")
.addSentence("US speed skaters won gold, silver, and bronze with Bonnie Blair taking both a gold and a Bronze.");
refSummaries2 = new HashSet<IRougeSummaryModel>();
refSummaries2.add(refSummary_2_1);
refSummaries2.add(refSummary_2_2);
}
@Test
public void rouge2_SameTextAsGoldStandard()
{
Set<IRougeSummaryModel> s = new HashSet<IRougeSummaryModel>();
s.add(testSummary1);
rouge = new RougeN(testSummary1, s, Integer.MAX_VALUE, Integer.MAX_VALUE, 2, 'A', 0.5);
Map<ScoreType, Double> results = rouge.computeNGramScore();
assertEquals(1, results.get(ScoreType.R), Math.pow(10, -5));
assertEquals(1, results.get(ScoreType.P), Math.pow(10, -5));
assertEquals(1, results.get(ScoreType.F), Math.pow(10, -5));
}
@Test
public void rouge2_TextDoesntMatchGoldStandard()
{
IRougeSummaryModel text = new RougeSummaryModel(null);
text.setTitle("hello world");
IRougeSummaryModel gs = new RougeSummaryModel(null);
gs.setTitle("nothing of the above");
Set<IRougeSummaryModel> s = new HashSet<IRougeSummaryModel>();
s.add(gs);
rouge = new RougeN(text, s, Integer.MAX_VALUE, Integer.MAX_VALUE, 2, 'A', 0.5);
Map<ScoreType, Double> results = rouge.computeNGramScore();
assertEquals(0, results.get(ScoreType.R), Math.pow(10, -5));
assertEquals(0, results.get(ScoreType.P), Math.pow(10, -5));
assertEquals(0, results.get(ScoreType.F), Math.pow(10, -5));
}
@Test
public void rouge2_textAndTwoGoldStandard()
{
rouge = new RougeN(testSummary1, refSummaries1, Integer.MAX_VALUE, Integer.MAX_VALUE, 2, 'A', 0.5);
Map<ScoreType, Double> results = rouge.computeNGramScore();
assertEquals(0.33659, results.get(ScoreType.R), Math.pow(10, -5));
assertEquals(0.32857, results.get(ScoreType.P), Math.pow(10, -5));
assertEquals(0.33253, results.get(ScoreType.F), Math.pow(10, -5));
}
@Test
public void rouge2_TwoTextsWithTwoGoldStandards()
{
rouge = new RougeN(testSummary1, refSummaries1, Integer.MAX_VALUE, Integer.MAX_VALUE, 2, 'A', 0.5);
Map<ScoreType, Double> resultsTest1 = rouge.computeNGramScore();
rouge = new RougeN(testSummary2, refSummaries2, Integer.MAX_VALUE, Integer.MAX_VALUE, 2, 'A', 0.5);
Map<ScoreType, Double> resultsTest2 = rouge.computeNGramScore();
double r = (resultsTest1.get(ScoreType.R) + resultsTest2.get(ScoreType.R))/2.0;
double p = (resultsTest1.get(ScoreType.P) + resultsTest2.get(ScoreType.P))/2.0;
double f = (resultsTest1.get(ScoreType.F) + resultsTest2.get(ScoreType.F))/2.0;
assertEquals(0.24391, r, Math.pow(10, -5));
assertEquals(0.24179, p, Math.pow(10, -5));
assertEquals(0.24281, f, Math.pow(10, -5));
}
@Test
public void rouge1_SameTextAsGoldStandard()
{
Set<IRougeSummaryModel> s = new HashSet<IRougeSummaryModel>();
s.add(testSummary1);
rouge = new RougeN(testSummary1, s, Integer.MAX_VALUE, Integer.MAX_VALUE, 1, 'A', 0.5);
Map<ScoreType, Double> results = rouge.computeNGramScore();
assertEquals(1, results.get(ScoreType.R), Math.pow(10, -5));
assertEquals(1, results.get(ScoreType.P), Math.pow(10, -5));
assertEquals(1, results.get(ScoreType.F), Math.pow(10, -5));
}
@Test
public void rouge1_TextDoesntMatchGoldStandard()
{
IRougeSummaryModel text = new RougeSummaryModel(null);
text.setTitle("hello world");
IRougeSummaryModel gs = new RougeSummaryModel(null);
gs.setTitle("nothing of the above");
Set<IRougeSummaryModel> s = new HashSet<IRougeSummaryModel>();
s.add(gs);
rouge = new RougeN(text, s, Integer.MAX_VALUE, Integer.MAX_VALUE, 1, 'A', 0.5);
Map<ScoreType, Double> results = rouge.computeNGramScore();
assertEquals(0, results.get(ScoreType.R), Math.pow(10, -5));
assertEquals(0, results.get(ScoreType.P), Math.pow(10, -5));
assertEquals(0, results.get(ScoreType.F), Math.pow(10, -5));
}
@Test
public void rouge1_textAndTwoGoldStandard()
{
rouge = new RougeN(testSummary1, refSummaries1, Integer.MAX_VALUE, Integer.MAX_VALUE, 1, 'A', 0.5);
Map<ScoreType, Double> results = rouge.computeNGramScore();
assertEquals(0.61353, results.get(ScoreType.R), Math.pow(10, -5));
assertEquals(0.59906, results.get(ScoreType.P), Math.pow(10, -5));
assertEquals(0.60621, results.get(ScoreType.F), Math.pow(10, -5));
}
@Test
public void rouge1_TwoTextsWithTwoGoldStandards()
{
rouge = new RougeN(testSummary1, refSummaries1, Integer.MAX_VALUE, Integer.MAX_VALUE, 1, 'A', 0.5);
Map<ScoreType, Double> resultsTest1 = rouge.computeNGramScore();
rouge = new RougeN(testSummary2, refSummaries2, Integer.MAX_VALUE, Integer.MAX_VALUE, 1, 'A', 0.5);
Map<ScoreType, Double> resultsTest2 = rouge.computeNGramScore();
double r = (resultsTest1.get(ScoreType.R) + resultsTest2.get(ScoreType.R))/2.0;
double p = (resultsTest1.get(ScoreType.P) + resultsTest2.get(ScoreType.P))/2.0;
double f = (resultsTest1.get(ScoreType.F) + resultsTest2.get(ScoreType.F))/2.0;
assertEquals(0.50725, r, Math.pow(10, -5));
assertEquals(0.50498, p, Math.pow(10, -5));
assertEquals(0.50604, f, Math.pow(10, -5));
}
}