package ivory.regression.sigir2013.cdec;
import ivory.core.eval.Qrels;
import ivory.regression.GroundTruth;
import ivory.regression.GroundTruth.Metric;
import ivory.smrf.retrieval.Accumulator;
import ivory.sqe.retrieval.QueryEngine;
import ivory.sqe.retrieval.RunQueryEngine;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import junit.framework.JUnit4TestAdapter;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.junit.Test;
import com.google.common.collect.Maps;
import edu.umd.cloud9.collection.DocnoMapping;
import edu.umd.cloud9.io.map.HMapSFW;
public class EnZh_NTCIR8 {
private QueryEngine qe;
private static String PATH = "en-zh.ntcir8";
private static String LANGUAGE = "zh";
private static int numTopics = 73;
private static float expectedTokenMAP = 0.1497f;
private static Map<Integer,float[]> expectedMAPs = new HashMap<Integer,float[]>();
private static Map<Integer, String[]> grammar_AP = new HashMap<Integer, String[]>();
private static Map<Integer, String[]> Nbest_AP = new HashMap<Integer, String[]>();
private static Map<Integer, String[]> Onebest_AP = new HashMap<Integer, String[]>();
private static Map<Integer, String[]> Interp_AP = new HashMap<Integer, String[]>();
public static void initialize() {
expectedMAPs.put(0, new float[]{0.1880f, 0.1547f, 0.1594f, 0.1865f}); // "one2none" -> grammar,1best,10best,interp
expectedMAPs.put(1, new float[]{0.1697f, 0.1553f, 0.1590f, 0.178f}); // "one2one" -> grammar,1best,10best,interp
expectedMAPs.put(2, new float[]{0.1818f, 0.1555f, 0.1591f, 0.1848f}); // "one2many" -> grammar,1best,10best,interp
grammar_AP.put(0, new String[] {
"78", "0.2479","77", "0.2809","35", "0.0020","36", "0.0031","33", "0.3581","39", "0.0391","38", "0.0","43", "0.0806","42", "0.2518","41", "0.1074","40", "1.0E-4","82", "0.286","83", "0.1818","80", "0.0904","87", "0.2764","84", "0.3939","85", "0.0773","67", "0.1329","66", "0.0136","69", "0.0","68", "0.646","23", "0.1587","26", "0.0245","28", "0.4728","29", "0.0995","2", "0.0488","30", "0.2472","6", "0.0926","5", "0.0023","32", "0.3786","70", "0.025","9", "0.1704","71", "0.5145","72", "0.38","73", "0.0049","74", "0.0666","75", "0.0031","76", "0.232","59", "0.0324","58", "0.0937","57", "0.2235","19", "0.2872","56", "0.2682","18", "0.053","15", "0.2904","16", "0.0491","12", "0.0379","64", "0.4162","65", "0.445","62", "0.5079","63", "0.1861","99", "0.3417","61", "0.1299","100", "0.1019","98", "0.4168","49", "0.0754","97", "0.0077","48", "0.0013","96", "0.0333","95", "0.0018","94", "0.4423","45", "0.2568","93", "0.1276","44", "0.3982","92", "0.2369","47", "0.0","91", "0.5586","46", "0.2252","90", "0.5052","51", "0.0","52", "0.1534","53", "0.0","54", "0.4284",
});
grammar_AP.put(1, new String[] {
"78", "0.2608","77", "0.1306","35", "0.0020","36", "0.0016","33", "0.3547","39", "0.0439","38", "0.0","43", "0.0617","42", "0.2446","41", "0.0516","40", "1.0E-4","82", "0.2914","83", "0.1779","80", "0.0667","87", "0.2769","84", "0.4004","85", "0.0727","67", "0.1346","66", "0.0123","69", "0.0","68", "0.6548","23", "0.1558","26", "0.032","28", "0.4284","29", "0.0942","2", "0.1212","30", "0.2911","6", "0.0394","5", "0.0032","32", "0.3324","70", "0.0195","9", "0.1454","71", "0.5083","72", "0.3891","73", "0.0053","74", "0.0537","75", "0.0029","76", "0.0514","59", "0.042","58", "0.092","57", "0.1243","19", "0.1819","56", "0.2222","18", "0.0492","15", "0.1185","16", "0.0452","12", "0.0259","64", "0.3718","65", "0.4375","62", "0.487","63", "0.0178","99", "0.5261","61", "0.1116","100", "0.052","98", "0.3959","49", "0.0278","97", "0.0076","48", "2.0E-4","96", "0.037","95", "0.0","94", "0.405","45", "0.2249","93", "0.1401","44", "0.3063","92", "0.2351","47", "0.0","91", "0.5516","46", "0.2209","90", "0.5214","51", "0.0","52", "0.1534","53", "0.0","54", "0.3419",
});
grammar_AP.put(2, new String[] {
"78", "0.2962","77", "0.2647","35", "0.0020","36", "0.0040","33", "0.3514","39", "0.0371","38", "0.0","43", "0.0678","42", "0.254","41", "0.1051","40", "1.0E-4","82", "0.2937","83", "0.1807","80", "0.0882","87", "0.2799","84", "0.4044","85", "0.0614","67", "0.1455","66", "0.0128","69", "0.0","68", "0.6555","23", "0.1562","26", "0.0288","28", "0.4664","29", "0.0994","2", "0.0397","30", "1.0E-4","6", "0.0405","5", "0.0015","32", "0.3421","70", "0.0248","9", "0.3192","71", "0.5066","72", "0.3865","73", "0.0039","74", "0.0672","75", "0.0030","76", "0.2454","59", "0.0342","58", "0.092","57", "0.1478","19", "0.2888","56", "0.2765","18", "0.0538","15", "0.398","16", "0.0498","12", "0.0338","64", "0.4185","65", "0.4425","62", "0.5036","63", "0.062","99", "0.305","61", "0.1279","100", "0.0424","98", "0.383","49", "0.0743","97", "0.0108","48", "0.0014","96", "0.0332","95", "0.0010","94", "0.4198","45", "0.1729","93", "0.129","44", "0.3489","92", "0.2401","47", "0.0","91", "0.5315","46", "0.2283","90", "0.4945","51", "0.0","52", "0.1534","53", "0.0","54", "0.534",
});
Nbest_AP.put(0, new String[] {
"78", "0.2782","77", "0.3064","35", "0.0026","36", "0.0032","33", "0.3505","39", "0.1631","38", "0.0","43", "0.054","42", "0.1508","41", "0.2385","40", "1.0E-4","82", "0.2901","83", "0.1208","80", "0.0274","87", "0.297","84", "0.1132","85", "0.0529","67", "0.1098","66", "0.0071","69", "0.0","68", "0.6759","23", "0.15","26", "0.1279","28", "0.0868","29", "0.0379","2", "0.0077","30", "0.0349","6", "0.068","5", "0.0","32", "0.3927","70", "0.0363","9", "0.31","71", "0.4973","72", "0.1549","73", "0.0294","74", "0.073","75", "0.0041","76", "0.0165","59", "0.0182","58", "0.0868","57", "0.3707","19", "0.2208","56", "0.1549","18", "0.0304","15", "0.4959","16", "0.057","12", "0.0449","64", "0.5126","65", "0.4075","62", "0.4841","63", "0.0026","99", "0.0673","61", "0.0138","100", "0.0067","98", "0.4255","49", "0.0","97", "4.0E-4","48", "0.0","96", "0.0266","95", "0.0","94", "0.505","45", "0.273","93", "0.0517","44", "0.3197","92", "0.2045","47", "0.0","91", "0.6822","46", "0.1578","90", "0.5209","51", "0.0","52", "0.1534","53", "0.0","54", "0.0754",
});
Nbest_AP.put(1, new String[] {
"78", "0.2782","77", "0.3064","35", "0.0026","36", "0.0032","33", "0.3505","39", "0.1631","38", "0.0","43", "0.054","42", "0.1508","41", "0.2385","40", "1.0E-4","82", "0.2901","83", "0.1208","80", "0.0274","87", "0.297","84", "0.1132","85", "0.0529","67", "0.1098","66", "0.0071","69", "0.0","68", "0.6759","23", "0.15","26", "0.1279","28", "0.0868","29", "0.0143","2", "0.0086","30", "0.0144","6", "0.0556","5", "0.0","32", "0.3927","70", "0.0363","9", "0.31","71", "0.4973","72", "0.1549","73", "0.0294","74", "0.073","75", "0.0041","76", "0.0314","59", "0.0182","58", "0.0868","57", "0.3707","19", "0.2208","56", "0.1549","18", "0.0304","15", "0.4959","16", "0.0469","12", "0.0449","64", "0.5126","65", "0.4075","62", "0.4841","63", "0.0067","99", "0.0673","61", "0.0138","100", "0.0067","98", "0.4255","49", "0.0","97", "4.0E-4","48", "0.0","96", "0.0296","95", "0.0","94", "0.505","45", "0.273","93", "0.0517","44", "0.3197","92", "0.2045","47", "0.0","91", "0.6822","46", "0.1578","90", "0.5209","51", "0.0","52", "0.1534","53", "0.0","54", "0.0876",
});
Nbest_AP.put(2, new String[] {
"78", "0.2782","77", "0.3064","35", "0.0026","36", "0.0032","33", "0.3505","39", "0.1631","38", "0.0","43", "0.054","42", "0.1508","41", "0.2385","40", "1.0E-4","82", "0.2901","83", "0.1208","80", "0.0274","87", "0.297","84", "0.1132","85", "0.0529","67", "0.1098","66", "0.0071","69", "0.0","68", "0.6759","23", "0.15","26", "0.1279","28", "0.0868","29", "0.0353","2", "0.0077","30", "0.0301","6", "0.0185","5", "0.0","32", "0.3927","70", "0.0363","9", "0.31","71", "0.4973","72", "0.1549","73", "0.0294","74", "0.073","75", "0.0041","76", "0.0415","59", "0.0182","58", "0.0868","57", "0.3707","19", "0.2208","56", "0.1549","18", "0.0304","15", "0.4959","16", "0.057","12", "0.0449","64", "0.5126","65", "0.4075","62", "0.4841","63", "0.0079","99", "0.0673","61", "0.0138","100", "0.0067","98", "0.4255","49", "0.0","97", "4.0E-4","48", "0.0","96", "0.0265","95", "0.0","94", "0.505","45", "0.273","93", "0.0517","44", "0.3197","92", "0.2045","47", "0.0","91", "0.6822","46", "0.1578","90", "0.5209","51", "0.0","52", "0.1534","53", "0.0","54", "0.0754",
});
Onebest_AP.put(0, new String[] {
"78", "0.2793","77", "0.2578","35", "0.0026","36", "0.0032","33", "0.3129","39", "0.1864","38", "0.0","43", "0.0478","42", "0.1508","41", "0.2377","40", "1.0E-4","82", "0.242","83", "0.1208","80", "0.0274","87", "0.297","84", "0.2594","85", "0.0534","67", "0.1098","66", "0.0064","69", "0.0","68", "0.6808","23", "0.1517","26", "0.1375","28", "0.0868","29", "0.0279","2", "0.0104","30", "0.0","6", "0.0138","5", "0.0","32", "0.4093","70", "0.0365","9", "0.2594","71", "0.4512","72", "0.1549","73", "0.0294","74", "0.0737","75", "0.0041","76", "0.0223","59", "0.0211","58", "0.0961","57", "0.3423","19", "0.2208","56", "0.2019","18", "0.0148","15", "0.4841","16", "0.0451","12", "0.0439","64", "0.6059","65", "0.4075","62", "0.4604","63", "6.0E-4","99", "0.0468","61", "0.0169","100", "0.0064","98", "0.4255","49", "0.0","97", "0.0010","48", "0.0","96", "0.027","95", "0.0","94", "0.3776","45", "0.3499","93", "0.0114","44", "0.1842","92", "0.2045","47", "0.0","91", "0.6822","46", "0.1578","90", "0.5209","51", "0.0","52", "0.1534","53", "0.0","54", "0.0375",
});
Onebest_AP.put(1, new String[] {
"78", "0.2793","77", "0.2578","35", "0.0026","36", "0.0032","33", "0.3129","39", "0.1864","38", "0.0","43", "0.0478","42", "0.1508","41", "0.2377","40", "1.0E-4","82", "0.242","83", "0.1208","80", "0.0274","87", "0.297","84", "0.2594","85", "0.0534","67", "0.1098","66", "0.0064","69", "0.0","68", "0.6808","23", "0.1517","26", "0.1375","28", "0.0868","29", "0.0044","2", "0.0104","30", "0.0","6", "0.0032","5", "0.0","32", "0.4093","70", "0.0365","9", "0.2594","71", "0.4512","72", "0.1549","73", "0.0294","74", "0.0737","75", "0.0041","76", "0.0402","59", "0.0211","58", "0.0961","57", "0.3423","19", "0.2208","56", "0.2019","18", "0.0148","15", "0.4841","16", "0.0505","12", "0.0439","64", "0.6059","65", "0.4075","62", "0.4604","63", "0.0112","99", "0.0468","61", "0.0169","100", "0.0064","98", "0.4255","49", "0.0","97", "0.0010","48", "0.0","96", "0.027","95", "0.0","94", "0.3776","45", "0.3499","93", "0.0114","44", "0.1842","92", "0.2045","47", "0.0","91", "0.6822","46", "0.1578","90", "0.5209","51", "0.0","52", "0.1534","53", "0.0","54", "0.0814",
});
Onebest_AP.put(2, new String[] {
"78", "0.2793","77", "0.2578","35", "0.0026","36", "0.0032","33", "0.3129","39", "0.1864","38", "0.0","43", "0.0478","42", "0.1508","41", "0.2377","40", "1.0E-4","82", "0.242","83", "0.1208","80", "0.0274","87", "0.297","84", "0.2594","85", "0.0534","67", "0.1098","66", "0.0064","69", "0.0","68", "0.6808","23", "0.1517","26", "0.1375","28", "0.0868","29", "0.0044","2", "0.0101","30", "0.0","6", "0.0032","5", "0.0","32", "0.4093","70", "0.0365","9", "0.2594","71", "0.4512","72", "0.1549","73", "0.0294","74", "0.0737","75", "0.0041","76", "0.0571","59", "0.0211","58", "0.0961","57", "0.3423","19", "0.2208","56", "0.2019","18", "0.0148","15", "0.4841","16", "0.0486","12", "0.0439","64", "0.6059","65", "0.4075","62", "0.4604","63", "0.0148","99", "0.0468","61", "0.0169","100", "0.0064","98", "0.4255","49", "0.0","97", "0.0010","48", "0.0","96", "0.027","95", "0.0","94", "0.3776","45", "0.3499","93", "0.0114","44", "0.1842","92", "0.2045","47", "0.0","91", "0.6822","46", "0.1578","90", "0.5209","51", "0.0","52", "0.1534","53", "0.0","54", "0.0814",
});
Interp_AP.put(0, new String[] {
"78", "0.3117","77", "0.2626","35", "0.0019","36", "0.0032","33", "0.3612","39", "0.1128","38", "0.0","43", "0.0684","42", "0.2023","41", "0.1481","40", "1.0E-4","82", "0.3154","83", "0.1547","80", "0.0874","87", "0.291","84", "0.2803","85", "0.0841","67", "0.1293","66", "0.0092","69", "0.0","68", "0.6616","23", "0.1548","26", "0.122","28", "0.4529","29", "0.1322","2", "0.2133","30", "0.2369","6", "0.0929","5", "0.0","32", "0.3759","70", "0.041","9", "0.1922","71", "0.5078","72", "0.3931","73", "0.0065","74", "0.0677","75", "0.0033","76", "0.1293","59", "0.0344","58", "0.0935","57", "0.3787","19", "0.2659","56", "0.242","18", "0.0453","15", "0.4009","16", "0.0483","12", "0.0455","64", "0.48","65", "0.448","62", "0.5043","63", "0.0129","99", "0.264","61", "0.0522","100", "0.0413","98", "0.3921","49", "0.0033","97", "0.0021","48", "2.0E-4","96", "0.0292","95", "6.0E-4","94", "0.4622","45", "0.2911","93", "0.0945","44", "0.2969","92", "0.2217","47", "0.0","91", "0.6577","46", "0.2068","90", "0.4516","51", "0.0","52", "0.1495","53", "0.1198","54", "0.269"
});
Interp_AP.put(1, new String[] {
"78", "0.2957","77", "0.2206","35", "0.0019","36", "0.0025","33", "0.3588","39", "0.1034","38", "0.0","43", "0.0697","42", "0.2007","41", "0.1322","40", "1.0E-4","82", "0.313","83", "0.1344","80", "0.0756","87", "0.2898","84", "0.2934","85", "0.0794","67", "0.1297","66", "0.0090","69", "0.0","68", "0.663","23", "0.1565","26", "0.1255","28", "0.4333","29", "0.1258","2", "0.1445","30", "0.2712","6", "0.0584","5", "0.0","32", "0.3683","70", "0.0357","9", "0.179","71", "0.5073","72", "0.3882","73", "0.0065","74", "0.0661","75", "0.0033","76", "0.0722","59", "0.0361","58", "0.0923","57", "0.3411","19", "0.2409","56", "0.2545","18", "0.0424","15", "0.3543","16", "0.0428","12", "0.0454","64", "0.4586","65", "0.4451","62", "0.5002","63", "0.011","99", "0.1807","61", "0.0256","100", "0.0299","98", "0.3927","49", "0.0012","97", "0.0021","48", "0.0","96", "0.0316","95", "1.0E-4","94", "0.4673","45", "0.2908","93", "0.0959","44", "0.2853","92", "0.2209","47", "0.0","91", "0.6526","46", "0.2112","90", "0.4519","51", "0.0","52", "0.1495","53", "0.1198","54", "0.2068"
});
Interp_AP.put(2, new String[] {
"78", "0.3167","77", "0.2599","35", "0.0019","36", "0.0033","33", "0.3573","39", "0.1078","38", "0.0","43", "0.0679","42", "0.2039","41", "0.147","40", "1.0E-4","82", "0.3175","83", "0.1541","80", "0.082","87", "0.291","84", "0.257","85", "0.0732","67", "0.1302","66", "0.0092","69", "0.0","68", "0.6626","23", "0.1562","26", "0.125","28", "0.4415","29", "0.1302","2", "0.1573","30", "0.2587","6", "0.0582","5", "0.0","32", "0.3711","70", "0.04","9", "0.2236","71", "0.5076","72", "0.383","73", "0.0065","74", "0.0679","75", "0.0034","76", "0.1072","59", "0.0353","58", "0.0933","57", "0.378","19", "0.2664","56", "0.2486","18", "0.0453","15", "0.4233","16", "0.0514","12", "0.0459","64", "0.4827","65", "0.4462","62", "0.5034","63", "0.0139","99", "0.267","61", "0.0392","100", "0.0264","98", "0.3932","49", "0.0026","97", "0.0025","48", "2.0E-4","96", "0.03","95", "3.0E-4","94", "0.4591","45", "0.3061","93", "0.0957","44", "0.2947","92", "0.2229","47", "0.0","91", "0.6519","46", "0.2098","90", "0.4246","51", "0.0","52", "0.1495","53", "0.1198","54", "0.2776"
});
}
private static String[] baseline_token_AP = new String[] {
"78", "0.2855","77", "0.1152","35", "0.0021","36", "0.0028","33", "0.354","39", "0.0542","38", "0.0","43", "0.0723","42", "0.2187","41", "0.0712","40", "0.0","82", "0.173","83", "0.1171","80", "0.0968","87", "0.2274","84", "0.3617","85", "0.06","67", "0.1404","66", "0.0118","69", "0.0","68", "0.6597","23", "0.1503","26", "0.0056","28", "0.4382","29", "0.1725","2", "0.1554","30", "0.1201","6", "0.0755","5", "0.0024","32", "0.1401","70", "0.0015","9", "0.1323","71", "0.4966","72", "0.3776","73", "7.0E-4","74", "0.0384","75", "0.0041","76", "0.3721","59", "0.0329","58", "0.0956","57", "0.1211","19", "0.2074","56", "0.1257","18", "0.0","15", "0.2479","16", "0.0552","12", "0.0476","64", "0.4514","65", "0.4463","62", "0.4401","63", "0.0063","99", "0.3902","61", "0.0106","100", "0.1015","98", "0.3637","49", "0.0","97", "0.0062","48", "2.0E-4","96", "0.0306","95", "7.0E-4","94", "0.1811","45", "0.1176","93", "0.1338","44", "0.026","92", "0.212","47", "0.0","91", "0.6703","46", "0.2169","90", "0.0233","51", "0.0","52", "0.1413","53", "0.1198","54", "0.1988",
};
public EnZh_NTCIR8() {
super();
qe = new QueryEngine();
}
@Test
public void runRegressions() throws Exception {
initialize();
runRegression(0); // "one2none"
runRegression(1); // "one2one"
runRegression(2); // "one2many"
verifyAllResults(qe.getModels(), qe.getAllResults(), qe.getDocnoMapping(),
new Qrels("data/"+ PATH + "/qrels."+ PATH + ".txt"));
}
public void runRegression(int heuristic) throws Exception {
/////// baseline-token
Configuration conf = RunQueryEngine.parseArgs(new String[] {
"--xml", "data/"+ PATH + "/run_en-" + LANGUAGE + ".token.xml",
"--queries_path", "data/"+ PATH + "/cdec/title_en-" + LANGUAGE + "-trans10-filtered.xml", "--one2many", heuristic + "", "--is_stemming", "--is_doc_stemmed" });
FileSystem fs = FileSystem.getLocal(conf);
// no need to repeat token-based case for other heuristics
if (heuristic == 0) {
qe.init(conf, fs);
qe.runQueries(conf);
}
/////// 1-best
conf = RunQueryEngine.parseArgs(new String[] {
"--xml", "data/"+ PATH + "/run_en-" + LANGUAGE + ".1best.xml",
"--queries_path", "data/"+ PATH + "/cdec/title_en-" + LANGUAGE + "-trans1-filtered.xml", "--one2many", heuristic + "", "--is_stemming", "--is_doc_stemmed" });
qe.init(conf, fs);
qe.runQueries(conf);
/////// grammar
conf = RunQueryEngine.parseArgs(new String[] {
"--xml", "data/"+ PATH + "/run_en-" + LANGUAGE + ".grammar.xml",
"--queries_path", "data/"+ PATH + "/cdec/title_en-" + LANGUAGE + "-trans10-filtered.xml", "--one2many", heuristic + "", "--is_stemming", "--is_doc_stemmed" });
qe.init(conf, fs);
qe.runQueries(conf);
/////// 10-best
conf = RunQueryEngine.parseArgs(new String[] {
"--xml", "data/"+ PATH + "/run_en-" + LANGUAGE + ".10best.xml",
"--queries_path", "data/"+ PATH + "/cdec/title_en-" + LANGUAGE + "-trans10-filtered.xml", "--one2many", heuristic + "", "--is_stemming", "--is_doc_stemmed" });
qe.init(conf, fs);
qe.runQueries(conf);
/////// interp
conf = RunQueryEngine.parseArgs(new String[] {
"--xml", "data/"+ PATH + "/run_en-" + LANGUAGE + ".interp.xml",
"--queries_path", "data/"+ PATH + "/cdec/title_en-" + LANGUAGE + "-trans10-filtered.xml", "--one2many", heuristic + "", "--is_stemming", "--is_doc_stemmed" });
qe.init(conf, fs);
qe.runQueries(conf);
}
public static void verifyAllResults(Set<String> models,
Map<String, Map<String, Accumulator[]>> results, DocnoMapping mapping, Qrels qrels) {
Map<String, GroundTruth> g = Maps.newHashMap();
g.put("en-" + LANGUAGE + ".token_10-0-100-100_0", new GroundTruth(Metric.AP, numTopics, baseline_token_AP, expectedTokenMAP));
for (int heuristic=0; heuristic <= 2; heuristic++) {
g.put("en-" + LANGUAGE + ".grammar_10-0-0-100_" + heuristic, new GroundTruth(Metric.AP, numTopics, grammar_AP.get(heuristic), expectedMAPs.get(heuristic)[0]));
g.put("en-" + LANGUAGE + ".1best_1-0-0-100_" + heuristic, new GroundTruth(Metric.AP, numTopics, Onebest_AP.get(heuristic), expectedMAPs.get(heuristic)[1]));
g.put("en-" + LANGUAGE + ".10best_10-100-0-100_" + heuristic, new GroundTruth(Metric.AP, numTopics, Nbest_AP.get(heuristic), expectedMAPs.get(heuristic)[2]));
g.put("en-" + LANGUAGE + ".interp_10-30-30-100_" + heuristic, new GroundTruth(Metric.AP, numTopics, Interp_AP.get(heuristic), expectedMAPs.get(heuristic)[3]));
}
for (String model : models) {
System.err.println("Verifying results of model \"" + model + "\"");
GroundTruth groundTruth = g.get(model);
Map<String, Accumulator[]> result = results.get(model);
groundTruth.verify(result, mapping, qrels);
System.err.println("Done!");
}
}
public static junit.framework.Test suite() {
return new JUnit4TestAdapter(EnZh_NTCIR8.class);
}
public static void main(String[] args) {
grammar_AP.put(2, new String[] {
"78", "0.2962","77", "0.2647","35", "0.0020","36", "0.0040","33", "0.3514","39", "0.0371","38", "0.0","43", "0.0678","42", "0.254","41", "0.1051","40", "1.0E-4","82", "0.2937","83", "0.1807","80", "0.0882","87", "0.2799","84", "0.4044","85", "0.0614","67", "0.1455","66", "0.0128","69", "0.0","68", "0.6555","23", "0.1562","26", "0.0288","28", "0.4664","29", "0.0994","2", "0.0397","30", "1.0E-4","6", "0.0405","5", "0.0015","32", "0.3421","70", "0.0248","9", "0.3192","71", "0.5066","72", "0.3865","73", "0.0039","74", "0.0672","75", "0.0030","76", "0.2454","59", "0.0342","58", "0.092","57", "0.1478","19", "0.2888","56", "0.2765","18", "0.0538","15", "0.398","16", "0.0498","12", "0.0338","64", "0.4185","65", "0.4425","62", "0.5036","63", "0.062","99", "0.305","61", "0.1279","100", "0.0424","98", "0.383","49", "0.0743","97", "0.0108","48", "0.0014","96", "0.0332","95", "0.0010","94", "0.4198","45", "0.1729","93", "0.129","44", "0.3489","92", "0.2401","47", "0.0","91", "0.5315","46", "0.2283","90", "0.4945","51", "0.0","52", "0.1534","53", "0.0","54", "0.534",
});
Onebest_AP.put(2, new String[] {
"78", "0.2793","77", "0.2578","35", "0.0026","36", "0.0032","33", "0.3129","39", "0.1864","38", "0.0","43", "0.0478","42", "0.1508","41", "0.2377","40", "1.0E-4","82", "0.242","83", "0.1208","80", "0.0274","87", "0.297","84", "0.2594","85", "0.0534","67", "0.1098","66", "0.0064","69", "0.0","68", "0.6808","23", "0.1517","26", "0.1375","28", "0.0868","29", "0.0044","2", "0.0101","30", "0.0","6", "0.0032","5", "0.0","32", "0.4093","70", "0.0365","9", "0.2594","71", "0.4512","72", "0.1549","73", "0.0294","74", "0.0737","75", "0.0041","76", "0.0571","59", "0.0211","58", "0.0961","57", "0.3423","19", "0.2208","56", "0.2019","18", "0.0148","15", "0.4841","16", "0.0486","12", "0.0439","64", "0.6059","65", "0.4075","62", "0.4604","63", "0.0148","99", "0.0468","61", "0.0169","100", "0.0064","98", "0.4255","49", "0.0","97", "0.0010","48", "0.0","96", "0.027","95", "0.0","94", "0.3776","45", "0.3499","93", "0.0114","44", "0.1842","92", "0.2045","47", "0.0","91", "0.6822","46", "0.1578","90", "0.5209","51", "0.0","52", "0.1534","53", "0.0","54", "0.0814",
});
Nbest_AP.put(2, new String[] {
"78", "0.2782","77", "0.3064","35", "0.0026","36", "0.0032","33", "0.3505","39", "0.1631","38", "0.0","43", "0.054","42", "0.1508","41", "0.2385","40", "1.0E-4","82", "0.2901","83", "0.1208","80", "0.0274","87", "0.297","84", "0.1132","85", "0.0529","67", "0.1098","66", "0.0071","69", "0.0","68", "0.6759","23", "0.15","26", "0.1279","28", "0.0868","29", "0.0353","2", "0.0077","30", "0.0301","6", "0.0185","5", "0.0","32", "0.3927","70", "0.0363","9", "0.31","71", "0.4973","72", "0.1549","73", "0.0294","74", "0.073","75", "0.0041","76", "0.0415","59", "0.0182","58", "0.0868","57", "0.3707","19", "0.2208","56", "0.1549","18", "0.0304","15", "0.4959","16", "0.057","12", "0.0449","64", "0.5126","65", "0.4075","62", "0.4841","63", "0.0079","99", "0.0673","61", "0.0138","100", "0.0067","98", "0.4255","49", "0.0","97", "4.0E-4","48", "0.0","96", "0.0265","95", "0.0","94", "0.505","45", "0.273","93", "0.0517","44", "0.3197","92", "0.2045","47", "0.0","91", "0.6822","46", "0.1578","90", "0.5209","51", "0.0","52", "0.1534","53", "0.0","54", "0.0754",
});
Interp_AP.put(2, new String[] {
"78", "0.3167","77", "0.2599","35", "0.0019","36", "0.0033","33", "0.3573","39", "0.1078","38", "0.0","43", "0.0679","42", "0.2039","41", "0.147","40", "1.0E-4","82", "0.3175","83", "0.1541","80", "0.082","87", "0.291","84", "0.257","85", "0.0732","67", "0.1302","66", "0.0092","69", "0.0","68", "0.6626","23", "0.1562","26", "0.125","28", "0.4415","29", "0.1302","2", "0.1573","30", "0.2587","6", "0.0582","5", "0.0","32", "0.3711","70", "0.04","9", "0.2236","71", "0.5076","72", "0.383","73", "0.0065","74", "0.0679","75", "0.0034","76", "0.1072","59", "0.0353","58", "0.0933","57", "0.378","19", "0.2664","56", "0.2486","18", "0.0453","15", "0.4233","16", "0.0514","12", "0.0459","64", "0.4827","65", "0.4462","62", "0.5034","63", "0.0139","99", "0.267","61", "0.0392","100", "0.0264","98", "0.3932","49", "0.0026","97", "0.0025","48", "2.0E-4","96", "0.03","95", "3.0E-4","94", "0.4591","45", "0.3061","93", "0.0957","44", "0.2947","92", "0.2229","47", "0.0","91", "0.6519","46", "0.2098","90", "0.4246","51", "0.0","52", "0.1495","53", "0.1198","54", "0.2776"
});
HMapSFW tenbestAPMap = array2Map(Nbest_AP.get(2));
HMapSFW onebestAPMap = array2Map(Onebest_AP.get(2));
HMapSFW grammarAPMap = array2Map(grammar_AP.get(2));
HMapSFW tokenAPMap = array2Map(baseline_token_AP);
HMapSFW gridAPMap = array2Map(Interp_AP.get(2));
System.out.println(countNumberOfImprovedTopics(tokenAPMap, gridAPMap));
System.out.println(countNumberOfImprovedTopics(tokenAPMap, tenbestAPMap));
System.out.println(countNumberOfImprovedTopics(tokenAPMap, onebestAPMap));
System.out.println(countNumberOfImprovedTopics(tokenAPMap, grammarAPMap));
System.out.println(countNumberOfNegligibleTopics(tokenAPMap, gridAPMap));
System.out.println(countNumberOfNegligibleTopics(tokenAPMap, tenbestAPMap));
System.out.println(countNumberOfNegligibleTopics(tokenAPMap, onebestAPMap));
System.out.println(countNumberOfNegligibleTopics(tokenAPMap, grammarAPMap));
}
private static int countNumberOfImprovedTopics(HMapSFW tokenAPMap, HMapSFW gridAPMap) {
int cnt = 0;
for (String key : tokenAPMap.keySet()) {
float difference = gridAPMap.get(key) - tokenAPMap.get(key);
if ( difference > 0.001 ) {
cnt++;
}
}
return cnt;
}
private static int countNumberOfNegligibleTopics(HMapSFW tokenAPMap, HMapSFW gridAPMap) {
int cnt = 0;
for (String key : tokenAPMap.keySet()) {
float difference = gridAPMap.get(key) - tokenAPMap.get(key);
if ( difference > -0.001 && difference < 0.001 ) {
cnt++;
}
}
return cnt;
}
private static HMapSFW array2Map(String[] array) {
HMapSFW map = new HMapSFW();
for ( int i = 0; i < array.length; i += 2 ) {
map.put(array[i], Float.parseFloat(array[i+1]));
}
return map;
}
}