package ivory.regression.cikm2010;
import ivory.core.eval.Qrels;
import ivory.regression.GroundTruth;
import ivory.regression.GroundTruth.Metric;
import ivory.smrf.retrieval.Accumulator;
import ivory.smrf.retrieval.BatchQueryRunner;
import java.util.HashMap;
import java.util.Map;
import junit.framework.JUnit4TestAdapter;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.log4j.Logger;
import org.junit.Test;
import edu.umd.cloud9.collection.DocnoMapping;
/* Note: different metrics are optimized separately */
public class Wt10g_Desc_Indep {
private static final Logger sLogger = Logger.getLogger(Wt10g_Desc_Indep.class);
private static String[] x10_rawAP = new String[] {
"501","0.4911","502","0.1483","503","0.1958","504","0.3081","505","0.3241",
"506","0.1699","507","0.0950","508","0.2180","509","0.2241","510","0.5216",
"511","0.1821","512","0.1451","513","0.2027","514","0.0928","515","0.0755",
"516","0.0945","517","0.0617","518","0.0649","519","0.3050","520","0.1654",
"521","0.0067","522","0.1813","523","0.3767","524","0.1031","525","0.0425",
"526","0.0963","527","0.0965","528","0.3805","529","0.3959","530","0.4780",
"531","0.0096","532","0.2586","533","0.2020","534","0.0064","535","0.0113",
"536","0.0505","537","0.0001","538","0.0761","539","0.0434","540","0.0269",
"541","0.0178","542","0.2026","543","0.0007","544","0.4864","545","0.3351",
"546","0.1294","547","0.0000","548","0.3333","549","0.2715","550","0.0618"};
private static String[] x15_rawAP = new String[] {
"501","0.5363","502","0.1072","503","0.1373","504","0.2698","505","0.3827",
"506","0.1277","507","0.1013","508","0.2605","509","0.3505","510","0.4973",
"511","0.2912","512","0.1872","513","0.2100","514","0.0761","515","0.1835",
"516","0.0856","517","0.0673","518","0.0099","519","0.2923","520","0.1804",
"521","0.0070","522","0.2517","523","0.2519","524","0.1858","525","0.1442",
"526","0.0881","527","0.2569","528","0.3880","529","0.3090","530","0.3757",
"531","0.1016","532","0.2685","533","0.2094","534","0.0062","535","0.0264",
"536","0.2520","537","0.0062","538","0.0459","539","0.0457","540","0.0209",
"541","0.0112","542","0.1292","543","0.0010","544","0.4085","545","0.2596",
"546","0.1369","547","0.0203","548","0.2917","549","0.3421","550","0.0994"};
private static String[] x20_rawAP = new String[] {
"501","0.5140","502","0.0766","503","0.1173","504","0.2495","505","0.3840",
"506","0.1027","507","0.0686","508","0.2357","509","0.3400","510","0.5346",
"511","0.3609","512","0.1851","513","0.2352","514","0.0850","515","0.2824",
"516","0.0639","517","0.0607","518","0.0173","519","0.2652","520","0.1429",
"521","0.0079","522","0.3338","523","0.2554","524","0.1059","525","0.1423",
"526","0.0792","527","0.2356","528","0.5256","529","0.2834","530","0.2849",
"531","0.1087","532","0.2700","533","0.2030","534","0.0050","535","0.0181",
"536","0.2612","537","0.0035","538","0.0406","539","0.1202","540","0.0225",
"541","0.2263","542","0.1652","543","0.0010","544","0.3919","545","0.3624",
"546","0.1431","547","0.1605","548","0.2917","549","0.3326","550","0.0830"};
private static String[] x25_rawAP = new String[] {
"501","0.5278","502","0.0766","503","0.1143","504","0.2495","505","0.4366",
"506","0.1027","507","0.0966","508","0.2489","509","0.3340","510","0.5904",
"511","0.3730","512","0.1851","513","0.2302","514","0.1033","515","0.2875",
"516","0.0639","517","0.0595","518","0.0433","519","0.2688","520","0.1637",
"521","0.0319","522","0.3525","523","0.2341","524","0.1107","525","0.1490",
"526","0.0792","527","0.3179","528","0.5219","529","0.2904","530","0.2943",
"531","0.1124","532","0.2789","533","0.2161","534","0.0040","535","0.0173",
"536","0.2611","537","0.0035","538","0.0427","539","0.1160","540","0.0220",
"541","0.2231","542","0.1599","543","0.0010","544","0.3597","545","0.4644",
"546","0.1612","547","0.1605","548","0.2429","549","0.3321","550","0.1224"};
private static String[] x30_rawAP = new String[] {
"501","0.5290","502","0.0950","503","0.1086","504","0.2487","505","0.4585",
"506","0.1027","507","0.0952","508","0.2580","509","0.3286","510","0.6282",
"511","0.3730","512","0.1969","513","0.2302","514","0.1037","515","0.2886",
"516","0.0639","517","0.0588","518","0.0458","519","0.2682","520","0.1660",
"521","0.0236","522","0.3577","523","0.2341","524","0.1117","525","0.1471",
"526","0.0792","527","0.3754","528","0.5218","529","0.2756","530","0.3008",
"531","0.1192","532","0.2817","533","0.2150","534","0.0039","535","0.0326",
"536","0.2604","537","0.0045","538","0.0413","539","0.1160","540","0.0219",
"541","0.2231","542","0.1510","543","0.0010","544","0.3643","545","0.5137",
"546","0.1612","547","0.1605","548","0.2667","549","0.3293","550","0.1299"};
private static String[] x35_rawAP = new String[] {
"501","0.5278","502","0.0950","503","0.1352","504","0.2487","505","0.4585",
"506","0.1027","507","0.0948","508","0.2630","509","0.3311","510","0.6273",
"511","0.3675","512","0.1969","513","0.2430","514","0.1069","515","0.2908",
"516","0.0639","517","0.0590","518","0.0527","519","0.2702","520","0.1664",
"521","0.0243","522","0.3323","523","0.2224","524","0.1133","525","0.1384",
"526","0.0792","527","0.3745","528","0.5217","529","0.2749","530","0.3061",
"531","0.1178","532","0.2754","533","0.2315","534","0.0044","535","0.0483",
"536","0.2625","537","0.0047","538","0.0394","539","0.0974","540","0.0217",
"541","0.2255","542","0.1538","543","0.0010","544","0.3670","545","0.5132",
"546","0.1744","547","0.1605","548","0.2667","549","0.3277","550","0.1236"};
private static String[] x40_rawAP = new String[] {
"501","0.5132","502","0.1154","503","0.1353","504","0.2454","505","0.4638",
"506","0.1027","507","0.0955","508","0.2914","509","0.3303","510","0.6273",
"511","0.4139","512","0.1979","513","0.2623","514","0.1062","515","0.2896",
"516","0.0640","517","0.0591","518","0.0537","519","0.2682","520","0.1661",
"521","0.0249","522","0.4165","523","0.2392","524","0.1136","525","0.1400",
"526","0.0774","527","0.3767","528","0.5217","529","0.2743","530","0.2839",
"531","0.1238","532","0.2782","533","0.2368","534","0.0044","535","0.0459",
"536","0.2520","537","0.0045","538","0.0400","539","0.1062","540","0.0217",
"541","0.2881","542","0.1640","543","0.0010","544","0.3680","545","0.5287",
"546","0.1837","547","0.1573","548","0.2429","549","0.3330","550","0.1242"};
private static String[] x45_rawAP = new String[] {
"501","0.5037","502","0.1154","503","0.1388","504","0.2454","505","0.4704",
"506","0.1027","507","0.0942","508","0.3106","509","0.3350","510","0.6292",
"511","0.4170","512","0.1979","513","0.2733","514","0.1065","515","0.3082",
"516","0.0640","517","0.0597","518","0.0510","519","0.2672","520","0.1664",
"521","0.0248","522","0.4170","523","0.2511","524","0.1109","525","0.1446",
"526","0.0774","527","0.3757","528","0.5216","529","0.2738","530","0.2708",
"531","0.1624","532","0.2781","533","0.2389","534","0.0044","535","0.0438",
"536","0.2517","537","0.0043","538","0.0406","539","0.1110","540","0.0218",
"541","0.3005","542","0.1873","543","0.0010","544","0.3687","545","0.5259",
"546","0.1708","547","0.1573","548","0.2250","549","0.3498","550","0.1239"};
private static String[] x50_rawAP = new String[] {
"501","0.4961","502","0.1154","503","0.1382","504","0.2427","505","0.4704",
"506","0.1027","507","0.0942","508","0.3070","509","0.3346","510","0.6223",
"511","0.4170","512","0.1979","513","0.2733","514","0.1035","515","0.3086",
"516","0.0640","517","0.0606","518","0.0496","519","0.2664","520","0.1667",
"521","0.0297","522","0.4170","523","0.2511","524","0.1097","525","0.1509",
"526","0.0774","527","0.3685","528","0.5216","529","0.2739","530","0.2668",
"531","0.1585","532","0.2758","533","0.2379","534","0.0044","535","0.0436",
"536","0.2536","537","0.0044","538","0.0406","539","0.1110","540","0.0218",
"541","0.3005","542","0.1873","543","0.0009","544","0.3684","545","0.5356",
"546","0.1708","547","0.1573","548","0.2250","549","0.3484","550","0.1237"};
@Test
public void runRegression() throws Exception {
Map<String, GroundTruth> g = new HashMap<String, GroundTruth>();
g.put("indep-x1.0", new GroundTruth("indep-x1.0", Metric.AP, 50, x10_rawAP, 0.1753f));
g.put("indep-x1.5", new GroundTruth("indep-x1.5", Metric.AP, 50, x15_rawAP, 0.1859f));
g.put("indep-x2.0", new GroundTruth("indep-x2.0", Metric.AP, 50, x20_rawAP, 0.1957f));
g.put("indep-x2.5", new GroundTruth("indep-x2.5", Metric.AP, 50, x25_rawAP, 0.2048f));
g.put("indep-x3.0", new GroundTruth("indep-x3.0", Metric.AP, 50, x30_rawAP, 0.2094f));
g.put("indep-x3.5", new GroundTruth("indep-x3.5", Metric.AP, 50, x35_rawAP, 0.2101f));
g.put("indep-x4.0", new GroundTruth("indep-x4.0", Metric.AP, 50, x40_rawAP, 0.2155f));
g.put("indep-x4.5", new GroundTruth("indep-x4.5", Metric.AP, 50, x45_rawAP, 0.2178f));
g.put("indep-x5.0", new GroundTruth("indep-x5.0", Metric.AP, 50, x50_rawAP, 0.2173f));
Qrels qrels = new Qrels("data/wt10g/qrels.wt10g.all");
String[] params = new String[] {
"data/wt10g/run.wt10g.CIKM2010.desc.indep.xml",
"data/wt10g/queries.wt10g.501-550.desc.xml" };
FileSystem fs = FileSystem.getLocal(new Configuration());
BatchQueryRunner qr = new BatchQueryRunner(params, fs);
long start = System.currentTimeMillis();
qr.runQueries();
long end = System.currentTimeMillis();
sLogger.info("Total query time: " + (end - start) + "ms");
DocnoMapping mapping = qr.getDocnoMapping();
for (String model : qr.getModels()) {
sLogger.info("Verifying results of model \"" + model + "\"");
Map<String, Accumulator[]> results = qr.getResults(model);
g.get(model).verify(results, mapping, qrels);
sLogger.info("Done!");
}
}
public static junit.framework.Test suite() {
return new JUnit4TestAdapter(Wt10g_Desc_Indep.class);
}
}