Package ivory.regression

Examples of ivory.regression.GroundTruth


  private static String[] bm25_wsd_rawAP = new String[] { "1", "0.5352", "10", "0.1917", "11", "0.4116", "12", "0.0966", "13", "0.0002", "14", "0.1139", "15", "0.3569", "16", "0.3284", "17", "0.2603", "18", "0.2706", "19", "0.0000", "2", "0.5062", "20", "0.0000", "21", "0.2752", "22", "0.4790", "23", "0.0538", "24", "0.1191", "25", "0.2733", "26", "0.0955", "27", "0.2817", "28", "0.3874", "29", "0.0274", "3", "0.0014", "30", "0.2503", "31", "0.4732", "32", "0.0420", "33", "0.4752", "34", "0.0020", "35", "0.4381", "36", "0.0938", "37", "0.0711", "38", "0.1018", "39", "0.1715", "4", "0.0258", "40", "0.1854", "41", "0.3584", "42", "0.1228", "43", "0.6643", "44", "0.0628", "45", "0.3177", "46", "0.7270", "47", "0.6123", "48", "0.1982", "49", "0.3494", "5", "0.2062", "50", "0.0711", "6", "0.1155", "7", "0.0242", "8", "0.0553", "9", "0.1714" };

  @Test
  public void runRegression() throws Exception {
    Map<String, GroundTruth> g = Maps.newHashMap();
    g.put("ql-base", new GroundTruth("ql-base", Metric.AP, 50, ql_base_rawAP, 0.2134f));
    g.put("ql-sd", new GroundTruth("ql-sd", Metric.AP, 50, ql_sd_rawAP, 0.2223f));
    g.put("ql-wsd", new GroundTruth("ql-wsd", Metric.AP, 50, ql_wsd_rawAP, 0.2283f));
    g.put("bm25-base", new GroundTruth("bm25-base", Metric.AP, 50, bm25_base_rawAP, 0.2167f));
    g.put("bm25-sd", new GroundTruth("bm25-sd", Metric.AP, 50, bm25_sd_rawAP, 0.2280f));
    g.put("bm25-wsd", new GroundTruth("bm25-wsd", Metric.AP, 50, bm25_wsd_rawAP, 0.2290f));

    Qrels qrels = new Qrels("data/clue/qrels.web09catB.txt");

    String[] params = new String[] {
        "data/clue/run.web09catB.all.spam.xml",
View Full Code Here


  public static void verifyAllResults(Set<String> models,
      Map<String, Map<String, Accumulator[]>> results, DocnoMapping mapping, Qrels qrels) {

    Map<String, GroundTruth> g = Maps.newHashMap();

    g.put("en-zh.token_0-0-0-0", new GroundTruth(Metric.AP, 73, baseline_token_p005_c95_Zh_NTCIR8_AP, 0.1497f));
    g.put("en-zh.phrase_10-0-0-100", new GroundTruth(Metric.AP, 73, phrase_p005_c95_Zh_NTCIR8_AP, 0.1873f));
    g.put("en-zh.1best_1-100-0-100", new GroundTruth(Metric.AP, 73, Onebest_p005_c95_Zh_NTCIR8_AP, 0.1519f));
    g.put("en-zh.10best_10-100-0-100", new GroundTruth(Metric.AP, 73, Nbest_p005_c95_Zh_NTCIR8_AP, 0.1707f));
    g.put("en-zh.gridbest_10-20-10-100", new GroundTruth(Metric.AP, 73, Gridbest_p005_c95_Zh_NTCIR8_AP, 0.1932f));   // nbest=0.2, bitext=0.1 scfg=0.7

    for (String model : models) {
      LOG.info("Verifying results of model \"" + model + "\"");

      g.get(model).verify(results.get(model), mapping, qrels);
View Full Code Here

  public static void verifyAllResults(Set<String> models,
      Map<String, Map<String, Accumulator[]>> results, DocnoMapping mapping, Qrels qrels) {

    Map<String, GroundTruth> g = Maps.newHashMap();

    g.put("en-ar.token_0-0-0-0", new GroundTruth(Metric.AP, 50, baseline_token_p005_c95_Ar_TREC02_AP, 0.2714f));
    g.put("en-ar.phrase_10-0-0-100", new GroundTruth(Metric.AP, 50, phrase_p005_c95_Ar_TREC02_AP, 0.3060f));
    g.put("en-ar.1best_1-100-0-100", new GroundTruth(Metric.AP, 50, Onebest_p005_c95_Ar_TREC02_AP, 0.2474f));
    g.put("en-ar.10best_10-100-0-100", new GroundTruth(Metric.AP, 50, Nbest_p005_c95_Ar_TREC02_AP, 0.2431f));
    g.put("en-ar.gridbest_10-0-0-100", new GroundTruth(Metric.AP, 50, Gridbest_p005_c95_Ar_TREC02_AP, 0.3060f));   // scfg=1.0

    for (String model : models) {
      LOG.info("Verifying results of model \"" + model + "\"");

      g.get(model).verify(results.get(model), mapping, qrels);
View Full Code Here

  public static void verifyAllResults(Set<String> models,
      Map<String, Map<String, Accumulator[]>> results, DocnoMapping mapping, Qrels qrels) {

    Map<String, GroundTruth> g = Maps.newHashMap();

    g.put("en-fr.token_0-0-0-0", new GroundTruth(Metric.AP, 50, baseline_token_p005_c95_Fr_CLEF06_AP, 0.2617f));
    g.put("en-fr.phrase_10-0-0-100", new GroundTruth(Metric.AP, 50, phrase_p005_c95_Fr_CLEF06_AP, 0.2868f));
    g.put("en-fr.1best_1-100-0-100", new GroundTruth(Metric.AP, 50, Onebest_p005_c95_Fr_CLEF06_AP, 0.2829f));
    g.put("en-fr.10best_10-100-0-100", new GroundTruth(Metric.AP, 50, Nbest_p005_c95_Fr_CLEF06_AP, 0.2979f));
    g.put("en-fr.gridbest_10-30-30-100", new GroundTruth(Metric.AP, 50, Gridbest_p005_c95_Fr_CLEF06_AP, 0.3084f));      // nbest=0.3, bitext=0.3 scfg=0.4

    for (String model : models) {
      LOG.info("Verifying results of model \"" + model + "\"");

      g.get(model).verify(results.get(model), mapping, qrels);
View Full Code Here

  @Test
  public void runRegression() throws Exception {
    Map<String, GroundTruth> g = new HashMap<String, GroundTruth>();

    g.put("Gov2-Cascade-0.9", new GroundTruth("Gov2-Cascade-0.9", Metric.NDCG20, 75, p9, 0.4457f));
    g.put("Gov2-Cascade-0.7", new GroundTruth("Gov2-Cascade-0.7", Metric.NDCG20, 75, p7, 0.4625f));
    g.put("Gov2-Cascade-0.5", new GroundTruth("Gov2-Cascade-0.5", Metric.NDCG20, 75, p5, 0.4634f));
    g.put("Gov2-Cascade-0.3", new GroundTruth("Gov2-Cascade-0.3", Metric.NDCG20, 75, p3, 0.4653f));
    g.put("Gov2-Cascade-0.1", new GroundTruth("Gov2-Cascade-0.1", Metric.NDCG20, 75, p1, 0.4745f));

    GradedQrels qrels = new GradedQrels("data/gov2/qrels.gov2.all");

    String[] params = new String[] {
        "data/gov2/run.gov2.SIGIR2011.varying.tradeoff.cascade.xml",
View Full Code Here

      "46", "0.8000", "47", "0.6000", "48", "0.1000", "49", "0.4000", "50", "0.0000" };

  @Test
  public void runRegression() throws Exception {
    Map<String, GroundTruth> g = Maps.newHashMap();
    g.put("UMHOO-BM25-catB", new GroundTruth(Metric.AP, 50, bm25_rawAP, 0.2051f));
    g.put("UMHOO-QL-catB", new GroundTruth(Metric.AP, 50, ql_rawAP, 0.1931f));

    Map<String, GroundTruth> h = Maps.newHashMap();
    h.put("UMHOO-BM25-catB", new GroundTruth(Metric.P10, 50, bm25_rawP10, 0.3720f));
    h.put("UMHOO-QL-catB", new GroundTruth(Metric.P10, 50, ql_rawP10, 0.3380f));

    Qrels qrels = new Qrels("data/clue/qrels.web09catB.txt");

    String[] params = new String[] {
        "data/clue/run.web09catB.xml",
View Full Code Here

  private static String[] bm25_wsd_rawAP = new String[] { "1", "0.5352", "10", "0.1917", "11", "0.4116", "12", "0.0966", "13", "0.0002", "14", "0.1139", "15", "0.3569", "16", "0.3284", "17", "0.2603", "18", "0.2706", "19", "0.0000", "2", "0.5062", "20", "0.0000", "21", "0.2752", "22", "0.4790", "23", "0.0538", "24", "0.1191", "25", "0.2733", "26", "0.0955", "27", "0.2817", "28", "0.3874", "29", "0.0274", "3", "0.0014", "30", "0.2503", "31", "0.4732", "32", "0.0420", "33", "0.4752", "34", "0.0020", "35", "0.4381", "36", "0.0938", "37", "0.0711", "38", "0.1018", "39", "0.1715", "4", "0.0258", "40", "0.1854", "41", "0.3584", "42", "0.1228", "43", "0.6643", "44", "0.0628", "45", "0.3177", "46", "0.7270", "47", "0.6123", "48", "0.1982", "49", "0.3494", "5", "0.2062", "50", "0.0711", "6", "0.1155", "7", "0.0242", "8", "0.0553", "9", "0.1714" };

  @Test
  public void runRegression() throws Exception {
    Map<String, GroundTruth> g = Maps.newHashMap();
    g.put("ql-base", new GroundTruth("ql-base", Metric.AP, 50, ql_base_rawAP, 0.2134f));
    g.put("ql-sd", new GroundTruth("ql-sd", Metric.AP, 50, ql_sd_rawAP, 0.2223f));
    g.put("ql-wsd", new GroundTruth("ql-wsd", Metric.AP, 50, ql_wsd_rawAP, 0.2283f));
    g.put("bm25-base", new GroundTruth("bm25-base", Metric.AP, 50, bm25_base_rawAP, 0.2167f));
    g.put("bm25-sd", new GroundTruth("bm25-sd", Metric.AP, 50, bm25_sd_rawAP, 0.2280f));
    g.put("bm25-wsd", new GroundTruth("bm25-wsd", Metric.AP, 50, bm25_wsd_rawAP, 0.2290f));

    Qrels qrels = new Qrels("data/clue/qrels.web09catB.txt");

    String[] params = new String[] {
        "data/clue/run.web09catB.all.spam.xml",
View Full Code Here

  public static void verifyAllResults(Set<String> models,
      Map<String, Map<String, Accumulator[]>> results, DocnoMapping mapping, Qrels qrels) {

    Map<String, GroundTruth> g = Maps.newHashMap();
    // One topic didn't contain qrels, so trec_eval only picked up 99.
    g.put("wt10g-dir-base", new GroundTruth(Metric.AP, 99, sDirBaseRawAP, 0.2093f));
    g.put("wt10g-bm25-base", new GroundTruth(Metric.AP, 99, sBm25BaseRawAP, 0.2105f));

    Map<String, GroundTruth> h = Maps.newHashMap();
    // One topic didn't contain qrels, so trec_eval only picked up 99.
    h.put("wt10g-dir-base", new GroundTruth(Metric.P10, 99, sDirBaseRawP10, 0.3131f));
    h.put("wt10g-bm25-base", new GroundTruth(Metric.P10, 99, sBm25BaseRawP10, 0.3202f));

    for (String model : models) {
      LOG.info("Verifying results of model \"" + model + "\"");

      Map<String, Accumulator[]> r = results.get(model);
View Full Code Here

  public static void verifyAllResults(Set<String> models,
          Map<String, Map<String, Accumulator[]>> results, DocnoMapping mapping, Qrels qrels) {

    Map<String, GroundTruth> g = Maps.newHashMap();
    // One topic didn't contain qrels, so trec_eval only picked up 49.
    g.put("genomics05-dir-base", new GroundTruth(Metric.AP, 49, sDirBaseRawAP, 0.2494f));
    g.put("genomics05-bm25-base", new GroundTruth(Metric.AP, 49, sBm25BaseRawAP, 0.2568f));

    Map<String, GroundTruth> h = Maps.newHashMap();
    // One topic didn't contain qrels, so trec_eval only picked up 49.
    h.put("genomics05-dir-base", new GroundTruth(Metric.P10, 49, sDirBaseRawP10, 0.4327f));
    h.put("genomics05-bm25-base", new GroundTruth(Metric.P10, 49, sBm25BaseRawP10, 0.4347f));

    for (String model : models) {
      LOG.info("Verifying results of model \"" + model + "\"");

      Map<String, Accumulator[]> r = results.get(model);
View Full Code Here

  public static void verifyAllResults(Set<String> models,
      Map<String, Map<String, Accumulator[]>> results, DocnoMapping mapping, Qrels qrels) {

    Map<String, GroundTruth> g = Maps.newHashMap();
    // One topic didn't contain qrels, so trec_eval only picked up 149.
    g.put("gov2-dir-base", new GroundTruth(Metric.AP, 149, sDirBaseRawAP, 0.3077f));
    g.put("gov2-dir-sd", new GroundTruth(Metric.AP, 149, sDirSDRawAP, 0.3239f));
    g.put("gov2-dir-fd", new GroundTruth(Metric.AP, 149, sDirFDRawAP, 0.3237f));
    g.put("gov2-bm25-base", new GroundTruth(Metric.AP, 149, sBm25BaseRawAP, 0.2999f));
    g.put("gov2-bm25-sd", new GroundTruth(Metric.AP, 149, sBm25SDRawAP, 0.3294f));
    g.put("gov2-bm25-fd", new GroundTruth(Metric.AP, 149, sBm25FDRawAP, 0.3295f));

    Map<String, GroundTruth> h = Maps.newHashMap();
    // One topic didn't contain qrels, so trec_eval only picked up 149.
    h.put("gov2-dir-base", new GroundTruth(Metric.P10, 149, sDirBaseRawP10, 0.5631f));
    h.put("gov2-dir-sd", new GroundTruth(Metric.P10, 149, sDirSDRawP10, 0.6007f));
    h.put("gov2-dir-fd", new GroundTruth(Metric.P10, 149, sDirFDRawP10, 0.5933f));
    h.put("gov2-bm25-base", new GroundTruth(Metric.P10, 149, sBm25BaseRawP10, 0.5846f));
    h.put("gov2-bm25-sd", new GroundTruth(Metric.P10, 149, sBm25SDRawP10, 0.6081f));
    h.put("gov2-bm25-fd", new GroundTruth(Metric.P10, 149, sBm25FDRawP10, 0.6094f));

    for (String model : models) {
      LOG.info("Verifying results of model \"" + model + "\"");

      Map<String, Accumulator[]> r = results.get(model);
View Full Code Here

TOP

Related Classes of ivory.regression.GroundTruth

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.