Examples of ivory.core.eval.GradedQrels

ivory.core.eval.GradedQrels

Representation of relevance judgments. In TREC parlance, qrels are judgments made by humans as to whether a document is relevant to an information need. Typically, qrels are created by a process known as "pooling" in large-scale system evaluations such as those at TREC.
@author Jimmy Lin @author Lidan Wang

    if (qrelsPath == null){
      System.out.println("Should have set qrelsPath!");
      System.exit(-1);
    }


    GradedQrels qrels = new GradedQrels(qrelsPath);
                DocnoMapping mapping = getDocnoMapping();


    if (K_val==0){
      //System.out.println("K value should be set already.");
      //System.exit(-1);
    }
    for (String queryID : queries.keySet()) {
      // Get the ranked list for this query.
      Accumulator[] list = runner.getResults(queryID);
      if (list == null) {
        LOG.info("null results for: " + queryID);
        continue;
      }


      float ndcg = (float) RankedListEvaluator.computeNDCG(kVal, list, mapping, qrels.getReldocsForQid(queryID, true)); 
      ndcgSum += ndcg;


      if (!internalDocno){
        if (qrels.getReldocsForQid(queryID, true).size()>0){ //if have qrels for this query
          //System.out.println("Lidan: NDCG for query "+queryID+" is "+ndcg);
          ndcgValues.add(ndcg+"");
          costKeys.add(modelID+" "+queryID); //save keys for cost. Evaluation metric is computed by the end of runQueries()
        }
      }

View Full Code Here

    g.put("Clue-Cascade", new GroundTruth("Clue-Cascade", Metric.NDCG20, 25, Cascade, 0.3061f));
    g.put("Clue-AdaRank", new GroundTruth("Clue-AdaRank", Metric.NDCG20, 25, AdaRank, 0.3095f));
    g.put("Clue-FeaturePrune", new GroundTruth("Clue-FeaturePrune", Metric.NDCG20, 25,
        FeaturePrune, 0.2966f));


    GradedQrels qrels = new GradedQrels("data/clue/qrels.web09catB.txt");


    String[] params = new String[] {
        "data/clue/run.clue.SIGIR2011.xml",
        "data/clue/queries.web09.26-50.xml" };

View Full Code Here

    g.put("Wt10g-Cascade", new GroundTruth("Wt10g-Cascade", Metric.NDCG20, 50, Cascade, 0.3560f));
    g.put("Wt10g-AdaRank", new GroundTruth("Wt10g-AdaRank", Metric.NDCG20, 50, AdaRank, 0.3549f));
    g.put("Wt10g-FeaturePrune", new GroundTruth("Wt10g-FeaturePrune", Metric.NDCG20, 50,
        FeaturePrune, 0.3486f));


    GradedQrels qrels = new GradedQrels("data/wt10g/qrels.wt10g.all");


    String[] params = new String[] {
        "data/wt10g/run.wt10g.SIGIR2011.xml",
        "data/wt10g/queries.wt10g.501-550.xml" };

View Full Code Here

    g.put("Wt10g-Cascade-0.7",
        new GroundTruth("Wt10g-Cascade-0.7", Metric.NDCG20, 50, p7, 0.3484f));
    g.put("Wt10g-Cascade-0.9",
        new GroundTruth("Wt10g-Cascade-0.9", Metric.NDCG20, 50, p9, 0.3407f));


    GradedQrels qrels = new GradedQrels("data/wt10g/qrels.wt10g.all");


    String[] params = new String[] {
        "data/wt10g/run.wt10g.SIGIR2011.varying.tradeoff.cascade.xml",
        "data/wt10g/queries.wt10g.501-550.xml" };

View Full Code Here

    g.put("Wt10g-FeaturePrune-0.7",
        new GroundTruth("Wt10g-FeaturePrune-0.7", Metric.NDCG20, 50, p7, 0.3444f));
    g.put("Wt10g-FeaturePrune-0.9",
        new GroundTruth("Wt10g-FeaturePrune-0.9", Metric.NDCG20, 50, p9, 0.3404f));


    GradedQrels qrels = new GradedQrels("data/wt10g/qrels.wt10g.all");


    String[] params = new String[] {
        "data/wt10g/run.wt10g.SIGIR2011.varying.tradeoff.featureprune.xml",
        "data/wt10g/queries.wt10g.501-550.xml" };

View Full Code Here

    g.put("Gov2-FeaturePrune-0.3",
        new GroundTruth("Gov2-FeaturePrune-0.3", Metric.NDCG20, 75, p3, 0.4661f));
    g.put("Gov2-FeaturePrune-0.1",
        new GroundTruth("Gov2-FeaturePrune-0.1", Metric.NDCG20, 75, p1, 0.4716f));


    GradedQrels qrels = new GradedQrels("data/gov2/qrels.gov2.all");


    String[] params = new String[] {
        "data/gov2/run.gov2.SIGIR2011.varying.tradeoff.featureprune.xml",
        "data/gov2/queries.gov2.title.776-850.xml" };

View Full Code Here

    g.put("Clue-Cascade-0.3", new GroundTruth("Clue-Cascade-0.3", Metric.NDCG20, 25, p3, 0.3054f));
    g.put("Clue-Cascade-0.5", new GroundTruth("Clue-Cascade-0.5", Metric.NDCG20, 25, p5, 0.3054f));
    g.put("Clue-Cascade-0.7", new GroundTruth("Clue-Cascade-0.7", Metric.NDCG20, 25, p7, 0.3006f));
    g.put("Clue-Cascade-0.9", new GroundTruth("Clue-Cascade-0.9", Metric.NDCG20, 25, p9, 0.2760f));


    GradedQrels qrels = new GradedQrels("data/clue/qrels.web09catB.txt");


    String[] params = new String[] {
        "data/clue/run.clue.SIGIR2011.varying.tradeoff.cascade.xml",
        "data/clue/queries.web09.26-50.xml" };

View Full Code Here

    g.put("Gov2-Cascade", new GroundTruth("Gov2-Cascade", Metric.NDCG20, 75, Cascade, 0.4744f));
    g.put("Gov2-AdaRank", new GroundTruth("Gov2-AdaRank", Metric.NDCG20, 75, AdaRank, 0.4737f));
    g.put("Gov2-FeaturePrune", new GroundTruth("Gov2-FeaturePrune", Metric.NDCG20, 75,
        FeaturePrune, 0.4716f));


    GradedQrels qrels = new GradedQrels("data/gov2/qrels.gov2.all");


    String[] params = new String[] {
        "data/gov2/run.gov2.SIGIR2011.xml",
        "data/gov2/queries.gov2.title.776-850.xml" };

View Full Code Here

    g.put("Gov2-Cascade-0.7", new GroundTruth("Gov2-Cascade-0.7", Metric.NDCG20, 75, p7, 0.4625f));
    g.put("Gov2-Cascade-0.5", new GroundTruth("Gov2-Cascade-0.5", Metric.NDCG20, 75, p5, 0.4634f));
    g.put("Gov2-Cascade-0.3", new GroundTruth("Gov2-Cascade-0.3", Metric.NDCG20, 75, p3, 0.4653f));
    g.put("Gov2-Cascade-0.1", new GroundTruth("Gov2-Cascade-0.1", Metric.NDCG20, 75, p1, 0.4745f));


    GradedQrels qrels = new GradedQrels("data/gov2/qrels.gov2.all");


    String[] params = new String[] {
        "data/gov2/run.gov2.SIGIR2011.varying.tradeoff.cascade.xml",
        "data/gov2/queries.gov2.title.776-850.xml" };

View Full Code Here

    g.put("Clue-FeaturePrune-0.7", new GroundTruth("Clue-FeaturePrune-0.7",
        Metric.NDCG20, 25, p7, 0.2934f));
    g.put("Clue-FeaturePrune-0.9", new GroundTruth("Clue-FeaturePrune-0.9",
        Metric.NDCG20, 25, p9, 0.2758f));


    GradedQrels qrels = new GradedQrels("data/clue/qrels.web09catB.txt");


    String[] params = new String[] {
            "data/clue/run.clue.SIGIR2011.varying.tradeoff.featureprune.xml",
            "data/clue/queries.web09.26-50.xml" };

View Full Code Here

0 1

TOP

Related Classes of ivory.core.eval.GradedQrels

ivory.cascade.retrieval.CascadeBatchQueryRunner

ivory.core.util.DelimitedValuesFileReader

ivory.regression.sigir2011.Clue_Cascade

ivory.regression.sigir2011.Clue_VaryingTradeoff_Cascade

ivory.regression.sigir2011.Clue_VaryingTradeoff_FeaturePrune

ivory.regression.sigir2011.Gov2_Cascade

ivory.regression.sigir2011.Gov2_VaryingTradeoff_Cascade

ivory.regression.sigir2011.Gov2_VaryingTradeoff_FeaturePrune

ivory.regression.sigir2011.Wt10g_Cascade

ivory.regression.sigir2011.Wt10g_VaryingTradeoff_Cascade

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.