Package edu.umd.cloud9.collection.trecweb

Examples of edu.umd.cloud9.collection.trecweb.Wt10gDocnoMapping

658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
  public DocnoMapping getDocnoMapping() throws IOException {
    return loadDocnoMapping(indexPath, fs);
  }

  public static DocnoMapping loadDocnoMapping(String indexPath, FileSystem fs) throws IOException {
    DocnoMapping mDocMapping = null;
    // load the docid to docno mappings
    try {
      LOG.info("Loading DocnoMapping file...");
      RetrievalEnvironment env = new RetrievalEnvironment(indexPath, fs);

      String className = env.readDocnoMappingClass();
      LOG.info(" - Class name: " + className);
      mDocMapping = (DocnoMapping) Class.forName(className).newInstance();

      Path mappingFile = env.getDocnoMappingData();
      LOG.info(" - File name: " + mappingFile);
      mDocMapping.loadMapping(mappingFile, fs);
      LOG.info("Done!");
    } catch (Exception e) {
      throw new IOException("Error initializing DocnoMapping!");
    }
    return mDocMapping;
View Full Code Here
359
360
361
362
363
364
365
366
367
368
369
    if (list.getLength() > 0) {  conf.set(Constants.StemmedStopwordListQ, list.item(0).getTextContent());  } 
  }

  static float eval(QueryEngine qe, Configuration conf, String setting){
    Qrels qrels = new Qrels(conf.get(Constants.QrelsPath));
    DocnoMapping mapping = qe.getDocnoMapping();
    float apSum = 0, p10Sum = 0;
    Map<String, Accumulator[]> results = qe.getResults();
    for (String qid : results.keySet()) {
      float ap = (float) RankedListEvaluator.computeAP(results.get(qid), mapping,
          qrels.getReldocsForQid(qid));
View Full Code Here
115
116
117
118
119
120
121
122
123
124
125
      return -1;
    }

    RetrievalEnvironment env = new RetrievalEnvironment(indexPath, fs);
    Path mappingFile = env.getDocnoMappingData();
    new ClueWarcDocnoMappingBuilder().build(new Path(collection), mappingFile, conf);

    conf.set(Constants.CollectionName, "ClueWeb:English:Segment" + segment);
    conf.set(Constants.CollectionPath, collection);
    conf.set(Constants.IndexPath, indexPath);
    conf.set(Constants.InputFormat, SequenceFileInputFormat.class.getCanonicalName());
View Full Code Here
82
83
84
85
86
87
88
89
90
91
92
93
94
    if (!fs.exists(p)) {
      sLogger.info("docno-mapping.dat doesn't exist, creating...");
      String[] arr = new String[] { collection, mappingDir.toString(), mappingFile.toString(),
          new Integer(numMappers).toString() };
      NumberTextDocuments tool = new NumberTextDocuments();
      tool.setConf(conf);
      tool.run(arr);

      fs.delete(mappingDir, true);
    }

    // Now we're ready to start the preprocessing pipeline... set
View Full Code Here
79
80
81
82
83
84
85
86
87
88
89
90
91
    // (sequentially-number integer). If it doesn't exist create it.
    Path mappingFile = env.getDocnoMappingData();
    if (!fs.exists(mappingFile)) {
      sLogger.info(mappingFile + " doesn't exist, creating...");
      String[] arr = new String[] { collection, indexPath + "/medline-docid-tmp",  mappingFile.toString(), new Integer(numMappers).toString() };
      NumberMedlineCitations tool = new NumberMedlineCitations();
      tool.setConf(conf);
      tool.run(arr);

      fs.delete(new Path(indexPath + "/medline-docid-tmp"), true);
    }

    // Now we're ready to start the preprocessing pipeline... set
View Full Code Here
82
83
84
85
86
87
88
89
90
91
92
93
94
    if (!fs.exists(mappingFile)) {
      LOG.info("docno-mapping.dat doesn't exist, creating...");
      String[] arr = new String[] { collection, mappingDir.toString(),
              mappingFile.toString() };
      NumberTrecDocuments2 tool = new NumberTrecDocuments2();
      tool.setConf(conf);
      tool.run(arr);

      fs.delete(mappingDir, true);
    }

    conf.set(Constants.CollectionName, "TREC_vol45");
View Full Code Here
82
83
84
85
86
87
88
89
90
91
92
93
94
    if (!fs.exists(mappingFile)) {
      LOG.info("docno-mapping.dat doesn't exist, creating...");
      String[] arr = new String[] { collection, mappingDir.toString(),
          mappingFile.toString(), "100" };
      NumberTrecWebDocuments tool = new NumberTrecWebDocuments();
      tool.setConf(conf);
      tool.run(arr);

      fs.delete(mappingDir, true);
    }

    conf.set(Constants.CollectionName, "Wt10g");
View Full Code Here
82
83
84
85
86
87
88
89
90
91
92
93
94
    if (!fs.exists(mappingFile)) {
      LOG.info("docno-mapping.dat doesn't exist, creating...");
      String[] arr = new String[] { collection, mappingDir.toString(),
          mappingFile.toString(), "100" };
      NumberTrecWebDocuments tool = new NumberTrecWebDocuments();
      tool.setConf(conf);
      tool.run(arr);

      fs.delete(mappingDir, true);
    }

    conf.set(Constants.CollectionName, "Gov2");
View Full Code Here
54
55
56
57
58
59
60
61
62
63
64
65
66
67
        "-" + DocnoMapping.BuilderUtils.COLLECTION_OPTION + "=" + repackedCollection,
        "-" + DocnoMapping.BuilderUtils.MAPPING_OPTION + "=" + wt10gMappingFile };

    IntegrationUtils.exec(Joiner.on(" ").join(args));

    Wt10gDocnoMapping mapping = new Wt10gDocnoMapping();
    mapping.loadMapping(new Path(wt10gMappingFile), fs);

    assertEquals("WTX001-B01-1", mapping.getDocid(1));
    assertEquals("WTX062-B34-37", mapping.getDocid(1000000));

    assertEquals(1, mapping.getDocno("WTX001-B01-1"));
    assertEquals(1000000, mapping.getDocno("WTX062-B34-37"));
  }
View Full Code Here
119
120
121
122
123
124
125
126
127
128
129
130
131
132
        "-" + DocnoMapping.BuilderUtils.FORMAT_OPTION + "=" + TrecWebDocumentInputFormat.class.getCanonicalName(),
        "-" + DocnoMapping.BuilderUtils.MAPPING_OPTION + "=" + wt10gMappingFile };

    IntegrationUtils.exec(Joiner.on(" ").join(args));

    Wt10gDocnoMapping mapping = new Wt10gDocnoMapping();
    mapping.loadMapping(new Path(wt10gMappingFile), fs);

    assertEquals("WTX001-B01-1", mapping.getDocid(1));
    assertEquals("WTX062-B34-37", mapping.getDocid(1000000));

    assertEquals(1, mapping.getDocno("WTX001-B01-1"));
    assertEquals(1000000, mapping.getDocno("WTX062-B34-37"));
  }
View Full Code Here
TOP

Related Classes of edu.umd.cloud9.collection.trecweb.Wt10gDocnoMapping

  • bak.pcj.IntIterator
  • edu.umd.cloud9.example.bfs.BfsNodeTest
  • edu.umd.cloud9.example.pagerank.PageRankNodeTest
  • edu.umd.cloud9.integration.collection.trecweb.IntegrationTest
  • edu.umd.cloud9.io.array.ArrayListOfIntsWritable
  • edu.umd.cloud9.io.fastutil.Int2FloatOpenHashMapWritableTest
  • edu.umd.cloud9.io.fastutil.Int2IntOpenHashMapWritableTest
  • edu.umd.cloud9.io.fastutil.String2FloatOpenHashMapWritableTest
  • edu.umd.cloud9.io.fastutil.String2IntOpenHashMapWritableTest
  • edu.umd.cloud9.io.map.HMapIDW
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.