Package edu.umd.cloud9.collection.wikipedia

Examples of edu.umd.cloud9.collection.wikipedia.WikipediaDocnoMapping


          "-output_path=" + indexRootPath + "/wiki-docid-tmp",
          "-output_file=" + mappingFile.toString(),
          "-wiki_language=" + collectionLang };
      LOG.info("Running BuildWikipediaDocnoMapping with args " + Arrays.toString(arr));

      BuildWikipediaDocnoMapping tool = new BuildWikipediaDocnoMapping();
      tool.setConf(conf);
      tool.run(arr);

      fs.delete(new Path(indexRootPath + "/wiki-docid-tmp"), true);
    } else {
      LOG.info("Docno mapping already exists at: " + mappingFile);
    }

    // Repack Wikipedia into sequential compressed block
    if (!fs.exists(new Path(seqCollection + "/part-00000"))) {
      LOG.info(seqCollection + " doesn't exist, creating...");
      String[] arr = new String[] { "-input=" + rawCollection,
          "-output=" + seqCollection,
          "-mapping_file=" + mappingFile.toString(),
          "-compression_type=block",
          "-wiki_language=" + collectionLang };
      LOG.info("Running RepackWikipedia with args " + Arrays.toString(arr));

      RepackWikipedia tool = new RepackWikipedia();
      tool.setConf(conf);
      tool.run(arr);
    } else {
      LOG.info("Repacked collection already exists at: " + seqCollection);     
    }

    conf.set(Constants.CollectionName, "Wikipedia-"+collectionLang);
View Full Code Here


        "-mapping_file=" + mappingFile.toString(),
        "-compression_type=block",
        "-wiki_language=" + collectionLang };
    LOG.info("Running RepackWikipedia with args " + Arrays.toString(arr));

    RepackWikipedia tool = new RepackWikipedia();
    tool.setConf(conf);
    tool.run(arr);

    conf.set(Constants.CollectionName, "Wikipedia-"+collectionLang);
    conf.setInt(Constants.NumMapTasks, numMappers);
    conf.setInt(Constants.NumReduceTasks, numReducers);
    conf.set(Constants.CollectionPath, seqCollection);
View Full Code Here

    // Repack Wikipedia into sequential compressed block
    p = new Path(seqCollection);
    if (!fs.exists(p)) {
      LOG.info(seqCollection + " doesn't exist, creating...");
      String[] arr = new String[] { rawCollection, seqCollection, mappingFile.toString(), "block"};
      RepackWikipedia tool = new RepackWikipedia();
      tool.setConf(conf);
      tool.run(arr);
    }

    conf.set("Ivory.CollectionName", "Wikipedia-"+collectionLang);
    conf.setInt("Ivory.NumMapTasks", numMappers);
    conf.setInt("Ivory.NumReduceTasks", numReducers);
View Full Code Here

          "-mapping_file=" + mappingFile.toString(),
          "-compression_type=block",
          "-wiki_language=" + collectionLang };
      LOG.info("Running RepackWikipedia with args " + Arrays.toString(arr));

      RepackWikipedia tool = new RepackWikipedia();
      tool.setConf(conf);
      tool.run(arr);
    } else {
      LOG.info("Repacked collection already exists at: " + seqCollection);     
    }

    conf.set(Constants.CollectionName, "Wikipedia-"+collectionLang);
View Full Code Here

          reader.close();
        } catch (IOException e1) {
        }
      }
     
      mDocMapping = new WikipediaDocnoMapping();
      FileSystem fs;
      try {
        fs = FileSystem.get(job);
        String indexPath = job.get("Ivory.IndexPath");
        RetrievalEnvironment env = new RetrievalEnvironment(indexPath, fs);
View Full Code Here

        sLogger.info("Loaded "+samplesMap.size()+" samples");
      }else{
        sLogger.info("No sample file read.");
      }

      mDocMapping = new WikipediaDocnoMapping();
      FileSystem fs;
      try {
        fs = FileSystem.get(job);
        String indexPath = job.get("Ivory.IndexPath");
        RetrievalEnvironment env = new RetrievalEnvironment(indexPath, fs);
View Full Code Here

          reader.close();
        } catch (IOException e1) {
        }
      }
     
      mDocMapping = new WikipediaDocnoMapping();
      FileSystem fs;
      try {
        fs = FileSystem.get(job);
        String indexPath = job.get("Ivory.IndexPath");
        RetrievalEnvironment env = new RetrievalEnvironment(indexPath, fs);
View Full Code Here

        "-" + WikipediaDocnoMappingBuilder.LANGUAGE_OPTION + "=" + language
    };

    List<Integer> counts = IntegrationUtils.execWiki(Joiner.on(" ").join(args));

    WikipediaDocnoMapping mapping = new WikipediaDocnoMapping();
    mapping.loadMapping(new Path(mappingFile), fs);

    System.out.println("DISAMBIGUATION = " + numDisamb + "; ARTICLE = " + numArticles + "; TOTAL = " + total);
    System.out.println("DOCNO 0 = " + mapping.getDocid(1));
    System.out.println("DOCNO 100000 = " + mapping.getDocid(100000));

    // docno to docid
    assertEquals(docid1, mapping.getDocid(1));
    assertEquals(docid2, mapping.getDocid(100000));

    // docid to docno
    assertEquals(1, mapping.getDocno(docid1));
    assertEquals(100000, mapping.getDocno(docid2));
   
    // # of disamb pages
    assertEquals(numDisamb, (int) counts.get(0));
    // # of articles
    assertEquals(numArticles, (int) counts.get(1));
View Full Code Here

          reader.close();
        } catch (IOException e1) {
        }
      }

      mDocMapping = new WikipediaDocnoMapping();
      FileSystem fs;
      try {
        fs = FileSystem.get(job);
        String indexPath = job.get("Ivory.IndexPath");
        RetrievalEnvironment env = new RetrievalEnvironment(indexPath, fs);
View Full Code Here

        sLogger.info("Loaded " + samplesMap.size() + " samples");
      } else {
        sLogger.info("No sample file read.");
      }

      mDocMapping = new WikipediaDocnoMapping();
      FileSystem fs;
      try {
        fs = FileSystem.get(job);
        String indexPath = job.get("Ivory.IndexPath");
        RetrievalEnvironment env = new RetrievalEnvironment(indexPath, fs);
View Full Code Here

TOP

Related Classes of edu.umd.cloud9.collection.wikipedia.WikipediaDocnoMapping

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.