Examples of uk.ac.cam.ch.wwmm.oscar3.indexersearcher.LuceneIndexerSearcher

Package uk.ac.cam.ch.wwmm.oscar3.indexersearcher

Examples of uk.ac.cam.ch.wwmm.oscar3.indexersearcher.LuceneIndexerSearcher

uk.ac.cam.ch.wwmm.oscar3.indexersearcher.LuceneIndexerSearcher
Indexes and searches papers in SciXML+Oscar using Apache Lucene. @author ptc24


  /**
   * @param args
   */
  public static void main(String[] args) throws Exception {
    LuceneIndexerSearcher lis = new LuceneIndexerSearcher(false);
    IndexSearcher is = lis.getIndexSearcher();
    IndexReader ir = lis.getIndexReader();
    
    List<String> docFiles = new ArrayList<String>();
    for(int i=0;i<ir.numDocs();i++) {
      docFiles.add(ir.document(i).getField("filename").stringValue().replaceAll("markedup", "source"));
    }
    
    TermEnum textEnum = ir.terms();
    Map<String,Integer> docFreqs = new HashMap<String,Integer>();
    while(textEnum.next()) {
      Term t = textEnum.term();
      if("txt".equals(t.field())) {
        String text = t.text();
        if(TermSets.getClosedClass().contains(text)) continue;
        if(!text.matches(".*[A-Za-z].*")) continue;
        int docFreq = ir.docFreq(t);
        if(docFreq > 1) {
          docFreqs.put(text, ir.docFreq(t));
        }
      }      
    }
    List<String> dfl = StringTools.getSortedList(docFreqs);


    List<Query> queries = new ArrayList<Query>();
    for(int i=0;i<50;i++) {
      queries.add(new TermQuery(new Term("txt", dfl.get(i))));
      System.out.println(dfl.get(i));
    }
    
    for(int i=0;i<10;i++) {
      Map<Integer,Integer> bestClusters = new HashMap<Integer,Integer>();
      Map<Integer,Float> bestClusterScores = new HashMap<Integer,Float>();
      List<Map<Integer,Float>> clusters = new ArrayList<Map<Integer,Float>>();
      for(int j=0;j<queries.size();j++) {
        clusters.add(new HashMap<Integer,Float>());
        VectorCollector vc = new VectorCollector();
        is.search(queries.get(j), vc);
        //System.out.println(vc.getResultsVector());
        for(Integer k : vc.getResultsVector().keySet()) {
          float score = vc.getResultsVector().get(k);
          if(score < 0.001) continue;
          if(!bestClusterScores.containsKey(k) || bestClusterScores.get(k) < score) {
            bestClusters.put(k, j);
            bestClusterScores.put(k, score);
          }
        }
      }
      for(Integer j : bestClusters.keySet()) {
        clusters.get(bestClusters.get(j)).put(j, bestClusterScores.get(j));
      }
      //for(Map<Integer,Float> cluster : clusters) System.out.println(cluster);
      queries.clear();
      for(int j=0;j<clusters.size();j++) {
        System.out.println("Size: " + clusters.get(j).size());
        /*if(i == 9) {
          for(Integer k : clusters.get(j).keySet()) {
            System.out.println(docFiles.get(k) + "\t" + bestClusterScores.get(k));
          }
        }*/
        //if(i == 9) ClusterAnalyser.excessAnalyseCluster(clusters.get(j), lis.getIndexReader(), 0.2, true);
        Map<String,Double> scores = ClusterAnalyser.simpleExcessAnalyseCluster(clusters.get(j), lis.getIndexReader(), 0.1);
        BooleanQuery bq = new BooleanQuery(false);
        List<String> terms = StringTools.getSortedList(scores);
        if(terms.size() > 20) terms = terms.subList(0, 20);
        for(String s : terms) {
          System.out.println(s + "\t" + scores.get(s));
          TermQuery tq = new TermQuery(new Term("txt", s));
          tq.setBoost(scores.get(s).floatValue());
          bq.add(new BooleanClause(tq, Occur.SHOULD));
        }
        System.out.println();
        queries.add(bq);
      }
      System.out.println();
    }
    List<Map<Integer,Float>> clusters = new ArrayList<Map<Integer,Float>>();
    final Map<Integer,Integer> clusterSizes = new HashMap<Integer,Integer>();
    for(int j=0;j<queries.size();j++) {
      VectorCollector vc = new VectorCollector();
      is.search(queries.get(j), vc);
      final Map<Integer,Float> cluster = new HashMap<Integer,Float>();
      //System.out.println(vc.getResultsVector());
      for(Integer k : vc.getResultsVector().keySet()) {
        float score = vc.getResultsVector().get(k);
        if(score < 0.2) continue;
        cluster.put(k, score);
      }
      clusters.add(cluster);
      clusterSizes.put(j, cluster.size());
    }
    List<Integer> clustersBySize = new ArrayList<Integer>(clusterSizes.keySet());
    Collections.sort(clustersBySize, Collections.reverseOrder(new Comparator<Integer>() {
      @SuppressWarnings("unchecked")
      public int compare(Integer o1, Integer o2) {
        return clusterSizes.get(o1).compareTo(clusterSizes.get(o2));
      }
    }));
    
    for(Integer j : clustersBySize) {
      final Map<Integer,Float> cluster = clusters.get(j);
      System.out.println("Size:\t" + cluster.size());
      List<Integer> list = new ArrayList<Integer>(cluster.keySet());
      Collections.sort(list, Collections.reverseOrder(new Comparator<Integer>() {
        @SuppressWarnings("unchecked")
        public int compare(Integer o1, Integer o2) {
          return cluster.get(o1).compareTo(cluster.get(o2));
        }
      }));
      for(Integer k : list) {
        System.out.println(docFiles.get(k) + "\t" + cluster.get(k));
      }
      ClusterAnalyser.excessAnalyseCluster(cluster, lis.getIndexReader(), 0.2, true);
      System.out.println();
    }




  }

View Full Code Here

      System.out.println(s + "\t" + tfIdf.get(s));
    }  
  }
  
  public static Map<String,Double> excessAnalyseCluster(Map<Integer,Float> cluster, IndexReader ir, double threshold, boolean enriched) throws Exception {
    LuceneChemicalIndex lci = new LuceneIndexerSearcher(false).getLci();
    Set<String> inchis = new HashSet<String>();
    Set<String> onts = new HashSet<String>();
    
    List<File> clusterFiles = new ArrayList<File>();
    for(Integer i : cluster.keySet()) {

View Full Code Here

    return scores;
  }


  
  public static void analyseCluster(Map<Integer,Float> cluster, IndexReader ir, DocVectorSimilarity similarity, double threshold) throws Exception {
    LuceneChemicalIndex lci = new LuceneIndexerSearcher(false).getLci();
    List<File> clusterFiles = new ArrayList<File>();
    Bag<String> dfs = new Bag<String>();
    Set<String> inchis = new HashSet<String>();
    Set<String> onts = new HashSet<String>();
    for(Integer i : cluster.keySet()) {

View Full Code Here

      System.out.println(term + "\t" + scores.get(term) + "\t" + overlaps.get(term));
    }
  }


  public static void main(String[] args) throws Exception {
    LuceneIndexerSearcher lis = new LuceneIndexerSearcher(false);
    IndexSearcher is = lis.getIndexSearcher();
    
    Stemmer stemmerTools = new Stemmer(new EnglishStemmer());
    
    //QueryParser qp = new Oscar3QueryParser("txt", new Oscar3Analyzer(), lis, false);
    //Query q = qp.parse("NaCl");
    
    String queryTerm = "content";
    //PhraseQuery pq = new PhraseQuery();
    //pq.add(new Term("txt", "aromatase"));
    //pq.add(new Term("txt", "inhibitors"));
    //Query q = new TermQuery(new Term("txt", queryTerm));
    Query q = new StemQuery(new Term("txt", queryTerm), stemmerTools);
    //q = pq;
    VectorCollector vc = new VectorCollector();
    is.search(q, vc);
    Map<String,Double> scores = simpleExcessAnalyseCluster(vc.getResultsVector(), lis.getIndexReader(), 0.01);
    for(String s : StringTools.getSortedList(scores)) {
      System.out.println(s + "\t" + scores.get(s));
    }


    //tfIdfAnalyseCluster(vc.getResultsVector(), lis.getIndexReader());

View Full Code Here

    //  return;
    //}
    
    ChiSquaredDistribution csd = new ChiSquaredDistributionImpl(1);
    List<Query> lq = new ArrayList<Query>();
    TextMiner tm = new TextMiner(new LuceneIndexerSearcher(false));


    System.out.println(tm.corpusBags.txtBag.totalCount());
    System.out.println(tm.corpusBags.txtBag.getCount("dextromethorphan"));
    
    q = new TermQuery(new Term("txt", "CYP2D6"));

View Full Code Here


  /**
   * @param args
   */
  public static void main(String[] args) throws Exception {
    LuceneIndexerSearcher lis = new LuceneIndexerSearcher(false);
    
    Query q = new FuzzyQuery(new Term("txt", "adrenaline"), 0.1f);
    Query qq = lis.getIndexSearcher().rewrite(q);
    
    if(qq instanceof BooleanQuery) {
      BooleanQuery bq = (BooleanQuery)qq;
      BooleanClause [] clauses = bq.getClauses();
      final HashMap<String,Float> hm = new HashMap<String,Float>();

View Full Code Here

  /**
   * @param args
   */
  public static void main(String[] args) throws Exception {
    // TODO Auto-generated method stub
    LuceneIndexerSearcher lis = new LuceneIndexerSearcher(false);
    IndexReader ir = lis.getIndexReader();
    Directory dir = new RAMDirectory(ir.directory());
    ir.close();
    IndexSearcher is = new IndexSearcher(dir);
    ir = is.getIndexReader();
    /*LuceneIndexerSearcher lis = new LuceneIndexerSearcher(false);

View Full Code Here

  
  /**
   * @param args
   */
  public static void main(String[] args) throws Exception {
    LuceneIndexerSearcher lis = new LuceneIndexerSearcher(false);
    IndexReader ir = lis.getIndexReader();
    Directory dir = new RAMDirectory(ir.directory());
    ir.close();
    IndexSearcher is = new IndexSearcher(dir);
    //IndexSearcher is = lis.getIndexSearcher();
    is.setSimilarity(new MySimilarity());

View Full Code Here

  }
  
  protected void doGet(HttpServletRequest request,
      HttpServletResponse response) throws ServletException, IOException {
    
    LuceneIndexerSearcher indexerSearcher;
    try {
      indexerSearcher = new LuceneIndexerSearcher(false);      
    } catch (Exception e) {
      PrintWriter out = response.getWriter();
      e.printStackTrace(out);
      return;
    }
    
    if(request.getParameter("suggest") != null) {
      try {
        String start = request.getParameter("suggest");
        boolean ontological = "yes".equals(request.getParameter("ontological"));
        String json = indexerSearcher.suggestJSON(start, ontological);
        response.setContentType("application/x-suggestions+json");
        response.getWriter().write(json);
      } catch (Exception e) {
        e.printStackTrace();
      }
      
      return;
    }
    
    if(request.getParameter("selectcorpus") != null) {
      SciXMLDocument doc = new SciXMLDocument();
      doc.setTitle("Select corpus for search");
      Element list = doc.addList();
      File [] dirs = new File(Oscar3Props.getInstance().workspace, "corpora").listFiles();
      for(int i=0;i<dirs.length;i++) {
        String name = dirs[i].getName();
        if(dirs[i].isDirectory() && !name.startsWith("."))  {
          Element a = doc.makeLink("Search?setcorpus=" + URLEncoder.encode(name, "UTF-8"), name);
          list.appendChild(doc.makeListItem(a));
        }
      }


      Element a = doc.makeLink("Search?settoscrap", "ScrapBook");
      list.appendChild(doc.makeListItem(a));


      a = doc.makeLink("Search?indexall", "Index entire workspace");
      list.appendChild(doc.makeListItem(a));


      doc.addServerProcessingInstructions();
      response.setContentType("application/xml");
      new Serializer(response.getOutputStream()).write(doc);
      return;
    }
    
    if(request.getParameter("settoscrap") != null) {
      response.setContentType("text/plain");
      PrintWriter out = response.getWriter();
      try {
        indexerSearcher = new LuceneIndexerSearcher(true);
        indexerSearcher.addScrapBook();
        out.println("Corpus changed to scrapbook OK!");
      } catch (Exception e) {
        out.println("Yikes, that didn't work!");
      }
      return;    
    }
    
    if(request.getParameter("indexall") != null) {
      response.setContentType("text/plain");
      PrintWriter out = response.getWriter();
      try {
        indexerSearcher = new LuceneIndexerSearcher(true);
        indexerSearcher.addScrapBook();
        indexerSearcher.addDirectory(new File(Oscar3Props.getInstance().workspace, "corpora"), out);
        out.println("Indexed entire workspace OK");
      } catch (Exception e) {
        out.println("Yikes, that didn't work!");
      }
      return;    
      
    }
    
    if(request.getParameter("setcorpus") != null) {
      response.setContentType("text/plain");
      PrintWriter out = response.getWriter();
      try {
        File activeCorpus = new File(new File(Oscar3Props.getInstance().workspace, "corpora"), request.getParameter("setcorpus"));
        indexerSearcher = new LuceneIndexerSearcher(true);
        indexerSearcher.addDirectory(activeCorpus, new PrintWriter(System.out, true));
        out.println("Corpus changed to " + request.getParameter("setcorpus") + " OK!");
      } catch (Exception e) {
        e.printStackTrace();
        out.println("Yikes, that didn't work!");
      }
      return;
    }


    if(request.getParameter("concordanceresults") != null) {
      response.setContentType("text/plain");
      PrintWriter out = response.getWriter();
      int number = Integer.parseInt(request.getParameter("number"));
      for(int i=0;i<number;i++) {
        out.println(request.getParameter("s" + i));
        out.println(URLDecoder.decode(request.getParameter("s" + i + "start"), "UTF-8"));
        out.println(URLDecoder.decode(request.getParameter("s" + i + "end"), "UTF-8"));
        out.println(URLDecoder.decode(request.getParameter("s" + i + "file"), "UTF-8"));
        out.println();
      }
      return;
    }
    
    if(request.getParameter("concordance") != null) {
      try {
        String word = request.getParameter("concordance");
        String word1 = null;
        String word2 = null;
        if(word.matches("\\S+\\s+\\S+")) {
          String [] ww = word.split("\\s+");
          word1 = ww[0];
          word2 = ww[1];
        }
        String mode = request.getParameter("mode");
        if("form".equals(mode)) {
          String optstr = request.getParameter("types");
          List<String> options = StringTools.arrayToList(optstr.split("\\s+"));
          List<ConcordanceEntry> ces = Concordance.makeConcordance(indexerSearcher.filesForWord(word), word, 150, mode);                  
          response.setContentType("text/html");
          PrintWriter out = response.getWriter();
          out.println("<html><head><title>foo</title></head><body><form method='POST' action='Search'>");
          out.println("<input name='concordanceresults' type='hidden' value='foo'>");          
          out.println("<table>");
          
          boolean shade = false;
          int i=0;
          for(ConcordanceEntry ce : ces) {
            out.print("<tr" + (shade ? " style='background-color: lightgrey'" : "") + ">");
            out.print("<td><tt>" + ce.text + "</tt></td>");
            out.print("<td><select name='s" + i + "'>");
            for(String option : options) {
              out.print("<option value='" + StringTools.urlEncodeUTF8NoThrow(option) + "'>");
              out.print(option);
              out.print("</option>");
            }
            out.print("</select></td>");
            out.println("</tr>");
            out.println("<input type='hidden' name='s" + i + "start' value='" + StringTools.urlEncodeUTF8NoThrow(ce.start) + "'>");
            out.println("<input type='hidden' name='s" + i + "end' value='" + StringTools.urlEncodeUTF8NoThrow(ce.end) + "'>");
            out.println("<input type='hidden' name='s" + i + "file' value='" + StringTools.urlEncodeUTF8NoThrow(ce.file) + "'>");
            shade = !shade;
            i++;
          }      
          out.println("</table>");
          out.println("<input type='hidden' name='number' value='" + i + "'>");
          out.println("<input type='submit' value='Submit'>");
          out.println("</form></body></html>");
        } else {
          response.setContentType("text/plain");  
          PrintWriter out = response.getWriter();
          long time = System.currentTimeMillis();
          List<String> concordance;
          if(word1 == null) {
            List<ConcordanceEntry> ces = Concordance.makeConcordance(indexerSearcher.filesForWord(word), word, 160, mode);                  
            concordance = new ArrayList<String>();
            for(ConcordanceEntry ce : ces) {
              concordance.add(ce.text);// + "\t" + ce.file + "\t" + ce.start + "\t" + ce.end);
            }
          } else {
            concordance = Concordance.biConcordance(indexerSearcher.filesForWordPair(word1, word2), word1, word2, 160, mode);                            
          }
          if(Oscar3Props.getInstance().verbose) System.out.println(System.currentTimeMillis() - time);
          for(String line : concordance) out.println(line);          
        }
      } catch (Exception e) {
        e.printStackTrace();
      }  
      return;
    }
    
    SciXMLDocument resultsDoc = null;
    
    if(request.getParameter("resultsType").equals("bigCompoundsList")) {
      try {
        resultsDoc = indexerSearcher.getBigCompoundsList();
      } catch (Exception e) {
        e.printStackTrace();
      }
    /*} else if(request.getParameter("resultsType").equals("pubchem")) {
      try {
        resultsDoc = indexerSearcher.getBigShoppingList();
        FetchFromPubChem.fillCNDFromCompoundsList(resultsDoc);
      } catch (Exception e) {
         e.printStackTrace();
      }*/
    } else {
      UserQuery.ResultsType rt;
      
      if(request.getParameter("resultsType").equals("compoundsList")) {
        rt = UserQuery.ResultsType.COMPOUNDSLIST;
      } else if(request.getParameter("resultsType").equals("hitsList")) {
        rt = UserQuery.ResultsType.HITSLIST;
      } else if(request.getParameter("resultsType").equals("assoc")) {
        rt = UserQuery.ResultsType.ASSOC;
      } else {
        rt = UserQuery.ResultsType.SNIPPETS;      
      }
      
      int size = 5;
      int skip = 0;
      
      if(request.getParameter("size") != null) size = Integer.parseInt(request.getParameter("size"));
      if(request.getParameter("skip") != null) skip = Integer.parseInt(request.getParameter("skip"));
      
      UserQuery uq = new UserQuery(rt, size, skip);
      if(request.getParameter("morelikethis") != null) {
        uq.setToMoreLikeThis(Integer.parseInt(request.getParameter("morelikethis")));
      } 
      if(request.getParameter("query") != null) {
        String query = request.getParameter("query");
        String queryType = request.getParameter("type");
        String parameter = request.getParameter("parameter");
        
        uq.addTerm(query, queryType, parameter);
        
        for(int i=2;request.getParameter("query" + Integer.toString(i)) != null;i++) {
          query = request.getParameter("query" + Integer.toString(i));
          queryType = request.getParameter("type" + Integer.toString(i));
          parameter = request.getParameter("parameter" + Integer.toString(i));
          uq.addTerm(query, queryType, parameter);      
        }
        
      }
    
      try {
        resultsDoc = indexerSearcher.getResultsByUserQuery(uq);
      } catch (Exception e) {
        e.printStackTrace();
      }
    }

View Full Code Here

    String c = hasCue ? "TRUE" : "FALSE";
    return new Event(c, words.toArray(new String[0]));
  }
  
  public static void main(String[] args) throws Exception {
    LuceneIndexerSearcher lis = new LuceneIndexerSearcher(false);
    IndexReader ir = lis.getIndexReader();
    
    int numDocs = ir.maxDoc();
    
    String c = "cholesterol";

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of uk.ac.cam.ch.wwmm.oscar3.indexersearcher.LuceneIndexerSearcher

dk.brics.automaton.Automaton

dk.brics.automaton.RunAutomaton

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.