Examples of uk.ac.cam.ch.wwmm.oscar3.recogniser.document.TokenSequenceSource

Package uk.ac.cam.ch.wwmm.oscar3.recogniser.document

Examples of uk.ac.cam.ch.wwmm.oscar3.recogniser.document.TokenSequenceSource

uk.ac.cam.ch.wwmm.oscar3.recogniser.document.TokenSequence
Takes a list of SciXML files, and provides an iterator over the TokenSequences corresponding to the usable parts of those documents. @author ptc24

    }
    
    //if(surface.matches("([Pp]oly).+")) features.add("polymer");
    //if(surface.matches(".+\\(\\d\\d\\d+\\)")) features.add("surfacenotation");
    
    Token t = state.procDoc.getTokenByStart(annot.getAttributeValue("from"));
    if(fPrevious && t != null) {
      Token tt = t.getNAfter(-1);
      if(tt != null) {
        if(TokenTypes.isRef(tt) && tt.getNAfter(-1) != null) {
          //  features.add(prefix + "skiprefprev");
          tt = tt.getNAfter(-1);
        }
        String ttv = tt.getValue();
        ttv = ttv.replaceAll("\\s+", "_");
        features.add("prev=" + ttv);
      }
    }
    
    t = state.procDoc.getTokenByEnd(annot.getAttributeValue("to"));
    if(fNext && t != null) {
      Token tt = t.getNAfter(1);
      if(tt != null && tt.getValue().equals("-")) {
        tt = t.getNAfter(2);
      }
      if(tt != null) {
        if(TokenTypes.isRef(tt) && tt.getNAfter(1) != null) {
          tt = tt.getNAfter(1);
        }
        
        String ttv = tt.getValue();
        ttv = ttv.replaceAll("\\s+", "_");
        features.add("next=" + ttv);
      }
    }

View Full Code Here

    Map<TokenSequence,Boolean> tokSeqs = new HashMap<TokenSequence,Boolean>();
    Map<TokenSequence,Bag<String>> tokSeqPRWs = new HashMap<TokenSequence,Bag<String>>();
    
    for(int i=0;i<n.size();i++) {
      Element e = (Element)n.get(i);
      Token token = procDoc.getTokenByStart(e.getAttributeValue("from"));
      if(token == null) token = procDoc.getTokenByEnd(e.getAttributeValue("to"));
      if(token == null) {
        System.out.println("Eeep!");
      } else {
        TokenSequence tokSeq = token.getTokenSequence();
        //boolean isReact = "REACT".equals(SafTools.getSlotValue(e, "subtype"));
        boolean isPubmed = f.toString().contains("pubmed");
        if(tokSeqs.containsKey(tokSeq)) {
          if(isPubmed) tokSeqs.put(tokSeq, true);
        } else {

View Full Code Here

   * 
   * @param word The string to test.
   */
  public static void destroyInstanceIfWordTokenises(String word) {
    if(myInstance == null) return;
    TokenSequence ts = Tokeniser.getInstance().tokenise(word);
    if(ts.getTokens().size() > 1) myInstance = null;
  }

View Full Code Here

   * 
   * @param word The string to test.
   */
  public static void destroyInstanceIfWordTokenises(String word) {
    if(myInstance == null) return;
    TokenSequence ts = Tokeniser.getInstance().tokenise(word);
    if(ts.getTokens().size() > 1) myInstance = null;
  }

View Full Code Here

        ontDFANumber++;
      }
      type = type + "_" + ontDFANumber;*/
    }


    TokenSequence ts = Tokeniser.getInstance().tokenise(ne);
    List<String> tokens = ts.getTokenStringList();


    if(!alwaysAdd && tokens.size() == 1 && !ne.contains("$")) return;
    StringBuffer sb = new StringBuffer();
    for(String token : tokens) {
      sb.append(getRepForToken(token));

View Full Code Here

                featuresForAbbrev.add(abbrMode + "abbr<" + thresh);
              }              
            }
          } else {
            int tokID = ne.getFirstToken().getId();
            TokenSequence tokSeq = ne.getFirstToken().getTokenSequence();
            int length = surf.length();
            boolean isAcro = false;
            if(allCaps.matcher(surf).matches()) {
              if(length <= (tokID - 1)) {
                isAcro = true;
                for(int i=0;i<length;i++) {
                  if(!tokSeq.getToken(tokID - length - 1 + i).getValue().toUpperCase().startsWith(surf.substring(i,i+1))) isAcro = false;
                }
                if(isAcro) {
                  featuresForAbbrev.add("allUpperAbbrev");
                }
              }

View Full Code Here

    double conf = ne.getConfidence();
    double confLog = Math.log(conf) - Math.log(1 - conf);


    List<String> features = new ArrayList<String>();
    
    TokenSequence t = ne.getTokens().get(0).getTokenSequence();
    int entityLength = ne.getTokens().size();
    int startID = ne.getTokens().get(0).getId();
    int endID = startID + entityLength - 1;
    String surf = ne.getSurface();

View Full Code Here

      
      Bag<String> wordCounts = new Bag<String>();
      
      ss.reset();
      for(String s : ss) {
        TokenSequence t = Tokeniser.getInstance().tokenise(s);
        for(String word : t.getTokenStringList()) {
          if(!word.matches(".*[a-z][a-z].*")) continue;
          word = StringTools.normaliseName(word);
          if(!knownWords.contains(NGram.parseWord(word))) wordCounts.add(word);
        }
      }

View Full Code Here

      Elements lineElems = elems.get(i).getChildElements();
      //List<String> wordsForWord = new ArrayList<String>();
      Set<String> wordsForWord = new HashSet<String>();
      for(int j=0;j<lineElems.size();j++) {
        String lineVal = lineElems.get(j).getValue();
        TokenSequence t = Tokeniser.getInstance().tokenise(lineVal);
        boolean found = false;
        List<String> tl = new ArrayList<String>();
        for(String s : t.getTokenStringList()) {
          if(s.compareToIgnoreCase(name) == 0) {
            tl.add("*WORD*");
            found = true;
          } else {
            //tl.add(StringTools.normaliseName2(s));

View Full Code Here

    
    int nb = 0;
    int b = 0;
    
    for(int i=0;i<n.size();i++) {
      TokenSequence t = Tokeniser.getInstance().tokenise(n.get(i).getValue());
      boolean isBoring = true;
      for(String s : t.getTokenStringList()) {
        s = StringTools.normaliseName(s);
        if(!s.matches(".*[A-Za-z].*")) continue;
        if(boring.contains(s)) continue;
        boring.add(s);
        isBoring = false;

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of uk.ac.cam.ch.wwmm.oscar3.recogniser.document.TokenSequenceSource

dk.brics.automaton.Automaton

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.