Package uk.ac.cam.ch.wwmm.oscar3.recogniser.document

Examples of uk.ac.cam.ch.wwmm.oscar3.recogniser.document.Token

57
58
59
60
61
62
63
64
65
66
67
68
     
      Bag<String> wordCounts = new Bag<String>();
     
      ss.reset();
      for(String s : ss) {
        TokenSequence t = Tokeniser.getInstance().tokenise(s);
        for(String word : t.getTokenStringList()) {
          if(!word.matches(".*[a-z][a-z].*")) continue;
          word = StringTools.normaliseName(word);
          if(!knownWords.contains(NGram.parseWord(word))) wordCounts.add(word);
        }
      } 
View Full Code Here
52
53
54
55
56
57
58
59
60
61
62
63
64
65
      Elements lineElems = elems.get(i).getChildElements();
      //List<String> wordsForWord = new ArrayList<String>();
      Set<String> wordsForWord = new HashSet<String>();
      for(int j=0;j<lineElems.size();j++) {
        String lineVal = lineElems.get(j).getValue();
        TokenSequence t = Tokeniser.getInstance().tokenise(lineVal);
        boolean found = false;
        List<String> tl = new ArrayList<String>();
        for(String s : t.getTokenStringList()) {
          if(s.compareToIgnoreCase(name) == 0) {
            tl.add("*WORD*");
            found = true;
          } else {
            //tl.add(StringTools.normaliseName2(s));
View Full Code Here
27
28
29
30
31
32
33
34
35
36
37
38
39
   
    int nb = 0;
    int b = 0;
   
    for(int i=0;i<n.size();i++) {
      TokenSequence t = Tokeniser.getInstance().tokenise(n.get(i).getValue());
      boolean isBoring = true;
      for(String s : t.getTokenStringList()) {
        s = StringTools.normaliseName(s);
        if(!s.matches(".*[A-Za-z].*")) continue;
        if(boring.contains(s)) continue;
        boring.add(s);
        isBoring = false;
View Full Code Here
58
59
60
61
62
63
64
65
66
67
68
69
   
    Bag<String> wordCounts = new Bag<String>();
   
    ss.reset();
    for(String s : ss) {
      TokenSequence t = Tokeniser.getInstance().tokenise(s);
      for(String word : t.getTokenStringList()) {
        if(!word.matches(".*[a-z][a-z].*")) continue;
        word = StringTools.normaliseName(word);
        wordCounts.add(word);
      }
    } 
View Full Code Here
96
97
98
99
100
101
102
103
104
105
106
107
108
    Collections.sort(offsetArray, offsetComparator);
    System.out.println(System.currentTimeMillis() - time); 
  }
 
  public int [] searchForString(String searchString) {
    TokenSequence ts = Tokeniser.getInstance().tokenise(searchString);
    List<Integer> tsl = new ArrayList<Integer>(ts.size() + 1);
    for(Token t : ts.getTokens()) {
      String s = t.getValue().toLowerCase();
      if(tokenIndex.containsKey(s)) {
        tsl.add(tokenIndex.get(s));
      } else {
        return null;
View Full Code Here
134
135
136
137
138
139
140
141
142
143
144
      Token token = procDoc.getTokenByStart(e.getAttributeValue("from"));
      if(token == null) token = procDoc.getTokenByEnd(e.getAttributeValue("to"));
      if(token == null) {
        System.out.println("Eeep!");
      } else {
        TokenSequence tokSeq = token.getTokenSequence();
        boolean isReact = "REACT".equals(SafTools.getSlotValue(e, "subtype"));
        if(tokSeqs.containsKey(tokSeq)) {
          if(isReact) tokSeqs.put(tokSeq, true);
        } else {
          tokSeqs.put(tokSeq, isReact);
View Full Code Here
26
27
28
29
30
31
32
33
34
35
36
        c = reader.read();
      }     
    } catch (Exception e) {
      throw new Error(e);
    }
    TokenSequence t = Tokeniser.getInstance().tokenise(sb.toString());
   
    TokenStream ts = new Oscar3TokenStream(t);
    ts = new Oscar3Filters(ts);
    return ts;
  }
View Full Code Here
134
135
136
137
138
139
140
141
142
143
144
      Token token = procDoc.getTokenByStart(e.getAttributeValue("from"));
      if(token == null) token = procDoc.getTokenByEnd(e.getAttributeValue("to"));
      if(token == null) {
        System.out.println("Eeep!");
      } else {
        TokenSequence tokSeq = token.getTokenSequence();
        //boolean isReact = "REACT".equals(SafTools.getSlotValue(e, "subtype"));
        boolean isPubmed = f.toString().contains("pubmed");
        if(tokSeqs.containsKey(tokSeq)) {
          if(isPubmed) tokSeqs.put(tokSeq, true);
        } else {
View Full Code Here
67
68
69
70
71
72
73
74
75
76
77
    //files = files.subList(0, 10);
   
   
    Bag<String> tokenBag = new Bag<String>();
   
    TokenSequenceSource tss = new TokenSequenceSource(files);
    int i=0;
    for(TokenSequence ts : tss) {
      for(Token t : ts.getTokens()) {
        tokenBag.add(t.getValue().intern());
      }
View Full Code Here
45
46
47
48
49
50
51
52
53
54
55
      corpusOffset++;
    }
  }
 
  public InverseSearcher(List<File> files) throws Exception {
    TokenSequenceSource tss = new TokenSequenceSource(files);

    corpusArray = new ArrayList<Integer>();
    offsetArray = new ArrayList<Integer>();
    tokenList = new ArrayList<String>();
    tokenIndex = new HashMap<String,Integer>();
View Full Code Here
TOP

Related Classes of uk.ac.cam.ch.wwmm.oscar3.recogniser.document.Token

  • dk.brics.automaton.Automaton
  • dk.brics.automaton.RunAutomaton
  • jregex.Pattern
  • nu.xom.Attribute
  • nu.xom.Builder
  • nu.xom.Document
  • nu.xom.Element
  • nu.xom.Elements
  • nu.xom.Node
  • nu.xom.Nodes
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.