Package uk.ac.cam.ch.wwmm.oscar3.recogniser.document

Examples of uk.ac.cam.ch.wwmm.oscar3.recogniser.document.Token

156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
          if(Oscar3Props.getInstance().verbose) System.out.println("NCM: " + ne.getValue());
        }
      }
    }
   
    ProcessingDocument procDoc = ProcessingDocumentFactory.getInstance().makeTokenisedDocument(doc, true, false, false);
   
  //NameRecogniser nr = new NameRecogniser();
    //nr.halfProcess(doc);
    //if(patternFeatures) {
    //  nr.findForReps(true);
    //} else {
      //nr.makeTokenisers(true);
    //}
    for(TokenSequence ts : procDoc.getTokenSequences()) {
      trainOnSentence(ts, domain);
    }
    if(Oscar3Props.getInstance().verbose) System.out.println(System.currentTimeMillis() - time);
  }
View Full Code Here
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
    for(int i=0;i<n.size();i++) n.get(i).detach();
    n = doc.query("//ne[@type='CPR']");
    for(int i=0;i<n.size();i++) XOMTools.removeElementPreservingText((Element)n.get(i));
   
   
    ProcessingDocument procDoc = ProcessingDocumentFactory.getInstance().makeTokenisedDocument(doc, true, false, false);
    //NameRecogniser nr = new NameRecogniser();
    //nr.halfProcess(doc);
    //if(patternFeatures) {
    //  nr.findForReps(true);
    //} else {
      //nr.makeTokenisers(true);
    //}
    for(TokenSequence ts : procDoc.getTokenSequences()) {
      cvFeatures(ts, domain);
    }
    if(Oscar3Props.getInstance().verbose) System.out.println(System.currentTimeMillis() - time);
  }
View Full Code Here
114
115
116
117
118
119
120
121
122
123
124
125
    //if(patternFeatures) {
    //  nr.findForReps(true);
    //} else {
      nr.makeTokenisers(true);
    //}*/
    ProcessingDocument procDoc = ProcessingDocumentFactory.getInstance().makeTokenisedDocument(doc, false, false, false);
   
    for(TokenSequence t : procDoc.getTokenSequences()) {
      trainOnSentence(t);
    }
    System.out.println(System.currentTimeMillis() - time);
  }
View Full Code Here
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
    //NameRecogniser nr = new NameRecogniser();
    //nr.halfProcess(doc);
    //nr.makeTokenisers(true);
    Set<String> testDataSections = new LinkedHashSet<String>();

    ProcessingDocument procDoc = ProcessingDocumentFactory.getInstance().makeTokenisedDocument(doc, false, false, false);
   
    for(TokenSequence tokSeq : procDoc.getTokenSequences()) {
      Nodes neNodes = tokSeq.getElem().query(".//ne");
      for(int k=0;k<neNodes.size();k++) {
        Element neElem = (Element)neNodes.get(k);
        String neStr = "["+ neElem.getAttributeValue("xtspanstart") + ":" + neElem.getAttributeValue("xtspanend") + "]";
        testDataSections.add(neStr);
      }
    }
    List<String> tdsl = new ArrayList<String>(testDataSections);
    Collections.sort(tdsl);
    System.out.println(tdsl);
     
    Set<String> results = new HashSet<String>();
    for(TokenSequence t : procDoc.getTokenSequences()) {
      results.addAll(testOnSentence(t));
    }
    List<String> rl = new ArrayList<String>(results);
    Collections.sort(rl);
    System.out.println(rl);
View Full Code Here
61
62
63
64
65
66
67
68
69
70
71
72
73
   
   
   
    for(File file : files) {
      System.out.println(file);
      ProcessingDocument procDoc = ProcessingDocumentFactory.getInstance().makeTokenisedDocument(new Builder().build(file), false, false, false);
      Bag<String> b = new Bag<String>();
      for(TokenSequence ts : procDoc.getTokenSequences()) {
        for(Token t : ts.getTokens()) {
          String s = t.getValue().intern();
          if(!TermSets.getClosedClass().contains(s.toLowerCase())) b.add(s);
        }
      }
View Full Code Here
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
    for(int i=0;i<nodes.size();i++) {
      XOMTools.removeElementPreservingText((Element)nodes.get(i));
    }
    Document safDoc = InlineToSAF.extractSAFs(doc, sourceDoc, "foo");

    ProcessingDocument procDoc = ProcessingDocumentFactory.getInstance().makeTokenisedDocument(sourceDoc, false, false, false);

    //NameRecogniser nr = new NameRecogniser();
    //nr.halfProcess(sourceDoc);
    //nr.makeTokenisers(false);

    Nodes n = safDoc.query("/saf/annot[slot[@name='type']['PRW']]");

    Map<TokenSequence,Boolean> tokSeqs = new HashMap<TokenSequence,Boolean>();
    Map<TokenSequence,Bag<String>> tokSeqPRWs = new HashMap<TokenSequence,Bag<String>>();
   
    for(int i=0;i<n.size();i++) {
      Element e = (Element)n.get(i);
      Token token = procDoc.getTokenByStart(e.getAttributeValue("from"));
      if(token == null) token = procDoc.getTokenByEnd(e.getAttributeValue("to"));
      if(token == null) {
        System.out.println("Eeep!");
      } else {
        TokenSequence tokSeq = token.getTokenSequence();
        boolean isReact = "REACT".equals(SafTools.getSlotValue(e, "subtype"));
View Full Code Here
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
      for(int i=0;i<nodes.size();i++) {
        XOMTools.removeElementPreservingText((Element)nodes.get(i));
      }
      Document safDoc = InlineToSAF.extractSAFs(doc, sourceDoc, "foo");

      ProcessingDocument procDoc = ProcessingDocumentFactory.getInstance().makeTokenisedDocument(sourceDoc, false, false, false);
      //NameRecogniser nr = new NameRecogniser();
      //nr.halfProcess(sourceDoc);
      //nr.makeTokenisers(false);
      Set<String> tokenSet = new HashSet<String>();
      Bag<String> tokenBag = new Bag<String>();
      for(TokenSequence t : procDoc.getTokenSequences()) {
        //System.out.println(t.getSourceString());
        for(Token token : t.getTokens()) {
          //tokenSet.add("stem=" + stemmer.getStem(token.getValue().toLowerCase()));
          //tokenSet.add(token.getValue().toLowerCase());
          tokenBag.add(token.getValue().toLowerCase());
View Full Code Here
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
      //files = FileTools.getFilesFromDirectoryByName(new File("/home/ptc24/oscarworkspace/corpora/paperset1"), "source.xml");
      //files = FileTools.getFilesFromDirectoryByName(new File("/home/ptc24/newows/corpora/paperset1"), "source.xml");
      //files = FileTools.getFilesFromDirectoryByName(new File("/home/ptc24/newows/corpora/BioIE"), "source.xml");
      files = FileTools.getFilesFromDirectoryByName(new File("/home/ptc24/newows/corpora/roughPubMed"), "source.xml");
      //files = FileTools.getFilesFromDirectoryByName(new File("/scratch/pubmed/2005"), "source.xml");
      StringSource ss = new StringSource(files, false);
     
      Bag<String> wordCounts = new Bag<String>();
     
      ss.reset();
      for(String s : ss) {
        TokenSequence t = Tokeniser.getInstance().tokenise(s);
        for(String word : t.getTokenStringList()) {
          if(!word.matches(".*[a-z][a-z].*")) continue;
          word = StringTools.normaliseName(word);
View Full Code Here
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
    Set<String> engWords = new HashSet<String>(NGramBuilder.getInstance().engSet);
   
    List<File> files = new ArrayList<File>();
    files = FileTools.getFilesFromDirectoryByName(new File("/home/ptc24/newows/corpora/roughPubMed"), "source.xml");

    StringSource ss = new StringSource(files, false);
   
    Bag<String> wordCounts = new Bag<String>();
   
    ss.reset();
    for(String s : ss) {
      TokenSequence t = Tokeniser.getInstance().tokenise(s);
      for(String word : t.getTokenStringList()) {
        if(!word.matches(".*[a-z][a-z].*")) continue;
        word = StringTools.normaliseName(word);
View Full Code Here
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
        neByLastToken.put(ne.getLastToken(), ne);
      }
    }

    for(NamedEntity ne : entities) {
      Token prev = ne.getFirstToken().getNAfter(-1);
      Token next = ne.getLastToken().getNAfter(1);
      if(prev != null && next != null && prev.getValue().equals("(") && next.getValue().equals(")")) {
        Token prev2 = ne.getFirstToken().getNAfter(-2);
        if(prev2 != null) {
          String surf = ne.getSurface();
          if(surf.matches(".*[A-Z]s") || prev2.getValue().endsWith("s")) surf = surf.substring(0, surf.length()-1);
          List<String> featuresForAbbrev;
          if(abbrevFeatures.containsKey(surf)) {
            featuresForAbbrev = abbrevFeatures.get(surf);
          } else {
            featuresForAbbrev = new ArrayList<String>();
View Full Code Here
TOP

Related Classes of uk.ac.cam.ch.wwmm.oscar3.recogniser.document.Token

  • dk.brics.automaton.Automaton
  • dk.brics.automaton.RunAutomaton
  • jregex.Pattern
  • nu.xom.Attribute
  • nu.xom.Builder
  • nu.xom.Document
  • nu.xom.Element
  • nu.xom.Elements
  • nu.xom.Node
  • nu.xom.Nodes
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.