Package uk.ac.cam.ch.wwmm.oscar3.preprocessor

Examples of uk.ac.cam.ch.wwmm.oscar3.preprocessor.PlainTextPreprocessor

61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
      for(int i=0;i<nodes.size();i++) {
        XOMTools.removeElementPreservingText((Element)nodes.get(i));
      }
      Document safDoc = InlineToSAF.extractSAFs(doc, sourceDoc, "foo");

      ProcessingDocument procDoc = ProcessingDocumentFactory.getInstance().makeTokenisedDocument(sourceDoc, false, false, false);
      //NameRecogniser nr = new NameRecogniser();
      //nr.halfProcess(sourceDoc);
      //nr.makeTokenisers(false);
      Set<String> tokenSet = new HashSet<String>();
      Bag<String> tokenBag = new Bag<String>();
      for(TokenSequence t : procDoc.getTokenSequences()) {
        //System.out.println(t.getSourceString());
        for(Token token : t.getTokens()) {
          //tokenSet.add("stem=" + stemmer.getStem(token.getValue().toLowerCase()));
          //tokenSet.add(token.getValue().toLowerCase());
          tokenBag.add(token.getValue().toLowerCase());
View Full Code Here
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
      //files = FileTools.getFilesFromDirectoryByName(new File("/home/ptc24/oscarworkspace/corpora/paperset1"), "source.xml");
      //files = FileTools.getFilesFromDirectoryByName(new File("/home/ptc24/newows/corpora/paperset1"), "source.xml");
      //files = FileTools.getFilesFromDirectoryByName(new File("/home/ptc24/newows/corpora/BioIE"), "source.xml");
      files = FileTools.getFilesFromDirectoryByName(new File("/home/ptc24/newows/corpora/roughPubMed"), "source.xml");
      //files = FileTools.getFilesFromDirectoryByName(new File("/scratch/pubmed/2005"), "source.xml");
      StringSource ss = new StringSource(files, false);
     
      Bag<String> wordCounts = new Bag<String>();
     
      ss.reset();
      for(String s : ss) {
        TokenSequence t = Tokeniser.getInstance().tokenise(s);
        for(String word : t.getTokenStringList()) {
          if(!word.matches(".*[a-z][a-z].*")) continue;
          word = StringTools.normaliseName(word);
View Full Code Here
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
    Set<String> engWords = new HashSet<String>(NGramBuilder.getInstance().engSet);
   
    List<File> files = new ArrayList<File>();
    files = FileTools.getFilesFromDirectoryByName(new File("/home/ptc24/newows/corpora/roughPubMed"), "source.xml");

    StringSource ss = new StringSource(files, false);
   
    Bag<String> wordCounts = new Bag<String>();
   
    ss.reset();
    for(String s : ss) {
      TokenSequence t = Tokeniser.getInstance().tokenise(s);
      for(String word : t.getTokenStringList()) {
        if(!word.matches(".*[a-z][a-z].*")) continue;
        word = StringTools.normaliseName(word);
View Full Code Here
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
        neByLastToken.put(ne.getLastToken(), ne);
      }
    }

    for(NamedEntity ne : entities) {
      Token prev = ne.getFirstToken().getNAfter(-1);
      Token next = ne.getLastToken().getNAfter(1);
      if(prev != null && next != null && prev.getValue().equals("(") && next.getValue().equals(")")) {
        Token prev2 = ne.getFirstToken().getNAfter(-2);
        if(prev2 != null) {
          String surf = ne.getSurface();
          if(surf.matches(".*[A-Z]s") || prev2.getValue().endsWith("s")) surf = surf.substring(0, surf.length()-1);
          List<String> featuresForAbbrev;
          if(abbrevFeatures.containsKey(surf)) {
            featuresForAbbrev = abbrevFeatures.get(surf);
          } else {
            featuresForAbbrev = new ArrayList<String>();
View Full Code Here
90
91
92
93
94
95
96
97
98
99
100
101
102
  public Document runGenia(Document sourceDoc) throws Exception {
    ProcessingDocument procDoc = ProcessingDocumentFactory.getInstance().makeTokenisedDocument(sourceDoc, false, false, true);
    Document safDoc = new Document(new Element("saf"));
    for(List<Token> sentence : procDoc.getSentences()) {
      if(sentence.size() > 0) {
        Token first = sentence.get(0);
        Token last = sentence.get(sentence.size()-1);
        Element sentenceAnnot = SafTools.makeAnnot(first.getStartXPoint(), last.getEndXPoint(),  "sentence");
        safDoc.getRootElement().appendChild(sentenceAnnot);
      }
    }
    for(TokenSequence ts : procDoc.getTokenSequences()) {
      for(Token t : ts.getTokens()) {
View Full Code Here
157
158
159
160
161
162
163
164
165
166
167
168
      if(line.matches("\\s*")) {
        writer.flush();
        line = null;
      } else {
        String [] sss = line.split("\t");
        Token t = sentence.get(index);
        t.setGeniaData(sss);
        line = bufferedReader.readLine();       
        index++;
      }
    }

View Full Code Here
213
214
215
216
217
218
219
220
221
222
    //  System.out.println();
    //}
  }
 
  private static NamedEntity makeNE(List<Token> neTokens, String neType) {
    Token firstToken = neTokens.get(0);
    Token lastToken = neTokens.get(neTokens.size()-1);
    String surf = firstToken.getTokenSequence().getSubstring(firstToken.getId(), lastToken.getId());
    return new NamedEntity(neTokens, surf, "GENIA-" + neType);
  }
View Full Code Here
161
162
163
164
165
166
167
168
169
170
171
172
  private void makeFeatures(int position) {
    List<String> local = features.get(position);
    List<String> contextable = contextableFeatures.get(position);
    List<String> bigramable = bigramableFeatures.get(position);

    Token token = tokSeq.getToken(position);
    String word = token.getValue();
    contextable.add(makeWordFeature(word));

    String normWord = StringTools.normaliseName(word);
    if (!word.equals(normWord)) {
      contextable.add(makeWordFeature(normWord));
View Full Code Here
32
33
34
35
36
37
38
39
40
41
42
43
  int truePos;
  int falsePos;
  int falseNeg;
 
  private String repForToken(TokenSequence t, int i) {
    Token token = t.getToken(i);
    String s = token.getValue();
    //if(s.equals("(")) {
    //  Token tt = token.getNAfter(1);
    //  if(tt != null && tt.getValue().equals("Found")) s += tt.getValue();
    //}
    //if(token.isCompRef()) return "$CR";
View Full Code Here
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
    String tag = bestLast;
    for(int i=t.size()-1;i>=0;i--) {
      tags.set(i, tag);
      tag = prevsByLabel.get(i).get(tag);
    }
    Token startToken = null;
    for(int i=0;i<t.size();i++) {
      //System.out.println(t.getToken(i).getValue() + "\t" + tags.get(i));
      if(startToken != null && !tags.get(i).matches("[IE]-ONT.*")) {
        //System.out.println(t.getSubstring(startToken.getId(), t.getToken(i-1).getId()));
        results.add("[" + startToken.getStart() + ":" + t.getToken(i-1).getEnd() + "]");
        startToken = null;
      }
      if(tags.get(i).equals("B-ONT")) startToken = t.getToken(i);
    }
    if(startToken != null) {
      results.add("[" + startToken.getStart() + ":" + t.getToken(t.size()-1).getEnd() + "]");
      //System.out.println(t.getSubstring(startToken.getId(), t.size()-1));
    }
    return results;
  }
View Full Code Here
TOP

Related Classes of uk.ac.cam.ch.wwmm.oscar3.preprocessor.PlainTextPreprocessor

  • dk.brics.automaton.Automaton
  • jregex.Pattern
  • nu.xom.Attribute
  • nu.xom.Builder
  • nu.xom.Document
  • nu.xom.Element
  • nu.xom.Elements
  • nu.xom.Node
  • nu.xom.Nodes
  • nu.xom.Serializer
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.