Package org.apache.uima.examples.opennlp

Examples of org.apache.uima.examples.opennlp.Token

204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
      int mapIdx = 0;

      // iterate over Tokens
      FSIterator tokenIterator = tokenIndex.subiterator(sentence);
      while (tokenIterator.hasNext()) {
        Token token = (Token) tokenIterator.next();

        String word = escapeToken(token.getCoveredText());

        int start = sentenceBuffer.length();
        int end = start + word.length();

        int origIdx = token.getBegin();
        for (mapIdx = start; mapIdx <= end; mapIdx++) {
          offsetMap.putMapping(mapIdx, origIdx);
          if (origIdx < token.getEnd())
            origIdx++;
        }

        sentenceBuffer.append(word + " ");
        wordList.add(word);
View Full Code Here
83
84
85
86
87
88
89
90
91
92
93
94
95
96
      String text = sentence.getCoveredText();
      Span[] tokenSpans = tokenizer.tokenizePos(text);
      for (int i = 0; i < tokenSpans.length; i++) {
        Span span = tokenSpans[i];
        Token token = new Token(aJCas);
        token.setBegin(sentence.getBegin() + span.getStart());
        token.setEnd(sentence.getBegin() + span.getEnd());
        token.setComponentId(COMPONENT_ID);
        token.addToIndexes();
      }
    }

  }
View Full Code Here
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
      wordList.clear();

      // iterate over Tokens
      FSIterator tokenIterator = tokenIndex.subiterator(sentence);
      while (tokenIterator.hasNext()) {
        Token token = (Token) tokenIterator.next();

        tokenList.add(token);
        wordList.add(token.getCoveredText());
      }

      for (int i = 0; i < numNefs; i++) {
        Constructor annotationMaker = neAnnotationMakers[i];
        finderTags = nameFinders[i].find(wordList, Collections.EMPTY_MAP);

        boolean inTag = false;
        int tagStart = 0;
        int tagEnd = 0;
        for (int j = 0; j < finderTags.size(); j++) {
          String tag = (String) finderTags.get(j);

          if (inTag) {
            // check for end tags
            if (tag.equals(NameFinderME.START) || tag.equals(NameFinderME.OTHER)) {
              // make annotation
              tagEnd = j - 1;
              Token startToken = (Token) tokenList.get(tagStart);
              Token endToken = (Token) tokenList.get(tagEnd);
              makeEntityAnnotation(annotationMaker, aJCas, startToken.getBegin(), endToken.getEnd());
              inTag = false;
            }
          }
          if (!inTag) {
            // check for start tags
View Full Code Here
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
      wordList.clear();

      // iterate over Tokens
      FSIterator tokenIterator = tokenIndex.subiterator(sentence);
      while (tokenIterator.hasNext()) {
        Token token = (Token) tokenIterator.next();

        tokenList.add(token);
        wordList.add(token.getCoveredText());
      }

      List wordTagList = tagger.tag(wordList);

      try {
        for (int i = 0; i < tokenList.size(); i++) {
          Token token = (Token) tokenList.get(i);
          String posTag = (String) wordTagList.get(i);
          token.setPosTag(posTag);
        }
      } catch (IndexOutOfBoundsException e) {
        System.err.println("POS tagger error - list of tags shorter than list of words");
      }
    }
View Full Code Here
TOP

Related Classes of org.apache.uima.examples.opennlp.Token

  • org.apache.uima.examples.opennlp.annotator.NEDetector
  • org.apache.uima.examples.opennlp.annotator.Parser
  • org.apache.uima.examples.opennlp.annotator.POSTagger
  • org.apache.uima.examples.opennlp.annotator.Tokenizer
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.