Package org.languagetool

Examples of org.languagetool.AnalyzedTokenReadings

164
165
166
167
168
169
170
171
172
173
174
          break;
        }
      }
      if (unified) {
        if (tokCnt == 0 || tokSequence.isEmpty()) {
          tokSequence.add(new AnalyzedTokenReadings(aToken, 0));
          List<Map<String, Set<String>>> equivList = new ArrayList<>();
          equivList.add(equivalencesMatched.get(tokCnt));
          tokSequenceEquivalences.add(equivList);
        } else {
          tokSequence.get(0).addReading(aToken);
View Full Code Here
225
226
227
228
229
230
231
232
233
234
235
        tokenFeaturesFound.set(i, tokenFeaturesFound.get(i) || allFeatsUnified);
        anyFeatUnified = anyFeatUnified || allFeatsUnified;
      }
      if (anyFeatUnified) {
        if (tokSequence.size() == readingsCounter) {
          tokSequence.add(new AnalyzedTokenReadings(aToken, 0));
          List<Map<String, Set<String>>> equivList = new ArrayList<>();
          equivList.add(equivalencesMatchedHere);
          tokSequenceEquivalences.add(equivList);
        } else {
          if (readingsCounter < tokSequence.size()) {
View Full Code Here
373
374
375
376
377
378
379
380
381
382
383
    return uTokens.toArray(new AnalyzedTokenReadings[uTokens.size()]);
  }

  private void addTokenToSequence(List<AnalyzedTokenReadings> tokenSequence, AnalyzedToken token, int pos) {
    if (tokenSequence.size() <= pos || tokenSequence.isEmpty()) {
      AnalyzedTokenReadings tmpATR = new AnalyzedTokenReadings(token, 0);
      tokenSequence.add(tmpATR);
    } else {
      tokenSequence.get(pos).addReading(token);
    }
  }
View Full Code Here
51
52
53
54
55
56
57
58
59
60
61
  @Test
  public void testPluralByPluralNoun() throws IOException {
    String input = "I/X have/N-VP ten/B-NP books/I-NP ./.";
    List<ChunkTaggedToken> tokens = makeTokens(input);
    tokens.remove(3);  // 'books'
    AnalyzedTokenReadings readings = new AnalyzedTokenReadings(Arrays.asList(
            new AnalyzedToken("books", "NNS", "book"),
            new AnalyzedToken("books", "VBZ", "book")),
            0
    );
    tokens.add(3, new ChunkTaggedToken("books", Collections.singletonList(new ChunkTag("I-NP")), readings));
View Full Code Here
77
78
79
80
81
82
83
84
85
86
87
88
89
    List<AnalyzedTokenReadings> result = new ArrayList<>();
    int pos = 0;
    while (tokenizer.hasMoreTokens()) {
      String token = tokenizer.nextToken();
      if (token.trim().isEmpty()) {
        result.add(new AnalyzedTokenReadings(new AnalyzedToken(token, null, null), pos));
      } else {
        result.add(new AnalyzedTokenReadings(new AnalyzedToken(token, "fake", "fake"), pos));
      }
      pos += token.length();
    }
    return result;
  }
View Full Code Here
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
      return false;
    }

    if (!precSpace && follSpace || tokens[i].isSentenceEnd()) {
      // exception for English inches, e.g., 20"
      final AnalyzedTokenReadings prevToken = tokens[i - 1];
      if ("\"".equals(tokenStr))
           {
        if (!symbolStack.empty() && "\"".equals(symbolStack.peek().getSymbol())) {
          return true;
        } else if (NUMBER.matcher(prevToken.getToken()).matches()) {
          return false;
        }
      }
      // Exception for English plural Saxon genitive
      if (("'".equals(tokenStr) || "’".equals(tokenStr)) && tokens[i].hasPosTag("POS")) {
        return false;
      }
      // puttin' on the Ritz
      if ("'".equals(tokenStr) && prevToken.hasPosTag("VBG")
          && prevToken.getToken().endsWith("in")) {
        return false;
      }
    }
    if (precSpace && !follSpace) {
      if ("'".equals(tokenStr) && i + 1 < tokens.length) {
View Full Code Here
TOP

Related Classes of org.languagetool.AnalyzedTokenReadings

  • org.languagetool.chunking.EnglishChunker
  • org.languagetool.chunking.EnglishChunkerTest
  • org.languagetool.chunking.EnglishChunkFilterTest
  • org.languagetool.dev.index.LanguageToolFilter
  • org.languagetool.rules.AbstractCompoundRule
  • org.languagetool.rules.ca.AccentuationCheckRule
  • org.languagetool.rules.ca.SimpleReplaceVerbsRule
  • org.languagetool.rules.ConfusionProbabilityRule
  • org.languagetool.rules.ConfusionProbabilityRuleTest
  • org.languagetool.rules.de.AgreementRule
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.