Package org.languagetool

Examples of org.languagetool.AnalyzedTokenReadings

129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
        while (l > rule.endPositionCorrection) {
          correctedEndPos -= tokenPositions[matchingTokens + l - 1];
          l--;
        }
      }
      AnalyzedTokenReadings firstMatchTokenObj = tokens[firstMatchToken + correctedStPos];
      boolean startsWithUppercase = StringTools
        .startsWithUppercase(firstMatchTokenObj.getToken())
        && !matchConvertsCase(rule.getSuggestionMatches())
        && !matchConvertsCase(rule.getSuggestionMatchesOutMsg());

      if (firstMatchTokenObj.isSentStart()
          && tokens.length > firstMatchToken + correctedStPos + 1) {
        // make uppercasing work also at sentence start:
        firstMatchTokenObj = tokens[firstMatchToken + correctedStPos + 1];
        startsWithUppercase = StringTools.startsWithUppercase(firstMatchTokenObj.getToken());
      }
      int fromPos = tokens[firstMatchToken + correctedStPos].getStartPos();
      // FIXME: this is fishy, assumes that comma should always come before whitespace:
      if (errMessage.contains(SUGGESTION_START_TAG + ",")
          && firstMatchToken + correctedStPos >= 1) {
View Full Code Here
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
    }
    if (isSpecialCase(sentence)) {
      return new RuleMatch[0];
    }
    boolean verbFound = false;
    AnalyzedTokenReadings lastToken = null;
    int i = 0;
    for (AnalyzedTokenReadings readings : sentence.getTokensWithoutWhitespace()) {
      if (readings.hasPartialPosTag("VER") || (!readings.isTagged() && !StringTools.isCapitalizedWord(readings.getToken()))) {  // ignore unknown words to avoid false alarms
        verbFound = true;
        break;
      } else if (i == 1 && verbAtSentenceStart(readings)) {
        verbFound = true;
        break;
      }
      lastToken = readings;
      i++;
    }
    if (!verbFound && lastToken != null && i - 1 >= MIN_TOKENS_FOR_ERROR) {
      RuleMatch match = new RuleMatch(this, 0, lastToken.getStartPos() + lastToken.getToken().length(), "Dieser Satz scheint kein Verb zu enthalten");
      return new RuleMatch[]{ match };
    }
    return new RuleMatch[0];
  }
View Full Code Here
107
108
109
110
111
112
113
114
115
116
117
118
  // we want to ignore headlines, and these usually don't end with [.?!]
  private boolean isRealSentence(AnalyzedSentence sentence) {
    AnalyzedTokenReadings[] tokens = sentence.getTokensWithoutWhitespace();
    if (tokens.length > 0) {
      AnalyzedTokenReadings lastToken = tokens[tokens.length - 1];
      String lastTokenStr = lastToken.getToken();
      if (lastTokenStr.equals(".") || lastTokenStr.equals("?") || lastTokenStr.equals("!")) {
        return true;
      }
    }
    return false;
View Full Code Here
34
35
36
37
38
39
40
41
42
43
    int pos = 0;
    for (String word : sentenceTokens) {
      final List<AnalyzedToken> l = new ArrayList<>();
      AnalyzedToken at = asAnalyzedToken(word);
      l.add(at);
      tokenReadings.add(new AnalyzedTokenReadings(l, pos));
      pos += at.getToken().length();
    }
    return tokenReadings;
  }
View Full Code Here
42
43
44
45
46
47
48
    return tokenReadings;
  }

  @Override
  public final AnalyzedTokenReadings createNullToken(final String token, final int startPos) {
    return new AnalyzedTokenReadings(new AnalyzedToken(token, null, null), startPos);
  }
View Full Code Here
80
81
82
83
84
85
86
87
88
89
90
    }
  }

  public AnalyzedTokenReadings lookup(final String word) throws IOException {
    final List<AnalyzedTokenReadings> result = tag(Collections.singletonList(word), false);
    final AnalyzedTokenReadings atr = result.get(0);
    if (atr.getAnalyzedToken(0).getPOSTag() == null) {
      return null;
    }
    return atr;
  }
View Full Code Here
134
135
136
137
138
139
140
141
142
143
        } else {
          l.add(new AnalyzedToken(word, null, null));
        }
      }

      tokenReadings.add(new AnalyzedTokenReadings(l.toArray(new AnalyzedToken[l.size()]), pos));
      pos += word.length();
    }
    return tokenReadings;
  }
View Full Code Here
200
201
202
203
204
205
206
    }
  }

  @Override
  public final AnalyzedTokenReadings createNullToken(final String token, final int startPos) {
    return new AnalyzedTokenReadings(new AnalyzedToken(token, null, null), startPos);
  }
View Full Code Here
59
60
61
62
63
64
65
  private AnalyzedTokenReadings[] getAnalyzedTokenReadings(final String input) throws IOException {
    return languageTool.getAnalyzedSentence(input).getTokensWithoutWhitespace();
  }

  private AnalyzedTokenReadings getAnalyzedTokenReadings(String token, String posTag, String lemma) {
    return new AnalyzedTokenReadings(new AnalyzedToken(token, posTag, lemma), 0);
  }
View Full Code Here
47
48
49
50
51
52
53
54
55
56
57
    for (String inputToken : tokens) {
      String[] parts = inputToken.split("/");
      String token = parts[0];
      String lemma = parts[1];
      String posTag = parts[2];
      tokenReadings.add(new AnalyzedTokenReadings(new AnalyzedToken(token, posTag, lemma), pos++));
    }
    if (tokenReadings.size() != 2) {
      throw new RuntimeException("Size of input not yet supported: " + tokenReadings.size());
    }
    AgreementSuggestor suggestor = new AgreementSuggestor(new German().getSynthesizer(), tokenReadings.get(0), tokenReadings.get(1));
View Full Code Here
TOP

Related Classes of org.languagetool.AnalyzedTokenReadings

  • org.languagetool.chunking.EnglishChunker
  • org.languagetool.chunking.EnglishChunkerTest
  • org.languagetool.chunking.EnglishChunkFilterTest
  • org.languagetool.dev.index.LanguageToolFilter
  • org.languagetool.rules.AbstractCompoundRule
  • org.languagetool.rules.ca.AccentuationCheckRule
  • org.languagetool.rules.ca.SimpleReplaceVerbsRule
  • org.languagetool.rules.ConfusionProbabilityRule
  • org.languagetool.rules.ConfusionProbabilityRuleTest
  • org.languagetool.rules.de.AgreementRule
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.