Package org.languagetool

Examples of org.languagetool.AnalyzedToken

159
160
161
162
163
164
165
166
167
168
169
170
      if (compoundParts != null) {
        // was originally a compound word
        final List<String> allButLastPart = compoundParts.subList(0, compoundParts.size() - 1);
        final String lemma = StringTools.listToString(allButLastPart, "")
            + StringTools.lowercaseFirstChar(taggerTokens[i]);
        l.add(new AnalyzedToken(word, taggerTokens[i + 1], lemma));
      } else {
        l.add(new AnalyzedToken(word, taggerTokens[i + 1], taggerTokens[i]));
      }
      i = i + 2;
    }
  }
View Full Code Here
200
201
202
203
204
205
206
    }
  }

  @Override
  public final AnalyzedTokenReadings createNullToken(final String token, final int startPos) {
    return new AnalyzedTokenReadings(new AnalyzedToken(token, null, null), startPos);
  }
View Full Code Here
205
206
207
208
209
210
211
    return new AnalyzedTokenReadings(new AnalyzedToken(token, null, null), startPos);
  }

  @Override
  public AnalyzedToken createToken(String token, String posTag) {
    return new AnalyzedToken(token, posTag, null);
  }
View Full Code Here
59
60
61
62
63
64
65
  private AnalyzedTokenReadings[] getAnalyzedTokenReadings(final String input) throws IOException {
    return languageTool.getAnalyzedSentence(input).getTokensWithoutWhitespace();
  }

  private AnalyzedTokenReadings getAnalyzedTokenReadings(String token, String posTag, String lemma) {
    return new AnalyzedTokenReadings(new AnalyzedToken(token, posTag, lemma), 0);
  }
View Full Code Here
47
48
49
50
51
52
53
54
55
56
57
    for (String inputToken : tokens) {
      String[] parts = inputToken.split("/");
      String token = parts[0];
      String lemma = parts[1];
      String posTag = parts[2];
      tokenReadings.add(new AnalyzedTokenReadings(new AnalyzedToken(token, posTag, lemma), pos++));
    }
    if (tokenReadings.size() != 2) {
      throw new RuntimeException("Size of input not yet supported: " + tokenReadings.size());
    }
    AgreementSuggestor suggestor = new AgreementSuggestor(new German().getSynthesizer(), tokenReadings.get(0), tokenReadings.get(1));
View Full Code Here
30
31
32
33
34
35
36
37
38
public class GermanHelperTest {
 
  @Test
  public void testHasReadingOfType() throws Exception {
    AnalyzedTokenReadings readings = new AnalyzedTokenReadings(new AnalyzedToken("der", "ART:DEF:DAT:SIN:FEM", null), 0);
    assertTrue(GermanHelper.hasReadingOfType(readings, GermanToken.POSType.DETERMINER));
    assertFalse(GermanHelper.hasReadingOfType(readings, GermanToken.POSType.NOMEN));
  }
View Full Code Here
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
    final Element elUpper = new Element("\\p{Lu}\\p{Ll}+", true, true, false);
    final Element elAllUpper = new Element("\\p{Lu}+$", true, true, false);
    unifierConfig.setEquivalence("case-sensitivity", "lowercase", elLower);
    unifierConfig.setEquivalence("case-sensitivity", "uppercase", elUpper);
    unifierConfig.setEquivalence("case-sensitivity", "alluppercase", elAllUpper);
    final AnalyzedToken lower1 = new AnalyzedToken("lower", "JJR", "lower");
    final AnalyzedToken lower2 = new AnalyzedToken("lowercase", "JJ", "lowercase");
    final AnalyzedToken upper1 = new AnalyzedToken("Uppercase", "JJ", "Uppercase");
    final AnalyzedToken upper2 = new AnalyzedToken("John", "NNP", "John");
    final AnalyzedToken upperAll1 = new AnalyzedToken("JOHN", "NNP", "John");
    final AnalyzedToken upperAll2 = new AnalyzedToken("JAMES", "NNP", "James");

    final Unifier uni = unifierConfig.createUnifier();

    final Map<String, List<String>> equiv = new HashMap<>();
    final List<String> list1 = new ArrayList<>();
View Full Code Here
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
    unifierConfig.setEquivalence("number", "plural",
        preparePOSElement(".*[\\.:]pl:.*"));

    final Unifier uni = unifierConfig.createUnifier();

    final AnalyzedToken sing1 = new AnalyzedToken("mały", "adj:sg:blahblah", "mały");
    final AnalyzedToken sing2 = new AnalyzedToken("człowiek", "subst:sg:blahblah", "człowiek");

    final Map<String, List<String>> equiv = new HashMap<>();
    final List<String> list1 = new ArrayList<>();
    list1.add("singular");
    equiv.put("number", list1);

    boolean satisfied = uni.isSatisfied(sing1, equiv);
    uni.startUnify();
    satisfied &= uni.isSatisfied(sing2, equiv);
    satisfied &= uni.getFinalUnificationValue(equiv);
    assertEquals(true, satisfied);
    uni.reset();

    //for multiple readings - OR for interpretations, AND for tokens
    AnalyzedToken sing1a = new AnalyzedToken("mały", "adj:pl:blahblah", "mały");
    satisfied = uni.isSatisfied(sing1, equiv);
    satisfied |= uni.isSatisfied(sing1a, equiv);
    uni.startUnify();
    satisfied &= uni.isSatisfied(sing2, equiv);
    satisfied &= uni.getFinalUnificationValue(equiv);
    assertEquals(true, satisfied);
    uni.reset();

    //check if any of the equivalences is there
    list1.add("plural");
    equiv.clear();
    equiv.put("number", list1);
    sing1a = new AnalyzedToken("mały", "adj:pl:blahblah", "mały");
    satisfied = uni.isSatisfied(sing1, equiv);
    satisfied |= uni.isSatisfied(sing1a, equiv);
    uni.startUnify();
    satisfied &= uni.isSatisfied(sing2, equiv);
    satisfied &= uni.getFinalUnificationValue(equiv);
    assertEquals(true, satisfied);
    uni.reset();

    //now test all possible feature equivalences by leaving type blank
    sing1a = new AnalyzedToken("mały", "adj:pl:blahblah", "mały");
    equiv.clear();
    equiv.put("number", null);
    satisfied = uni.isSatisfied(sing1, equiv);
    satisfied |= uni.isSatisfied(sing1a, equiv);
    uni.startUnify();
View Full Code Here
78
79
80
81
82
83
84
    // commented out as "a enumera" contains an extra form (.dict spelling error - "enumăm" instead of "enumăr"). To be fixed.
   
  }

  private AnalyzedToken dummyToken(String tokenStr) {
    return new AnalyzedToken(tokenStr, tokenStr, tokenStr);
  }
View Full Code Here
199
200
201
202
203
204
205
206
207
208
209
210
211
212
    mascElement.setPosElement(".*[\\.:]m", true, false);
    unifierConfig.setEquivalence("gender", "masculine", mascElement);

    final Unifier uni = unifierConfig.createUnifier();

    final AnalyzedToken sing1 = new AnalyzedToken("mały", "adj:sg:blahblah:m", "mały");
    final AnalyzedToken sing1a = new AnalyzedToken("mała", "adj:sg:blahblah:f", "mały");
    final AnalyzedToken sing1b = new AnalyzedToken("małe", "adj:pl:blahblah:m", "mały");
    final AnalyzedToken sing2 = new AnalyzedToken("człowiek", "subst:sg:blahblah:m", "człowiek");

    final Map<String, List<String>> equiv = new HashMap<>();
    equiv.put("number", null);
    equiv.put("gender", null);

View Full Code Here
TOP

Related Classes of org.languagetool.AnalyzedToken

  • org.apache.commons.lang.builder.EqualsBuilder
  • org.apache.commons.lang.builder.HashCodeBuilder
  • org.languagetool.chunking.EnglishChunkerTest
  • org.languagetool.chunking.EnglishChunkFilterTest
  • org.languagetool.dev.index.LanguageToolFilter
  • org.languagetool.dev.index.PatternRuleQueryBuilder
  • org.languagetool.rules.AbstractCompoundRule
  • org.languagetool.rules.ca.AccentuationCheckRule
  • org.languagetool.rules.ca.SimpleReplaceVerbsRule
  • org.languagetool.rules.ConfusionProbabilityRuleTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.