Examples of AnalyzedToken

Examples of org.languagetool.AnalyzedToken

33
34
35
36
37
38
39
40
41
42
43
44
45
    final List<AnalyzedTokenReadings> tokenReadings = new ArrayList<>();
    int pos = 0;

    for (String word : sentenceTokens) {
      final List<AnalyzedToken> l = new ArrayList<>();
      AnalyzedToken at = asAnalyzedToken(word);
      l.add(at);
      tokenReadings.add(new AnalyzedTokenReadings(l, pos));
      pos += at.getToken().length();
    }

    return tokenReadings;
  }
View Full Code Here

Examples of org.languagetool.AnalyzedToken

44
45
46
47
48
49
50
    return tokenReadings;
  }

  @Override
  public final AnalyzedTokenReadings createNullToken(final String token, final int startPos) {
    return new AnalyzedTokenReadings(new AnalyzedToken(token, null, null), startPos);
  }
View Full Code Here

Examples of org.languagetool.AnalyzedToken

49
50
51
52
53
54
55
    return new AnalyzedTokenReadings(new AnalyzedToken(token, null, null), startPos);
  }

  @Override
  public AnalyzedToken createToken(String token, String posTag) {
    return new AnalyzedToken(token, posTag, null);
  }
View Full Code Here

Examples of org.languagetool.AnalyzedToken

55
56
57
58
59
60
61
62
63
  }

  private AnalyzedToken asAnalyzedToken(final String word) {
    String[] parts = word.split(" ");
    if (parts.length != 3) {
      return new AnalyzedToken(" ", null, null);
    }
    return new AnalyzedToken(parts[0], parts[1], parts[2]);
  }
View Full Code Here

Examples of org.languagetool.AnalyzedToken

37
38
39
40
41
42
43
44
45
46
      final List<AnalyzedTokenReadings> tokenReadings = new ArrayList<>();
      for (String word : sentenceTokens) {
        final List<AnalyzedToken> l = new ArrayList<>();
        // a real tagger would need to assign a POS tag
        // in the next line instead of null:
        l.add(new AnalyzedToken(word, null, null));
        tokenReadings.add(new AnalyzedTokenReadings(l, 0));
      }
      return tokenReadings;
    }
View Full Code Here

Examples of org.languagetool.AnalyzedToken

45
46
47
48
49
50
51
52
      return tokenReadings;
    }

    @Override
    public AnalyzedTokenReadings createNullToken(String token, int startPos) {
      return new AnalyzedTokenReadings(new AnalyzedToken(token, null, null),
              startPos);
    }
View Full Code Here

Examples of org.languagetool.AnalyzedToken

51
52
53
54
55
56
57
              startPos);
    }

    @Override
    public AnalyzedToken createToken(String token, String posTag) {
      return new AnalyzedToken(token, posTag, null);
    }
View Full Code Here

Examples of org.languagetool.AnalyzedToken

26
27
28
29
30
31
32
import org.languagetool.AnalyzedToken;

public class SpanishSynthesizerTest extends TestCase {
    private final AnalyzedToken dummyToken(String tokenStr) {
      return new AnalyzedToken(tokenStr, tokenStr, tokenStr);
    }
View Full Code Here

Examples of org.languagetool.AnalyzedToken

216
217
218
219
220
221
222
223
224
225
226
   *
   * @param lemmaString String that specifies the base form.
   */
  public final void setLemmaString(final String lemmaString) {
    if (!StringTools.isEmpty(lemmaString)) {
      formattedToken = new AnalyzedTokenReadings(new AnalyzedToken(lemmaString,
          posTag, lemmaString), 0);
      staticLemma = true;
      postagRegexp = true;
      if (posTag != null) {
        pPosRegexMatch = Pattern.compile(posTag);
View Full Code Here

Examples of org.languagetool.AnalyzedToken

347
348
349
350
351
352
353
354
355
356
357
358
    if (this.suppressMisspelled && lang != null) {
      final List<String> formattedStringElements = Arrays.asList(formattedString);
      //tagger-based speller
      final List<AnalyzedTokenReadings> analyzed = lang.getTagger().tag(formattedStringElements);
      for (int i = 0; i < formattedString.length; i++) {
        final AnalyzedToken analyzedToken = analyzed.get(i).getAnalyzedToken(0);
        if (analyzedToken.getLemma() == null && analyzedToken.hasNoTag()) {
          formattedString[i] = "";
        }
      }
    }
    return formattedString;
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.