Package org.languagetool

Examples of org.languagetool.AnalyzedToken


import org.languagetool.AnalyzedToken;

public class EnglishSynthesizerTest extends TestCase {

  private AnalyzedToken dummyToken(String tokenStr) {
    return new AnalyzedToken(tokenStr, tokenStr, tokenStr);
  }
View Full Code Here


    for (String word : sentenceTokens) {

      final List<AnalyzedToken> l = new ArrayList<>();

      AnalyzedToken at = asAnalyzedToken(word);
      l.add(at);
      tokenReadings.add(new AnalyzedTokenReadings(l, pos));
      pos += at.getToken().length();
    }

    return tokenReadings;
  }
View Full Code Here

    return tokenReadings;
  }

  @Override
  public final AnalyzedTokenReadings createNullToken(final String token, final int startPos) {
    return new AnalyzedTokenReadings(new AnalyzedToken(token, null, null), startPos);
  }
View Full Code Here

    return new AnalyzedTokenReadings(new AnalyzedToken(token, null, null), startPos);
  }

  @Override
  public AnalyzedToken createToken(String token, String posTag) {
    return new AnalyzedToken(token, posTag, null);
  }
View Full Code Here

    return new AnalyzedToken(token, posTag, null);
  }

  private AnalyzedToken asAnalyzedToken(final String word) {
    if (word.indexOf("/") < 0) {
      return new AnalyzedToken(" ", null, null);
    }
    String[] parts = word.split("/");
    return new AnalyzedToken(parts[0], parts[1], null);
  }
View Full Code Here

    }

    // It must clear attributes, as it is creating new tokens.
    clearAttributes();
    final AnalyzedTokenReadings tr = tokenIter.next();
    AnalyzedToken at = tr.getAnalyzedToken(0);

    // add POS tag for sentence start.
    if (tr.isSentStart()) {
      // TODO: would be needed so negated tokens can match on something (see testNegatedMatchAtSentenceStart())
      // but breaks other cases:
      //termAtt.append("SENT_START");
      typeAtt.setType("pos");
      if (toLowerCase) {
        termAtt.append(POS_PREFIX.toLowerCase() + tr.getAnalyzedToken(0).getPOSTag().toLowerCase());
      } else {
        termAtt.append(POS_PREFIX + tr.getAnalyzedToken(0).getPOSTag());
      }
      return true;
    }

    // by pass the white spaces.
    if (tr.isWhitespace()) {
      return this.incrementToken();
    }

    offsetAtt.setOffset(tr.getStartPos(), tr.getStartPos() + at.getToken().length());

    for (int i = 0; i < tr.getReadingsLength(); i++) {
      at = tr.getAnalyzedToken(i);
      if (at.getPOSTag() != null) {
        if (toLowerCase) {
          posStack.push(POS_PREFIX.toLowerCase() + at.getPOSTag().toLowerCase());
        } else {
          posStack.push(POS_PREFIX + at.getPOSTag());
        }
      }
    }

    current = captureState();
View Full Code Here

    final Queue<AnalyzedTokenReadings> prevTokens = new ArrayBlockingQueue<>(MAX_TERMS);
    for (int i = 0; i < tokens.length + MAX_TERMS-1; i++) {
      final AnalyzedTokenReadings token;
      // we need to extend the token list so we find matches at the end of the original list:
      if (i >= tokens.length) {
        token = new AnalyzedTokenReadings(new AnalyzedToken("", "", null), prevTokens.peek().getStartPos());
      } else {
        token = tokens[i];
      }
      if (i == 0) {
        addToQueue(token, prevTokens);
View Full Code Here

        List<AnalyzedToken> additionalTaggedTokens = additionalTags(word);
        addTokens(additionalTaggedTokens, l);      
      }
     
      if (l.isEmpty()) {
        l.add(new AnalyzedToken(word, null, null));
      }
     
      tokenReadings.add(new AnalyzedTokenReadings(l, pos));
      pos += word.length();
    }
View Full Code Here

    }
    return aTokenList;
  }

  protected AnalyzedToken asAnalyzedToken(final String word, final WordData wd) {
    return new AnalyzedToken(
        word,
        StringTools.asString(wd.getTag()),
        StringTools.asString(wd.getStem()));
  }
View Full Code Here

   * org.languagetool.tagging.Tagger#createNullToken(java.lang.String
   * , int)
   */ 
  @Override
  public final AnalyzedTokenReadings createNullToken(final String token, final int startPos) {
    return new AnalyzedTokenReadings(new AnalyzedToken(token, null, null), startPos);
  }
View Full Code Here

TOP

Related Classes of org.languagetool.AnalyzedToken

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.