Examples of AnalyzedTokenReadings

org.languagetool.AnalyzedTokenReadings
An array of {@link AnalyzedToken}s used to store multiple POS tags and lemmas for a given single token. @author Marcin Milkowski

Examples of org.languagetool.AnalyzedTokenReadings

   * org.languagetool.tagging.Tagger#createNullToken(java.lang.String
   * , int)
   */  
  @Override
  public final AnalyzedTokenReadings createNullToken(final String token, final int startPos) {
    return new AnalyzedTokenReadings(new AnalyzedToken(token, null, null), startPos);
  }

View Full Code Here

Examples of org.languagetool.AnalyzedTokenReadings

    for (String word : sentenceTokens) {
      final List<AnalyzedToken> l = new ArrayList<>();
      // a real tagger would need to assign a POS tag
      // in the next line instead of null:
      l.add(new AnalyzedToken(word, null, null));      
      tokenReadings.add(new AnalyzedTokenReadings(l, 0));
    }
    return tokenReadings;
  }

View Full Code Here

Examples of org.languagetool.AnalyzedTokenReadings

    return tokenReadings;
  }


  @Override
  public AnalyzedTokenReadings createNullToken(String token, int startPos) {
    return new AnalyzedTokenReadings(new AnalyzedToken(token, null, null), startPos);
  }

View Full Code Here

Examples of org.languagetool.AnalyzedTokenReadings

                if (newTokenReadings[i].getLemma() == null) { //empty lemma
                  lemma = token;
                } else {
                  lemma = newTokenReadings[i].getLemma();
                }
                final AnalyzedTokenReadings toReplace = new AnalyzedTokenReadings(
                        new AnalyzedToken(token, newTokenReadings[i].getPOSTag(), lemma),
                        whTokens[fromPos].getStartPos());
                whTokens[position] = replaceTokens(whTokens[position], toReplace);
              }
            }
          } else if (matchElement == null) {
            String lemma = "";
            for (int l = 0; l < numRead; l++) {
              if (whTokens[fromPos].getAnalyzedToken(l).getPOSTag() != null
                      && (whTokens[fromPos].getAnalyzedToken(l).getPOSTag().equals(
                      disambiguatedPOS) && (whTokens[fromPos].getAnalyzedToken(l)
                      .getLemma() != null))) {
                lemma = whTokens[fromPos].getAnalyzedToken(l).getLemma();
              }
            }
            if (StringTools.isEmpty(lemma)) {
              lemma = whTokens[fromPos].getAnalyzedToken(0).getLemma();
            }


            final AnalyzedTokenReadings toReplace = new AnalyzedTokenReadings(
                    new AnalyzedToken(whTokens[fromPos].getToken(), disambiguatedPOS,
                            lemma), whTokens[fromPos].getStartPos());
            whTokens[fromPos] = replaceTokens(whTokens[fromPos], toReplace);
          } else {
            // using the match element

View Full Code Here

Examples of org.languagetool.AnalyzedTokenReadings

  
  private AnalyzedTokenReadings setAndAnnotate(final AnalyzedTokenReadings oldReading, 
          final AnalyzedToken newReading) {      
      final String old = oldReading.toString();
      final String prevAnot = oldReading.getHistoricalAnnotations();
      final AnalyzedTokenReadings newAtr = new AnalyzedTokenReadings(oldReading.getReadings(), 
              oldReading.getStartPos());
      newAtr.setWhitespaceBefore(oldReading.isWhitespaceBefore());
      newAtr.addReading(newReading);
      newAtr.setHistoricalAnnotations(
              annotateToken(prevAnot, old, newAtr.toString()));
      return newAtr;
  }

View Full Code Here

Examples of org.languagetool.AnalyzedTokenReadings

    for (int i = 0; i < anTokens.length; i++) {


      if (anTokens[i].getReadingsLength() > 1) {
        final AnalyzedToken[] firstToken = new AnalyzedToken[1];
        firstToken[0] = anTokens[i].getAnalyzedToken(0);
        output[i] = new AnalyzedTokenReadings(firstToken, anTokens[i].getStartPos());
      } else {
        output[i] = anTokens[i];
      }
    }
    return new AnalyzedSentence(output);

View Full Code Here

Examples of org.languagetool.AnalyzedTokenReadings

      }


      if (l.isEmpty()) {
        l.add(new AnalyzedToken(word, null, null));
      }      
      tokenReadings.add(new AnalyzedTokenReadings(l, pos));
      pos += word.length();
    }


    return tokenReadings;
  }

View Full Code Here

Examples of org.languagetool.AnalyzedTokenReadings

              continue;
            }
            l.add(new AnalyzedToken(word, null, null));
          }
        }
        tokenReadings.add(new AnalyzedTokenReadings(l, pos));
        pos += word.length();
        break;
      }
    }

View Full Code Here

Examples of org.languagetool.AnalyzedTokenReadings

        }
      } else {
        // Single letter word (no tag).
        l.add(new AnalyzedToken(word, null, null));
      }
      tokenReadings.add(new AnalyzedTokenReadings(l, 0));
    }
    return tokenReadings;
  }

View Full Code Here

Examples of org.languagetool.AnalyzedTokenReadings

    return tokenReadings;
  }


  @Override
  public AnalyzedTokenReadings createNullToken(String token, int startPos) {
    return new AnalyzedTokenReadings(
      new AnalyzedToken(token, null, null), startPos);
  }

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.