Package org.languagetool.tokenizers

Examples of org.languagetool.tokenizers.WordTokenizer

31
32
33
34
35
36
37
  private WordTokenizer tokenizer;

  @Override
  public void setUp() {
    tagger = new EsperantoTagger();
    tokenizer = new WordTokenizer();
  }
View Full Code Here
31
32
33
34
35
36
37
  private WordTokenizer tokenizer;
     
  @Override
  public void setUp() {
    tagger = new RussianTagger();
    tokenizer = new WordTokenizer();
  }
View Full Code Here
31
32
33
34
35
36
37
  private WordTokenizer tokenizer;
     
  @Override
  public void setUp() {
    tagger = new SlovakTagger();
    tokenizer = new WordTokenizer();
  }
View Full Code Here
30
31
32
33
34
35
36
37
38
39
40
41
  private static final String FILE = "/lt/performance-test/en.txt";

  protected void testPerformance(LanguageModel model, int ngramLength) throws Exception {
    try (FileInputStream fis = new FileInputStream(FILE)) {
      String content = StringTools.readStream(fis, "UTF-8");
      WordTokenizer wordTokenizer = new WordTokenizer();
      List<String> words = wordTokenizer.tokenize(content);
      String prevPrevWord = null;
      String prevWord = null;
      int i = 0;
      long totalMicros = 0;
      for (String word : words) {
View Full Code Here
41
42
43
44
45
46
47
  private WordTokenizer tokenizer;

  @Override
  public void setUp() {
    tagger = createTagger();
    tokenizer = new WordTokenizer();
  }
View Full Code Here
31
32
33
34
35
36
37
  private WordTokenizer tokenizer;

  @Override
  public void setUp() {
    tagger = new FrenchTagger();
    tokenizer = new WordTokenizer();
  }
View Full Code Here
42
43
44
45
46
47
48
49
50
51
52
  private JLanguageTool lt;
 
  @Override
  public void setUp() throws IOException {
    tagger = new FrenchTagger();
    tokenizer = new WordTokenizer();
    French language = new French();
    sentenceTokenizer = new SRXSentenceTokenizer(language);
    disambiguator = new XmlRuleDisambiguator(language);
    disamb2 = new DemoDisambiguator();   
    lt = new JLanguageTool(language);
View Full Code Here
31
32
33
34
35
36
37
  private WordTokenizer tokenizer;
     
  @Override
  public void setUp() {
    tagger = new SwedishTagger();
    tokenizer = new WordTokenizer();
  }
View Full Code Here
31
32
33
34
35
36
37
  private WordTokenizer tokenizer;
     
  @Override
  public void setUp() {
    tagger = new DutchTagger();
    tokenizer = new WordTokenizer();
  }
View Full Code Here
31
32
33
34
35
36
37
  private WordTokenizer tokenizer;

  @Override
  public void setUp() {
    tagger = new ItalianTagger();
    tokenizer = new WordTokenizer();
  }
View Full Code Here
TOP

Related Classes of org.languagetool.tokenizers.WordTokenizer

  • org.languagetool.languagemodel.LanguageModelTest
  • org.languagetool.tagging.ca.CatalanTaggerTest
  • org.languagetool.tagging.disambiguation.PolishDisambiguatorTest
  • org.languagetool.tagging.disambiguation.rules.en.EnglishRuleDisambiguatorTest
  • org.languagetool.tagging.disambiguation.rules.fr.FrenchRuleDisambiguatorTest
  • org.languagetool.tagging.en.EnglishTaggerTest
  • org.languagetool.tagging.eo.EsperantoTaggerTest
  • org.languagetool.tagging.es.SpanishTaggerTest
  • org.languagetool.tagging.fr.FrenchTaggerTest
  • org.languagetool.tagging.gl.GalicianTaggerTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.