Examples of SentenceTokenizer

Examples of org.apache.lucene.analysis.cn.smart.SentenceTokenizer

164
165
166
167
168
169
170
171
172
173
174
      throws IOException {
    SavedStreams streams = (SavedStreams) getPreviousTokenStream();
    if (streams == null) {
      streams = new SavedStreams();
      setPreviousTokenStream(streams);
      streams.tokenStream = new SentenceTokenizer(reader);
      streams.filteredTokenStream = new WordTokenFilter(streams.tokenStream);
      streams.filteredTokenStream = new PorterStemFilter(streams.filteredTokenStream);
      if (!stopWords.isEmpty()) {
        streams.filteredTokenStream = new StopFilter(matchVersion, streams.filteredTokenStream, stopWords, false);
      }
View Full Code Here

Examples of org.languagetool.tokenizers.SentenceTokenizer

37
38
39
40
41
42
43
44
45
 
  @Override
  public void setUp() {
    tagger = new EnglishTagger();
    tokenizer = new WordTokenizer();
    sentenceTokenizer = new SentenceTokenizer();
    disambiguator = new EnglishRuleDisambiguator();
    disamb2 = new DemoDisambiguator();
  }
View Full Code Here

Examples of org.languagetool.tokenizers.SentenceTokenizer

38
39
40
41
42
43
44
45
46
  @Override
  public void setUp() {
    tagger = new RomanianTagger();
    tokenizer = new RomanianWordTokenizer();
    sentenceTokenizer = new SentenceTokenizer();
    disambiguator = new RomanianRuleDisambiguator();
    disamb2 = new DemoDisambiguator();
  }
View Full Code Here

Examples of org.languagetool.tokenizers.SentenceTokenizer

41
42
43
44
45
46
47
48
49
50
51
 
  @Override
  public void setUp() {
    tagger = new FrenchTagger();
    tokenizer = new WordTokenizer();
    sentenceTokenizer = new SentenceTokenizer();
    disambiguator = new FrenchRuleDisambiguator();
    disamb2 = new DemoDisambiguator();   
    try {
      lt = new JLanguageTool(new French());
    } catch (IOException e) {
View Full Code Here

Examples of org.languagetool.tokenizers.SentenceTokenizer

108
109
110
111
112
113
114
115
116
117
118
      printMatches(ruleMatches, prevMatches, contents, contextSize);
    }

    //display stats if it's not in a buffered mode
    if (xmlMode == StringTools.XmlPrintMode.NORMAL_XML) {
      SentenceTokenizer sentenceTokenizer = lt.getLanguage().getSentenceTokenizer();
      int sentenceCount = sentenceTokenizer.tokenize(contents).size();
      displayTimeStats(startTime, sentenceCount, apiFormat);
    }
    return ruleMatches.size();
  }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.