Package org.languagetool.tokenizers

Examples of org.languagetool.tokenizers.Tokenizer.tokenize()

74
75
76
77
78
79
80
81
82
83
84
      System.out.println("Checking " + file.getAbsolutePath());
      String text = StringTools.readFile(new FileInputStream(file.getAbsolutePath()));
      text = textFilter.filter(text);
      if (CHECK_BY_SENTENCE) {
        final Tokenizer sentenceTokenizer = langTool.getLanguage().getSentenceTokenizer();
        final List<String> sentences = sentenceTokenizer.tokenize(text);
        for (String sentence : sentences) {
          Tools.checkText(sentence, langTool, false, 1000);
        }
      } else {
        Tools.checkText(text, langTool);
View Full Code Here
87
88
89
90
91
92
93
94
95
96
97
        while (scanner.hasNextLine()) {
          String line = scanner.nextLine();
          int sepPos = separatorChar.isEmpty() ? -1 : line.indexOf(separatorChar);
          String occurrences = sepPos != -1 ? line.substring(sepPos + separatorChar.length()) : "";
          String lineWithoutOcc = sepPos != -1 ? line.substring(0, sepPos) : line;
          List<String> tokens = wordTokenizer.tokenize(lineWithoutOcc);
          for (String token : tokens) {
            if (token.length() > 0) {
              out.write(token);
              if (sepPos != -1) {
                out.write(separatorChar);
View Full Code Here
73
74
75
76
77
78
79
80
81
82
83
      System.out.println("Checking " + file.getAbsolutePath());
      String text = StringTools.readStream(new FileInputStream(file.getAbsolutePath()), "utf-8");
      text = textFilter.filter(text);
      if (CHECK_BY_SENTENCE) {
        final Tokenizer sentenceTokenizer = langTool.getLanguage().getSentenceTokenizer();
        final List<String> sentences = sentenceTokenizer.tokenize(text);
        for (String sentence : sentences) {
          CommandLineTools.checkText(sentence, langTool, false, 1000);
        }
      } else {
        CommandLineTools.checkText(text, langTool);
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.