Package org.apache.ctakes.core.nlp.tokenizer

Examples of org.apache.ctakes.core.nlp.tokenizer.Tokenizer

58
59
60
61
62
63
64
65
66
67
68
   * The file is delimited with "|" and has two fields:<br>
   * hyphen-term|frequency
   */
  public HyphenTextModifierImpl(String hyphenfilename, int windowSize) {
    iv_windowSize = windowSize;
    iv_tokenizer = new Tokenizer();
    BufferedReader br;
    try {
      br = new BufferedReader(new FileReader(new File(hyphenfilename)));

      String line = "";
View Full Code Here
54
55
56
57
58
59
60
61
62
63
64
  /*
   * DECPRECATED: Use InputSteam instead of filename
   */
  public HyphenTextModifierImpl(String hyphenfilename, int windowSize) {
    iv_windowSize = windowSize;
    iv_tokenizer = new Tokenizer();
    BufferedReader br;
    try {
      br = new BufferedReader(new FileReader(new File(hyphenfilename)));

      String line = "";
View Full Code Here
92
93
94
95
96
97
98
99
100
101
102
   * The file is delimited with "|" and has two fields:<br>
   * hyphen-term|frequency
   */
  public HyphenTextModifierImpl(InputStream hyphenfilename, int windowSize) {
    iv_windowSize = windowSize;
    iv_tokenizer = new Tokenizer();
    BufferedReader br;
    try {
      br = new BufferedReader(new InputStreamReader(hyphenfilename));
      String line = "";

View Full Code Here
54
55
56
57
58
59
60
61
62
63
64
  /*
   * DECPRECATED: Uses InputSteam instead
   */
  public HyphenTextModifierImpl(String hyphenfilename, int windowSize) {
    iv_windowSize = windowSize;
    iv_tokenizer = new Tokenizer();
    BufferedReader br;
    try {
      br = new BufferedReader(new FileReader(new File(hyphenfilename)));

      String line = "";
View Full Code Here
92
93
94
95
96
97
98
99
100
101
102
   * The file is delimited with "|" and has two fields:<br>
   * hyphen-term|frequency
   */
  public HyphenTextModifierImpl(InputStream hyphenfilename, int windowSize) {
    iv_windowSize = windowSize;
    iv_tokenizer = new Tokenizer();
    BufferedReader br;
    try {
      br = new BufferedReader(new InputStreamReader(hyphenfilename));
      String line = "";

View Full Code Here
107
108
109
110
111
112
113
114
115
116
117
118
119
        .getResourceObject(HYPH_FREQ_TABLE_RESRC_KEY);
    if (strIntMapResrc == null) {
      logger.warn("Unable to locate resource with key="
          + HYPH_FREQ_TABLE_RESRC_KEY
          + ".  Proceeding without hyphenation support.");
      tokenizer = new Tokenizer();
    } else {
      logger.info("Hyphen dictionary: " + strIntMapResrc.toString());
      Map<String, Integer> hyphMap = strIntMapResrc.getMap();
      tokenizer = new Tokenizer(hyphMap, freqCutoff);
    }

  }
View Full Code Here
TOP

Related Classes of org.apache.ctakes.core.nlp.tokenizer.Tokenizer

  • org.apache.ctakes.core.ae.TokenizerAnnotator
  • org.apache.ctakes.core.ci.HyphenTextModifierImpl
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.