Package opennlp.tools.tokenize.lang

Examples of opennlp.tools.tokenize.lang.Factory

69
70
71
72
73
74
75
76
77
78
   * @param cg
   */
  public TokSpanEventStream(ObjectStream<TokenSample> tokenSamples,
        boolean skipAlphaNumerics, TokenContextGenerator cg) {
    super(tokenSamples);
    Factory factory = new Factory();
    this.alphaNumeric = factory.getAlphanumeric(null);
    this.skipAlphaNumerics = skipAlphaNumerics;
    this.cg = cg;
  }
View Full Code Here
69
70
71
72
73
74
75
76
77
78
   * @param cg
   */
  public TokSpanEventStream(ObjectStream<TokenSample> tokenSamples,
        boolean skipAlphaNumerics, TokenContextGenerator cg) {
    super(tokenSamples);
    Factory factory = new Factory();
    this.alphaNumeric = factory.getAlphanumeric(null);
    this.skipAlphaNumerics = skipAlphaNumerics;
    this.cg = cg;
  }
View Full Code Here
119
120
121
122
123
124
125
  private List<Double> tokProbs;

  private List<Span> newTokens;

  public TokenizerME(TokenizerModel model) {
    this(model, new Factory());
  }
View Full Code Here
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
   */
  public static TokenizerModel train(String languageCode,
      ObjectStream<TokenSample> samples, Dictionary abbreviations,
      boolean useAlphaNumericOptimization, TrainingParameters mlParams)
      throws IOException {
    Factory factory = new Factory();

    Map<String, String> manifestInfoEntries = new HashMap<String, String>();

    EventStream eventStream = new TokSpanEventStream(samples,
        useAlphaNumericOptimization, factory.getAlphanumeric(languageCode),
        factory.createTokenContextGenerator(languageCode,
            getAbbreviations(abbreviations)));

    AbstractModel maxentModel = TrainUtil.train(eventStream,
        mlParams.getSettings(), manifestInfoEntries);

View Full Code Here
164
165
166
167
168
169
170
171
172
173
174
          this.alphaNumericPattern = Pattern.compile(prop);
        }
      }
      // could not load from manifest, will get from language dependent factory
      if (this.alphaNumericPattern == null) {
        Factory f = new Factory();
        this.alphaNumericPattern = f.getAlphanumeric(languageCode);
      }
    }
    return this.alphaNumericPattern;
  }
View Full Code Here
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
  /**
   * Gets the context generator
   */
  public TokenContextGenerator getContextGenerator() {
    Factory f = new Factory();
    Set<String> abbs = null;
    Dictionary abbDict = getAbbreviationDictionary();
    if (abbDict != null) {
      abbs = abbDict.asStringSet();
    } else {
      abbs = Collections.emptySet();
    }
    return f.createTokenContextGenerator(getLanguageCode(), abbs);
  }
View Full Code Here
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
   */
  public static TokenizerModel train(String languageCode,
      ObjectStream<TokenSample> samples, Dictionary abbreviations,
      boolean useAlphaNumericOptimization, TrainingParameters mlParams)
      throws IOException {
    Factory factory = new Factory();

    Map<String, String> manifestInfoEntries = new HashMap<String, String>();

    EventStream eventStream = new TokSpanEventStream(samples,
        useAlphaNumericOptimization, factory.getAlphanumeric(languageCode),
        factory.createTokenContextGenerator(languageCode,
            getAbbreviations(abbreviations)));

    AbstractModel maxentModel = TrainUtil.train(eventStream,
        mlParams.getSettings(), manifestInfoEntries);

View Full Code Here
69
70
71
72
73
74
75
76
77
78
   * @param cg
   */
  public TokSpanEventStream(ObjectStream<TokenSample> tokenSamples,
        boolean skipAlphaNumerics, TokenContextGenerator cg) {
    super(tokenSamples);
    Factory factory = new Factory();
    this.alphaNumeric = factory.getAlphanumeric(null);
    this.skipAlphaNumerics = skipAlphaNumerics;
    this.cg = cg;
  }
View Full Code Here
174
175
176
177
178
179
180
181
182
183
184
          this.alphaNumericPattern = Pattern.compile(prop);
        }
      }
      // could not load from manifest, will get from language dependent factory
      if (this.alphaNumericPattern == null) {
        Factory f = new Factory();
        this.alphaNumericPattern = f.getAlphanumeric(languageCode);
      }
    }
    return this.alphaNumericPattern;
  }
View Full Code Here
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
   * Gets the context generator
   *
   * @return a new instance of the context generator
   */
  public TokenContextGenerator getContextGenerator() {
    Factory f = new Factory();
    Set<String> abbs = null;
    Dictionary abbDict = getAbbreviationDictionary();
    if (abbDict != null) {
      abbs = abbDict.asStringSet();
    } else {
      abbs = Collections.emptySet();
    }
    return f.createTokenContextGenerator(getLanguageCode(), abbs);
  }
View Full Code Here
TOP

Related Classes of opennlp.tools.tokenize.lang.Factory

  • opennlp.tools.tokenize.DefaultTokenContextGenerator
  • opennlp.tools.tokenize.TokenizerFactory
  • opennlp.tools.tokenize.TokenizerME
  • opennlp.tools.tokenize.TokSpanEventStream
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.