Package opennlp.tools.tokenize

Examples of opennlp.tools.tokenize.TokenizerModel

68
69
70
71
72
73
74
75
     *
     * @throws IOException
     */
    public ApacheExtractor() throws IOException {
        nameFinder = new NameFinderME(new TokenNameFinderModel(ApacheExtractor.class.getResourceAsStream(pathToNERModel)));
        tokenizer = new TokenizerME(new TokenizerModel(ApacheExtractor.class.getResourceAsStream(pathToTokenizerModel)));
        sentenceDetector = new SentenceDetectorME(new SentenceModel(ApacheExtractor.class.getResourceAsStream(pathToSentenceDetectorModel)));
    }
View Full Code Here
139
140
141
142
143
144
145
146
147
148
149
150
151
152
    }

    protected Tokenizer loadTokenizer(String pathPrefix, FileSystem fs) throws IOException {
        Path tokenizerHdfsPath = new Path(pathPrefix + "/en-token.bin");

        TokenizerModel tokenizerModel = null;
        InputStream tokenizerModelInputStream = fs.open(tokenizerHdfsPath);
        try {
            tokenizerModel = new TokenizerModel(tokenizerModelInputStream);
        } finally {
            tokenizerModelInputStream.close();
        }

        return new TokenizerME(tokenizerModel);
View Full Code Here
135
136
137
138
139
140
141
142
143
144
145
146
147
148
    }

    protected Tokenizer loadTokenizer(String pathPrefix, FileSystem fs) throws IOException {
        Path tokenizerHdfsPath = new Path(pathPrefix + "/en-token.bin");

        TokenizerModel tokenizerModel = null;
        InputStream tokenizerModelInputStream = fs.open(tokenizerHdfsPath);
        try {
            tokenizerModel = new TokenizerModel(tokenizerModelInputStream);
        } finally {
            tokenizerModelInputStream.close();
        }

        return new TokenizerME(tokenizerModel);
View Full Code Here
229
230
231
232
233
234
235
236
237
238
239
    // if trace file
    // serialize events ...
   
    InputStream additionalTrainingDataIn = null;
    Writer samplesOut = null;
    TokenizerModel tokenModel;
   
    try {
      if (additionalTrainingDataFile != null) {
       
        if (mLogger.isLoggable(Level.INFO)) {
View Full Code Here
30
31
32
33
34
35
36
    return model;
  }

  @Override
  protected TokenizerModel loadModel(InputStream in) throws IOException {
    return new TokenizerModel(in);
  }
View Full Code Here
81
82
83
84
85
86
87
88
89
90
91
  public void initialize(UimaContext context)
      throws ResourceInitializationException {

    super.initialize(context);

    TokenizerModel model;

    try {
      TokenizerModelResource modelResource = (TokenizerModelResource) context
          .getResourceObject(UimaUtil.MODEL_PARAMETER);
View Full Code Here
109
110
111
112
113
114
115
116
117
118
119
        trainingDataInFile, params.getEncoding());
   
    if(mlParams == null)
      mlParams = createTrainingParameters(params.getIterations(), params.getCutoff());

    TokenizerModel model;
    try {
      Dictionary dict = loadDict(params.getAbbDict());
      model = opennlp.tools.tokenize.TokenizerME.train(params.getLang(),
          sampleStream, dict, params.getAlphaNumOpt(), mlParams);
    } catch (IOException e) {
View Full Code Here
36
37
38
39
40
41
42
  }
 
  @Override
  protected TokenizerModel loadModel(InputStream modelIn) throws IOException,
      InvalidFormatException {
    return new TokenizerModel(modelIn);
  }
View Full Code Here
80
81
82
83
84
85
86
87
88
89
90
  public void initialize(UimaContext context)
      throws ResourceInitializationException {

    super.initialize(context);

    TokenizerModel model;

    try {
      TokenizerModelResource modelResource = (TokenizerModelResource) context
          .getResourceObject(UimaUtil.MODEL_PARAMETER);
View Full Code Here
214
215
216
217
218
219
220
221
222
223
224
    GIS.PRINT_MESSAGES = false;
  
    ObjectStream<TokenSample> samples = ObjectStreamUtils.createObjectStream(tokenSamples);
   
    InputStream additionalTrainingDataIn = null;
    TokenizerModel tokenModel;
   
    try {
      if (additionalTrainingDataFile != null) {
       
        if (mLogger.isLoggable(Level.INFO)) {
View Full Code Here
TOP

Related Classes of opennlp.tools.tokenize.TokenizerModel

  • com.bericotech.clavin.extractor.ApacheExtractor
  • edu.washington.cs.knowitall.util.DefaultObjects
  • functionality.Models
  • io.lumify.opennlpDictionary.OpenNLPDictionaryExtractorGraphPropertyWorker
  • io.lumify.opennlpme.OpenNLPMaximumEntropyExtractorGraphPropertyWorker
  • ivory.core.tokenize.OpenNLPTokenizer
  • net.sf.nlpshell.Main
  • opennlp.maxent.io.BinaryGISModelReader
  • opennlp.model.AbstractModel
  • opennlp.tools.cmdline.tokenizer.TokenizerMEEvaluatorTool
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.