Package opennlp.tools.tokenize

Examples of opennlp.tools.tokenize.TokenizerModel

244
245
246
247
248
249
250
251
252
253
254
        if(modelName == null){
            return openNLP.getTokenizer(language);
        } else if(SIMPLE_MODEL_NAME.equals(modelName)){
            return SimpleTokenizer.INSTANCE;
        } else { //try to load the configured model
            TokenizerModel model;
            try {
                model = openNLP.getModel(TokenizerModel.class, modelName, null);
            } catch (Exception e) {
                throw new EngineException("Error while loading the configured OpenNLP "
                    + "TokenizerModel '"+modelName+"' ("+getClass().getSimpleName()+" | name="
View Full Code Here
78
79
80
81
82
83
84
85
86
87
88
    }

    File modelOutFile = params.getModel();
    CmdLineUtil.checkOutputFile("tokenizer model", modelOutFile);

    TokenizerModel model;
    try {
      Dictionary dict = loadDict(params.getAbbDict());

      TokenizerFactory tokFactory = TokenizerFactory.create(
          params.getFactory(), params.getLang(), dict,
View Full Code Here
34
35
36
37
38
39
40
    super("Tokenizer");
  }

  @Override
  protected TokenizerModel loadModel(InputStream modelIn) throws IOException {
    return new TokenizerModel(modelIn);
  }
View Full Code Here
43
44
45
46
47
48
49
50
51
52
53
  }

  public void run(String format, String[] args) {
    super.run(format, args);

    TokenizerModel model = new TokenizerModelLoader().load(params.getModel());

    TokenizerEvaluationMonitor misclassifiedListener = null;
    if (params.getMisclassified()) {
      misclassifiedListener = new TokenEvaluationErrorListener();
    }
View Full Code Here
36
37
38
39
40
41
42
43
44
45
46
  public void run(String[] args) {
    if (args.length != 1) {
      System.out.println(getHelp());
    } else {

      TokenizerModel model = new TokenizerModelLoader().load(new File(args[0]));

      CommandLineTokenizer tokenizer =
        new CommandLineTokenizer(new opennlp.tools.tokenize.TokenizerME(model));

      tokenizer.process();
View Full Code Here
49
50
51
52
53
54
55
56
57
58
59
  public ObjectStream<NameSample> create(String[] args) {

    Parameters params = ArgumentParser.parse(args, Parameters.class);

    TokenizerModel tokenizerModel = new TokenizerModelLoader().load(params.getTokenizerModel());
    Tokenizer tokenizer = new TokenizerME(tokenizerModel);

    ObjectStream<String> mucDocStream = new FileToStringSampleStream(
        new DirectorySampleStream(params.getData(), new FileFilter() {
View Full Code Here
138
139
140
141
142
143
144
145
146
147
148
    Tokenizer tokenizer = WhitespaceTokenizer.INSTANCE;

    if (params.getTokenizerModel() != null) {
      try {
        tokenizer = new TokenizerME(new TokenizerModel(params.getTokenizerModel()));
      } catch (IOException e) {
        throw new TerminateToolException(-1, "Failed to load tokenizer model!", e);
      }
    }
    else if (params.getRuleBasedTokenizer() != null) {
View Full Code Here
83
84
85
86
87
88
89
90
91
92
93
  public void initialize(UimaContext context)
      throws ResourceInitializationException {

    super.initialize(context);

    TokenizerModel model;

    try {
      TokenizerModelResource modelResource = (TokenizerModelResource) context
          .getResourceObject(UimaUtil.MODEL_PARAMETER);
View Full Code Here
231
232
233
234
235
236
237
238
239
240
241
    // if trace file
    // serialize events ...

    InputStream additionalTrainingDataIn = null;
    Writer samplesOut = null;
    TokenizerModel tokenModel;

    try {
      if (additionalTrainingDataFile != null) {

        if (mLogger.isLoggable(Level.INFO)) {
View Full Code Here
75
76
77
78
79
80
81
82
83
84
85
86
  }

  public void setTokenizer(FileSystem fs, Path p){
    try {
      FSDataInputStream in = fs.open(p);
      TokenizerModel model;
      model = new TokenizerModel(in);
      tokenizer = new TokenizerME(model);
    }
    catch (IOException e) {
      e.printStackTrace();
    }
View Full Code Here
TOP

Related Classes of opennlp.tools.tokenize.TokenizerModel

  • com.bericotech.clavin.extractor.ApacheExtractor
  • edu.washington.cs.knowitall.util.DefaultObjects
  • functionality.Models
  • io.lumify.opennlpDictionary.OpenNLPDictionaryExtractorGraphPropertyWorker
  • io.lumify.opennlpme.OpenNLPMaximumEntropyExtractorGraphPropertyWorker
  • ivory.core.tokenize.OpenNLPTokenizer
  • net.sf.nlpshell.Main
  • opennlp.maxent.io.BinaryGISModelReader
  • opennlp.model.AbstractModel
  • opennlp.tools.cmdline.tokenizer.TokenizerMEEvaluatorTool
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.