Package opennlp.tools.tokenize

Examples of opennlp.tools.tokenize.TokenizerModel

30
31
32
33
34
35
36
    return model;
  }

  @Override
  protected TokenizerModel loadModel(InputStream in) throws IOException {
    return new TokenizerModel(in);
  }
View Full Code Here
57
58
59
60
61
62
63
64
65
66
67
    EvaluatorParams params = ArgumentParser.parse(args,
        EvaluatorParams.class);

    Charset encoding = params.getEncoding();

    TokenizerModel model = new TokenizerModelLoader().load(params.getModel());

    TokenizerEvaluationMonitor missclassifiedListener = null;
    if (params.getMisclassified()) {
      missclassifiedListener = new TokenEvaluationErrorListener();
    }
View Full Code Here
42
43
44
45
46
47
48
49
50
51
52
    if (args.length != 1) {
      System.out.println(getHelp());
      throw new TerminateToolException(1);
    }
   
    TokenizerModel model = new TokenizerModelLoader().load(new File(args[0]));
   
    CommandLineTokenizer tokenizer =
      new CommandLineTokenizer(new opennlp.tools.tokenize.TokenizerME(model));
   
    tokenizer.process();
View Full Code Here
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
 
  /**
   * @return tokenizer model
   */
  public TokenizerModel tokModel() {
    TokenizerModel model = null;
    try {
      /* Load the tokenization model from file */
      InputStream modelIn = this.getClass().getClassLoader().getResourceAsStream("en-token.bin");
      model = new TokenizerModel(modelIn);
      return model;
    }
    catch (Exception e) {
      e.printStackTrace();
    }
View Full Code Here
48
49
50
51
52
53
54
55
56
57
58
  public ObjectStream<NameSample> create(String[] args) {

    Parameters params = ArgumentParser.parse(args, Parameters.class);

    TokenizerModel tokenizerModel = new TokenizerModelLoader().load(params.getTokenizerModel());
    Tokenizer tokenizer = new TokenizerME(tokenizerModel);

    ObjectStream<String> mucDocStream = new FileToStringSampleStream(
        new DirectorySampleStream(params.getData(), new FileFilter() {
View Full Code Here
77
78
79
80
81
82
83
84
85
86
87
    }

    File modelOutFile = params.getModel();
    CmdLineUtil.checkOutputFile("tokenizer model", modelOutFile);

    TokenizerModel model;
    try {
      Dictionary dict = loadDict(params.getAbbDict());

      TokenizerFactory tokFactory = TokenizerFactory.create(
          params.getFactory(), params.getLang(), dict,
View Full Code Here
34
35
36
37
38
39
40
    super("Tokenizer");
  }
 
  @Override
  protected TokenizerModel loadModel(InputStream modelIn) throws IOException {
    return new TokenizerModel(modelIn);
  }
View Full Code Here
43
44
45
46
47
48
49
50
51
52
53
  }
 
  public void run(String format, String[] args) {
    super.run(format, args);

    TokenizerModel model = new TokenizerModelLoader().load(params.getModel());

    TokenizerEvaluationMonitor misclassifiedListener = null;
    if (params.getMisclassified()) {
      misclassifiedListener = new TokenEvaluationErrorListener();
    }
View Full Code Here
36
37
38
39
40
41
42
43
44
45
46
  public void run(String[] args) {
    if (args.length != 1) {
      System.out.println(getHelp());
    } else {
   
      TokenizerModel model = new TokenizerModelLoader().load(new File(args[0]));

      CommandLineTokenizer tokenizer =
        new CommandLineTokenizer(new opennlp.tools.tokenize.TokenizerME(model));

      tokenizer.process();
View Full Code Here
78
79
80
81
82
83
84
85
86
87
88
    Parameters params = ArgumentParser.parse(args, Parameters.class);
   
    ParserModel parserModel = new ParserModelLoader().load(params.getParserModel());
    Parser parser =  ParserFactory.create(parserModel);
   
    TokenizerModel tokenizerModel = new TokenizerModelLoader().load(params.getTokenizerModel());
    Tokenizer tokenizer = new TokenizerME(tokenizerModel);
   
    ObjectStream<String> mucDocStream = new FileToStringSampleStream(
        new DirectorySampleStream(params.getData(), new FileFilter() {
         
View Full Code Here
TOP

Related Classes of opennlp.tools.tokenize.TokenizerModel

  • com.bericotech.clavin.extractor.ApacheExtractor
  • edu.washington.cs.knowitall.util.DefaultObjects
  • functionality.Models
  • io.lumify.opennlpDictionary.OpenNLPDictionaryExtractorGraphPropertyWorker
  • io.lumify.opennlpme.OpenNLPMaximumEntropyExtractorGraphPropertyWorker
  • ivory.core.tokenize.OpenNLPTokenizer
  • net.sf.nlpshell.Main
  • opennlp.maxent.io.BinaryGISModelReader
  • opennlp.model.AbstractModel
  • opennlp.tools.cmdline.tokenizer.TokenizerMEEvaluatorTool
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.