Package opennlp.tools.tokenize

Examples of opennlp.tools.tokenize.DetokenizationDictionary

55
56
57
58
59
60
61
62
63
64
65
    ObjectStream<POSSample> posSampleStream = new WordTagSampleStreamFactory()
        .create(params);

    Detokenizer detokenizer;
    try {
      detokenizer = new DictionaryDetokenizer(new DetokenizationDictionary(
          new FileInputStream(new File(params.getDetokenizer()))));
    } catch (IOException e) {
      System.err.println("Error while loading detokenizer dict: "
          + e.getMessage());
      throw new TerminateToolException(-1);
View Full Code Here
54
55
56
57
58
59
60
61
62
63
64
   
    ObjectStream<POSSample> samples = new ConllXPOSSampleStreamFactory().create(params);
   
    Detokenizer detokenizer;
    try {
      detokenizer = new DictionaryDetokenizer(new DetokenizationDictionary(new FileInputStream(new File(params.getDetokenizer()))));
    } catch (IOException e) {
      System.err.println("Error while loading detokenizer dict: " + e.getMessage());
      throw new TerminateToolException(-1);
    }
   
View Full Code Here
31
32
33
34
35
36
37
  }
 
  @Override
  protected DetokenizationDictionary loadModel(InputStream modelIn)
      throws IOException, InvalidFormatException {
    return new DetokenizationDictionary(modelIn);
  }
View Full Code Here
56
57
58
59
60
61
62
63
64
65
66
        .create(params);

    // TODO: Move this to a factory method
    Detokenizer detokenizer;
    try {
      detokenizer = new DictionaryDetokenizer(new DetokenizationDictionary(
          new FileInputStream(new File(params.getDetokenizer()))));
    } catch (IOException e) {
      System.err.println("Error while loading detokenizer dict: "
          + e.getMessage());
      throw new TerminateToolException(-1);
View Full Code Here
55
56
57
58
59
60
61
62
63
64
65
        .create(params);

    // TODO: Move this to a factory method
    Detokenizer detokenizer;
    try {
      detokenizer = new DictionaryDetokenizer(new DetokenizationDictionary(
          new FileInputStream(new File(params.getDetokenizer()))));
    } catch (IOException e) {
      System.err.println("Error while loading detokenizer dict: "
          + e.getMessage());
      throw new TerminateToolException(-1);
View Full Code Here
55
56
57
58
59
60
61
62
63
64
65
        .create(params);

    // TODO: Move this to a factory method
    Detokenizer detokenizer;
    try {
      detokenizer = new DictionaryDetokenizer(new DetokenizationDictionary(
          new FileInputStream(new File(params.getDetokenizer()))));
    } catch (IOException e) {
      System.err.println("Error while loading detokenizer dict: "
          + e.getMessage());
      throw new TerminateToolException(-1);
View Full Code Here
59
60
61
62
63
64
65
66
67
68
69
    ObjectStream<POSSample> posSampleStream =
        new ConllXPOSSampleStreamFactory().create(params);
   
    Detokenizer detokenizer;
    try {
      detokenizer = new DictionaryDetokenizer(new DetokenizationDictionary(new FileInputStream(new File(params.getDetokenizer()))));
    } catch (IOException e) {
      System.err.println("Error while loading detokenizer dict: " + e.getMessage());
      throw new TerminateToolException(-1);
    }
   
View Full Code Here
29
30
31
32
33
34
35
    super("detokenizer dictionary");
  }
 
  @Override
  protected DetokenizationDictionary loadModel(InputStream modelIn) throws IOException {
    return new DetokenizationDictionary(modelIn);
  }
View Full Code Here
36
37
38
39
40
41
42
43
44
45
46
    super(params);
  }

  protected Detokenizer createDetokenizer(DetokenizerParameter p) {
    try {
      return new DictionaryDetokenizer(new DetokenizationDictionary(
          new FileInputStream(new File(p.getDetokenizer()))));
    } catch (IOException e) {
      throw new TerminateToolException(-1, "IO error while loading detokenizer dict: " + e.getMessage(), e);
    }
  }
View Full Code Here
29
30
31
32
33
34
35
    super("detokenizer dictionary");
  }

  @Override
  protected DetokenizationDictionary loadModel(InputStream modelIn) throws IOException {
    return new DetokenizationDictionary(modelIn);
  }
View Full Code Here
TOP

Related Classes of opennlp.tools.tokenize.DetokenizationDictionary

  • opennlp.tools.cmdline.tokenizer.DetokenizationDictionaryLoader
  • opennlp.tools.dictionary.serializer.Attributes
  • opennlp.tools.dictionary.serializer.Entry
  • opennlp.tools.dictionary.serializer.EntryInserter
  • opennlp.tools.formats.ConllXSentenceSampleStreamFactory
  • opennlp.tools.formats.ConllXTokenSampleStreamFactory
  • opennlp.tools.formats.DetokenizerSampleStreamFactory
  • opennlp.tools.formats.NameToSentenceSampleStreamFactory
  • opennlp.tools.formats.NameToTokenSampleStreamFactory
  • opennlp.tools.formats.POSToSentenceSampleStreamFactory
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.