Package opennlp.tools.tokenize

Examples of opennlp.tools.tokenize.DictionaryDetokenizer

55
56
57
58
59
60
61
62
63
64
65
    ObjectStream<POSSample> posSampleStream = new WordTagSampleStreamFactory()
        .create(params);

    Detokenizer detokenizer;
    try {
      detokenizer = new DictionaryDetokenizer(new DetokenizationDictionary(
          new FileInputStream(new File(params.getDetokenizer()))));
    } catch (IOException e) {
      System.err.println("Error while loading detokenizer dict: "
          + e.getMessage());
      throw new TerminateToolException(-1);
View Full Code Here
54
55
56
57
58
59
60
61
62
63
64
   
    ObjectStream<POSSample> samples = new ConllXPOSSampleStreamFactory().create(params);
   
    Detokenizer detokenizer;
    try {
      detokenizer = new DictionaryDetokenizer(new DetokenizationDictionary(new FileInputStream(new File(params.getDetokenizer()))));
    } catch (IOException e) {
      System.err.println("Error while loading detokenizer dict: " + e.getMessage());
      throw new TerminateToolException(-1);
    }
   
View Full Code Here
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
    if (args.length != 1) {
      System.out.println(getHelp());
      throw new TerminateToolException(1);
    }
   
    Detokenizer detokenizer = new DictionaryDetokenizer(
        new DetokenizationDictionaryLoader().load(new File(args[0])));
   
    ObjectStream<String> tokenizedLineStream =
      new PlainTextByLineStream(new InputStreamReader(System.in));
   
    PerformanceMonitor perfMon = new PerformanceMonitor(System.err, "sent");
    perfMon.start();
   
    try {
      String tokenizedLine;
      while ((tokenizedLine = tokenizedLineStream.read()) != null) {
       
        // white space tokenize line
        String tokens[] = WhitespaceTokenizer.INSTANCE.tokenize(tokenizedLine);
       
        DetokenizationOperation operations[] = detokenizer.detokenize(tokens);
       
        System.out.println(detokenize(tokens, operations));
       
        perfMon.incrementCounter();
      }
View Full Code Here
56
57
58
59
60
61
62
63
64
65
66
        .create(params);

    // TODO: Move this to a factory method
    Detokenizer detokenizer;
    try {
      detokenizer = new DictionaryDetokenizer(new DetokenizationDictionary(
          new FileInputStream(new File(params.getDetokenizer()))));
    } catch (IOException e) {
      System.err.println("Error while loading detokenizer dict: "
          + e.getMessage());
      throw new TerminateToolException(-1);
View Full Code Here
55
56
57
58
59
60
61
62
63
64
65
        .create(params);

    // TODO: Move this to a factory method
    Detokenizer detokenizer;
    try {
      detokenizer = new DictionaryDetokenizer(new DetokenizationDictionary(
          new FileInputStream(new File(params.getDetokenizer()))));
    } catch (IOException e) {
      System.err.println("Error while loading detokenizer dict: "
          + e.getMessage());
      throw new TerminateToolException(-1);
View Full Code Here
55
56
57
58
59
60
61
62
63
64
65
        .create(params);

    // TODO: Move this to a factory method
    Detokenizer detokenizer;
    try {
      detokenizer = new DictionaryDetokenizer(new DetokenizationDictionary(
          new FileInputStream(new File(params.getDetokenizer()))));
    } catch (IOException e) {
      System.err.println("Error while loading detokenizer dict: "
          + e.getMessage());
      throw new TerminateToolException(-1);
View Full Code Here
59
60
61
62
63
64
65
66
67
68
69
    ObjectStream<POSSample> posSampleStream =
        new ConllXPOSSampleStreamFactory().create(params);
   
    Detokenizer detokenizer;
    try {
      detokenizer = new DictionaryDetokenizer(new DetokenizationDictionary(new FileInputStream(new File(params.getDetokenizer()))));
    } catch (IOException e) {
      System.err.println("Error while loading detokenizer dict: " + e.getMessage());
      throw new TerminateToolException(-1);
    }
   
View Full Code Here
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
   
    if (args.length != 1) {
      System.out.println(getHelp());
    } else {
   
      Detokenizer detokenizer = new DictionaryDetokenizer(
          new DetokenizationDictionaryLoader().load(new File(args[0])));

      ObjectStream<String> tokenizedLineStream =
        new PlainTextByLineStream(new InputStreamReader(System.in));

      PerformanceMonitor perfMon = new PerformanceMonitor(System.err, "sent");
      perfMon.start();

      try {
        String tokenizedLine;
        while ((tokenizedLine = tokenizedLineStream.read()) != null) {

          // white space tokenize line
          String tokens[] = WhitespaceTokenizer.INSTANCE.tokenize(tokenizedLine);

          System.out.println(detokenizer.detokenize(tokens, null));

          perfMon.incrementCounter();
        }
      }
      catch (IOException e) {
View Full Code Here
36
37
38
39
40
41
42
43
44
45
46
    super(params);
  }

  protected Detokenizer createDetokenizer(DetokenizerParameter p) {
    try {
      return new DictionaryDetokenizer(new DetokenizationDictionary(
          new FileInputStream(new File(p.getDetokenizer()))));
    } catch (IOException e) {
      throw new TerminateToolException(-1, "IO error while loading detokenizer dict: " + e.getMessage(), e);
    }
  }
View Full Code Here
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
    if (args.length != 1) {
      System.out.println(getHelp());
    } else {
          try {
      Detokenizer detokenizer = new DictionaryDetokenizer(
          new DetokenizationDictionaryLoader().load(new File(args[0])));

      ObjectStream<String> tokenizedLineStream =
        new PlainTextByLineStream(new SystemInputStreamFactory(), SystemInputStreamFactory.encoding());

      PerformanceMonitor perfMon = new PerformanceMonitor(System.err, "sent");
      perfMon.start();


        String tokenizedLine;
        while ((tokenizedLine = tokenizedLineStream.read()) != null) {

          // white space tokenize line
          String tokens[] = WhitespaceTokenizer.INSTANCE.tokenize(tokenizedLine);

          System.out.println(detokenizer.detokenize(tokens, null));

          perfMon.incrementCounter();
        }
              perfMon.stopAndPrintFinalResult();
      }
View Full Code Here
TOP

Related Classes of opennlp.tools.tokenize.DictionaryDetokenizer

  • opennlp.tools.cmdline.tokenizer.DictionaryDetokenizerTool
  • opennlp.tools.formats.ConllXSentenceSampleStreamFactory
  • opennlp.tools.formats.ConllXTokenSampleStreamFactory
  • opennlp.tools.formats.DetokenizerSampleStreamFactory
  • opennlp.tools.formats.NameToSentenceSampleStreamFactory
  • opennlp.tools.formats.NameToTokenSampleStreamFactory
  • opennlp.tools.formats.POSToSentenceSampleStreamFactory
  • opennlp.tools.formats.POSToTokenSampleStreamFactory
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.