Package opennlp.tools.tokenize.Detokenizer

Examples of opennlp.tools.tokenize.Detokenizer.DetokenizationOperation

108
109
110
111
112
113
114
115
116
117
118
      while ((tokenizedLine = tokenizedLineStream.read()) != null) {
       
        // white space tokenize line
        String tokens[] = WhitespaceTokenizer.INSTANCE.tokenize(tokenizedLine);
       
        DetokenizationOperation operations[] = detokenizer.detokenize(tokens);
       
        System.out.println(detokenize(tokens, operations));
       
        perfMon.incrementCounter();
      }
View Full Code Here
54
55
56
57
58
59
60
61
62
63
64
   
    StringBuilder documentBuilder = new StringBuilder();
   
    for (String sentenceTokens[] : sentences) {
     
      DetokenizationOperation operations[] = detokenizer.detokenize(sentenceTokens);
     
      String sampleSentence = DictionaryDetokenizerTool.detokenize(sentenceTokens, operations);
     
      int beginIndex = documentBuilder.length();
      documentBuilder.append(sampleSentence);
View Full Code Here
44
45
46
47
48
49
50
51
52
53
54
      };
   
    DetokenizationDictionary dict = new DetokenizationDictionary(tokens, operations);
    Detokenizer detokenizer = new DictionaryDetokenizer(dict);
   
    DetokenizationOperation detokenizeOperations[] =
      detokenizer.detokenize(new String[]{"Simple",  "test", "."});
   
    assertEquals(DetokenizationOperation.NO_OPERATION, detokenizeOperations[0]);
    assertEquals(DetokenizationOperation.NO_OPERATION, detokenizeOperations[1]);
    assertEquals(DetokenizationOperation.MERGE_TO_LEFT, detokenizeOperations[2]);
View Full Code Here
69
70
71
72
73
74
75
76
77
78
79
  public void testDetokenizeToString() throws IOException {
   
    Detokenizer detokenizer = createLatinDetokenizer();
   
    String tokens[] = new String[]{"A", "test", ",", "(", "string", ")", "."};
    DetokenizationOperation operations[] = detokenizer.detokenize(tokens);
     
    String sentence = DictionaryDetokenizerTool.detokenize(tokens, operations);
     
    assertEquals("A test, (string).", sentence);
  }
View Full Code Here
44
45
46
47
48
49
50
51
52
53
54
      };
   
    DetokenizationDictionary dict = new DetokenizationDictionary(tokens, operations);
    Detokenizer detokenizer = new DictionaryDetokenizer(dict);
   
    DetokenizationOperation detokenizeOperations[] =
      detokenizer.detokenize(new String[]{"Simple",  "test", ".", "co", "-", "worker"});
   
    assertEquals(DetokenizationOperation.NO_OPERATION, detokenizeOperations[0]);
    assertEquals(DetokenizationOperation.NO_OPERATION, detokenizeOperations[1]);
    assertEquals(DetokenizationOperation.MERGE_TO_LEFT, detokenizeOperations[2]);
View Full Code Here
44
45
46
47
48
49
50
51
52
53
54
      };

    DetokenizationDictionary dict = new DetokenizationDictionary(tokens, operations);
    Detokenizer detokenizer = new DictionaryDetokenizer(dict);

    DetokenizationOperation detokenizeOperations[] =
      detokenizer.detokenize(new String[]{"Simple",  "test", ".", "co", "-", "worker"});

    assertEquals(DetokenizationOperation.NO_OPERATION, detokenizeOperations[0]);
    assertEquals(DetokenizationOperation.NO_OPERATION, detokenizeOperations[1]);
    assertEquals(DetokenizationOperation.MERGE_TO_LEFT, detokenizeOperations[2]);
View Full Code Here
TOP

Related Classes of opennlp.tools.tokenize.Detokenizer.DetokenizationOperation

  • opennlp.tools.cmdline.tokenizer.DictionaryDetokenizerTool
  • opennlp.tools.sentdetect.SentenceSample
  • opennlp.tools.tokenize.DictionaryDetokenizerTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.