Package opennlp.tools.util

Examples of opennlp.tools.util.PlainTextByLineStream

322
323
324
325
326
327
328
329
330
331
332
    trainingData.append("c\n");
    trainingData.append("\n");
    trainingData.append("d\n");
   
    ObjectStream<String> untokenizedLineStream =
      new PlainTextByLineStream(new StringReader(trainingData.toString()));
   
    ObjectStream<NameSample> trainingStream = new NameSampleDataStream(untokenizedLineStream);
   
    assertFalse(trainingStream.read().isClearAdaptiveDataSet());
    assertFalse(trainingStream.read().isClearAdaptiveDataSet());
View Full Code Here
85
86
87
88
89
90
91
92
93
94
95
 
  @Test
  public void testIncludeFeats() throws IOException {
    // add one sentence with includeFeats = true
    ADPOSSampleStream stream = new ADPOSSampleStream(
        new PlainTextByLineStream(
            ADParagraphStreamTest.class
                .getResourceAsStream("/opennlp/tools/formats/ad.sample"),
            "UTF-8"), false, true);

    POSSample sample = stream.read();
View Full Code Here
339
340
341
342
343
344
345
346
347
348
349
  public void testHtmlNameSampleParsing() throws IOException {
    InputStream in = getClass().getClassLoader().getResourceAsStream(
        "opennlp/tools/namefind/html1.train");
   
    NameSampleDataStream ds = new NameSampleDataStream(
        new PlainTextByLineStream(new InputStreamReader(in, "UTF-8")));

    NameSample ns = ds.read();
   
    assertEquals(1, ns.getSentence().length);
    assertEquals("<html>", ns.getSentence()[0]);
View Full Code Here
54
55
56
57
58
59
60
61
62
63
64
  public void setup() throws IOException {
    InputStream in = ADSentenceSampleStreamTest.class
        .getResourceAsStream("/opennlp/tools/formats/ad.sample");

    ADSentenceSampleStream stream = new ADSentenceSampleStream(
        new PlainTextByLineStream(in, "UTF-8"), true);

    SentenceSample sample = stream.read();

    while (sample != null) {
      System.out.println(sample.getDocument());
View Full Code Here
58
59
60
61
62
63
64
65
66
67
68
      SentenceModel model = new SentenceModelLoader().load(new File(args[0]));

      SentenceDetectorME sdetector = new SentenceDetectorME(model);

      ObjectStream<String> paraStream =
        new ParagraphStream(new PlainTextByLineStream(new InputStreamReader(System.in)));

      PerformanceMonitor perfMon = new PerformanceMonitor(System.err, "sent");
      perfMon.start();

      try {
View Full Code Here
65
66
67
68
69
70
71
  }
 
  private static ADSentenceStream openData() throws IOException {
    InputStream in = ADParagraphStreamTest.class.getResourceAsStream("/opennlp/tools/formats/ad.sample");
   
    return new ADSentenceStream(new PlainTextByLineStream(in, "UTF-8"));
  }
View Full Code Here
69
70
71
72
73
74
75
76
77
78
79
  public void setup() throws IOException {
    InputStream in = ADParagraphStreamTest.class
  .getResourceAsStream("/opennlp/tools/formats/ad.sample");

    ADChunkSampleStream stream = new ADChunkSampleStream(
      new PlainTextByLineStream(in, "UTF-8"));

    ChunkSample sample = stream.read();

    while (sample != null) {
      samples.add(sample);
View Full Code Here
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
   
      Detokenizer detokenizer = new DictionaryDetokenizer(
          new DetokenizationDictionaryLoader().load(new File(args[0])));

      ObjectStream<String> tokenizedLineStream =
        new PlainTextByLineStream(new InputStreamReader(System.in));

      PerformanceMonitor perfMon = new PerformanceMonitor(System.err, "sent");
      perfMon.start();

      try {
        String tokenizedLine;
        while ((tokenizedLine = tokenizedLineStream.read()) != null) {

          // white space tokenize line
          String tokens[] = WhitespaceTokenizer.INSTANCE.tokenize(tokenizedLine);

          System.out.println(detokenizer.detokenize(tokens, null));
View Full Code Here
52
53
54
55
56
57
58
59
60
61
62
      DoccatModel model = new DoccatModelLoader().load(new File(args[0]));

      DocumentCategorizerME doccat = new DocumentCategorizerME(model);

      ObjectStream<String> documentStream = new ParagraphStream(
          new PlainTextByLineStream(new InputStreamReader(System.in)));

      PerformanceMonitor perfMon = new PerformanceMonitor(System.err, "doc");
      perfMon.start();

      try {
View Full Code Here
42
43
44
45
46
47
48
49
50
      throws IOException {
    InputStream in = ChunkerFactoryTest.class.getClassLoader()
        .getResourceAsStream("opennlp/tools/chunker/test.txt");
    Reader sentences = new InputStreamReader(in);

    ChunkSampleStream stream = new ChunkSampleStream(new PlainTextByLineStream(
        sentences));
    return stream;
  }
View Full Code Here
TOP

Related Classes of opennlp.tools.util.PlainTextByLineStream

  • com.tamingtext.opennlp.NameFinderTest
  • io.lumify.opennlpDictionary.OpenNLPDictionaryExtractorGraphPropertyWorker
  • io.lumify.opennlpme.OpenNLPMaximumEntropyExtractorGraphPropertyWorker
  • opennlp.tools.chunker.ChunkerDetailedFMeasureListenerTest
  • opennlp.tools.chunker.ChunkerEvaluatorTest
  • opennlp.tools.chunker.ChunkerFactoryTest
  • opennlp.tools.chunker.ChunkerME
  • opennlp.tools.chunker.ChunkerMETest
  • opennlp.tools.chunker.ChunkSampleStreamTest
  • opennlp.tools.chunker.ChunkSampleTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.