Package opennlp.tools.tokenize

Examples of opennlp.tools.tokenize.TokenSample

204
205
206
207
208
209
210
    Span[] spans = openNLPSpans.toArray(new Span[openNLPSpans.size()]);
   
    Arrays.sort(spans);
   
    tokenSamples.add(new TokenSample(sentence.getCoveredText(), spans));
  }
View Full Code Here
57
58
59
60
61
62
63
64
65
66
67
        .getResource("opennlp/tools/formats/ad.sample").toURI());
    String[] args = { "-data", data.getCanonicalPath(), "-encoding", "UTF-8",
        "-lang", "pt", "-detokenizer", dict.getCanonicalPath() };
    ObjectStream<TokenSample> tokenSampleStream = factory.create(args);

    TokenSample sample = tokenSampleStream.read();

    while (sample != null) {
      samples.add(sample);
      sample = tokenSampleStream.read();
    }
View Full Code Here
118
119
120
121
122
123
124
      line = in.readLine();
    } catch (IOException e) {
      e.printStackTrace();
      line = null;
    }
    return new TokenSample(sb.toString(),spans.toArray(new Span[spans.size()]));
  }
View Full Code Here
146
147
148
149
150
151
152
153
154
155
156
157
158
159
      }
      ai++;
    }
    TokenSampleStream tss = new TokenSampleStream(System.in);
    while(tss.hasNext()) {
      TokenSample ts = tss.next();
      String text = ts.getText();
      System.out.println(text);
      Span[] tokenSpans = ts.getTokenSpans();
      int ti=0;
      if (showSpans) {
        for (int i=0;i<text.length();i++) {
          if (ti-1 >= 0 && i==tokenSpans[ti-1].getEnd()-1) {
            System.out.print("]");
View Full Code Here
194
195
196
197
198
199
200
    Span[] spans = openNLPSpans.toArray(new Span[openNLPSpans.size()]);
   
    Arrays.sort(spans);
   
    tokenSamples.add(new TokenSample(sentence.getCoveredText(), spans));
  }
View Full Code Here
39
40
41
42
43
44
45
46
47
48
49
50
51
  }
 
  public TokenSample read() throws IOException {
    NameSample nameSample = samples.read();
   
    TokenSample tokenSample = null;
   
    if (nameSample != null ) {
      tokenSample = new TokenSample(detokenizer, nameSample.getSentence());
    }
   
    return tokenSample;
  }
View Full Code Here
44
45
46
47
48
49
50
51
52
53
54
55
56
 
  public TokenSample read() throws IOException {
   
    POSSample posSample = samples.read();
   
    TokenSample tokenSample = null;
   
    if (posSample != null ) {
      tokenSample = new TokenSample(detokenizer, posSample.getSentence());
    }
   
    return tokenSample;
  }
View Full Code Here
39
40
41
42
43
44
45
46
47
48
49
50
51
  }
 
  public TokenSample read() throws IOException {
    NameSample nameSample = samples.read();
   
    TokenSample tokenSample = null;
   
    if (nameSample != null ) {
      tokenSample = new TokenSample(detokenizer, nameSample.getSentence());
    }
   
    return tokenSample;
  }
View Full Code Here
44
45
46
47
48
49
50
51
52
53
54
55
56
 
  public TokenSample read() throws IOException {
   
    POSSample posSample = samples.read();
   
    TokenSample tokenSample = null;
   
    if (posSample != null ) {
      tokenSample = new TokenSample(detokenizer, posSample.getSentence());
    }
   
    return tokenSample;
  }
View Full Code Here
118
119
120
121
122
123
124
      line = in.readLine();
    } catch (IOException e) {
      e.printStackTrace();
      line = null;
    }
    return new TokenSample(sb.toString(),spans.toArray(new Span[spans.size()]));
  }
View Full Code Here
TOP

Related Classes of opennlp.tools.tokenize.TokenSample

  • opennlp.tools.formats.ad.ADTokenSampleStreamTest
  • opennlp.tools.formats.convert.NameToTokenSampleStream
  • opennlp.tools.formats.convert.POSToTokenSampleStream
  • opennlp.tools.formats.NameToTokenSampleStream
  • opennlp.tools.formats.POSToTokenSampleStream
  • opennlp.tools.tokenize.lang.en.TokenSampleStream
  • opennlp.tools.util.Span
  • opennlp.uima.tokenize.TokenizerTrainer
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.