Package opennlp.tools.tokenize

Examples of opennlp.tools.tokenize.TokenSample

146
147
148
149
150
151
152
153
154
155
156
157
158
159
      }
      ai++;
    }
    TokenSampleStream tss = new TokenSampleStream(System.in);
    while(tss.hasNext()) {
      TokenSample ts = tss.next();
      String text = ts.getText();
      System.out.println(text);
      Span[] tokenSpans = ts.getTokenSpans();
      int ti=0;
      if (showSpans) {
        for (int i=0;i<text.length();i++) {
          if (ti-1 >= 0 && i==tokenSpans[ti-1].getEnd()-1) {
            System.out.print("]");
View Full Code Here
39
40
41
42
43
44
45
46
47
48
49
50
51
  }

  public TokenSample read() throws IOException {
    NameSample nameSample = samples.read();

    TokenSample tokenSample = null;

    if (nameSample != null ) {
      tokenSample = new TokenSample(detokenizer, nameSample.getSentence());
    }

    return tokenSample;
  }
View Full Code Here
44
45
46
47
48
49
50
51
52
53
54
55
56
  public TokenSample read() throws IOException {

    POSSample posSample = samples.read();

    TokenSample tokenSample = null;

    if (posSample != null ) {
      tokenSample = new TokenSample(detokenizer, posSample.getSentence());
    }

    return tokenSample;
  }
View Full Code Here
118
119
120
121
122
123
124
      line = in.readLine();
    } catch (IOException e) {
      e.printStackTrace();
      line = null;
    }
    return new TokenSample(sb.toString(),spans.toArray(new Span[spans.size()]));
  }
View Full Code Here
146
147
148
149
150
151
152
153
154
155
156
157
158
159
      }
      ai++;
    }
    TokenSampleStream tss = new TokenSampleStream(System.in);
    while(tss.hasNext()) {
      TokenSample ts = tss.next();
      String text = ts.getText();
      System.out.println(text);
      Span[] tokenSpans = ts.getTokenSpans();
      int ti=0;
      if (showSpans) {
        for (int i=0;i<text.length();i++) {
          if (ti-1 >= 0 && i==tokenSpans[ti-1].getEnd()-1) {
            System.out.print("]");
View Full Code Here
206
207
208
209
210
211
212
    Span[] spans = openNLPSpans.toArray(new Span[openNLPSpans.size()]);

    Arrays.sort(spans);

    tokenSamples.add(new TokenSample(sentence.getCoveredText(), spans));
  }
View Full Code Here
TOP

Related Classes of opennlp.tools.tokenize.TokenSample

  • opennlp.tools.formats.ad.ADTokenSampleStreamTest
  • opennlp.tools.formats.convert.NameToTokenSampleStream
  • opennlp.tools.formats.convert.POSToTokenSampleStream
  • opennlp.tools.formats.NameToTokenSampleStream
  • opennlp.tools.formats.POSToTokenSampleStream
  • opennlp.tools.tokenize.lang.en.TokenSampleStream
  • opennlp.tools.util.Span
  • opennlp.uima.tokenize.TokenizerTrainer
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.