Package opennlp.tools.tokenize

Examples of opennlp.tools.tokenize.SimpleTokenizer

61
62
63
64
65
66
67
68
69
70
71
72
  @Test
  public void testSingleTokeNameAtSentenceStart() {

    String sentence = "Max a b c d";

    SimpleTokenizer tokenizer = SimpleTokenizer.INSTANCE;
    String tokens[] = tokenizer.tokenize(sentence);

    Span names[] = mNameFinder.find(tokens);

    assertTrue(names.length == 1);
    assertTrue(names[0].getStart() == 0 && names[0].getEnd() == 1);
View Full Code Here
74
75
76
77
78
79
80
81
82
83
84
85
  @Test
  public void testSingleTokeNameInsideSentence() {
    String sentence = "a b  Max c d";

    SimpleTokenizer tokenizer = SimpleTokenizer.INSTANCE;
    String tokens[] = tokenizer.tokenize(sentence);

    Span names[] = mNameFinder.find(tokens);

    assertTrue(names.length == 1);
    assertTrue(names[0].getStart() == 2 && names[0].getEnd() == 3);
View Full Code Here
87
88
89
90
91
92
93
94
95
96
97
98
  @Test
  public void testSingleTokeNameAtSentenceEnd() {
    String sentence = "a b c Max";

    SimpleTokenizer tokenizer = SimpleTokenizer.INSTANCE;
    String tokens[] = tokenizer.tokenize(sentence);

    Span names[] = mNameFinder.find(tokens);

    assertTrue(names.length == 1);
    assertTrue(names[0].getStart() == 3 && names[0].getEnd() == 4);
View Full Code Here
57
58
59
60
61
62
63
64
65
66
67
68
  @Test
  public void testSingleTokeNameAtSentenceStart() {

    String sentence = "Max a b c d";

    SimpleTokenizer tokenizer = SimpleTokenizer.INSTANCE;
    String tokens[] = tokenizer.tokenize(sentence);

    Span names[] = mNameFinder.find(tokens);

    assertTrue(names.length == 1);
    assertTrue(names[0].getStart() == 0 && names[0].getEnd() == 1);
View Full Code Here
70
71
72
73
74
75
76
77
78
79
80
81
  @Test
  public void testSingleTokeNameInsideSentence() {
    String sentence = "a b  Max c d";

    SimpleTokenizer tokenizer = SimpleTokenizer.INSTANCE;
    String tokens[] = tokenizer.tokenize(sentence);

    Span names[] = mNameFinder.find(tokens);

    assertTrue(names.length == 1);
    assertTrue(names[0].getStart() == 2 && names[0].getEnd() == 3);
View Full Code Here
83
84
85
86
87
88
89
90
91
92
93
94
  @Test
  public void testSingleTokeNameAtSentenceEnd() {
    String sentence = "a b c Max";

    SimpleTokenizer tokenizer = SimpleTokenizer.INSTANCE;
    String tokens[] = tokenizer.tokenize(sentence);

    Span names[] = mNameFinder.find(tokens);

    assertTrue(names.length == 1);
    assertTrue(names[0].getStart() == 3 && names[0].getEnd() == 4);
View Full Code Here
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
   * @throws IOException
   */
  private static void processText(TreebankNameFinder[] finders, String[] tags, BufferedReader input) throws IOException {
    Span[][] nameSpans = new Span[finders.length][];
    String[][] nameOutcomes = new String[finders.length][];
    opennlp.tools.tokenize.Tokenizer tokenizer = new SimpleTokenizer();
    StringBuffer output = new StringBuffer();
    for (String line = input.readLine(); null != line; line = input.readLine()) {
      if (line.equals("")) {
        clearPrevTokenMaps(finders);
        System.out.println();
        continue;
      }
      output.setLength(0);
      Span[] spans = tokenizer.tokenizePos(line);
      String[] tokens = Span.spansToStrings(spans,line);
      for (int fi = 0, fl = finders.length; fi < fl; fi++) {
        nameSpans[fi] = finders[fi].nameFinder.find(tokens);
        //System.err.println("EnglighNameFinder.processText: "+tags[fi] + " " + java.util.Arrays.asList(finderTags[fi]));
        nameOutcomes[fi] = NameFinderEventStream.generateOutcomes(nameSpans[fi], null, tokens.length);
View Full Code Here
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
        }

        NameFinderME finder = new NameFinderME((TokenNameFinderModel)nameFinderModel);

        List<SurfaceFormOccurrence> sfOccurrences = new ArrayList<SurfaceFormOccurrence>();
        Tokenizer tokenizer = new SimpleTokenizer();
        for (int i = 0; i < sentences.length; i++) {
            String sentence = sentences[i];
            //LOG.debug("Sentence: " + sentence);

            // extract the names in the current sentence
            String[] tokens = tokenizer.tokenize(sentence);
            Span[] tokenspan = tokenizer.tokenizePos(sentence);
            Span[] nameSpans = finder.find(tokens);
            double[] probs = finder.probs();

            if (nameSpans != null && nameSpans.length > 0) {
                //System.out.println("Tokens: " +(new ArrayList(Arrays.asList(tokens))).toString());
View Full Code Here
61
62
63
64
65
66
67
68
69
70
71
72
  @Test
  public void testSingleTokeNameAtSentenceStart() {

    String sentence = "Max a b c d";

    SimpleTokenizer tokenizer = SimpleTokenizer.INSTANCE;
    String tokens[] = tokenizer.tokenize(sentence);

    Span names[] = mNameFinder.find(tokens);

    assertTrue(names.length == 1);
    assertTrue(names[0].getStart() == 0 && names[0].getEnd() == 1);
View Full Code Here
74
75
76
77
78
79
80
81
82
83
84
85
  @Test
  public void testSingleTokeNameInsideSentence() {
    String sentence = "a b  Max c d";

    SimpleTokenizer tokenizer = SimpleTokenizer.INSTANCE;
    String tokens[] = tokenizer.tokenize(sentence);

    Span names[] = mNameFinder.find(tokens);

    assertTrue(names.length == 1);
    assertTrue(names[0].getStart() == 2 && names[0].getEnd() == 3);
View Full Code Here
TOP

Related Classes of opennlp.tools.tokenize.SimpleTokenizer

  • opennlp.tools.lang.english.TreebankNameFinder
  • opennlp.tools.namefind.DictionaryNameFinderTest
  • opennlp.tools.util.Span
  • org.dbpedia.spotlight.spot.NESpotter
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.