Package opennlp.tools.util

Examples of opennlp.tools.util.StringList

254
255
256
257
258
259
260
261
262
263
264
    if (cutoffUnder > 0 || cutoffOver < Integer.MAX_VALUE) {

      for (Iterator<StringList> it = iterator(); it.hasNext();) {

        StringList ngram = it.next();

        int count = getCount(ngram);

        if (count < cutoffUnder ||
            count > cutoffOver) {
View Full Code Here
317
318
319
320
321
322
323
324
325
326
327
            return mDictionaryIterator.hasNext();
          }

          public Entry next() {

            StringList tokens = mDictionaryIterator.next();

            Attributes attributes = new Attributes();

            attributes.setValue(COUNT, Integer.toString(getCount(tokens)));
View Full Code Here
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
    DictionarySerializer.create(in, new EntryInserter() {
      public void insert(Entry entry) throws InvalidFormatException {

        String operationString = entry.getAttributes().getValue("operation");

        StringList word = entry.getTokens();

        if (word.size() != 1)
          throw new InvalidFormatException("Each entry must have exactly one token! "+word);
       
        // parse operation
        Operation operation = Operation.parse(operationString);
       
        if (operation == null)
            throw new InvalidFormatException("Unkown operation type: " + operationString);
       
        operationTable.put(word.getToken(0), operation);
      }});
  }
View Full Code Here
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
    this.lineStream = new PlainTextByLineStream(in, this.encoding);
  }

  public StringList read() throws IOException {
    String line = lineStream.read();
    StringList name = null;

    if ((line != null) &&
        (!StringUtil.isEmpty(line))) {
      String name2;
      // find the location of the name separator in the line of data.
      int pos = line.indexOf(' ');
      if ((pos != -1)) {
        String parsed = line.substring(0, pos);
        // the data is in ALL CAPS ... so the easiest way is to convert
        // back to standard mixed case.
        if ((parsed.length() > 2) &&
            (parsed.startsWith("MC"))) {
          name2 = parsed.substring(0,1).toUpperCase(locale) +
                  parsed.substring(1,2).toLowerCase(locale) +
                  parsed.substring(2,3).toUpperCase(locale) +
                  parsed.substring(3).toLowerCase(locale);
        } else {
          name2 = parsed.substring(0,1).toUpperCase(locale) +
                  parsed.substring(1).toLowerCase(locale);
        }
        name = new StringList(new String[]{name2});
      }
    }

    return name;
  }
View Full Code Here
136
137
138
139
140
141
142
143
144
145
146
        String token = iterator.next();

        Attributes attributes = new Attributes();
        attributes.setValue("operation", getOperation(token).toString());

        return new Entry(new StringList(token), attributes);
      }

      public void remove() {
        throw new UnsupportedOperationException();
      }
View Full Code Here
40
41
42
43
44
45
46
47
48
49
50
51
   */
  public Index(Iterator<StringList> tokenLists) {

    while (tokenLists.hasNext()) {

      StringList tokens = tokenLists.next();

      for (int i = 0; i < tokens.size(); i++) {
        this.tokens.add(tokens.getToken(i));
      }
    }
  }
View Full Code Here
216
217
218
219
220
221
222
223
224
225
226
          return dictionaryIterator.hasNext();
        }

        public Entry next() {

          StringList tokens = dictionaryIterator.next();
         
          return new Entry(tokens, new Attributes());
        }

        public void remove() {
View Full Code Here
288
289
290
291
292
293
294
295
296
297
298
        int tokenIndex = 0;
        while (whiteSpaceTokenizer.hasMoreTokens()) {
          tokens[tokenIndex++] = whiteSpaceTokenizer.nextToken();
        }

        dictionary.put(new StringList(tokens));
      }
    }

    return dictionary;
  }
View Full Code Here
339
340
341
342
343
344
345
346
347
348
349
        boolean result = false;

        if (obj instanceof String) {
          String str = (String) obj;

          result = entrySet.contains(new StringListWrapper(new StringList(str)));

        }

        return result;
      }
View Full Code Here
50
51
52
53
54
55
56
57
58
59
60
61
    NGramModel model = new NGramModel();
    model.add(tokens[index], minLength, maxLength);

    for (Iterator<StringList> it = model.iterator(); it.hasNext();) {

      StringList tokenList = it.next();

      if (tokenList.size() > 0) {
        features.add("ng=" + tokenList.getToken(0).toLowerCase());
      }
    }
  }
View Full Code Here
TOP

Related Classes of opennlp.tools.util.StringList

  • io.lumify.opennlpDictionary.OpenNLPDictionaryExtractorGraphPropertyWorker
  • io.lumify.opennlpDictionary.OpenNLPDictionaryExtractorGraphPropertyWorkerTest
  • opennlp.tools.cmdline.namefind.CensusDictionaryCreatorTool
  • opennlp.tools.coref.CorefModel
  • opennlp.tools.dictionary.Dictionary
  • opennlp.tools.dictionary.DictionaryAsSetCaseInsensitiveTest
  • opennlp.tools.dictionary.DictionaryAsSetCaseSensitiveTest
  • opennlp.tools.dictionary.DictionaryTest
  • opennlp.tools.dictionary.Index
  • opennlp.tools.dictionary.serializer.DictionarySerializer
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.