Package opennlp.tools.dictionary

Examples of opennlp.tools.dictionary.Dictionary

195
196
197
198
199
200
201
202
203
204
205
206
207
        System.exit(1);
      }
      ai++;
    }
    HeadRules rules = new opennlp.tools.parser.lang.en.HeadRules(args[ai++]);
    Dictionary dict = null;
    if (ai < args.length) {
      dict = new Dictionary(new FileInputStream(args[ai++]),true);
    }
    if (fun) {
      Parse.useFunctionTags(true);
    }
    ObjectStream<Event> es = new ParserEventStream(new ParseSampleStream(new PlainTextByLineStream(new java.io.InputStreamReader(System.in))), rules, etype, dict);
View Full Code Here
136
137
138
139
140
141
142
143
144
145
146
      if (this.factory == null) {
        this.factory = POSTaggerFactory.create(this.factoryClassName, null,
            null);
      }

      Dictionary ngramDict = this.factory.getDictionary();
      if (ngramDict == null) {
        if(this.ngramCutoff != null) {
          System.err.print("Building ngram dictionary ... ");
          ngramDict = POSTaggerME.buildNGramDictionary(trainingSampleStream,
              this.ngramCutoff);
View Full Code Here
67
68
69
70
71
72
73
74
75
76
77
    }

    File modelOutFile = params.getModel();
    CmdLineUtil.checkOutputFile("pos tagger model", modelOutFile);

    Dictionary ngramDict = null;

    Integer ngramCutoff = params.getNgram();

    if (ngramCutoff != null) {
      System.err.print("Building ngram dictionary ... ");
View Full Code Here
47
48
49
50
51
52
53
54
55
56
57
58
  public String getShortDescription() {
    return "trainer for the learnable tokenizer";
  }

  static Dictionary loadDict(File f) throws IOException {
    Dictionary dict = null;
    if (f != null) {
      CmdLineUtil.checkInputFile("abb dict", f);
      dict = new Dictionary(new FileInputStream(f));
    }
    return dict;
  }
View Full Code Here
80
81
82
83
84
85
86
87
88
89
90
    File modelOutFile = params.getModel();
    CmdLineUtil.checkOutputFile("tokenizer model", modelOutFile);

    TokenizerModel model;
    try {
      Dictionary dict = loadDict(params.getAbbDict());

      TokenizerFactory tokFactory = TokenizerFactory.create(
          params.getFactory(), params.getLang(), dict,
          params.getAlphaNumOpt(), null);
      model = opennlp.tools.tokenize.TokenizerME.train(sampleStream,
View Full Code Here
60
61
62
63
64
65
66
67
68
69
70
    if (params.getMisclassified()) {
      listener = new TokenEvaluationErrorListener();
    }

    try {
      Dictionary dict = TokenizerTrainerTool.loadDict(params.getAbbDict());

      TokenizerFactory tokFactory = TokenizerFactory.create(
          params.getFactory(), params.getLang(), dict,
          params.getAlphaNumOpt(), null);
      validator = new opennlp.tools.tokenize.TokenizerCrossValidator(mlParams,
View Full Code Here
59
60
61
62
63
64
65
66
67
68
69
  @Test
  public void testPOSTaggerWithCustomFactory() throws IOException {
    DummyPOSDictionary posDict = new DummyPOSDictionary(
        POSDictionary.create(POSDictionaryTest.class
            .getResourceAsStream("TagDictionaryCaseSensitive.xml")));
    Dictionary dic = POSTaggerME.buildNGramDictionary(createSampleStream(), 0);

    POSModel posModel = trainPOSModel(ModelType.MAXENT,
        new DummyPOSTaggerFactory(dic, posDict));

    POSTaggerFactory factory = posModel.getFactory();
View Full Code Here
86
87
88
89
90
91
92
93
94
95
96
  @Test
  public void testPOSTaggerWithDefaultFactory() throws IOException {
    POSDictionary posDict = POSDictionary.create(POSDictionaryTest.class
            .getResourceAsStream("TagDictionaryCaseSensitive.xml"));
    Dictionary dic = POSTaggerME.buildNGramDictionary(createSampleStream(), 0);

    POSModel posModel = trainPOSModel(ModelType.MAXENT,
        new POSTaggerFactory(dic, posDict));

    POSTaggerFactory factory = posModel.getFactory();
View Full Code Here
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
   *    built from the input file.
   * @throws IOException IOException
   */
  public static Dictionary createDictionary(ObjectStream<StringList> sampleStream) throws IOException {

    Dictionary mNameDictionary = new Dictionary(true);
    StringList entry;

    entry = sampleStream.read();
    while (entry != null) {
      if (!mNameDictionary.contains(entry)) {
        mNameDictionary.put(entry);
      }
      entry = sampleStream.read();
    }

    return mNameDictionary;
View Full Code Here
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
    FileInputStream sampleDataIn = CmdLineUtil.openInFile(testData);
    ObjectStream<StringList> sampleStream = new NameFinderCensus90NameStream(sampleDataIn,
        Charset.forName(params.getEncoding()));

    Dictionary mDictionary;
    try {
      System.out.println("Creating Dictionary...");
      mDictionary = createDictionary(sampleStream);
    } catch (IOException e) {
      throw new TerminateToolException(-1, "IO error while reading training data or indexing data: "
          + e.getMessage(), e);
    } finally {
      try {
        sampleStream.close();
      } catch(IOException e) {
        // sorry this can fail..
      }
    }

    System.out.println("Saving Dictionary...");

    OutputStream out = null;

    try {
      out = new FileOutputStream(dictOutFile);
      mDictionary.serialize(out);
    } catch (IOException e) {
      throw new TerminateToolException(-1, "IO error while writing dictionary file: "
          + e.getMessage(), e);
    }
    finally {
View Full Code Here
TOP

Related Classes of opennlp.tools.dictionary.Dictionary

  • io.lumify.opennlpDictionary.OpenNLPDictionaryExtractorGraphPropertyWorker
  • io.lumify.opennlpDictionary.OpenNLPDictionaryExtractorGraphPropertyWorkerTest
  • opennlp.tools.cmdline.dictionary.DictionaryBuilderTool
  • opennlp.tools.cmdline.namefind.CensusDictionaryCreatorTool
  • opennlp.tools.cmdline.parser.BuildModelUpdaterTool
  • opennlp.tools.cmdline.parser.CheckModelUpdaterTool
  • opennlp.tools.cmdline.parser.ParserTrainerTool
  • opennlp.tools.cmdline.postag.POSTaggerTrainerTool
  • opennlp.tools.cmdline.sentdetect.SentenceDetectorCrossValidatorTool
  • opennlp.tools.cmdline.sentdetect.SentenceDetectorTrainerTool
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.