Examples of opennlp.tools.ngram.Dictionary

opennlp.tools.ngram.Dictionary
This class allows for the the loading of n-gram dictionaries to facilitate feature generation for n-gram based models. @see MutableDictionary @author Tom Morton

    events = ec.getEvents();
    //System.err.println("POSEventStream.addNewEvents: got "+events.length+" events");
  }


  public static void main(String[] args) throws java.io.IOException {
    EventStream es = new POSEventStream(new opennlp.maxent.PlainTextByLineDataStream(new java.io.InputStreamReader(System.in)),new Dictionary(args[0]));
    while (es.hasNext()) {
      System.out.println(es.nextEvent());
    }
  }

View Full Code Here

      if (encoding == null) {
        if (dict == null) {
          es = new POSEventStream(new PlainTextByLineDataStream(new InputStreamReader(new FileInputStream(inFile))));
        }
        else {
          es = new POSEventStream(new PlainTextByLineDataStream(new InputStreamReader(new FileInputStream(inFile))), new Dictionary(dict));
        }
      }
      else {
        if (dict == null) {
          es = new POSEventStream(new PlainTextByLineDataStream(new InputStreamReader(new FileInputStream(inFile),encoding)));
        }
        else {
          es = new POSEventStream(new PlainTextByLineDataStream(new InputStreamReader(new FileInputStream(inFile),encoding)), new Dictionary(dict));
        }
      }
      mod = train(es, iterations, cutoff);
      System.out.println("Saving the model as: " + outFile);
      new SuffixSensitiveGISModelWriter(mod, outFile).persist();

View Full Code Here

      mdict.persist(dictFile);
    }
    if (tag || all) {
      System.err.println("Training tagger");
      System.err.println("Loading Dictionary");
      Dictionary tridict = new Dictionary(dictFile.toString());
      opennlp.maxent.EventStream tes = new ParserEventStream(new opennlp.maxent.PlainTextByLineDataStream(new java.io.FileReader(inFile)), rules, EventTypeEnum.TAG,tridict);
      GISModel tagModel = train(tes, iterations, cutoff);
      System.out.println("Saving the tagger model as: " + tagFile);
      new opennlp.maxent.io.SuffixSensitiveGISModelWriter(tagModel, tagFile).persist();
    }


    if (chunk || all) {
      System.err.println("Training chunker");
      opennlp.maxent.EventStream ces = new ParserEventStream(new opennlp.maxent.PlainTextByLineDataStream(new java.io.FileReader(inFile)), rules, EventTypeEnum.CHUNK);
      GISModel chunkModel = train(ces, iterations, cutoff);
      System.out.println("Saving the chunker model as: " + chunkFile);
      new opennlp.maxent.io.SuffixSensitiveGISModelWriter(chunkModel, chunkFile).persist();
    }


    if (build || all) {
      System.err.println("Loading Dictionary");
      Dictionary tridict = new Dictionary(dictFile.toString());
      System.err.println("Training builder");
      opennlp.maxent.EventStream bes = new ParserEventStream(new opennlp.maxent.PlainTextByLineDataStream(new java.io.FileReader(inFile)), rules, EventTypeEnum.BUILD,tridict);
      GISModel buildModel = train(bes, iterations, cutoff);
      System.out.println("Saving the build model as: " + buildFile);
      new opennlp.maxent.io.SuffixSensitiveGISModelWriter(buildModel, buildFile).persist();

View Full Code Here

      dictFile = args[ai++];
    }
    
    if (tagdict != null) {
      if (dictFile != null) {
        tagger = new PosTagger(model,new Dictionary(dictFile),new POSDictionary(tagdict,caseSensitive));
      }
      else {
        tagger = new PosTagger(model,new POSDictionary(tagdict,caseSensitive));
      }
    }
    else {
      if (dictFile != null) {
        tagger = new PosTagger(model,new Dictionary(dictFile));
      }
      else {
        tagger = new PosTagger(model,(Dictionary)null);
      }
    }

View Full Code Here

        System.exit(1);
      }
      ai++;
    }
    HeadRules rules = new opennlp.tools.lang.english.HeadRules(args[ai++]);
    Dictionary dict = null;
    if (ai < args.length) {
      dict = new Dictionary(args[ai++]);
    }
    if (fun) {
      Parse.useFunctionTags(true);
    }
    opennlp.maxent.EventStream es = new ParserEventStream(new opennlp.maxent.PlainTextByLineDataStream(new java.io.InputStreamReader(System.in)), rules, etype, dict);

View Full Code Here

TOP

Related Classes of opennlp.tools.ngram.Dictionary

opennlp.tools.lang.english.PosTagger

opennlp.tools.parser.ParserEventStream

opennlp.tools.parser.ParserME

opennlp.tools.postag.POSEventStream

opennlp.tools.postag.POSTaggerME

opennlp.tools.util.NumberedSet

java.io.BufferedReader

java.util.HashSet

java.io.File

java.io.InputStreamReader

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.