Examples of opennlp.model.AbstractModel

opennlp.model.AbstractModel

      int iterations = 100;
      if (args.length > ai) {
        cutoff = Integer.parseInt(args[ai++]);
        iterations = Integer.parseInt(args[ai++]);
      }
      AbstractModel mod;
      if (dict != null) {
        buildDictionary(dict, inFile, cutoff);
      }
      if (sequence) {
        POSSampleSequenceStream ss;

View Full Code Here

     if (generator != null)
       featureGenerator = generator;
     else
       featureGenerator = createFeatureGenerator();


     AbstractModel nameFinderModel;


     if (!TrainUtil.isSequenceTraining(trainParams.getSettings())) {
       EventStream eventStream = new NameFinderEventStream(samples, type,
           new DefaultNameContextGenerator(featureGenerator));

View Full Code Here

    }
    ParserEventTypeEnum etype = null;
    boolean fun = false;
    int ai = 0;
    Dictionary dict = null;
    AbstractModel model = null;


    while (ai < args.length && args[ai].startsWith("-")) {
      if (args[ai].equals("-build")) {
        etype = ParserEventTypeEnum.BUILD;
      }
      else if (args[ai].equals("-attach")) {
        etype = ParserEventTypeEnum.ATTACH;
      }
      else if (args[ai].equals("-chunk")) {
        etype = ParserEventTypeEnum.CHUNK;
      }
      else if (args[ai].equals("-check")) {
        etype = ParserEventTypeEnum.CHECK;
      }
      else if (args[ai].equals("-tag")) {
        etype = ParserEventTypeEnum.TAG;
      }
      else if (args[ai].equals("-fun")) {
        fun = true;
      }
      else if (args[ai].equals("-dict")) {
        ai++;
        dict = new Dictionary(new FileInputStream(args[ai]));
      }
      else if (args[ai].equals("-model")) {
        ai++;
        model = (new SuffixSensitiveGISModelReader(new File(args[ai]))).getModel();
      }
      else {
        System.err.println("Invalid option " + args[ai]);
        System.exit(1);
      }
      ai++;
    }
    HeadRules rules = new opennlp.tools.parser.lang.en.HeadRules(args[ai++]);
    if (fun) {
      Parse.useFunctionTags(true);
    }
    opennlp.model.EventStream es = new ParserEventStream(new ParseSampleStream(new PlainTextByLineStream(new java.io.InputStreamReader(System.in))), rules, etype, dict);
    while (es.hasNext()) {
      Event e = es.next();
      if (model != null) {
        System.out.print(model.eval(e.getContext())[model.getIndex(e.getOutcome())]+" ");
      }
      System.out.println(e);
    }
  }

View Full Code Here


    String languageCode = args[ai++];
    String packageName = args[ai++];
    String modelName = args[ai];


    AbstractModel model = new GenericModelReader(new File(modelName)).getModel();
    SentenceModel packageModel = new SentenceModel(languageCode, model,
        useTokenEnd, abbreviations, (char[]) null);
    packageModel.serialize(new FileOutputStream(packageName));
  }

View Full Code Here


    // TODO: Fix the EventStream to throw exceptions when training goes wrong
    EventStream eventStream = new SDEventStream(samples,
        sdFactory.getSDContextGenerator(), sdFactory.getEndOfSentenceScanner());


    AbstractModel sentModel = TrainUtil.train(eventStream,
        mlParams.getSettings(), manifestInfoEntries);


    return new SentenceModel(languageCode, sentModel, manifestInfoEntries,
        sdFactory);
  }

View Full Code Here

      modelWriter.persist();
      modelWriter.close();
      
      GenericModelReader modelReader = new GenericModelReader(new BinaryFileDataReader(
          new ByteArrayInputStream(modelBytes.toByteArray())));
      AbstractModel readModel = modelReader.getModel();
      QNModel deserModel = (QNModel) readModel;
      
      assertTrue(trainedModel.equals(deserModel)); 
      
      String[] features2Classify = new String[] {"feature2","feature3", "feature3", "feature3","feature3", "feature3", "feature3","feature3", "feature3", "feature3","feature3", "feature3"};

View Full Code Here

    
    // build
    System.err.println("Training builder");
    opennlp.model.EventStream bes = new ParserEventStream(parseSamples, rules, ParserEventTypeEnum.BUILD, mdict);
    HashSumEventStream hsbes = new HashSumEventStream(bes);
    AbstractModel buildModel = train(hsbes, iterations, cut);
    manifestInfoEntries.put("Training-Builder-Eventhash", 
        hsbes.calculateHashSum().toString(16));
    
    parseSamples.reset();
    
    // tag
    POSModel posModel = POSTaggerME.train(languageCode, new PosSampleStream(parseSamples), 
        ModelType.MAXENT, null, null, cut, iterations);
    
    parseSamples.reset();
    
    // chunk
    ChunkerModel chunkModel = ChunkerME.train(languageCode, 
        new ChunkSampleStream(parseSamples), cut, iterations,
        new ChunkContextGenerator());
    
    parseSamples.reset();
    
    // check
    System.err.println("Training checker");
    opennlp.model.EventStream kes = new ParserEventStream(parseSamples, rules, ParserEventTypeEnum.CHECK);
    HashSumEventStream hskes = new HashSumEventStream(kes);
    AbstractModel checkModel = train(hskes, iterations, cut);
    manifestInfoEntries.put("Training-Checker-Eventhash", 
        hskes.calculateHashSum().toString(16));
    
    // TODO: Remove cast for HeadRules
    return new ParserModel(languageCode, buildModel, checkModel,

View Full Code Here

    if (build || all) {
      System.err.println("Loading Dictionary");
      Dictionary tridict = new Dictionary(new FileInputStream(dictFile.toString()),true);
      System.err.println("Training builder");
      opennlp.model.EventStream bes = new ParserEventStream(new ParseSampleStream(new PlainTextByLineStream(new java.io.FileReader(inFile))), rules, ParserEventTypeEnum.BUILD,tridict);
      AbstractModel buildModel = train(bes, iterations, cutoff);
      System.out.println("Saving the build model as: " + buildFile);
      new opennlp.maxent.io.SuffixSensitiveGISModelWriter(buildModel, buildFile).persist();
    }


    if (check || all) {
      System.err.println("Training checker");
      opennlp.model.EventStream kes = new ParserEventStream(new ParseSampleStream(new PlainTextByLineStream(new java.io.FileReader(inFile))), rules, ParserEventTypeEnum.CHECK);
      AbstractModel checkModel = train(kes, iterations, cutoff);
      System.out.println("Saving the check model as: " + checkFile);
      new opennlp.maxent.io.SuffixSensitiveGISModelWriter(checkModel, checkFile).persist();
    }
  }

View Full Code Here

  public static void main(String[] args) throws java.io.IOException {
    if (args.length == 0) {
      System.err.println("Usage: PerceptronModel modelname < contexts");
      System.exit(1);
    }
    AbstractModel m = new PerceptronModelReader(new File(args[0])).getModel();
    BufferedReader in = new BufferedReader(new InputStreamReader(System.in));
    DecimalFormat df = new java.text.DecimalFormat(".###");
    for (String line = in.readLine(); line != null; line = in.readLine()) {
      String[] context = line.split(" ");
      double[] dist = m.eval(context);
      for (int oi=0;oi<dist.length;oi++) {
        System.out.print("["+m.getOutcome(oi)+" "+df.format(dist[oi])+"] ");
      }
      System.out.println();
    }
  }

View Full Code Here

    if (args.length != 6){
      System.err.println("ParserModel packageName buildModel checkModel headRules chunkerModel posModel");
      System.exit(1);
    }


    AbstractModel buildModel = readModel(args[1]);


    AbstractModel checkModel = readModel(args[2]);


    opennlp.tools.parser.lang.en.HeadRules headRules =
        new opennlp.tools.parser.lang.en.HeadRules(args[3]);


    ChunkerModel chunkerModel = new ChunkerModel(new FileInputStream(args[4]));

View Full Code Here

0 1 2 3 4 5 6

TOP

Related Classes of opennlp.model.AbstractModel

CreateModel

opennlp.maxent.GISModel

opennlp.maxent.io.SuffixSensitiveGISModelReader

opennlp.maxent.MaxentPrepAttachTest

opennlp.maxent.ModelTrainer

opennlp.maxent.quasinewton.QNTrainerTest

opennlp.perceptron.PerceptronModel

opennlp.perceptron.PerceptronPrepAttachTest

opennlp.tools.chunker.ChunkerME

opennlp.tools.chunker.ChunkerModel

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.