Examples of opennlp.tools.ml.model.TwoPassDataIndexer

opennlp.tools.ml.model.TwoPassDataIndexer
Collecting event and context counts by making two passes over the events. The first pass determines which contexts will be used by the model, and the second pass creates the events in memory containing only the contexts which will be used. This greatly reduces the amount of memory required for storing the events. During the first pass a temporary event file is created which is read during the second pass.

    return train(languageCode, parseSamples, rules, params);
  }


  @Deprecated
  public static AbstractModel train(ObjectStream<Event>  es, int iterations, int cut) throws java.io.IOException {
    return opennlp.tools.ml.maxent.GIS.trainModel(iterations, new TwoPassDataIndexer(es, cut));
  }

View Full Code Here

   * @deprecated Please do not use anymore, use the ObjectStream train methods instead! This method
   * will be removed soon.
   */
  @Deprecated
  public static AbstractModel train(ObjectStream<Event> es, int iterations, int cut) throws java.io.IOException {
    return opennlp.tools.ml.maxent.GIS.trainModel(iterations, new TwoPassDataIndexer(es, cut));
  }

View Full Code Here

    DataIndexer indexer = null;


    if (DATA_INDEXER_ONE_PASS_VALUE.equals(dataIndexerName)) {
      indexer = new OnePassDataIndexer(events, cutoff, sortAndMerge);
    } else if (DATA_INDEXER_TWO_PASS_VALUE.equals(dataIndexerName)) {
      indexer = new TwoPassDataIndexer(events, cutoff, sortAndMerge);
    } else {
      throw new IllegalStateException("Unexpected data indexer name: "
          + dataIndexerName);
    }
    return indexer;

View Full Code Here


  @Test
  public void testPerceptronOnPrepAttachData() throws IOException {
    MaxentModel model =
        new PerceptronTrainer().trainModel(400,
        new TwoPassDataIndexer(createTrainingStream(), 1, false), 1);


    testModel(model, 0.7650408516959644);
  }

View Full Code Here


  @Test
  public void testQNOnPrepAttachData() throws IOException {
    AbstractModel model = 
        new QNTrainer(true).trainModel(
            100, new TwoPassDataIndexer(createTrainingStream(), 1));


    testModel(model, 0.8155484030700668);
  }

View Full Code Here


  @Test
  public void testMaxentOnPrepAttachData() throws IOException {
    AbstractModel model =
        new GISTrainer(true).trainModel(100,
        new TwoPassDataIndexer(createTrainingStream(), 1), 1);


    testModel(model, 0.7997028967566229);
  }

View Full Code Here


  @Test
  public void testMaxentOnPrepAttachData2Threads() throws IOException {
    AbstractModel model =
        new GISTrainer(true).trainModel(100,
            new TwoPassDataIndexer(createTrainingStream(), 1),
            new UniformPrior(), 1, 2);


    testModel(model, 0.7997028967566229);
  }

View Full Code Here

TOP

Related Classes of opennlp.tools.ml.model.TwoPassDataIndexer

opennlp.tools.ml.AbstractEventTrainer

opennlp.tools.ml.maxent.MaxentPrepAttachTest

opennlp.tools.ml.maxent.quasinewton.QNPrepAttachTest

opennlp.tools.ml.perceptron.PerceptronPrepAttachTest

opennlp.tools.parser.chunking.Parser

opennlp.tools.parser.treeinsert.Parser

java.io.File

java.io.OutputStreamWriter

java.io.FileOutputStream

java.io.Writer

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.