Examples of TwoPassDataIndexer

opennlp.maxent.TwoPassDataIndexer
opennlp.model.TwoPassDataIndexer
Collecting event and context counts by making two passes over the events. The first pass determines which contexts will be used by the model, and the second pass creates the events in memory containing only the contexts which will be used. This greatly reduces the amount of memory required for storing the events. During the first pass a temporary event file is created which is read during the second pass.
opennlp.tools.ml.model.TwoPassDataIndexer
Collecting event and context counts by making two passes over the events. The first pass determines which contexts will be used by the model, and the second pass creates the events in memory containing only the contexts which will be used. This greatly reduces the amount of memory required for storing the events. During the first pass a temporary event file is created which is read during the second pass.

Examples of opennlp.maxent.TwoPassDataIndexer

  public double[] probs() {
    return bestSequence.getProbs();
  }


  private static GISModel train(EventStream es, int iterations, int cut) throws IOException {
    return GIS.trainModel(iterations, new TwoPassDataIndexer(es, cut));
  }

View Full Code Here

Examples of opennlp.maxent.TwoPassDataIndexer

    }
    return (orderedTags);
  }


  public static GISModel train(EventStream es, int iterations, int cut) throws IOException {
    return opennlp.maxent.GIS.trainModel(iterations, new TwoPassDataIndexer(es, cut));
  }

View Full Code Here

Examples of opennlp.maxent.TwoPassDataIndexer

    return bestSequence.getProbs();
  }


  
  private static GISModel train(opennlp.maxent.EventStream es, int iterations, int cut) throws java.io.IOException {
    return opennlp.maxent.GIS.trainModel(iterations, new TwoPassDataIndexer(es, cut));
  }

View Full Code Here

Examples of opennlp.maxent.TwoPassDataIndexer

    //System.err.println("collapsedPunctuation: collapsedParses"+collapsedParses);
    return (Parse[]) collapsedParses.toArray(new Parse[collapsedParses.size()]);
  }


  public static GISModel train(opennlp.maxent.EventStream es, int iterations, int cut) throws java.io.IOException {
    return opennlp.maxent.GIS.trainModel(iterations, new TwoPassDataIndexer(es, cut));
  }

View Full Code Here

Examples of opennlp.maxent.TwoPassDataIndexer

    }
    return (Span[]) tokens.toArray(new Span[tokens.size()]);
  }


  public static void train(EventStream evc, File output) throws IOException {
    GISModel tokModel = opennlp.maxent.GIS.trainModel(100,new TwoPassDataIndexer(evc, 5),true);
    new SuffixSensitiveGISModelWriter(tokModel, output).persist();
  }

View Full Code Here

Examples of opennlp.maxent.TwoPassDataIndexer

    modelsByType = new HashMap<String,GISModel>();
    for(String type : eventsByType.keySet()) {
      DataIndexer di = null;
      List<Event> evs = eventsByType.get(type);
      if(evs.size() == 1) evs.add(evs.get(0));
      di = new TwoPassDataIndexer(new EventCollectorAsStream(new SimpleEventCollector(evs)), 1);
      modelsByType.put(type, GIS.trainModel(trainingCycles, di));
    }
  }

View Full Code Here

Examples of opennlp.maxent.TwoPassDataIndexer

    if(useUber) {
      List<Event> evs = new ArrayList<Event>();
      for(String prevTagg : evsByPrev.keySet()) {
        evs.addAll(evsByPrev.get(prevTagg));
      }
      DataIndexer di = new TwoPassDataIndexer(new EventCollectorAsStream(new SimpleEventCollector(evs)), featureCutOff);
      ubermodel = GIS.trainModel(trainingCycles, di);
    } else {
      for(String prevTagg : evsByPrev.keySet()) {
        if(Oscar3Props.getInstance().verbose) System.out.println(prevTagg);
        List<Event> evs = evsByPrev.get(prevTagg);
        if(featureSel) {
          evs = new FeatureSelector().selectFeatures(evs);            
        }    
        if(evs.size() == 1) {
          evs.add(evs.get(0));
        }
        DataIndexer di = null;
        try {
          di = new TwoPassDataIndexer(new EventCollectorAsStream(new SimpleEventCollector(evs)), featureCutOff);
          gmByPrev.put(prevTagg, GIS.trainModel(trainingCycles, di));
        } catch (Exception e) {
          di = new TwoPassDataIndexer(new EventCollectorAsStream(new SimpleEventCollector(evs)), 1);        
          gmByPrev.put(prevTagg, GIS.trainModel(trainingCycles, di));
        }  
      }
    }
  }

View Full Code Here

Examples of opennlp.maxent.TwoPassDataIndexer

      String outcome = "E";
      if(isChem.get(s)) outcome = "C";
      events.add(new Event(outcome, getFeatures(s)));
    }
    SimpleEventCollector sec = new SimpleEventCollector(events);
    DataIndexer di = new TwoPassDataIndexer(new EventCollectorAsStream(sec), 1);
    GISModel gis = GIS.trainModel(100, di);
    
    System.out.println(gis.getIndex("FOO"));
    if(true) return;
    
    
    long time = System.currentTimeMillis();
    StringGISModelWriter sgmw = new StringGISModelWriter(gis);
    sgmw.persist();
    String modelStr = sgmw.toString();


    //System.out.println(modelStr);
    
    StringGISModelReader sgmr = new StringGISModelReader(modelStr);
    gis = sgmr.getModel();
    System.out.println("Round trip in " + (System.currentTimeMillis() - time));
    System.out.println(modelStr.length());
    
    //if(true) return;
    
    GISModel gis2 = null;
    
    if(false) {
      List<Event> events2 = new ArrayList<Event>();
      for(String s : batch2) {
        double prob = gis.eval(getFeatures(s))[gis.getIndex("C")];
        String outcome = "E";
        if(isChem.get(s)) outcome = "C";
        events.add(new Event(outcome, probToFeatures(prob)));
      }
      sec = new SimpleEventCollector(events);
      di = new TwoPassDataIndexer(new EventCollectorAsStream(sec), 2);
      gis2 = GIS.trainModel(1000, di);      
    }
    
    double score = 0.0;
    List<Double> trialScores = new ArrayList<Double>();

View Full Code Here

Examples of opennlp.maxent.TwoPassDataIndexer


      Event event = new Event(type, f.toArray(new String[0]));
      events.add(event);
    }
    if(events.size() == 1) events.add(events.get(0));
    DataIndexer di = new TwoPassDataIndexer(new EventCollectorAsStream(new SimpleEventCollector(events)), 3);
    GISModel gm = GIS.trainModel(100, di);
    
    ClassificationEvaluator ce = new ClassificationEvaluator();


    List<Double> chemList = new ArrayList<Double>();

View Full Code Here

Examples of opennlp.maxent.TwoPassDataIndexer

      trainEvents.add(new Event(be.getClassLabel(), be.getFeatures().getSet().toArray(new String[0])));
    }
    for(BagEvent be : testBagEvents) {
      testEvents.add(new Event(be.getClassLabel(), be.getFeatures().getSet().toArray(new String[0])));
    }
    DataIndexer di = new TwoPassDataIndexer(new EventCollectorAsStream(new SimpleEventCollector(trainEvents)), 1);
    GISModel gm = GIS.trainModel(100, di);
    
    //ClassificationEvaluator ce = new ClassificationEvaluator();
    
    for(Event event : testEvents) {

View Full Code Here

0 1 2 3 4

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.