Examples of edu.stanford.nlp.stats.ClassicCounter

edu.stanford.nlp.stats.ClassicCounter
A specialized kind of hash table (or map) for storing numeric counts for objects. It works like a Map, but with different methods for easily getting/setting/incrementing counts for objects and computing various functions with the counts. The Counter constructor and addAll method can be used to copy another Counter's contents over.
Implementation notes: You shouldn't casually add further methods to this interface. Rather, they should be added to the {@link Counters} class.Note that this class stores a totalCount field as well as the map. This makes certain operations much more efficient, but means that any methods that change the map must also update totalCount appropriately. If you use the setCount method, then you cannot go wrong. This class is not threadsafe: If multiple threads are accessing the same counter, then access should be synchronized externally to this class. @author Dan Klein (klein@cs.stanford.edu) @author Joseph Smarr (jsmarr@stanford.edu) @author Teg Grenager @author Galen Andrew @author Christopher Manning @author Kayur Patel (kdpatel@cs)

public class QuasiDeterminizer implements TransducerGraph.GraphProcessor {




  public TransducerGraph processGraph(TransducerGraph graph) {
    // compute lambda function
    ClassicCounter lambda = computeLambda(graph); // not destructive
    // do the pushing
    TransducerGraph result = pushLambdas(graph, lambda); // creates a new one
    return result;
  }

View Full Code Here

  /**
   * Takes time linear in number of arcs.
   */
  public static ClassicCounter computeLambda(TransducerGraph graph) {
    LinkedList queue = new LinkedList();
    ClassicCounter lambda = new ClassicCounter();
    ClassicCounter length = new ClassicCounter();
    Map first = new HashMap();
    Set nodes = graph.getNodes();
    for (Object node : nodes) {
      lambda.setCount(node, 0);
      length.setCount(node, Double.POSITIVE_INFINITY);
    }
    Set endNodes = graph.getEndNodes();
    for (Object o : endNodes) {
      lambda.setCount(o, 0);
      length.setCount(o, 0);
      queue.addLast(o);
    }
    // Breadth first search
    // get the first node from the queue
    Object node = null;
    try {
      node = queue.removeFirst();
    } catch (NoSuchElementException e) {
    }
    while (node != null) {
      double oldLen = length.getCount(node);
      Set arcs = graph.getArcsByTarget(node);
      if (arcs != null) {
        for (Object arc1 : arcs) {
          TransducerGraph.Arc arc = (TransducerGraph.Arc) arc1;
          Object newNode = arc.getSourceNode();
          Comparable a = (Comparable) arc.getInput();
          double k = ((Double) arc.getOutput()).doubleValue();
          double newLen = length.getCount(newNode);
          if (newLen == Double.POSITIVE_INFINITY) {
            // we are discovering this
            queue.addLast(newNode);
          }
          Comparable f = (Comparable) first.get(newNode);
          if (newLen == Double.POSITIVE_INFINITY || (newLen == oldLen + 1 && a.compareTo(f) < 0)) { // f can't be null, since we have a newLen
            // we do this to this to newNode when we have new info, possibly many times
            first.put(newNode, a); // ejecting old one if necessary
            length.setCount(newNode, oldLen + 1); // this may already be the case
            lambda.setCount(newNode, k + lambda.getCount(node));
          }
        }
      }
      // get a new node from the queue

View Full Code Here

  /**
   * If markovOrder is zero, we always transition back to the start state
   * If markovOrder is negative, we assume that it is infinite
   */
  public static TransducerGraph createGraphFromPaths(List paths, int markovOrder) {
    ClassicCounter pathCounter = new ClassicCounter();
    for (Object o : paths) {
      pathCounter.incrementCount(o);
    }
    return createGraphFromPaths(pathCounter, markovOrder);
  }

View Full Code Here

  public void train(Collection<Pair<?,?>> data) {
    for (Pair p : data) {
      Object seen = p.first();
      Object hidden = p.second();
      if (!hiddenToSeen.keySet().contains(hidden)) {
        hiddenToSeen.put(hidden, new ClassicCounter());
      }
      hiddenToSeen.get(hidden).incrementCount(seen);


      if (!seenToHidden.keySet().contains(seen)) {
        seenToHidden.put(seen, new ClassicCounter());
      }
      seenToHidden.get(seen).incrementCount(hidden);
    }
  }

View Full Code Here

  public RandomWalk(Collection<Pair<?,?>> data, int steps) {
    train(data);
    for (Iterator i = seenToHidden.keySet().iterator(); i.hasNext();) {
      Object seen = i.next();
      if (!model.containsKey(seen)) {
        model.put(seen, new ClassicCounter());
      }
      for (Iterator j = hiddenToSeen.keySet().iterator(); j.hasNext();) {
        Object hidden = j.next();
        model.get(seen).setCount(hidden, score(seen, hidden, steps));
      }

View Full Code Here

    List right = rightSisterLabels(t, p);


    String label = t.label().value();


    if (!nodeRules.containsKey(label)) {
      nodeRules.put(label, new ClassicCounter());
    }


    if (!rightRules.containsKey(label)) {
      rightRules.put(label, new HashMap());
    }

View Full Code Here

  protected void sideCounters(String label, List rewrite, List sideSisters, Map sideRules) {
    for (Iterator i = sideSisters.iterator(); i.hasNext();) {
      String sis = (String) i.next();


      if (!((Map) sideRules.get(label)).containsKey(sis)) {
        ((Map) sideRules.get(label)).put(sis, new ClassicCounter());
      }


      ((ClassicCounter) ((HashMap) sideRules.get(label)).get(sis)).incrementCount(rewrite);
    }
  }

View Full Code Here

    ArrayList topScores = new ArrayList();


    for (Iterator it = nodeRules.keySet().iterator(); it.hasNext();) {
      ArrayList answers = new ArrayList();
      String label = (String) it.next();
      ClassicCounter cntr = (ClassicCounter) nodeRules.get(label);
      double support = (cntr.totalCount());
      System.out.println("Node " + label + " support is " + support);




      for (Iterator it2 = ((HashMap) leftRules.get(label)).keySet().iterator(); it2.hasNext();) {
        String sis = (String) it2.next();
        ClassicCounter cntr2 = (ClassicCounter) ((HashMap) leftRules.get(label)).get(sis);
        double support2 = (cntr2.totalCount());


        /* alternative 1: use full distribution to calculate score */
        double kl = Counters.klDivergence(cntr2, cntr);


        /* alternative 2: hold out test-context data to calculate score */
        /* this doesn't work because it can lead to zero-probability
         * data points hence infinite divergence */
        //   Counter tempCounter = new Counter();
        //   tempCounter.addCounter(cntr2);
        //   for(Iterator i = tempCounter.seenSet().iterator(); i.hasNext();) {
        //     Object o = i.next();
        //     tempCounter.setCount(o,-1*tempCounter.countOf(o));
        //   }
        //   System.out.println(tempCounter); //debugging
        //   tempCounter.addCounter(cntr);
        //   System.out.println(tempCounter); //debugging
        //   System.out.println(cntr);
        //   double kl = cntr2.klDivergence(tempCounter);
        /* alternative 2 ends here */


        String annotatedLabel = label + "=l=" + sis;
        System.out.println("KL(" + annotatedLabel + "||" + label + ") = " + nf.format(kl) + "\t" + "support(" + sis + ") = " + support2);
        answers.add(new Pair(annotatedLabel, new Double(kl * support2)));
        topScores.add(new Pair(annotatedLabel, new Double(kl * support2)));
      }


      for (Iterator it2 = ((HashMap) rightRules.get(label)).keySet().iterator(); it2.hasNext();) {
        String sis = (String) it2.next();
        ClassicCounter cntr2 = (ClassicCounter) ((HashMap) rightRules.get(label)).get(sis);
        double support2 = (cntr2.totalCount());
        double kl = Counters.klDivergence(cntr2, cntr);
        String annotatedLabel = label + "=r=" + sis;
        System.out.println("KL(" + annotatedLabel + "||" + label + ") = " + nf.format(kl) + "\t" + "support(" + sis + ") = " + support2);
        answers.add(new Pair(annotatedLabel, new Double(kl * support2)));
        topScores.add(new Pair(annotatedLabel, new Double(kl * support2)));

View Full Code Here

TOP

Related Classes of edu.stanford.nlp.stats.ClassicCounter

edu.stanford.nlp.fsm.QuasiDeterminizer

edu.stanford.nlp.fsm.TransducerGraph

edu.stanford.nlp.parser.lexparser.RandomWalk

edu.stanford.nlp.parser.lexparser.SisterAnnotationStats

edu.stanford.nlp.util.MutableDouble

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.