Package edu.stanford.nlp.international.french.scripts

Source Code of edu.stanford.nlp.international.french.scripts.MWEFrequencyDist

package edu.stanford.nlp.international.french.scripts;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.util.List;
import java.util.Set;

import edu.stanford.nlp.ling.CoreLabel;
import edu.stanford.nlp.stats.Counter;
import edu.stanford.nlp.stats.TwoDimensionalCounter;
import edu.stanford.nlp.trees.Tree;
import edu.stanford.nlp.trees.TreeReader;
import edu.stanford.nlp.trees.TreeReaderFactory;
import edu.stanford.nlp.trees.international.french.FrenchTreeReaderFactory;
import edu.stanford.nlp.trees.tregex.TregexParseException;
import edu.stanford.nlp.trees.tregex.TregexMatcher;
import edu.stanford.nlp.trees.tregex.TregexPattern;
import edu.stanford.nlp.util.Generics;

/**
* Prints a frequency distribution of MWEs in French.
*
* @author Spence Green
*
*/
public final class MWEFrequencyDist {

  private MWEFrequencyDist() {};
 
  public static void main(String[] args) {
    if(args.length != 1) {
      System.err.printf("Usage: java %s file%n", MWEFrequencyDist.class.getName());
      System.exit(-1);
    }
   
    final File treeFile = new File(args[0]);
    TwoDimensionalCounter<String,String> mweLabelToString = new TwoDimensionalCounter<String,String>();
    Set<String> uniquePOSSequences = Generics.newHashSet();
   
    try {
      BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(treeFile), "UTF-8"));
      TreeReaderFactory trf = new FrenchTreeReaderFactory();
      TreeReader tr = trf.newTreeReader(br);

      final TregexPattern pMWE = TregexPattern.compile("/^MW/");
      for(Tree t; (t = tr.readTree()) != null;) {
        //Count MWE statistics
        TregexMatcher m = pMWE.matcher(t);
        while(m.findNextMatchingNode()) {
          Tree match = m.getMatch();
          String label = match.value();
          List<CoreLabel> yield = match.taggedLabeledYield();
          StringBuilder termYield = new StringBuilder();
          StringBuilder posYield = new StringBuilder();
          for(CoreLabel cl : yield) {
            termYield.append(cl.word()).append(" ");
            posYield.append(cl.tag()).append(" ");
          }
          mweLabelToString.incrementCount(label, termYield.toString().trim());
          uniquePOSSequences.add(posYield.toString().trim());
        }
      }
      tr.close(); //Closes the underlying reader
     
      System.out.printf("Type\t#Type\t#Single\t%%Single\t%%Total%n");
     
      double nMWEs = mweLabelToString.totalCount();
      int nAllSingletons = 0;
      int nTokens = 0;
      for(String mweLabel : mweLabelToString.firstKeySet()) {
        int nSingletons = 0;
        double totalCount = mweLabelToString.totalCount(mweLabel);
        Counter<String> mc = mweLabelToString.getCounter(mweLabel);
        for(String term : mc.keySet()) {
          if(mc.getCount(term) == 1.0)
            nSingletons++;
          nTokens += term.split("\\s+").length * (int) mc.getCount(term);
        }
        nAllSingletons += nSingletons;
        System.out.printf("%s\t%d\t%d\t%.2f\t%.2f%n",mweLabel,(int) totalCount,nSingletons, 100.0 * nSingletons / totalCount, 100.0 * totalCount / nMWEs);
      }
      System.out.printf("TOTAL:\t%d\t%d\t%.2f%n", (int) nMWEs,nAllSingletons, 100.0 * nAllSingletons / nMWEs);
      System.out.println("#tokens = " + nTokens);
      System.out.println("#unique MWE POS sequences = " + uniquePOSSequences.size());

    } catch (UnsupportedEncodingException e) {
      e.printStackTrace();

    } catch (FileNotFoundException e) {
      e.printStackTrace();

    } catch (TregexParseException e) {
      e.printStackTrace();

    } catch (IOException e) {
      e.printStackTrace();
    }
  }
}
TOP

Related Classes of edu.stanford.nlp.international.french.scripts.MWEFrequencyDist

TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.