Package edu.stanford.nlp.process

Examples of edu.stanford.nlp.process.PTBTokenizer$PTBTokenizerFactory

56
57
58
59
60
61
62
63
64
65
66
67
      int pNum = 0;
      int sNum = 0;
      int wNum = 0;


      PTBTokenizer ptb = PTBTokenizer.newPTBTokenizer(new BufferedReader(new StringReader(doc)), false, true);
      List<CoreLabel> words = ptb.tokenize();

      List<CoreLabel> result = new ArrayList<CoreLabel>();

      CoreLabel prev = null;
      String prevString = "";
View Full Code Here
68
69
70
71
72
73
74
75
76
77
78
79
80
81
        // Output bag
        DataBag bagOfTokens = bagFactory.newDefaultBag();
               
        StringReader textInput = new StringReader(input.get(0).toString());
        PTBTokenizer ptbt = new PTBTokenizer(textInput, new CoreLabelTokenFactory(), "");

        for (CoreLabel label; ptbt.hasNext(); ) {
          label = (CoreLabel)ptbt.next();
          Tuple termText = tupleFactory.newTuple(label.toString());
          bagOfTokens.add(termText);
        }
       
        return bagOfTokens;
View Full Code Here
32
33
34
35
36
37
38
39
40
41
42
43
  }
 
 
  public String tag(String s) {
    Reader r = new StringReader(s);
    PTBTokenizer ptb = new PTBTokenizer(r);
    String tokenText = PTBTokenizer.ptb2Text(ptb.tokenize());
    Debug.printSeparate(tokenText);
    try {
      String tagged = MaxentTagger.tagString(tokenText);
      tagged = tagged.replaceAll("(bxgqwp.*?)/[^ ]*", "$1/DATE");
      Debug.printSeparate(tagged);
View Full Code Here
TOP

Related Classes of edu.stanford.nlp.process.PTBTokenizer$PTBTokenizerFactory

  • edu.stanford.nlp.ling.CoreLabel
  • edu.stanford.nlp.sequences.MUCDocumentReaderAndWriter$MUCDocumentParser
  • edu.stanford.nlp.util.Timing
  • org.timedex.extraction.PartOfSpeechTagger
  • varaha.text.StanfordTokenize
  • edu.stanford.nlp.io.RuntimeIOException
  • java.util.regex.Matcher
  • java.text.NumberFormat
  • java.util.regex.Pattern
  • java.text.DecimalFormat
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.