Package edu.stanford.nlp.parser.lexparser

Examples of edu.stanford.nlp.parser.lexparser.Options

19
20
21
22
23
24
25
26
27
28
29
* @author Ahmed
*/
public class StringToWordsTokenizer {
    public static ArrayList<Word> tokenize(String in){
        ArrayList<Word> lst;
         Options op = new Options();
        TokenizerFactory<? extends HasWord> tf = op.tlpParams.treebankLanguagePack().getTokenizerFactory();
        Tokenizer<? extends HasWord> tokenizer = tf.getTokenizer(new BufferedReader(new StringReader((String) in)));
        lst = (ArrayList<Word>) tokenizer.tokenize();
        return lst;
    }
View Full Code Here
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
  static final boolean DEBUG = false;

  public FilterConfusingRules(LexicalizedParser parser) {
    BinaryGrammar binaryGrammar = parser.bg;
    UnaryGrammar unaryGrammar = parser.ug;
    Options op = parser.getOp();
    Index<String> stateIndex = parser.stateIndex;
   
    for (UnaryRule unaryRule : unaryGrammar) {
      // only make one matrix for each parent state, and only use the
      // basic category for that     
      String childState = stateIndex.get(unaryRule.child);
      String childBasic = op.langpack().basicCategory(childState);
     
      unaryRules.add(childBasic);
    }
   
    for (BinaryRule binaryRule : binaryGrammar) {
      // only make one matrix for each parent state, and only use the
      // basic category for that
      String leftState = stateIndex.get(binaryRule.leftChild);
      String leftBasic = op.langpack().basicCategory(leftState);
      String rightState = stateIndex.get(binaryRule.rightChild);
      String rightBasic = op.langpack().basicCategory(rightState);
     
      binaryRules.add(leftBasic, rightBasic);
    }

    if (DEBUG) {
View Full Code Here
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
      }
    }

    String[] newArgs = unusedArgs.toArray(new String[unusedArgs.size()]);
    LexicalizedParser underlyingParser = null;
    Options options = null;
    LexicalizedParser combinedParser = null;
    if (baseModelPaths != null) {
      List<DVModel> dvparsers = new ArrayList<DVModel>();
      for (String baseModelPath : baseModelPaths) {
        System.err.println("Loading serialized DVParser from " + baseModelPath);
        LexicalizedParser dvparser = LexicalizedParser.loadModel(baseModelPath);
        Reranker reranker = dvparser.reranker;
        if (!(reranker instanceof DVModelReranker)) {
          throw new IllegalArgumentException("Expected parsers with DVModel embedded");
        }
        dvparsers.add(((DVModelReranker) reranker).getModel());
        if (underlyingParser == null) {
          underlyingParser = dvparser;
          options = underlyingParser.getOp();
          // TODO: other parser's options?
          options.setOptions(newArgs);
        }
        System.err.println("... done");
      }
      combinedParser = LexicalizedParser.copyLexicalizedParser(underlyingParser);
      CombinedDVModelReranker reranker = new CombinedDVModelReranker(options, dvparsers);
View Full Code Here
80
81
82
83
84
85
86
87
88
89
90
        tb.loadPath(args[i]);
      }
    }

    PrintWriter pw = tlpp.pw();
    Options op = new Options();
    Options.LexOptions lexOptions = op.lexOptions;
    if(lang == Language.French) {
      lexOptions.useUnknownWordSignatures = 1;
      lexOptions.smartMutation = false;
      lexOptions.unknownSuffixSize = 2;
View Full Code Here
80
81
82
83
84
85
86
87
88
89
90
  private transient Function<List<? extends HasWord>, List<TaggedWord>> tagger;
  private transient String taggerPath;

  public Function<List<? extends HasWord>, List<TaggedWord>> loadTagger() {
    Options op = getOp();
    if (op.testOptions.preTag) {
      synchronized(this) { // TODO: rather coarse synchronization
        if (!op.testOptions.taggerSerializedFile.equals(taggerPath)) {
          taggerPath = op.testOptions.taggerSerializedFile;
          tagger = ReflectionLoading.loadByReflection("edu.stanford.nlp.tagger.maxent.MaxentTagger", taggerPath);
View Full Code Here
38
39
40
41
42
43
44
45
    for (String text : TEST_TREES) {
      Tree tree = Tree.valueOf(text);
      treebank.add(tree);
    }

    List<Tree> binarizedTrees = ShiftReduceParser.binarizeTreebank(treebank, new Options());
    return binarizedTrees;
  }
View Full Code Here
59
60
61
62
63
64
65
66
67
68
69
  };

  Debinarizer debinarizer = new Debinarizer(false);

  public void setUp() {
    Options op = new Options();
    Treebank treebank = op.tlpParams.memoryTreebank();
   
    treebank.addAll(Arrays.asList(correctTrees));
    binarizedTrees = ShiftReduceParser.binarizeTreebank(treebank, op);
  }
View Full Code Here
60
61
62
63
64
65
66
67
68
69
70
      assertEquals(tree, state.stack.peek());
    }
  }

  Tree convertTree(String treeText) {
    Options op = new Options();
    HeadFinder binaryHeadFinder = new BinaryHeadFinder(op.tlpParams.headFinder());
    Tree tree = Tree.valueOf(treeText);
    Trees.convertToCoreLabels(tree);
    tree.percolateHeadAnnotations(binaryHeadFinder);
    return tree;
View Full Code Here
142
143
144
145
146
147
148
149
150
151
152
    ChineseTreebankParserParams ctpp = new ChineseTreebankParserParams();
    ctpp.charTags = true;
    // TODO: these options are getting clobbered by reading in the
    // parser object (unless it's a text file parser?)
    Options op = new Options(ctpp);
    op.doDep = false;
    op.testOptions.maxLength = 90;

    LexicalizedParser lp;
    try {
View Full Code Here
TOP

Related Classes of edu.stanford.nlp.parser.lexparser.Options

  • edu.stanford.nlp.parser.common.ParserGrammar
  • edu.stanford.nlp.parser.dvparser.CombineDVModels
  • edu.stanford.nlp.parser.dvparser.FilterConfusingRules
  • edu.stanford.nlp.parser.eval.UNKPrinter
  • edu.stanford.nlp.parser.shiftreduce.OracleTest
  • edu.stanford.nlp.parser.shiftreduce.ReorderingOracleTest
  • edu.stanford.nlp.parser.shiftreduce.ShiftReduceParserTest
  • edu.stanford.nlp.trees.CompositeTreeTransformer
  • edu.stanford.nlp.trees.international.pennchinese.CharacterLevelTagExtender
  • edu.stanford.nlp.trees.TreeTransformer
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.