Package joshua.corpus.vocab

Examples of joshua.corpus.vocab.SymbolTable

54
55
56
57
58
59
60
61
62
63
64
   
    String f_dev_hg_prefix=args[0].trim();
    String f_dev_items = f_dev_hg_prefix +".items";
    String f_dev_rules = f_dev_hg_prefix +".rules";
   
    SymbolTable p_symbol = new BuildinSymbol(null);
    int baseline_lm_feat_id =0;
    int num_sents =5;
    double scale=1.0;
    DefaultSemiringParser ds = new HypLenExpectation(1,0,scale);
    DiskHyperGraph dhg_test = new DiskHyperGraph(p_symbol, baseline_lm_feat_id, true, null); //have model costs stored
View Full Code Here
80
81
82
83
84
85
86
87
88
89
90
//  ======================== main method ================================     
   public static void main(String[] args)   throws IOException{ 
       if(args.length<2){
         System.out.println("Wrong command, it should be: java ConfusionExtractor  f_input_grammar f_confusion_grammar");
       }
      SymbolTable symbolTbl = new BuildinSymbol(null);
      String fInputGrammar = args[0];
      String fConfusiongrammar = args[1];
     
      ArrayList<Double> featureWeights = new ArrayList<Double>();
      for(int i=2; i<args.length; i++){
View Full Code Here
488
489
490
491
492
493
494
495
496
497
498
//======================== main method ================================     
   public static void main(String[] args)   throws IOException{ 
       if(args.length<3){
         System.out.println("Wrong command, it should be: java ConfusionExtractor f_hypergraphs_items f_hypergraphs_grammar f_confusion_grammar total_num_sent");
       }
      SymbolTable p_symbol = new BuildinSymbol(null);
      int baseline_lm_feat_id=0;//TODO
      boolean saveModelCosts = true;
      boolean itemSpecific=false;
     
      String f_hypergraphs = args[0];
View Full Code Here
695
696
697
698
699
700
701
702
703
704
705
       
        int[] patternWords = pattern.getWordIDs();
       
        // 6: Q_alphaX <-- Q_alpha
        {
          SymbolTable vocab = (suffixArray==null) ? null : suffixArray.getVocabulary();
          Pattern xpattern = new Pattern(vocab, patternWords, X);
         
//          HierarchicalPhrases phrasesWithFinalX = new HierarchicalPhrases(xpattern, node.sourceHierarchicalPhrases);
          MatchedHierarchicalPhrases phrasesWithFinalX;
          if (suffixArray==null) {
View Full Code Here
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
   * Sets the symbol table to the provided object, and changes
   * migrates all internal data to use the new mappings
   * provided by that object.
   */
  public void setSymbolTable(ExternalizableSymbolTable vocab) {
    SymbolTable oldVocab = this.symbolTable;
   
    for (int i=0; i<corpus.length; i++) {
     
      int oldID = corpus[i];
      String word = oldVocab.getWord(oldID);
      int newID = vocab.getID(word);
     
      corpus[i] = newID;
    }
   
View Full Code Here
94
95
96
97
98
99
100
101
102
103
104
    int numSrcWords =1;
    if(args.length>=4)
      numSrcWords = new Integer(args[3].trim());
   
   
    SymbolTable symbolTbl = new BuildinSymbol(null);
    int ngramStateID =0;
   
    double sumEntropy = 0;
   
    EntropyOnHGUsingIO ds = new EntropyOnHGUsingIO(1.0);
View Full Code Here
68
69
70
71
72
73
74
75
76
77
78
    int numSrcWords =1;
    if(args.length>=4)
      numSrcWords = new Integer(args[3].trim());
   
   
    SymbolTable p_symbol = new BuildinSymbol(null);
    int ngramStateID =0;
   
    double sumEntropy = 0;
   
    DefaultInsideSemiringParser ds = new EntropyOnHG(1.0);
View Full Code Here
36
37
38
39
40
41
42
43
44
45
46
  private final HierarchicalPhrases matchedPhrases;
 
  XNode(RootNode parent) {
    super(parent);
   
    SymbolTable vocab = (parent.parallelCorpus==null || parent.parallelCorpus.getSourceCorpus()==null) ? null : parent.parallelCorpus.getSourceCorpus().getVocabulary();
   
    this.matchedPhrases =
      HierarchicalPhrases.emptyList(
          vocab,
          SymbolTable.X);
View Full Code Here
62
63
64
65
66
67
68
69
70
71
72
    int numSrcWords =1;
    if(args.length>=4)
      numSrcWords = new Integer(args[3].trim());
   
   
    SymbolTable symbolTbl = new BuildinSymbol(null);
    int ngramStateID =0;
   
    double sumEntropy = 0;
   
    DefaultSemiringParser ds = new EntropyOnHG(1,0,scale);
View Full Code Here
36
37
38
39
40
41
42
43
44
 
  MatchedHierarchicalPhrases matchedPhrases;
 
  BotNode(ParallelCorpusGrammarFactory parallelCorpus, RootNode root) {
    super(parallelCorpus, PrefixTree.BOT_NODE_ID);
    SymbolTable symbolTable = (parallelCorpus==null) ? null : parallelCorpus.getSuffixArray().getVocabulary();
    this.matchedPhrases = HierarchicalPhrases.emptyList(symbolTable);
    this.children = new BotMap<Integer,Node>(root);
  }
View Full Code Here
TOP

Related Classes of joshua.corpus.vocab.SymbolTable

  • joshua.corpus.AbstractPhrase
  • joshua.corpus.alignment.AlignmentsTest
  • joshua.corpus.CorpusArray
  • joshua.corpus.lexprob.AbstractLexProbs
  • joshua.corpus.suffix_array.AbstractSuffixArray
  • joshua.corpus.suffix_array.FrequentPhrases
  • joshua.corpus.suffix_array.HierarchicalPhraseTest
  • joshua.corpus.suffix_array.PhrasePairCollocations
  • joshua.decoder.ff.lm.buildin_lm.TrieLM
  • joshua.decoder.ff.lm.srilm.VocabularyWriter
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.