Package joshua.corpus.vocab

Examples of joshua.corpus.vocab.SymbolTable

159
160
161
162
163
164
165
166
    this.alignments = alignments;
    this.sourceSuffixArray = sourceSuffixArray;
    this.sampleSize = sampleSize;
    this.models = models;
   
    SymbolTable vocab = sourceSuffixArray.getVocabulary();
    this.nonterminalIDs = new int[]{vocab.addNonterminal(SymbolTable.X1_STRING), vocab.addNonterminal(SymbolTable.X2_STRING)};
  }
View Full Code Here
86
87
88
89
90
91
92
93
94
95
96
        targetSuffixArray, alignments,
        lexProbs, models, sampleSize,
        maxPhraseSpan, maxPhraseLength,
        minNonterminalSpan, maxNonterminalSpan);
   
    SymbolTable vocab = new Vocabulary();
   
    Corpus corpus = suffixArray.getCorpus();
   
    NGramLanguageModel largeLM = new LMGrammarJAVA(
        vocab,
View Full Code Here
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
public abstract class AbstractLexProbs implements LexicalProbabilities {

  /* See Javadoc for LexicalProbabilities#getTargetGivenSourceAlignments(Pattern,Pattern). */
  public AlignmentGrid getTargetGivenSourceAlignments(Pattern targetPattern, Pattern sourcePattern) {
   
    SymbolTable sourceVocab = getSourceVocab();
    SymbolTable targetVocab = getTargetVocab();
   
    StringBuilder alignmentPoints = new StringBuilder();
   
    for (IndexedInt indexedTarget : Lists.eachWithIndex(targetPattern.getWordIDs())) {
      final int targetWord = indexedTarget.getValue();
      final int targetIndex = indexedTarget.getIndex();
     
      if (targetVocab.isNonterminal(targetWord)) {
        //TODO Do something special here
       
      } else {
       
        float max = targetGivenSource(targetWord, null);
View Full Code Here
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
  }
 
  /* See Javadoc for LexicalProbabilities#getSourceGivenTargetAlignments(Pattern,Pattern). */
  public AlignmentGrid getSourceGivenTargetAlignments(Pattern sourcePattern, Pattern targetPattern) {
   
    SymbolTable sourceVocab = getSourceVocab();
    SymbolTable targetVocab = getTargetVocab();
   
    StringBuilder alignmentPoints = new StringBuilder();
   
    for (IndexedInt indexedSource : Lists.eachWithIndex(sourcePattern.getWordIDs())) {     
      int sourceWord = indexedSource.getValue();
      int sourceIndex = indexedSource.getIndex();
     
      if (sourceVocab.isNonterminal(sourceWord)) {
        //TODO Do something special here
       
      } else {
     
        float max = sourceGivenTarget(sourceWord, null);
        Integer bestTargetIndex = null;
       
        for (IndexedInt indexedTarget : Lists.eachWithIndex(targetPattern.getWordIDs())) {       
          int targetWord = indexedTarget.getValue();
          int targetIndex = indexedTarget.getIndex();
         
          if (! targetVocab.isNonterminal(targetWord)) {
            float score = this.sourceGivenTarget(sourceWord, targetWord);
            if (score > max) {
              max = score;
              bestTargetIndex = targetIndex;
            }
View Full Code Here
48
49
50
51
52
53
54
55
56
57
58
    String outVocabFile = args[1].trim();
    
   
    // Load the lm file so that the SRI toolkit will set up the map
    int lmOrder = 1;
    SymbolTable symbolTable = new SrilmSymbol(lmOrder);
    new LMGrammarSRILM((SrilmSymbol)symbolTable, lmOrder, lmFile);
   
   
    // Write the map to a temporary file
    File tmpFile = File.createTempFile("srilm", "out");
View Full Code Here
163
164
165
166
167
168
169
170
171
172
173
      System.out.println("num of args is "+ args.length);
      for(int i=0; i <args.length; i++)System.out.println("arg is: " + args[i]);
      System.exit(0);   
    }
    long start_time = System.currentTimeMillis();
    SymbolTable symbolTbl = new BuildinSymbol(null); 
    boolean is_using_crf =  new Boolean(args[0].trim());
    HGDiscriminativeLearner.usingCRF=is_using_crf;
    String f_l_train_items=args[1].trim();
    String f_l_train_rules=args[2].trim();
    String f_l_orc_items=args[3].trim();
View Full Code Here
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
  }
 
 
  public static void main(String[] args) {
   
    SymbolTable symbolTable = new BuildinSymbol();
    //init symbol
   
    NULL_ALIGN_WRD_SYM_ID = symbolTable.addTerminal(NULL_ALIGN_WRD_SYM);
    NON_TERMINAL_TAG_SYM_ID = symbolTable.addNonterminal(NON_TERMINAL_TAG_SYM);
   
    //read weights files
    eweights_table = readWeightFile("C:\\data_disk\\java_work_space\\SyntaxMT\\phraseExtraction\\lex.f2e.gz");
    fweights_table = readWeightFile("C:\\data_disk\\java_work_space\\SyntaxMT\\phraseExtraction\\lex.e2f.gz");
   
View Full Code Here
34
35
36
37
38
39
40
41
42
43
44
    int num_sents=new Integer(args[2].trim());
    String f_config=args[3].trim();//be careful with the weights
   
    //set up models
    VariationalDecoderConfiguration.readConfigFile(f_config);
    SymbolTable symbolTbl = new BuildinSymbol(null); 
    List<FeatureFunction> featFunctions = new ArrayList<FeatureFunction>();
    HashMap<VariationalNgramApproximator, FeatureTemplateBasedFF> approximatorMap = new HashMap<VariationalNgramApproximator, FeatureTemplateBasedFF> ();
    VariationalDecoderConfiguration.initializeModels(f_config, symbolTbl, featFunctions, approximatorMap);   
    double insideOutsideScalingFactor =  VariationalDecoderConfiguration.insideoutsideScalingFactor;       
   
View Full Code Here
127
128
129
130
131
132
133
134
135
136
137
 
    int baseline_lm_feat_id = 0;//???????
   
    int max_num_words =25;
   
    SymbolTable p_symbol = new BuildinSymbol(null);
    KBestExtractor kbest_extractor = new KBestExtractor(p_symbol, true, false, false, false,  false, true);//????????????
    ApproximateFilterHGByOneString filter = new ApproximateFilterHGByOneString(p_symbol,baseline_lm_feat_id,baseline_lm_order);
    StringSumInHG p_sumer = new StringSumInHG(p_symbol, kbest_extractor, filter);
   
    //#### process test set
View Full Code Here
39
40
41
42
43
44
45
46
47
48
49
  private final MatchedHierarchicalPhrases matchedPhrases;
 
  RootNode(PrefixTree tree, int incomingArcValue) {
    super(tree.parallelCorpus, 1);
    this.tree = tree;
    SymbolTable vocab = tree.vocab;
    this.matchedPhrases = HierarchicalPhrases.emptyList(vocab);
    Suffixes suffixArray = tree.suffixArray;
    if (suffixArray != null) {
      setBounds(0, suffixArray.size()-1);
    }
View Full Code Here
TOP

Related Classes of joshua.corpus.vocab.SymbolTable

  • joshua.corpus.AbstractPhrase
  • joshua.corpus.alignment.AlignmentsTest
  • joshua.corpus.CorpusArray
  • joshua.corpus.lexprob.AbstractLexProbs
  • joshua.corpus.suffix_array.AbstractSuffixArray
  • joshua.corpus.suffix_array.FrequentPhrases
  • joshua.corpus.suffix_array.HierarchicalPhraseTest
  • joshua.corpus.suffix_array.PhrasePairCollocations
  • joshua.decoder.ff.lm.buildin_lm.TrieLM
  • joshua.decoder.ff.lm.srilm.VocabularyWriter
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.