Package joshua.decoder.hypergraph

Examples of joshua.decoder.hypergraph.DiskHyperGraph

146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
   
    long start_time0 = System.currentTimeMillis();
    long time_on_reading = 0;
    long time_on_orc_extract = 0;
    BufferedReader t_reader_ref = FileUtility.getReadFileStream(f_ref_files);
    DiskHyperGraph dhg_read  = new DiskHyperGraph(p_symbolTable, baseline_lm_feat_id, true, null);
 
    dhg_read.initRead(f_hypergraphs, f_rule_tbl, null);
   
    OracleExtractionHG orc_extractor = new OracleExtractionHG(p_symbolTable, baseline_lm_feat_id);
    String ref_sent= null;
    long start_time = System.currentTimeMillis();
    int sent_id=0;
    while( (ref_sent=FileUtility.read_line_lzf(t_reader_ref))!= null ){
      System.out.println("############Process sentence " + sent_id);
      start_time = System.currentTimeMillis();
      sent_id++;
      //if(sent_id>10)break;
     
      HyperGraph hg = dhg_read.readHyperGraph();
      if(hg==null)continue;
      String orc_sent=null;
      double orc_bleu=0;
     
      //System.out.println("read disk hyp: " + (System.currentTimeMillis()-start_time));
View Full Code Here
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
     
      BufferedWriter out1best = FileUtilityOld.getWriteFileStream(reranked1bestFile);
     
      RescorerHGSimple reranker = new RescorerHGSimple();
     
      DiskHyperGraph diskHG = new DiskHyperGraph(symbolTbl, ngramStateID, saveModelCosts, null);
      diskHG.initRead(testNodesFile, testRulesFile, null);
      for(int sent_id=0; sent_id < numSent; sent_id ++){
        System.out.println("#Process sentence " + sent_id);
        HyperGraph testHG = diskHG.readHyperGraph();
        ((TableBasedBaselineFT) baselineFeature).setBaselineScoreTbl( reranker.collectTransitionLogPs(testHG) );
        HyperGraph rerankedOnebestHG = reranker.rerankHGAndGet1best(testHG, modelTbl, restrictedFeatureSet, featTemplates, isAvgModel);
        System.out.println("bestScore=" + rerankedOnebestHG.goalNode.bestHyperedge.bestDerivationLogP );
        String reranked_1best = ViterbiExtractor.extractViterbiString(symbolTbl, rerankedOnebestHG.goalNode);
       
View Full Code Here
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
    boolean useTreeNbest = false;
    boolean addCombinedCost = true; 
    KBestExtractor kbestExtractor = new KBestExtractor(symbolTbl, useUniqueNbest, useTreeNbest, false, addCombinedCost, false, true);
   
   
    DiskHyperGraph diskHG = new DiskHyperGraph(symbolTbl, ngramStateID, saveModelCosts, null);
    diskHG.initRead(testNodesFile, testRulesFile, null);
    for(int sentID=0; sentID < numSent; sentID ++){
      System.out.println("#Process sentence " + sentID);
      HyperGraph testHG = diskHG.readHyperGraph();
      baselineFeature.collectTransitionLogPs(testHG);
      reranker.rankHG(testHG);
   
      try{
        kbestExtractor.lazyKBestExtractOnHG(testHG, features, topN, sentID, out1best);
View Full Code Here
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
    //#### extract feat tbl
    HashMap<String, Double> tbl_feats = new HashMap<String, Double>();
       
    for(int fid=0; fid < testItemsFiles.size(); fid++){
      System.out.println("############Process file id " + fid);
      DiskHyperGraph dhg_train = new DiskHyperGraph(p_symbol, ngramStateID, saveModelScore, null);
      dhg_train.initRead((String)testItemsFiles.get(fid), (String)testRulesFiles.get(fid),null);   
      int total_num_sent = new Integer((String)l_num_sents.get(fid));
      for(int sent_id=0; sent_id < total_num_sent; sent_id ++){
        System.out.println("############Process sentence " + sent_id);
        HyperGraph hg_train = dhg_train.readHyperGraph();       
        FeatureExtractionHG.featureExtractionOnHG(hg_train, tbl_feats, null, featureTemplates);
      }
    }
    System.out.println("===feature table size is " + tbl_feats.size());
    //#### write hashtable
View Full Code Here
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
     
    //#####begin to do training
    int sentID=0;   
    for(int fid=0; fid < l_file_train_items.size(); fid++){
      System.out.println("############Process file id " + fid);
      DiskHyperGraph diskHG = new DiskHyperGraph(p_symbol, ngramStateID, saveModelCosts, null);
      diskHG.initRead(l_file_train_items.get(fid), l_file_train_rules.get(fid),tbl_sent_selected);
       
      int total_num_sent = new Integer((String)l_num_sents.get(fid));
      for(int sent_id=0; sent_id < total_num_sent; sent_id ++){
        System.out.println("#Process sentence " + sentID);
        HyperGraph hg = diskHG.readHyperGraph();
        if(hg!=null)//sent is not skipped
          FeatureExtractionHG.featureExtractionOnHG(hg,featureIntersectionSet, restrictedFeatureSet,  featTemplates);
     
        sentID++;
      }
View Full Code Here
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
 
  public void  decodingTestSet(String testItemsFile, String testRulesFile, int numSents, String nbestFile) {

    BufferedWriter nbestWriter =  FileUtilityOld.getWriteFileStream(nbestFile); 
    System.out.println("############Process file  " + testItemsFile);
    DiskHyperGraph diskHG = new DiskHyperGraph(symbolTbl, VariationalDecoderConfiguration.ngramStateID, true, null); //have model costs stored
    diskHG.initRead(testItemsFile, testRulesFile,null);
     
    for(int sentID=0; sentID < numSents; sentID ++){
      System.out.println("#Process sentence " + sentID);
      HyperGraph testhg = diskHG.readHyperGraph();
      /*if(use_constituent_decoding)
        vdecoder.constitudent_decoding(hg_test, sent_id, t_writer_nbest);
      else*/
        decoding(testhg, sentID, nbestWriter);
     
View Full Code Here
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
    int g_sent_id=0;
    for(int loop_id=0; loop_id<max_loop; loop_id++){
      System.out.println("###################################Loop " + loop_id);
      for(int fid=0; fid < l_file_train_items.size(); fid++){
        System.out.println("############Process file id " + fid);
        DiskHyperGraph dhg_train = new DiskHyperGraph(symbolTbl, ngramStateID, saveModelCosts, null);
        dhg_train.initRead((String)l_file_train_items.get(fid), (String)l_file_train_rules.get(fid),tbl_sent_selected);
        DiskHyperGraph dhg_orc =null;
        BufferedReader t_reader_orc =null;
        if(l_file_orc_rules!=null){
          dhg_orc = new DiskHyperGraph(symbolTbl, ngramStateID, saveModelCosts, null);
          dhg_orc.initRead((String)l_file_orc_items.get(fid), (String)l_file_orc_rules.get(fid), tbl_sent_selected);
        }else{
          t_reader_orc = FileUtilityOld.getReadFileStream((String)l_file_orc_items.get(fid),"UTF-8");
        }
         
        int total_num_sent = new Integer((String)l_num_sents.get(fid));
        for(int sent_id=0; sent_id < total_num_sent; sent_id ++){
          System.out.println("#Process sentence " + g_sent_id);
          HyperGraph hg_train = dhg_train.readHyperGraph();
          HyperGraph hg_orc =null;
          String hyp_oracle =null;
          if(l_file_orc_rules!=null)
            hg_orc = dhg_orc.readHyperGraph();
          else
            hyp_oracle = FileUtilityOld.readLineLzf(t_reader_orc);
          if(hg_train!=null){//sent is not skipped
            if(l_file_orc_rules!=null)
              hgdl.processOneSent( hg_train, hg_orc, null, l_feat_templates, l_feat_templates_nobaseline);
View Full Code Here
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
    int ngramStateID = 0;
    DefaultSemiringParser parserEntropyP = new CrossEntropyOnHG(1, 0, scale, pFeatFunctions, pFeatFunctions);
    DefaultSemiringParser parserEntropyQ = new CrossEntropyOnHG(1, 0, scale, qFeatFunctions, qFeatFunctions);
    DefaultSemiringParser parserCrossentropyPQ = new CrossEntropyOnHG(1, 0, scale, pFeatFunctions, qFeatFunctions);
   
    DiskHyperGraph diskHG = new DiskHyperGraph(symbolTbl, ngramStateID, true, null); //have model costs stored
    diskHG.initRead(testItemsFile, testRulesFile, null);   
    for(int sent_id=0; sent_id < num_sents; sent_id ++){
      System.out.println("#Process sentence " + sent_id);
      HyperGraph testHG = diskHG.readHyperGraph();
     
      //################setup the model: including estimation of variational model
      //### step-1: run inside-outside
      //note, inside and outside will use the transition_cost of each hyperedge, this cost is already linearly interpolated
      TrivialInsideOutside insideOutsider = new TrivialInsideOutside();
View Full Code Here
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
   
    //#### process test set
    BufferedWriter t_writer_nbest =  FileUtilityOld.getWriteFileStream(f_nbest); 
    BufferedWriter t_writer_1best =  FileUtilityOld.getWriteFileStream(f_1best);
    System.out.println("############Process file  " + f_test_items);
    DiskHyperGraph dhg_test = new DiskHyperGraph(p_symbol, baseline_lm_feat_id, true, null); //have model costs stored
    dhg_test.initRead(f_test_items, f_test_rules,null);
     
    for(int sent_id=0; sent_id < num_sents; sent_id ++){
      System.out.println("#Process sentence " + sent_id);
      HyperGraph hg_test = dhg_test.readHyperGraph();     
      //if(sent_id==1)System.exit(1);
      //generate a unique nbest of strings based on viterbi cost
      ArrayList<String> nonUniqueNbestStrings = new ArrayList<String>();
      kbest_extractor.lazyKBestExtractOnHG(hg_test, null, topN, sent_id, nonUniqueNbestStrings);
     
View Full Code Here
138
139
140
141
142
143
144
145
146
147
148
   }
  
   private void initDiskReading(){
    logger.info("initialize reading hypergraphss..............");
    
     diskHG = new DiskHyperGraph(symbolTbl, baselineLMFeatID, true, null); //have model costs stored
         diskHG.initRead(diskHGFilePrefix+".hg.items", diskHGFilePrefix+".hg.rules",null);
        
         //=== references files, they are needed only when we want annote the hypergraph with risk           
         refFileReaders = new BufferedReader[referenceFiles.length];
     for(int i=0; i<referenceFiles.length; i++)
View Full Code Here
TOP

Related Classes of joshua.decoder.hypergraph.DiskHyperGraph

  • joshua.corpus.vocab.BuildinSymbol
  • joshua.corpus.vocab.SymbolTable
  • joshua.decoder.DecoderThread
  • joshua.decoder.ff.state_maintenance.NgramDPState
  • joshua.decoder.ff.tm.hiero.DiskHyperGraphFormatReader
  • joshua.decoder.ff.tm.Rule
  • joshua.discriminative.feature_related.FeatureIntersection
  • joshua.discriminative.feature_related.FeatureSelectionHG
  • joshua.discriminative.ranker.HGRanker
  • joshua.discriminative.ranker.RescorerHGSimple
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.