Package joshua.decoder.hypergraph

Examples of joshua.decoder.hypergraph.HyperGraph

160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
      System.out.println("############Process sentence " + sent_id);
      start_time = System.currentTimeMillis();
      sent_id++;
      //if(sent_id>10)break;
     
      HyperGraph hg = dhg_read.readHyperGraph();
      if(hg==null)continue;
      String orc_sent=null;
      double orc_bleu=0;
     
      //System.out.println("read disk hyp: " + (System.currentTimeMillis()-start_time));
      time_on_reading += System.currentTimeMillis()-start_time;
      start_time = System.currentTimeMillis();
     
      if(orc_extract_nbest){
        Object[] res = orc_extractor.oracle_extract_nbest(kbest_extractor, hg, topN, do_ngram_clip_nbest, ref_sent);
        orc_sent = (String) res[0];
        orc_bleu = (Double) res[1];
      }else{       
        HyperGraph hg_oracle = orc_extractor.oracle_extract_hg(hg, hg.sentLen, lm_order, ref_sent);
        orc_sent =  ViterbiExtractor.extractViterbiString(p_symbolTable, hg_oracle.goalNode);
        orc_bleu = orc_extractor.get_best_goal_cost(hg, orc_extractor.g_tbl_split_virtual_items);
       
        time_on_orc_extract += System.currentTimeMillis()-start_time;
        System.out.println("num_virtual_items: " + orc_extractor.g_num_virtual_items + " num_virtual_dts: " + orc_extractor.g_num_virtual_deductions);
View Full Code Here
68
69
70
71
72
73
74
75
76
 
  //get the 1best tree hg, the 1-best is ranked by the split hypergraph, but the return hypergraph is in the form of the original hg 
  public HyperGraph get_1best_tree_hg(HyperGraph original_hg, HashMap<HGNode, ArrayList<VirtualItem>> g_tbl_split_virtual_items){
    VirtualItem virutal_goal_item =  get_virtual_goal_item(original_hg, g_tbl_split_virtual_items);
    HGNode onebest_goal_item = clone_item_with_best_deduction(virutal_goal_item);   
    HyperGraph res = new HyperGraph(onebest_goal_item, -1, -1, original_hg.sentID, original_hg.sentLen);//TODO: number of items/deductions
    get_1best_tree_item(virutal_goal_item, onebest_goal_item);
    return res;
  }
View Full Code Here
252
253
254
255
256
257
258
259
260
261
262
263
264
     
      DiskHyperGraph diskHG = new DiskHyperGraph(symbolTbl, ngramStateID, saveModelCosts, null);
      diskHG.initRead(testNodesFile, testRulesFile, null);
      for(int sent_id=0; sent_id < numSent; sent_id ++){
        System.out.println("#Process sentence " + sent_id);
        HyperGraph testHG = diskHG.readHyperGraph();
        ((TableBasedBaselineFT) baselineFeature).setBaselineScoreTbl( reranker.collectTransitionLogPs(testHG) );
        HyperGraph rerankedOnebestHG = reranker.rerankHGAndGet1best(testHG, modelTbl, restrictedFeatureSet, featTemplates, isAvgModel);
        System.out.println("bestScore=" + rerankedOnebestHG.goalNode.bestHyperedge.bestDerivationLogP );
        String reranked_1best = ViterbiExtractor.extractViterbiString(symbolTbl, rerankedOnebestHG.goalNode);
       
        FileUtilityOld.writeLzf(out1best, reranked_1best + "\n");       
      }
View Full Code Here
196
197
198
199
200
201
202
203
204
205
206
   
    DiskHyperGraph diskHG = new DiskHyperGraph(symbolTbl, ngramStateID, saveModelCosts, null);
    diskHG.initRead(testNodesFile, testRulesFile, null);
    for(int sentID=0; sentID < numSent; sentID ++){
      System.out.println("#Process sentence " + sentID);
      HyperGraph testHG = diskHG.readHyperGraph();
      baselineFeature.collectTransitionLogPs(testHG);
      reranker.rankHG(testHG);
   
      try{
        kbestExtractor.lazyKBestExtractOnHG(testHG, features, topN, sentID, out1best);
View Full Code Here
94
95
96
97
98
99
100
101
102
103
104
      DiskHyperGraph dhg_train = new DiskHyperGraph(p_symbol, ngramStateID, saveModelScore, null);
      dhg_train.initRead((String)testItemsFiles.get(fid), (String)testRulesFiles.get(fid),null);   
      int total_num_sent = new Integer((String)l_num_sents.get(fid));
      for(int sent_id=0; sent_id < total_num_sent; sent_id ++){
        System.out.println("############Process sentence " + sent_id);
        HyperGraph hg_train = dhg_train.readHyperGraph();       
        FeatureExtractionHG.featureExtractionOnHG(hg_train, tbl_feats, null, featureTemplates);
      }
    }
    System.out.println("===feature table size is " + tbl_feats.size());
    //#### write hashtable
View Full Code Here
95
96
97
98
99
100
101
102
103
104
105
      diskHG.initRead(l_file_train_items.get(fid), l_file_train_rules.get(fid),tbl_sent_selected);
       
      int total_num_sent = new Integer((String)l_num_sents.get(fid));
      for(int sent_id=0; sent_id < total_num_sent; sent_id ++){
        System.out.println("#Process sentence " + sentID);
        HyperGraph hg = diskHG.readHyperGraph();
        if(hg!=null)//sent is not skipped
          FeatureExtractionHG.featureExtractionOnHG(hg,featureIntersectionSet, restrictedFeatureSet,  featTemplates);
     
        sentID++;
      }
View Full Code Here
340
341
342
343
344
345
346
      System.exit(1);
    }
   
    if(logger.isLoggable(Level.FINE))
      logger.fine("Finished expand");
    return new HyperGraph(this.goalBin.getSortedNodes().get(0), -1, -1, this.segmentID, foreignSentenceLength);
  }
View Full Code Here
106
107
108
109
110
111
112
113
114
115
116
    DiskHyperGraph diskHG = new DiskHyperGraph(symbolTbl, VariationalDecoderConfiguration.ngramStateID, true, null); //have model costs stored
    diskHG.initRead(testItemsFile, testRulesFile,null);
     
    for(int sentID=0; sentID < numSents; sentID ++){
      System.out.println("#Process sentence " + sentID);
      HyperGraph testhg = diskHG.readHyperGraph();
      /*if(use_constituent_decoding)
        vdecoder.constitudent_decoding(hg_test, sent_id, t_writer_nbest);
      else*/
        decoding(testhg, sentID, nbestWriter);
     
View Full Code Here
74
75
76
77
78
79
80
81
82
83
84
      //inside_outside.sanity_check_hg(hg_full);
      FeatureExtractionHG.featureExtractionOnHG(fullHG, insideOutsider, modelFeatsTbl, restrictedFeatureSet, featTemplates); 
      insideOutsider.clearState();
      //sent_rerank_1best_debug = RescorerHG.rerank_hg_and_get_1best_string(hg_full, p_optimizer.get_sum_model(), g_baseline_scale, g_restricted_feature_set, l_feat_templates_nobaseline, false);
    }else{//perceptron 
      HyperGraph  rerankedOnebest = reranker.rerankHGAndGet1best(fullHG, optimizer.getSumModel(), restrictedFeatureSet, featTemplatesNobaseline, false);
      FeatureExtractionHG.featureExtractionOnHG(rerankedOnebest, modelFeatsTbl, restrictedFeatureSet, featTemplates);
      //sent_rerank_1best_debug = HyperGraph.extract_best_string(hg_reranked_1best.goal_item);
    }
   
    //####feature extraction on emperical, get  g_tbl_feats_emperical
View Full Code Here
304
305
306
307
308
309
310
311
312
313
314
315
        }
         
        int total_num_sent = new Integer((String)l_num_sents.get(fid));
        for(int sent_id=0; sent_id < total_num_sent; sent_id ++){
          System.out.println("#Process sentence " + g_sent_id);
          HyperGraph hg_train = dhg_train.readHyperGraph();
          HyperGraph hg_orc =null;
          String hyp_oracle =null;
          if(l_file_orc_rules!=null)
            hg_orc = dhg_orc.readHyperGraph();
          else
            hyp_oracle = FileUtilityOld.readLineLzf(t_reader_orc);
View Full Code Here
TOP

Related Classes of joshua.decoder.hypergraph.HyperGraph

  • joshua.decoder.chart_parser.Chart
  • joshua.decoder.DecoderThread
  • joshua.discriminative.feature_related.FeatureIntersection
  • joshua.discriminative.feature_related.FeatureSelectionHG
  • joshua.discriminative.monolingual_parser.MonolingualDecoderThread
  • joshua.discriminative.ranker.HGRanker
  • joshua.discriminative.ranker.RescorerHGSimple
  • joshua.discriminative.semiring_parsing.EntropyOnHG
  • joshua.discriminative.semiring_parsing.HypLenExpectation
  • joshua.discriminative.semiring_parsing.HypLenSquareExpectation
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.