Examples of joshua.decoder.hypergraph.KBestExtractor$DerivationState

joshua.decoder.hypergraph.KBestExtractor
This class implements lazy k-best extraction on a hyper-graph. To seed the kbest extraction, it only needs that each hyperedge should have the best_cost properly set, and it does not require any list being sorted. Instead, the priority queue heap_cands will do internal sorting In fact, the real crucial cost is the transition-cost at each hyperedge. We store the best-cost instead of the transition cost since it is easy to do pruning and find one-best. Moreover, the transition cost can be recovered by get_transition_cost(), though somewhat expensive. To recover the model cost for each individual model, we should either have access to the model, or store the model cost in the hyperedge. (For example, in the case of disk-hypergraph, we need to store all these model cost at each hyperedge.) @author Zhifei Li, @version $LastChangedDate: 2010-02-08 13:03:13 -0600 (Mon, 08 Feb 2010) $

    int baseline_lm_feat_id = 0; 
    //??????????????????????????????????????
    
    SymbolTable p_symbolTable = new BuildinSymbol(null);
    
    KBestExtractor kbest_extractor = null;
    int topN = 300;//TODO
    boolean extract_unique_nbest = true;//TODO
    boolean do_ngram_clip_nbest = true; //TODO
    if (orc_extract_nbest) {
      System.out.println("oracle extraction from nbest list");
      kbest_extractor = new KBestExtractor(p_symbolTable, extract_unique_nbest, false, false, false,  false, true);
    }
    
    BufferedWriter orc_out = FileUtility.getWriteFileStream(f_orc_out);
    
    long start_time0 = System.currentTimeMillis();

View Full Code Here

          
    int topN=3;
    boolean useUniqueNbest =true;
    boolean useTreeNbest = false;
    boolean addCombinedCost = true;  
    KBestExtractor kbestExtractor = new KBestExtractor(symbolTbl, useUniqueNbest, useTreeNbest, false, addCombinedCost, false, true);
    
    
    DiskHyperGraph diskHG = new DiskHyperGraph(symbolTbl, ngramStateID, saveModelCosts, null); 
    diskHG.initRead(testNodesFile, testRulesFile, null);
    for(int sentID=0; sentID < numSent; sentID ++){
      System.out.println("#Process sentence " + sentID);
      HyperGraph testHG = diskHG.readHyperGraph();
      baselineFeature.collectTransitionLogPs(testHG);
      reranker.rankHG(testHG);
    
      try{
        kbestExtractor.lazyKBestExtractOnHG(testHG, features, topN, sentID, out1best);
      } catch (IOException e) {
        e.printStackTrace();
      }
              
    }

View Full Code Here

    this.approximatorMap = new HashMap<VariationalNgramApproximator, FeatureTemplateBasedFF>();    
    VariationalDecoderConfiguration.initializeModels(configFile, this.symbolTbl, this.featFunctions, this.approximatorMap);    
    this.insideOutsideScalingFactor =  VariationalDecoderConfiguration.insideoutsideScalingFactor;


    //this.kbestExtractor = new KBestExtractor(p_symbol);  
    this.kbestExtractor = new KBestExtractor(this.symbolTbl, this.useUniqueNbest, this.useTreeNbest, false, this.addCombinedCost, false, true);
  }

View Full Code Here

    int baseline_lm_feat_id = 0;//???????
    
    int max_num_words =25;
    
    SymbolTable p_symbol = new BuildinSymbol(null);
    KBestExtractor kbest_extractor = new KBestExtractor(p_symbol, true, false, false, false,  false, true);//????????????
    ApproximateFilterHGByOneString filter = new ApproximateFilterHGByOneString(p_symbol,baseline_lm_feat_id,baseline_lm_order);
    StringSumInHG p_sumer = new StringSumInHG(p_symbol, kbest_extractor, filter);
    
    //#### process test set
    BufferedWriter t_writer_nbest =  FileUtilityOld.getWriteFileStream(f_nbest);  
    BufferedWriter t_writer_1best =  FileUtilityOld.getWriteFileStream(f_1best);
    System.out.println("############Process file  " + f_test_items);
    DiskHyperGraph dhg_test = new DiskHyperGraph(p_symbol, baseline_lm_feat_id, true, null); //have model costs stored
    dhg_test.initRead(f_test_items, f_test_rules,null);
      
    for(int sent_id=0; sent_id < num_sents; sent_id ++){
      System.out.println("#Process sentence " + sent_id);
      HyperGraph hg_test = dhg_test.readHyperGraph();      
      //if(sent_id==1)System.exit(1);
      //generate a unique nbest of strings based on viterbi cost
      ArrayList<String> nonUniqueNbestStrings = new ArrayList<String>();
      kbest_extractor.lazyKBestExtractOnHG(hg_test, null, topN, sent_id, nonUniqueNbestStrings);
      
      double max_prob = Double.NEGATIVE_INFINITY;
      String max_string = "";
      
      //chech if the sentence is too long

View Full Code Here

    this.testFile        = testFile;
    this.nbestFile       = nbestFile;
    this.oracleFile      = oracleFile;
    this.startSentenceID = startSentenceID;
    
    this.kbestExtractor = new KBestExtractor(
      this.symbolTable,
      JoshuaConfiguration.use_unique_nbest,
      JoshuaConfiguration.use_tree_nbest,
      JoshuaConfiguration.include_align_index,
      JoshuaConfiguration.add_combined_cost,

View Full Code Here

    int baseline_lm_feat_id = 0; 
    //??????????????????????????????????????
    
    SymbolTable p_symbolTable = new BuildinSymbol(null);
    
    KBestExtractor kbestExtractor =null;
    int topN=300;//TODO
    boolean extract_unique_nbest = true;//TODO
    boolean do_ngram_clip_nbest = true; //TODO
    if(orc_extract_nbest==true){
      System.out.println("oracle extraction from nbest list");
      kbestExtractor = new KBestExtractor(p_symbolTable, extract_unique_nbest, false, false, false,  false, true);
    }
    
    BufferedWriter orc_out = FileUtility.getWriteFileStream(f_orc_out); 
    boolean rerankKbestOracles = true;
    BufferedWriter rerankOrcOut=null;
    if(rerankKbestOracles==true){
      rerankOrcOut = FileUtility.getWriteFileStream(f_orc_out+".rerank");
    }
    
    long start_time0 = System.currentTimeMillis();
    long time_on_reading = 0;
    long time_on_orc_extract = 0;
    BufferedReader t_reader_ref = FileUtility.getReadFileStream(f_ref_files);
    
    DiskHyperGraph dhg_read  = new DiskHyperGraph(p_symbolTable, baseline_lm_feat_id, saveModelScores, null);
    
  
    dhg_read.initRead(f_hypergraphs, f_rule_tbl, null);
    
    KBestExtractor oracleKbestExtractor = new KBestExtractor(p_symbolTable, extract_unique_nbest, false, false, true,  false, true);//extract kbest oracles
    KBestExtractor rerankOracleKbestExtractor = new KBestExtractor(p_symbolTable, extract_unique_nbest, false, false, false,  false, true);//extract kbest oracles
    int topKOracles= 500;//TODO
    //OracleExtractionOnHGV2 orc_extractor = new OracleExtractionOnHGV2(p_symbolTable, baseline_lm_feat_id);
    OracleExtractionOnHGV3 orc_extractor = new OracleExtractionOnHGV3(p_symbolTable);
    String ref_sent= null;
    int sent_id=0;

View Full Code Here

    HGRanker reranker = new HGRanker(features);  
    
    
    //==== kbest 
    boolean addCombinedCost = false;  
    KBestExtractor kbestExtractor = new KBestExtractor(symbolTbl, MRConfig.use_unique_nbest, MRConfig.use_tree_nbest, false, addCombinedCost, false, true);
    
    //==== loop
    HyperGraphFactory hgFactory = new HyperGraphFactory(curHypFilePrefix, referenceFiles, MRConfig.ngramStateID,  symbolTbl, true);
    hgFactory.startLoop();
    for(int sentID=0; sentID< this.numTrainingSentence; sentID ++){      
      HGAndReferences res = hgFactory.nextHG();
      reranker.rankHG(res.hg);//reset best pointer and transition prob
    
      String hypSent = kbestExtractor.getKthHyp(res.hg.goalNode, 1, -1, null, null);
      double bleu = BLEU.computeSentenceBleu(res.referenceSentences, hypSent);
      bleuSum  += bleu;
      
      double googleGain = BLEU.computeLinearCorpusGain(MRConfig.linearCorpusGainThetas, res.referenceSentences, hypSent);
      googleGainSum += googleGain;

View Full Code Here

    this.nbestFile      = nbest_file_in;


    t_writer_nbest =  FileUtility.getWriteFileStream(nbestFile);
    
    
    this.kbestExtractor = new KBestExtractor(this.symbolTable, JoshuaConfiguration.use_unique_nbest, JoshuaConfiguration.use_tree_nbest, 
        JoshuaConfiguration.include_align_index, JoshuaConfiguration.add_combined_cost,  true, true);
    
  }

View Full Code Here

  public NbestCrunching(SymbolTable symbolTbl, double insideOutsideScalingFactor, int topN){
    this.scalingFactor = insideOutsideScalingFactor;
    this.topN = topN;
    
    this.symbolTbl = symbolTbl;
    this.kbestExtractor = new KBestExtractor(this.symbolTbl, this.useUniqueNbest, this.useTreeNbest, false, this.addCombinedCost,  false, true);
  }

View Full Code Here

  public EstimateEntropyGapOnNbest(SymbolTable symbol_, double scalingFactor, int topn){
    this.scalingFactor = scalingFactor;
    this.topN = topn;
    
    this.symbolTbl = symbol_;
    this.kbestExtractor = new KBestExtractor(this.symbolTbl, this.useUniqueNbest, this.useTreeNbest, false, this.addCombinedCost,  false, true);
  }

View Full Code Here

TOP

Related Classes of joshua.decoder.hypergraph.KBestExtractor$DerivationState

joshua.decoder.DecoderThread

joshua.decoder.ff.tm.Rule

joshua.discriminative.monolingual_parser.NbestDecoderThread

joshua.discriminative.ranker.HGRanker

joshua.discriminative.training.oracle.OracleExtractionOnHGV2

joshua.discriminative.training.oracle.StringSumInHG

joshua.discriminative.training.risk_annealer.hypergraph.HGMinRiskDAMert

joshua.discriminative.variational_decoder.nbest.EstimateEntropyGapOnNbest

joshua.discriminative.variational_decoder.nbest.NbestCrunching

joshua.discriminative.variational_decoder.VariationalDecoder

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.