private int topN=300;

  private boolean useUniqueNbest =true;

  private boolean useTreeNbest = false;

  private boolean addCombinedCost = true; 

 

  private KBestExtractor kbestExtractor;

 

  SymbolTable symbolTbl;

  List<FeatureFunction> featFunctions ; //for HG reranking and kbest extraction

  HashMap<VariationalNgramApproximator, FeatureTemplateBasedFF> approximatorMap;

 

 

  double insideOutsideScalingFactor= 0.5;

  HGRanker ranker;

 

  public VariationalDecoder(){

    //do nothing;

  }

 

 

 

 

  public HyperGraph decoding(HyperGraph hg, int sentenceID, BufferedWriter out) {   

 

    //=== step-1: run inside-outside

    //note, inside and outside will use the transition_cost of each hyperedge, this cost is already linearly interpolated

    TrivialInsideOutside pInsideOutside = new TrivialInsideOutside();

    pInsideOutside.runInsideOutside(hg, 0, 1, insideOutsideScalingFactor);//ADD_MODE=0=sum; LOG_SEMIRING=1;

   

    //=== initialize baseline table

    //TODO:????????????????????

    ((EdgeTblBasedBaselineFF)featFunctions.get(0)).collectTransitionLogPs(hg);

   

    //=== step-2: model extraction based on the definition of Q

    for(Map.Entry<VariationalNgramApproximator, FeatureTemplateBasedFF> entry : approximatorMap.entrySet()){

      VariationalNgramApproximator approximator = entry.getKey();

      FeatureTemplateBasedFF featureFunction = entry.getValue();

      HashMap<String, Double> model = approximator.estimateModel(hg, pInsideOutside);

      featureFunction.setModel(model);     

    }

   

    //clean up

    pInsideOutside.clearState();

   

    //=== step-3: rank the HG using the baseline and variational feature

    this.ranker.rankHG(hg);

    //=== step-4: kbest extraction from the reranked HG: remember to add the new feature function into the model list

    try{

      kbestExtractor.lazyKBestExtractOnHG(hg, this.featFunctions, this.topN, sentenceID, out);

    } catch (IOException e) {

      e.printStackTrace();

    }

    return hg;

  }

 

 

  public void initializeDecoder(String configFile){

    VariationalDecoderConfiguration.readConfigFile(configFile);

    this.symbolTbl = new BuildinSymbol(null);

    this.featFunctions = new ArrayList<FeatureFunction>();

    this.ranker = new HGRanker(featFunctions);

    this.approximatorMap = new HashMap<VariationalNgramApproximator, FeatureTemplateBasedFF>();   

    VariationalDecoderConfiguration.initializeModels(configFile, this.symbolTbl, this.featFunctions, this.approximatorMap);   

    this.insideOutsideScalingFactor =  VariationalDecoderConfiguration.insideoutsideScalingFactor;

    //this.kbestExtractor = new KBestExtractor(p_symbol); 

    this.kbestExtractor = new KBestExtractor(this.symbolTbl, this.useUniqueNbest, this.useTreeNbest, false, this.addCombinedCost, false, true);

  }

 

  public void  decodingTestSet(String testItemsFile, String testRulesFile, int numSents, String nbestFile) {

    BufferedWriter nbestWriter =  FileUtilityOld.getWriteFileStream(nbestFile); 

    System.out.println("############Process file  " + testItemsFile);

    DiskHyperGraph diskHG = new DiskHyperGraph(symbolTbl, VariationalDecoderConfiguration.ngramStateID, true, null); //have model costs stored

    diskHG.initRead(testItemsFile, testRulesFile,null);

     

    for(int sentID=0; sentID < numSents; sentID ++){

      System.out.println("#Process sentence " + sentID);

      HyperGraph testhg = diskHG.readHyperGraph();

      /*if(use_constituent_decoding)

        vdecoder.constitudent_decoding(hg_test, sent_id, t_writer_nbest);

      else*/

        decoding(testhg, sentID, nbestWriter);

     

      //Thread.sleep(3000);

    }

    FileUtilityOld.closeWriteFile(nbestWriter);

  }

 

  public void writeConfigFile(double[] newWeights, String template, String fileToWrite){

    BufferedReader configReader = FileUtilityOld.getReadFileStream(template);

    BufferedWriter configWriter =  FileUtilityOld.getWriteFileStream(fileToWrite);   

    String line;

    int featID = 0;

    while ((line = FileUtilityOld.readLineLzf(configReader)) != null) {

      line = line.trim();

      if (line.matches("^\\s*\\#.*$") || line.matches("^\\s*$") || line.indexOf("=") != -1) {//comment, empty line, or parameter lines: just copy

         FileUtilityOld.writeLzf(configWriter, line + "\n");

      }else{//models: replace the weight

        String[] fds = line.split("\\s+");

        StringBuffer new_line = new StringBuffer();

        if(fds[fds.length-1].matches("^[\\d\\.\\-\\+]+")==false){System.out.println("last field is not a number, must be wrong; the field is: " + fds[fds.length-1]); System.exit(1);};

        for(int i=0; i<fds.length-1; i++){

          new_line.append(fds[i]);

          new_line.append(" ");

        }

        new_line.append(newWeights[featID++]); 

        FileUtilityOld.writeLzf(configWriter, new_line.toString() + "\n");

      }       

    }

    if(featID!=newWeights.length){System.out.println("number of models does not match number of weights, must be wrong"); System.exit(1);};

    FileUtilityOld.closeReadFile(configReader);

    FileUtilityOld.closeWriteFile(configWriter);   

  }

 

  /*this assumes that the weight_vector is ordered according to the decoder config file

   * */

  public void changeFeatureWeightVector(double[] weight_vector){

    if(featFunctions.size()!=weight_vector.length){

      System.out.println("In updateFeatureWeightVector: number of weights does not match number of feature functions");

      System.exit(0);

    }   

    for(int i=0; i<featFunctions.size(); i++){

      FeatureFunction ff = featFunctions.get(i);

      double old_weight = ff.getWeight();

      ff.setWeight(weight_vector[i]);

      System.out.println("Feature function : " + ff.getClass().getSimpleName() + "; weight changed from " + old_weight + " to " + ff.getWeight());

    }

  }

 

 

 

 

  //============================ main function ==============================

  public static void main(String[] args) throws InterruptedException, IOException {

    /*//##read configuration information

    if(args.length<8){

      System.out.println("wrong command, correct command should be: java Perceptron_HG is_crf lf_train_items lf_train_rules lf_orc_items lf_orc_rules f_l_num_sents f_data_sel f_model_out_prefix use_tm_feat use_lm_feat use_edge_bigram_feat_only f_feature_set use_joint_tm_lm_feature");

      System.out.println("num of args is "+ args.length);

      for(int i=0; i <args.length; i++)System.out.println("arg is: " + args[i]);

      System.exit(0);   

    }*/

    if(args.length!=5){

      System.out.println("Wrong number of parameters, it must be  5");

      System.exit(1);

    }

   

    //long start_time = System.currentTimeMillis();   

    String testItemsFile=args[0].trim();

    String testRulesFile=args[1].trim();

    int numSents=new Integer(args[2].trim());

    String nbestFile=args[3].trim();

    String configFile=args[4].trim();

   

    VariationalDecoder vdecoder = new VariationalDecoder();

    vdecoder.initializeDecoder(configFile);

    vdecoder.decodingTestSet(testItemsFile, testRulesFile, numSents, nbestFile);

  }

 

 

 

  /*

  //return changed hg

  public HyperGraph decoding_old(HyperGraph hg, int sentenceID, BufferedWriter out){   

    //### step-1: run inside-outside

    TrivialInsideOutside p_inside_outside = new TrivialInsideOutside();

    p_inside_outside.run_inside_outside(hg, 0, 1);//ADD_MODE=0=sum; LOG_SEMIRING=1;

   

   

    //### step-2: model extraction based on the definition of Q

    

    //step-2.1: baseline feature: do not use insideoutside

    this.baseline_feat_tbl.clear();

    FeatureExtractionHG.feature_extraction_hg(hg, this.baseline_feat_tbl, null, this.baseline_ft_template); 

    

    //step-2.2: ngram feature

    this.ngram_feat_tbl.clear();

    FeatureExtractionHG.feature_extraction_hg(hg, p_inside_outside, this.ngram_feat_tbl, null, this.ngram_ft_template); 

    VariationalLMFeature.getNormalizedLM(this.ngram_feat_tbl, this.baseline_lm_order, false, false, false); 

   

    //step-2.2: constituent feature

    this.constituent_feat_tbl.clear();

    FeatureExtractionHG.feature_extraction_hg(hg, p_inside_outside, this.constituent_feat_tbl, null, this.constituent_template); 

   

    //clean

    p_inside_outside.clear_state();

   

    //### step-3: rank the HG using the baseline and variational feature

    l_feat_functions.clear();   

   

    FeatureFunction base_ff =  new BaselineFF(this.baseline_feat_id, this.baseline_weight, this.baseline_feat_tbl);

    l_feat_functions.add(base_ff);

     

    FeatureFunction variational_ff =  new VariationalLMFeature(this.baseline_lm_feat_id, this.baseline_lm_order, this.p_symbol, this.ngram_feat_tbl,  this.ngam_weight, false, false, false);

    l_feat_functions.add(variational_ff);

   

    FeatureFunction constituent_ff =  new BaselineFF(this.constituent_feat_id, this.constituent_weight, this.constituent_feat_tbl);

    l_feat_functions.add(constituent_ff);

   

    RankHG.rankHG(hg, l_feat_functions);

   

       

    //### step-4: kbest extraction from the reranked HG: remember to add the new feature function into the model list

    //two features: baseline feature, and reranking feature

    KbestExtraction kbestExtractor = new KbestExtraction(p_symbol);

    kbestExtractor.lazy_k_best_extract_hg(hg, this.l_feat_functions, this.topN, this.use_unique_nbest, sentenceID, out, this.use_tree_nbest, this.add_combined_cost);

   

    return hg;

  }

 

 

 

 

//  return changed hg

  public HyperGraph constitudent_decoding(HyperGraph hg, int sentenceID, BufferedWriter out){   

   

    ConstituentVariationalDecoder decoder = new ConstituentVariationalDecoder();

    decoder.decoding(hg);

     

    //### step-2: model extraction based on the definition of Q

    

    //step-2.1: baseline feature: do not use insideoutside

    this.baseline_feat_tbl.clear();

    FeatureExtractionHG.feature_extraction_hg(hg, this.baseline_feat_tbl, null, this.baseline_ft_template); 

   

   

    //### step-3: rank the HG using the baseline and variational feature

    l_feat_functions.clear();   

   

    FeatureFunction base_ff =  new BaselineFF(this.baseline_feat_id, this.baseline_weight, this.baseline_feat_tbl);

    l_feat_functions.add(base_ff);

     

    //RankHG.rankHG(hg, l_feat_functions);

   

   

    //### step-4: kbest extraction from the reranked HG: remember to add the new feature function into the model list

    //two features: baseline feature, and reranking feature

    this.kbestExtractor.lazy_k_best_extract_hg(hg, this.l_feat_functions, this.topN, this.use_unique_nbest, sentenceID, out, this.use_tree_nbest, this.add_combined_cost);

   

    return hg;

  }

*/