Package joshua.decoder.chart_parser

Source Code of joshua.decoder.chart_parser.Chart

/* This file is part of the Joshua Machine Translation System.
*
* Joshua is free software; you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as
* published by the Free Software Foundation; either version 2.1
* of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free
* Software Foundation, Inc., 59 Temple Place, Suite 330, Boston,
* MA 02111-1307 USA
*/
package joshua.decoder.chart_parser;

import java.util.ArrayList;
import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;

import joshua.corpus.vocab.SymbolTable;
import joshua.decoder.JoshuaConfiguration;
import joshua.decoder.chart_parser.DotChart.DotNode;
import joshua.decoder.ff.FeatureFunction;
import joshua.decoder.ff.state_maintenance.StateComputer;
import joshua.decoder.ff.tm.Grammar;
import joshua.decoder.ff.tm.Rule;
import joshua.decoder.ff.tm.RuleCollection;
import joshua.decoder.ff.tm.Trie;
import joshua.decoder.hypergraph.HGNode;
import joshua.decoder.hypergraph.HyperGraph;
import joshua.decoder.segment_file.ConstraintSpan;
import joshua.lattice.Arc;
import joshua.lattice.Lattice;
import joshua.lattice.Node;


/**
* Chart class this class implements chart-parsing:
* (1) seeding the chart
* (2) cky main loop over bins,
* (3) identify applicable rules in each bin
*
* Note: the combination operation will be done in Cell
*
* Signatures of class:
* Cell: i, j
* SuperNode (used for CKY check): i,j, lhs
* HGNode ("or" node): i,j, lhs, edge ngrams
* HyperEdge ("and" node)
*
* index of sentences: start from zero
* index of cell: cell (i,j) represent span of words indexed [i,j-1]
* where i is in [0,n-1] and j is in [1,n]
*
* @author Zhifei Li, <zhifei.work@gmail.com>
* @version $LastChangedDate: 2010-02-03 14:58:06 -0600 (Wed, 03 Feb 2010) $
*/

public class Chart {
   

  //===========================================================
  // Satistics
  //===========================================================
 
  /**
   * how many items have been pruned away because its cost
   * is greater than the cutoff in calling
   * chart.add_deduction_in_chart()
   */
  int nPreprunedEdges           = 0;
 
  int nPreprunedFuzz1     = 0;
  int nPreprunedFuzz2     = 0;
  int nPrunedItems              = 0;
  int nMerged              = 0;
  int nAdded               = 0;
  int nDotitemAdded       = 0; // note: there is no pruning in dot-item
  int nCalledComputeNode = 0;
 
  int              segmentID;
 
//===============================================================
// Private instance fields (maybe could be protected instead)
//===============================================================
  private Cell[][] cells; // note that in some cell, it might be null
  private int foreignSentenceLength;
  private List<FeatureFunction> featureFunctions; 
  private List<StateComputer> stateComputers; 
  private  Grammar[]       grammars;
  private  DotChart[]      dotcharts; // each grammar should have a dotchart associated with it
  private Cell              goalBin;
  private int              goalSymbolID = -1;
  private Lattice<Integer> sentence; // a list of foreign words
 
 
   
  private Combiner combiner = null;
  private ManualConstraintsHandler manualConstraintsHandler;
 
  //===========================================================
  // Decoder-wide fields
  //===========================================================
 
  /**
   * Shared symbol table for source language terminals, target
   * language terminals, and shared nonterminals.
   * <p>
   * It may be that separate tables should be maintained for
   * the source and target languages.
   * <p>
   * This class adds an untranslated word ID to the symbol
   * table. The Bin class adds a goal symbol nonterminal to
   * the symbol table.
   * <p>
   */
  private SymbolTable symbolTable;
 
 
//===============================================================
// Static fields
//===============================================================
 
  //===========================================================
  // Time-profiling variables for debugging
  //===========================================================
  // These are only referenced in a commented out logger. They are never set.
  //private static long g_time_lm                = 0;
  //private static long g_time_score_sent        = 0;
  //private static long g_time_check_nonterminal = 0;
 
 
  //===========================================================
  // Logger
  //===========================================================
  private static final Logger logger =
    Logger.getLogger(Chart.class.getName());
 
 
 
 
//===============================================================
// Constructors
//===============================================================
 
  /**TODO: Once the Segment interface is adjusted to provide a Latice<String> for the sentence() method,
   * we should just accept a Segment instead of the sentence, segmentID, and constraintSpans parameters.
   * We have the symbol table already, so we can do the integerization here instead of in DecoderThread.
   * GrammarFactory.getGrammarForSentence will want the integerized sentence as well,
   * but then we'll need to adjust that interface to deal with (non-trivial) lattices too. Of course,
   * we get passed the grammars too so we could move all of that into here.
   */
 
  public Chart(
    Lattice<Integer>           sentence,
    List<FeatureFunction> featureFunctions,
    List<StateComputer> stateComputers,
    SymbolTable                symbolTable,
    int                        segmentID,
    Grammar[]                  grammars,
    boolean                    hasLM,
    String                     goalSymbol,
    List<ConstraintSpan>       constraintSpans
    )
  {
    this.sentence         = sentence;
    this.foreignSentenceLength   = sentence.size() - 1;
    this.featureFunctions = featureFunctions;
    this.stateComputers = stateComputers;
    this.symbolTable      = symbolTable;
   
    // TODO: this is very memory-expensive
    this.cells         = new Cell[foreignSentenceLength][foreignSentenceLength+1];
   
    this.segmentID    = segmentID;
    this.goalSymbolID = this.symbolTable.addNonterminal(goalSymbol);
    this.goalBin      = new Cell(this, this.goalSymbolID);
    this.grammars = grammars;
   
    // each grammar will have a dot chart
    this.dotcharts = new DotChart[this.grammars.length];
    for (int i = 0; i < this.grammars.length; i++)
      this.dotcharts[i] = new DotChart(this.sentence, this.grammars[i], this);
   
   
    if(JoshuaConfiguration.useCubePrune)//TODO: should not directly refer to JoshuaConfiguration
      combiner = new CubePruneCombiner(this.featureFunctions, this.stateComputers);
    else
      combiner = new ExhaustiveCombiner(this.featureFunctions, this.stateComputers);

    //============== begin to do initialization work

    //TODO: which grammar should we use to create a mannual rule?, grammar[1] is the regular grammar
    manualConstraintsHandler = new ManualConstraintsHandler(symbolTable, this, grammars[1], constraintSpans);
   
   
    /**add OOV rules;
     * this should be called after the manual constraints have been set up
     * Different grammar differ in hasRuleForSpan, defaultOwner, and defaultLHSSymbol
     **/
    // TODO: the transition cost for phrase model, arity penalty, word penalty are all zero, except the LM cost
    for (Node<Integer> node : sentence) {
      for (Arc<Integer> arc : node.getOutgoingArcs()) {
        // create a rule, but do not add into the grammar trie
        // TODO: which grammar should we use to create an OOV rule?
//        this is the regular grammar
        int sourceWord = arc.getLabel();
        int targetWord = symbolTable.addTerminal( symbolTable.getWord(sourceWord)+"_OOV");
        Rule rule = this.grammars[1].constructOOVRule(
          this.featureFunctions.size(), sourceWord, targetWord, hasLM);
     
        if (manualConstraintsHandler.containHardRuleConstraint(node.getNumber(), arc.getTail().getNumber())) {
          //do not add the oov axiom
          if (logger.isLoggable(Level.FINE))
            logger.fine("Using hard rule constraint for span " + node.getNumber() + ", " + arc.getTail().getNumber());
        } else {
          //System.out.println(rule.toString(symbolTable));
          addAxiom(node.getNumber(), arc.getTail().getNumber(), rule, new SourcePath().extend(arc));
        }
      }
    }
   
 
   
    if (logger.isLoggable(Level.FINE))
      logger.fine("Finished seeding chart.");
  }

 
//===============================================================
// The primary method for filling in the chart
//===============================================================
 
  /**
   * Construct the hypergraph with the help from DotChart.
   */
 
  /** a parser that can handle:
   * - multiple grammars
   * - on the fly binarization
   * - unary rules (without cycle)
   * */
 
  public HyperGraph expand() {
   
    if (logger.isLoggable(Level.FINE))
      logger.fine("Begin expand.");
   
    for (int width = 1; width <= foreignSentenceLength; width++) {
      for (int i = 0; i <= foreignSentenceLength - width; i++) {
        int j = i + width;
        if (logger.isLoggable(Level.FINEST))
          logger.finest(String.format("Processing span (%d, %d)",i,j));
       
       
        //(1)=== expand the cell in dotchart
        if (logger.isLoggable(Level.FINEST))
          logger.finest("Expanding cell");
        for (int k = 0; k < this.grammars.length; k++) {
          /**each dotChart can act individually (without consulting other dotCharts)
           * because it either consumes the source input or the complete nonTerminals,
           * which are both grammar-independent
           **/
          this.dotcharts[k].expandDotCell(i,j);
        }
     
       
        //(2)=== populate COMPLETE rules into Chart: the regular CKY part
        if (logger.isLoggable(Level.FINEST))
          logger.finest("Adding complete items into chart");
        for (int k = 0; k < this.grammars.length; k++) {
         
          if (this.grammars[k].hasRuleForSpan(i, j, foreignSentenceLength)
            && null != this.dotcharts[k].getDotCell(i, j)) {
           
            for (DotNode dotNode: this.dotcharts[k].getDotCell(i, j).getDotNodes()) {
              RuleCollection ruleCollection = dotNode.getTrieNode().getRules();
              if (ruleCollection != null) { // have rules under this trienode
                // TODO: filter the rule according to LHS constraint               
                completeCell(i, j, dotNode, ruleCollection.getSortedRules(), ruleCollection.getArity(), dotNode.getSourcePath());                 
               
              }
            }
          }
        }       
       
        //(3)=== process unary rules (e.g., S->X, NP->NN), just add these items in chart, assume acyclic
        if (logger.isLoggable(Level.FINEST))
          logger.finest("Adding unary items into chart");
       
        /**zhifei replaced the following code to address an interaction problem between different grammars
         * the problem is: if [X]->[NT,1],[NT,1] in a regular grammar, but  [S]->[X,1],[X,1] is in a glue grammar;
         * then [S]->[NT,1],[NT,1] may not be achievable, depending on which grammar is processed first.
         */
        if(false){//behavior depend on the order of the grammars got processed, which is bad
          for (int k = 0; k < this.grammars.length; k++) {
            if (this.grammars[k].hasRuleForSpan(i, j, foreignSentenceLength)) {
              addUnaryNodesPerGrammar(this.grammars[k],i,j);//single-branch path
            }
          }
        }else{//behavior does not depend on the order of the grammars got processed
          addUnaryNodes(this.grammars,i,j);
        }       
       
       
        //(4)=== in dot_cell(i,j), add dot-nodes that start from the /complete/ superIterms in chart_cell(i,j)
        if (logger.isLoggable(Level.FINEST))
          logger.finest("Initializing new dot-items that start from complete items in this cell");
        for (int k = 0; k < this.grammars.length; k++) {
          if (this.grammars[k].hasRuleForSpan(i, j, foreignSentenceLength)) {
            this.dotcharts[k].startDotItems(i,j);
          }
        }       
       
        //(5)=== sort the nodes in the cell
        /**Cube-pruning requires the nodes being sorted, when prunning for later/wider cell.
         * Cuebe-pruning will see superNode, which contains a list of nodes.
         * getSortedNodes() will make the nodes in the superNode get sorted*/
        if (null != this.cells[i][j]) {
          this.cells[i][j].getSortedNodes();
        }
      }
    }
   
    logStatistics(Level.INFO);

    // transition_final: setup a goal item, which may have many deductions
    if (null != this.cells[0][foreignSentenceLength]) {
      this.goalBin.transitToGoal(this.cells[0][foreignSentenceLength], this.featureFunctions, this.foreignSentenceLength);       
    } else {
      logger.severe(
        "No complete item in the cell(0," + foreignSentenceLength + "); possible reasons: " +
        "(1) your grammar does not have any valid derivation for the source sentence; " +
        "(2) too aggressive pruning");
      System.exit(1);
    }
   
    if(logger.isLoggable(Level.FINE))
      logger.fine("Finished expand");
    return new HyperGraph(this.goalBin.getSortedNodes().get(0), -1, -1, this.segmentID, foreignSentenceLength);
  }
 
 
  public Cell getCell(int i, int j){
    return this.cells[i][j];
  }
 
 
//===============================================================
// Private methods
//===============================================================
 
  private void logStatistics(Level level) {
    if (logger.isLoggable(level)) {
      logger.log(level,
        String.format("ADDED: %d; MERGED: %d; PRUNED: %d; PRE-PRUNED: %d, FUZZ1: %d, FUZZ2: %d; DOT-ITEMS ADDED: %d",
          this.nAdded,
          this.nMerged,
          this.nPrunedItems,
          this.nPreprunedEdges,
          this.nPreprunedFuzz1,
          this.nPreprunedFuzz2,
          this.nDotitemAdded));
    }
  }
 
 
  /**
   * agenda based extension: this is necessary in case more
   * than two unary rules can be applied in topological order
   * s->x; ss->s for unary rules like s->x, once x is complete,
   * then s is also complete
   */
  private int addUnaryNodes(Grammar[] grs, int i, int j) {   
   
    Cell chartBin = this.cells[i][j];
    if (null == chartBin) {
      return 0;
    }
    int qtyAdditionsToQueue = 0;
    ArrayList<HGNode> queue  = new ArrayList<HGNode>( chartBin.getSortedNodes() );
   
    while (queue.size() > 0) {
      HGNode node = queue.remove(0);
      for(Grammar gr : grs){
        if (! gr.hasRuleForSpan(i, j, foreignSentenceLength))
          continue;
       
        Trie childNode = gr.getTrieRoot().matchOne(node.lhs); // match rule and complete part
        if (childNode != null
          && childNode.getRules() != null
          && childNode.getRules().getArity() == 1) { // have unary rules under this trienode
         
          ArrayList<HGNode> antecedents = new ArrayList<HGNode>();
          antecedents.add(node);
          List<Rule> rules = childNode.getRules().getSortedRules();
         
          for (Rule rule : rules) { // for each unary rules               
            ComputeNodeResult states = new ComputeNodeResult(this.featureFunctions, rule, antecedents, i, j, new SourcePath(), stateComputers, this.segmentID);
            HGNode resNode = chartBin.addHyperEdgeInCell(states, rule, i, j, antecedents, new SourcePath(), true);
            if (null != resNode) {
              queue.add(resNode);
              qtyAdditionsToQueue++;
            }
          }
        }
      }
    }
    return qtyAdditionsToQueue;
  }
 
  /**
     * agenda based extension: this is necessary in case more than two unary rules can be applied in topological order s->x; ss->s
     * for unary rules like s->x, once x is complete, then s is also complete
     */
    private int addUnaryNodesPerGrammar(Grammar gr, int i, int j) {
     
            Cell chartCell = this.cells[i][j];
            if (null == chartCell) {
                    return 0;
            }
            int qtyAdditionsToQueue = 0;
            ArrayList<HGNode> queue
                    = new ArrayList<HGNode>(chartCell.getSortedNodes());


            while (queue.size() > 0) {
              HGNode item = (HGNode)queue.remove(0);
                Trie child_tnode = gr.getTrieRoot().matchOne(item.lhs);//match rule and complete part
                if (child_tnode != null
                && child_tnode.getRules() != null
                && child_tnode.getRules().getArity() == 1) {//have unary rules under this trienode
                        ArrayList<HGNode> l_ants = new ArrayList<HGNode>();
                        l_ants.add(item);
                        List<Rule> rules =
                                child_tnode.getRules().getSortedRules();

                        for (Rule rule : rules){//for each unary rules
                          ComputeNodeResult states = new ComputeNodeResult(this.featureFunctions, rule, l_ants, i, j, new SourcePath(), stateComputers, this.segmentID);
                            HGNode res_item = chartCell.addHyperEdgeInCell(states, rule, i, j, l_ants, new SourcePath(), false);
                            if (null != res_item) {
                                    queue.add(res_item);
                                    qtyAdditionsToQueue++;
                            }
                        }
                }
            }
            return qtyAdditionsToQueue;
    }

 
  /** axiom is for rules with zero-arity */
  public void addAxiom(int i, int j, Rule rule, SourcePath srcPath) {
    if (null == this.cells[i][j]) {
      this.cells[i][j] = new Cell(this, this.goalSymbolID);
    }   
    combiner.addAxiom(this, this.cells[i][j], i, j, rule, srcPath);
  }
 
 
 
  private void completeCell(int i, int j, DotNode dotNode, List<Rule> sortedRules, int arity, SourcePath srcPath) {
   
    if (manualConstraintsHandler.containHardRuleConstraint(i, j)) {
      if (logger.isLoggable(Level.FINE))
        logger.fine("Hard rule constraint for span " +i +", " + j);
      return; //do not add any nodes
    }
   
    if (null == this.cells[i][j]) {
      this.cells[i][j] = new Cell(this, this.goalSymbolID);
    }
    // combinations: rules, antecent items
    List<Rule> filteredRules =  manualConstraintsHandler.filterRules(i,j, sortedRules);
    if(arity==0)
      combiner.addAxioms(this, this.cells[i][j], i, j, filteredRules, srcPath);
    else
      //this.cells[i][j].completeCell(i, j, dt.l_ant_super_items, filterRules(i,j,rb.getSortedRules()), rb.getArity(), srcPath);
      combiner.combine(this, this.cells[i][j], i, j,  dotNode.getAntSuperNodes(), filteredRules, arity, srcPath);
 
 
 
 
}
TOP

Related Classes of joshua.decoder.chart_parser.Chart

TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.