*

* Joshua is free software; you can redistribute it and/or modify it

* under the terms of the GNU Lesser General Public License as published by

* the Free Software Foundation; either version 2.1 of the License, or

* (at your option) any later version.

*

* This library is distributed in the hope that it will be useful, but

* WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY

* or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public

* License for more details.

*

* You should have received a copy of the GNU Lesser General Public License

* along with this library; if not, write to the Free Software Foundation,

* Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA

*/

* Front to end tests to extract rules from a sample corpus using suffix arrays.

*

* @author Lane Schwartz

* @version $LastChangedDate:2008-09-18 12:47:23 -0500 (Thu, 18 Sep 2008) $

*/

  // The following ruby one-liner can convert a Hiero grammar

  //     into useful Java assertions for unit testing:

  //

  // cat hiero.grammar | sort | ruby -e 'STDIN.each_line{ |line| words=line.strip.split(" ||| "); puts "verifyLine(lines.get(n++), \"#{words[0]}\", \"#{words[1]}\", \"#{words[2]}\");"; }' > java-rules.txt

 

  // Also

  // cat hiero.grammar | sort | ruby -e 'STDIN.each_line{ |line| words=line.strip.split(" ||| "); puts "#{words[0]} ||| #{words[1]} ||| #{words[2]}"; }' > hiero.grammar.clean

 

  String sourceFileName;

  String targetFileName;

  String alignmentFileName;

  String alignmentsType;

  // ä == \u00E4

  // ü == \u00FC

  // ñ == \u00F1

  // í == \u00ED

 

  @Parameters({"alignmentsType"})

  @Test

  public void setup(String alignmentsType) throws IOException {

   

    // Tell System.out and System.err to use UTF8

    FormatUtil.useUTF8();

       

    String sourceCorpusString =

      "it makes him and it mars him , it sets him on yet it takes him off .";

   

    {

      File sourceFile = File.createTempFile("source", new Date().toString());

      PrintStream sourcePrintStream = new PrintStream(sourceFile, "UTF-8");

      sourcePrintStream.println(sourceCorpusString);

      sourcePrintStream.close();

      sourceFileName = sourceFile.getAbsolutePath();

    }

 

    String targetCorpusString =

      "das macht ihn und es besch\u00E4digt ihn , es setzt ihn auf und es f\u00FChrt ihn aus .";

   

   

    {

      File targetFile = File.createTempFile("target", new Date().toString());

      PrintWriter targetPrintStream = new PrintWriter(targetFile, "UTF-8");

      targetPrintStream.println(targetCorpusString);

      targetPrintStream.close();

      targetFileName = targetFile.getAbsolutePath();

    }

   

    String alignmentString =

      "0-0 1-1 2-2 3-3 4-4 5-5 6-6 7-7 8-8 9-9 10-10 11-11 12-12 13-13 14-14 15-15 16-16 17-17";

   

    {

      File alignmentFile = File.createTempFile("alignment", new Date().toString());

      PrintStream alignmentPrintStream = new PrintStream(alignmentFile);

      alignmentPrintStream.println(alignmentString);

      alignmentPrintStream.close();

      alignmentFileName = alignmentFile.getAbsolutePath();

    }

    this.alignmentsType = alignmentsType;

  }

  /**

   * Extracts rules and returns the file name where the extracted rules are stored.

   *

   * @param testCorpusString

   * @param sentenceInitialX TODO

   * @param sentenceFinalX TODO

   * @param violatingX TODO

   * @return

   * @throws IOException

   */

  private List<String> extractRules(String testCorpusString, boolean sentenceInitialX, boolean sentenceFinalX, boolean violatingX) throws IOException {

    return extractRules(sourceFileName, targetFileName, alignmentFileName, testCorpusString, sentenceInitialX, sentenceFinalX, violatingX, false, 2);

  }

 

  private List<String> extractRules(String sourceFileName, String targetFileName, String alignmentFileName, String testCorpusString, boolean sentenceInitialX, boolean sentenceFinalX, boolean violatingX, boolean printPrefixTree, int minNonterminalSpan) throws IOException {

   

    String testFileName;

    {

      File testFile = File.createTempFile("test", new Date().toString());

      PrintStream testPrintStream = new PrintStream(testFile, "UTF-8");

      testPrintStream.println(testCorpusString);

      testPrintStream.close();

      testFileName = testFile.getAbsolutePath();

    }

   

    // Filename of the extracted rules file.

    String rulesFileName;

    { 

      File rulesFile = File.createTempFile("rules", new Date().toString());

      rulesFileName = rulesFile.getAbsolutePath();

    }

   

    String joshDirName;

    {

      File joshDir = File.createTempFile(new Date().toString(), "josh");

      joshDirName = joshDir.getAbsolutePath();

      joshDir.delete();

    }

   

    Compile compileJoshDir = new Compile();

    compileJoshDir.setSourceCorpus(sourceFileName);

    compileJoshDir.setTargetCorpus(targetFileName);

    compileJoshDir.setAlignments(alignmentFileName);

    compileJoshDir.setOutputDir(joshDirName);

    compileJoshDir.execute();

   

    ExtractRules extractRules = new ExtractRules();

    extractRules.setSentenceInitialX(sentenceInitialX);

    extractRules.setSentenceFinalX(sentenceFinalX);

    extractRules.setEdgeXViolates(violatingX);

    extractRules.setMaxPhraseLength(5);

    extractRules.setMinNonterminalSpan(minNonterminalSpan);

    extractRules.setJoshDir(joshDirName);

    extractRules.setTestFile(testFileName);

    extractRules.setOutputFile(rulesFileName);

    try {

      extractRules.execute();

    } catch (ClassNotFoundException e) {

      e.printStackTrace();

      Assert.fail(e.getLocalizedMessage());

    }

   

    Scanner scanner = new Scanner(new File(rulesFileName));

   

    ArrayList<String> lines = new ArrayList<String>();

   

    while (scanner.hasNextLine()) {

     

      String line = scanner.nextLine();

      lines.add(line);

     

    }

   

   

    Collections.sort(lines);

   

    return lines;

  }

 

  public void verifyLine(String line, String lhs, String sourceRHS, String targetRHS)  {

    String[] part = line.split(" \\|\\|\\| ");

    Assert.assertEquals(part[0], lhs);

    Assert.assertEquals(part[1], sourceRHS);

    Assert.assertEquals(part[2], targetRHS);

  }

 

  @Test

  public void europarlSmall100() throws IOException {

    extractEuroparlSmall100(false);

  }

 

//  public void europarlSmall100ViolatingX() throws IOException {

//    extractEuroparlSmall100(true);

//  }

 

  private static class WTF extends RuntimeException {

    WTF(String msg) { super(msg); }

  }

  public void extractEuroparlSmall100(boolean violatingX) throws IOException {

   

    String sourceFileName = "data/europarl.es.small.100";

    String targetFileName = "data/europarl.en.small.100";

    String alignmentFileName = "data/es_en_europarl_alignments.txt.small.100";

   

    String testSentence = "declaro reanudado el per\u00EDodo de sesiones del parlamento europeo , interrumpido el viernes 17 de diciembre pasado , y reitero a sus se\u00F1or\u00EDas mi deseo de que hayan tenido unas buenas vacaciones .";

   

    boolean printPrefixTree = false;

   

    int minNonterminalSpan = 2;

   

    List<String> lines = extractRules(sourceFileName, targetFileName, alignmentFileName, testSentence, true, true, violatingX, printPrefixTree, minNonterminalSpan);

    int expectedLines = 525;

    if (violatingX) expectedLines += 14;

   

//      System.err.println(line);

//    }

   

//    if (lines.size() != expectedLines) {

//      throw new WTF("lines.size()=="+lines.size() + " but expectedLines=="+expectedLines);

//    }

    Assert.assertEquals(lines.size(), expectedLines);

   

    int n = 0;

    verifyLine(lines.get(n++), "[X]", ", [X,1] , [X,2]", ", [X,1] , [X,2]");

    verifyLine(lines.get(n++), "[X]", ", [X,1] , [X,2]", ", [X,1] [X,2]");

    verifyLine(lines.get(n++), "[X]", ", [X,1] , [X,2]", ", [X,2] [X,1]");

    verifyLine(lines.get(n++), "[X]", ", [X,1] , [X,2]", ", you [X,1] [X,2]");

    if (violatingX) verifyLine(lines.get(n++), "[X]", ", [X,1] , y [X,2]", ", [X,1] , and [X,2]"); // Added in buggy

    verifyLine(lines.get(n++), "[X]", ", [X,1] , y", ", [X,1] , and");

    verifyLine(lines.get(n++), "[X]", ", [X,1] , y", ", you [X,1] and");

    verifyLine(lines.get(n++), "[X]", ", [X,1] ,", ", [X,1] ,");

    verifyLine(lines.get(n++), "[X]", ", [X,1] a [X,2]", ", [X,1] to wish [X,2]");

    verifyLine(lines.get(n++), "[X]", ", [X,1] a [X,2]", "too [X,1] to [X,2]");

    verifyLine(lines.get(n++), "[X]", ", [X,1] a sus se\u00F1or\u00EDas", ", [X,1] to wish you");

    verifyLine(lines.get(n++), "[X]", ", [X,1] a", ", [X,1] to");

    verifyLine(lines.get(n++), "[X]", ", [X,1] a", "too [X,1] to");

    verifyLine(lines.get(n++), "[X]", ", [X,1] de [X,2]", ", [X,1] [X,2]");

    verifyLine(lines.get(n++), "[X]", ", [X,1] de [X,2]", ", [X,1] in [X,2]");

    verifyLine(lines.get(n++), "[X]", ", [X,1] de [X,2]", ", [X,1] of [X,2]");

    verifyLine(lines.get(n++), "[X]", ", [X,1] de [X,2]", ", [X,1] shall do [X,2]");

    verifyLine(lines.get(n++), "[X]", ", [X,1] de [X,2]", ", still [X,1] in [X,2]");

    verifyLine(lines.get(n++), "[X]", ", [X,1] de [X,2]", ", still [X,1] of [X,2]");

    if (violatingX) verifyLine(lines.get(n++), "[X]", ", [X,1] de que [X,2]", ", [X,1] that [X,2]"); // Added in buggy

    verifyLine(lines.get(n++), "[X]", ", [X,1] de que", ", [X,1] that");

    verifyLine(lines.get(n++), "[X]", ", [X,1] de que", ", [X,1] to the start of");

    verifyLine(lines.get(n++), "[X]", ", [X,1] de", ", [X,1] attention to");

    verifyLine(lines.get(n++), "[X]", ", [X,1] de", ", [X,1] in");

    verifyLine(lines.get(n++), "[X]", ", [X,1] de", ", [X,1] of");

    verifyLine(lines.get(n++), "[X]", ", [X,1] de", ", still [X,1] in");

    verifyLine(lines.get(n++), "[X]", ", [X,1] de", ", still [X,1] of");

    verifyLine(lines.get(n++), "[X]", ", [X,1] deseo [X,2]", ", [X,1] wish [X,2]");

    verifyLine(lines.get(n++), "[X]", ", [X,1] deseo de [X,2]", ", [X,1] wish of [X,2]");

    verifyLine(lines.get(n++), "[X]", ", [X,1] deseo de", ", [X,1] wish of");

    verifyLine(lines.get(n++), "[X]", ", [X,1] deseo", ", [X,1] wish");

    verifyLine(lines.get(n++), "[X]", ", [X,1] el [X,2] ,", ", [X,1] [X,2] ,");

    verifyLine(lines.get(n++), "[X]", ", [X,1] el [X,2]", ", [X,1] [X,2]");

    verifyLine(lines.get(n++), "[X]", ", [X,1] el [X,2]", ", [X,1] the [X,2]");

    verifyLine(lines.get(n++), "[X]", ", [X,1] el", ", [X,1] the");

    verifyLine(lines.get(n++), "[X]", ", [X,1] el", ", [X,1] there");

    if (violatingX) verifyLine(lines.get(n++), "[X]", ", [X,1] que [X,2]", ", [X,1] that [X,2]"); // Added in buggy

    verifyLine(lines.get(n++), "[X]", ", [X,1] que", ", [X,1] that");

    verifyLine(lines.get(n++), "[X]", ", [X,1] que", ", [X,1] which");

    verifyLine(lines.get(n++), "[X]", ", [X,1] sus se\u00F1or\u00EDas", ", [X,1] wish you");

    if (violatingX) verifyLine(lines.get(n++), "[X]", ", [X,1] y [X,2]", ", [X,1] and [X,2]"); // Added in buggy

    verifyLine(lines.get(n++), "[X]", ", [X,1] y", ", [X,1] and");

    verifyLine(lines.get(n++), "[X]", ", [X,1]", ", [X,1]");

    verifyLine(lines.get(n++), "[X]", ", [X,1]", ", and [X,1]");

    verifyLine(lines.get(n++), "[X]", ", [X,1]", ", still [X,1]");

    verifyLine(lines.get(n++), "[X]", ", [X,1]", ", there is [X,1]");

    verifyLine(lines.get(n++), "[X]", ", [X,1]", ", which [X,1]");

    verifyLine(lines.get(n++), "[X]", ", [X,1]", ", you [X,1]");

    verifyLine(lines.get(n++), "[X]", ", [X,1]", "and urging [X,1]");

    verifyLine(lines.get(n++), "[X]", ", [X,1]", "i [X,1]");

    verifyLine(lines.get(n++), "[X]", ", [X,1]", "too [X,1]");

    verifyLine(lines.get(n++), "[X]", ", y [X,1] de que", ", and [X,1] that");

    verifyLine(lines.get(n++), "[X]", ", y [X,1] sus se\u00F1or\u00EDas", ", and [X,1] wish you");

    verifyLine(lines.get(n++), "[X]", ", y [X,1]", ", and [X,1]");

    verifyLine(lines.get(n++), "[X]", ", y reitero [X,1]", ", and i would like once again [X,1]");

    verifyLine(lines.get(n++), "[X]", ", y reitero a [X,1]", ", and i would like once again to wish [X,1]");

    verifyLine(lines.get(n++), "[X]", ", y reitero a", ", and i would like once again to");

    verifyLine(lines.get(n++), "[X]", ", y reitero", ", and i would like once again");

    verifyLine(lines.get(n++), "[X]", ", y", ", and");

    verifyLine(lines.get(n++), "[X]", ",", ", which");

    verifyLine(lines.get(n++), "[X]", ",", ",");

    verifyLine(lines.get(n++), "[X]", ",", "and");

    verifyLine(lines.get(n++), "[X]", ",", "i");

    verifyLine(lines.get(n++), "[X]", ",", "too");

    verifyLine(lines.get(n++), "[X]", ".", ".");

    verifyLine(lines.get(n++), "[X]", ".", "?");

    verifyLine(lines.get(n++), "[X]", ".", "materialise");

    verifyLine(lines.get(n++), "[X]", "17 [X,1] , [X,2]", "17 [X,1] , [X,2]");

    verifyLine(lines.get(n++), "[X]", "17 [X,1] , y reitero", "17 [X,1] , and i would like once again");

    verifyLine(lines.get(n++), "[X]", "17 [X,1] , y", "17 [X,1] , and");

    verifyLine(lines.get(n++), "[X]", "17 [X,1] ,", "17 [X,1] ,");

    verifyLine(lines.get(n++), "[X]", "17 [X,1] reitero", "17 [X,1] i would like once again");

    verifyLine(lines.get(n++), "[X]", "17 [X,1] y reitero", "17 [X,1] and i would like once again");

    verifyLine(lines.get(n++), "[X]", "17 [X,1] y", "17 [X,1] and");

    verifyLine(lines.get(n++), "[X]", "17 [X,1]", "17 [X,1]");

    verifyLine(lines.get(n++), "[X]", "17 de diciembre pasado ,", "17 december 1999 ,");

    verifyLine(lines.get(n++), "[X]", "17 de diciembre pasado [X,1]", "17 december 1999 [X,1]");

    verifyLine(lines.get(n++), "[X]", "17 de diciembre pasado", "17 december 1999");

    verifyLine(lines.get(n++), "[X]", "17", "17");

    if (violatingX) verifyLine(lines.get(n++), "[X]", "[X,1] , [X,2] , y", "[X,1] , [X,2] , and"); // Added in buggy

    verifyLine(lines.get(n++), "[X]", "[X,1] , [X,2] ,", ", [X,2] [X,1] ,");

    verifyLine(lines.get(n++), "[X]", "[X,1] , [X,2] ,", "[X,1] , [X,2] ,");

    verifyLine(lines.get(n++), "[X]", "[X,1] , [X,2] ,", "[X,1] [X,2] ,");

    verifyLine(lines.get(n++), "[X]", "[X,1] , [X,2] 17", "[X,1] [X,2] 17");

    verifyLine(lines.get(n++), "[X]", "[X,1] , [X,2] a", "[X,1] , [X,2] to");

    if (violatingX) verifyLine(lines.get(n++), "[X]", "[X,1] , [X,2] de que", "[X,1] , [X,2] that"); // Added in buggy

    if (violatingX) verifyLine(lines.get(n++), "[X]", "[X,1] , [X,2] de que", "[X,1] [X,2] to the start of"); // Added in buggy

    verifyLine(lines.get(n++), "[X]", "[X,1] , [X,2] de", "[X,1] , [X,2] in");

    verifyLine(lines.get(n++), "[X]", "[X,1] , [X,2] de", "[X,1] , [X,2] of");

    verifyLine(lines.get(n++), "[X]", "[X,1] , [X,2] de", "[X,1] [X,2] of");

    verifyLine(lines.get(n++), "[X]", "[X,1] , [X,2] deseo de", "[X,1] , [X,2] wish of");

    verifyLine(lines.get(n++), "[X]", "[X,1] , [X,2] deseo", "[X,1] , [X,2] wish");

    verifyLine(lines.get(n++), "[X]", "[X,1] , [X,2] el", "[X,1] , [X,2] the");

    verifyLine(lines.get(n++), "[X]", "[X,1] , [X,2] el", "[X,1] , [X,2] there");

    if (violatingX) verifyLine(lines.get(n++), "[X]", "[X,1] , [X,2] que", "[X,1] , [X,2] that"); // Added in buggy start

    if (violatingX) verifyLine(lines.get(n++), "[X]", "[X,1] , [X,2] que", "[X,1] [X,2] that"); // Added in buggy start

    if (violatingX) verifyLine(lines.get(n++), "[X]", "[X,1] , [X,2] y", "[X,1] , [X,2] and"); // Added in buggy start

    verifyLine(lines.get(n++), "[X]", "[X,1] , [X,2]", ", [X,2] [X,1]");

    verifyLine(lines.get(n++), "[X]", "[X,1] , [X,2]", "[X,1] , [X,2]");

    verifyLine(lines.get(n++), "[X]", "[X,1] , [X,2]", "[X,1] , you [X,2]");

    verifyLine(lines.get(n++), "[X]", "[X,1] , [X,2]", "[X,1] i [X,2]");

    verifyLine(lines.get(n++), "[X]", "[X,1] , [X,2]", "[X,1] in sri lanka and urging [X,2]");

    verifyLine(lines.get(n++), "[X]", "[X,1] , [X,2]", "[X,1] one , there is [X,2]");

    verifyLine(lines.get(n++), "[X]", "[X,1] , interrumpido [X,2] 17", "[X,1] adjourned [X,2] 17");

    verifyLine(lines.get(n++), "[X]", "[X,1] , interrumpido [X,2]", "[X,1] adjourned [X,2]");

    verifyLine(lines.get(n++), "[X]", "[X,1] , interrumpido el viernes", "[X,1] adjourned on friday");

    verifyLine(lines.get(n++), "[X]", "[X,1] , interrumpido", "[X,1] adjourned");

    verifyLine(lines.get(n++), "[X]", "[X,1] , y [X,2]", "[X,1] , and [X,2]");

    verifyLine(lines.get(n++), "[X]", "[X,1] , y [X,2]", "[X,1] and [X,2]");

    verifyLine(lines.get(n++), "[X]", "[X,1] , y reitero a", "[X,1] , and i would like once again to");

    verifyLine(lines.get(n++), "[X]", "[X,1] , y reitero", "[X,1] , and i would like once again");

    verifyLine(lines.get(n++), "[X]", "[X,1] , y", "[X,1] , and");

    verifyLine(lines.get(n++), "[X]", "[X,1] , y", "[X,1] and");

    verifyLine(lines.get(n++), "[X]", "[X,1] ,", "[X,1] ,");

    verifyLine(lines.get(n++), "[X]", "[X,1] ,", "[X,1] i");

    verifyLine(lines.get(n++), "[X]", "[X,1] ,", "[X,1] in sri lanka and");

    verifyLine(lines.get(n++), "[X]", "[X,1] ,", "[X,1] one ,");

    verifyLine(lines.get(n++), "[X]", "[X,1] ,", "[X,1] to point out ,");

    verifyLine(lines.get(n++), "[X]", "[X,1] .", "[X,1] .");

    verifyLine(lines.get(n++), "[X]", "[X,1] .", "[X,1] ?");

    verifyLine(lines.get(n++), "[X]", "[X,1] 17 [X,2] , y", "[X,1] 17 [X,2] , and");

    verifyLine(lines.get(n++), "[X]", "[X,1] 17 [X,2] ,", "[X,1] 17 [X,2] ,");

    verifyLine(lines.get(n++), "[X]", "[X,1] 17 [X,2] y", "[X,1] 17 [X,2] and");

    verifyLine(lines.get(n++), "[X]", "[X,1] 17 [X,2]", "[X,1] 17 [X,2]");

    verifyLine(lines.get(n++), "[X]", "[X,1] 17 de diciembre pasado", "[X,1] 17 december 1999");

    verifyLine(lines.get(n++), "[X]", "[X,1] 17", "[X,1] 17");

    verifyLine(lines.get(n++), "[X]", "[X,1] a [X,2] de", "[X,1] to [X,2] of");

    verifyLine(lines.get(n++), "[X]", "[X,1] a [X,2] que", "[X,1] for [X,2] which");

    verifyLine(lines.get(n++), "[X]", "[X,1] a [X,2] que", "[X,1] on [X,2] that");

    verifyLine(lines.get(n++), "[X]", "[X,1] a [X,2]", "[X,1] on [X,2]");

    verifyLine(lines.get(n++), "[X]", "[X,1] a [X,2]", "[X,1] to [X,2]");

    verifyLine(lines.get(n++), "[X]", "[X,1] a [X,2]", "[X,1] to wish [X,2]");

    verifyLine(lines.get(n++), "[X]", "[X,1] a sus se\u00F1or\u00EDas", "[X,1] to wish you");

    verifyLine(lines.get(n++), "[X]", "[X,1] a", "[X,1] on");

    verifyLine(lines.get(n++), "[X]", "[X,1] a", "[X,1] to");

    verifyLine(lines.get(n++), "[X]", "[X,1] de [X,2] ,", "[X,1] [X,2] ,");

    verifyLine(lines.get(n++), "[X]", "[X,1] de [X,2] ,", "[X,1] in [X,2] ,");

    verifyLine(lines.get(n++), "[X]", "[X,1] de [X,2] ,", "[X,1] of [X,2] ,");

    verifyLine(lines.get(n++), "[X]", "[X,1] de [X,2] .", "[X,1] [X,2] .");

    verifyLine(lines.get(n++), "[X]", "[X,1] de [X,2] .", "[X,1] for topical [X,2] .");

    verifyLine(lines.get(n++), "[X]", "[X,1] de [X,2] .", "[X,1] of [X,2] .");

    verifyLine(lines.get(n++), "[X]", "[X,1] de [X,2] .", "[X,1] shall do [X,2] .");

    verifyLine(lines.get(n++), "[X]", "[X,1] de [X,2] .", "[X,2] [X,1] .");

    verifyLine(lines.get(n++), "[X]", "[X,1] de [X,2] del", "[X,2] 's [X,1] of");

    verifyLine(lines.get(n++), "[X]", "[X,1] de [X,2] el", "[X,1] [X,2] the");

    verifyLine(lines.get(n++), "[X]", "[X,1] de [X,2] el", "[X,1] of [X,2] the");

    verifyLine(lines.get(n++), "[X]", "[X,1] de [X,2] parlamento", "[X,1] with [X,2] house");

    verifyLine(lines.get(n++), "[X]", "[X,1] de [X,2] y", "[X,1] in [X,2] and");

    verifyLine(lines.get(n++), "[X]", "[X,1] de [X,2] y", "[X,1] of [X,2] and");

    verifyLine(lines.get(n++), "[X]", "[X,1] de [X,2]", "[X,1] 's [X,2]");

    verifyLine(lines.get(n++), "[X]", "[X,1] de [X,2]", "[X,1] [X,2] of");

    verifyLine(lines.get(n++), "[X]", "[X,1] de [X,2]", "[X,1] attention to [X,2]");

    verifyLine(lines.get(n++), "[X]", "[X,1] de [X,2]", "[X,1] by [X,2]");

    verifyLine(lines.get(n++), "[X]", "[X,1] de [X,2]", "[X,1] for topical [X,2]");

    verifyLine(lines.get(n++), "[X]", "[X,1] de [X,2]", "[X,1] in [X,2]");

    verifyLine(lines.get(n++), "[X]", "[X,1] de [X,2]", "[X,1] of [X,2]");

    verifyLine(lines.get(n++), "[X]", "[X,1] de [X,2]", "[X,1] to [X,2]");

    verifyLine(lines.get(n++), "[X]", "[X,1] de [X,2]", "[X,1] with [X,2]");

    verifyLine(lines.get(n++), "[X]", "[X,1] de [X,2]", "[X,2] 's [X,1] for");

    verifyLine(lines.get(n++), "[X]", "[X,1] de [X,2]", "[X,2] 's [X,1]");

    verifyLine(lines.get(n++), "[X]", "[X,1] de [X,2]", "[X,2] [X,1] of");

    verifyLine(lines.get(n++), "[X]", "[X,1] de [X,2]", "of [X,1] [X,2]");

    verifyLine(lines.get(n++), "[X]", "[X,1] de [X,2]", "of [X,2] [X,1]");

    verifyLine(lines.get(n++), "[X]", "[X,1] de diciembre pasado ,", "[X,1] december 1999 ,");

    verifyLine(lines.get(n++), "[X]", "[X,1] de diciembre pasado [X,2]", "[X,1] december 1999 [X,2]");

    verifyLine(lines.get(n++), "[X]", "[X,1] de diciembre pasado", "[X,1] december 1999");

    verifyLine(lines.get(n++), "[X]", "[X,1] de que [X,2]", "[X,1] that [X,2]");

    verifyLine(lines.get(n++), "[X]", "[X,1] de que", "[X,1] that");

    verifyLine(lines.get(n++), "[X]", "[X,1] de que", "[X,1] to the start of");

    verifyLine(lines.get(n++), "[X]", "[X,1] de sesiones del [X,2]", "[X,1] session of [X,2]");

    verifyLine(lines.get(n++), "[X]", "[X,1] de sesiones del", "[X,1] session of");

    verifyLine(lines.get(n++), "[X]", "[X,1] de sesiones", "[X,1] part-session");

    verifyLine(lines.get(n++), "[X]", "[X,1] de", "'s [X,1]");

    verifyLine(lines.get(n++), "[X]", "[X,1] de", "[X,1] 's");

    verifyLine(lines.get(n++), "[X]", "[X,1] de", "[X,1] attention to");

    verifyLine(lines.get(n++), "[X]", "[X,1] de", "[X,1] by");

    verifyLine(lines.get(n++), "[X]", "[X,1] de", "[X,1] for topical");

    verifyLine(lines.get(n++), "[X]", "[X,1] de", "[X,1] for");

    verifyLine(lines.get(n++), "[X]", "[X,1] de", "[X,1] in");

    verifyLine(lines.get(n++), "[X]", "[X,1] de", "[X,1] of");

    verifyLine(lines.get(n++), "[X]", "[X,1] de", "[X,1] on");

    verifyLine(lines.get(n++), "[X]", "[X,1] de", "[X,1] to");

    verifyLine(lines.get(n++), "[X]", "[X,1] de", "[X,1] with");

    verifyLine(lines.get(n++), "[X]", "[X,1] de", "of [X,1]");

    verifyLine(lines.get(n++), "[X]", "[X,1] del [X,2] ,", "[X,1] by [X,2] ,");

    verifyLine(lines.get(n++), "[X]", "[X,1] del [X,2] ,", "[X,1] of [X,2] ,");

    verifyLine(lines.get(n++), "[X]", "[X,1] del [X,2]", "[X,1] by [X,2]");

    verifyLine(lines.get(n++), "[X]", "[X,1] del [X,2]", "[X,1] from [X,2]");

    verifyLine(lines.get(n++), "[X]", "[X,1] del [X,2]", "[X,1] of [X,2]");

    verifyLine(lines.get(n++), "[X]", "[X,1] del [X,2]", "[X,1] of the [X,2]");

    verifyLine(lines.get(n++), "[X]", "[X,1] del [X,2]", "[X,1] of this [X,2]");

    verifyLine(lines.get(n++), "[X]", "[X,1] del parlamento [X,2]", "[X,2] parliament [X,1]");

    verifyLine(lines.get(n++), "[X]", "[X,1] del parlamento", "parliament [X,1]");

    verifyLine(lines.get(n++), "[X]", "[X,1] del", "'s [X,1]");

    verifyLine(lines.get(n++), "[X]", "[X,1] del", "[X,1] 's");

    verifyLine(lines.get(n++), "[X]", "[X,1] del", "[X,1] by");

    verifyLine(lines.get(n++), "[X]", "[X,1] del", "[X,1] from");

    verifyLine(lines.get(n++), "[X]", "[X,1] del", "[X,1] of the");

    verifyLine(lines.get(n++), "[X]", "[X,1] del", "[X,1] of");

    verifyLine(lines.get(n++), "[X]", "[X,1] deseo [X,2] .", "[X,1] wish [X,2] .");

    verifyLine(lines.get(n++), "[X]", "[X,1] deseo [X,2]", "[X,1] wish [X,2]");

    verifyLine(lines.get(n++), "[X]", "[X,1] deseo de [X,2] .", "[X,1] wish of [X,2] .");

    verifyLine(lines.get(n++), "[X]", "[X,1] deseo de [X,2]", "[X,1] wish of [X,2]");

    verifyLine(lines.get(n++), "[X]", "[X,1] deseo de", "[X,1] wish of");