Examples of joshua.corpus.suffix_array.HierarchicalPhrases

joshua.corpus.suffix_array.HierarchicalPhrases
HierarchicalPhrases represents a list of matched hierarchical phrases.
TODO Add unit tests for this class. @author Lane Schwartz @since Jan 9 2009 @version $LastChangedDate: 2010-02-01 14:37:27 -0600 (Mon, 01 Feb 2010) $

    
    Pattern     pattern = new Pattern(vocab, vocab.getIDs("de sesiones del parlamento europeo"));
    int[]       terminalSequenceStartIndices = {4};
    int[]       sentenceNumbers = {0};
    
    HierarchicalPhrases phrases =
      new HierarchicalPhrases(pattern, terminalSequenceStartIndices, sentenceNumbers);
    
    int arity = phrases.arity();
    int n = phrases.getNumberOfTerminalSequences();
    
    Assert.assertEquals(arity, 0);
    Assert.assertEquals(n, 1);
    
  }

View Full Code Here

  private HierarchicalPhrases getSourcePhrase(String sourcePhrase, int startIndex, int endIndex) {
    Pattern pattern = new Pattern(sourceVocab, sourceVocab.getIDs(sourcePhrase));
    int[] terminalSequenceStartIndices = {startIndex};
    int[]       sentenceNumbers = {0};
    
    HierarchicalPhrases phrases =
      new HierarchicalPhrases(pattern, terminalSequenceStartIndices, sentenceNumbers);
    
    return phrases;
  }

View Full Code Here

      Pattern     pattern = new Pattern(vocab, vocab.getIDs("de sesiones del parlamento europeo"));
      int[]       terminalSequenceStartIndices = {4};
      int[]       sentenceNumbers = {0};
      int phraseIndex = 0;
      
      HierarchicalPhrases phrases =
        new HierarchicalPhrases(pattern, terminalSequenceStartIndices, sentenceNumbers);
      
      
      
      Assert.assertFalse(alignments.hasAlignedTerminal(0 , phrases, phraseIndex));
      Assert.assertFalse(alignments.hasAlignedTerminal(1 , phrases, phraseIndex));
      Assert.assertFalse(alignments.hasAlignedTerminal(2 , phrases, phraseIndex));
      Assert.assertFalse(alignments.hasAlignedTerminal(3 , phrases, phraseIndex));


      Assert.assertTrue(alignments.hasAlignedTerminal(4 , phrases, phraseIndex));
      Assert.assertTrue(alignments.hasAlignedTerminal(5 , phrases, phraseIndex));
      Assert.assertTrue(alignments.hasAlignedTerminal(6 , phrases, phraseIndex));
      Assert.assertTrue(alignments.hasAlignedTerminal(7 , phrases, phraseIndex));
      Assert.assertTrue(alignments.hasAlignedTerminal(8 , phrases, phraseIndex));


      Assert.assertFalse(alignments.hasAlignedTerminal(9 , phrases, phraseIndex));
      Assert.assertFalse(alignments.hasAlignedTerminal(10 , phrases, phraseIndex));
      Assert.assertFalse(alignments.hasAlignedTerminal(11 , phrases, phraseIndex));
    }
  
    
    {
      Pattern     pattern = new Pattern(vocab, vocab.getIDs(","));
      int[]       terminalSequenceStartIndices = {9};
      int[]       sentenceNumbers = {0};
      int phraseIndex = 0;


      HierarchicalPhrases phrases =
        new HierarchicalPhrases(pattern, terminalSequenceStartIndices, sentenceNumbers);
      
      Assert.assertFalse(alignments.hasAlignedTerminal(0 , phrases, phraseIndex));
      Assert.assertFalse(alignments.hasAlignedTerminal(1 , phrases, phraseIndex));
      Assert.assertFalse(alignments.hasAlignedTerminal(2 , phrases, phraseIndex));
      Assert.assertFalse(alignments.hasAlignedTerminal(3 , phrases, phraseIndex));

View Full Code Here

      Pattern     pattern = new Pattern(new Pattern(new Pattern(vocab, vocab.getIDs("de sesiones")), SymbolTable.X), vocab.getIDs("europo"));// del parlamento europeo"));
      int[]       terminalSequenceStartIndices = {4,8};
      int[]       sentenceNumbers = {0};
      int phraseIndex = 0;
      
      HierarchicalPhrases phrases =
        new HierarchicalPhrases(pattern, terminalSequenceStartIndices, sentenceNumbers);
//      HierarchicalPhrase phrase = new HierarchicalPhrase(pattern, terminalSequenceStartIndices, terminalSequenceEndIndices, sourceCorpusArray, length);


      Assert.assertFalse(alignments.hasAlignedTerminal(0 , phrases, phraseIndex));
      Assert.assertFalse(alignments.hasAlignedTerminal(1 , phrases, phraseIndex));
      Assert.assertFalse(alignments.hasAlignedTerminal(2 , phrases, phraseIndex));
      Assert.assertFalse(alignments.hasAlignedTerminal(3 , phrases, phraseIndex));


      Assert.assertTrue(alignments.hasAlignedTerminal(4 , phrases, phraseIndex));
      Assert.assertTrue(alignments.hasAlignedTerminal(5 , phrases, phraseIndex));
      Assert.assertTrue(alignments.hasAlignedTerminal(6 , phrases, phraseIndex));
      Assert.assertTrue(alignments.hasAlignedTerminal(7 , phrases, phraseIndex));
      Assert.assertFalse(alignments.hasAlignedTerminal(8 , phrases, phraseIndex));


      Assert.assertFalse(alignments.hasAlignedTerminal(9 , phrases, phraseIndex));
      Assert.assertFalse(alignments.hasAlignedTerminal(10 , phrases, phraseIndex));
      Assert.assertFalse(alignments.hasAlignedTerminal(11 , phrases, phraseIndex));


    }
    
    {
      Pattern     pattern = new Pattern(new Pattern(new Pattern(vocab, vocab.getIDs(", y")), PrefixTree.X), vocab.getIDs("sus"));// del parlamento europeo"));
      int[]       terminalSequenceStartIndices = {17,21};
      int[]       sentenceNumbers = {0};
      int phraseIndex = 0;
      
      HierarchicalPhrases phrases =
        new HierarchicalPhrases(pattern, terminalSequenceStartIndices, sentenceNumbers);
      
      Assert.assertFalse(alignments.hasAlignedTerminal(12 , phrases, phraseIndex));
      Assert.assertFalse(alignments.hasAlignedTerminal(13 , phrases, phraseIndex));
      Assert.assertFalse(alignments.hasAlignedTerminal(14 , phrases, phraseIndex));


      Assert.assertTrue(alignments.hasAlignedTerminal(15, phrases, phraseIndex));
      Assert.assertTrue(alignments.hasAlignedTerminal(16, phrases, phraseIndex));
      
      Assert.assertFalse(alignments.hasAlignedTerminal(17, phrases, phraseIndex));
      Assert.assertFalse(alignments.hasAlignedTerminal(18, phrases, phraseIndex));
      Assert.assertFalse(alignments.hasAlignedTerminal(19, phrases, phraseIndex));
      Assert.assertFalse(alignments.hasAlignedTerminal(20, phrases, phraseIndex));
      Assert.assertFalse(alignments.hasAlignedTerminal(21, phrases, phraseIndex));
      Assert.assertFalse(alignments.hasAlignedTerminal(22, phrases, phraseIndex));
      Assert.assertFalse(alignments.hasAlignedTerminal(23, phrases, phraseIndex));
      
      Assert.assertTrue(alignments.hasAlignedTerminal(24, phrases, phraseIndex));


      Assert.assertFalse(alignments.hasAlignedTerminal(25, phrases, phraseIndex));
      Assert.assertFalse(alignments.hasAlignedTerminal(26, phrases, phraseIndex));
      Assert.assertFalse(alignments.hasAlignedTerminal(27, phrases, phraseIndex));
      
    }
    
    {


      Pattern     pattern = new Pattern(new Pattern(new Pattern(vocab, vocab.getIDs(",")), PrefixTree.X), vocab.getIDs(", y"));// del parlamento europeo"));
      int[]       terminalSequenceStartIndices = {9,17};
      int[]       sentenceNumbers = {0};
      int phraseIndex = 0;
      
      HierarchicalPhrases phrases =
        new HierarchicalPhrases(pattern, terminalSequenceStartIndices, sentenceNumbers);
      
      Assert.assertFalse(alignments.hasAlignedTerminal(9, phrases, phraseIndex));
      
      Assert.assertFalse(alignments.hasAlignedTerminal(10 , phrases, phraseIndex));
      Assert.assertFalse(alignments.hasAlignedTerminal(11 , phrases, phraseIndex));

View Full Code Here

    // "das macht ihn und es besch\u00E4digt ihn , es setzt ihn auf und es f\u00FChrt ihn aus ."


    int phraseIndex = 0;


    {
      HierarchicalPhrases phrases = getSourcePhrase("it", 0, 1);
      HierarchicalPhrase targetPhrase = getTargetPhrase("das", 0, 1);
      float sourceGivenTarget = lexProbs.lexProbSourceGivenTarget(phrases, phraseIndex, targetPhrase);
      float targetGivenSource = lexProbs.lexProbTargetGivenSource(phrases, phraseIndex, targetPhrase);
      Assert.assertEquals(sourceGivenTarget, 1.0f);  // lex P(it | das)
      Assert.assertEquals(targetGivenSource, 0.25f);// lex P(das | it)
    }


    {
      HierarchicalPhrases phrases = getSourcePhrase("makes", 1, 2);
      HierarchicalPhrase targetPhrase = getTargetPhrase("macht", 1, 2);
      float sourceGivenTarget = lexProbs.lexProbSourceGivenTarget(phrases, phraseIndex, targetPhrase);
      float targetGivenSource = lexProbs.lexProbTargetGivenSource(phrases, phraseIndex, targetPhrase);
      Assert.assertEquals(sourceGivenTarget, 1.0f); // lex P(makes | macht)
      Assert.assertEquals(targetGivenSource, 1.0f);// lex P(macht | makes)
    }


    {
      HierarchicalPhrases phrases = getSourcePhrase("him", 2, 3);
      HierarchicalPhrase targetPhrase = getTargetPhrase("ihn", 2, 3);
      float sourceGivenTarget = lexProbs.lexProbSourceGivenTarget(phrases, phraseIndex, targetPhrase);
      float targetGivenSource = lexProbs.lexProbTargetGivenSource(phrases, phraseIndex, targetPhrase);
      Assert.assertEquals(sourceGivenTarget, 1.0f);
      Assert.assertEquals(targetGivenSource, 1.0f);  
    }


    {
      HierarchicalPhrases phrases = getSourcePhrase("and", 3, 4);
      HierarchicalPhrase targetPhrase = getTargetPhrase("und", 3, 4);
      float sourceGivenTarget = lexProbs.lexProbSourceGivenTarget(phrases, phraseIndex, targetPhrase);
      float targetGivenSource = lexProbs.lexProbTargetGivenSource(phrases, phraseIndex, targetPhrase);
      Assert.assertEquals(sourceGivenTarget, 0.5f); // P(and | und)
      Assert.assertEquals(targetGivenSource, 1.0f);// P(und | and)
    }


    {
      HierarchicalPhrases phrases = getSourcePhrase("it", 4, 5);
      HierarchicalPhrase targetPhrase = getTargetPhrase("es", 4, 5);
      float sourceGivenTarget = lexProbs.lexProbSourceGivenTarget(phrases, phraseIndex, targetPhrase);
      float targetGivenSource = lexProbs.lexProbTargetGivenSource(phrases, phraseIndex, targetPhrase);
      Assert.assertEquals(sourceGivenTarget, 1.0f);  // lex P(it | es)
      Assert.assertEquals(targetGivenSource, 0.75f);// lex P(es | it)
    }


    {
      HierarchicalPhrases phrases = getSourcePhrase("mars", 5, 6);
      HierarchicalPhrase targetPhrase = getTargetPhrase("besch\u00E4digt", 5, 6);
      float sourceGivenTarget = lexProbs.lexProbSourceGivenTarget(phrases, phraseIndex, targetPhrase);
      float targetGivenSource = lexProbs.lexProbTargetGivenSource(phrases, phraseIndex, targetPhrase);
      Assert.assertEquals(sourceGivenTarget, 1.0f);
      Assert.assertEquals(targetGivenSource, 1.0f);  
    }


    {
      HierarchicalPhrases phrases = getSourcePhrase("him", 6, 7);
      HierarchicalPhrase targetPhrase = getTargetPhrase("ihn", 6, 7);
      float sourceGivenTarget = lexProbs.lexProbSourceGivenTarget(phrases, phraseIndex, targetPhrase);
      float targetGivenSource = lexProbs.lexProbTargetGivenSource(phrases, phraseIndex, targetPhrase);
      Assert.assertEquals(sourceGivenTarget, 1.0f);
      Assert.assertEquals(targetGivenSource, 1.0f);  
    }


    {
      HierarchicalPhrases phrases = getSourcePhrase(",", 7, 8);
      HierarchicalPhrase targetPhrase = getTargetPhrase(",", 7, 8);
      float sourceGivenTarget = lexProbs.lexProbSourceGivenTarget(phrases, phraseIndex, targetPhrase);
      float targetGivenSource = lexProbs.lexProbTargetGivenSource(phrases, phraseIndex, targetPhrase);
      Assert.assertEquals(sourceGivenTarget, 1.0f);
      Assert.assertEquals(targetGivenSource, 1.0f);  
    }


    {
      HierarchicalPhrases phrases = getSourcePhrase("it", 8, 9);
      HierarchicalPhrase targetPhrase = getTargetPhrase("es", 8, 9);
      float sourceGivenTarget = lexProbs.lexProbSourceGivenTarget(phrases, phraseIndex, targetPhrase);
      float targetGivenSource = lexProbs.lexProbTargetGivenSource(phrases, phraseIndex, targetPhrase);
      Assert.assertEquals(sourceGivenTarget, 1.0f);  // lex P(it | es)
      Assert.assertEquals(targetGivenSource, 0.75f);// lex P(es | it)
    }


    {
      HierarchicalPhrases phrases = getSourcePhrase("sets", 9, 10);
      HierarchicalPhrase targetPhrase = getTargetPhrase("setzt", 9, 10);
      float sourceGivenTarget = lexProbs.lexProbSourceGivenTarget(phrases, phraseIndex, targetPhrase);
      float targetGivenSource = lexProbs.lexProbTargetGivenSource(phrases, phraseIndex, targetPhrase);
      Assert.assertEquals(sourceGivenTarget, 1.0f);
      Assert.assertEquals(targetGivenSource, 1.0f);  
    }


    {
      HierarchicalPhrases phrases = getSourcePhrase("him", 10, 11);
      HierarchicalPhrase targetPhrase = getTargetPhrase("ihn", 10, 11);
      float sourceGivenTarget = lexProbs.lexProbSourceGivenTarget(phrases, phraseIndex, targetPhrase);
      float targetGivenSource = lexProbs.lexProbTargetGivenSource(phrases, phraseIndex, targetPhrase);
      Assert.assertEquals(sourceGivenTarget, 1.0f);
      Assert.assertEquals(targetGivenSource, 1.0f);  
    }


    {
      HierarchicalPhrases phrases = getSourcePhrase("on", 11, 12);
      HierarchicalPhrase targetPhrase = getTargetPhrase("auf", 11, 12);
      float sourceGivenTarget = lexProbs.lexProbSourceGivenTarget(phrases, phraseIndex, targetPhrase);
      float targetGivenSource = lexProbs.lexProbTargetGivenSource(phrases, phraseIndex, targetPhrase);
      Assert.assertEquals(sourceGivenTarget, 1.0f);
      Assert.assertEquals(targetGivenSource, 1.0f);  
    }


    {
      HierarchicalPhrases phrases = getSourcePhrase("yet", 12, 13);
      HierarchicalPhrase targetPhrase = getTargetPhrase("und", 12, 13);
      float sourceGivenTarget = lexProbs.lexProbSourceGivenTarget(phrases, phraseIndex, targetPhrase);
      float targetGivenSource = lexProbs.lexProbTargetGivenSource(phrases, phraseIndex, targetPhrase);
      Assert.assertEquals(sourceGivenTarget, 0.5f); // P(yet | und)
      Assert.assertEquals(targetGivenSource, 1.0f);// P(und | yet)
    }


    {
      HierarchicalPhrases phrases = getSourcePhrase("it", 13, 14);
      HierarchicalPhrase targetPhrase = getTargetPhrase("es", 13, 14);
      float sourceGivenTarget = lexProbs.lexProbSourceGivenTarget(phrases, phraseIndex, targetPhrase);
      float targetGivenSource = lexProbs.lexProbTargetGivenSource(phrases, phraseIndex, targetPhrase);
      Assert.assertEquals(sourceGivenTarget, 1.0f);  // lex P(it | es)
      Assert.assertEquals(targetGivenSource, 0.75f);// lex P(es | it)
    }


    {
      HierarchicalPhrases phrases = getSourcePhrase("takes", 14, 15);
      HierarchicalPhrase targetPhrase = getTargetPhrase("f\u00FChrt", 14, 15);
      float sourceGivenTarget = lexProbs.lexProbSourceGivenTarget(phrases, phraseIndex, targetPhrase);
      float targetGivenSource = lexProbs.lexProbTargetGivenSource(phrases, phraseIndex, targetPhrase);
      Assert.assertEquals(sourceGivenTarget, 1.0f);
      Assert.assertEquals(targetGivenSource, 1.0f);  
    }


    {
      HierarchicalPhrases phrases = getSourcePhrase("him", 15, 16);
      HierarchicalPhrase targetPhrase = getTargetPhrase("ihn", 15, 16);
      float sourceGivenTarget = lexProbs.lexProbSourceGivenTarget(phrases, phraseIndex, targetPhrase);
      float targetGivenSource = lexProbs.lexProbTargetGivenSource(phrases, phraseIndex, targetPhrase);
      Assert.assertEquals(sourceGivenTarget, 1.0f);
      Assert.assertEquals(targetGivenSource, 1.0f);  
    }


    {
      HierarchicalPhrases phrases = getSourcePhrase("off", 16, 17);
      HierarchicalPhrase targetPhrase = getTargetPhrase("aus", 16, 17);
      float sourceGivenTarget = lexProbs.lexProbSourceGivenTarget(phrases, phraseIndex, targetPhrase);
      float targetGivenSource = lexProbs.lexProbTargetGivenSource(phrases, phraseIndex, targetPhrase);
      Assert.assertEquals(sourceGivenTarget, 1.0f);
      Assert.assertEquals(targetGivenSource, 1.0f);  
    }


    {
      HierarchicalPhrases phrases = getSourcePhrase(".", 17, 18);
      HierarchicalPhrase targetPhrase = getTargetPhrase(".", 17, 18);
      float sourceGivenTarget = lexProbs.lexProbSourceGivenTarget(phrases, phraseIndex, targetPhrase);
      float targetGivenSource = lexProbs.lexProbTargetGivenSource(phrases, phraseIndex, targetPhrase);
      Assert.assertEquals(sourceGivenTarget, 1.0f);
      Assert.assertEquals(targetGivenSource, 1.0f);  
    }


    ///////////


    {
      HierarchicalPhrases phrases = getSourcePhrase("yet it", 12, 14);
      HierarchicalPhrase targetPhrase = getTargetPhrase("und es", 12, 14);
      float sourceGivenTarget = lexProbs.lexProbSourceGivenTarget(phrases, phraseIndex, targetPhrase);
      float targetGivenSource = lexProbs.lexProbTargetGivenSource(phrases, phraseIndex, targetPhrase);
      Assert.assertEquals(sourceGivenTarget, 0.5f * 1.0f);  // lex P(yet it | und es)
      Assert.assertEquals(targetGivenSource, 1.0f * 0.75f);// lex P(und es | yet it)
    }


    ///////////


    {
      HierarchicalPhrases phrases = getSourcePhrase("of the session", 19, 22);
      HierarchicalPhrase targetPhrase = getTargetPhrase("der sitzungsperiode", 19, 21);
      float sourceGivenTarget = lexProbs.lexProbSourceGivenTarget(phrases, phraseIndex, targetPhrase);
      float targetGivenSource = lexProbs.lexProbTargetGivenSource(phrases, phraseIndex, targetPhrase);
      Assert.assertEquals(sourceGivenTarget, 0.5f * 0.5f * 1.0f);  // lex P(of the session | der sitzungsperiode)
      Assert.assertEquals(targetGivenSource, 0.5f*((1.0f/3.0f) + (1.0f/3.0f)) * (1.0f/3.0f));// lex P(der sitzungsperiode | of the session)
    }


    {
      HierarchicalPhrases phrases = getSourcePhrase("thunder ; lightning", 29, 32);
      HierarchicalPhrase targetPhrase = getTargetPhrase("blitzen", 28, 29);
      float sourceGivenTarget = lexProbs.lexProbSourceGivenTarget(phrases, phraseIndex, targetPhrase);
      float targetGivenSource = lexProbs.lexProbTargetGivenSource(phrases, phraseIndex, targetPhrase);
      Assert.assertEquals(sourceGivenTarget, 0.5f * (1.0f/3.0f) * 0.5f);  // lex P(thunder ; lightning | blitzen)
      Assert.assertEquals(targetGivenSource, ((1.0f/2.0f) * (1.0f + 1.0f)));// lex P(blitzen | thunder ; lightning)

View Full Code Here

//        terminalSequenceStartIndices,
//        terminalSequenceEndIndices,
//        sourceCorpusArray,
//        terminalSequenceEndIndices[terminalSequenceEndIndices.length-1] - terminalSequenceStartIndices[0]);
    
    HierarchicalPhrases phrases =
      new HierarchicalPhrases(pattern, terminalSequenceStartIndices, sentenceNumbers);
    
//    Pair<Float,Float> results;
    
    //  "it makes him and it mars him , it sets him on yet it takes him off .";
    // "das macht ihn und es besch\u00E4digt ihn , es setzt ihn auf und es f\u00FChrt ihn aus ."

View Full Code Here

TOP

Related Classes of joshua.corpus.suffix_array.HierarchicalPhrases

joshua.corpus.alignment.AlignmentsTest

joshua.corpus.lexprob.BetterLexProbsTest

joshua.corpus.suffix_array.HierarchicalPhraseTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.