Examples of SuffixArray

com.googlecode.gaal.suffix.api.SuffixArray
joshua.corpus.suffix_array.SuffixArray
SuffixArray is the main class for producing suffix arrays from corpora, and manipulating them once created. Suffix arrays are a space economical way of storing a corpus and allowing very quick searching of any substring within the corpus. A suffix array contains a list of references to every point in a corpus, and each reference denotes the suffix starting at that point and continuing to the end of the corpus. The suffix array is sorted alphabetically, so any substring within the corpus can be found with a binary search in O(log n) time, where n is the length of the corpus. @author Colin Bannard @since 10 December 2004 @author Josh Schroeder @since 2 Jan 2005 @author Chris Callison-Burch @since 9 February 2005 @version $LastChangedDate:2008-07-30 17:15:52 -0400 (Wed, 30 Jul 2008) $

Examples of com.googlecode.gaal.suffix.api.SuffixArray

    }


    private void visualizeInverseSuffixTable(String fileName, String caption, String label) throws IOException {
        buffer.append(String.format(LATEX_INPUT, fileName));
        Writer out = new FileWriter(DOC_PATH + fileName);
        SuffixArray sa = new LinearizedSuffixTreeImpl(corpus.sequence(), corpus.alphabetSize());
        LaTeXVisualizer.visualizeInverseSuffixTable(out, caption, label, sa, corpus);
        out.flush();
        out.close();
    }

View Full Code Here

Examples of joshua.corpus.suffix_array.SuffixArray

      alignmentFileName = alignmentFile.getAbsolutePath();
    }
    
    CorpusArray sourceCorpusArray =
      SuffixArrayFactory.createCorpusArray(sourceFileName);
    SuffixArray sourceSuffixArray = 
      SuffixArrayFactory.createSuffixArray(sourceCorpusArray, SuffixArray.DEFAULT_CACHE_CAPACITY);
    
    CorpusArray targetCorpusArray =
      SuffixArrayFactory.createCorpusArray(targetFileName);
    SuffixArray targetSuffixArray = 
      SuffixArrayFactory.createSuffixArray(targetCorpusArray, SuffixArray.DEFAULT_CACHE_CAPACITY);


    Alignments alignmentArray = SuffixArrayFactory.createAlignments(alignmentFileName, sourceSuffixArray, targetSuffixArray);


    return new SampledLexProbs(Integer.MAX_VALUE, sourceSuffixArray, targetSuffixArray, alignmentArray, Cache.DEFAULT_CAPACITY, false);

View Full Code Here

Examples of joshua.corpus.suffix_array.SuffixArray

      }
    }
    Alignments alignments = new AlignmentArray(alignedTargetIndices, alignedSourceIndices, 1);
    
    CorpusArray targetCorpus = new CorpusArray(sentenceF, sentenceStartPositions, vocab);
    SuffixArray targetSuffixes = new SuffixArray(targetCorpus);


    CorpusArray sourceCorpus = new CorpusArray(sentence, sentenceStartPositions, vocab);
    SuffixArray sourceSuffixes = new SuffixArray(sourceCorpus);
    ParallelCorpusGrammarFactory parallelCorpus = new ParallelCorpusGrammarFactory(sourceSuffixes, targetSuffixes, alignments, null, Integer.MAX_VALUE, maxPhraseSpan, maxPhraseLength, maxNonterminals, 2, Float.MIN_VALUE, JoshuaConfiguration.phrase_owner, JoshuaConfiguration.default_non_terminal, JoshuaConfiguration.oovFeatureCost);
    
//    tree = new PrefixTree(vocab, maxPhraseSpan, maxPhraseLength, maxNonterminals);
    tree = new PrefixTree(parallelCorpus);
    Assert.assertNotNull(tree);

View Full Code Here

Examples of joshua.corpus.suffix_array.SuffixArray

      for(int i = 0; i < corpusSentence.size(); i++) {
        corpus[i] = corpusSentence.getWordID(i);
      }
      
      CorpusArray corpusArray = new CorpusArray(corpus, sentenceStartPositions, sourceVocab);
      suffixArray = new SuffixArray(corpusArray);
      




      int[] targetSentenceStartPositions = {0};
      
      BasicPhrase targetCorpusSentence = new BasicPhrase(targetCorpusString, targetVocab);
      Assert.assertEquals(targetCorpusSentence.size(), 18);
      
      int[] targetCorpus = new int[targetCorpusSentence.size()];
      for(int i = 0; i < targetCorpusSentence.size(); i++) {
        targetCorpus[i] = targetCorpusSentence.getWordID(i);
      }
      


      


      
      CorpusArray targetCorpusArray = new CorpusArray(targetCorpus, targetSentenceStartPositions, targetVocab);
      targetSuffixArray = new SuffixArray(targetCorpusArray);


      
      int[] lowestAlignedTargetIndex = new int[corpusSentence.size()];
      int[] highestAlignedTargetIndex = new int[corpusSentence.size()];
      int[] lowestAlignedSourceIndex = new int[targetCorpusSentence.size()];

View Full Code Here

Examples of joshua.corpus.suffix_array.SuffixArray

    }
    
    CorpusArray corpusArray = new CorpusArray(corpus, sentences, vocab);
    
    if (binaryFileName==null || binaryFileName.trim().length()==0)
      suffixArray = new SuffixArray(corpusArray);
    else
      suffixArray = new MemoryMappedSuffixArray(binaryFileName, corpusArray, MemoryMappedSuffixArray.DEFAULT_CACHE_CAPACITY);
    
  }

View Full Code Here

Examples of joshua.corpus.suffix_array.SuffixArray

    }
    
    this.sourceCorpusArray =
      SuffixArrayFactory.createCorpusArray(sourceFileName);
    this.sourceVocab = (Vocabulary) sourceCorpusArray.getVocabulary();
    SuffixArray sourceSuffixArray =
      SuffixArrayFactory.createSuffixArray(sourceCorpusArray, SuffixArray.DEFAULT_CACHE_CAPACITY);


    this.targetCorpusArray =
      SuffixArrayFactory.createCorpusArray(targetFileName);
    this.targetVocab = (Vocabulary) targetCorpusArray.getVocabulary();
    SuffixArray targetSuffixArray = 
      SuffixArrayFactory.createSuffixArray(targetCorpusArray, SuffixArray.DEFAULT_CACHE_CAPACITY);


    this.alignmentArray =
      SuffixArrayFactory.createAlignments(alignmentFileName, sourceSuffixArray, targetSuffixArray);

View Full Code Here

Examples of joshua.corpus.suffix_array.SuffixArray

    
    targetCorpusArray =
      SuffixArrayFactory.createCorpusArray(targetFileName);


    if (alignmentsType.equals("AlignmentArray")) {
      SuffixArray targetSuffixArray = 
        SuffixArrayFactory.createSuffixArray(targetCorpusArray, SuffixArray.DEFAULT_CACHE_CAPACITY);
      SuffixArray sourceSuffixArray = 
        SuffixArrayFactory.createSuffixArray(sourceCorpusArray, SuffixArray.DEFAULT_CACHE_CAPACITY);
      alignments = 
        SuffixArrayFactory.createAlignments(alignmentFileName, sourceSuffixArray, targetSuffixArray);
    } else if (alignmentsType.equals("AlignmentGrids")) {
      alignments = new AlignmentGrids(new Scanner(new File(alignmentFileName)), sourceCorpusArray, targetCorpusArray, 3);

View Full Code Here

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.