Examples of joshua.corpus.suffix_array.mm.MemoryMappedSuffixArray

joshua.corpus.suffix_array.mm.MemoryMappedSuffixArray

    //////////////////////////////////
    Suffixes sourceSuffixArray;
    String binarySourceSuffixArrayFileName = sourceSuffixesFileName;
    if (binaryCorpus) {
      if (logger.isLoggable(Level.INFO)) logger.info("Constructing source language suffix array from binary file " + binarySourceSuffixArrayFileName);
      sourceSuffixArray = new MemoryMappedSuffixArray(binarySourceSuffixArrayFileName, sourceCorpusArray, cacheSize);
    } else {
      if (logger.isLoggable(Level.INFO)) logger.info("Constructing source language suffix array from source corpus.");
      sourceSuffixArray = SuffixArrayFactory.createSuffixArray(sourceCorpusArray, cacheSize);
    }
    
    


        
    //////////////////////////////////
    // Target language corpus array //
    //////////////////////////////////
    final Corpus targetCorpusArray;
    if (binaryCorpus) {
      if (logger.isLoggable(Level.INFO)) logger.info("Constructing memory mapped target language corpus array.");
      targetCorpusArray = new MemoryMappedCorpusArray(commonVocab, targetFileName);
    } else {
      if (logger.isLoggable(Level.INFO)) logger.info("Constructing target language corpus array.");
      targetCorpusArray = SuffixArrayFactory.createCorpusArray(targetFileName, commonVocab, numTargetWords, numTargetSentences);
    }
    


    //////////////////////////////////
    // Target language suffix array //
    //////////////////////////////////
    Suffixes targetSuffixArray;
    String binaryTargetSuffixArrayFileName = targetSuffixesFileName;
    if (binaryCorpus) {
      if (logger.isLoggable(Level.INFO)) logger.info("Constructing target language suffix array from binary file " + binaryTargetSuffixArrayFileName);
      targetSuffixArray = new MemoryMappedSuffixArray(binaryTargetSuffixArrayFileName, targetCorpusArray, cacheSize);
    } else {
      if (logger.isLoggable(Level.INFO)) logger.info("Constructing target language suffix array from target corpus.");
      targetSuffixArray = SuffixArrayFactory.createSuffixArray(targetCorpusArray, cacheSize);
    }

View Full Code Here

    
    if (logger.isLoggable(Level.INFO))
      logger.info("Reading source language suffix array from " +
        binarySourceSuffixesFileName);
    Suffixes sourceSuffixArray =
      new MemoryMappedSuffixArray(
          binarySourceSuffixesFileName,
          sourceCorpusArray,
          maxCacheSize);


    
    if (logger.isLoggable(Level.INFO))
      logger.info("Reading target language corpus from " +
        binaryTargetCorpusFileName);
    Corpus targetCorpusArray =
      new MemoryMappedCorpusArray(
        this.symbolTable, binaryTargetCorpusFileName);
    
    if (logger.isLoggable(Level.INFO))
      logger.info("Reading target language suffix array from " +
        binaryTargetSuffixesFileName);
    Suffixes targetSuffixArray =
      new MemoryMappedSuffixArray(
          binaryTargetSuffixesFileName,
          targetCorpusArray,
          maxCacheSize);
    
    String binaryAlignmentFileName =

View Full Code Here


          logger.fine("Loading source corpus...");
          Corpus sourceCorpus = new MemoryMappedCorpusArray(commonVocab, binarySourceFileName);


          logger.fine("Loading source suffix array...");
          Suffixes sourceSuffixes = new MemoryMappedSuffixArray(binarySourceSuffixesFileName, sourceCorpus);
          
          logger.fine("Loading target corpus...");    
          Corpus targetCorpus = new MemoryMappedCorpusArray(commonVocab, binaryTargetFileName);
          
          logger.fine("Loading target suffix array...");
          Suffixes targetSuffixes = new MemoryMappedSuffixArray(binarySourceSuffixesFileName, sourceCorpus);


          logger.fine("Loading alignment grids...");
          Alignments alignments = new MemoryMappedAlignmentGrids(binaryAlignmentFileName, sourceCorpus, targetCorpus);
          
//          ParallelCorpusGrammarFactory parallelCorpus = new ParallelCorpusGrammarFactory(

View Full Code Here

      logger.info("Constructing corpus array from file " + binaryCorpusFileName);
      if (logger.isLoggable(Level.INFO)) logger.info("Constructing memory mapped source language corpus array.");
      corpusArray = new MemoryMappedCorpusArray(symbolTable, binaryCorpusFileName);


      logger.info("Constructing suffix array from file " + binarySuffixArrayFileName);
      suffixArray = new MemoryMappedSuffixArray(binarySuffixArrayFileName, corpusArray, Cache.DEFAULT_CAPACITY);




    } else {


      System.err.println("Usage: java " + SuffixArray.class.getName() + " source.vocab source.corpus source.suffixes");

View Full Code Here

    CorpusArray corpusArray = new CorpusArray(corpus, sentences, vocab);
    
    if (binaryFileName==null || binaryFileName.trim().length()==0)
      suffixArray = new SuffixArray(corpusArray);
    else
      suffixArray = new MemoryMappedSuffixArray(binaryFileName, corpusArray, MemoryMappedSuffixArray.DEFAULT_CACHE_CAPACITY);
    
  }

View Full Code Here

TOP

Related Classes of joshua.corpus.suffix_array.mm.MemoryMappedSuffixArray

joshua.corpus.suffix_array.FrequentPhrases

joshua.corpus.suffix_array.SuffixArrayTest

joshua.decoder.JoshuaDecoder

joshua.prefix_tree.ExtractRules

joshua.ui.alignment.GridViewer

java.nio.channels.FileChannel

java.nio.IntBuffer

joshua.corpus.suffix_array.MismatchedCorpusException

java.io.RandomAccessFile

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.