Examples of Vocabulary

Examples of com.github.pmerienne.trident.ml.nlp.Vocabulary

25
26
27
28
29
30
31
32
33
34
35
36
37
38
public class VocabularyTest {

  @Test
  public void testCount() {
    Vocabulary vocabulary = new Vocabulary();
    vocabulary.add("only one");
    vocabulary.add("we are 2");
    vocabulary.add("we are 2");

    assertEquals(1, vocabulary.count("only one").intValue());
    assertEquals(2, vocabulary.count("we are 2").intValue());
    assertEquals(0, vocabulary.count("I'm not here").intValue());
  }
View Full Code Here

Examples of com.github.pmerienne.trident.ml.nlp.Vocabulary

37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
    assertEquals(0, vocabulary.count("I'm not here").intValue());
  }

  @Test
  public void testFrequency() {
    Vocabulary vocabulary = new Vocabulary();
    vocabulary.add("only one");
    vocabulary.add("we are 2");
    vocabulary.add("we are 2");
    vocabulary.add("I like kitten");

    assertEquals(0.25, vocabulary.frequency("only one"), 0.001);
    assertEquals(0.5, vocabulary.frequency("we are 2"), 0.001);
    assertEquals(0.0, vocabulary.frequency("I'm not here"), 0.001);
  }
View Full Code Here

Examples of com.github.pmerienne.trident.ml.nlp.Vocabulary

50
51
52
53
54
55
56
57
58
59
60
61
    assertEquals(0.0, vocabulary.frequency("I'm not here"), 0.001);
  }

  @Test
  public void testAddAll() {
    Vocabulary vocabulary = new Vocabulary();
    vocabulary.addAll(Arrays.asList("only one", "we are 2", "we are 2"));

    assertEquals(1, vocabulary.count("only one").intValue());
    assertEquals(2, vocabulary.count("we are 2").intValue());
    assertEquals(0, vocabulary.count("I'm not here").intValue());
  }
View Full Code Here

Examples of com.github.pmerienne.trident.ml.nlp.Vocabulary

61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
  }

  @Test
  public void testLimit() {
    // Given
    Vocabulary vocabulary = new Vocabulary();
    for (int i = 1; i < 6; i++) {
      for (int j = 0; j < i; j++) {
        vocabulary.add("we are " + i);
      }
    }

    // When
    vocabulary.limitWords(3);

    // Then
    assertEquals(3, vocabulary.wordCount().intValue());
    assertEquals(0, vocabulary.count("we are 1").intValue());
    assertEquals(0, vocabulary.count("we are 2").intValue());
    assertEquals(3, vocabulary.count("we are 3").intValue());
    assertEquals(4, vocabulary.count("we are 4").intValue());
    assertEquals(5, vocabulary.count("we are 5").intValue());

  }
View Full Code Here

Examples of com.luxoft.dnepr.courses.regular.unit14.model.Vocabulary

20
21
22
23
24
25
26
27
28
29
30
        initialize();
    }

    private void initialize() {
        try {
            vocabulary = new Vocabulary();
        } catch (IOException e) {
            System.out.println(String.format(ERROR_MESSAGE, Vocabulary.SONNETS_FILE));
        }
        anInterface = new ConsoleInterface(System.in, System.out);
    }
View Full Code Here

Examples of joshua.corpus.vocab.Vocabulary

75
76
77
78
79
80
81
82
83
84
85
        sourcePrintStream.println(sentence);
      }
      sourcePrintStream.close();
      String sourceCorpusFileName = sourceFile.getAbsolutePath();
     
      Vocabulary symbolTable = new Vocabulary();
      int[] sourceLengths = Vocabulary.initializeVocabulary(sourceCorpusFileName, symbolTable, true);
      Assert.assertEquals(sourceLengths.length, 2);
      int numberOfSentences = sourceLengths[1];
     
      Corpus sourceCorpus = SuffixArrayFactory.createCorpusArray(sourceCorpusFileName, symbolTable, sourceLengths[0], sourceLengths[1]);
View Full Code Here

Examples of joshua.corpus.vocab.Vocabulary

79
80
81
82
83
84
85
86
87
88
89
    this.vocab = vocab;
  }

  public ArpaFile(String arpaFileName) throws IOException {
    this.arpaFile = new File(arpaFileName);
    this.vocab = new Vocabulary();
   
//    final Scanner scanner = new Scanner(arpaFile);
   
//    // Eat initial header lines
//    while (scanner.hasNextLine()) {
View Full Code Here

Examples of joshua.corpus.vocab.Vocabulary

220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
   
    ////////////////////////////////
    // Common vocabulary          //
    ////////////////////////////////
    if (logger.isLoggable(Level.INFO)) logger.info("Constructing empty common vocabulary");
    Vocabulary commonVocab = new Vocabulary();
    int numSourceWords, numSourceSentences;
    int numTargetWords, numTargetSentences;
    String binaryCommonVocabFileName = this.commonVocabFileName;
    if (binaryCorpus) {
      if (logger.isLoggable(Level.INFO)) logger.info("Initializing common vocabulary from binary file " + binaryCommonVocabFileName);
      ObjectInput in = BinaryIn.vocabulary(binaryCommonVocabFileName);
      commonVocab.readExternal(in);
     
      numSourceWords = Integer.MIN_VALUE;
      numSourceSentences = Integer.MIN_VALUE;
     
      numTargetWords = Integer.MIN_VALUE;
View Full Code Here

Examples of joshua.corpus.vocab.Vocabulary

86
87
88
89
90
91
92
93
94
95
96
        targetSuffixArray, alignments,
        lexProbs, models, sampleSize,
        maxPhraseSpan, maxPhraseLength,
        minNonterminalSpan, maxNonterminalSpan);
   
    SymbolTable vocab = new Vocabulary();
   
    Corpus corpus = suffixArray.getCorpus();
   
    NGramLanguageModel largeLM = new LMGrammarJAVA(
        vocab,
View Full Code Here

Examples of joshua.corpus.vocab.Vocabulary

59
60
61
62
63
64
65
66
67
68
69
70
71
72
    srilm.write_default_vocab_map(tmpFile.getAbsolutePath());
   
   
    // Create a vocabulary object from using the SRILM integer mappings
    Scanner scanner = new Scanner(tmpFile);
    Vocabulary vocab = Vocabulary.getVocabFromSRILM(scanner);
//    vocab.fixVocabulary();
   
   
    // Write the vocabulary to disk in binary format
    ObjectOutput out = new BinaryOut(outVocabFile);
    vocab.writeExternal(out);
   
  }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.