Package org.apache.mahout.common.nlp

Examples of org.apache.mahout.common.nlp.NGrams

72
73
74
75
76
77
78
79
80
81
82
  @Override
  public void map(Text key, Text value,
                  OutputCollector<StringTuple,DoubleWritable> output, Reporter reporter) throws IOException {
    String label = key.toString();
   
    List<String> ngrams = new NGrams(value.toString(), gramSize).generateNGramsWithoutLabel();
   
    try {
      ClassifierResult result = classifier.classifyDocument(ngrams.toArray(new String[ngrams.size()]),
        defaultCategory);
     
View Full Code Here
250
251
252
253
254
255
256
257
258
259
260
       
        long lineNum = 0;
        for (String line : new FileLineIterable(new File(file.getPath()), Charset.forName(params
            .get("encoding")), false)) {
         
          Map<String,List<String>> document = new NGrams(line, Integer.parseInt(params.get("gramSize")))
              .generateNGrams();
          for (Map.Entry<String,List<String>> stringListEntry : document.entrySet()) {
            String correctLabel = stringListEntry.getKey();
            List<String> strings = stringListEntry.getValue();
            TimingStatistics.Call call = operationStats.newCall();
View Full Code Here
86
87
88
89
90
91
92
93
94
95
96
    ClassifierContext classifier = new ClassifierContext(algorithm, datastore);
    classifier.initialize();
    ResultAnalyzer resultAnalyzer = new ResultAnalyzer(classifier.getLabels(), params.get("defaultCat"));
   
    for (String[] entry : ClassifierData.DATA) {
      List<String> document = new NGrams(entry[1], Integer.parseInt(params.get("gramSize")))
          .generateNGramsWithoutLabel();
      assertEquals(3, classifier.classifyDocument(document.toArray(new String[document.size()]),
        params.get("defaultCat"), 100).length);
      ClassifierResult result = classifier.classifyDocument(document.toArray(new String[document.size()]), params
          .get("defaultCat"));
View Full Code Here
137
138
139
140
141
142
143
144
145
146
147
    Datastore datastore = new InMemoryBayesDatastore(params);
    ClassifierContext classifier = new ClassifierContext(algorithm, datastore);
    classifier.initialize();
    ResultAnalyzer resultAnalyzer = new ResultAnalyzer(classifier.getLabels(), params.get("defaultCat"));
    for (String[] entry : ClassifierData.DATA) {
      List<String> document = new NGrams(entry[1], Integer.parseInt(params.get("gramSize")))
          .generateNGramsWithoutLabel();
      assertEquals(3, classifier.classifyDocument(document.toArray(new String[document.size()]),
        params.get("defaultCat"), 100).length);
      ClassifierResult result = classifier.classifyDocument(document.toArray(new String[document.size()]), params
          .get("defaultCat"));
View Full Code Here
174
175
176
177
178
179
180
181
182
183
184
    StringBuilder line = new StringBuilder();
    for (String token : document) {
      line.append(token).append(' ');
    }
   
    List<String> doc = new NGrams(line.toString(), gramSize).generateNGramsWithoutLabel();
   
    log.info("Done converting");
    log.info("Classifying document: {}", docPath);
    ClassifierResult category = classifier.classifyDocument(doc.toArray(new String[doc.size()]), defaultCat);
    log.info("Category for {} is {}", docPath, category);
View Full Code Here
64
65
66
67
68
69
70
71
72
73
74
    //String line = value.toString();
    String label = key.toString();

    Map<String, int[]> wordList = new HashMap<String, int[]>(1000);

    List<String> ngrams  = new NGrams(value.toString(), gramSize).generateNGramsWithoutLabel();

    for (String ngram : ngrams) {
      int[] count = wordList.get(ngram);
      if (count == null) {
        count = new int[1];
View Full Code Here
73
74
75
76
77
78
79
80
81
82
83

    //StringBuilder builder = new StringBuilder(label);
    //builder.ensureCapacity(32);// make sure we have a reasonably size buffer to
                               // begin with
    List<String> ngrams  = new NGrams(value.toString(), gramSize).generateNGramsWithoutLabel();
   
    try {
      ClassifierResult result = classifier.classifyDocument( ngrams
          .toArray(new String[ngrams.size()]), defaultCategory);
    
View Full Code Here
272
273
274
275
276
277
278
279
280
281
282
        long lineNum = 0;
        for (String line : new FileLineIterable(new File(file.getPath()),
            Charset.forName(params.get("encoding")), false)) {

          Map<String, List<String>> document = new NGrams(line, Integer
              .parseInt(params.get("gramSize"))).generateNGrams();
          for (Map.Entry<String, List<String>> stringListEntry : document
              .entrySet()) {
            List<String> strings = stringListEntry.getValue();
            TimingStatistics.Call call = operationStats.newCall();
View Full Code Here
207
208
209
210
211
212
213
214
215
216
217
    StringBuilder line = new StringBuilder();
    for (String token : document) {
      line.append(token).append(' ');
    }

    List<String> doc = new NGrams(line.toString(), gramSize)
        .generateNGramsWithoutLabel();

    log.info("Done converting");
    log.info("Classifying document: {}", docPath);
    ClassifierResult category = classifier.classifyDocument(doc
View Full Code Here
161
162
163
164
165
166
167
168
169
170
171
    StringBuilder line = new StringBuilder();
    for (String token : document) {
      line.append(token).append(' ');
    }
   
    List<String> doc = new NGrams(line.toString(), gramSize).generateNGramsWithoutLabel();
   
    log.info("Done converting");
    log.info("Classifying document: {}", docPath);
    ClassifierResult category = classifier.classifyDocument(doc.toArray(new String[doc.size()]), defaultCat);
    log.info("Category for {} is {}", docPath, category);
View Full Code Here
TOP

Related Classes of org.apache.mahout.common.nlp.NGrams

  • org.apache.mahout.classifier.bayes.BayesClassifierSelfTest
  • org.apache.mahout.classifier.bayes.mapreduce.bayes.BayesClassifierMapper
  • org.apache.mahout.classifier.bayes.mapreduce.common.BayesFeatureMapper
  • org.apache.mahout.classifier.bayes.TestClassifier
  • org.apache.mahout.classifier.Classify
  • java.util.StringTokenizer
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.