Package org.apache.mahout.common

Examples of org.apache.mahout.common.StringTuple

116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
                                                                   PathType.GLOB,
                                                                   null,
                                                                   null,
                                                                   true,
                                                                   conf)) {
      StringTuple key = record.getFirst();
      DoubleWritable value = record.getSecond();
      // Sum of weights in a Label
      if (key.stringAt(0).equals(BayesConstants.LABEL_THETA_NORMALIZER)) {
        datastore.setThetaNormalizer(key.stringAt(1), value.get());
        if (++count % 50000 == 0) {
          log.info("Read {} theta norms", count);
        }
      }
    }
View Full Code Here
137
138
139
140
141
142
143
144
145
146
147
148
149
                                                                   PathType.GLOB,
                                                                   null,
                                                                   null,
                                                                   true,
                                                                   conf)) {
      StringTuple key = record.getFirst();
      DoubleWritable value = record.getSecond();
      if (key.stringAt(0).equals(BayesConstants.TOTAL_SUM)) {
        // Sum of weights for all Features and all Labels
        datastore.setSigmaJSigmaK(value.get());
        log.info("{}", value.get());
      }
    }
View Full Code Here
157
158
159
160
161
162
163
164
165
166
167
168
169
170
                                                                   PathType.GLOB,
                                                                   null,
                                                                   null,
                                                                   true,
                                                                   conf)) {
      StringTuple key = record.getFirst();
      DoubleWritable value = record.getSecond();
      if (key.stringAt(0).equals(BayesConstants.LABEL_SUM)) {
        // Sum of counts of labels
        labelSum.put(key.stringAt(1), value.get());
      }
    }
    return labelSum;
  }
View Full Code Here
177
178
179
180
181
182
183
184
185
186
187
188
189
190
                                                                   PathType.GLOB,
                                                                   null,
                                                                   null,
                                                                   true,
                                                                   conf)) {
      StringTuple key = record.getFirst();
      DoubleWritable value = record.getSecond();
      // Count of Documents in a Label
      if (key.stringAt(0).equals(BayesConstants.LABEL_COUNT)) {
        labelDocumentCounts.put(key.stringAt(1), value.get());
      }
    }
    return labelDocumentCounts;
  }
View Full Code Here
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
                                                                   PathType.GLOB,
                                                                   null,
                                                                   null,
                                                                   true,
                                                                   conf)) {
      StringTuple key = record.getFirst();
      DoubleWritable value = record.getSecond();
      if (weightSum.size() > 1) {
        throw new IllegalStateException("Incorrect Sum File");
      } else if (key.stringAt(0).equals(BayesConstants.TOTAL_SUM)) {
        weightSum.put(BayesConstants.TOTAL_SUM, value.get());
      }
    }
    return weightSum.get(BayesConstants.TOTAL_SUM);
  }
View Full Code Here
219
220
221
222
223
224
225
226
227
228
229
230
231
                                                                   true,
                                                                   conf)) {
      if (weightSum.size() > 1) {
        throw new IllegalStateException("Incorrect vocabCount File");
      }
      StringTuple key = record.getFirst();
      DoubleWritable value = record.getSecond();
      if (key.stringAt(0).equals(BayesConstants.FEATURE_SET_SIZE)) {
        weightSum.put(BayesConstants.FEATURE_SET_SIZE, value.get());
      }
    }
    return weightSum.get(BayesConstants.FEATURE_SET_SIZE);
  }
View Full Code Here
93
94
95
96
97
98
99
100
101
102
103
104
105
106
                                                                   PathType.GLOB,
                                                                   null,
                                                                   null,
                                                                   true,
                                                                   conf)) {
      StringTuple key = record.getFirst();
      DoubleWritable value = record.getSecond();
      String correctLabel = key.stringAt(1);
      String classifiedLabel = key.stringAt(2);
      Map<String,Integer> rowMatrix = confusionMatrix.get(correctLabel);
      if (rowMatrix == null) {
        rowMatrix = new HashMap<String,Integer>();
      }
      Integer count = Double.valueOf(value.get()).intValue();
View Full Code Here
67
68
69
70
71
72
73
74
75
    String label = key.stringAt(1);
   
    reporter.setStatus("Bayes Theta Normalizer Mapper: " + label);
   
    double weight = Math.log((value.get() + alphaI) / (labelWeightSum.get(label) + vocabCount));
    StringTuple thetaNormalizerTuple = new StringTuple(BayesConstants.LABEL_THETA_NORMALIZER);
    thetaNormalizerTuple.add(label);
    output.collect(thetaNormalizerTuple, new DoubleWritable(weight));
  }
View Full Code Here
57
58
59
60
61
62
63
64
65
66
67
68
69
70
    FileStatus[] statuses = fs.listStatus(output);
    assertEquals(1, statuses.length);
    Path filePath = statuses[0].getPath();
    SequenceFile.Reader reader = new SequenceFile.Reader(fs, filePath, configuration);
    Text key = reader.getKeyClass().asSubclass(Text.class).newInstance();
    StringTuple value = reader.getValueClass().asSubclass(StringTuple.class).newInstance();

    reader.next(key, value);
    assertEquals(documentId1, key.toString());
    assertEquals(Arrays.asList("test", "document", "processor"), value.getEntries());
    reader.next(key, value);
    assertEquals(documentId2, key.toString());
    assertEquals(Arrays.asList("another", "one"), value.getEntries());
  }
View Full Code Here
39
40
41
42
43
44
45
46
47
48
49
50
51
  @Override
  protected void map(Text key, Text value, Context context) throws IOException, InterruptedException {
    TokenStream stream = analyzer.tokenStream(key.toString(), new StringReader(value.toString()));
    CharTermAttribute termAtt = stream.addAttribute(CharTermAttribute.class);
    StringTuple document = new StringTuple();
    while (stream.incrementToken()) {
      if (termAtt.length() > 0) {
        document.add(new String(termAtt.buffer(), 0, termAtt.length()));
      }
    }
    context.write(key, document);
  }
View Full Code Here
TOP

Related Classes of org.apache.mahout.common.StringTuple

  • com.digitalpebble.behemoth.mahout.BehemothTokenizerMapper
  • com.digitalpebble.behemoth.mahout.LuceneTokenizerMapper
  • org.apache.hadoop.io.Text
  • org.apache.mahout.classifier.bayes.io.SequenceFileModelReader
  • org.apache.mahout.classifier.bayes.mapreduce.bayes.BayesClassifierDriver
  • org.apache.mahout.classifier.bayes.mapreduce.bayes.BayesClassifierMapper
  • org.apache.mahout.classifier.bayes.mapreduce.bayes.BayesThetaNormalizerMapper
  • org.apache.mahout.classifier.bayes.mapreduce.cbayes.CBayesThetaNormalizerMapper
  • org.apache.mahout.classifier.bayes.mapreduce.common.BayesFeatureMapper
  • org.apache.mahout.classifier.bayes.mapreduce.common.BayesFeatureOutputFormat
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.