Package org.apache.mahout.math

Examples of org.apache.mahout.math.MatrixSlice

183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
    FileSystem fs = FileSystem.get(conf);
    SequenceFile.Writer seqWriter = new SequenceFile.Writer(fs, conf, path, IntWritable.class, VectorWritable.class);
    VectorWritable vw = new VectorWritable();
    IntWritable iw = new IntWritable();
    for(Map.Entry<MatrixSlice,EigenStatus> pruneSlice : prunedEigenMeta) {
      MatrixSlice s = pruneSlice.getKey();
      EigenStatus meta = pruneSlice.getValue();
      EigenVector ev = new EigenVector((DenseVector)s.vector(),
                                       meta.getEigenValue(),
                                       Math.abs(1-meta.getCosAngle()),
                                       s.index());
      log.info("appending " + ev.getName() + " to " + path.toString());
      vw.set(ev);
      iw.set(s.index());
      seqWriter.append(iw, vw);
    }
    seqWriter.close();
  }
View Full Code Here
243
244
245
246
247
248
249
    public MatrixSlice next() {
      if(!hasBuffered && !hasNext()) {
        throw new NoSuchElementException();
      }
      hasBuffered = false;
      return new MatrixSlice(v.get(), i.get());
    }
View Full Code Here
66
67
68
69
70
71
72
73
74
75
76
77
78
79
    Iterator<MatrixSlice> mIt = m.iterateAll();
    Iterator<MatrixSlice> mttIt = mtt.iterateAll();
    Map<Integer, Vector> mMap = new HashMap<Integer,Vector>();
    Map<Integer, Vector> mttMap = new HashMap<Integer, Vector>();
    while(mIt.hasNext() && mttIt.hasNext()) {
      MatrixSlice ms = mIt.next();
      mMap.put(ms.index(), ms.vector());
      MatrixSlice mtts = mttIt.next();
      mttMap.put(mtts.index(), mtts.vector());
    }
    for(Map.Entry<Integer, Vector> entry : mMap.entrySet()) {
      Integer key = entry.getKey();
      Vector value = entry.getValue();
      if(value == null || mttMap.get(key) == null) {
View Full Code Here
161
162
163
164
165
166
167
168
169
170
171
172
        return new Iterator<VectorWritable>() {
          @Override
          public boolean hasNext() { return it.hasNext(); }
          @Override
          public VectorWritable next() {
            MatrixSlice slice = it.next();
            v.set(slice.vector());
            return v;
          }
          @Override
          public void remove() { it.remove(); }
        };
View Full Code Here
140
141
142
143
144
145
146
147
148
149
150
                                                                  true,
                                                                  conf),
          new Function<Pair<IntWritable,VectorWritable>,MatrixSlice>() {
            @Override
            public MatrixSlice apply(Pair<IntWritable, VectorWritable> from) {
              return new MatrixSlice(from.getSecond().get(), from.getFirst().get());
            }
          });
    } catch (IOException ioe) {
      throw new IllegalStateException(ioe);
    }
View Full Code Here
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
    log.info(info.toString());

    RunningAverage avg = new FullRunningAverage();
    Iterator<MatrixSlice> sliceIterator = preferences.iterateAll();
    while (sliceIterator.hasNext()) {
      MatrixSlice slice = sliceIterator.next();
      Iterator<Vector.Element> elementIterator = slice.vector().iterateNonZero();
      while (elementIterator.hasNext()) {
        Vector.Element e = elementIterator.next();
        if (!Double.isNaN(e.get())) {
          double pref = e.get();
          double estimate = u.viewRow(slice.index()).dot(m.viewRow(e.index()));
          double err = pref - estimate;
          avg.addDatum(err * err);
          log.info("Comparing preference of user [{}] towards item [{}], was [{}] estimate is [{}]",
                   new Object[] { slice.index(), e.index(), pref, estimate });
        }
      }
    }
    double rmse = Math.sqrt(avg.getAverage());
    log.info("RMSE: {}", rmse);
View Full Code Here
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
    log.info(info.toString());

    RunningAverage avg = new FullRunningAverage();
    Iterator<MatrixSlice> sliceIterator = preferences.iterateAll();
    while (sliceIterator.hasNext()) {
      MatrixSlice slice = sliceIterator.next();
      for (Vector.Element e : slice.vector()) {
        if (!Double.isNaN(e.get())) {
          double pref = e.get();
          double estimate = u.viewRow(slice.index()).dot(m.viewRow(e.index()));
          double confidence = 1 + alpha * observations.getQuick(slice.index(), e.index());
          double err = confidence * (pref - estimate) * (pref - estimate);
          avg.addDatum(err);
          log.info("Comparing preference of user [{}] towards item [{}], was [{}] with confidence [{}] " +
                       "estimate is [{}]", new Object[]{slice.index(), e.index(), pref, confidence, estimate});
        }
      }
    }
    double rmse = Math.sqrt(avg.getAverage());
    log.info("RMSE: {}", rmse);
View Full Code Here
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
  protected static String preferencesAsText(Matrix preferences) {
    StringBuilder prefsAsText = new StringBuilder();
    String separator = "";
    Iterator<MatrixSlice> sliceIterator = preferences.iterateAll();
    while (sliceIterator.hasNext()) {
      MatrixSlice slice = sliceIterator.next();
      Iterator<Vector.Element> elementIterator = slice.vector().iterateNonZero();
      while (elementIterator.hasNext()) {
        Vector.Element e = elementIterator.next();
        if (!Double.isNaN(e.get())) {
          prefsAsText.append(separator).append(slice.index()).append(',').append(e.index()).append(',').append(e.get());
          separator = "\n";
        }
      }
    }
    return prefsAsText.toString();
View Full Code Here
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
    SequenceFile.Writer seqWriter = new SequenceFile.Writer(fs, conf, path, IntWritable.class, VectorWritable.class);
    try {
      IntWritable iw = new IntWritable();
      int numEigensWritten = 0;
      for (Map.Entry<MatrixSlice, EigenStatus> pruneSlice : prunedEigenMeta) {
        MatrixSlice s = pruneSlice.getKey();
        EigenStatus meta = pruneSlice.getValue();
        EigenVector ev = new EigenVector(s.vector(),
                                         meta.getEigenValue(),
                                         Math.abs(1 - meta.getCosAngle()),
                                         s.index());
        log.info("appending {} to {}", ev, path);
        Writable vw = new VectorWritable(ev);
        iw.set(s.index());
        seqWriter.append(iw, vw);

        // increment the number of eigenvectors written and see if we've
        // reached our specified limit, or if we wish to write all eigenvectors
        // (latter is built-in, since numEigensWritten will always be > 0
View Full Code Here
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
    Iterator<MatrixSlice> docIterator = matrix.iterator();
    Iterator<MatrixSlice> docTopicIterator = docTopicCounts.iterator();
    double perplexity = 0;
    double matrixNorm = 0;
    while(docIterator.hasNext() && docTopicIterator.hasNext()) {
      MatrixSlice docSlice = docIterator.next();
      MatrixSlice topicSlice = docTopicIterator.next();
      int docId = docSlice.index();
      Vector document = docSlice.vector();
      Vector topicDist = topicSlice.vector();
      if(testFraction == 0 || docId % (1/testFraction) == 0) {
        trainSync(document, topicDist, false, 10);
        perplexity += readModel.perplexity(document, topicDist);
        matrixNorm += document.norm(1);
      }
View Full Code Here
TOP

Related Classes of org.apache.mahout.math.MatrixSlice

  • org.apache.mahout.cf.taste.hadoop.als.ParallelALSFactorizationJobTest
  • org.apache.mahout.cf.taste.impl.recommender.svd.ALSWRFactorizerTest
  • org.apache.mahout.clustering.lda.cvb.ModelTrainer
  • org.apache.mahout.math.hadoop.decomposer.EigenVerificationJob
  • org.apache.mahout.math.hadoop.DistributedRowMatrix
  • org.apache.mahout.math.hadoop.DistributedRowMatrix$DistributedMatrixIterator
  • org.apache.mahout.math.hadoop.TestDistributedRowMatrix
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.