Package org.apache.mahout.math

Examples of org.apache.mahout.math.Matrix

69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
    randomWalkWithRestart.run(new String[]{"--vertices", verticesFile.getAbsolutePath(),
        "--edges", edgesFile.getAbsolutePath(), "--sourceVertexIndex", String.valueOf(2),
        "--output", outputDir.getAbsolutePath(), "--numIterations", String.valueOf(2),
        "--stayingProbability", String.valueOf(0.75), "--tempDir", tempDir.getAbsolutePath()});

    Matrix expectedAdjacencyMatrix = new DenseMatrix(new double[][] {
        { 0, 1, 1, 0 },
        { 0, 1, 0, 1 },
        { 1, 1, 1, 1 },
        { 0, 1, 0, 0 } });

    int numVertices = HadoopUtil.readInt(new Path(tempDir.getAbsolutePath(), AdjacencyMatrixJob.NUM_VERTICES), conf);
    assertEquals(4, numVertices);
    Matrix actualAdjacencyMatrix = MathHelper.readMatrix(conf, new Path(tempDir.getAbsolutePath(),
        AdjacencyMatrixJob.ADJACENCY_MATRIX + "/part-r-00000"), numVertices, numVertices);

    StringBuilder info = new StringBuilder();
    info.append("\nexpected adjacency matrix\n\n");
    info.append(MathHelper.nice(expectedAdjacencyMatrix));
    info.append("\nactual adjacency matrix \n\n");
    info.append(MathHelper.nice(actualAdjacencyMatrix));
    info.append('\n');
    log.info(info.toString());

    Matrix expectedTransitionMatrix = new DenseMatrix(new double[][] {
        { 0,     0,     0.1875, 0    },
        { 0.375, 0.375, 0.1875, 0.75 },
        { 0.375, 0,     0.1875, 0    },
        { 0,     0.375, 0.1875, 0    } });

    Matrix actualTransitionMatrix = MathHelper.readMatrix(conf, new Path(tempDir.getAbsolutePath(),
        "transitionMatrix/part-r-00000"), numVertices, numVertices);

    info = new StringBuilder();
    info.append("\nexpected transition matrix\n\n");
    info.append(MathHelper.nice(expectedTransitionMatrix));
View Full Code Here
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
    pageRank.setConf(conf);
    pageRank.run(new String[] { "--vertices", verticesFile.getAbsolutePath(), "--edges", edgesFile.getAbsolutePath(),
        "--output", outputDir.getAbsolutePath(), "--numIterations", "3", "--stayingProbability", "0.8",
        "--tempDir", tempDir.getAbsolutePath() });

    Matrix expectedAdjacencyMatrix = new DenseMatrix(new double[][] {
        { 0, 1, 1, 1 },
        { 1, 0, 0, 1 },
        { 0, 0, 1, 0 },
        { 0, 1, 1, 0 } });

    int numVertices = HadoopUtil.readInt(new Path(tempDir.getAbsolutePath(), AdjacencyMatrixJob.NUM_VERTICES), conf);
    assertEquals(4, numVertices);
    Matrix actualAdjacencyMatrix = MathHelper.readMatrix(conf, new Path(tempDir.getAbsolutePath(),
        AdjacencyMatrixJob.ADJACENCY_MATRIX + "/part-r-00000"), numVertices, numVertices);

    StringBuilder info = new StringBuilder();
    info.append("\nexpected adjacency matrix\n\n");
    info.append(MathHelper.nice(expectedAdjacencyMatrix));
    info.append("\nactual adjacency matrix \n\n");
    info.append(MathHelper.nice(actualAdjacencyMatrix));
    info.append('\n');
    log.info(info.toString());

    Matrix expectedTransitionMatrix = new DenseMatrix(new double[][] {
        { 0.0,         0.4, 0.0, 0.0 },
        { 0.266666667, 0.0, 0.0, 0.4 },
        { 0.266666667, 0.0, 0.8, 0.4 },
        { 0.266666667, 0.4, 0.0, 0.0 } });

    Matrix actualTransitionMatrix = MathHelper.readMatrix(conf, new Path(tempDir.getAbsolutePath(),
        "transitionMatrix/part-r-00000"), numVertices, numVertices);

    info = new StringBuilder();
    info.append("\nexpected transition matrix\n\n");
    info.append(MathHelper.nice(expectedTransitionMatrix));
View Full Code Here
184
185
186
187
188
189
190
191
192
 
  private static void exportText(Path inputPath, PrintStream out) throws IOException {
    MatrixWritable mw = new MatrixWritable();
    Text key = new Text();
    readSeqFile(inputPath, key, mw);
    Matrix m = mw.get();
    ConfusionMatrix cm = new ConfusionMatrix(m);
    out.println(cm.toString());
  }
View Full Code Here
195
196
197
198
199
200
201
202
203
204
205
    MatrixWritable mw = new MatrixWritable();
    Text key = new Text();
    readSeqFile(inputPath, key, mw);
    String fileName = inputPath.getName();
    fileName = fileName.substring(fileName.lastIndexOf('/') + 1, fileName.length());
    Matrix m = mw.get();
    ConfusionMatrix cm = new ConfusionMatrix(m);
    if (wrapHtml) {
      printHeader(out, fileName);
    }
    out.println("<p/>");
View Full Code Here
383
384
385
386
387
388
389
390
391
392
393
        conf.set("fs.default.name", dfsNameNode);
      }
      String[] terms = loadDictionary(dictDirString, conf);
      logTime("dictionary loading", System.nanoTime() - start);
      start = System.nanoTime();
      Matrix corpus = loadVectors(inputDirString, conf);
      logTime("vector seqfile corpus loading", System.nanoTime() - start);
      start = System.nanoTime();
      InMemoryCollapsedVariationalBayes0 cvb0 =
          new InMemoryCollapsedVariationalBayes0(corpus, terms, numTopics, alpha, eta,
                                                 numTrainThreads, numUpdateThreads, modelCorpusFraction, 1234);
View Full Code Here
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
    rowSimilarityJob.setConf(conf);
    rowSimilarityJob.run(new String[] { "--input", inputFile.getAbsolutePath(), "--output", outputDir.getAbsolutePath(),
        "--numberOfColumns", String.valueOf(5), "--similarityClassname", TanimotoCoefficientSimilarity.class.getName(),
        "--tempDir", tmpDir.getAbsolutePath() });

    Matrix similarityMatrix = MathHelper.readMatrix(conf, new Path(outputDir.getAbsolutePath(), "part-r-00000"), 3, 3);

    assertNotNull(similarityMatrix);
    assertEquals(3, similarityMatrix.numCols());
    assertEquals(3, similarityMatrix.numRows());

    assertEquals(1.0, similarityMatrix.get(0, 0), EPSILON);
    assertEquals(1.0, similarityMatrix.get(1, 1), EPSILON);
    assertEquals(1.0, similarityMatrix.get(2, 2), EPSILON);
    assertEquals(0.0, similarityMatrix.get(2, 0), EPSILON);
    assertEquals(0.0, similarityMatrix.get(2, 1), EPSILON);
    assertEquals(0.0, similarityMatrix.get(0, 2), EPSILON);
    assertEquals(0.0, similarityMatrix.get(1, 2), EPSILON);
    assertEquals(0.666666, similarityMatrix.get(0, 1), EPSILON);
    assertEquals(0.666666, similarityMatrix.get(1, 0), EPSILON);
  }
View Full Code Here
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
    rowSimilarityJob.run(new String[] { "--input", inputFile.getAbsolutePath(), "--output", outputDir.getAbsolutePath(),
        "--numberOfColumns", String.valueOf(6), "--similarityClassname", TanimotoCoefficientSimilarity.class.getName(),
        "--maxSimilaritiesPerRow", String.valueOf(1), "--excludeSelfSimilarity", String.valueOf(true),
        "--tempDir", tmpDir.getAbsolutePath() });

    Matrix similarityMatrix = MathHelper.readMatrix(conf, new Path(outputDir.getAbsolutePath(), "part-r-00000"), 3, 3);

    assertNotNull(similarityMatrix);
    assertEquals(3, similarityMatrix.numCols());
    assertEquals(3, similarityMatrix.numRows());

    assertEquals(0.0, similarityMatrix.get(0, 0), EPSILON);
    assertEquals(0.5, similarityMatrix.get(0, 1), EPSILON);
    assertEquals(0.0, similarityMatrix.get(0, 2), EPSILON);

    assertEquals(0.5, similarityMatrix.get(1, 0), EPSILON);
    assertEquals(0.0, similarityMatrix.get(1, 1), EPSILON);
    assertEquals(0.0, similarityMatrix.get(1, 2), EPSILON);

    assertEquals(0.4, similarityMatrix.get(2, 0), EPSILON);
    assertEquals(0.0, similarityMatrix.get(2, 1), EPSILON);
    assertEquals(0.0, similarityMatrix.get(2, 2), EPSILON);
  }
View Full Code Here
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
  public Vector topicSums() {
    return topicSums;
  }

  private static Pair<Matrix,Vector> randomMatrix(int numTopics, int numTerms, Random random) {
    Matrix topicTermCounts = new DenseMatrix(numTopics, numTerms);
    Vector topicSums = new DenseVector(numTopics);
    if(random != null) {
      for(int x = 0; x < numTopics; x++) {
        for(int term = 0; term < numTerms; term++) {
          topicTermCounts.viewRow(x).set(term, random.nextDouble());
        }
      }
    }
    for(int x = 0; x < numTopics; x++) {
      topicSums.set(x, random == null ? 1.0 : topicTermCounts.viewRow(x).norm(1));
    }
    return Pair.of(topicTermCounts, topicSums);
  }
View Full Code Here
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
    }
  }

  public static Matrix sampledCorpus(Matrix matrix, Random random,
      int numDocs, int numSamples, int numTopicsPerDoc) {
    Matrix corpus = new SparseRowMatrix(numDocs, matrix.numCols());
    LDASampler modelSampler = new LDASampler(matrix, random);
    Vector topicVector = new DenseVector(matrix.numRows());
    for(int i = 0; i < numTopicsPerDoc; i++) {
      int topic = random.nextInt(topicVector.size());
      topicVector.set(topic, topicVector.get(topic) + 1);
    }
    for(int docId = 0; docId < numDocs; docId++) {
      for(int sample : modelSampler.sample(topicVector, numSamples)) {
        corpus.set(docId, sample, corpus.get(docId, sample) + 1);
      }
    }
    return corpus;
  }
View Full Code Here
209
210
211
212
213
214
215
216
217
218
219
220
221
    }
    if(rows.isEmpty()) {
      throw new IOException(Arrays.toString(modelPaths) + " have no vectors in it");
    }
    numTopics++;
    Matrix model = new DenseMatrix(numTopics, numTerms);
    Vector topicSums = new DenseVector(numTopics);
    for(Pair<Integer, Vector> pair : rows) {
      model.viewRow(pair.getFirst()).assign(pair.getSecond());
      topicSums.set(pair.getFirst(), pair.getSecond().norm(1));
    }
    return Pair.of(model, topicSums);
  }
View Full Code Here
TOP

Related Classes of org.apache.mahout.math.Matrix

  • com.mapr.stats.bandit.ContextualBayesBanditTest
  • org.apache.mahout.cf.taste.hadoop.als.ParallelALSFactorizationJobTest
  • org.apache.mahout.cf.taste.impl.recommender.svd.ImplicitLinearRegressionFactorizer
  • org.apache.mahout.classifier.AbstractVectorClassifier
  • org.apache.mahout.classifier.ConfusionMatrix
  • org.apache.mahout.classifier.ConfusionMatrixDumper
  • org.apache.mahout.classifier.ConfusionMatrixTest
  • org.apache.mahout.classifier.discriminative.PerceptronTrainerTest
  • org.apache.mahout.classifier.discriminative.WinnowTrainerTest
  • org.apache.mahout.classifier.evaluation.AucTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.