Package org.apache.mahout.clustering.iterator

Examples of org.apache.mahout.clustering.iterator.ClusterWritable

184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
  public void writeToSeqFiles(Path path) throws IOException {
    writePolicy(policy, path);
    Configuration config = new Configuration();
    FileSystem fs = FileSystem.get(path.toUri(), config);
    SequenceFile.Writer writer = null;
    ClusterWritable cw = new ClusterWritable();
    for (int i = 0; i < models.size(); i++) {
      try {
        Cluster cluster = models.get(i);
        cw.setValue(cluster);
        writer = new SequenceFile.Writer(fs, config,
            new Path(path, "part-" + String.format(Locale.ENGLISH, "%05d", i)), IntWritable.class,
            ClusterWritable.class);
        Writable key = new IntWritable(i);
        writer.append(key, cw);
View Full Code Here
162
163
164
165
166
167
168
169
170
171
172
173
    List<Cluster> clusterModels = Lists.newArrayList();
    Path finalClustersPath = finalClustersPath(conf, clusterOutputPath);
    Iterator<?> it = new SequenceFileDirValueIterator<Writable>(finalClustersPath, PathType.LIST,
        PathFilters.partFilter(), null, false, conf);
    while (it.hasNext()) {
      ClusterWritable next = (ClusterWritable) it.next();
      Cluster cluster = next.getValue();
      cluster.configure(conf);
      clusterModels.add(cluster);
    }
    return clusterModels;
  }
View Full Code Here
115
116
117
118
119
120
121
122
123
124
125
126
    FileStatus[] clusterFiles = fileSystem.listStatus(clusterOutputPath, PathFilters.finalPartFilter());
    Iterator<?> it = new SequenceFileDirValueIterator<Writable>(
        clusterFiles[0].getPath(), PathType.LIST, PathFilters.partFilter(),
        null, false, conf);
    while (it.hasNext()) {
      ClusterWritable next = (ClusterWritable) it.next();
      Cluster cluster = next.getValue();
      cluster.configure(conf);
      clusters.add(cluster);
    }
    return clusters;
  }
View Full Code Here
50
51
52
53
54
55
56
57
58
59
60
61
  public static void configureWithClusterInfo(Configuration conf, Path clusterPath, Collection<Cluster> clusters) {
    for (Writable value : new SequenceFileDirValueIterable<Writable>(clusterPath, PathType.LIST,
        PathFilters.partFilter(), conf)) {
      Class<? extends Writable> valueClass = value.getClass();
      if (valueClass.equals(ClusterWritable.class)) {
        ClusterWritable clusterWritable = (ClusterWritable) value;
        value = clusterWritable.getValue();
        valueClass = value.getClass();
      }
      log.debug("Read 1 Cluster from {}", clusterPath);
     
      if (valueClass.equals(Kluster.class)) {
View Full Code Here
96
97
98
99
100
101
102
103
104
105
106
107
                maxEigens.put(index, v);
                Text newText = new Text(key.toString());
                chosenTexts.put(index, newText);
                Kluster newCluster = new Kluster(value.get(), index, measure);
                newCluster.observe(value.get(), 1);
                ClusterWritable clusterWritable = new ClusterWritable();
                clusterWritable.setValue(newCluster);
                chosenClusters.put(index, clusterWritable);
              }
            }
          }
        }
View Full Code Here
48
49
50
51
52
53
54
55
56
57
58
59
    for (Writable value : new SequenceFileDirValueIterable<Writable>(clusterPath, PathType.LIST,
        PathFilters.partFilter(), conf)) {
      Class<? extends Writable> valueClass = value.getClass();
     
      if (valueClass.equals(ClusterWritable.class)) {
        ClusterWritable clusterWritable = (ClusterWritable) value;
        value = clusterWritable.getValue();
        valueClass = value.getClass();
      }
     
      if (valueClass.equals(Kluster.class)) {
        // get the cluster info
View Full Code Here
212
213
214
215
216
217
218
219
220
221
222
223
      SequenceFile.Writer writer = new SequenceFile.Writer(fs, conf, new Path(
          output, "part-m-" + part++), Text.class, ClusterWritable.class);
      try {
        for (VectorWritable value : new SequenceFileValueIterable<VectorWritable>(s.getPath(), conf)) {
          MeanShiftCanopy initialCanopy = MeanShiftCanopy.initialCanopy(value.get(), id++, measure);
          ClusterWritable clusterWritable = new ClusterWritable();
          clusterWritable.setValue(initialCanopy);
          writer.append(new Text(), clusterWritable);
        }
      } finally {
        Closeables.close(writer, false);
      }
View Full Code Here
326
327
328
329
330
331
332
333
334
335
336
337
                AbstractCluster.formatVector(cluster.getCenter(), null),
                cluster.getNumObservations(),
                AbstractCluster.formatVector(cluster.getRadius(), null),
                clustersOut.getName());
          }
          ClusterWritable clusterWritable = new ClusterWritable();
          clusterWritable.setValue(cluster);
          writer.append(new Text(cluster.getIdentifier()), clusterWritable);
        }
      } finally {
        Closeables.close(writer, false);
      }
View Full Code Here
486
487
488
489
490
491
492
493
494
495
496
497
          output, "part-m-" + part++), IntWritable.class,
          WeightedVectorWritable.class);
      try {
        for (Pair<Writable, ClusterWritable> record
            : new SequenceFileIterable<Writable, ClusterWritable>(s.getPath(), conf)) {
          ClusterWritable clusterWritable = record.getSecond();
          MeanShiftCanopy canopy = (MeanShiftCanopy) clusterWritable.getValue();
          MeanShiftCanopy closest = MeanShiftCanopyClusterer.findCoveringCanopy(canopy, clusters);
          writer.append(new IntWritable(closest.getId()), new WeightedVectorWritable(1, canopy.getCenter()));
        }
      } finally {
        Closeables.close(writer, false);
View Full Code Here
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
    Path path = new Path(output, "clusters-0-final/part-r-00000");
    FileSystem fs = FileSystem.get(path.toUri(), config);
    SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, config);
    try {
      Writable key = new Text();
      ClusterWritable clusterWritable = new ClusterWritable();
      assertTrue("more to come", reader.next(key, clusterWritable));
      assertEquals("1st key", "C-0", key.toString());

      List<Pair<Double,Double>> refCenters = Lists.newArrayList();
      refCenters.add(new Pair<Double,Double>(1.8,1.8));
      refCenters.add(new Pair<Double,Double>(4.433333333333334, 4.433333333333334));
      Pair<Double,Double> c = new Pair<Double,Double>(clusterWritable.getValue().getCenter().get(0),
                                                      clusterWritable.getValue().getCenter().get(1));
      assertTrue("center "+c+" not found", findAndRemove(c, refCenters, EPSILON));
      assertTrue("more to come", reader.next(key, clusterWritable));
      assertEquals("2nd key", "C-1", key.toString());
      c = new Pair<Double,Double>(clusterWritable.getValue().getCenter().get(0),
                                  clusterWritable.getValue().getCenter().get(1));
      assertTrue("center "+c+" not found", findAndRemove(c, refCenters, EPSILON));
      assertFalse("more to come", reader.next(key, clusterWritable));
    } finally {
      Closeables.close(reader, true);
    }
View Full Code Here
TOP

Related Classes of org.apache.mahout.clustering.iterator.ClusterWritable

  • org.apache.mahout.clustering.canopy.CanopyDriver
  • org.apache.mahout.clustering.canopy.CanopyReducer
  • org.apache.mahout.clustering.canopy.TestCanopyCreation
  • org.apache.mahout.clustering.classify.ClusterClassificationDriver
  • org.apache.mahout.clustering.classify.ClusterClassificationMapper
  • org.apache.mahout.clustering.classify.ClusterClassifier
  • org.apache.mahout.clustering.fuzzykmeans.FuzzyKMeansUtil
  • org.apache.mahout.clustering.kmeans.EigenSeedGenerator
  • org.apache.mahout.clustering.kmeans.KMeansUtil
  • org.apache.mahout.clustering.kmeans.RandomSeedGenerator
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.