Package org.apache.mahout.common

Examples of org.apache.mahout.common.FileLineIterable

87
88
89
90
91
92
93
94
95
  }

  private static void loadPopulation(FileSystem fs, Path f,
                                     Collection<DummyCandidate> population) throws IOException {
    FSDataInputStream in = fs.open(f);
    for (String line : new FileLineIterable(in)) {
      population.add(StringUtils.<DummyCandidate>fromString(line));
    }
  }
View Full Code Here
35
36
37
38
39
40
41
42
43
44
45
   
    DataSet dataset = FileInfoParser.parseFile(fs, inpath);
    DataSet.initialize(dataset);

    DataLine dl = new DataLine();
    for (String line : new FileLineIterable(new File("target/test-classes/wdbc/wdbc.data"))) {
      dl.set(line);
      for (int index = 0; index < dataset.getNbAttributes(); index++) {
        if (dataset.isNumerical(index)) {
          assertInRange(dl.getAttribut(index), dataset.getMin(index), dataset
              .getMax(index));
View Full Code Here
189
190
191
192
193
194
195
196
197
198
199
200
    SequenceFile.Writer writer = new SequenceFile.Writer(fs, conf, path, Text.class, TopKStringPatterns.class);
   
    FPGrowth<String> fp = new FPGrowth<String>();
    Set<String> features = new HashSet<String>();
   
    fp.generateTopKFrequentPatterns(new StringRecordIterator(new FileLineIterable(new File(input), encoding,
        false), pattern), fp.generateFList(new StringRecordIterator(new FileLineIterable(new File(input),
        encoding, false), pattern), minSupport), minSupport, maxHeapSize, features,
      new StringOutputConverter(new SequenceFileOutputCollector<Text,TopKStringPatterns>(writer)),
      new ContextStatusUpdater(null));
    writer.close();
   
View Full Code Here
237
238
239
240
241
242
243
244
245
246
247
          log.info("Testing: {}", file);
        }
        TimingStatistics operationStats = new TimingStatistics();
       
        long lineNum = 0;
        for (String line : new FileLineIterable(new File(file.getPath()), Charset.forName(params
            .get("encoding")), false)) {
         
          Map<String,List<String>> document = new NGrams(line, Integer.parseInt(params.get("gramSize")))
              .generateNGrams();
          for (Map.Entry<String,List<String>> stringListEntry : document.entrySet()) {
View Full Code Here
93
94
95
96
97
98
99
100
101
102
103
    Charset charset = Charset.forName("UTF-8");
    BayesFileFormatter.collapse("animal", analyzer, input, charset, new File(out, "animal"));
    files = out.listFiles();
    assertEquals("files Size: " + files.length + " is not: " + 1, 1, files.length);
    int count = 0;
    for (String line : new FileLineIterable(files[0])) {
      assertTrue("line does not start with label", line.startsWith("animal"));
      System.out.println("Line: " + line);
      count++;
    }
    assertEquals(count + " does not equal: " + words.length, count, words.length);
View Full Code Here
84
85
86
87
88
89
90
91
92
93
94
      itemIDPadded.insert(0, '0');
    }
    List<Preference> prefs = new ArrayList<Preference>();
    File movieFile = new File(new File(dataDirectory, "training_set"), "mv_00" + itemIDPadded + ".txt");
    try {
      for (String line : new FileLineIterable(movieFile, true)) {
        int firstComma = line.indexOf(',');
        Integer userID = Integer.valueOf(line.substring(0, firstComma));
        int secondComma = line.indexOf(',', firstComma + 1);
        float rating = Float.parseFloat(line.substring(firstComma + 1, secondComma));
        prefs.add(new GenericPreference(userID, itemID, rating));
View Full Code Here
55
56
57
58
59
60
61
62
63
64
65
      resultFile.delete();
    }
    PrintWriter writer = null;
    try {
      writer = new PrintWriter(new OutputStreamWriter(new FileOutputStream(resultFile), Charset.forName("UTF-8")));
      for (String line : new FileLineIterable(originalFile, false)) {
        String convertedLine = COLON_DELIMITER_PATTERN.matcher(line.substring(0, line.lastIndexOf(COLON_DELIMTER))).replaceAll(",");
        writer.println(convertedLine);
      }
      writer.flush();
    } catch (IOException ioe) {
View Full Code Here
60
61
62
63
64
65
66
67
68
69
70
    File resultFile = new File(new File(System.getProperty("java.io.tmpdir")), "taste.bookcrossing.txt");
    resultFile.delete();
    PrintWriter writer = null;
    try {
      writer = new PrintWriter(new OutputStreamWriter(new FileOutputStream(resultFile), Charset.forName("UTF-8")));
      for (String line : new FileLineIterable(originalFile, true)) {
        // 0 ratings are basically "no rating", ignore them (thanks h.9000)
        if (line.endsWith("\"0\"")) {
          continue;
        }
        // Delete replace anything that isn't numeric, or a semicolon delimiter. Make comma the delimiter.
View Full Code Here
62
63
64
65
66
67
68
69
70
71
72
   
    Map<String, List<String>> byUserEntryCache = new FastMap<String, List<String>>(100000);
   
    for (File byItemFile : byItemDirectory.listFiles()) {
      log.info("Processing {}", byItemFile);
      Iterator<String> lineIterator = new FileLineIterable(byItemFile, false).iterator();
      String line = lineIterator.next();
      String movieIDString = line.substring(0, line.length() - 1);
      while (lineIterator.hasNext()) {
        line = lineIterator.next();
        int firstComma = line.indexOf(',');
View Full Code Here
80
81
82
83
84
85
86
87
88
89
90
    FastByIDMap<Collection<Preference>> userIDPrefMap = new FastByIDMap<Collection<Preference>>();
   
    int counter = 0;
    FilenameFilter filenameFilter = new MovieFilenameFilter();
    for (File movieFile : new File(dataDirectory, "training_set").listFiles(filenameFilter)) {
      Iterator<String> lineIterator = new FileLineIterable(movieFile, false).iterator();
      String line = lineIterator.next();
      long movieID = Long.parseLong(line.substring(0, line.length() - 1)); // strip colon
      while (lineIterator.hasNext()) {
        line = lineIterator.next();
        if (++counter % 100000 == 0) {
View Full Code Here
TOP

Related Classes of org.apache.mahout.common.FileLineIterable

  • org.apache.mahout.cf.taste.example.bookcrossing.BookCrossingDataModel
  • org.apache.mahout.cf.taste.example.grouplens.GroupLensDataModel
  • org.apache.mahout.cf.taste.example.netflix.NetflixDataModel
  • org.apache.mahout.cf.taste.example.netflix.NetflixFileDataModel
  • org.apache.mahout.cf.taste.example.netflix.TransposeToByUser
  • org.apache.mahout.cf.taste.hadoop.item.AggregateAndRecommendReducer
  • org.apache.mahout.cf.taste.hadoop.item.RecommenderJobTest
  • org.apache.mahout.cf.taste.hadoop.item.RecommenderMapper
  • org.apache.mahout.cf.taste.hadoop.item.UserVectorSplitterMapper
  • org.apache.mahout.cf.taste.hadoop.pseudo.RecommenderReducer
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.