Package org.apache.mahout.common.iterator

Examples of org.apache.mahout.common.iterator.FileLineIterable

93
94
95
96
97
98
99
100
101
102
103
  /**
   * Test Parallel FPGrowth on retail data using top-level runPFPGrowth() method
   */
  @Test
  public void testRetailDataMinSup100() throws Exception {
    StringRecordIterator it = new StringRecordIterator(new FileLineIterable(Resources.getResource(
      "retail_results_with_min_sup_100.dat").openStream()), "\\s+");
    Map<Set<String>,Long> expectedResults = Maps.newHashMap();
    while (it.hasNext()) {
      Pair<List<String>,Long> next = it.next();
      List<String> items = Lists.newArrayList(next.getFirst());
View Full Code Here
146
147
148
149
150
151
152
153
154
155
156
  /**
   * Test Parallel FPG on retail data, running various stages individually
   */
  @Test
  public void testRetailDataMinSup100InSteps() throws Exception {
    StringRecordIterator it = new StringRecordIterator(new FileLineIterable(Resources.getResource(
      "retail_results_with_min_sup_100.dat").openStream()), "\\s+");  
    Map<Set<String>,Long> expectedResults = Maps.newHashMap();
    while (it.hasNext()) {
      Pair<List<String>,Long> next = it.next();
      List<String> items = Lists.newArrayList(next.getFirst());
View Full Code Here
179
180
181
182
183
184
185
186
187
188
189
     */
    HadoopUtil.delete(conf, outPath);
   
    Set<String> categories = Sets.newHashSet();
    if (!catFile.isEmpty()) {
      for (String line : new FileLineIterable(new File(catFile))) {
        categories.add(line.trim().toLowerCase(Locale.ENGLISH));
      }
    }
   
    Stringifier<Set<String>> setStringifier =
View Full Code Here
157
158
159
160
161
162
163
164
165
166
167
      String messageId = null;
      boolean inBody = false;
      Pattern quotedTextPattern = options.getQuotedTextPattern();

      for (String nextLine : new FileLineIterable(mailBoxInputStream, options.getCharset(), false, filename)) {
        if (!options.isStripQuotedText() || !quotedTextPattern.matcher(nextLine).find()) {
          for (int i = 0; i < matches.length; i++) {
            Matcher matcher = matches[i];
            matcher.reset(nextLine);
            if (matcher.matches()) {
View Full Code Here
37
38
39
40
41
42
43
44
45
46
47
   
    DataSet dataset = FileInfoParser.parseFile(fs, inpath);
    DataSet.initialize(dataset);

    DataLine dl = new DataLine();
    for (String line : new FileLineIterable(new File(Resources.getResource("wdbc/wdbc.data").getPath()))) {
      dl.set(line);
      for (int index = 0; index < dataset.getNbAttributes(); index++) {
        if (dataset.isNumerical(index)) {
          CDMutationTest.assertInRange(dl.getAttribute(index), dataset.getMin(index), dataset
              .getMax(index));
View Full Code Here
173
174
175
176
177
178
179
180
181
182
183
        boolean inBody = false;
        Matcher subjectMatcher = SUBJECT_PREFIX.matcher("");
        Matcher messageIdMatcher = MESSAGE_ID_PREFIX.matcher("");
        Matcher messageBoundaryMatcher = MESSAGE_START.matcher("");
       
        for (String nextLine : new FileLineIterable(current, charset, false)) {

          // subject may come before message ID
          subjectMatcher.reset(nextLine);
          if (subjectMatcher.matches()) {
            file.append(subjectMatcher.group(1)).append('\n');
View Full Code Here
85
86
87
88
89
90
91
92
93
94
95
      fs.listStatus(fst.getPath(),
                    new SequenceFilesFromCsvFilter(conf, prefix + Path.SEPARATOR + current.getName(),
                        this.options, writer));
    } else {
      InputStream in = fs.open(fst.getPath());
      for (CharSequence aFit : new FileLineIterable(in, charset, false)) {
        String[] columns = TAB.split(aFit);
        log.info("key : {}, value : {}", columns[keyColumn], columns[valueColumn]);
        String key = columns[keyColumn];
        String value = columns[valueColumn];
        writer.write(prefix + key, value);
View Full Code Here
47
48
49
50
51
52
53
54
55
56
57
      InputStream in = null;
      try {
        in = fs.open(fst.getPath());

        StringBuilder file = new StringBuilder();
        for (String aFit : new FileLineIterable(in, charset, false)) {
          file.append(aFit).append('\n');
        }
        String name = current.getName().equals(fst.getPath().getName())
            ? current.getName()
            : current.getName() + Path.SEPARATOR + fst.getPath().getName();
View Full Code Here
55
56
57
58
59
60
61
62
63
64
65
        Path unqualifiedUsersFilePath = new Path(usersFilePathString);
        FileSystem fs = FileSystem.get(unqualifiedUsersFilePath.toUri(), jobConf);
        usersToRecommendFor = new FastIDSet();
        Path usersFilePath = unqualifiedUsersFilePath.makeQualified(fs);
        in = fs.open(usersFilePath);
        for (String line : new FileLineIterable(in)) {
          usersToRecommendFor.add(Long.parseLong(line));
        }
      } finally {
        IOUtils.closeStream(in);
      }
View Full Code Here
162
163
164
165
166
167
168
169
170
171
172
             + "org.apache.hadoop.io.serializer.WritableSerialization");
    // Dont ever forget this. People should keep track of how hadoop conf
    // parameters can make or break a piece of code
   
    Set<String> categories = new HashSet<String>();
    for (String line : new FileLineIterable(new File(catFile))) {
      categories.add(line.trim().toLowerCase(Locale.ENGLISH));
    }
   
    DefaultStringifier<Set<String>> setStringifier =
        new DefaultStringifier<Set<String>>(conf, GenericsUtil.getClass(categories));
View Full Code Here
TOP

Related Classes of org.apache.mahout.common.iterator.FileLineIterable

  • mia.recommender.ch05.GenderRescorer
  • org.apache.mahout.cf.taste.example.bookcrossing.BookCrossingDataModel
  • org.apache.mahout.cf.taste.example.grouplens.BooleanDataAndDateDataModel
  • org.apache.mahout.cf.taste.example.grouplens.GroupLensDataModel
  • org.apache.mahout.cf.taste.example.kddcup.track2.TrackItemSimilarity
  • org.apache.mahout.cf.taste.example.netflix.NetflixDataModel
  • org.apache.mahout.cf.taste.example.netflix.NetflixFileDataModel
  • org.apache.mahout.cf.taste.example.netflix.TransposeToByUser
  • org.apache.mahout.cf.taste.hadoop.example.als.netflix.NetflixDatasetConverter
  • org.apache.mahout.cf.taste.hadoop.item.AggregateAndRecommendReducer
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.