Examples of org.apache.mahout.common.iterator.FileLineIterator

Package org.apache.mahout.common.iterator

Examples of org.apache.mahout.common.iterator.FileLineIterator

org.apache.mahout.common.iterator.FileLineIterator
Iterates over the lines of a text file. This assumes the text file's lines are delimited in a manner consistent with how {@link BufferedReader} defines lines.
This class will uncompress files that end in .zip or .gz accordingly, too.

    
    StringBuilder content = new StringBuilder();
    content.append(header);
    NumberFormat decimalFormatter = new DecimalFormat("0000");
    File dumpFile = new File(dumpFilePath);
    FileLineIterator it;
    if (dumpFilePath.endsWith(".bz2")) {
      // default compression format from http://download.wikimedia.org
      CompressionCodec codec = new BZip2Codec();
      it = new FileLineIterator(codec.createInputStream(new FileInputStream(dumpFile)));
    } else {
      // assume the user has previously de-compressed the dump file
      it = new FileLineIterator(dumpFile);
    }
    int filenumber = 0;
    while (it.hasNext()) {
      String thisLine = it.next();
      if (thisLine.trim().startsWith("<page>")) {
        boolean end = false;
        while (!thisLine.trim().startsWith("</page>")) {
          content.append(thisLine).append('\n');
          if (it.hasNext()) {
            thisLine = it.next();
          } else {
            end = true;
            break;
          }
        }

View Full Code Here

      FSDataOutputStream outputStream = fs.create(new Path(outputPath, "trainingSet/ratings.tsv"));
      writer = new BufferedWriter(new OutputStreamWriter(outputStream, Charsets.UTF_8));


      int ratingsProcessed = 0;
      for (File movieRatings : new File(trainingDataDir).listFiles()) {
        FileLineIterator lines = null;
        try  {
          lines = new FileLineIterator(movieRatings);
          boolean firstLineRead = false;
          String movieID = null;
          while (lines.hasNext()) {
            String line = lines.next();
            if (firstLineRead) {
              String[] tokens = SEPARATOR.split(line);
              String userID = tokens[0];
              String rating = tokens[1];
              writer.write(userID + TAB + movieID + TAB + rating + NEWLINE);

View Full Code Here


  public DataFileIterator(File dataFile) throws IOException {
    if (dataFile == null || dataFile.isDirectory() || !dataFile.exists()) {
      throw new IllegalArgumentException("Bad data file: " + dataFile);
    }
    lineIterator = new FileLineIterator(dataFile);
  }

View Full Code Here

      try {


        averageDiffs.clear();
        allRecommendableItemIDs.clear();
        
        FileLineIterator iterator = new FileLineIterator(dataFile, false);
        String firstLine = iterator.peek();
        while (firstLine.length() == 0 || firstLine.charAt(0) == COMMENT_CHAR) {
          iterator.next();
          firstLine = iterator.peek();
        }
        char delimiter = FileDataModel.determineDelimiter(firstLine);
        long averageCount = 0L;
        while (iterator.hasNext()) {
          averageCount = processLine(iterator.next(), delimiter, averageCount);
        }
        
        pruneInconsequentialDiffs();
        updateAllRecommendableItems();

View Full Code Here

   * @param path
   *          data path
   */
  public static int[] extractLabels(Dataset dataset, FileSystem fs, Path path) throws IOException {
    FSDataInputStream input = fs.open(path);
    FileLineIterator iterator = new FileLineIterator(input);
    
    int[] labels = new int[dataset.nbInstances()];
    DataConverter converter = new DataConverter(dataset);
    
    int index = 0;
    
    while (iterator.hasNext()) {
      labels[index++] = converter.convert(0, iterator.next()).getLabel();
    }
    
    iterator.close();
    
    return labels;
  }

View Full Code Here


    this.dataFile = dataFile.getAbsoluteFile();
    this.lastModified = dataFile.lastModified();
    this.lastUpdateFileModified = readLastUpdateFileModified();


    FileLineIterator iterator = new FileLineIterator(dataFile, false);
    String firstLine = iterator.peek();
    while (firstLine.length() == 0 || firstLine.charAt(0) == COMMENT_CHAR) {
      iterator.next();
      firstLine = iterator.peek();
    }
    iterator.close();


    delimiter = determineDelimiter(firstLine);
    delimiterPattern = Pattern.compile(String.valueOf(delimiter));
    String[] firstLineSplit = delimiterPattern.split(firstLine);
    // If preference value exists and isn't empty then the file is specifying pref values

View Full Code Here

    if (hasPrefValues) {


      if (loadFreshData) {


        FastByIDMap<Collection<Preference>> data = new FastByIDMap<Collection<Preference>>();
        FileLineIterator iterator = new FileLineIterator(dataFile, false);
        processFile(iterator, data, timestamps, false);


        for (File updateFile : findUpdateFilesAfter(newLastModified)) {
          processFile(new FileLineIterator(updateFile, false), data, timestamps, false);
        }


        return new GenericDataModel(GenericDataModel.toDataMap(data, true), timestamps);


      } else {


        FastByIDMap<PreferenceArray> rawData = ((GenericDataModel) delegate).getRawUserData();


        for (File updateFile : findUpdateFilesAfter(Math.max(oldLastUpdateFileModifieid, newLastModified))) {
          processFile(new FileLineIterator(updateFile, false), rawData, timestamps, true);
        }


        return new GenericDataModel(rawData, timestamps);


      }


    } else {


      if (loadFreshData) {


        FastByIDMap<FastIDSet> data = new FastByIDMap<FastIDSet>();
        FileLineIterator iterator = new FileLineIterator(dataFile, false);
        processFileWithoutID(iterator, data, timestamps);


        for (File updateFile : findUpdateFilesAfter(newLastModified)) {
          processFileWithoutID(new FileLineIterator(updateFile, false), data, timestamps);
        }


        return new GenericBooleanPrefDataModel(data, timestamps);


      } else {


        FastByIDMap<FastIDSet> rawData = ((GenericBooleanPrefDataModel) delegate).getRawUserData();


        for (File updateFile : findUpdateFilesAfter(Math.max(oldLastUpdateFileModifieid, newLastModified))) {
          processFileWithoutID(new FileLineIterator(updateFile, false), rawData, timestamps);
        }


        return new GenericBooleanPrefDataModel(rawData, timestamps);


      }

View Full Code Here

   * <pre>
   * term DocFreq Index
   * </pre>
   */
  private static String[] loadTermDictionary(InputStream is) throws IOException {
    FileLineIterator it = new FileLineIterator(is);
    
    int numEntries = Integer.parseInt(it.next());
    String[] result = new String[numEntries];
    
    while (it.hasNext()) {
      String line = it.next();
      if (line.startsWith("#")) {
        continue;
      }
      String[] tokens = VectorHelper.TAB_PATTERN.split(line);
      if (tokens.length < 3) {

View Full Code Here

    
    StringBuilder content = new StringBuilder();
    content.append(header);
    NumberFormat decimalFormatter = new DecimalFormat("0000");
    File dumpFile = new File(dumpFilePath);
    FileLineIterator it;
    if (dumpFilePath.endsWith(".bz2")) {
      // default compression format from http://download.wikimedia.org
      CompressionCodec codec = new BZip2Codec();
      it = new FileLineIterator(codec.createInputStream(new FileInputStream(dumpFile)));
    } else {
      // assume the user has previously de-compressed the dump file
      it = new FileLineIterator(dumpFile);
    }
    int filenumber = 0;
    while (it.hasNext()) {
      String thisLine = it.next();
      if (thisLine.trim().startsWith("<page>")) {
        boolean end = false;
        while (!thisLine.trim().startsWith("</page>")) {
          content.append(thisLine).append('\n');
          if (it.hasNext()) {
            thisLine = it.next();
          } else {
            end = true;
            break;
          }
        }

View Full Code Here

  }
  
  @Override
  protected DataModel buildModel() throws IOException {
    FastByIDMap<Collection<Preference>> data = new FastByIDMap<Collection<Preference>>();
    FileLineIterator iterator = new FileLineIterator(getDataFile(), false);
    FastByIDMap<FastByIDMap<Long>> timestamps = new FastByIDMap<FastByIDMap<Long>>();
    processFile(iterator, data, timestamps, false);
    return new GenericDataModel(GenericDataModel.toDataMap(data, true));
  }

View Full Code Here

0 1 2 3

TOP

Related Classes of org.apache.mahout.common.iterator.FileLineIterator

org.apache.mahout.cf.taste.example.jester.JesterDataModel

org.apache.mahout.cf.taste.example.kddcup.DataFileIterator

org.apache.mahout.cf.taste.hadoop.example.als.netflix.NetflixDatasetConverter

org.apache.mahout.cf.taste.impl.model.file.FileDataModel

org.apache.mahout.cf.taste.impl.recommender.slopeone.file.FileDiffStorage

org.apache.mahout.cf.taste.impl.similarity.file.FileItemItemSimilarityIterator

org.apache.mahout.classifier.bayes.BayesFileFormatterTest

org.apache.mahout.classifier.bayes.WikipediaXmlSplitter

org.apache.mahout.df.data.Data

org.apache.mahout.text.wikipedia.WikipediaXmlSplitter

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.