Examples of net.myrrix.common.collection.FastIDSet

net.myrrix.common.collection.FastIDSet
Based on Mahout's {@code FastIDSet}. @author Sean Owen @author Mahout @since 1.0

    Preconditions.checkNotNull(client);
    Preconditions.checkNotNull(dataDirectory);
    Preconditions.checkArgument(steps > 0);  
    
    log.info("Reading IDs...");    
    FastIDSet userIDsSet = new FastIDSet();
    FastIDSet itemIDsSet = new FastIDSet();
    Splitter comma = Splitter.on(',');
    for (File f : dataDirectory.listFiles(new PatternFilenameFilter(".+\\.csv(\\.(zip|gz))?"))) {
      for (CharSequence line : new FileLineIterable(f)) {
        Iterator<String> it = comma.split(line).iterator();
        userIDsSet.add(Long.parseLong(it.next()));
        itemIDsSet.add(Long.parseLong(it.next()));
      }
    }
    
    this.client = client;    
    this.uniqueUserIDs = userIDsSet.toArray();
    this.uniqueItemIDs = itemIDsSet.toArray();
    this.steps = steps;
  }

View Full Code Here

    }


    modelFile = new File(inputDir, "model.bin.gz");
    appendFile = new File(inputDir, "append.bin.gz");


    recentlyActiveUsers = new FastIDSet();
    recentlyActiveItems = new FastIDSet();
    hasher = new OneWayMigrator();
    
    loader = new GenerationLoader(recentlyActiveUsers, recentlyActiveItems, this);


    countdownToRebuild = WRITES_BETWEEN_REBUILD;

View Full Code Here

                    ? null
                    : new FastByIDMap<FastIDSet>(10000);
            theCurrentGeneration = new Generation(newKnownItemsIDs,
                                                  new FastByIDMap<float[]>(10000),
                                                  new FastByIDMap<float[]>(10000),
                                                  new FastIDSet(1000),
                                                  new FastIDSet(1000));
          }
          
            
          log.info("Computing model from input in {}", inputDir);


          FastByIDMap<FastIDSet> knownItemIDs;
          if (Boolean.valueOf(System.getProperty(Generation.NO_KNOWN_ITEMS_KEY))) {
            knownItemIDs = null;
          } else {
            knownItemIDs = new FastByIDMap<FastIDSet>(10000);
          }
          FastByIDMap<FastByIDFloatMap> RbyRow = new FastByIDMap<FastByIDFloatMap>(10000);
          FastByIDMap<FastByIDFloatMap> RbyColumn = new FastByIDMap<FastByIDFloatMap>(10000);
          FastIDSet itemTagIDs = new FastIDSet(1000);
          FastIDSet userTagIDs = new FastIDSet(1000);
          InputFilesReader.readInputFiles(knownItemIDs, RbyRow, RbyColumn, itemTagIDs, userTagIDs, inputDir);
      
          if (!RbyRow.isEmpty() && !RbyColumn.isEmpty()) {
            // Compute latest generation:
            MatrixFactorizer als;

View Full Code Here

    Preconditions.checkArgument(inputDir.isDirectory());
    
    FastByIDMap<FastIDSet> knownItemIDs = new FastByIDMap<FastIDSet>(10000);
    FastByIDMap<FastByIDFloatMap> RbyRow = new FastByIDMap<FastByIDFloatMap>(10000);
    FastByIDMap<FastByIDFloatMap> RbyColumn = new FastByIDMap<FastByIDFloatMap>(10000);
    FastIDSet itemTagIDs = new FastIDSet(1000);
    FastIDSet userTagIDs = new FastIDSet(1000);
    InputFilesReader.readInputFiles(knownItemIDs, RbyRow, RbyColumn, itemTagIDs, userTagIDs, inputDir);


    int numUsers = RbyRow.size();
    int numItems = RbyColumn.size();
    if (numUsers == 0 || numItems == 0) {

View Full Code Here

        } else {
          MatrixUtils.addTo(userID, itemID, value, rbyRow, rbyColumn);
        }


        if (knownItemIDs != null) {
          FastIDSet itemIDs = knownItemIDs.get(userID);
          if (Float.isNaN(value)) {
            // Remove, not set
            if (itemIDs != null) {
              itemIDs.remove(itemID);
              if (itemIDs.isEmpty()) {
                knownItemIDs.remove(userID);
              }
            }
          } else {
            if (itemIDs == null) {
              itemIDs = new FastIDSet();
              knownItemIDs.put(userID, itemIDs);
            }
            itemIDs.add(itemID);
          }
        }


        if (lines % 1000000 == 0) {
          log.info("Finished {} lines", lines);

View Full Code Here

    
    if (updatedKnownItemIDs != null) {
      updateMap(currentGeneration.getKnownItemIDs(), updatedKnownItemIDs, currentGeneration.getKnownItemLock().writeLock());
    }
    
    FastIDSet updatedXKeys = keysToSet(newX);
    FastIDSet updatedYKeys = keysToSet(newY);
    FastIDSet updatedUserIDsForKnownItems = updatedKnownItemIDs == null ? null : keysToSet(updatedKnownItemIDs);
    
    synchronized (lockForRecent) {
      // Not recommended to set this to 'false' -- may be useful in rare cases      
      if (Boolean.parseBoolean(System.getProperty("model.removeNotUpdatedData", "true"))) {
        log.info("Pruning old entries...");

View Full Code Here

      writeLock.unlock();
    }
  }
  
  private static FastIDSet keysToSet(FastByIDMap<?> map) {
    FastIDSet result = new FastIDSet(map.size());
    LongPrimitiveIterator it = map.keySetIterator();
    while (it.hasNext()) {
      result.add(it.nextLong());
    }
    return result;
  }

View Full Code Here


  private static void printKnownItems(FastByIDMap<FastIDSet> known, Appendable out) throws IOException {
    if (known != null) {
      for (FastByIDMap.MapEntry<FastIDSet> entry : known.entrySet()) {
        long id = entry.getKey();
        FastIDSet keys = entry.getValue();
        StringBuilder line = new StringBuilder();
        line.append(id);
        for (long key : keys) {
          line.append('\t').append(key);
        }

View Full Code Here

                                   RescorerProvider provider, // ignored
                                   Multimap<Long,RecommendedItem> testData) throws TasteException {
    FastByIDMap<FastIDSet> converted = new FastByIDMap<FastIDSet>(testData.size());
    for (long userID : testData.keySet()) {
      Collection<RecommendedItem> userTestData = testData.get(userID);
      FastIDSet itemIDs = new FastIDSet(userTestData.size());
      converted.put(userID, itemIDs);
      for (RecommendedItem datum : userTestData) {
        itemIDs.add(datum.getItemID());
      }
    }
    return evaluate(recommender, converted);
  }

View Full Code Here

    
    Processor<Long> processor = new Processor<Long>() {
      private final RandomGenerator random = RandomManager.getRandom();      
      @Override
      public void process(Long userID, long count) throws ExecutionException {
        FastIDSet testItemIDs = testData.get(userID);
        int numTest = testItemIDs.size();  
        for (int i = 0; i < numTest; i++) {
  
          long randomTestItemID;
          long randomTrainingItemID;
          synchronized (random) {
            randomTestItemID = RandomUtils.randomFrom(testItemIDs, random);
            do {
              randomTrainingItemID = allItemIDs[random.nextInt(allItemIDs.length)];
            } while (testItemIDs.contains(randomTrainingItemID));
          }
  
          float relevantEstimate;
          float nonRelevantEstimate;
          try {

View Full Code Here

0 1 2 3 4

TOP

Related Classes of net.myrrix.common.collection.FastIDSet

net.myrrix.client.CLI

net.myrrix.client.ClientRecommender

net.myrrix.client.SimpleTest

net.myrrix.client.translating.TranslatingClientRecommender

net.myrrix.common.LoadRunner

net.myrrix.common.math.MatrixUtils

net.myrrix.common.random.RandomUtils

net.myrrix.online.candidate.ExampleCandidateFilter

net.myrrix.online.candidate.LocationSensitiveHash

net.myrrix.online.candidate.LocationSensitiveHashTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.