Package net.myrrix.common.collection

Examples of net.myrrix.common.collection.FastIDSet$KeyIterator

63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
    Preconditions.checkNotNull(client);
    Preconditions.checkNotNull(dataDirectory);
    Preconditions.checkArgument(steps > 0); 
   
    log.info("Reading IDs...");   
    FastIDSet userIDsSet = new FastIDSet();
    FastIDSet itemIDsSet = new FastIDSet();
    Splitter comma = Splitter.on(',');
    for (File f : dataDirectory.listFiles(new PatternFilenameFilter(".+\\.csv(\\.(zip|gz))?"))) {
      for (CharSequence line : new FileLineIterable(f)) {
        Iterator<String> it = comma.split(line).iterator();
        userIDsSet.add(Long.parseLong(it.next()));
        itemIDsSet.add(Long.parseLong(it.next()));
      }
    }
   
    this.client = client;   
    this.uniqueUserIDs = userIDsSet.toArray();
    this.uniqueItemIDs = itemIDsSet.toArray();
    this.steps = steps;
  }
View Full Code Here
113
114
115
116
117
118
119
120
121
122
123
124
    }

    modelFile = new File(inputDir, "model.bin.gz");
    appendFile = new File(inputDir, "append.bin.gz");

    recentlyActiveUsers = new FastIDSet();
    recentlyActiveItems = new FastIDSet();
    hasher = new OneWayMigrator();
   
    loader = new GenerationLoader(recentlyActiveUsers, recentlyActiveItems, this);

    countdownToRebuild = WRITES_BETWEEN_REBUILD;
View Full Code Here
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
                    ? null
                    : new FastByIDMap<FastIDSet>(10000);
            theCurrentGeneration = new Generation(newKnownItemsIDs,
                                                  new FastByIDMap<float[]>(10000),
                                                  new FastByIDMap<float[]>(10000),
                                                  new FastIDSet(1000),
                                                  new FastIDSet(1000));
          }
         
           
          log.info("Computing model from input in {}", inputDir);

          FastByIDMap<FastIDSet> knownItemIDs;
          if (Boolean.valueOf(System.getProperty(Generation.NO_KNOWN_ITEMS_KEY))) {
            knownItemIDs = null;
          } else {
            knownItemIDs = new FastByIDMap<FastIDSet>(10000);
          }
          FastByIDMap<FastByIDFloatMap> RbyRow = new FastByIDMap<FastByIDFloatMap>(10000);
          FastByIDMap<FastByIDFloatMap> RbyColumn = new FastByIDMap<FastByIDFloatMap>(10000);
          FastIDSet itemTagIDs = new FastIDSet(1000);
          FastIDSet userTagIDs = new FastIDSet(1000);
          InputFilesReader.readInputFiles(knownItemIDs, RbyRow, RbyColumn, itemTagIDs, userTagIDs, inputDir);
     
          if (!RbyRow.isEmpty() && !RbyColumn.isEmpty()) {
            // Compute latest generation:
            MatrixFactorizer als;
View Full Code Here
66
67
68
69
70
71
72
73
74
75
76
77
    Preconditions.checkArgument(inputDir.isDirectory());
   
    FastByIDMap<FastIDSet> knownItemIDs = new FastByIDMap<FastIDSet>(10000);
    FastByIDMap<FastByIDFloatMap> RbyRow = new FastByIDMap<FastByIDFloatMap>(10000);
    FastByIDMap<FastByIDFloatMap> RbyColumn = new FastByIDMap<FastByIDFloatMap>(10000);
    FastIDSet itemTagIDs = new FastIDSet(1000);
    FastIDSet userTagIDs = new FastIDSet(1000);
    InputFilesReader.readInputFiles(knownItemIDs, RbyRow, RbyColumn, itemTagIDs, userTagIDs, inputDir);

    int numUsers = RbyRow.size();
    int numItems = RbyColumn.size();
    if (numUsers == 0 || numItems == 0) {
View Full Code Here
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
        } else {
          MatrixUtils.addTo(userID, itemID, value, rbyRow, rbyColumn);
        }

        if (knownItemIDs != null) {
          FastIDSet itemIDs = knownItemIDs.get(userID);
          if (Float.isNaN(value)) {
            // Remove, not set
            if (itemIDs != null) {
              itemIDs.remove(itemID);
              if (itemIDs.isEmpty()) {
                knownItemIDs.remove(userID);
              }
            }
          } else {
            if (itemIDs == null) {
              itemIDs = new FastIDSet();
              knownItemIDs.put(userID, itemIDs);
            }
            itemIDs.add(itemID);
          }
        }

        if (lines % 1000000 == 0) {
          log.info("Finished {} lines", lines);
View Full Code Here
61
62
63
64
65
66
67
68
69
70
71
72
73
   
    if (updatedKnownItemIDs != null) {
      updateMap(currentGeneration.getKnownItemIDs(), updatedKnownItemIDs, currentGeneration.getKnownItemLock().writeLock());
    }
   
    FastIDSet updatedXKeys = keysToSet(newX);
    FastIDSet updatedYKeys = keysToSet(newY);
    FastIDSet updatedUserIDsForKnownItems = updatedKnownItemIDs == null ? null : keysToSet(updatedKnownItemIDs);
   
    synchronized (lockForRecent) {
      // Not recommended to set this to 'false' -- may be useful in rare cases     
      if (Boolean.parseBoolean(System.getProperty("model.removeNotUpdatedData", "true"))) {
        log.info("Pruning old entries...");       
View Full Code Here
124
125
126
127
128
129
130
131
132
133
134
135
      writeLock.unlock();
    }
  }
 
  private static FastIDSet keysToSet(FastByIDMap<?> map) {
    FastIDSet result = new FastIDSet(map.size());
    LongPrimitiveIterator it = map.keySetIterator();
    while (it.hasNext()) {
      result.add(it.nextLong());
    }
    return result;
  }
View Full Code Here
113
114
115
116
117
118
119
120
121
122
123
  private static void printKnownItems(FastByIDMap<FastIDSet> known, Appendable out) throws IOException {
    if (known != null) {
      for (FastByIDMap.MapEntry<FastIDSet> entry : known.entrySet()) {
        long id = entry.getKey();
        FastIDSet keys = entry.getValue();
        StringBuilder line = new StringBuilder();
        line.append(id);
        for (long key : keys) {
          line.append('\t').append(key);
        }
View Full Code Here
63
64
65
66
67
68
69
70
71
72
73
74
75
                                   RescorerProvider provider, // ignored
                                   Multimap<Long,RecommendedItem> testData) throws TasteException {
    FastByIDMap<FastIDSet> converted = new FastByIDMap<FastIDSet>(testData.size());
    for (long userID : testData.keySet()) {
      Collection<RecommendedItem> userTestData = testData.get(userID);
      FastIDSet itemIDs = new FastIDSet(userTestData.size());
      converted.put(userID, itemIDs);
      for (RecommendedItem datum : userTestData) {
        itemIDs.add(datum.getItemID());
      }
    }
    return evaluate(recommender, converted);
  }
View Full Code Here
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
   
    Processor<Long> processor = new Processor<Long>() {
      private final RandomGenerator random = RandomManager.getRandom();     
      @Override
      public void process(Long userID, long count) throws ExecutionException {
        FastIDSet testItemIDs = testData.get(userID);
        int numTest = testItemIDs.size(); 
        for (int i = 0; i < numTest; i++) {
 
          long randomTestItemID;
          long randomTrainingItemID;
          synchronized (random) {
            randomTestItemID = RandomUtils.randomFrom(testItemIDs, random);
            do {
              randomTrainingItemID = allItemIDs[random.nextInt(allItemIDs.length)];
            } while (testItemIDs.contains(randomTrainingItemID));
          }
 
          float relevantEstimate;
          float nonRelevantEstimate;
          try {
View Full Code Here
TOP

Related Classes of net.myrrix.common.collection.FastIDSet$KeyIterator

  • net.myrrix.client.CLI
  • net.myrrix.client.ClientRecommender
  • net.myrrix.client.SimpleTest
  • net.myrrix.client.translating.TranslatingClientRecommender
  • net.myrrix.common.LoadRunner
  • net.myrrix.common.math.MatrixUtils
  • net.myrrix.common.random.RandomUtils
  • net.myrrix.online.candidate.ExampleCandidateFilter
  • net.myrrix.online.candidate.LocationSensitiveHash
  • net.myrrix.online.candidate.LocationSensitiveHashTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.