Package ivory.lsh.data

Examples of ivory.lsh.data.PermutationByBit

230
231
232
233
234
235
236
237
238
239
240
241
242
    if(fs.exists(new Path(randomPermFile))){
      sLogger.info("Random permutations output path already exists!");
      return randomPermFile;
    }
    SequenceFile.Writer writer = SequenceFile.createWriter(fs, job, new Path(randomPermFile), IntWritable.class, ArrayListOfIntsWritable.class);
    Permutation p = new PermutationByBit(numBits);
    for(int i=0;i<numOfPermutations;i++){
      ArrayListOfIntsWritable perm = p.nextPermutation();
      writer.append(new IntWritable(i), perm);
      sLogger.debug(i +":"+perm);
    }
    writer.close();
    sLogger.info("Random permutations written.");
View Full Code Here
181
182
183
184
185
186
187
188
189
190
191
    PwsimEnvironment.numOfPermutations = 0;  //doesn't matter. not used to compute signatures
   
    if(fs.exists(new Path(randomPermFile))){
      sLogger.info("Random permutations output path already exists!");
    }else{
      Permutation p = new PermutationByBit(vocabSize);
      Permutation.writeToFile(p, numInts, fs, job, randomPermFile); 
    }
    DistributedCache.addCacheFile(new URI(randomPermFile), job);

    sLogger.info("Computing signatures...");
View Full Code Here
171
172
173
174
175
176
177
178
179
180
181
    String randomPermFile = PwsimEnvironment.getPermutationsFile(dir, fs, vocabSize, numInts);

    if (fs.exists(new Path(randomPermFile))) {
      sLogger.info("Random permutations output path already exists!");
    } else {
      Permutation p = new PermutationByBit(vocabSize);
      Permutation.writeToFile(p, numInts, fs, job, randomPermFile);
    }
    DistributedCache.addCacheFile(new URI(randomPermFile), job);

    sLogger.info("Computing signatures...");
View Full Code Here
230
231
232
233
234
235
236
237
238
239
240
241
242
      return randomPermFile;
    }

    SequenceFile.Writer writer = SequenceFile.createWriter(fs, job, new Path(randomPermFile),
        IntWritable.class, ArrayListOfIntsWritable.class);
    Permutation p = new PermutationByBit(numBits);
    for (int i = 0; i < numOfPermutations; i++) {
      ArrayListOfIntsWritable perm = p.nextPermutation();
      writer.append(new IntWritable(i), perm);
      sLogger.debug(i + ":" + perm);
    }
    writer.close();
    sLogger.info("Random permutations written.");
View Full Code Here
186
187
188
189
190
191
192
193
194
195
196
    PwsimEnvironment.numOfPermutations = 0; // doesn't matter. not used to compute signatures

    if (fs.exists(new Path(randomPermFile))) {
      sLogger.info("Random permutations output path already exists!");
    } else {
      Permutation p = new PermutationByBit(vocabSize);
      Permutation.writeToFile(p, numInts, fs, job, randomPermFile);
    }
    DistributedCache.addCacheFile(new URI(randomPermFile), job);

    sLogger.info("Computing signatures...");
View Full Code Here
241
242
243
244
245
246
247
248
249
250
251
252
253
      sLogger.info("Random permutations output path already exists!");
      return randomPermFile;
    }
    SequenceFile.Writer writer = SequenceFile.createWriter(fs, job, new Path(randomPermFile),
        IntWritable.class, ArrayListOfIntsWritable.class);
    Permutation p = new PermutationByBit(numBits);
    for (int i = 0; i < numOfPermutations; i++) {
      ArrayListOfIntsWritable perm = p.nextPermutation();
      writer.append(new IntWritable(i), perm);
      sLogger.debug(i + ":" + perm);
    }
    writer.close();
    sLogger.info("Random permutations written.");
View Full Code Here
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
    return s;
  }

  @Test
  public void testPermute() throws IOException {
    PermutationByBit p = new PermutationByBit(D);

    MinhashSignature s = getRandomSignature();
    System.out.println(s);

    int loopcnt = 0;
    MinhashSignature permutedS = new MinhashSignature(D);
    while (loopcnt++ < 10) {
      ArrayListOfIntsWritable a = p.nextPermutation();
      s.perm(a, permutedS);
      for (int i = 0; i < s.size(); i++) {
        assertTrue(permutedS.containsTerm(s.get(i)));
      }
      assertTrue(permutedS.size() == s.size());
View Full Code Here
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
    return s;
  }

  @Test
  public void testPermuteBit() throws IOException {
    PermutationByBit p = new PermutationByBit(D);

    NBitSignature s = getRandomSignature();

    int cntBits = s.countSetBits();

    int loopcnt = 0;
    NBitSignature permutedS = new NBitSignature(D);
    while (loopcnt++ < 100) {
      ArrayListOfIntsWritable a = p.nextPermutation();
      s.perm(a, permutedS);
      System.out.println(permutedS.countSetBits());
      assertTrue(cntBits == permutedS.countSetBits());
      // System.out.println(permutedS);
    }
View Full Code Here
294
295
296
297
298
299
300
301
302
303
304
305
  //   
  // }

  @Test
  public void testSubSignature() {
    PermutationByBit p = new PermutationByBit(D);
    ArrayListOfIntsWritable a = p.nextPermutation();

    for (int i = 0; i < 100000; i++) {

      NBitSignature s = getRandomSignature();
View Full Code Here
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
    return s;
  }

  @Test
  public void testPermute() throws IOException {
    PermutationByBit p = new PermutationByBit(D);

    MinhashSignature s = getRandomSignature();
    System.out.println(s);

    int loopcnt = 0;
    MinhashSignature permutedS = new MinhashSignature(D);
    while (loopcnt++ < 10) {
      ArrayListOfIntsWritable a = p.nextPermutation();
      s.perm(a, permutedS);
      for (int i = 0; i < s.size(); i++) {
        assertTrue(permutedS.containsTerm(s.get(i)));
      }
      assertTrue(permutedS.size() == s.size());
View Full Code Here
TOP

Related Classes of ivory.lsh.data.PermutationByBit

  • edu.umd.cloud9.io.array.ArrayListOfIntsWritable
  • ivory.lsh.MinhashSignatureTest
  • ivory.lsh.projection.ComputeSignaturesMinhash
  • ivory.lsh.pwsim.GenerateChunkedPermutedTables
  • ivory.lsh.Signature64Test
  • ivory.lsh.SignatureTest
  • java.util.Random
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.