} else {
index.writeIndex(DataGenerator.randomIterator(lookupHits, size, hitrate, minStrLen, maxStrLen, minChar, maxChar));
}
} else {
// populate the lookup-hits table
DiskIndex diskIndexTmp = new DiskIndex(path, new DefaultByteRangeComparator(), compress, mmap);
Iterator<Entry<byte[], byte[]>> itTmp = diskIndexTmp.rangeLookup(null, null, true);
while(itTmp.hasNext()) {
if(generator.nextInt() % hitrate == 0)
lookupHits.add(itTmp.next().getKey());
}
diskIndexTmp.destroy();
}
// do a warm-up phase to trick the JVM
int warmups = 5;
while(warmups-- > 0) {
int readEntries = 10000;
// read the disk index
DiskIndex diskIndex = new DiskIndex(path, new DefaultByteRangeComparator(), compress, mmap);
Iterator<Entry<byte[], byte[]>> it = diskIndex.rangeLookup(null, null, true);
while(it.hasNext() && readEntries-- > 0) it.next();
diskIndex.destroy();
}
// clear caches...
Runtime.getRuntime().exec("/bin/sync");
Runtime.getRuntime().exec("/bin/echo 3 > /proc/sys/vm/drop_caches");
// run garbage collection to remove any existing mmap:ed pages
Runtime.getRuntime().gc();
// read the disk index
DiskIndex diskIndex = new DiskIndex(path, new DefaultByteRangeComparator(), compress, mmap);
Iterator<Entry<byte[], byte[]>> it = diskIndex.rangeLookup(null, null, true);
/* iterate over all data in the disk index to measure the prefix lookup throughput */
long iterStart = System.currentTimeMillis();
while(it.hasNext()) it.next();
long iterTime = System.currentTimeMillis() - iterStart;
// Iterator<Entry<ReusableBuffer, ReusableBuffer>> it =
// diskIndex.rangeLookup(null, null);
// while (it.hasNext())
// System.out.println(new String(it.next().getKey().array()));
if(verbose)
System.out.println("performing " + lookups + " random lookups ..." + " hits size: " + lookupHits.size());
// look up each element
int hits = 0;
long sumLookups = 0;
Collections.shuffle(lookupHits);
/* random lookups, this should put random blocks into memory */
for (int i = 0; i < lookups; i++) {
byte[] key;
/* pick a random element that is in the index according to the given hitrate */
if(generator.nextInt() % hitrate == 0) {
key = lookupHits.get(Math.abs(generator.nextInt()) % lookupHits.size());
} else {
key = DataGenerator.createRandomString(minChar, maxChar, maxStrLen+1, maxStrLen*2).getBytes();
}
long t0 = System.currentTimeMillis();
byte[] result = diskIndex.lookup(key);
sumLookups += System.currentTimeMillis() - t0;
if (result != null)
hits++;
//if (i % 100000 == 0 && verbose)
// System.out.println(i);
}
/* random scans */
long scanTotal = 0;
for(int i=0; i < scans; i++) {
byte[] from;
byte[] to;
int firstIndex = Math.abs(generator.nextInt()) % lookupHits.size();
from = lookupHits.get(firstIndex);
to = lookupHits.get(firstIndex + (Math.abs(generator.nextInt()) % (lookupHits.size() - firstIndex)));
Iterator<Entry<byte[], byte[]>> tmpIt = diskIndex.rangeLookup(from, to, true);
/* iterate over all data returned by the range scan */
long scanStart = System.currentTimeMillis();
while(tmpIt.hasNext()) tmpIt.next();
scanTotal += System.currentTimeMillis() - scanStart;
}
System.out.print(path + ", ");
System.out.print(size + ", ");
System.out.print(lookups + ", ");
System.out.print(hits + ", ");
System.out.print(sumLookups + ", ");
/* number of scans */
System.out.print(scans + ", ");
/* total time for scans */
System.out.print(scanTotal + ", ");
// lookups/s (lookup throughput)
System.out.print((int) Math.ceil(((double) lookups / (((double) sumLookups) / 1000.0))) + ", ");
System.out.print((int) Math.ceil((double) iterTime) + ", ");
// entries/s (scan throughput)
System.out.println((int) Math.ceil(((double) size / (((double) iterTime) / 1000.0))));
diskIndex.destroy();
}