final DataInterface dataInterface = createDataInterface(dataType, cachingType, factory);
dataInterface.dropAllData();
final BufferedReader rdr = new BufferedReader(new FileReader(largeTextFile));
//first fill database by writing data
MutableLong numberOfItemsWritten = new MutableLong(0);
CountDownLatch writeLatch = new CountDownLatch(numberOfThreads);
for (int i = 0; i < numberOfThreads; i++) {
new BigramTestsThread(dataType, numberOfItemsWritten, numberOfItems, rdr, dataInterface, writeLatch, false).start();
}
writeLatch.await();
dataInterface.flush();
//now start threads that will write and read data simultaneously
dataInterface.optimizeForReading();
MutableLong numberOfItemsRead = new MutableLong(0);
numberOfItemsWritten = new MutableLong(0);
CountDownLatch readLatch = new CountDownLatch(numberOfThreads / 2);
writeLatch = new CountDownLatch(numberOfThreads / 2);
long start = System.nanoTime();
for (int i = 0; i < numberOfThreads; i++) {
boolean isReadThread = i % 2 == 0;
new BigramTestsThread(dataType, isReadThread ? numberOfItemsRead : numberOfItemsWritten, Math.min(100 * 1024 * 1024, numberOfItems), rdr, dataInterface, isReadThread ? readLatch : writeLatch, isReadThread).start();
}
readLatch.await(); //this assumes that reading data is faster than writing data.
long endOfRead = System.nanoTime();
writeLatch.await();
dataInterface.flush();
long endOfWrite = System.nanoTime();
double readsPerSecond = numberOfItemsRead.longValue() * 1e9 / (endOfRead - start);
double writesPerSecond = numberOfItemsWritten.longValue() * 1e9 / (endOfWrite - start);
dataInterface.close();
UI.write(factory.getClass().getSimpleName() + " threads " + numberOfThreads + " items " + numberOfItems + " write " + NumUtils.fmt(writesPerSecond) + " read " + NumUtils.fmt(readsPerSecond));
}