/**
*
*/
package com.browseengine.bobo.util.test;
import java.util.Arrays;
import java.util.Random;
import junit.framework.TestCase;
import org.apache.lucene.util.OpenBitSet;
import com.browseengine.bobo.util.BigIntBuffer;
import com.browseengine.bobo.util.BigNestedIntArray;
import com.browseengine.bobo.util.BigNestedIntArray.BufferedLoader;
import com.browseengine.bobo.util.BigNestedIntArray.Loader;
public class BigNestedIntArrayTest extends TestCase {
public void testBasic() throws Throwable {
int maxId = 3000;
int[] count = new int[maxId];
BufferedLoader loader = new BufferedLoader(maxId);
for (int id = 0; id < maxId; id++) {
for (int val = 0; val < 2000; val += (id + 1)) {
if (loader.add(id, val)) count[id]++;
}
}
BigNestedIntArray nestedArray = new BigNestedIntArray();
nestedArray.load(maxId, loader);
int[] buf = new int[1024];
for (int id = 0; id < maxId; id++) {
int cnt = nestedArray.getData(id, buf);
assertEquals("item count", count[id], cnt);
if (cnt > 0) {
int val = 0;
for (int i = 0; i < cnt; i++) {
assertEquals("item[" + i + "]", val, buf[i]);
val += (id + 1);
}
}
}
}
public void testSparseIds() throws Throwable {
int maxId = 100000;
int[] count = new int[maxId];
BufferedLoader loader = new BufferedLoader(maxId);
for (int id = 0; id < maxId; id += ((id >> 2) + 1)) {
for (int val = 0; val < 3000; val += (id + 1)) {
if (loader.add(id, val)) count[id]++;
}
}
BigNestedIntArray nestedArray = new BigNestedIntArray();
nestedArray.load(maxId, loader);
int[] buf = new int[1024];
for (int id = 0; id < maxId; id++) {
int cnt = nestedArray.getData(id, buf);
assertEquals("item count", count[id], cnt);
if (cnt > 0) {
int val = 0;
for (int i = 0; i < cnt; i++) {
assertEquals("item[" + i + "]", val, buf[i]);
val += (id + 1);
}
}
}
}
public void testBufferedLoaderReuse() throws Throwable {
int maxId = 5000;
int[] maxNumItems = { 25, 50, 20, 100, 15, 500, 10, 1000, 5, 2000, 2 };
int[][] count = new int[maxNumItems.length][maxId];
BigIntBuffer buffer = new BigIntBuffer();
BufferedLoader loader = new BufferedLoader(maxId, BigNestedIntArray.MAX_ITEMS, buffer);
BigNestedIntArray[] nestedArray = new BigNestedIntArray[maxNumItems.length];
for (int i = 0; i < maxNumItems.length; i++) {
for (int id = 0; id < maxId; id++) {
int cnt = id % (maxNumItems[i] + 1);
for (int val = 0; val < cnt; val++) {
if (loader.add(id, val)) count[i][id]++;
}
}
nestedArray[i] = new BigNestedIntArray();
nestedArray[i].load(maxId, loader);
loader.reset(maxId, BigNestedIntArray.MAX_ITEMS, buffer);
}
for (int i = 0; i < maxNumItems.length; i++) {
int[] buf = new int[1024];
for (int id = 0; id < maxId; id++) {
int cnt = nestedArray[i].getData(id, buf);
assertEquals("count[" + i + "," + id + "]", count[i][id], cnt);
if (cnt > 0) {
for (int val = 0; val < cnt; val++) {
assertEquals("item[" + i + "," + id + "," + val + "]", val, buf[val]);
}
}
}
}
}
public void testMemoryReuse() throws Throwable {
int maxId = 4096;
int[] maxNumItems = { 1, 1, 2, 2, 3, 3, 3, 3, 1, 1 };
int[] minNumItems = { 1, 1, 0, 1, 0, 0, 2, 3, 1, 0 };
int[] count = new int[maxId];
BigIntBuffer buffer = new BigIntBuffer();
BufferedLoader loader = null;
BigNestedIntArray nestedArray = new BigNestedIntArray();
Random rand = new Random();
for (int i = 0; i < maxNumItems.length; i++) {
loader = new BufferedLoader(maxId, BigNestedIntArray.MAX_ITEMS, buffer);
for (int id = 0; id < maxId; id++) {
count[id] = 0;
int cnt = Math.max(rand.nextInt(maxNumItems[i] + 1), minNumItems[i]);
for (int val = 0; val < cnt; val++) {
if (loader.add(id, val)) count[id]++;
}
}
nestedArray.load(maxId, loader);
int[] buf = new int[1024];
for (int id = 0; id < maxId; id++) {
int cnt = nestedArray.getData(id, buf);
assertEquals("count[" + i + "," + id + "]", count[id], cnt);
if (cnt > 0) {
for (int val = 0; val < cnt; val++) {
assertEquals("item[" + i + "," + id + "," + val + "]", val, buf[val]);
}
}
}
if (i == 0) {
maxId = maxId * 2;
count = new int[maxId];
}
}
}
public void testAllocThenAddData() throws Throwable {
int maxId = 5000;
int[] maxNumItems = { 25, 50, 20, 100, 15, 500, 10, 1000, 5, 1024, 2 };
int[][] count = new int[maxNumItems.length][maxId];
AllocOnlyTestLoader loader = new AllocOnlyTestLoader(maxId);
BigNestedIntArray[] nestedArray = new BigNestedIntArray[maxNumItems.length];
for (int i = 0; i < maxNumItems.length; i++) {
for (int id = 0; id < maxId; id++) {
int cnt = id % (maxNumItems[i] + 1);
loader.addSize(id, cnt);
count[i][id] = cnt;
}
nestedArray[i] = new BigNestedIntArray();
nestedArray[i].load(maxId, loader);
loader.reset();
for (int id = 0; id < maxId; id++) {
for (int data = 0; data < count[i][id]; data++) {
nestedArray[i].addData(id, data);
}
}
}
for (int i = 0; i < maxNumItems.length; i++) {
int[] buf = new int[1024];
for (int id = 0; id < maxId; id++) {
int cnt = nestedArray[i].getData(id, buf);
assertEquals("count[" + i + "," + id + "]", count[i][id], cnt);
if (cnt > 0) {
for (int val = 0; val < cnt; val++) {
assertEquals("item[" + i + "," + id + "," + val + "]", val, buf[val]);
}
}
}
}
}
/**
* A loader that allocate data storage wihtout loading data to BigNestedIntArray.
* Note that this loader supports only non-negative integer data.
*/
public final static class AllocOnlyTestLoader extends Loader {
private final int[] _maxNumItems;
public AllocOnlyTestLoader(int maxdoc) {
_maxNumItems = new int[maxdoc];
}
public void addSize(int docid, int size) {
_maxNumItems[docid] = size;
}
public void reset() {
Arrays.fill(_maxNumItems, 0);
}
@Override
public void load() throws Exception {
for (int i = 0; i < _maxNumItems.length; i++) {
if (_maxNumItems[i] > 0) {
allocate(i, _maxNumItems[i], true);
}
}
}
}
public void testMaxItems() throws Throwable {
int maxId = 5000;
int[] maxNumItems = { 25, 50, 20, 100, 15, 500, 10, 1000, 5, 1024, 2 };
int[][] count = new int[maxNumItems.length][maxId];
AllocOnlyTestLoader loader = new AllocOnlyTestLoader(maxId);
BigNestedIntArray[] nestedArray = new BigNestedIntArray[maxNumItems.length];
for (int i = 0; i < maxNumItems.length; i++) {
for (int id = 0; id < maxId; id++) {
int cnt = id % 2000;
loader.addSize(id, cnt);
count[i][id] = cnt;
}
nestedArray[i] = new BigNestedIntArray();
nestedArray[i].setMaxItems(maxNumItems[i]);
nestedArray[i].load(maxId, loader);
loader.reset();
for (int id = 0; id < maxId; id++) {
boolean failed = false;
for (int data = 0; data < count[i][id]; data++) {
if (nestedArray[i].addData(id, data)) {
if (!failed && (data + 1 > maxNumItems[i])) {
failed = true;
assertEquals("maxItems", data, maxNumItems[i]);
}
}
}
}
}
}
public void testCountNoReturnWithFilter() throws Throwable {
int maxId = 20;
int numVals = 10;
int[] count = new int[numVals];
BufferedLoader loader = new BufferedLoader(maxId);
for (int val = 0; val < numVals; val++) {
for (int i = 0; i < maxId - val; i++) {
loader.add(i, val);
}
}
BigNestedIntArray nestedArray = new BigNestedIntArray();
nestedArray.load(maxId, loader);
OpenBitSet filter = new OpenBitSet(numVals);
for (int i = 0; i < numVals; i++) {
if (i % 2 == 0) {
filter.set(i);
}
}
for (int i = 0; i < maxId; i++) {
nestedArray.countNoReturnWithFilter(i, count, filter);
}
for (int i = 0; i < numVals; i++) {
if (i % 2 == 0) {
assertTrue(count[i] == maxId - i);
} else {
assertTrue(count[i] == 0);
}
}
return;
}
}