package dovetaildb.bagindex;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Iterator;
import java.util.Map;
import java.util.Random;
import junit.framework.TestCase;
import dovetaildb.bagindex.BlueSteelBagIndex.MergeIterator;
import dovetaildb.bagindex.BlueSteelBagIndex.PostingNode;
import dovetaildb.bagindex.BlueSteelBagIndex.TokenRec;
import dovetaildb.bagindex.BlueSteelBagIndex.TokenTable;
import dovetaildb.bagindex.BlueSteelBagIndex.SegmentPush;
import dovetaildb.bagindex.BlueSteelBagIndex.TraversalStack;
import dovetaildb.bytes.ArrayBytes;
public class BlueSteelBagIndexTest extends BagIndexTest {
private ArrayBytes bytes(String s) {
try {
return new ArrayBytes(s.getBytes("utf-8"));
} catch (UnsupportedEncodingException e) {
throw new RuntimeException(e);
}
}
public void testMergeIterator() {
ArrayList<String> alpha = new ArrayList<String>();
alpha.add("Phil");
alpha.add("Zax");
ArrayList<String> beta = new ArrayList<String>();
beta.add("Alice");
beta.add("Phil");
class StringMergeIterator extends MergeIterator<String,String> {
public StringMergeIterator(Iterator<String> a, Iterator<String> b) {
super(a,b);
}
public int compare(String a, String b) {
return a.compareTo(b);
}
}
StringMergeIterator i = new StringMergeIterator(alpha.iterator(), beta.iterator());
String l,r;
i.next(); l = i.getLeft(); r = i.getRight();
assertEquals(null, l);
assertEquals("Alice", r);
i.next(); l = i.getLeft(); r = i.getRight();
assertEquals("Phil", l);
assertEquals("Phil", r);
i.next(); l = i.getLeft(); r = i.getRight();
assertEquals("Zax", l);
assertEquals(null, r);
i.next(); l = i.getLeft(); r = i.getRight();
assertEquals(null, l);
assertEquals(null, r);
}
public void compareSegments(SegmentPush push, long[] docIds) {
TraversalStack traversal = new TraversalStack(push);
int i=0;
while(traversal.nextOrDown()) {
assertEquals(docIds[i++], traversal.getCurrent().getDocId());
}
assertFalse(traversal.nextOrDown());
}
public void testSpliceSegmentsIntoSegmentPush() {
ArrayList<PostingNode> nodes = new ArrayList<PostingNode>();
PostingNode node3 = new PostingNode(3, new ArrayBytes(new byte[]{53}));
nodes.add(node3);
node3.setIdxPos(333);
ArrayList<PostingNode> subnodes1 = new ArrayList<PostingNode>();
PostingNode node5 = new PostingNode(5, new ArrayBytes(new byte[]{53}));
node5.setIdxPos(555);
subnodes1.add(node5);
PostingNode node6 = new PostingNode(6, new ArrayBytes(new byte[]{53}));
subnodes1.add(node6);
SegmentPush sub1 = new SegmentPush(subnodes1);
PostingNode node7 = new PostingNode(sub1, 7, new ArrayBytes(new byte[]{57}));
nodes.add(node7);
PostingNode node9 = new PostingNode(null, 9, new ArrayBytes(new byte[]{59}));
nodes.add(node9);
SegmentPush push = new SegmentPush(nodes);
SegmentPush o;
List<EditRec> edits = new ArrayList<EditRec>();
// edits.add(new EditRec(1, new ArrayBytes(new byte[]{11}), false));
// o = BlueSteelBagIndex.spliceEditsIntoSegmentPush(edits, push);
// assertEquals(6, o.getCount());
// Iterator<PostingNode> i = o.iterator();
// assertTrue(i.hasNext());
// assertEquals(1, i.next().getDocId());
// assertTrue(i.hasNext());
// PostingNode n = i.next();
// assertEquals(9, n.getDocId());
// assertEquals(333, n.getPush().getIdxPos());
edits = new ArrayList<EditRec>();
edits.add(new EditRec(2, new ArrayBytes(new byte[]{0}), false));
o = BlueSteelBagIndex.spliceEditsIntoSegmentPush(edits, push);
compareSegments(o, new long[]{2,3,5,6,7,9});
edits = new ArrayList<EditRec>();
edits.add(new EditRec(9, new ArrayBytes(new byte[]{127}), false));
o = BlueSteelBagIndex.spliceEditsIntoSegmentPush(edits, push);
compareSegments(o, new long[]{3,5,6,7,9,9});
edits = new ArrayList<EditRec>();
edits.add(new EditRec(7, new ArrayBytes(new byte[]{0}), false));
o = BlueSteelBagIndex.spliceEditsIntoSegmentPush(edits, push);
compareSegments(o, new long[]{3,5,6,7,7,9});
edits = new ArrayList<EditRec>();
edits.add(new EditRec(3, new ArrayBytes(new byte[]{127}), false));
o = BlueSteelBagIndex.spliceEditsIntoSegmentPush(edits, push);
compareSegments(o, new long[]{3,3,5,6,7,9});
edits = new ArrayList<EditRec>();
edits.add(new EditRec(5, new ArrayBytes(new byte[]{127}), false));
o = BlueSteelBagIndex.spliceEditsIntoSegmentPush(edits, push);
compareSegments(o, new long[]{3,5,5,6,7,9});
edits = new ArrayList<EditRec>();
edits.add(new EditRec(6, new ArrayBytes(new byte[]{127}), false));
o = BlueSteelBagIndex.spliceEditsIntoSegmentPush(edits, push);
compareSegments(o, new long[]{3,5,6,6,7,9});
edits = new ArrayList<EditRec>();
edits.add(new EditRec(8, new ArrayBytes(new byte[]{127}), false));
o = BlueSteelBagIndex.spliceEditsIntoSegmentPush(edits, push);
compareSegments(o, new long[]{3,5,6,7,8,9});
edits = new ArrayList<EditRec>();
edits.add(makeEdit(node3, true));
o = BlueSteelBagIndex.spliceEditsIntoSegmentPush(edits, push);
compareSegments(o, new long[]{5,6,7,9});
edits = new ArrayList<EditRec>();
edits.add(makeEdit(node5, true));
o = BlueSteelBagIndex.spliceEditsIntoSegmentPush(edits, push);
compareSegments(o, new long[]{3,6,7,9});
edits = new ArrayList<EditRec>();
edits.add(makeEdit(node6, true));
o = BlueSteelBagIndex.spliceEditsIntoSegmentPush(edits, push);
compareSegments(o, new long[]{3,5,7,9});
edits = new ArrayList<EditRec>();
edits.add(makeEdit(node7, true));
o = BlueSteelBagIndex.spliceEditsIntoSegmentPush(edits, push);
compareSegments(o, new long[]{3,5,6,9});
edits = new ArrayList<EditRec>();
edits.add(makeEdit(node9, true));
o = BlueSteelBagIndex.spliceEditsIntoSegmentPush(edits, push);
compareSegments(o, new long[]{3,5,6,7});
edits = new ArrayList<EditRec>();
edits.add(new EditRec(2, new ArrayBytes(new byte[]{0}), false));
edits.add(makeEdit(node3, true));
o = BlueSteelBagIndex.spliceEditsIntoSegmentPush(edits, push);
compareSegments(o, new long[]{2,5,6,7,9});
edits = new ArrayList<EditRec>();
edits.add(makeEdit(node3, true));
edits.add(new EditRec(4, new ArrayBytes(new byte[]{0}), false));
o = BlueSteelBagIndex.spliceEditsIntoSegmentPush(edits, push);
compareSegments(o, new long[]{4,5,6,7,9});
edits = new ArrayList<EditRec>();
edits.add(new EditRec(7, new ArrayBytes(new byte[]{0}), false));
edits.add(makeEdit(node7, true));
o = BlueSteelBagIndex.spliceEditsIntoSegmentPush(edits, push);
compareSegments(o, new long[]{3,5,6,7,9});
edits = new ArrayList<EditRec>();
edits.add(makeEdit(node9, true));
edits.add(new EditRec(10, new ArrayBytes(new byte[]{0}), false));
o = BlueSteelBagIndex.spliceEditsIntoSegmentPush(edits, push);
compareSegments(o, new long[]{3,5,6,7,10});
edits = new ArrayList<EditRec>();
edits.add(new EditRec(8, new ArrayBytes(new byte[]{0}), false));
edits.add(makeEdit(node9, true));
o = BlueSteelBagIndex.spliceEditsIntoSegmentPush(edits, push);
compareSegments(o, new long[]{3,5,6,7,8});
edits = new ArrayList<EditRec>();
edits.add(makeEdit(node3, true));
edits.add(makeEdit(node5, true));
edits.add(makeEdit(node6, true));
edits.add(makeEdit(node7, true));
edits.add(makeEdit(node9, true));
o = BlueSteelBagIndex.spliceEditsIntoSegmentPush(edits, push);
compareSegments(o, new long[]{});
edits = new ArrayList<EditRec>();
edits.add(makeEdit(node6, true));
edits.add(makeEdit(node7, true));
edits.add(makeEdit(node9, true));
o = BlueSteelBagIndex.spliceEditsIntoSegmentPush(edits, push);
compareSegments(o, new long[]{3,5});
}
private EditRec makeEdit(PostingNode node, boolean isDel) {
return new EditRec(node.getDocId(), node.getToken(), isDel);
}
public void testBalanceSegmentPush() {
SegmentPush o;
ArrayList<PostingNode> nodes = new ArrayList<PostingNode>();
nodes.add(new PostingNode(1L, new ArrayBytes(new byte[]{11})));
nodes.add(new PostingNode(2L, new ArrayBytes(new byte[]{22})));
nodes.add(new PostingNode(3L, new ArrayBytes(new byte[]{33})));
nodes.add(new PostingNode(4L, new ArrayBytes(new byte[]{44})));
nodes.add(new PostingNode(5L, new ArrayBytes(new byte[]{55})));
nodes.add(new PostingNode(6L, new ArrayBytes(new byte[]{66})));
nodes.add(new PostingNode(7L, new ArrayBytes(new byte[]{77})));
o=BlueSteelBagIndex.balanceSegmentPush(new SegmentPush(nodes), 4);
compareSegments(o, new long[]{1,2,3,4,5,6,7});
assertEquals(7, o.getCount());
assertEquals(4, o.nodes.size());
o=BlueSteelBagIndex.balanceSegmentPush(new SegmentPush(nodes), 2);
compareSegments(o, new long[]{1,2,3,4,5,6,7});
assertEquals(7, o.getCount());
assertEquals(2, o.nodes.size());
}
public void testApplyEditsToTokenTable() {
List<EditRec> edits;
TokenTable o;
TokenRec tr;
ArrayList<TokenRec> tokenRecs = new ArrayList<TokenRec>();
//tokenRecs.add(new TokenRec(0, null, null)); //byte token, TokenTable tokenTable, SegmentPush segmentPush
TokenTable root = new TokenTable(null, tokenRecs);
edits = new ArrayList<EditRec>();
edits.add(new EditRec(1, bytes("age:13"), false));
edits.add(new EditRec(1, bytes("name:phil"), false));
o=BlueSteelBagIndex.applyEditsToTokenTable(edits, root);
System.out.println(o);
Iterator<TokenRec> i=o.getTokenRecs().iterator();
assertTrue(i.hasNext());
tr = i.next();
assertEquals('a', tr.getToken());
compareSegments(tr.getSegmentPush(), new long[]{1});
assertTrue(i.hasNext());
tr = i.next();
assertEquals('n', tr.getToken());
compareSegments(tr.getSegmentPush(), new long[]{1});
assertFalse(i.hasNext());
edits = new ArrayList<EditRec>();
edits.add(new EditRec(2, bytes("age:14"), false));
edits.add(new EditRec(2, bytes("name:joe"), false));
o=BlueSteelBagIndex.applyEditsToTokenTable(edits, o);
System.out.println(o);
}
public void testApplyDocEdits_complex() {
/*
Random r = new Random(12345);
ArrayList<EditRec> edits;
DocRec head = null;
long ct = 0;
RecBuffer<MemDocRec> results = null;
edits = new ArrayList<EditRec>();
HashSet<String> seen = new HashSet<String>();
for(int i=0; i<1024; i++) {
long id = r.nextInt(100);
byte[] field = new byte[r.nextInt(10)];
for(int j=0; j<field.length; j++) {
field[j] = (new byte[]{'a','b','c','d','e','f','g','h','i','j'})[r.nextInt(10)];
}
String rep = new String(field)+" "+id;
if (!seen.contains(rep)) {
seen.add(rep);
edits.add(new EditRec(id, new ArrayBytes(field), false));
}
}
// System.out.println(edits);
List<EditRec> removeQueue = new ArrayList<EditRec>(edits.subList(0, edits.size()/2));
List<EditRec> addQueue = new ArrayList<EditRec>(edits.subList(edits.size()/2, edits.size()));
edits = new ArrayList<EditRec>(removeQueue);
Collections.sort(edits);
results = ImmutableBagIndex.applyDocEdits(edits, head, 0);
head = results.head; ct = results.cumulativeCount();
//System.out.println(results);
long expectedCt = removeQueue.size();
assertEquals(expectedCt, ct);
for(EditRec edit : removeQueue) { edit.isDelete = true; }
int modCap=1;
while( (!addQueue.isEmpty()) && (!removeQueue.isEmpty()) ) {
results = ImmutableBagIndex.balance(results, r.nextInt(10)+5);
head = results.head; ct = results.cumulativeCount();
//System.out.println(results);
assertEquals(expectedCt, ct);
int numRemovals = r.nextInt(Math.min(modCap, removeQueue.size()+1));
int numAdds = r.nextInt(Math.min(modCap, addQueue.size()+1));
edits = new ArrayList<EditRec>();
edits.addAll(removeQueue.subList(0, numRemovals));
edits.addAll( addQueue.subList(0, numAdds));
Collections.sort(edits);
removeQueue = removeQueue.subList(numRemovals, removeQueue.size());
addQueue = addQueue.subList(numAdds, addQueue.size());
expectedCt += numAdds - numRemovals;
System.out.println(ImmutableBagIndex.docListToString(new ImmutableBagIndex.BoundedDocRec(results.head,results.ct),""));
System.out.println(edits.toString());
results = ImmutableBagIndex.applyDocEdits(edits, head, ct);
head = results.head; ct = results.cumulativeCount();
System.out.println(ImmutableBagIndex.docListToString(new ImmutableBagIndex.BoundedDocRec(results.head,results.ct),""));
assertEquals(expectedCt, ct);
modCap += 2;
}
*/
}
@Override
protected BagIndex createIndex() {
return new MemoryBlueSteelBagIndex();
}
}