package org.hibernate.search.test.filter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.BitSet;
import java.util.Collections;
import java.util.List;
import java.util.Random;
import junit.framework.TestCase;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.util.DocIdBitSet;
import org.hibernate.search.filter.AndDocIdSet;
/**
* Functionality testcase for org.hibernate.search.filter.AndDocIdSet.
* There is a main method to run some very approximate performance
* comparisons with the use of java.util.BitSet ands.
* The numbers show the AndDocIdSet should be used only when it's not
* possible to rely on a BitSet; in this class however we use BitSet
* as it's useful to test the implementation.
*
* @see AndDocIdSet
* @see BitSet
* @author Sanne Grinovero
*/
public class AndDocIdSetsTest extends TestCase {
static final List<Integer> testDataFrom0to9 = toImmutableList( 0,1,2,3,4,5,6,7,8,9 );
static final List<Integer> testDataFrom1to10 = toImmutableList( 1,2,3,4,5,6,7,8,9,10 );
static final List<Integer> testDataFrom1to9 = toImmutableList( 1,2,3,4,5,6,7,8,9 );
private static final List<Integer> toImmutableList(int... is) {
List<Integer> l = new ArrayList<Integer>( is.length );
for ( int i=0; i<is.length; i++ ) {
l.add( Integer.valueOf( is[i] ) );
}
return Collections.unmodifiableList( l );
}
@SuppressWarnings("unchecked")
List<Integer> andLists(List<Integer>... lists) {
if (lists.length==0) {
return Collections.EMPTY_LIST;
}
List<Integer> result = new ArrayList<Integer>( lists[0] );
for (int i=1; i<lists.length; i++) {
result.retainAll( lists[i] );
}
return result;
}
// auto-testing of test utility methods for AND operations on test arrays
@SuppressWarnings("unchecked")
public void testAndingArrays() {
List<Integer> andLists = andLists( testDataFrom0to9, testDataFrom1to10 );
assertTrue( andLists.containsAll( testDataFrom1to9 ) );
assertFalse( andLists.contains( Integer.valueOf( 0 ) ) );
assertFalse( andLists.contains( Integer.valueOf( 10 ) ) );
assertTrue( andLists.equals( testDataFrom1to9 ) );
DocIdSet docIdSet0_9 = arrayToDocIdSet(testDataFrom0to9);
DocIdSet docIdSet1_10 = arrayToDocIdSet(testDataFrom1to10);
DocIdSet docIdSet1_9 = arrayToDocIdSet(testDataFrom1to9);
assertTrue( docIdSetsEqual( docIdSet0_9, docIdSet0_9 ) );
assertTrue( docIdSetsEqual( docIdSet1_10, docIdSet1_10 ) );
assertFalse( docIdSetsEqual( docIdSet1_10, docIdSet1_9 ) );
assertFalse( docIdSetsEqual( docIdSet0_9, docIdSet1_9 ) );
}
// auto-testing of test utility methods for conversion in DocIdSetIterator
public void testIteratorMatchesTestArray() throws IOException {
DocIdSet docIdSet0_9 = arrayToDocIdSet(testDataFrom0to9);
DocIdSetIterator docIdSetIterator = docIdSet0_9.iterator();
assertTrue( docIdSetIterator.next() );
assertEquals( 0, docIdSetIterator.doc() );
assertTrue( docIdSetIterator.skipTo(9) );
assertFalse( docIdSetIterator.skipTo(10) );
}
public void testAndDocIdSets() {
List<DocIdSet> filters = new ArrayList<DocIdSet>( 2 );
filters.add( arrayToDocIdSet( testDataFrom0to9 ) );
filters.add( arrayToDocIdSet( testDataFrom1to10 ) );
DocIdSet expected = arrayToDocIdSet( testDataFrom1to9 );
DocIdSet testedSet = new AndDocIdSet( filters, 10 );
assertTrue( docIdSetsEqual( expected, testedSet ) );
}
public void testOnRandomBigArrays(){
onRandomBigArraysTest( 13L );
onRandomBigArraysTest( 9L );
onRandomBigArraysTest( 71L );
}
public void onRandomBigArraysTest(long randomSeed) {
List<BitSet> filtersData = makeRandomBitSetList( randomSeed, 4, 1000000, 1500000 );
BitSet expectedBitset = applyANDOnBitSets( filtersData );
List<DocIdSet> filters = toDocIdSetList( filtersData );
DocIdBitSet expectedDocIdSet = new DocIdBitSet( expectedBitset );
DocIdSet testedSet = new AndDocIdSet( filters, 1500000 );
assertTrue( docIdSetsEqual(expectedDocIdSet, testedSet) );
}
private static List<DocIdSet> toDocIdSetList(List<BitSet> filtersData) {
List<DocIdSet> docIdSets = new ArrayList<DocIdSet>( filtersData.size() );
for (BitSet bitSet : filtersData) {
docIdSets.add ( new DocIdBitSet(bitSet) );
}
return docIdSets;
}
public static void main(String[] args) throws IOException {
compareAndingPerformance( 8, 1000000, 1500000 );
compareAndingPerformance( 4, 1000000, 1500000 );
compareAndingPerformance( 2, 1000000, 1500000 );
compareAndingPerformance( 2, 100000000, 150000000 );
compareAndingPerformance( 4, 100000000, 150000000 );
compareAndingPerformance( 8, 100000000, 150000000 );
}
private static void compareAndingPerformance(final int listSize,
final int minBitsSize, final int maxBitsSize) throws IOException {
List<BitSet> filtersData = makeRandomBitSetList( 13L, listSize, minBitsSize, maxBitsSize );
DocIdSet andedByBitsResult = null;
DocIdSet andedByIterationResult = null;
{
long startTime = System.currentTimeMillis();
for ( int i=0; i<1000; i++ ) {
BitSet expectedBitset = applyANDOnBitSets( filtersData );
andedByBitsResult = new DocIdBitSet( expectedBitset );
// iteration is needed to have a fair comparison with other impl:
iterateOnResults( andedByBitsResult );
}
long totalTimeMs = System.currentTimeMillis() - startTime;
System.out.println( "Time to \"AND " + listSize +
" BitSets and iterate on results\" 1000 times: " +
totalTimeMs + "ms. (" +
minBitsSize +" minimum BitSet size)");
}
List<DocIdSet> docIdSetList = toDocIdSetList( filtersData );
{
long startTime = System.currentTimeMillis();
for ( int i=0; i<1000; i++ ) {
andedByIterationResult = new AndDocIdSet( docIdSetList, maxBitsSize );
// iteration is needed because the AND is been done lazily on iterator access:
iterateOnResults( andedByIterationResult );
}
long totalTimeMs = System.currentTimeMillis() - startTime;
System.out.println( "Time to \"use AndDocIdSet iterator on " + listSize +
" Filters and iterate on results\" 1000 times: " +
totalTimeMs + "ms. (" +
minBitsSize +" minimum BitSet size)");
}
System.out.println(" Results are same: " + docIdSetsEqual( andedByBitsResult, andedByIterationResult ) );
}
private static int iterateOnResults(DocIdSet docIdBitSet) throws IOException {
DocIdSetIterator iterator = docIdBitSet.iterator();
int i = 0;
while ( iterator.next() ) {
i += iterator.doc();
}
return i;
}
private static final BitSet applyANDOnBitSets(final List<BitSet> filtersData) {
BitSet andedBitSet = null;
for (BitSet bits : filtersData) {
if ( andedBitSet==null ) {
andedBitSet = (BitSet) bits.clone();
}
else {
andedBitSet.and( bits );
}
}
return andedBitSet;
}
private static List<BitSet> makeRandomBitSetList(final long randomSeed, final int listSize,
final int minBitsSize, final int maxBitsSize) {
Random r = new Random( randomSeed ); //have a fixed Seed for repeatable tests
List<BitSet> restulList = new ArrayList<BitSet>( listSize );
for (int i=0; i<listSize; i++) {
int arraySize = minBitsSize + r.nextInt( maxBitsSize-minBitsSize );
restulList.add( makeRandomBitSet( r, arraySize) );
}
return restulList;
}
private static BitSet makeRandomBitSet(final Random randomSource, final int maxSize){
BitSet bitSet = new BitSet();
for ( int datai=0; datai<maxSize; datai++ ) {
// each bit has 50% change to be set:
if ( randomSource.nextBoolean() ) bitSet.set( datai );
}
return bitSet;
}
/**
* converts a list of Integers representing Document ids
* into a Lucene DocIdSet
* @param docIdList
* @return
*/
public DocIdSet arrayToDocIdSet(List<Integer> docIdList) {
BitSet bitset = new BitSet();
for (int i : docIdList) {
bitset.set(i);
}
return new DocIdBitSet(bitset);
}
/**
* @param a
* @param b
* @return true if the two DocIdSet are equal: contain the same number of ids, same order and all are equal
*/
public static final boolean docIdSetsEqual(DocIdSet expected, DocIdSet tested) {
DocIdSetIterator iterA = expected.iterator();
DocIdSetIterator iterB = tested.iterator();
boolean nextA = false;
boolean nextB = false;
try{
do {
nextA = iterA.next();
nextB = iterB.next();
if ( nextA!=nextB ) {
return false;
}
else if ( nextA==false) {
return true;
}
else if ( iterA.doc() != iterB.doc() ) {
return false;
}
} while ( nextA && nextB );
}
catch (IOException ioe) {
fail( "these DocIdSetIterator instances should not throw any exceptions" );
}
return true;
}
}