ColumnDictionaryEntryStore adder = hasMultipleValues
? new MultiValColumnDictionaryEntryStore()
: new SingleValColumnDictionaryEntryStore();
final BitmapFactory bitmapFactory = bitmapSerdeFactory.getBitmapFactory();
MutableBitmap nullSet = null;
int rowCount = 0;
for (Rowboat theRow : theRows) {
if (dimIndex > theRow.getDims().length) {
if (nullSet == null) {
nullSet = bitmapFactory.makeEmptyMutableBitmap();
}
nullSet.add(rowCount);
adder.add(null);
} else {
int[] dimVals = theRow.getDims()[dimIndex];
if (dimVals == null || dimVals.length == 0) {
if (nullSet == null) {
nullSet = bitmapFactory.makeEmptyMutableBitmap();
}
nullSet.add(rowCount);
}
adder.add(dimVals);
}
rowCount++;
}
final Iterable<String> dimensionValues = dimensionValuesLookup.get(dimension);
GenericIndexed<String> dictionary = GenericIndexed.fromIterable(
dimensionValues,
GenericIndexed.stringStrategy
);
boolean bumpDictionary = false;
if (hasMultipleValues) {
final List<List<Integer>> vals = ((MultiValColumnDictionaryEntryStore) adder).get();
if (nullSet != null) {
log.info("Dimension[%s] has null rows.", dimension);
if (Iterables.getFirst(dimensionValues, "") != null) {
bumpDictionary = true;
log.info("Dimension[%s] has no null value in the dictionary, expanding...", dimension);
final List<String> nullList = Lists.newArrayList();
nullList.add(null);
dictionary = GenericIndexed.fromIterable(
Iterables.concat(nullList, dimensionValues),
GenericIndexed.stringStrategy
);
final int dictionarySize = dictionary.size();
multiValCol = VSizeIndexed.fromIterable(
FunctionalIterable
.create(vals)
.transform(
new Function<List<Integer>, VSizeIndexedInts>()
{
@Override
public VSizeIndexedInts apply(final List<Integer> input)
{
if (input == null) {
return VSizeIndexedInts.fromList(
new AbstractList<Integer>()
{
@Override
public Integer get(int index)
{
return 0;
}
@Override
public int size()
{
return 1;
}
}, dictionarySize
);
}
return VSizeIndexedInts.fromList(
new AbstractList<Integer>()
{
@Override
public Integer get(int index)
{
Integer val = input.get(index);
if (val == null) {
return 0;
}
return val + 1;
}
@Override
public int size()
{
return input.size();
}
},
dictionarySize
);
}
}
)
);
} else {
final int dictionarySize = dictionary.size();
multiValCol = VSizeIndexed.fromIterable(
FunctionalIterable
.create(vals)
.transform(
new Function<List<Integer>, VSizeIndexedInts>()
{
@Override
public VSizeIndexedInts apply(List<Integer> input)
{
if (input == null) {
return VSizeIndexedInts.fromList(
new AbstractList<Integer>()
{
@Override
public Integer get(int index)
{
return 0;
}
@Override
public int size()
{
return 1;
}
}, dictionarySize
);
}
return VSizeIndexedInts.fromList(
input,
dictionarySize
);
}
}
)
);
}
} else {
final int dictionarySize = dictionary.size();
multiValCol = VSizeIndexed.fromIterable(
FunctionalIterable
.create(vals)
.transform(
new Function<List<Integer>, VSizeIndexedInts>()
{
@Override
public VSizeIndexedInts apply(List<Integer> input)
{
return VSizeIndexedInts.fromList(
input,
dictionarySize
);
}
}
)
);
}
} else {
final List<Integer> vals = ((SingleValColumnDictionaryEntryStore) adder).get();
if (nullSet != null) {
log.info("Dimension[%s] has null rows.", dimension);
if (Iterables.getFirst(dimensionValues, "") != null) {
bumpDictionary = true;
log.info("Dimension[%s] has no null value in the dictionary, expanding...", dimension);
final List<String> nullList = Lists.newArrayList();
nullList.add(null);
dictionary = GenericIndexed.fromIterable(
Iterables.concat(nullList, dimensionValues),
GenericIndexed.stringStrategy
);
singleValCol = VSizeIndexedInts.fromList(
new AbstractList<Integer>()
{
@Override
public Integer get(int index)
{
Integer val = vals.get(index);
if (val == null) {
return 0;
}
return val + 1;
}
@Override
public int size()
{
return vals.size();
}
}, dictionary.size()
);
} else {
singleValCol = VSizeIndexedInts.fromList(
new AbstractList<Integer>()
{
@Override
public Integer get(int index)
{
Integer val = vals.get(index);
if (val == null) {
return 0;
}
return val;
}
@Override
public int size()
{
return vals.size();
}
}, dictionary.size()
);
}
} else {
singleValCol = VSizeIndexedInts.fromList(vals, dictionary.size());
}
}
// Make bitmap indexes
List<MutableBitmap> mutableBitmaps = Lists.newArrayList();
for (String dimVal : dimensionValues) {
List<Iterable<Integer>> convertedInverteds = Lists.newArrayListWithCapacity(adapters.size());
for (int j = 0; j < adapters.size(); ++j) {
convertedInverteds.add(
new ConvertingIndexedInts(
adapters.get(j).getBitmapIndex(dimension, dimVal), rowNumConversions.get(j)
)
);
}
MutableBitmap bitset = bitmapSerdeFactory.getBitmapFactory().makeEmptyMutableBitmap();
for (Integer row : CombiningIterable.createSplatted(
convertedInverteds,
Ordering.<Integer>natural().nullsFirst()
)) {
if (row != INVALID_ROW) {
bitset.add(row);
}
}
mutableBitmaps.add(bitset);
}
GenericIndexed<ImmutableBitmap> bitmaps;
if (nullSet != null) {
final ImmutableBitmap theNullSet = bitmapFactory.makeImmutableBitmap(nullSet);
if (bumpDictionary) {
bitmaps = GenericIndexed.fromIterable(
Iterables.concat(
Arrays.asList(theNullSet),
Iterables.transform(
mutableBitmaps,
new Function<MutableBitmap, ImmutableBitmap>()
{
@Override
public ImmutableBitmap apply(MutableBitmap input)
{
return bitmapFactory.makeImmutableBitmap(input);
}
}
)
),
bitmapSerdeFactory.getObjectStrategy()
);
} else {
Iterable<ImmutableBitmap> immutableBitmaps = Iterables.transform(
mutableBitmaps,
new Function<MutableBitmap, ImmutableBitmap>()
{
@Override
public ImmutableBitmap apply(MutableBitmap input)
{
return bitmapFactory.makeImmutableBitmap(input);
}
}
);
bitmaps = GenericIndexed.fromIterable(
Iterables.concat(
Arrays.asList(
theNullSet.union(Iterables.getFirst(immutableBitmaps, null))
),
Iterables.skip(immutableBitmaps, 1)
),
bitmapSerdeFactory.getObjectStrategy()
);
}
} else {
bitmaps = GenericIndexed.fromIterable(
Iterables.transform(
mutableBitmaps,
new Function<MutableBitmap, ImmutableBitmap>()
{
@Override
public ImmutableBitmap apply(MutableBitmap input)
{
return bitmapFactory.makeImmutableBitmap(input);
}
}
),
bitmapSerdeFactory.getObjectStrategy()
);