/*
* This software is subject to the terms of the Eclipse Public License v1.0
* Agreement, available at the following URL:
* http://www.eclipse.org/legal/epl-v10.html.
* You must accept the terms of that agreement to use this software.
*
* Copyright (c) 2002-2013 Pentaho Corporation.. All rights reserved.
*/
package mondrian.olap.fun;
import mondrian.calc.*;
import mondrian.calc.impl.AbstractListCalc;
import mondrian.calc.impl.DelegatingTupleList;
import mondrian.mdx.*;
import mondrian.olap.*;
import mondrian.olap.type.Type;
import mondrian.resource.MondrianResource;
import org.apache.log4j.Logger;
import org.olap4j.impl.Olap4jUtil;
import java.util.*;
import static mondrian.olap.fun.NativizeSetFunDef.NativeElementType.*;
/**
* Definition of the <code>NativizeSet</code> MDX function.
*
* @author jrand
* @since Oct 14, 2009
*/
public class NativizeSetFunDef extends FunDefBase {
/*
* Static final fields.
*/
protected static final Logger LOGGER =
Logger.getLogger(NativizeSetFunDef.class);
private static final String SENTINEL_PREFIX = "_Nativized_Sentinel_";
private static final String MEMBER_NAME_PREFIX = "_Nativized_Member_";
private static final String SET_NAME_PREFIX = "_Nativized_Set_";
private static final List<Class<? extends FunDef>> functionWhitelist =
Arrays.<Class<? extends FunDef>>asList(
CacheFunDef.class,
SetFunDef.class,
CrossJoinFunDef.class,
NativizeSetFunDef.class);
static final ReflectiveMultiResolver Resolver = new ReflectiveMultiResolver(
"NativizeSet",
"NativizeSet(<Set>)",
"Tries to natively evaluate <Set>.",
new String[] {"fxx"},
NativizeSetFunDef.class);
/*
* Instance final fields.
*/
private final SubstitutionMap substitutionMap = new SubstitutionMap();
private final HashSet<Dimension> dimensions =
new LinkedHashSet<Dimension>();
private boolean isFirstCompileCall = true;
/*
* Instance non-final fields.
*/
private Exp originalExp;
private static final String ESTIMATE_MESSAGE =
"isHighCardinality=%b: estimate=%,d threshold=%,d";
private static final String PARTIAL_ESTIMATE_MESSAGE =
"isHighCardinality=%b: partial estimate=%,d threshold=%,d";
public NativizeSetFunDef(FunDef dummyFunDef) {
super(dummyFunDef);
LOGGER.debug("---- NativizeSetFunDef constructor");
}
public Exp createCall(Validator validator, Exp[] args) {
LOGGER.debug("NativizeSetFunDef createCall");
ResolvedFunCall call =
(ResolvedFunCall) super.createCall(validator, args);
call.accept(new FindLevelsVisitor(substitutionMap, dimensions));
return call;
}
public Calc compileCall(ResolvedFunCall call, ExpCompiler compiler) {
LOGGER.debug("NativizeSetFunDef compileCall");
Exp funArg = call.getArg(0);
if (MondrianProperties.instance().UseAggregates.get()
|| MondrianProperties.instance().ReadAggregates.get())
{
return funArg.accept(compiler);
}
final Calc[] calcs = {compiler.compileList(funArg, true)};
final int arity = calcs[0].getType().getArity();
assert arity >= 0;
if (arity == 1 || substitutionMap.isEmpty()) {
IterCalc calc = (IterCalc) funArg.accept(compiler);
final boolean highCardinality =
arity == 1
&& isHighCardinality(funArg, compiler.getEvaluator());
if (calc == null) {
// This can happen under JDK1.4: caller wants iterator
// implementation, but compiler can only provide list.
// Fall through and use native.
} else if (calc instanceof ListCalc) {
return new NonNativeListCalc((ListCalc) calc, highCardinality);
} else {
return new NonNativeIterCalc(calc, highCardinality);
}
}
if (isFirstCompileCall) {
isFirstCompileCall = false;
originalExp = funArg.clone();
Query query = compiler.getEvaluator().getQuery();
call.accept(
new AddFormulasVisitor(query, substitutionMap, dimensions));
call.accept(new TransformToFormulasVisitor(query));
query.resolve();
}
return new NativeListCalc(
call, calcs, compiler, substitutionMap, originalExp);
}
private boolean isHighCardinality(Exp funArg, Evaluator evaluator) {
Level level = findLevel(funArg);
if (level != null) {
int cardinality =
evaluator.getSchemaReader()
.getLevelCardinality(level, false, true);
final int minThreshold = MondrianProperties.instance()
.NativizeMinThreshold.get();
final boolean isHighCard = cardinality > minThreshold;
logHighCardinality(
ESTIMATE_MESSAGE, minThreshold, cardinality, isHighCard);
return isHighCard;
}
return false;
}
private Level findLevel(Exp exp) {
exp.accept(new FindLevelsVisitor(substitutionMap, dimensions));
final Collection<Level> levels = substitutionMap.values();
if (levels.size() == 1) {
return levels.iterator().next();
}
return null;
}
private static void logHighCardinality(
final String estimateMessage,
long nativizeMinThreshold,
long estimatedCardinality,
boolean highCardinality)
{
LOGGER.debug(
String.format(
estimateMessage,
highCardinality,
estimatedCardinality,
nativizeMinThreshold));
}
static class NonNativeCalc implements Calc {
final Calc parent;
final boolean nativeEnabled;
protected NonNativeCalc(Calc parent, final boolean nativeEnabled) {
assert parent != null;
this.parent = parent;
this.nativeEnabled = nativeEnabled;
}
public Object evaluate(final Evaluator evaluator) {
evaluator.setNativeEnabled(nativeEnabled);
return parent.evaluate(evaluator);
}
public boolean dependsOn(final Hierarchy hierarchy) {
return parent.dependsOn(hierarchy);
}
public Type getType() {
return parent.getType();
}
public void accept(final CalcWriter calcWriter) {
parent.accept(calcWriter);
}
public ResultStyle getResultStyle() {
return parent.getResultStyle();
}
/**
* {@inheritDoc}
*
* Default implementation just does 'instanceof TargetClass'. Subtypes
* that are wrappers should override.
*/
public boolean isWrapperFor(Class<?> iface) {
return iface.isInstance(this);
}
/**
* {@inheritDoc}
*
* Default implementation just casts to TargetClass.
* Subtypes that are wrappers should override.
*/
public <T> T unwrap(Class<T> iface) {
return iface.cast(this);
}
}
static class NonNativeIterCalc
extends NonNativeCalc
implements IterCalc
{
protected NonNativeIterCalc(IterCalc parent, boolean highCardinality) {
super(parent, highCardinality);
}
IterCalc parent() {
return (IterCalc) parent;
}
public TupleIterable evaluateIterable(Evaluator evaluator) {
evaluator.setNativeEnabled(nativeEnabled);
return parent().evaluateIterable(evaluator);
}
}
static class NonNativeListCalc
extends NonNativeCalc
implements ListCalc
{
protected NonNativeListCalc(ListCalc parent, boolean highCardinality) {
super(parent, highCardinality);
}
ListCalc parent() {
return (ListCalc) parent;
}
public TupleList evaluateList(Evaluator evaluator) {
evaluator.setNativeEnabled(nativeEnabled);
return parent().evaluateList(evaluator);
}
public TupleIterable evaluateIterable(Evaluator evaluator) {
return evaluateList(evaluator);
}
}
public static class NativeListCalc extends AbstractListCalc {
private final SubstitutionMap substitutionMap;
private final ListCalc simpleCalc;
private final ExpCompiler compiler;
private final Exp originalExp;
protected NativeListCalc(
ResolvedFunCall call,
Calc[] calcs,
ExpCompiler compiler,
SubstitutionMap substitutionMap,
Exp originalExp)
{
super(call, calcs);
LOGGER.debug("---- NativeListCalc constructor");
this.substitutionMap = substitutionMap;
this.simpleCalc = (ListCalc) calcs[0];
this.compiler = compiler;
this.originalExp = originalExp;
}
public TupleList evaluateList(Evaluator evaluator) {
return computeTuples(evaluator);
}
public TupleList computeTuples(Evaluator evaluator) {
TupleList simplifiedList = evaluateSimplifiedList(evaluator);
if (simplifiedList.isEmpty()) {
return simplifiedList;
}
if (!isHighCardinality(evaluator, simplifiedList)) {
return evaluateNonNative(evaluator);
}
return evaluateNative(evaluator, simplifiedList);
}
private TupleList evaluateSimplifiedList(Evaluator evaluator) {
final int savepoint = evaluator.savepoint();
try {
evaluator.setNonEmpty(false);
evaluator.setNativeEnabled(false);
TupleList simplifiedList =
simpleCalc.evaluateList(evaluator);
dumpListToLog("simplified list", simplifiedList);
return simplifiedList;
} finally {
evaluator.restore(savepoint);
}
}
private TupleList evaluateNonNative(Evaluator evaluator) {
LOGGER.debug(
"Disabling native evaluation. originalExp="
+ originalExp);
ListCalc calc =
compiler.compileList(getOriginalExp(evaluator.getQuery()));
final int savepoint = evaluator.savepoint();
try {
evaluator.setNonEmpty(true);
evaluator.setNativeEnabled(false);
TupleList members = calc.evaluateList(evaluator);
return members;
} finally {
evaluator.restore(savepoint);
}
}
private TupleList evaluateNative(
Evaluator evaluator, TupleList simplifiedList)
{
CrossJoinAnalyzer analyzer =
new CrossJoinAnalyzer(simplifiedList, substitutionMap);
String crossJoin = analyzer.getCrossJoinExpression();
// If the crossjoin expression is empty, then the simplified list
// already contains the fully evaluated tuple list, so we can
// return it now without any additional work.
if (crossJoin.length() == 0) {
return simplifiedList;
}
// Force non-empty to true to create the native list.
LOGGER.debug(
"crossjoin reconstituted from simplified list: "
+ String.format(
"%n"
+ crossJoin.replaceAll(",", "%n, ")));
final int savepoint = evaluator.savepoint();
try {
evaluator.setNonEmpty(true);
evaluator.setNativeEnabled(true);
TupleList members = analyzer.mergeCalcMembers(
evaluateJoinExpression(evaluator, crossJoin));
return members;
} finally {
evaluator.restore(savepoint);
}
}
private Exp getOriginalExp(final Query query) {
originalExp.accept(
new TransformFromFormulasVisitor(query, compiler));
if (originalExp instanceof NamedSetExpr) {
//named sets get their evaluator cached in RolapResult.
//We do not want to use the cached evaluator, so pass along the
//expression instead.
return ((NamedSetExpr) originalExp).getNamedSet().getExp();
}
return originalExp;
}
private boolean isHighCardinality(
Evaluator evaluator, TupleList simplifiedList)
{
Util.assertTrue(!simplifiedList.isEmpty());
SchemaReader schema = evaluator.getSchemaReader();
List<Member> tuple = simplifiedList.get(0);
long nativizeMinThreshold =
MondrianProperties.instance().NativizeMinThreshold.get();
long estimatedCardinality = simplifiedList.size();
for (Member member : tuple) {
String memberName = member.getName();
if (memberName.startsWith(MEMBER_NAME_PREFIX)) {
Level level = member.getLevel();
Dimension dimension = level.getDimension();
Hierarchy hierarchy = dimension.getHierarchy();
String levelName = getLevelNameFromMemberName(memberName);
Level hierarchyLevel =
Util.lookupHierarchyLevel(hierarchy, levelName);
long levelCardinality =
getLevelCardinality(schema, hierarchyLevel);
estimatedCardinality *= levelCardinality;
if (estimatedCardinality >= nativizeMinThreshold) {
logHighCardinality(
PARTIAL_ESTIMATE_MESSAGE,
nativizeMinThreshold,
estimatedCardinality,
true);
return true;
}
}
}
boolean isHighCardinality =
(estimatedCardinality >= nativizeMinThreshold);
logHighCardinality(
ESTIMATE_MESSAGE,
nativizeMinThreshold,
estimatedCardinality,
isHighCardinality);
return isHighCardinality;
}
private long getLevelCardinality(SchemaReader schema, Level level) {
if (cardinalityIsKnown(level)) {
return level.getApproxRowCount();
}
return schema.getLevelCardinality(level, false, true);
}
private boolean cardinalityIsKnown(Level level) {
return level.getApproxRowCount() > 0;
}
private TupleList evaluateJoinExpression(
Evaluator evaluator, String crossJoinExpression)
{
Exp unresolved =
evaluator.getQuery().getConnection()
.parseExpression(crossJoinExpression);
Exp resolved = compiler.getValidator().validate(unresolved, false);
ListCalc calc = compiler.compileList(resolved);
return calc.evaluateList(evaluator);
}
}
static class FindLevelsVisitor extends MdxVisitorImpl {
private final SubstitutionMap substitutionMap;
private final Set<Dimension> dimensions;
public FindLevelsVisitor(
SubstitutionMap substitutionMap, HashSet<Dimension> dimensions)
{
this.substitutionMap = substitutionMap;
this.dimensions = dimensions;
}
@Override
public Object visit(ResolvedFunCall call) {
if (call.getFunDef() instanceof LevelMembersFunDef) {
if (call.getArg(0) instanceof LevelExpr) {
Level level = ((LevelExpr) call.getArg(0)).getLevel();
substitutionMap.put(createMemberId(level), level);
dimensions.add(level.getDimension());
}
} else if (
functionWhitelist.contains(call.getFunDef().getClass()))
{
for (Exp arg : call.getArgs()) {
arg.accept(this);
}
}
turnOffVisitChildren();
return null;
}
@Override
public Object visit(MemberExpr member) {
dimensions.add(member.getMember().getDimension());
return null;
}
}
static class AddFormulasVisitor extends MdxVisitorImpl {
private final Query query;
private final Collection<Level> levels;
private final Set<Dimension> dimensions;
public AddFormulasVisitor(
Query query,
SubstitutionMap substitutionMap,
Set<Dimension> dimensions)
{
LOGGER.debug("---- AddFormulasVisitor constructor");
this.query = query;
this.levels = substitutionMap.values();
this.dimensions = dimensions;
}
@Override
public Object visit(ResolvedFunCall call) {
if (call.getFunDef() instanceof NativizeSetFunDef) {
addFormulasToQuery();
}
turnOffVisitChildren();
return null;
}
private void addFormulasToQuery() {
LOGGER.debug("FormulaResolvingVisitor addFormulas");
List<Formula> formulas = new ArrayList<Formula>();
for (Level level : levels) {
Formula memberFormula = createDefaultMemberFormula(level);
formulas.add(memberFormula);
formulas.add(createNamedSetFormula(level, memberFormula));
}
for (Dimension dim : dimensions) {
Level level = dim.getHierarchy().getLevels()[0];
formulas.add(createSentinelFormula(level));
}
query.addFormulas(formulas.toArray(new Formula[formulas.size()]));
}
private Formula createSentinelFormula(Level level) {
Id memberId = createSentinelId(level);
Exp memberExpr = query.getConnection()
.parseExpression("101010");
LOGGER.debug(
"createSentinelFormula memberId="
+ memberId
+ " memberExpr="
+ memberExpr);
return new Formula(memberId, memberExpr, new MemberProperty[0]);
}
private Formula createDefaultMemberFormula(Level level) {
Id memberId = createMemberId(level);
Exp memberExpr =
new UnresolvedFunCall(
"DEFAULTMEMBER",
Syntax.Property,
new Exp[] {hierarchyId(level)});
LOGGER.debug(
"createLevelMembersFormulas memberId="
+ memberId
+ " memberExpr="
+ memberExpr);
return new Formula(memberId, memberExpr, new MemberProperty[0]);
}
private Formula createNamedSetFormula(
Level level, Formula memberFormula)
{
Id setId = createSetId(level);
Exp setExpr = query.getConnection()
.parseExpression(
"{"
+ memberFormula.getIdentifier().toString()
+ "}");
LOGGER.debug(
"createNamedSetFormula setId="
+ setId
+ " setExpr="
+ setExpr);
return new Formula(setId, setExpr);
}
}
static class TransformToFormulasVisitor extends MdxVisitorImpl {
private final Query query;
public TransformToFormulasVisitor(Query query) {
LOGGER.debug("---- TransformToFormulasVisitor constructor");
this.query = query;
}
@Override
public Object visit(ResolvedFunCall call) {
LOGGER.debug("visit " + call);
Object result = null;
if (call.getFunDef() instanceof LevelMembersFunDef) {
result = replaceLevelMembersReferences(call);
} else if (
functionWhitelist.contains(call.getFunDef().getClass()))
{
result = visitCallArguments(call);
}
turnOffVisitChildren();
return result;
}
private Object replaceLevelMembersReferences(ResolvedFunCall call) {
LOGGER.debug("replaceLevelMembersReferences " + call);
Level level = ((LevelExpr) call.getArg(0)).getLevel();
Id setId = createSetId(level);
Formula formula = query.findFormula(setId.toString());
Exp exp = Util.createExpr(formula.getNamedSet());
return query.createValidator().validate(exp, false);
}
private Object visitCallArguments(ResolvedFunCall call) {
Exp[] exps = call.getArgs();
LOGGER.debug("visitCallArguments " + call);
for (int i = 0; i < exps.length; i++) {
Exp transformedExp = (Exp) exps[i].accept(this);
if (transformedExp != null) {
exps[i] = transformedExp;
}
}
if (exps.length > 1
&& call.getFunDef() instanceof SetFunDef)
{
return flattenSetFunDef(call);
}
return null;
}
private Object flattenSetFunDef(ResolvedFunCall call) {
List<Exp> newArgs = new ArrayList<Exp>();
flattenSetMembers(newArgs, call.getArgs());
addSentinelMembers(newArgs);
if (newArgs.size() != call.getArgCount()) {
return new ResolvedFunCall(
call.getFunDef(),
newArgs.toArray(new Exp[newArgs.size()]),
call.getType());
}
return null;
}
private void flattenSetMembers(List<Exp> result, Exp[] args) {
for (Exp arg : args) {
if (arg instanceof ResolvedFunCall
&& ((ResolvedFunCall)arg).getFunDef() instanceof SetFunDef)
{
flattenSetMembers(result, ((ResolvedFunCall)arg).getArgs());
} else {
result.add(arg);
}
}
}
private void addSentinelMembers(List<Exp> args) {
Exp prev = args.get(0);
for (int i = 1; i < args.size(); i++) {
Exp curr = args.get(i);
if (prev.toString().equals(curr.toString())) {
OlapElement element = null;
if (curr instanceof NamedSetExpr) {
element = ((NamedSetExpr) curr).getNamedSet();
} else if (curr instanceof MemberExpr) {
element = ((MemberExpr) curr).getMember();
}
if (element != null) {
Level level = element.getHierarchy().getLevels()[0];
Id memberId = createSentinelId(level);
Formula formula =
query.findFormula(memberId.toString());
args.add(i++, Util.createExpr(formula.getMdxMember()));
}
}
prev = curr;
}
}
}
static class TransformFromFormulasVisitor extends MdxVisitorImpl {
private final Query query;
private final ExpCompiler compiler;
public TransformFromFormulasVisitor(Query query, ExpCompiler compiler) {
LOGGER.debug("---- TransformFromFormulasVisitor constructor");
this.query = query;
this.compiler = compiler;
}
@Override
public Object visit(ResolvedFunCall call) {
LOGGER.debug("visit " + call);
Object result;
result = visitCallArguments(call);
turnOffVisitChildren();
return result;
}
@Override
public Object visit(NamedSetExpr namedSetExpr) {
String exprName = namedSetExpr.getNamedSet().getName();
Exp membersExpr;
if (exprName.contains(SET_NAME_PREFIX)) {
String levelMembers = exprName.replaceAll(
SET_NAME_PREFIX, "\\[")
.replaceAll("_$", "\\]")
.replaceAll("_", "\\]\\.\\[")
+ ".members";
membersExpr =
query.getConnection().parseExpression(levelMembers);
membersExpr =
compiler.getValidator().validate(membersExpr, false);
} else {
membersExpr = namedSetExpr.getNamedSet().getExp();
}
return membersExpr;
}
private Object visitCallArguments(ResolvedFunCall call) {
Exp[] exps = call.getArgs();
LOGGER.debug("visitCallArguments " + call);
for (int i = 0; i < exps.length; i++) {
Exp transformedExp = (Exp) exps[i].accept(this);
if (transformedExp != null) {
exps[i] = transformedExp;
}
}
return null;
}
}
private static class SubstitutionMap {
private final Map<String, Level> map = new HashMap<String, Level>();
public boolean isEmpty() {
return map.isEmpty();
}
public boolean contains(Member member) {
return map.containsKey(toKey(member));
}
public Level get(Member member) {
return map.get(toKey(member));
}
public Level put(Id id, Level level) {
return map.put(toKey(id), level);
}
public Collection<Level> values() {
return map.values();
}
@Override
public String toString() {
return map.toString();
}
private String toKey(Id id) {
return id.toString();
}
private String toKey(Member member) {
return member.getUniqueName();
}
}
public static class CrossJoinAnalyzer {
private final int arity;
private final Member[] tempTuple;
private final List<Member> tempTupleAsList;
private final int[] nativeIndices;
private final int resultLimit;
private final List<Collection<String>> nativeMembers;
private final ReassemblyGuide reassemblyGuide;
private final TupleList resultList;
public CrossJoinAnalyzer(
TupleList simplifiedList, SubstitutionMap substitutionMap)
{
long nativizeMaxResults =
MondrianProperties.instance().NativizeMaxResults.get();
arity = simplifiedList.getArity();
tempTuple = new Member[arity];
tempTupleAsList = Arrays.asList(tempTuple);
resultLimit = nativizeMaxResults <= 0
? Integer.MAX_VALUE
: (int) Math.min(nativizeMaxResults, Integer.MAX_VALUE);
resultList = TupleCollections.createList(arity);
reassemblyGuide = classifyMembers(simplifiedList, substitutionMap);
nativeMembers = findNativeMembers();
nativeIndices = findNativeIndices();
}
public ReassemblyGuide classifyMembers(
TupleList simplifiedList,
SubstitutionMap substitutionMap)
{
ReassemblyGuide guide = new ReassemblyGuide(0);
List<ReassemblyCommand> cmdTuple =
new ArrayList<ReassemblyCommand>(arity);
for (List<Member> srcTuple : simplifiedList) {
cmdTuple.clear();
for (Member mbr : srcTuple) {
cmdTuple.add(zz(substitutionMap, mbr));
}
guide.addCommandTuple(cmdTuple);
}
return guide;
}
private ReassemblyCommand zz(
SubstitutionMap substitutionMap, Member mbr)
{
ReassemblyCommand c;
if (substitutionMap.contains(mbr)) {
c =
new ReassemblyCommand(
substitutionMap.get(mbr), LEVEL_MEMBERS);
} else if (mbr.getName().startsWith(SENTINEL_PREFIX)) {
c =
new ReassemblyCommand(mbr, SENTINEL);
} else {
NativeElementType nativeType = !isNativeCompatible(mbr)
? NON_NATIVE
: mbr.getMemberType() == Member.MemberType.REGULAR
? ENUMERATED_VALUE
: OTHER_NATIVE;
c = new ReassemblyCommand(mbr, nativeType);
}
return c;
}
private List<Collection<String>> findNativeMembers() {
List<Collection<String>> nativeMembers =
new ArrayList<Collection<String>>(arity);
for (int i = 0; i < arity; i++) {
nativeMembers.add(new LinkedHashSet<String>());
}
findNativeMembers(reassemblyGuide, nativeMembers);
return nativeMembers;
}
private void findNativeMembers(
ReassemblyGuide guide,
List<Collection<String>> nativeMembers)
{
List<ReassemblyCommand> commands = guide.getCommands();
Set<NativeElementType> typesToAdd =
ReassemblyCommand.getMemberTypes(commands);
if (typesToAdd.contains(LEVEL_MEMBERS)) {
typesToAdd.remove(ENUMERATED_VALUE);
}
int index = guide.getIndex();
for (ReassemblyCommand command : commands) {
NativeElementType type = command.getMemberType();
if (type.isNativeCompatible() && typesToAdd.contains(type)) {
nativeMembers.get(index).add(command.getElementName());
}
if (command.hasNextGuide()) {
findNativeMembers(command.forNextCol(), nativeMembers);
}
}
}
private int[] findNativeIndices() {
int[] indices = new int[arity];
int nativeColCount = 0;
for (int i = 0; i < arity; i++) {
Collection<String> natives = nativeMembers.get(i);
if (!natives.isEmpty()) {
indices[nativeColCount++] = i;
}
}
if (nativeColCount == arity) {
return indices;
}
int[] result = new int[nativeColCount];
System.arraycopy(indices, 0, result, 0, nativeColCount);
return result;
}
private boolean isNativeCompatible(Member member) {
return member.isParentChildLeaf()
|| (!member.isMeasure()
&& !member.isCalculated() && !member.isAll());
}
private String getCrossJoinExpression() {
return formatCrossJoin(nativeMembers);
}
private String formatCrossJoin(List<Collection<String>> memberLists) {
StringBuilder buf = new StringBuilder();
String left = toCsv(memberLists.get(0));
String right =
memberLists.size() == 1
? ""
: formatCrossJoin(memberLists.subList(1, memberLists.size()));
if (left.length() == 0) {
buf.append(right);
} else {
if (right.length() == 0) {
buf.append("{").append(left).append("}");
} else {
buf.append("CrossJoin(")
.append("{").append(left).append("},")
.append(right).append(")");
}
}
return buf.toString();
}
private TupleList mergeCalcMembers(TupleList nativeValues) {
TupleList nativeList =
adaptList(nativeValues, arity, nativeIndices);
dumpListToLog("native list", nativeList);
mergeCalcMembers(reassemblyGuide, new Range(nativeList), null);
dumpListToLog("result list", resultList);
return resultList;
}
private void mergeCalcMembers(
ReassemblyGuide guide, Range range, Set<List<Member>> history)
{
int col = guide.getIndex();
if (col == arity - 1) {
if (history == null) {
appendMembers(guide, range);
} else {
appendMembers(guide, range, history);
}
return;
}
for (ReassemblyCommand command : guide.getCommands()) {
ReassemblyGuide nextGuide = command.forNextCol();
tempTuple[col] = null;
switch (command.getMemberType()) {
case NON_NATIVE:
tempTuple[col] = command.getMember();
mergeCalcMembers(
nextGuide,
range,
(history == null
? new HashSet<List<Member>>()
: history));
break;
case ENUMERATED_VALUE:
Member value = command.getMember();
Range valueRange = range.subRangeForValue(value, col);
if (!valueRange.isEmpty()) {
mergeCalcMembers(nextGuide, valueRange, history);
}
break;
case LEVEL_MEMBERS:
Level level = command.getLevel();
Range levelRange = range.subRangeForValue(level, col);
for (Range subRange : levelRange.subRanges(col)) {
mergeCalcMembers(nextGuide, subRange, history);
}
break;
case OTHER_NATIVE:
for (Range subRange : range.subRanges(col)) {
mergeCalcMembers(nextGuide, subRange, history);
}
break;
default:
throw Util.unexpected(command.getMemberType());
}
}
}
private void appendMembers(ReassemblyGuide guide, Range range) {
int col = guide.getIndex();
for (ReassemblyCommand command : guide.getCommands()) {
switch (command.getMemberType()) {
case NON_NATIVE:
tempTuple[col] = command.getMember();
appendTuple(range.getTuple(), tempTupleAsList);
break;
case ENUMERATED_VALUE:
Member value = command.getMember();
Range valueRange = range.subRangeForValue(value, col);
if (!valueRange.isEmpty()) {
appendTuple(valueRange.getTuple());
}
break;
case LEVEL_MEMBERS:
case OTHER_NATIVE:
for (List<Member> tuple : range.getTuples()) {
appendTuple(tuple);
}
break;
default:
throw Util.unexpected(command.getMemberType());
}
}
}
private void appendMembers(
ReassemblyGuide guide, Range range, Set<List<Member>> history)
{
int col = guide.getIndex();
for (ReassemblyCommand command : guide.getCommands()) {
switch (command.getMemberType()) {
case NON_NATIVE:
tempTuple[col] = command.getMember();
if (range.isEmpty()) {
appendTuple(tempTupleAsList, history);
} else {
appendTuple(range.getTuple(), tempTupleAsList, history);
}
break;
case ENUMERATED_VALUE:
Member value = command.getMember();
Range valueRange = range.subRangeForValue(value, col);
if (!valueRange.isEmpty()) {
appendTuple(
valueRange.getTuple(), tempTupleAsList, history);
}
break;
case LEVEL_MEMBERS:
case OTHER_NATIVE:
tempTuple[col] = null;
for (List<Member> tuple : range.getTuples()) {
appendTuple(tuple, tempTupleAsList, history);
}
break;
default:
throw Util.unexpected(command.getMemberType());
}
}
}
private void appendTuple(
List<Member> nonNatives,
Set<List<Member>> history)
{
if (history.add(nonNatives)) {
appendTuple(nonNatives);
}
}
private void appendTuple(
List<Member> natives,
List<Member> nonNatives,
Set<List<Member>> history)
{
List<Member> copy = copyOfTuple(natives, nonNatives);
if (history.add(copy)) {
appendTuple(copy);
}
}
private void appendTuple(
List<Member> natives,
List<Member> nonNatives)
{
appendTuple(copyOfTuple(natives, nonNatives));
}
private void appendTuple(List<Member> tuple) {
resultList.add(tuple);
checkNativeResultLimit(resultList.size());
}
private List<Member> copyOfTuple(
List<Member> natives,
List<Member> nonNatives)
{
Member[] copy = new Member[arity];
for (int i = 0; i < arity; i++) {
copy[i] =
(nonNatives.get(i) == null)
? natives.get(i)
: nonNatives.get(i);
}
return Arrays.asList(copy);
}
/**
* Check the resultSize against the result limit setting. Throws
* LimitExceededDuringCrossjoin exception if limit exceeded.
* <p/>
* It didn't seem appropriate to use the existing Mondrian
* ResultLimit property, since the meaning and use of that
* property seems to be a bit ambiguous, otherwise we could
* simply call Util.checkCJResultLimit.
*
* @param resultSize Result limit
* @throws mondrian.olap.ResourceLimitExceededException
*
*/
private void checkNativeResultLimit(int resultSize) {
// Throw an exeption if the size of the crossjoin exceeds the result
// limit.
if (resultLimit < resultSize) {
throw MondrianResource.instance()
.LimitExceededDuringCrossjoin.ex(resultSize, resultLimit);
}
}
public TupleList adaptList(
final TupleList sourceList,
final int destSize,
final int[] destIndices)
{
if (sourceList.isEmpty()) {
return TupleCollections.emptyList(destIndices.length);
}
checkNativeResultLimit(sourceList.size());
TupleList destList =
new DelegatingTupleList(
destSize,
new AbstractList<List<Member>>() {
@Override
public List<Member> get(int index) {
final List<Member> sourceTuple =
sourceList.get(index);
final Member[] members = new Member[destSize];
for (int i = 0; i < destIndices.length; i++) {
members[destIndices[i]] = sourceTuple.get(i);
}
return Arrays.asList(members);
}
@Override
public int size() {
return sourceList.size();
}
}
);
// The mergeCalcMembers method in this file assumes that the
// resultList is random access - that calls to get(n) are constant
// cost, regardless of n. Unfortunately, the TraversalList objects
// created by HighCardSqlTupleReader are implemented using linked
// lists, leading to pathologically long run times.
// This presumes that the ResultStyle is LIST
if (LOGGER.isDebugEnabled()) {
String sourceListType =
sourceList.getClass().getSimpleName();
String sourceElementType =
String.format("Member[%d]", destSize);
LOGGER.debug(
String.format(
"returning native %s<%s> without copying to new list.",
sourceListType,
sourceElementType));
}
return destList;
}
}
// REVIEW: Can we remove this class, and simply use TupleList?
static class Range {
private final TupleList list;
private final int from;
private final int to;
public Range(TupleList list)
{
this(list, 0, list.size());
}
private Range(TupleList list, int from, int to) {
if (from < 0) {
throw new IllegalArgumentException("from is must be >= 0");
}
if (to > list.size()) {
throw new IllegalArgumentException(
"to must be <= to list size");
}
if (from > to) {
throw new IllegalArgumentException("from must be <= to");
}
this.list = list;
this.from = from;
this.to = to;
}
public boolean isEmpty() {
return size() == 0;
}
public int size() {
return to - from;
}
public List<Member> getTuple() {
if (from >= list.size()) {
throw new NoSuchElementException();
}
return list.get(from);
}
public List<List<Member>> getTuples() {
if (from == 0 && to == list.size()) {
return list;
}
return list.subList(from, to);
}
public Member getMember(int cursor, int col) {
return list.get(cursor).get(col);
}
public String toString() {
return "[" + from + " : " + to + "]";
}
private Range subRange(int fromRow, int toRow) {
return new Range(list, fromRow, toRow);
}
public Range subRangeForValue(Member value, int col) {
int startAt = nextMatching(value, from, col);
int endAt = nextNonMatching(value, startAt + 1, col);
return subRange(startAt, endAt);
}
public Range subRangeForValue(Level level, int col) {
int startAt = nextMatching(level, from, col);
int endAt = nextNonMatching(level, startAt + 1, col);
return subRange(startAt, endAt);
}
public Range subRangeStartingAt(int startAt, int col) {
Member value = list.get(startAt).get(col);
int endAt = nextNonMatching(value, startAt + 1, col);
return subRange(startAt, endAt);
}
private int nextMatching(Member value, int startAt, int col) {
for (int cursor = startAt; cursor < to; cursor++) {
if (value.equals(list.get(cursor).get(col))) {
return cursor;
}
}
return to;
}
private int nextMatching(Level level, int startAt, int col) {
for (int cursor = startAt; cursor < to; cursor++) {
if (level.equals(list.get(cursor).get(col).getLevel())) {
return cursor;
}
}
return to;
}
private int nextNonMatching(Member value, int startAt, int col) {
if (value == null) {
return nextNonNull(startAt, col);
}
for (int cursor = startAt; cursor < to; cursor++) {
if (!value.equals(list.get(cursor).get(col))) {
return cursor;
}
}
return to;
}
private int nextNonMatching(Level level, int startAt, int col) {
if (level == null) {
return nextNonNull(startAt, col);
}
for (int cursor = startAt; cursor < to; cursor++) {
if (!level.equals(list.get(cursor).get(col).getLevel())) {
return cursor;
}
}
return to;
}
private int nextNonNull(int startAt, int col) {
for (int cursor = startAt; cursor < to; cursor++) {
if (list.get(cursor).get(col) != null) {
return cursor;
}
}
return to;
}
public Iterable<Range> subRanges(final int col) {
final Range parent = this;
return new Iterable<Range>() {
final int rangeCol = col;
public Iterator<Range> iterator() {
return new RangeIterator(parent, rangeCol);
}
};
}
public Iterable<Member> getMembers(final int col) {
return new Iterable<Member>() {
public Iterator<Member> iterator() {
return new Iterator<Member>() {
private int cursor = from;
public boolean hasNext() {
return cursor < to;
}
public Member next() {
if (!hasNext()) {
throw new NoSuchElementException();
}
return getMember(cursor++, col);
}
public void remove() {
throw new UnsupportedOperationException();
}
};
}
};
}
}
public static class RangeIterator
implements Iterator<Range>
{
private final Range parent;
private final int col;
private Range precomputed;
public RangeIterator(Range parent, int col) {
this.parent = parent;
this.col = col;
precomputed = next(parent.from);
}
public boolean hasNext() {
return precomputed != null;
}
private Range next(int cursor) {
return (cursor >= parent.to)
? null
: parent.subRangeStartingAt(cursor, col);
}
public Range next() {
if (precomputed == null) {
throw new NoSuchElementException();
}
Range it = precomputed;
precomputed = next(precomputed.to);
return it;
}
public void remove() {
throw new UnsupportedOperationException();
}
}
private static class ReassemblyGuide {
private final int index;
private final List<ReassemblyCommand> commands =
new ArrayList<ReassemblyCommand>();
public ReassemblyGuide(int index) {
this.index = index;
}
public int getIndex() {
return index;
}
public List<ReassemblyCommand> getCommands() {
return Collections.unmodifiableList(commands);
}
private void addCommandTuple(List<ReassemblyCommand> commandTuple) {
ReassemblyCommand curr = currentCommand(commandTuple);
if (index < commandTuple.size() - 1) {
curr.forNextCol(index + 1).addCommandTuple(commandTuple);
}
}
private ReassemblyCommand currentCommand(
List<ReassemblyCommand> commandTuple)
{
ReassemblyCommand curr = commandTuple.get(index);
ReassemblyCommand prev = commands.isEmpty()
? null : commands.get(commands.size() - 1);
if (prev != null && prev.getMemberType() == SENTINEL) {
commands.set(commands.size() - 1, curr);
} else if (prev == null
|| !prev.getElement().equals(curr.getElement()))
{
commands.add(curr);
} else {
curr = prev;
}
return curr;
}
public String toString() {
return "" + index + ":" + commands.toString()
.replaceAll("=null", "").replaceAll("=", " ") + " ";
}
}
private static class ReassemblyCommand {
private final OlapElement element;
private final String elementName;
private final NativeElementType memberType;
private ReassemblyGuide nextColGuide;
public ReassemblyCommand(
Member member,
NativeElementType memberType)
{
this.element = member;
this.memberType = memberType;
this.elementName = member.toString();
}
public ReassemblyCommand(
Level level,
NativeElementType memberType)
{
this.element = level;
this.memberType = memberType;
this.elementName = level.toString() + ".members";
}
public OlapElement getElement() {
return element;
}
public String getElementName() {
return elementName;
}
public Member getMember() {
return (Member) element;
}
public Level getLevel() {
return (Level) element;
}
public boolean hasNextGuide() {
return nextColGuide != null;
}
public ReassemblyGuide forNextCol() {
return nextColGuide;
}
public ReassemblyGuide forNextCol(int index) {
if (nextColGuide == null) {
nextColGuide = new ReassemblyGuide(index);
}
return nextColGuide;
}
public NativeElementType getMemberType() {
return memberType;
}
public static Set<NativeElementType> getMemberTypes(
Collection<ReassemblyCommand> commands)
{
Set<NativeElementType> types =
Olap4jUtil.enumSetNoneOf(NativeElementType.class);
for (ReassemblyCommand command : commands) {
types.add(command.getMemberType());
}
return types;
}
@Override
public String toString() {
return memberType.toString() + ": " + getElementName();
}
}
enum NativeElementType {
LEVEL_MEMBERS(true),
ENUMERATED_VALUE(true),
OTHER_NATIVE(true),
NON_NATIVE(false),
SENTINEL(false);
private final boolean isNativeCompatible;
private NativeElementType(boolean isNativeCompatible) {
this.isNativeCompatible = isNativeCompatible;
}
public boolean isNativeCompatible() {
return isNativeCompatible;
}
}
private static Id createSentinelId(Level level) {
return hierarchyId(level)
.append(q(createMangledName(level, SENTINEL_PREFIX)));
}
private static Id createMemberId(Level level) {
return hierarchyId(level)
.append(q(createMangledName(level, MEMBER_NAME_PREFIX)));
}
private static Id createSetId(Level level) {
return new Id(
q(createMangledName(level, SET_NAME_PREFIX)));
}
private static Id hierarchyId(Level level) {
Id id = new Id(q(level.getDimension().getName()));
if (MondrianProperties.instance().SsasCompatibleNaming.get()) {
id = id.append(q(level.getHierarchy().getName()));
}
return id;
}
private static Id.Segment q(String s) {
return new Id.NameSegment(s);
}
private static String createMangledName(Level level, String prefix) {
return prefix
+ level.getUniqueName().replaceAll("[\\[\\]]", "")
.replaceAll("\\.", "_")
+ "_";
}
private static void dumpListToLog(
String heading, TupleList list)
{
if (LOGGER.isDebugEnabled()) {
LOGGER.debug(
String.format(
"%s created with %,d rows.", heading, list.size()));
StringBuilder buf = new StringBuilder(Util.nl);
for (List<Member> element : list) {
buf.append(Util.nl);
buf.append(element);
}
LOGGER.debug(buf.toString());
}
}
private static <T> String toCsv(Collection<T> list) {
StringBuilder buf = new StringBuilder();
String sep = "";
for (T element : list) {
buf.append(sep).append(element);
sep = ", ";
}
return buf.toString();
}
private static String getLevelNameFromMemberName(String memberName) {
// we assume that the last token is the level name
String tokens[] = memberName.split("_");
return tokens[tokens.length - 1];
}
}
// End NativizeSetFunDef.java