/**************************************************************************************
* Copyright (C) 2008 EsperTech, Inc. All rights reserved. *
* http://esper.codehaus.org *
* http://www.espertech.com *
* ---------------------------------------------------------------------------------- *
* The software in this package is published under the terms of the GPL license *
* a copy of which has been included with this distribution in the license.txt file. *
**************************************************************************************/
package com.espertech.esper.epl.core;
import com.espertech.esper.client.EPException;
import com.espertech.esper.client.EventType;
import com.espertech.esper.client.annotation.HookType;
import com.espertech.esper.collection.Pair;
import com.espertech.esper.core.context.util.ContextPropertyRegistry;
import com.espertech.esper.core.service.ExprEvaluatorContextStatement;
import com.espertech.esper.core.service.StatementContext;
import com.espertech.esper.epl.agg.rollup.GroupByRollupNodeBase;
import com.espertech.esper.epl.agg.rollup.GroupByRollupPerLevelExpression;
import com.espertech.esper.epl.agg.rollup.GroupByRollupPlanDesc;
import com.espertech.esper.epl.agg.rollup.GroupByRollupPlanHook;
import com.espertech.esper.epl.agg.service.AggregationGroupByRollupDesc;
import com.espertech.esper.epl.agg.service.AggregationGroupByRollupLevel;
import com.espertech.esper.epl.agg.service.AggregationServiceFactoryDesc;
import com.espertech.esper.epl.agg.service.AggregationServiceFactoryFactory;
import com.espertech.esper.epl.expression.*;
import com.espertech.esper.epl.spec.*;
import com.espertech.esper.event.NativeEventType;
import com.espertech.esper.util.CollectionUtil;
import com.espertech.esper.util.JavaClassHelper;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import java.util.*;
/**
* Factory for output processors. Output processors process the result set of a join or of a view
* and apply aggregation/grouping, having and some output limiting logic.
* <p>
* The instance produced by the factory depends on the presence of aggregation functions in the select list,
* the presence and nature of the group-by clause.
* <p>
* In case (1) and (2) there are no aggregation functions in the select clause.
* <p>
* Case (3) is without group-by and with aggregation functions and without non-aggregated properties
* in the select list: <pre>select sum(volume) </pre>.
* Always produces one row for new and old data, aggregates without grouping.
* <p>
* Case (4) is without group-by and with aggregation functions but with non-aggregated properties
* in the select list: <pre>select price, sum(volume) </pre>.
* Produces a row for each event, aggregates without grouping.
* <p>
* Case (5) is with group-by and with aggregation functions and all selected properties are grouped-by.
* in the select list: <pre>select customerId, sum(volume) group by customerId</pre>.
* Produces a old and new data row for each group changed, aggregates with grouping, see
* {@link ResultSetProcessorRowPerGroup}
* <p>
* Case (6) is with group-by and with aggregation functions and only some selected properties are grouped-by.
* in the select list: <pre>select customerId, supplierId, sum(volume) group by customerId</pre>.
* Produces row for each event, aggregates with grouping.
*/
public class ResultSetProcessorFactoryFactory
{
/**
* Returns the result set process for the given select expression, group-by clause and
* having clause given a set of types describing each stream in the from-clause.
* @param statementSpec - a subset of the statement specification
* @param stmtContext - engine and statement and agent-instance level services
* @param typeService - for information about the streams in the from clause
* @param viewResourceDelegate - delegates views resource factory to expression resources requirements
* @param isUnidirectionalStream - true if unidirectional join for any of the streams
* @param allowAggregation - indicator whether to allow aggregation functions in any expressions
* @return result set processor instance
* @throws ExprValidationException when any of the expressions is invalid
*/
public static ResultSetProcessorFactoryDesc getProcessorPrototype(StatementSpecCompiled statementSpec,
StatementContext stmtContext,
StreamTypeService typeService,
ViewResourceDelegateUnverified viewResourceDelegate,
boolean[] isUnidirectionalStream,
boolean allowAggregation,
ContextPropertyRegistry contextPropertyRegistry,
SelectExprProcessorDeliveryCallback selectExprProcessorCallback
)
throws ExprValidationException
{
OrderByItem[] orderByListUnexpanded = statementSpec.getOrderByList();
SelectClauseSpecCompiled selectClauseSpec = statementSpec.getSelectClauseSpec();
InsertIntoDesc insertIntoDesc = statementSpec.getInsertIntoDesc();
ExprNode optionalHavingNode = statementSpec.getHavingExprRootNode();
OutputLimitSpec outputLimitSpec = statementSpec.getOutputLimitSpec();
boolean isUnidirectional = false;
for (int i = 0; i < isUnidirectionalStream.length; i++)
{
isUnidirectional |= isUnidirectionalStream[i];
}
// Expand any instances of select-clause names in the
// order-by clause with the full expression
List<OrderByItem> orderByList = expandColumnNames(selectClauseSpec.getSelectExprList(), orderByListUnexpanded);
// Validate selection expressions, if any (could be wildcard i.e. empty list)
List<SelectClauseExprCompiledSpec> namedSelectionList = new LinkedList<SelectClauseExprCompiledSpec>();
ExprEvaluatorContextStatement evaluatorContextStmt = new ExprEvaluatorContextStatement(stmtContext);
boolean allowRollup = statementSpec.getGroupByExpressions() != null && statementSpec.getGroupByExpressions().getGroupByRollupLevels() != null;
ExprValidationContext validationContext = new ExprValidationContext(typeService, stmtContext.getMethodResolutionService(), viewResourceDelegate, stmtContext.getSchedulingService(), stmtContext.getVariableService(),evaluatorContextStmt, stmtContext.getEventAdapterService(), stmtContext.getStatementName(), stmtContext.getStatementId(), stmtContext.getAnnotations(), stmtContext.getContextDescriptor(), false, allowRollup);
validateSelectAssignColNames(selectClauseSpec, namedSelectionList, validationContext);
if (statementSpec.getGroupByExpressions() != null && statementSpec.getGroupByExpressions().getSelectClausePerLevel() != null) {
ExprNodeUtility.getValidatedSubtree(ExprNodeOrigin.GROUPBY, statementSpec.getGroupByExpressions().getSelectClausePerLevel(), validationContext);
}
boolean isUsingWildcard = selectClauseSpec.isUsingWildcard();
// Validate stream selections, if any (such as stream.*)
boolean isUsingStreamSelect = false;
for (SelectClauseElementCompiled compiled : selectClauseSpec.getSelectExprList())
{
if (!(compiled instanceof SelectClauseStreamCompiledSpec))
{
continue;
}
SelectClauseStreamCompiledSpec streamSelectSpec = (SelectClauseStreamCompiledSpec) compiled;
int streamNum = Integer.MIN_VALUE;
boolean isFragmentEvent = false;
boolean isProperty = false;
Class propertyType = null;
isUsingStreamSelect = true;
for (int i = 0; i < typeService.getStreamNames().length; i++)
{
String streamName = streamSelectSpec.getStreamName();
if (typeService.getStreamNames()[i].equals(streamName))
{
streamNum = i;
break;
}
// see if the stream name is known as a nested event type
EventType candidateProviderOfFragments = typeService.getEventTypes()[i];
// for the native event type we don't need to fragment, we simply use the property itself since all wrappers understand Java objects
if (!(candidateProviderOfFragments instanceof NativeEventType) && (candidateProviderOfFragments.getFragmentType(streamName) != null))
{
streamNum = i;
isFragmentEvent = true;
break;
}
}
// stream name not found
if (streamNum == Integer.MIN_VALUE)
{
// see if the stream name specified resolves as a property
PropertyResolutionDescriptor desc = null;
try
{
desc = typeService.resolveByPropertyName(streamSelectSpec.getStreamName(), false);
}
catch (StreamTypesException e)
{
// not handled
}
if (desc == null)
{
throw new ExprValidationException("Stream selector '" + streamSelectSpec.getStreamName() + ".*' does not match any stream name in the from clause");
}
isProperty = true;
propertyType = desc.getPropertyType();
streamNum = desc.getStreamNum();
}
streamSelectSpec.setStreamNumber(streamNum);
streamSelectSpec.setFragmentEvent(isFragmentEvent);
streamSelectSpec.setProperty(isProperty, propertyType);
}
// Validate having clause, if present
if (optionalHavingNode != null) {
optionalHavingNode = ExprNodeUtility.getValidatedSubtree(ExprNodeOrigin.HAVING, optionalHavingNode, validationContext);
if (statementSpec.getGroupByExpressions() != null) {
ExprNodeUtility.getValidatedSubtree(ExprNodeOrigin.GROUPBY, statementSpec.getGroupByExpressions().getOptHavingNodePerLevel(), validationContext);
}
}
// Validate order-by expressions, if any (could be empty list for no order-by)
for (int i = 0; i < orderByList.size(); i++)
{
ExprNode orderByNode = orderByList.get(i).getExprNode();
// Ensure there is no subselects
ExprNodeSubselectDeclaredDotVisitor visitor = new ExprNodeSubselectDeclaredDotVisitor();
orderByNode.accept(visitor);
if (visitor.getSubselects().size() > 0)
{
throw new ExprValidationException("Subselects not allowed within order-by clause");
}
Boolean isDescending = orderByList.get(i).isDescending();
OrderByItem validatedOrderBy = new OrderByItem(ExprNodeUtility.getValidatedSubtree(ExprNodeOrigin.ORDERBY, orderByNode, validationContext), isDescending);
orderByList.set(i, validatedOrderBy);
if (statementSpec.getGroupByExpressions() != null && statementSpec.getGroupByExpressions().getOptOrderByPerLevel() != null) {
ExprNodeUtility.getValidatedSubtree(ExprNodeOrigin.GROUPBY, statementSpec.getGroupByExpressions().getOptOrderByPerLevel(), validationContext);
}
}
// Get the select expression nodes
List<ExprNode> selectNodes = new ArrayList<ExprNode>();
for(SelectClauseExprCompiledSpec element : namedSelectionList)
{
selectNodes.add(element.getSelectExpression());
}
// Get the order-by expression nodes
List<ExprNode> orderByNodes = new ArrayList<ExprNode>();
for(OrderByItem element : orderByList) {
orderByNodes.add(element.getExprNode());
}
// Determine aggregate functions used in select, if any
List<ExprAggregateNode> selectAggregateExprNodes = new LinkedList<ExprAggregateNode>();
for (SelectClauseExprCompiledSpec element : namedSelectionList) {
ExprAggregateNodeUtil.getAggregatesBottomUp(element.getSelectExpression(), selectAggregateExprNodes);
}
if (statementSpec.getGroupByExpressions() != null) {
ExprAggregateNodeUtil.getAggregatesBottomUp(statementSpec.getGroupByExpressions().getSelectClausePerLevel(), selectAggregateExprNodes);
}
if (!allowAggregation && !selectAggregateExprNodes.isEmpty()) {
throw new ExprValidationException("Aggregation functions are not allowed in this context");
}
// Determine if we have a having clause with aggregation
List<ExprAggregateNode> havingAggregateExprNodes = new LinkedList<ExprAggregateNode>();
Set<Pair<Integer, String>> propertiesAggregatedHaving = new HashSet<Pair<Integer, String>>();
if (optionalHavingNode != null) {
ExprAggregateNodeUtil.getAggregatesBottomUp(optionalHavingNode, havingAggregateExprNodes);
if (statementSpec.getGroupByExpressions() != null) {
ExprAggregateNodeUtil.getAggregatesBottomUp(statementSpec.getGroupByExpressions().getOptHavingNodePerLevel(), havingAggregateExprNodes);
}
propertiesAggregatedHaving = ExprNodeUtility.getAggregatedProperties(havingAggregateExprNodes);
}
if (!allowAggregation && !havingAggregateExprNodes.isEmpty()) {
throw new ExprValidationException("Aggregation functions are not allowed in this context");
}
// Determine if we have a order-by clause with aggregation
List<ExprAggregateNode> orderByAggregateExprNodes = new LinkedList<ExprAggregateNode>();
if (orderByNodes != null && !orderByNodes.isEmpty())
{
for (ExprNode orderByNode : orderByNodes) {
ExprAggregateNodeUtil.getAggregatesBottomUp(orderByNode, orderByAggregateExprNodes);
}
if (statementSpec.getGroupByExpressions() != null) {
ExprAggregateNodeUtil.getAggregatesBottomUp(statementSpec.getGroupByExpressions().getOptOrderByPerLevel(), orderByAggregateExprNodes);
}
if (!allowAggregation && !orderByAggregateExprNodes.isEmpty()) {
throw new ExprValidationException("Aggregation functions are not allowed in this context");
}
}
// Analyze rollup
GroupByRollupInfo groupByInfo = analyzeValidateGroupBy(statementSpec.getGroupByExpressions(), validationContext);
ExprNode[] groupByNodesValidated = groupByInfo == null ? new ExprNode[0] : groupByInfo.getExprNodes();
AggregationGroupByRollupDesc groupByRollupDesc = groupByInfo == null ? null : groupByInfo.getRollupDesc();
// Construct the appropriate aggregation service
boolean hasGroupBy = groupByNodesValidated.length > 0;
AggregationServiceFactoryDesc aggregationServiceFactory = AggregationServiceFactoryFactory.getService(
selectAggregateExprNodes, groupByNodesValidated, havingAggregateExprNodes, orderByAggregateExprNodes, Collections.<ExprAggregateNodeGroupKey>emptyList(), hasGroupBy, evaluatorContextStmt, statementSpec.getAnnotations(), stmtContext.getVariableService(), typeService.getEventTypes().length > 1, false,
statementSpec.getFilterRootNode(), statementSpec.getHavingExprRootNode(),
stmtContext.getAggregationServiceFactoryService(), typeService.getEventTypes(), stmtContext.getMethodResolutionService(), groupByRollupDesc,
statementSpec.getOptionalContextName());
boolean useCollatorSort = false;
if (stmtContext.getConfigSnapshot() != null) {
useCollatorSort = stmtContext.getConfigSnapshot().getEngineDefaults().getLanguage().isSortUsingCollator();
}
// Construct the processor for sorting output events
OrderByProcessorFactory orderByProcessorFactory = OrderByProcessorFactoryFactory.getProcessor(namedSelectionList,
groupByNodesValidated, orderByList, statementSpec.getRowLimitSpec(), stmtContext.getVariableService(), useCollatorSort, statementSpec.getOptionalContextName());
// Construct the processor for evaluating the select clause
SelectExprEventTypeRegistry selectExprEventTypeRegistry = new SelectExprEventTypeRegistry(stmtContext.getStatementName(), stmtContext.getStatementEventTypeRef());
SelectExprProcessor selectExprProcessor = SelectExprProcessorFactory.getProcessor(Collections.<Integer>emptyList(), selectClauseSpec.getSelectExprList(), isUsingWildcard, insertIntoDesc, statementSpec.getForClauseSpec(), typeService, stmtContext.getEventAdapterService(), stmtContext.getStatementResultService(), stmtContext.getValueAddEventService(), selectExprEventTypeRegistry, stmtContext.getMethodResolutionService(), evaluatorContextStmt,
stmtContext.getVariableService(), stmtContext.getTimeProvider(), stmtContext.getEngineURI(), stmtContext.getStatementId(), stmtContext.getStatementName(), stmtContext.getAnnotations(), stmtContext.getContextDescriptor(), stmtContext.getConfigSnapshot(), selectExprProcessorCallback, stmtContext.getNamedWindowService());
// Get a list of event properties being aggregated in the select clause, if any
Set<Pair<Integer, String>> propertiesGroupBy = ExprNodeUtility.getGroupByPropertiesValidateHasOne(groupByNodesValidated);
// Figure out all non-aggregated event properties in the select clause (props not under a sum/avg/max aggregation node)
Set<Pair<Integer, String>> nonAggregatedPropsSelect = ExprNodeUtility.getNonAggregatedProps(typeService.getEventTypes(), selectNodes, contextPropertyRegistry);
if (optionalHavingNode != null) {
ExprNodeUtility.addNonAggregatedProps(optionalHavingNode, nonAggregatedPropsSelect);
}
// Validate that group-by is filled with sensible nodes (identifiers, and not part of aggregates selected, no aggregates)
validateGroupByNoAggregations(groupByNodesValidated);
// Validate the having-clause (selected aggregate nodes and all in group-by are allowed)
boolean hasAggregation = (!selectAggregateExprNodes.isEmpty()) || (!havingAggregateExprNodes.isEmpty()) || (!orderByAggregateExprNodes.isEmpty()) || (!propertiesAggregatedHaving.isEmpty());
if (optionalHavingNode != null && hasAggregation)
{
validateHaving(propertiesGroupBy, optionalHavingNode);
}
// We only generate Remove-Stream events if they are explicitly selected, or the insert-into requires them
boolean isSelectRStream = (statementSpec.getSelectStreamSelectorEnum() == SelectClauseStreamSelectorEnum.RSTREAM_ISTREAM_BOTH
|| statementSpec.getSelectStreamSelectorEnum() == SelectClauseStreamSelectorEnum.RSTREAM_ONLY);
if ((statementSpec.getInsertIntoDesc() != null) && (statementSpec.getInsertIntoDesc().getStreamSelector().isSelectsRStream()))
{
isSelectRStream = true;
}
ExprEvaluator optionHavingEval = optionalHavingNode == null ? null : optionalHavingNode.getExprEvaluator();
// (1)
// There is no group-by clause and no aggregate functions with event properties in the select clause and having clause (simplest case)
if ((groupByNodesValidated.length == 0) && (selectAggregateExprNodes.isEmpty()) && (havingAggregateExprNodes.isEmpty()))
{
// Determine if any output rate limiting must be performed early while processing results
// Snapshot output does not count in terms of limiting output for grouping/aggregation purposes
boolean isOutputLimitingNoSnapshot = (outputLimitSpec != null) && (outputLimitSpec.getDisplayLimit() != OutputLimitLimitType.SNAPSHOT);
// (1a)
// There is no need to perform select expression processing, the single view itself (no join) generates
// events in the desired format, therefore there is no output processor. There are no order-by expressions.
if (orderByNodes.isEmpty() && optionalHavingNode == null && !isOutputLimitingNoSnapshot && statementSpec.getRowLimitSpec() == null)
{
log.debug(".getProcessor Using no result processor");
ResultSetProcessorHandThrougFactory factory = new ResultSetProcessorHandThrougFactory(selectExprProcessor, isSelectRStream);
return new ResultSetProcessorFactoryDesc(factory, orderByProcessorFactory, aggregationServiceFactory);
}
// (1b)
// We need to process the select expression in a simple fashion, with each event (old and new)
// directly generating one row, and no need to update aggregate state since there is no aggregate function.
// There might be some order-by expressions.
log.debug(".getProcessor Using ResultSetProcessorSimple");
ResultSetProcessorSimpleFactory factory = new ResultSetProcessorSimpleFactory(selectExprProcessor, optionHavingEval, isSelectRStream);
return new ResultSetProcessorFactoryDesc(factory, orderByProcessorFactory, aggregationServiceFactory);
}
// (2)
// A wildcard select-clause has been specified and the group-by is ignored since no aggregation functions are used, and no having clause
boolean isLast = statementSpec.getOutputLimitSpec() != null && statementSpec.getOutputLimitSpec().getDisplayLimit() == OutputLimitLimitType.LAST;
boolean isFirst = statementSpec.getOutputLimitSpec() != null && statementSpec.getOutputLimitSpec().getDisplayLimit() == OutputLimitLimitType.FIRST;
if ((namedSelectionList.isEmpty()) && (propertiesAggregatedHaving.isEmpty()) && (havingAggregateExprNodes.isEmpty()) && !isLast && !isFirst)
{
log.debug(".getProcessor Using ResultSetProcessorSimple");
ResultSetProcessorSimpleFactory factory = new ResultSetProcessorSimpleFactory(selectExprProcessor, optionHavingEval, isSelectRStream);
return new ResultSetProcessorFactoryDesc(factory, orderByProcessorFactory, aggregationServiceFactory);
}
if ((groupByNodesValidated.length == 0) && hasAggregation)
{
// (3)
// There is no group-by clause and there are aggregate functions with event properties in the select clause (aggregation case)
// or having class, and all event properties are aggregated (all properties are under aggregation functions).
if ((nonAggregatedPropsSelect.isEmpty()) && (!isUsingWildcard) && (!isUsingStreamSelect) && (viewResourceDelegate == null || viewResourceDelegate.getPreviousRequests().isEmpty()))
{
log.debug(".getProcessor Using ResultSetProcessorRowForAll");
ResultSetProcessorRowForAllFactory factory = new ResultSetProcessorRowForAllFactory(selectExprProcessor, optionHavingEval, isSelectRStream, isUnidirectional);
return new ResultSetProcessorFactoryDesc(factory, orderByProcessorFactory, aggregationServiceFactory);
}
// (4)
// There is no group-by clause but there are aggregate functions with event properties in the select clause (aggregation case)
// or having clause and not all event properties are aggregated (some properties are not under aggregation functions).
log.debug(".getProcessor Using ResultSetProcessorAggregateAll");
ResultSetProcessorAggregateAllFactory factory = new ResultSetProcessorAggregateAllFactory(selectExprProcessor, optionHavingEval, isSelectRStream, isUnidirectional);
return new ResultSetProcessorFactoryDesc(factory, orderByProcessorFactory, aggregationServiceFactory);
}
// Handle group-by cases
if (groupByNodesValidated.length == 0)
{
throw new IllegalStateException("Unexpected empty group-by expression list");
}
// Figure out if all non-aggregated event properties in the select clause are listed in the group by
boolean allInGroupBy = true;
String notInGroupByReason = null;
if (isUsingStreamSelect) {
allInGroupBy = false;
notInGroupByReason = "stream select";
}
for (Pair<Integer, String> nonAggregatedProp : nonAggregatedPropsSelect) {
if (!propertiesGroupBy.contains(nonAggregatedProp)) {
allInGroupBy = false;
notInGroupByReason = "property '" + nonAggregatedProp.getSecond() + "'";
}
}
// Wildcard select-clause means we do not have all selected properties in the group
if (isUsingWildcard)
{
allInGroupBy = false;
notInGroupByReason = "wildcard select";
}
// Figure out if all non-aggregated event properties in the order-by clause are listed in the select expression
Set<Pair<Integer, String>> nonAggregatedPropsOrderBy = ExprNodeUtility.getNonAggregatedProps(typeService.getEventTypes(), orderByNodes, contextPropertyRegistry);
boolean allInSelect = true;
for (Pair<Integer, String> nonAggregatedProp : nonAggregatedPropsOrderBy)
{
if (!nonAggregatedPropsSelect.contains(nonAggregatedProp))
{
allInSelect = false;
}
}
// Wildcard select-clause means that all order-by props in the select expression
if (isUsingWildcard)
{
allInSelect = true;
}
// (4)
// There is a group-by clause, and all event properties in the select clause that are not under an aggregation
// function are listed in the group-by clause, and if there is an order-by clause, all non-aggregated properties
// referred to in the order-by clause also appear in the select (output one row per group, not one row per event)
ExprEvaluator[] groupByEval = ExprNodeUtility.getEvaluators(groupByNodesValidated);
if (allInGroupBy && allInSelect)
{
boolean noDataWindowSingleStream = typeService.getIStreamOnly()[0] && typeService.getEventTypes().length < 2;
log.debug(".getProcessor Using ResultSetProcessorRowPerGroup");
ResultSetProcessorFactory factory;
if (groupByRollupDesc != null) {
GroupByRollupPerLevelExpression perLevelExpression = getRollUpPerLevelExpressions(statementSpec, groupByNodesValidated, groupByRollupDesc, stmtContext, selectExprEventTypeRegistry, evaluatorContextStmt, insertIntoDesc, typeService, validationContext);
factory = new ResultSetProcessorRowPerGroupRollupFactory(perLevelExpression, groupByNodesValidated, groupByEval, isSelectRStream, isUnidirectional, outputLimitSpec, orderByProcessorFactory != null, noDataWindowSingleStream, groupByRollupDesc, typeService.getEventTypes().length > 1);
}
else {
factory = new ResultSetProcessorRowPerGroupFactory(selectExprProcessor, groupByNodesValidated, groupByEval, optionHavingEval, isSelectRStream, isUnidirectional, outputLimitSpec, orderByProcessorFactory != null, noDataWindowSingleStream);
}
return new ResultSetProcessorFactoryDesc(factory, orderByProcessorFactory, aggregationServiceFactory);
}
if (groupByRollupDesc != null) {
throw new ExprValidationException("Group-by with rollup requires a fully-aggregated query, the query is not full-aggregated because of " + notInGroupByReason);
}
// (6)
// There is a group-by clause, and one or more event properties in the select clause that are not under an aggregation
// function are not listed in the group-by clause (output one row per event, not one row per group)
log.debug(".getProcessor Using ResultSetProcessorAggregateGrouped");
ResultSetProcessorAggregateGroupedFactory factory = new ResultSetProcessorAggregateGroupedFactory(selectExprProcessor, groupByNodesValidated, groupByEval, optionHavingEval, isSelectRStream, isUnidirectional, outputLimitSpec, orderByProcessorFactory != null);
return new ResultSetProcessorFactoryDesc(factory, orderByProcessorFactory, aggregationServiceFactory);
}
private static GroupByRollupInfo analyzeValidateGroupBy(GroupByClauseExpressions groupBy, ExprValidationContext validationContext)
throws ExprValidationException
{
if (groupBy == null) {
return null;
}
ExprNode[] validated = new ExprNode[groupBy.getGroupByNodes().length];
ExprNodeSubselectDeclaredDotVisitor visitor = new ExprNodeSubselectDeclaredDotVisitor();
ExprNodeGroupingVisitorWParent groupingVisitor = new ExprNodeGroupingVisitorWParent();
for (int i = 0; i < validated.length; i++) {
groupBy.getGroupByNodes()[i].accept(visitor);
if (visitor.getSubselects().size() > 0) {
throw new ExprValidationException("Subselects not allowed within group-by");
}
validated[i] = ExprNodeUtility.getValidatedSubtree(ExprNodeOrigin.GROUPBY, groupBy.getGroupByNodes()[i], validationContext);
validated[i].accept(groupingVisitor);
if (!groupingVisitor.getGroupingIdNodes().isEmpty()) {
throw ExprGroupingIdNode.makeException("grouping_id");
}
if (!groupingVisitor.getGroupingNodes().isEmpty()) {
throw ExprGroupingIdNode.makeException("grouping");
}
}
if (groupBy.getGroupByRollupLevels() == null) {
return new GroupByRollupInfo(validated, null);
}
AggregationGroupByRollupDesc rollup = AggregationGroupByRollupDesc.make(groupBy.getGroupByRollupLevels());
// callback when hook reporting enabled
try {
GroupByRollupPlanHook hook = (GroupByRollupPlanHook) JavaClassHelper.getAnnotationHook(validationContext.getAnnotations(), HookType.INTERNAL_GROUPROLLUP_PLAN, GroupByRollupPlanHook.class, null);
if (hook != null) {
hook.query(new GroupByRollupPlanDesc(validated, rollup));
}
}
catch (ExprValidationException e) {
throw new EPException("Failed to obtain hook for " + HookType.INTERNAL_QUERY_PLAN);
}
return new GroupByRollupInfo(validated, rollup);
}
private static GroupByRollupPerLevelExpression getRollUpPerLevelExpressions(StatementSpecCompiled statementSpec, ExprNode[] groupByNodesValidated, AggregationGroupByRollupDesc groupByRollupDesc, StatementContext stmtContext, SelectExprEventTypeRegistry selectExprEventTypeRegistry, ExprEvaluatorContextStatement evaluatorContextStmt, InsertIntoDesc insertIntoDesc, StreamTypeService typeService, ExprValidationContext validationContext)
throws ExprValidationException
{
int numLevels = groupByRollupDesc.getLevels().length;
GroupByClauseExpressions groupByExpressions = statementSpec.getGroupByExpressions();
// allocate
SelectExprProcessor[] processors = new SelectExprProcessor[numLevels];
ExprEvaluator[] havingClauses = null;
if (groupByExpressions.getOptHavingNodePerLevel() != null) {
havingClauses = new ExprEvaluator[numLevels];
}
OrderByElement[][] orderByElements = null;
if (groupByExpressions.getOptOrderByPerLevel() != null) {
orderByElements = new OrderByElement[numLevels][];
}
// for each expression in the group-by clause determine which properties it refers to
Set<Pair<Integer, String>>[] propsPerGroupByExpr = (Set<Pair<Integer, String>>[]) new Set[groupByNodesValidated.length];
for (int i = 0; i < groupByNodesValidated.length; i++) {
propsPerGroupByExpr[i] = ExprNodeUtility.getGroupByPropertiesValidateHasOne(new ExprNode[] {groupByNodesValidated[i]});
}
// for each level obtain a separate select expression processor
for (int i = 0; i < numLevels; i++) {
AggregationGroupByRollupLevel level = groupByRollupDesc.getLevels()[i];
// determine properties rolled up for this level
Set<Pair<Integer, String>> rolledupProps = getRollupProperties(level, propsPerGroupByExpr);
ExprNode[] selectClauseLevel = groupByExpressions.getSelectClausePerLevel()[i];
SelectClauseElementCompiled[] selectClause = getRollUpSelectClause(statementSpec.getSelectClauseSpec(), selectClauseLevel, level, rolledupProps, groupByNodesValidated, validationContext);
processors[i] = SelectExprProcessorFactory.getProcessor(Collections.<Integer>emptyList(), selectClause, false, insertIntoDesc, statementSpec.getForClauseSpec(), typeService, stmtContext.getEventAdapterService(), stmtContext.getStatementResultService(), stmtContext.getValueAddEventService(), selectExprEventTypeRegistry, stmtContext.getMethodResolutionService(), evaluatorContextStmt,
stmtContext.getVariableService(), stmtContext.getTimeProvider(), stmtContext.getEngineURI(), stmtContext.getStatementId(), stmtContext.getStatementName(), stmtContext.getAnnotations(), stmtContext.getContextDescriptor(), stmtContext.getConfigSnapshot(), null, stmtContext.getNamedWindowService());
if (havingClauses != null) {
havingClauses[i] = rewriteRollupValidateExpression(ExprNodeOrigin.HAVING, groupByExpressions.getOptHavingNodePerLevel()[i], validationContext, rolledupProps, groupByNodesValidated, level).getExprEvaluator();
}
if (orderByElements != null) {
orderByElements[i] = rewriteRollupOrderBy(statementSpec.getOrderByList(), groupByExpressions.getOptOrderByPerLevel()[i], validationContext, rolledupProps, groupByNodesValidated, level);
}
}
return new GroupByRollupPerLevelExpression(processors, havingClauses, orderByElements);
}
private static OrderByElement[] rewriteRollupOrderBy(OrderByItem[] items, ExprNode[] orderByList, ExprValidationContext validationContext, Set<Pair<Integer, String>> rolledupProps, ExprNode[] groupByNodes, AggregationGroupByRollupLevel level)
throws ExprValidationException
{
OrderByElement[] elements = new OrderByElement[orderByList.length];
for (int i = 0; i < orderByList.length; i++) {
ExprNode validated = rewriteRollupValidateExpression(ExprNodeOrigin.ORDERBY, orderByList[i], validationContext, rolledupProps, groupByNodes, level);
elements[i] = new OrderByElement(validated, validated.getExprEvaluator(), items[i].isDescending());
}
return elements;
}
private static Set<Pair<Integer, String>> getRollupProperties(AggregationGroupByRollupLevel level, Set<Pair<Integer, String>>[] propsPerGroupByExpr) {
// determine properties rolled up for this level
Set<Pair<Integer, String>> rolledupProps = new HashSet<Pair<Integer, String>>();
for (int i = 0; i < propsPerGroupByExpr.length; i++) {
if (level.isAggregationTop()) {
rolledupProps.addAll(propsPerGroupByExpr[i]);
}
else {
boolean rollupContainsGroupExpr = false;
for (int num : level.getRollupKeys()) {
if (num == i) {
rollupContainsGroupExpr = true;
break;
}
}
if (!rollupContainsGroupExpr) {
rolledupProps.addAll(propsPerGroupByExpr[i]);
}
}
}
return rolledupProps;
}
private static SelectClauseElementCompiled[] getRollUpSelectClause(SelectClauseSpecCompiled selectClauseSpec, ExprNode[] selectClauseLevel, AggregationGroupByRollupLevel level, Set<Pair<Integer, String>> rolledupProps, ExprNode[] groupByNodesValidated, ExprValidationContext validationContext)
throws ExprValidationException
{
SelectClauseElementCompiled[] rewritten = new SelectClauseElementCompiled[selectClauseSpec.getSelectExprList().length];
for (int i = 0; i < rewritten.length; i++) {
SelectClauseElementCompiled spec = selectClauseSpec.getSelectExprList()[i];
if (!(spec instanceof SelectClauseExprCompiledSpec)) {
throw new ExprValidationException("Group-by clause with roll-up does not allow wildcard");
}
SelectClauseExprCompiledSpec exprSpec = (SelectClauseExprCompiledSpec) spec;
ExprNode validated = rewriteRollupValidateExpression(ExprNodeOrigin.SELECT, selectClauseLevel[i], validationContext, rolledupProps, groupByNodesValidated, level);
rewritten[i] = new SelectClauseExprCompiledSpec(validated, exprSpec.getAssignedName(), exprSpec.getProvidedName(), exprSpec.isEvents());
}
return rewritten;
}
private static ExprNode rewriteRollupValidateExpression(ExprNodeOrigin exprNodeOrigin,
ExprNode exprNode,
ExprValidationContext validationContext,
Set<Pair<Integer, String>> rolledupProps,
ExprNode[] groupByNodes,
AggregationGroupByRollupLevel level)
throws ExprValidationException
{
// rewrite grouping expressions
ExprNodeGroupingVisitorWParent groupingVisitor = new ExprNodeGroupingVisitorWParent();
exprNode.accept(groupingVisitor);
for (Pair<ExprNode, ExprGroupingNode> groupingNodePair : groupingVisitor.getGroupingNodes()) {
// obtain combination - always a single one as grouping nodes cannot have
int[] combination = getGroupExprCombination(groupByNodes, groupingNodePair.getSecond().getChildNodes());
boolean found = false;
int[] rollupIndexes = level.isAggregationTop() ? new int[0] : level.getRollupKeys();
for (int index : rollupIndexes) {
if (index == combination[0]) {
found = true;
break;
}
}
int result = found ? 0 : 1;
ExprConstantNodeImpl constant = new ExprConstantNodeImpl(result, Integer.class);
if (groupingNodePair.getFirst() != null) {
ExprNodeUtility.replaceChildNode(groupingNodePair.getFirst(), groupingNodePair.getSecond(), constant);
}
else {
exprNode = constant;
}
}
// rewrite grouping id expressions
for (Pair<ExprNode, ExprGroupingIdNode> groupingIdNodePair : groupingVisitor.getGroupingIdNodes()) {
int[] combination = getGroupExprCombination(groupByNodes, groupingIdNodePair.getSecond().getChildNodes());
int result = 0;
for (int i = 0; i < combination.length; i++) {
int index = combination[i];
boolean found = false;
int[] rollupIndexes = level.isAggregationTop() ? new int[0] : level.getRollupKeys();
for (int rollupIndex : rollupIndexes) {
if (index == rollupIndex) {
found = true;
break;
}
}
if (!found) {
result = result + pow2(combination.length - i - 1);
}
}
ExprConstantNodeImpl constant = new ExprConstantNodeImpl(result, Integer.class);
if (groupingIdNodePair.getFirst() != null) {
ExprNodeUtility.replaceChildNode(groupingIdNodePair.getFirst(), groupingIdNodePair.getSecond(), constant);
}
else {
exprNode = constant;
}
}
// rewrite properties
ExprNodeIdentifierCollectVisitorWContainer identVisitor = new ExprNodeIdentifierCollectVisitorWContainer();
exprNode.accept(identVisitor);
for (Pair<ExprNode, ExprIdentNode> node : identVisitor.getExprProperties()) {
boolean rewrite = false;
for (Pair<Integer, String> rolledupProp : rolledupProps) {
if (rolledupProp.getFirst() == node.getSecond().getStreamId() && rolledupProp.getSecond().equals(node.getSecond().getResolvedPropertyName())) {
rewrite = true;
break;
}
}
if (node.getFirst() != null && (node.getFirst() instanceof ExprPreviousNode || node.getFirst() instanceof ExprPriorNode)) {
rewrite = false;
}
if (!rewrite) {
continue;
}
ExprConstantNodeImpl constant = new ExprConstantNodeImpl(null, node.getSecond().getExprEvaluator().getType());
if (node.getFirst() != null) {
ExprNodeUtility.replaceChildNode(node.getFirst(), node.getSecond(), constant);
}
else {
exprNode = constant;
}
}
return ExprNodeUtility.getValidatedSubtree(exprNodeOrigin, exprNode, validationContext);
}
private static int[] getGroupExprCombination(ExprNode[] groupByNodes, ExprNode[] childNodes)
throws ExprValidationException
{
Set<Integer> indexes = new TreeSet<Integer>();
for (ExprNode child : childNodes) {
boolean found = false;
for (int i = 0; i < groupByNodes.length; i++) {
if (ExprNodeUtility.deepEquals(child, groupByNodes[i])) {
if (indexes.contains(i)) {
throw new ExprValidationException("Duplicate expression '" + ExprNodeUtility.toExpressionStringMinPrecedence(child) + "' among grouping function parameters");
}
indexes.add(i);
found = true;
}
}
if (!found) {
throw new ExprValidationException("Failed to find expression '" + ExprNodeUtility.toExpressionStringMinPrecedence(child) + "' among group-by expressions");
}
}
return CollectionUtil.intArray(indexes);
}
private static void validateSelectAssignColNames(SelectClauseSpecCompiled selectClauseSpec, List<SelectClauseExprCompiledSpec> namedSelectionList, ExprValidationContext validationContext)
throws ExprValidationException
{
for (int i = 0; i < selectClauseSpec.getSelectExprList().length; i++)
{
// validate element
SelectClauseElementCompiled element = selectClauseSpec.getSelectExprList()[i];
if (element instanceof SelectClauseExprCompiledSpec)
{
SelectClauseExprCompiledSpec expr = (SelectClauseExprCompiledSpec) element;
ExprNode validatedExpression = ExprNodeUtility.getValidatedSubtree(ExprNodeOrigin.SELECT, expr.getSelectExpression(), validationContext);
// determine an element name if none assigned
String asName = expr.getAssignedName();
if (asName == null) {
asName = ExprNodeUtility.toExpressionStringMinPrecedence(validatedExpression);
}
expr.setAssignedName(asName);
expr.setSelectExpression(validatedExpression);
namedSelectionList.add(expr);
}
}
}
private static void validateHaving(Set<Pair<Integer, String>> propertiesGroupedBy,
ExprNode havingNode)
throws ExprValidationException
{
List<ExprAggregateNode> aggregateNodesHaving = new LinkedList<ExprAggregateNode>();
if (aggregateNodesHaving != null)
{
ExprAggregateNodeUtil.getAggregatesBottomUp(havingNode, aggregateNodesHaving);
}
// Any non-aggregated properties must occur in the group-by clause (if there is one)
if (!propertiesGroupedBy.isEmpty())
{
ExprNodeIdentifierVisitor visitor = new ExprNodeIdentifierVisitor(true);
havingNode.accept(visitor);
List<Pair<Integer, String>> allPropertiesHaving = visitor.getExprProperties();
Set<Pair<Integer, String>> aggPropertiesHaving = ExprNodeUtility.getAggregatedProperties(aggregateNodesHaving);
allPropertiesHaving.removeAll(aggPropertiesHaving);
allPropertiesHaving.removeAll(propertiesGroupedBy);
if (!allPropertiesHaving.isEmpty())
{
String name = allPropertiesHaving.iterator().next().getSecond();
throw new ExprValidationException("Non-aggregated property '" + name + "' in the HAVING clause must occur in the group-by clause");
}
}
}
private static List<OrderByItem> expandColumnNames(SelectClauseElementCompiled[] selectionList, OrderByItem[] orderByUnexpanded)
{
if (orderByUnexpanded.length == 0) {
return Collections.emptyList();
}
// copy list to modify
List<OrderByItem> expanded = new ArrayList<OrderByItem>();
for (OrderByItem item : orderByUnexpanded) {
expanded.add(item.copy());
}
// expand
for(SelectClauseElementCompiled selectElement : selectionList)
{
// process only expressions
if (!(selectElement instanceof SelectClauseExprCompiledSpec))
{
continue;
}
SelectClauseExprCompiledSpec selectExpr = (SelectClauseExprCompiledSpec) selectElement;
String name = selectExpr.getAssignedName();
if(name != null)
{
ExprNode fullExpr = selectExpr.getSelectExpression();
for(ListIterator<OrderByItem> iterator = expanded.listIterator(); iterator.hasNext(); )
{
OrderByItem orderByElement = iterator.next();
ExprNode swapped = ColumnNamedNodeSwapper.swap(orderByElement.getExprNode(), name, fullExpr);
OrderByItem newOrderByElement = new OrderByItem(swapped, orderByElement.isDescending());
iterator.set(newOrderByElement);
}
}
}
return expanded;
}
private static void validateGroupByNoAggregations(ExprNode[] groupByNodes)
throws ExprValidationException
{
// Make sure there is no aggregate function in group-by
List<ExprAggregateNode> aggNodes = new LinkedList<ExprAggregateNode>();
for (ExprNode groupByNode : groupByNodes)
{
ExprAggregateNodeUtil.getAggregatesBottomUp(groupByNode, aggNodes);
if (!aggNodes.isEmpty())
{
throw new ExprValidationException("Group-by expressions cannot contain aggregate functions");
}
}
}
private static int pow2(int exponent) {
if (exponent == 0) {
return 1;
}
int result = 2;
for (int i = 0; i < exponent - 1; i++) {
result = 2*result;
}
return result;
}
private static class GroupByRollupInfo {
private final ExprNode[] exprNodes;
private final AggregationGroupByRollupDesc rollupDesc;
private GroupByRollupInfo(ExprNode[] exprNodes, AggregationGroupByRollupDesc rollupDesc) {
this.exprNodes = exprNodes;
this.rollupDesc = rollupDesc;
}
public ExprNode[] getExprNodes() {
return exprNodes;
}
public AggregationGroupByRollupDesc getRollupDesc() {
return rollupDesc;
}
}
private static class GroupByExpressionInfo {
private final List<ExprNode> expressions;
private final List<GroupByRollupNodeBase> nodes;
private GroupByExpressionInfo(List<ExprNode> expressions, List<GroupByRollupNodeBase> nodes) {
this.expressions = expressions;
this.nodes = nodes;
}
public List<ExprNode> getExpressions() {
return expressions;
}
public List<GroupByRollupNodeBase> getNodes() {
return nodes;
}
}
private static final Log log = LogFactory.getLog(ResultSetProcessorFactoryFactory.class);
}