/**
* Licensed to Jasig under one or more contributor license
* agreements. See the NOTICE file distributed with this work
* for additional information regarding copyright ownership.
* Jasig licenses this file to you under the Apache License,
* Version 2.0 (the "License"); you may not use this file
* except in compliance with the License. You may obtain a
* copy of the License at:
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.jasig.portal.events.aggr;
import java.io.Serializable;
import java.util.Collection;
import java.util.Collections;
import java.util.EnumMap;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import javax.annotation.Resource;
import javax.persistence.EntityManager;
import javax.persistence.FlushModeType;
import org.apache.commons.lang.mutable.MutableInt;
import org.apache.commons.lang.mutable.MutableObject;
import org.hibernate.Cache;
import org.hibernate.Session;
import org.hibernate.SessionFactory;
import org.hibernate.metadata.ClassMetadata;
import org.hibernate.type.CollectionType;
import org.hibernate.type.Type;
import org.jasig.portal.IPortalInfoProvider;
import org.jasig.portal.concurrency.locking.IClusterLockService;
import org.jasig.portal.events.PortalEvent;
import org.jasig.portal.events.aggr.IEventAggregatorStatus.ProcessingType;
import org.jasig.portal.events.aggr.dao.DateDimensionDao;
import org.jasig.portal.events.aggr.dao.IEventAggregationManagementDao;
import org.jasig.portal.events.aggr.session.EventSession;
import org.jasig.portal.events.aggr.session.EventSessionDao;
import org.jasig.portal.events.handlers.db.IPortalEventDao;
import org.jasig.portal.jpa.BaseAggrEventsJpaDao;
import org.jasig.portal.jpa.BaseRawEventsJpaDao.RawEventsTransactional;
import org.jasig.portal.spring.context.ApplicationEventFilter;
import org.jasig.portal.utils.cache.CacheKey;
import org.joda.time.DateTime;
import org.joda.time.Period;
import org.joda.time.ReadablePeriod;
import org.springframework.aop.framework.AopProxyUtils;
import org.springframework.beans.factory.DisposableBean;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.transaction.TransactionStatus;
import org.springframework.transaction.support.TransactionCallback;
import com.google.common.base.Function;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableMap.Builder;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
@Service
public class PortalRawEventsAggregatorImpl extends BaseAggrEventsJpaDao implements PortalRawEventsAggregator, DisposableBean {
private static final String EVENT_SESSION_CACHE_KEY_SOURCE = AggregateEventsHandler.class.getName() + "-EventSession";
private IClusterLockService clusterLockService;
private IPortalEventProcessingManager portalEventAggregationManager;
private PortalEventDimensionPopulator portalEventDimensionPopulator;
private IEventAggregationManagementDao eventAggregationManagementDao;
private IPortalInfoProvider portalInfoProvider;
private IPortalEventDao portalEventDao;
private AggregationIntervalHelper intervalHelper;
private EventSessionDao eventSessionDao;
private DateDimensionDao dateDimensionDao;
private Set<IntervalAwarePortalEventAggregator<PortalEvent>> intervalAwarePortalEventAggregators = Collections.emptySet();
private Set<SimplePortalEventAggregator<PortalEvent>> simplePortalEventAggregators = Collections.emptySet();
private List<ApplicationEventFilter<PortalEvent>> applicationEventFilters = Collections.emptyList();
private int eventAggregationBatchSize = 10000;
private int intervalAggregationBatchSize = 5;
private int cleanUnclosedAggregationsBatchSize = 1000;
private int cleanUnclosedIntervalsBatchSize = 315;
private ReadablePeriod aggregationDelay = Period.seconds(30);
private final Map<Class<?>, List<String>> entityCollectionRoles = new HashMap<Class<?>, List<String>>();
private volatile boolean shutdown = false;
@Autowired
public void setDateDimensionDao(DateDimensionDao dateDimensionDao) {
this.dateDimensionDao = dateDimensionDao;
}
@Autowired
public void setPortalEventAggregationManager(IPortalEventProcessingManager portalEventAggregationManager) {
this.portalEventAggregationManager = portalEventAggregationManager;
}
@Autowired
public void setClusterLockService(IClusterLockService clusterLockService) {
this.clusterLockService = clusterLockService;
}
@Autowired
public void setPortalEventDimensionPopulator(PortalEventDimensionPopulator portalEventDimensionPopulator) {
this.portalEventDimensionPopulator = portalEventDimensionPopulator;
}
@Autowired
public void setEventAggregationManagementDao(IEventAggregationManagementDao eventAggregationManagementDao) {
this.eventAggregationManagementDao = eventAggregationManagementDao;
}
@Autowired
public void setPortalInfoProvider(IPortalInfoProvider portalInfoProvider) {
this.portalInfoProvider = portalInfoProvider;
}
@Autowired
public void setPortalEventDao(IPortalEventDao portalEventDao) {
this.portalEventDao = portalEventDao;
}
@Autowired
public void setIntervalHelper(AggregationIntervalHelper intervalHelper) {
this.intervalHelper = intervalHelper;
}
@Autowired
public void setEventSessionDao(EventSessionDao eventSessionDao) {
this.eventSessionDao = eventSessionDao;
}
@Autowired
@SuppressWarnings({ "rawtypes", "unchecked" })
public void setPortalEventAggregators(Set<IPortalEventAggregator<PortalEvent>> portalEventAggregators) {
final com.google.common.collect.ImmutableSet.Builder<IntervalAwarePortalEventAggregator<PortalEvent>> intervalAwarePortalEventAggregatorsBuilder = ImmutableSet.builder();
final com.google.common.collect.ImmutableSet.Builder<SimplePortalEventAggregator<PortalEvent>> simplePortalEventAggregatorsBuilder = ImmutableSet.builder();
for (final IPortalEventAggregator<PortalEvent> portalEventAggregator : portalEventAggregators) {
if (portalEventAggregator instanceof IntervalAwarePortalEventAggregator) {
intervalAwarePortalEventAggregatorsBuilder.add((IntervalAwarePortalEventAggregator)portalEventAggregator);
}
else if (portalEventAggregator instanceof SimplePortalEventAggregator) {
simplePortalEventAggregatorsBuilder.add((SimplePortalEventAggregator)portalEventAggregator);
}
}
this.intervalAwarePortalEventAggregators = intervalAwarePortalEventAggregatorsBuilder.build();
this.simplePortalEventAggregators = simplePortalEventAggregatorsBuilder.build();
}
@Resource(name="aggregatorEventFilters")
public void setApplicationEventFilters(List<ApplicationEventFilter<PortalEvent>> applicationEventFilters) {
this.applicationEventFilters = applicationEventFilters;
}
@Value("${org.jasig.portal.events.aggr.PortalRawEventsAggregatorImpl.aggregationDelay:PT30S}")
public void setAggregationDelay(ReadablePeriod aggregationDelay) {
this.aggregationDelay = aggregationDelay;
}
@Value("${org.jasig.portal.events.aggr.PortalRawEventsAggregatorImpl.eventAggregationBatchSize:10000}")
public void setEventAggregationBatchSize(int eventAggregationBatchSize) {
this.eventAggregationBatchSize = eventAggregationBatchSize;
}
@Value("${org.jasig.portal.events.aggr.PortalRawEventsAggregatorImpl.intervalAggregationBatchSize:5}")
public void setIntervalAggregationBatchSize(int intervalAggregationBatchSize) {
this.intervalAggregationBatchSize = intervalAggregationBatchSize;
}
@Value("${org.jasig.portal.events.aggr.PortalRawEventsAggregatorImpl.cleanUnclosedAggregationsBatchSize:1000}")
public void setCleanUnclosedAggregationsBatchSize(int cleanUnclosedAggregationsBatchSize) {
this.cleanUnclosedAggregationsBatchSize = cleanUnclosedAggregationsBatchSize;
}
@Value("${org.jasig.portal.events.aggr.PortalRawEventsAggregatorImpl.cleanUnclosedIntervalsBatchSize:300}")
public void setCleanUnclosedIntervalsBatchSize(int cleanUnclosedIntervalsBatchSize) {
this.cleanUnclosedIntervalsBatchSize = cleanUnclosedIntervalsBatchSize;
}
public void setShutdown(boolean shutdown) {
this.shutdown = shutdown;
}
@Override
public void destroy() throws Exception {
this.shutdown = true;
}
private void checkShutdown() {
if (shutdown) {
//Mark ourselves as interupted and throw an exception
Thread.currentThread().interrupt();
throw new RuntimeException("uPortal is shutting down, throwing an exception to stop processing");
}
}
@RawEventsTransactional
@Override
public EventProcessingResult doAggregateRawEvents() {
//Do RawTX around AggrTX. The AggrTX is MUCH more likely to fail than the RawTX and this results in both rolling back
return this.getTransactionOperations().execute(new TransactionCallback<EventProcessingResult>() {
@Override
public EventProcessingResult doInTransaction(TransactionStatus status) {
return doAggregateRawEventsInternal();
}
});
}
@AggrEventsTransactional
@Override
public void evictAggregates(Map<Class<?>, Collection<Serializable>> entitiesToEvict) {
int evictedEntities = 0;
int evictedCollections = 0;
final Session session = getEntityManager().unwrap(Session.class);
final SessionFactory sessionFactory = session.getSessionFactory();
final Cache cache = sessionFactory.getCache();
for (final Entry<Class<?>, Collection<Serializable>> evictedEntityEntry : entitiesToEvict.entrySet()) {
final Class<?> entityClass = evictedEntityEntry.getKey();
final List<String> collectionRoles = getCollectionRoles(sessionFactory, entityClass);
for (final Serializable id : evictedEntityEntry.getValue()) {
cache.evictEntity(entityClass, id);
evictedEntities++;
for (final String collectionRole : collectionRoles) {
cache.evictCollection(collectionRole, id);
evictedCollections++;
}
}
}
logger.debug("Evicted {} entities and {} collections from hibernate caches", evictedEntities, evictedCollections);
}
@Override
@AggrEventsTransactional
public EventProcessingResult doCloseAggregations() {
if (!this.clusterLockService.isLockOwner(AGGREGATION_LOCK_NAME)) {
throw new IllegalStateException("The cluster lock " + AGGREGATION_LOCK_NAME + " must be owned by the current thread and server");
}
final IEventAggregatorStatus cleanUnclosedStatus = eventAggregationManagementDao.getEventAggregatorStatus(ProcessingType.CLEAN_UNCLOSED, true);
//Update status with current server name
final String serverName = this.portalInfoProvider.getUniqueServerName();
cleanUnclosedStatus.setServerName(serverName);
cleanUnclosedStatus.setLastStart(new DateTime());
//Determine date of most recently aggregated data
final IEventAggregatorStatus eventAggregatorStatus = eventAggregationManagementDao.getEventAggregatorStatus(ProcessingType.AGGREGATION, false);
if (eventAggregatorStatus == null || eventAggregatorStatus.getLastEventDate() == null) {
//Nothing has been aggregated, skip unclosed cleanup
cleanUnclosedStatus.setLastEnd(new DateTime());
eventAggregationManagementDao.updateEventAggregatorStatus(cleanUnclosedStatus);
return new EventProcessingResult(0, null, null, true);
}
final DateTime lastAggregatedDate = eventAggregatorStatus.getLastEventDate();
//If lastCleanUnclosedDate is null use the oldest date dimension as there can be
//no aggregations that exist before it
final DateTime lastCleanUnclosedDate;
if (cleanUnclosedStatus.getLastEventDate() == null) {
final DateDimension oldestDateDimension = this.dateDimensionDao.getOldestDateDimension();
lastCleanUnclosedDate = oldestDateDimension.getDate().toDateTime();
}
else {
lastCleanUnclosedDate = cleanUnclosedStatus.getLastEventDate();
}
if (!(lastCleanUnclosedDate.isBefore(lastAggregatedDate))) {
logger.debug("No events aggregated since last unclosed aggregation cleaning, skipping clean: {}", lastAggregatedDate);
return new EventProcessingResult(0, lastCleanUnclosedDate, lastAggregatedDate, true);
}
//Switch to flush on commit to avoid flushes during queries
final EntityManager entityManager = this.getEntityManager();
entityManager.flush();
entityManager.setFlushMode(FlushModeType.COMMIT);
//Track the number of closed aggregations and the last date of a cleaned interval
int closedAggregations = 0;
int cleanedIntervals = 0;
DateTime cleanUnclosedEnd;
final Thread currentThread = Thread.currentThread();
final String currentName = currentThread.getName();
try {
currentThread.setName(currentName + "-" + lastCleanUnclosedDate + "-" + lastAggregatedDate);
//Local caches used to reduce db io
final IntervalsForAggregatorHelper intervalsForAggregatorHelper = new IntervalsForAggregatorHelper();
final Map<AggregationInterval, AggregationIntervalInfo> previousIntervals = new HashMap<AggregationInterval, AggregationIntervalInfo>();
//A DateTime within the next interval to close aggregations in
DateTime nextIntervalDate = lastCleanUnclosedDate;
do {
//Reset our goal of catching up to the last aggregated event on every iteration
cleanUnclosedEnd = lastAggregatedDate;
//For each interval the aggregator supports, cleanup the unclosed aggregations
for (final AggregationInterval interval : intervalsForAggregatorHelper.getHandledIntervals()) {
final AggregationIntervalInfo previousInterval = previousIntervals.get(interval);
if (previousInterval != null && nextIntervalDate.isBefore(previousInterval.getEnd())) {
logger.debug("{} interval before {} has already been cleaned during this execution, ignoring", interval, previousInterval.getEnd());
continue;
}
//The END date of the last clean session will find us the next interval to clean
final AggregationIntervalInfo nextIntervalToClean = intervalHelper.getIntervalInfo(interval, nextIntervalDate);
previousIntervals.put(interval, nextIntervalToClean);
if (nextIntervalToClean == null) {
continue;
}
final DateTime start = nextIntervalToClean.getStart();
final DateTime end = nextIntervalToClean.getEnd();
if (!end.isBefore(lastAggregatedDate)) {
logger.debug("{} interval between {} and {} is still active, ignoring", new Object[] { interval, start, end});
continue;
}
//Track the oldest interval end, this ensures that nothing is missed
if (end.isBefore(cleanUnclosedEnd)) {
cleanUnclosedEnd = end;
}
logger.debug("Cleaning unclosed {} aggregations between {} and {}", new Object[] { interval, start, end});
for (final IntervalAwarePortalEventAggregator<PortalEvent> portalEventAggregator : intervalAwarePortalEventAggregators) {
checkShutdown();
final Class<? extends IPortalEventAggregator<?>> aggregatorType = getClass(portalEventAggregator);
//Get aggregator specific interval info config
final AggregatedIntervalConfig aggregatorIntervalConfig = intervalsForAggregatorHelper.getAggregatorIntervalConfig(aggregatorType);
//If the aggregator is being used for the specified interval call cleanUnclosedAggregations
if (aggregatorIntervalConfig.isIncluded(interval)) {
closedAggregations += portalEventAggregator.cleanUnclosedAggregations(start, end, interval);
}
}
cleanedIntervals++;
}
//Set the next interval to the end date from the last aggregation run
nextIntervalDate = cleanUnclosedEnd;
logger.debug("Closed {} aggregations across {} interval before {} with goal of {}", new Object[] { closedAggregations, cleanedIntervals, cleanUnclosedEnd, lastAggregatedDate});
//Loop until either the batchSize of cleaned aggregations has been reached or no aggregation work is done
} while (closedAggregations <= cleanUnclosedAggregationsBatchSize && cleanedIntervals <= cleanUnclosedIntervalsBatchSize && cleanUnclosedEnd.isBefore(lastAggregatedDate));
}
finally {
currentThread.setName(currentName);
}
//Update the status object and store it
cleanUnclosedStatus.setLastEventDate(cleanUnclosedEnd);
cleanUnclosedStatus.setLastEnd(new DateTime());
eventAggregationManagementDao.updateEventAggregatorStatus(cleanUnclosedStatus);
return new EventProcessingResult(closedAggregations, lastCleanUnclosedDate, lastAggregatedDate, !cleanUnclosedEnd.isBefore(lastAggregatedDate));
}
@SuppressWarnings("unchecked")
protected final <T> Class<T> getClass(T object) {
return (Class<T>)AopProxyUtils.ultimateTargetClass(object);
}
private List<String> getCollectionRoles(final SessionFactory sessionFactory, final Class<?> entityClass) {
List<String> collectionRoles = entityCollectionRoles.get(entityClass);
if (collectionRoles != null) {
return collectionRoles;
}
final com.google.common.collect.ImmutableList.Builder<String> collectionRolesBuilder = ImmutableList.builder();
final ClassMetadata classMetadata = sessionFactory.getClassMetadata(entityClass);
for (final Type type : classMetadata.getPropertyTypes()) {
if (type.isCollectionType()) {
collectionRolesBuilder.add(((CollectionType)type).getRole());
}
}
collectionRoles = collectionRolesBuilder.build();
entityCollectionRoles.put(entityClass, collectionRoles);
return collectionRoles;
}
private EventProcessingResult doAggregateRawEventsInternal() {
if (!this.clusterLockService.isLockOwner(AGGREGATION_LOCK_NAME)) {
throw new IllegalStateException("The cluster lock " + AGGREGATION_LOCK_NAME + " must be owned by the current thread and server");
}
if (!this.portalEventDimensionPopulator.isCheckedDimensions()) {
//First time aggregation has happened, run populateDimensions to ensure enough dimension data exists
final boolean populatedDimensions = this.portalEventAggregationManager.populateDimensions();
if (!populatedDimensions) {
this.logger.warn("Aborting raw event aggregation, populateDimensions returned false so the state of date/time dimensions is unknown");
return null;
}
}
//Flush any dimension creation before aggregation
final EntityManager entityManager = this.getEntityManager();
entityManager.flush();
entityManager.setFlushMode(FlushModeType.COMMIT);
final IEventAggregatorStatus eventAggregatorStatus = eventAggregationManagementDao.getEventAggregatorStatus(ProcessingType.AGGREGATION, true);
//Update status with current server name
final String serverName = this.portalInfoProvider.getUniqueServerName();
final String previousServerName = eventAggregatorStatus.getServerName();
if (previousServerName != null && !serverName.equals(previousServerName)) {
this.logger.debug("Last aggregation run on {} clearing all aggregation caches", previousServerName);
final Session session = getEntityManager().unwrap(Session.class);
final Cache cache = session.getSessionFactory().getCache();
cache.evictEntityRegions();
}
eventAggregatorStatus.setServerName(serverName);
//Calculate date range for aggregation
DateTime lastAggregated = eventAggregatorStatus.getLastEventDate();
if (lastAggregated == null) {
lastAggregated = portalEventDao.getOldestPortalEventTimestamp();
//No portal events to aggregate, skip aggregation
if (lastAggregated == null) {
return new EventProcessingResult(0, null, null, true);
}
//First time aggregation has run, initialize the CLEAN_UNCLOSED status to save catch-up time
final IEventAggregatorStatus cleanUnclosedStatus = eventAggregationManagementDao.getEventAggregatorStatus(ProcessingType.CLEAN_UNCLOSED, true);
AggregationIntervalInfo oldestMinuteInterval = this.intervalHelper.getIntervalInfo(AggregationInterval.MINUTE, lastAggregated);
cleanUnclosedStatus.setLastEventDate(oldestMinuteInterval.getStart().minusMinutes(1));
eventAggregationManagementDao.updateEventAggregatorStatus(cleanUnclosedStatus);
}
final DateTime newestEventTime = DateTime.now().minus(this.aggregationDelay).secondOfMinute().roundFloorCopy();
final Thread currentThread = Thread.currentThread();
final String currentName = currentThread.getName();
final MutableInt events = new MutableInt();
final MutableObject lastEventDate = new MutableObject(newestEventTime);
boolean complete;
try {
currentThread.setName(currentName + "-" + lastAggregated + "_" + newestEventTime);
logger.debug("Starting aggregation of events between {} (inc) and {} (exc)", lastAggregated, newestEventTime);
//Do aggregation, capturing the start and end dates
eventAggregatorStatus.setLastStart(DateTime.now());
complete = portalEventDao.aggregatePortalEvents(
lastAggregated, newestEventTime, this.eventAggregationBatchSize,
new AggregateEventsHandler(events, lastEventDate, eventAggregatorStatus));
eventAggregatorStatus.setLastEventDate((DateTime)lastEventDate.getValue());
eventAggregatorStatus.setLastEnd(DateTime.now());
}
finally {
currentThread.setName(currentName);
}
//Store the results of the aggregation
eventAggregationManagementDao.updateEventAggregatorStatus(eventAggregatorStatus);
complete = complete && (this.eventAggregationBatchSize <= 0 || events.intValue() < this.eventAggregationBatchSize);
return new EventProcessingResult(events.intValue(), lastAggregated, eventAggregatorStatus.getLastEventDate(), complete);
}
/**
* Helper class that loads and caches the interval configuration for each aggregator as well as the union of intervals handled by the set of aggregators.
*/
private final class IntervalsForAggregatorHelper {
private final Map<Class<? extends IPortalEventAggregator<?>>, AggregatedIntervalConfig> aggregatorIntervalConfigsCache = new HashMap<Class<? extends IPortalEventAggregator<?>>, AggregatedIntervalConfig>();
private final AggregatedIntervalConfig defaultAggregatedIntervalConfig;
private final Set<AggregationInterval> handledIntervals;
public IntervalsForAggregatorHelper() {
this.defaultAggregatedIntervalConfig = eventAggregationManagementDao.getDefaultAggregatedIntervalConfig();
//Create the set of intervals that are actually being aggregated
final Set<AggregationInterval> handledIntervalsNotIncluded = EnumSet.allOf(AggregationInterval.class);
final Set<AggregationInterval> handledIntervalsBuilder = EnumSet.noneOf(AggregationInterval.class);
for (final IntervalAwarePortalEventAggregator<PortalEvent> portalEventAggregator : intervalAwarePortalEventAggregators) {
final Class<? extends IPortalEventAggregator<?>> aggregatorType = PortalRawEventsAggregatorImpl.this.getClass(portalEventAggregator);
//Get aggregator specific interval info config
final AggregatedIntervalConfig aggregatorIntervalConfig = this.getAggregatorIntervalConfig(aggregatorType);
for (final Iterator<AggregationInterval> intervalsIterator = handledIntervalsNotIncluded.iterator(); intervalsIterator.hasNext(); ) {
final AggregationInterval interval = intervalsIterator.next();
if (aggregatorIntervalConfig.isIncluded(interval)) {
handledIntervalsBuilder.add(interval);
intervalsIterator.remove();
}
}
}
handledIntervals = Sets.immutableEnumSet(handledIntervalsBuilder);
}
/**
* @return All of the intervals that are actually handled by the current set of aggregators
*/
public Set<AggregationInterval> getHandledIntervals() {
return handledIntervals;
}
/**
* @return The interval config for the aggregator, returns the default config if no aggregator specific config is set
*/
public AggregatedIntervalConfig getAggregatorIntervalConfig(final Class<? extends IPortalEventAggregator<?>> aggregatorType) {
AggregatedIntervalConfig config = aggregatorIntervalConfigsCache.get(aggregatorType);
if (config != null) {
return config;
}
config = eventAggregationManagementDao.getAggregatedIntervalConfig(aggregatorType);
if (config == null) {
config = defaultAggregatedIntervalConfig;
}
aggregatorIntervalConfigsCache.put(aggregatorType, config);
return config;
}
}
private final class AggregateEventsHandler implements Function<PortalEvent, Boolean> {
//Event Aggregation Context - used by aggregators to track state
private final EventAggregationContext eventAggregationContext = new EventAggregationContextImpl();
private final MutableInt eventCounter;
private final MutableObject lastEventDate;
private final IEventAggregatorStatus eventAggregatorStatus;
private int intervalsCrossed = 0;
//Local tracking of the current aggregation interval and info about said interval
private final Map<AggregationInterval, AggregationIntervalInfo> currentIntervalInfo = new EnumMap<AggregationInterval, AggregationIntervalInfo>(AggregationInterval.class);
//Local caches of per-aggregator config data, shouldn't ever change for the duration of an aggregation run
private final IntervalsForAggregatorHelper intervalsForAggregatorHelper = new IntervalsForAggregatorHelper();
private final Map<Class<? extends IPortalEventAggregator<?>>, AggregatedGroupConfig> aggregatorGroupConfigs = new HashMap<Class<? extends IPortalEventAggregator<?>>, AggregatedGroupConfig>();
private final Map<Class<? extends IPortalEventAggregator<?>>, Map<AggregationInterval, AggregationIntervalInfo>> aggregatorReadOnlyIntervalInfo = new HashMap<Class<? extends IPortalEventAggregator<?>>, Map<AggregationInterval,AggregationIntervalInfo>>();
private final AggregatedGroupConfig defaultAggregatedGroupConfig;
private AggregateEventsHandler(MutableInt eventCounter, MutableObject lastEventDate, IEventAggregatorStatus eventAggregatorStatus) {
this.eventCounter = eventCounter;
this.lastEventDate = lastEventDate;
this.eventAggregatorStatus = eventAggregatorStatus;
this.defaultAggregatedGroupConfig = eventAggregationManagementDao.getDefaultAggregatedGroupConfig();
}
@Override
public Boolean apply(PortalEvent event) {
if (shutdown) {
//Mark ourselves as interupted and throw an exception
Thread.currentThread().interrupt();
throw new RuntimeException("uPortal is shutting down, throwing an exeption to stop aggregation");
}
final DateTime eventDate = event.getTimestampAsDate();
this.lastEventDate.setValue(eventDate);
//If no interval data yet populate it.
if (this.currentIntervalInfo.isEmpty()) {
initializeIntervalInfo(eventDate);
}
//Check each interval to see if an interval boundary has been crossed
boolean intervalCrossed = false;
for (final AggregationInterval interval : this.intervalsForAggregatorHelper.getHandledIntervals()) {
AggregationIntervalInfo intervalInfo = this.currentIntervalInfo.get(interval);
if (intervalInfo != null && !intervalInfo.getEnd().isAfter(eventDate)) { //if there is no IntervalInfo that interval must not be supported in the current environment
logger.debug("Crossing {} Interval, triggered by {}", interval, event);
this.doHandleIntervalBoundary(interval, this.currentIntervalInfo);
intervalInfo = intervalHelper.getIntervalInfo(interval, eventDate);
this.currentIntervalInfo.put(interval, intervalInfo);
this.aggregatorReadOnlyIntervalInfo.clear(); //Clear out cached per-aggregator interval info whenever a current interval info changes
intervalCrossed = true;
}
}
if (intervalCrossed) {
this.intervalsCrossed++;
//If we have crossed more intervals than the interval batch size return false to stop aggregation before handling the triggering event
if (this.intervalsCrossed >= intervalAggregationBatchSize) {
return false;
}
}
//Aggregate the event
this.doAggregateEvent(event);
//Update the status object with the event date
this.lastEventDate.setValue(eventDate);
//Continue processing
return true;
}
private void initializeIntervalInfo(final DateTime eventDate) {
final DateTime intervalDate;
final DateTime lastEventDate = this.eventAggregatorStatus.getLastEventDate();
if (lastEventDate != null) {
//If there was a previously aggregated event use that date to make sure an interval is not missed
intervalDate = lastEventDate;
}
else {
//Otherwise just use the current event date
intervalDate = eventDate;
}
for (final AggregationInterval interval : this.intervalsForAggregatorHelper.getHandledIntervals()) {
final AggregationIntervalInfo intervalInfo = intervalHelper.getIntervalInfo(interval, intervalDate);
if (intervalInfo != null) {
this.currentIntervalInfo.put(interval, intervalInfo);
}
else {
this.currentIntervalInfo.remove(interval);
}
}
}
private void doAggregateEvent(PortalEvent item) {
checkShutdown();
eventCounter.increment();
for (final ApplicationEventFilter<PortalEvent> applicationEventFilter : applicationEventFilters) {
if (!applicationEventFilter.supports(item)) {
logger.trace("Skipping event {} - {} excluded by filter {}", new Object[] { eventCounter, item,
applicationEventFilter });
return;
}
}
logger.trace("Aggregating event {} - {}", eventCounter, item);
//Load or create the event session
EventSession eventSession = getEventSession(item);
//Give each interval aware aggregator a chance at the event
for (final IntervalAwarePortalEventAggregator<PortalEvent> portalEventAggregator : intervalAwarePortalEventAggregators) {
if (checkSupports(portalEventAggregator, item)) {
final Class<? extends IPortalEventAggregator<?>> aggregatorType = PortalRawEventsAggregatorImpl.this.getClass(portalEventAggregator);
//Get aggregator specific interval info map
final Map<AggregationInterval, AggregationIntervalInfo> aggregatorIntervalInfo = this.getAggregatorIntervalInfo(aggregatorType);
//If there is an event session get the aggregator specific version of it
if (eventSession != null) {
final AggregatedGroupConfig aggregatorGroupConfig = getAggregatorGroupConfig(aggregatorType);
final CacheKey key = CacheKey.build(EVENT_SESSION_CACHE_KEY_SOURCE, eventSession, aggregatorGroupConfig);
EventSession filteredEventSession = this.eventAggregationContext.getAttribute(key);
if (filteredEventSession == null) {
filteredEventSession = new FilteredEventSession(eventSession, aggregatorGroupConfig);
this.eventAggregationContext.setAttribute(key, filteredEventSession);
}
eventSession = filteredEventSession;
}
//Aggregation magic happens here!
portalEventAggregator.aggregateEvent(item, eventSession, eventAggregationContext, aggregatorIntervalInfo);
}
}
//Give each simple aggregator a chance at the event
for (final SimplePortalEventAggregator<PortalEvent> portalEventAggregator : simplePortalEventAggregators) {
if (checkSupports(portalEventAggregator, item)) {
portalEventAggregator.aggregateEvent(item, eventSession);
}
}
}
/**
* @deprecated This method exists until uPortal 4.1 when IPortalEventAggregator#supports(Class) can be deleted
*/
@Deprecated
protected boolean checkSupports(IPortalEventAggregator<PortalEvent> portalEventAggregator, PortalEvent item) {
try {
return portalEventAggregator.supports(item);
}
catch (AbstractMethodError e) {
return portalEventAggregator.supports(item.getClass());
}
}
protected EventSession getEventSession(PortalEvent item) {
final String eventSessionId = item.getEventSessionId();
//First check the aggregation context for a cached session event, fall back
//to asking the DAO if nothing in the context, cache the result
final CacheKey key = CacheKey.build(EVENT_SESSION_CACHE_KEY_SOURCE, eventSessionId);
EventSession eventSession = this.eventAggregationContext.getAttribute(key);
if (eventSession == null) {
eventSession = eventSessionDao.getEventSession(item);
this.eventAggregationContext.setAttribute(key, eventSession);
}
//Record the session access
eventSession.recordAccess(item.getTimestampAsDate());
eventSessionDao.storeEventSession(eventSession);
return eventSession;
}
private void doHandleIntervalBoundary(AggregationInterval interval, Map<AggregationInterval, AggregationIntervalInfo> intervals) {
for (final IntervalAwarePortalEventAggregator<PortalEvent> portalEventAggregator : intervalAwarePortalEventAggregators) {
final Class<? extends IPortalEventAggregator<?>> aggregatorType = PortalRawEventsAggregatorImpl.this.getClass(portalEventAggregator);
final AggregatedIntervalConfig aggregatorIntervalConfig = this.intervalsForAggregatorHelper.getAggregatorIntervalConfig(aggregatorType);
//If the aggreagator is configured to use the interval notify it of the interval boundary
if (aggregatorIntervalConfig.isIncluded(interval)) {
final Map<AggregationInterval, AggregationIntervalInfo> aggregatorIntervalInfo = this.getAggregatorIntervalInfo(aggregatorType);
portalEventAggregator.handleIntervalBoundary(interval, eventAggregationContext, aggregatorIntervalInfo);
}
}
}
/**
* @return The interval info map for the aggregator
*/
protected Map<AggregationInterval, AggregationIntervalInfo> getAggregatorIntervalInfo(final Class<? extends IPortalEventAggregator<?>> aggregatorType) {
final AggregatedIntervalConfig aggregatorIntervalConfig = this.intervalsForAggregatorHelper.getAggregatorIntervalConfig(aggregatorType);
Map<AggregationInterval, AggregationIntervalInfo> intervalInfo = this.aggregatorReadOnlyIntervalInfo.get(aggregatorType);
if (intervalInfo == null) {
final Builder<AggregationInterval, AggregationIntervalInfo> intervalInfoBuilder = ImmutableMap.builder();
for (Map.Entry<AggregationInterval, AggregationIntervalInfo> intervalInfoEntry : this.currentIntervalInfo.entrySet()) {
final AggregationInterval key = intervalInfoEntry.getKey();
if (aggregatorIntervalConfig.isIncluded(key)) {
intervalInfoBuilder.put(key, intervalInfoEntry.getValue());
}
}
intervalInfo = intervalInfoBuilder.build();
aggregatorReadOnlyIntervalInfo.put(aggregatorType, intervalInfo);
}
return intervalInfo;
}
/**
* @return The group config for the aggregator, returns the default config if no aggregator specific config is set
*/
protected AggregatedGroupConfig getAggregatorGroupConfig(final Class<? extends IPortalEventAggregator<?>> aggregatorType) {
AggregatedGroupConfig config = this.aggregatorGroupConfigs.get(aggregatorType);
if (config == null) {
config = eventAggregationManagementDao.getAggregatedGroupConfig(aggregatorType);
if (config == null) {
config = this.defaultAggregatedGroupConfig;
}
this.aggregatorGroupConfigs.put(aggregatorType, config);
}
return config;
}
}
}