package com.linkedin.databus2.core.container;
/*
*
* Copyright 2013 LinkedIn Corp. All rights reserved
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*
*/
import com.linkedin.databus.core.DbusEvent;
import com.linkedin.databus.core.DbusEventInternalReadable;
import com.linkedin.databus.core.DbusEventInternalWritable;
import com.linkedin.databus.core.InvalidEventException;
import com.linkedin.databus.core.monitoring.mbean.AggregatedDbusEventsStatisticsCollector;
import com.linkedin.databus.core.monitoring.mbean.AggregatedDbusEventsTotalStats;
import com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector;
import com.linkedin.databus.core.monitoring.mbean.DbusEventsTotalStats;
import com.linkedin.databus.core.monitoring.mbean.StatsCollectors;
import com.linkedin.databus.core.test.DbusEventCorrupter;
import com.linkedin.databus.core.test.DbusEventGenerator;
import com.linkedin.databus.core.util.RngUtils;
import com.linkedin.databus2.core.container.netty.ServerContainer.GlobalStatsCalc;
import com.linkedin.databus2.test.TestUtil;
import java.util.Vector;
import junit.framework.Assert;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.testng.annotations.Test;
public class TestAggregatedDbusEventStatsCollector
{
public final static String MODULE = TestAggregatedDbusEventStatsCollector.class.getName();
public final static Logger LOG = Logger.getLogger(MODULE);
static
{
TestUtil.setupLogging(true, "/tmp/TestAggregatedDbusEventStatsCollector.log", Level.ERROR);
}
@Test
public void testAggregatedStats()
{
//aggregate stats collectors
int n[] = {2,10,100,1000};
for (int i : n)
{
runAggregateTestStats(i);
}
}
void runAggregateTestStats(int n)
{
try
{
DbusEventsStatisticsCollector aggregateEventStatsCollectors =
new AggregatedDbusEventsStatisticsCollector(0, "eventsInbound", true, true, null);
//collection of n+1 stats collectors;
StatsCollectors<DbusEventsStatisticsCollector> eventStatsCollectors =
new StatsCollectors<DbusEventsStatisticsCollector>(aggregateEventStatsCollectors);
//add new individual stats collectors
int maxEventsInWindow=10;
StatsWriter[] nStatsWriters = createStatsWriters(n, maxEventsInWindow);
for (StatsWriter sw : nStatsWriters)
{
eventStatsCollectors.addStatsCollector(sw.getStatsName(), sw.getEventsStatsCollector());
}
//aggregator thread; 250 ms poll time
GlobalStatsCalc agg = new GlobalStatsCalc(10);
agg.registerStatsCollector(eventStatsCollectors);
Thread aggThread = new Thread(agg);
aggThread.start();
//start writers
for (StatsWriter sw : nStatsWriters)
{
sw.start();
}
//Let the writers start
Thread.sleep(1000);
long startTimeMs = System.currentTimeMillis();
long durationInMs = 5*1000; //5s
DbusEventsTotalStats globalStats = aggregateEventStatsCollectors.getTotalStats();
long prevValue = 0, prevSize =0;
while (System.currentTimeMillis() < (startTimeMs+durationInMs))
{
//constraint checks;
//check that readers don't have partial updates or get initialized
long value = globalStats.getNumDataEvents();
long size = globalStats.getSizeDataEvents();
Assert.assertTrue(value > 0);
if (prevValue > 0 && (value != prevValue))
{
Assert.assertTrue(size != prevSize);
prevValue = value;
prevSize = size;
}
Assert.assertTrue(globalStats.getMaxSeenWinScn() > 0);
Thread.sleep(RngUtils.randomPositiveInt()%10+1);
}
//shut down
for (StatsWriter sw : nStatsWriters)
{
sw.shutdown();
sw.interrupt();
}
//Give a chance to catch up
Thread.sleep(1000);
agg.halt();
aggThread.interrupt();
//final tally aggregatedEventTotalStats = sum of all individual statsWriter objects in a thread free way
AggregatedDbusEventsTotalStats myTotalStats = new AggregatedDbusEventsTotalStats(StatsWriter.OWNERID, "mytotal", true, false, null);
for (DbusEventsStatisticsCollector s:eventStatsCollectors.getStatsCollectors())
{
DbusEventsTotalStats writerStat = s.getTotalStats();
//obviously - we assume this is correct here. we want to check that the updates happen correctly in a concurrent setting
myTotalStats.mergeStats(writerStat);
}
LOG.info("global = " + globalStats.getNumDataEvents() + " Sigma writers=" + myTotalStats.getNumDataEvents());
Assert.assertEquals("NumDataEvents mismatch for n = " + n,
globalStats.getNumDataEvents(),
myTotalStats.getNumDataEvents());
Assert.assertEquals("MaxSeenWinScn mismatch for n = " + n,
globalStats.getMaxSeenWinScn(),
myTotalStats.getMaxSeenWinScn());
}
catch (InterruptedException e)
{
Assert.assertTrue(false);
}
}
StatsWriter[] createStatsWriters(int n, int maxWindowSize)
{
StatsWriter[] sw = new StatsWriter[n];
for (int i=0; i < n; ++i)
{
sw[i] = new StatsWriter(i+1, maxWindowSize);
}
return sw;
}
/**
* Create a writer to a stats collector object it owns.
*
* @author snagaraj
*/
class StatsWriter extends Thread
{
private final DbusEventsStatisticsCollector _stats;
private final int _maxWindowSize ;
private final String _name;
private volatile boolean _shutdown = false;
public static final int OWNERID=-1;
public StatsWriter(int id, int maxWindowSize)
{
_maxWindowSize = maxWindowSize;
_name = "stats_" + id;
_stats = new DbusEventsStatisticsCollector(OWNERID,_name,true,true,null);
}
public DbusEventsStatisticsCollector getEventsStatsCollector()
{
return _stats;
}
public void shutdown()
{
_shutdown = true;
}
public String getStatsName()
{
return _name;
}
@Override
public void run()
{
int numEvents = _maxWindowSize*10;
Vector<DbusEvent> events = new Vector<DbusEvent>(numEvents);
long startScn = RngUtils.randomPositiveLong()%10000;
long prevScn = startScn-1;
while (!_shutdown)
{
int maxEventSize = 100;
int payloadSize = 5;
events.clear();
DbusEventGenerator eventGen = new DbusEventGenerator(startScn+1);
long newScn = eventGen.generateEvents(numEvents, _maxWindowSize, maxEventSize, payloadSize, true, events);
DbusEventInternalWritable p = null;
for (DbusEvent e : events)
{
if (p != null && (p.sequence() != e.sequence()))
{
//control event for prev sequence;
p.setSrcId((short)-1);
_stats.registerDataEvent(p);
}
_stats.registerDataEvent((DbusEventInternalReadable)e);
try
{
p = DbusEventCorrupter.makeWritable(e); // for testing only!
}
catch (InvalidEventException iee)
{
LOG.fatal("makeWritable() should not have thrown exception: " + iee);
}
}
if (p != null)
{
p.setSrcId((short)-1);
_stats.registerDataEvent(p);
}
_stats.registerBufferMetrics(startScn+1, newScn , prevScn, RngUtils.randomPositiveInt());
startScn = newScn;
if (!_shutdown)
{
try
{
Thread.sleep(RngUtils.randomPositiveInt()%50+10);
}
catch (InterruptedException e1)
{
// ignore
}
}
}
}
}
}