Examples of com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector

Package com.linkedin.databus.core.monitoring.mbean

Examples of com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector

com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector
Container for all monitoring mbeans

    // Dest Event consumer
    Vector<DbusEvent> dstTestEvents = new Vector<DbusEvent>();


    // emitter/producer/appender -> DEB -> writer -> pipe -> reader -> DEB -> consumer
    // Test configurations;
    DbusEventsStatisticsCollector emitterStats = new DbusEventsStatisticsCollector(1,"appenderStats",true,true,null);
    DbusEventsStatisticsCollector streamStats = new DbusEventsStatisticsCollector(1,"streamStats",true,true,null);
    DbusEventsStatisticsCollector clientStats = new DbusEventsStatisticsCollector(1,"clientStats",true,true,null);


    srcTestEvents.clear();
    dstTestEvents.clear();
    emitterStats.reset();  // a.k.a. producer, a.k.a. appender
    streamStats.reset();   // a.k.a. writer (== reader from relay buffer)
    clientStats.reset();   // a.k.a. reader (== writer into client buffer)


    assertEquals(0, dstTestEvents.size());


    assertTrue(runConstEventsReaderWriter(srcTestEvents,dstTestEvents,testInput,emitterStats,streamStats,clientStats));


    int numEvents = testInput.getNumEvents();
    // check data!
    assertEquals("numEvents generated", numEvents, srcTestEvents.size());
    if (!enableAsserts && numEvents != dstTestEvents.size())
    {
      LOG.info(testInput.getTestName() + " would fail if asserts were enabled:  numEvents seen by consumer = " +
               dstTestEvents.size() + " < " + numEvents);
      return false;
    }
    assertEquals(testInput.getTestName() + ": numEvents seen by consumer", numEvents, dstTestEvents.size());


    assertEquals(testInput.getTestName() + ": numDataEvents seen by producer",
                 srcTestEvents.size(),
                 emitterStats.getTotalStats().getNumDataEvents());
    assertEquals(testInput.getTestName() + ": numDataEvents seen by writer",
                 dstTestEvents.size(),
                 streamStats.getTotalStats().getNumDataEvents());
    assertEquals(testInput.getTestName() + ": numDataEvents seen by producer",
                 dstTestEvents.size(),
                 clientStats.getTotalStats().getNumDataEvents());
    assertTrue(testInput.getTestName() + ": numSysEvents seen by producer",
               emitterStats.getTotalStats().getNumSysEvents() != 0);
    assertEquals(testInput.getTestName() + ": numSysEvents seen by writer",
                 emitterStats.getTotalStats().getNumSysEvents(),
                 streamStats.getTotalStats().getNumSysEvents());
    assertEquals(testInput.getTestName() + ": numSysEvents seen by reader",
                 streamStats.getTotalStats().getNumSysEvents(),
                 clientStats.getTotalStats().getNumSysEvents());
    // check integrity of each event received
    for (int i=0; i < numEvents; ++i)
    {
      DbusEvent src = srcTestEvents.get(i);
      DbusEvent dst = dstTestEvents.get(i);

View Full Code Here

    // Dest Event consumer
    Vector<DbusEvent> dstTestEvents = new Vector<DbusEvent>();
    // num events
    int numEvents = 100;


    DbusEventsStatisticsCollector emitterStats =
        new DbusEventsStatisticsCollector(1,"appenderStats", true, true, null);
    DbusEventsStatisticsCollector streamStats =
        new DbusEventsStatisticsCollector(1, "streamStats", true, true, null);
    DbusEventsStatisticsCollector clientStats =
        new DbusEventsStatisticsCollector(1, "clientStats", true, true, null);




    int[][] listOfCorruptedIndices = new int[][] { {81}, {12}, {10,65}, {0}, {99} };
    EventCorruptionType[] corruptionType =
        new EventCorruptionType[] { EventCorruptionType.LENGTH,
                                    EventCorruptionType.PAYLOADCRC,
                                    EventCorruptionType.HEADERCRC,
                                    EventCorruptionType.PAYLOAD };
    int totalInvalid=0;
    for (int i=0; i < listOfCorruptedIndices.length; ++i)
    {
      for (int j=0; j < corruptionType.length; ++j)
      {
        log.info("starting test: corruptionType=" + corruptionType[j] + " corruptIndex=" + i);
        EventBufferTestInput invalidEventInput = new EventBufferTestInput();
        invalidEventInput.setNumEvents(numEvents)
                         .setWindowSize(numEvents/5)
                         .setProducerBufferSize(numEvents*2)
                         .setSharedBufferSize(numEvents*2)
                         .setStagingBufferSize(numEvents*2)
                         .setIndexSize(numEvents/10)
                         .setIndividualBufferSize(numEvents*2)
                         .setCorruptionType(corruptionType[j])
                         .setCorruptIndexList(listOfCorruptedIndices[i])
                         .setTestName("testEventReadBufferInvalidEvents");


        srcTestEvents.clear();
        dstTestEvents.clear();
        emitterStats.reset();
        streamStats.reset();
        clientStats.reset();


        boolean result = runConstEventsReaderWriter(srcTestEvents, dstTestEvents,
                                                    invalidEventInput, emitterStats,
                                                    streamStats, clientStats);


        assertTrue(result);
        //check data!
        assertEquals(numEvents, srcTestEvents.size());
        assertEquals(srcTestEvents.size(), emitterStats.getTotalStats().getNumDataEvents());
        totalInvalid += clientStats.getTotalStats().getNumInvalidEvents();
        //expect no data to be received;
        log.info(String.format("Read %d events until an invalid event was discovered with %s corruption \n",
                               dstTestEvents.size(),
                               corruptionType[j]));
        assertTrue(dstTestEvents.size() < numEvents);

View Full Code Here

         .setProducerBufferSize(numEvents/2)
         .setPayloadSize(100)
         .setDeleteInterval(1)
         .setProdQueuePolicy(QueuePolicy.OVERWRITE_ON_WRITE);
    input.setTestName("testStatsMinMaxScn");
    DbusEventsStatisticsCollector emitterStats = new DbusEventsStatisticsCollector(1,"appenderStats",true,true,null);


    DbusEventGenerator evGen = new DbusEventGenerator(startScn);
    if (evGen.generateEvents(numEvents,input.getWindowSize(),512,input.getPayloadSize(),true,srcTestEvents) <= 0)
    {
      fail();
      return;
    }
    //sleep 10 ms;


    int eventSize = srcTestEvents.get(0).size();
    DbusEventBuffer prodEventBuffer =
        new DbusEventBuffer(getConfig(input.getProducerBufferSize()*eventSize,
                                      input.getIndividualBufferSize()*eventSize,
                                      input.getIndexSize()*eventSize,
                                      input.getStagingBufferSize()*eventSize,
                                      AllocationPolicy.HEAP_MEMORY,
                                      input.getProdQueuePolicy(),
                                      input.getProdBufferAssertLevel()));
    DbusEventAppender eventProducer = new DbusEventAppender(srcTestEvents, prodEventBuffer,emitterStats);
    Thread tEmitter = new Thread(eventProducer);
    tEmitter.start();
    tEmitter.join();


    //sleep 10 ms;
    int msDelay = 10;
    Thread.sleep(msDelay);


    long min = (numWinScn-3)*input.getWindowSize() + startScn;
    long max=numWinScn*input.getWindowSize()+ startScn;
    //note : event generator generates events such that a one second lag exists between the latest event and prev event
    long expectedRange = (max-min)  + input.getWindowSize()-1;
    System.out.printf("Total timespan = %d\n",
        (srcTestEvents.get(numEvents-1).timestampInNanos() - srcTestEvents.get(0).timestampInNanos())/NANOSECONDS);
    System.out.printf("prevScn=%d\n", emitterStats.getTotalStats().getPrevScn());
    System.out.printf("min = %d , max=%d  buf=%d ,%d\n", emitterStats.getTotalStats().getMinScn(),
                      emitterStats.getTotalStats().getMaxScn(), prodEventBuffer.getMinScn(),
                      prodEventBuffer.lastWrittenScn());
    System.out.printf("timespan=%d , timeSinceLastEvent = %d , timeSinceLastAccess %d\n",
                      emitterStats.getTotalStats().getTimeSpan()/MILLISECONDS,
                      emitterStats.getTotalStats().getTimeSinceLastEvent(),
                      emitterStats.getTotalStats().getTimeSinceLastAccess());


    assertEquals(numEvents, srcTestEvents.size());
    assertEquals(numEvents, emitterStats.getTotalStats().getNumDataEvents());
    assertEquals(min, emitterStats.getTotalStats().getMinScn());
    assertEquals(max, emitterStats.getTotalStats().getMaxScn());
    assertEquals(min-input.getWindowSize(), emitterStats.getTotalStats().getPrevScn());
    assertEquals(emitterStats.getTotalStats().getSizeDataEvents()*numEvents, numEvents*eventSize);
    long tsSpanInSec = emitterStats.getTotalStats().getTimeSpan()/MILLISECONDS;
    assertEquals(expectedRange, tsSpanInSec);
    long tsSinceLastEvent = emitterStats.getTotalStats().getTimeSinceLastEvent();
    assertTrue(tsSinceLastEvent >= msDelay);
    assertTrue(emitterStats.getTotalStats().getTimeLag() >= 0);
    assertTrue(emitterStats.getTotalStats().getMinTimeLag() >= 0);
    assertTrue(emitterStats.getTotalStats().getMaxTimeLag() >= 0);


    DbusEventBuffer readEventBuffer =
        new DbusEventBuffer(getConfig(numEvents*2L*eventSize, numEvents*eventSize, (numEvents/10)*eventSize,
                                      numEvents*2*eventSize, AllocationPolicy.HEAP_MEMORY,
                                      input.getProdQueuePolicy(), input.getProdBufferAssertLevel()));

View Full Code Here

         .setProducerBufferSize(numEvents*2)
         .setPayloadSize(100)
         .setDeleteInterval(1)
         .setProdQueuePolicy(QueuePolicy.OVERWRITE_ON_WRITE);
    input.setTestName("testStreamScn");
    DbusEventsStatisticsCollector emitterStats = new DbusEventsStatisticsCollector(1,"appenderStats",true,true,null);
    DbusEventsStatisticsCollector clientStats = new DbusEventsStatisticsCollector(1,"clientStats",true,true,null);


    DbusEventGenerator evGen = new DbusEventGenerator();
    assertTrue(evGen.generateEvents(numEvents,input.getWindowSize(),512,input.getPayloadSize(),true,srcTestEvents) > 0);
    int eventSize = srcTestEvents.get(0).size();
    DbusEventBuffer prodEventBuffer =
        new DbusEventBuffer(getConfig(input.getProducerBufferSize()*eventSize,
                                      input.getIndividualBufferSize()*eventSize,
                                      input.getIndexSize()*eventSize,
                                      input.getStagingBufferSize()*eventSize,
                                      AllocationPolicy.HEAP_MEMORY, input.getProdQueuePolicy(),
                                      input.getProdBufferAssertLevel()));
    DbusEventBuffer readEventBuffer =
        new DbusEventBuffer(getConfig(input.getProducerBufferSize()*eventSize,
                                      input.getIndividualBufferSize()*eventSize, input.getIndexSize()*eventSize,
                                      input.getStagingBufferSize()*eventSize,
                                      AllocationPolicy.HEAP_MEMORY, input.getProdQueuePolicy(),
                                      input.getProdBufferAssertLevel()));
    DbusEventAppender eventProducer = new DbusEventAppender(srcTestEvents, prodEventBuffer,emitterStats);


    Vector<Long> seenScns = new Vector<Long>();
    Checkpoint cp = new Checkpoint();
    cp.setConsumptionMode(DbusClientMode.ONLINE_CONSUMPTION);
    boolean origEmptyValue  = prodEventBuffer.empty();


    //empty buffer; prevScn=-1 , minScn=-1 ; so no Scn not found exception
    cp.setWindowScn(2L);
    cp.setWindowOffset(-1);
    seenScns.clear();
    readEventBuffer.clear();
    boolean res = streamWriterReader(prodEventBuffer,input.getBatchSize()*eventSize,cp,"scn",readEventBuffer,seenScns);
    assertEquals(-1L, prodEventBuffer.getPrevScn());
    assertTrue(res);
    assertEquals(0, seenScns.size());


    //partial buffer; with no complete window written;  prevScn > sinceScn , minScn=-1 ; Scn not found exception thrown;
    cp.setWindowScn(2L);
    cp.setWindowOffset(-1);
    prodEventBuffer.setPrevScn(20L);
    prodEventBuffer.setEmpty(false);
    seenScns.clear();
    readEventBuffer.clear();
    assertEquals(-1L, prodEventBuffer.getMinScn());
    res = streamWriterReader(prodEventBuffer,input.getBatchSize()*eventSize,cp,"scn",readEventBuffer,seenScns);
    assertFalse(res);
    assertEquals(0, seenScns.size());
    //restore
    prodEventBuffer.setPrevScn(-1L);
    prodEventBuffer.setEmpty(origEmptyValue);


    //partial buffer; with no complete window written;  sinceScn >= prevScn , minScn=-1 ; no exception;
    cp.setWindowScn(45L);
    cp.setWindowOffset(-1);
    prodEventBuffer.setPrevScn(20L);
    prodEventBuffer.setEmpty(false);
    seenScns.clear();
    readEventBuffer.clear();
    res = streamWriterReader(prodEventBuffer,input.getBatchSize()*eventSize,cp,"scn",readEventBuffer,seenScns);
    assertTrue(res);
    assertEquals(0, seenScns.size());
    //restore
    prodEventBuffer.setPrevScn(-1L);
    prodEventBuffer.setEmpty(origEmptyValue);


    Thread tEmitter = new Thread(eventProducer);
    tEmitter.start();
    tEmitter.join();


    long minScn = emitterStats.getTotalStats().getMinScn();
    long maxScn = emitterStats.getTotalStats().getMaxScn();
    long prevScn = emitterStats.getTotalStats().getPrevScn();


    System.out.printf("minScn=%d,maxScn=%d,prevScn=%d,range=%d\n",minScn,maxScn,prevScn,emitterStats.getTotalStats().getTimeSpan());
    assertEquals(numEvents - 1, emitterStats.getTotalStats().getTimeSpan()/MILLISECONDS);
    assertEquals(prodEventBuffer.getTimestampOfFirstEvent(), emitterStats.getTotalStats().getTimestampMinScnEvent());


    //stream with scn < max; expect last window; not last 2
    cp.setWindowScn(maxScn-1);
    cp.setWindowOffset(-1);
    seenScns.clear();
    readEventBuffer.clear();
    streamWriterReader(prodEventBuffer,input.getBatchSize()*eventSize,cp,"scn",readEventBuffer,seenScns);
    assertEquals(1, seenScns.size());
    assertEquals(Long.valueOf(maxScn), seenScns.get(0));


    //set windowScn to maxScn; get >= behaviour here ; get the last window
    cp.setWindowScn(maxScn);
    cp.setWindowOffset(0);
    seenScns.clear();
    readEventBuffer.clear();
    streamWriterReader(prodEventBuffer,input.getBatchSize()*eventSize,cp,"scn",readEventBuffer,seenScns);
    assertEquals(1, seenScns.size());
    assertEquals(Long.valueOf(maxScn), seenScns.get(0));


    //stream with scn >= max ; get a window higher than max - expect nothing
    cp.setWindowScn(maxScn);
    cp.setWindowOffset(-1);
    readEventBuffer.clear();
    seenScns.clear();
    streamWriterReader(prodEventBuffer,input.getBatchSize()*eventSize,cp,"scn",readEventBuffer,seenScns);
    assertEquals(0, seenScns.size());


    //stream with scn > max
    cp.setWindowScn(maxScn+1);
    cp.setWindowOffset(-1);
    readEventBuffer.clear();
    seenScns.clear();
    streamWriterReader(prodEventBuffer,input.getBatchSize()*eventSize,cp,"scn",readEventBuffer,seenScns);
    assertEquals(0, seenScns.size());


    //stream with scn >= min
    cp.setWindowScn(minScn);
    cp.setWindowOffset(-1);
    readEventBuffer.clear();
    seenScns.clear();
    streamWriterReader(prodEventBuffer,input.getBatchSize()*eventSize,cp,"scn",readEventBuffer,seenScns);
    assertEquals(numScns-1, seenScns.size());
    assertTrue(seenScns.get(0)!=minScn);
    assertEquals(Long.valueOf(maxScn), seenScns.get(numScns-2));


    //stream with scn < min but >= prevScn
    cp.setWindowScn(prevScn);
    cp.setWindowOffset(-1);
    readEventBuffer.clear();
    seenScns.clear();
    streamWriterReader(prodEventBuffer,input.getBatchSize()*eventSize,cp,"scn",readEventBuffer,seenScns,clientStats);
    assertEquals(numScns, seenScns.size());
    assertEquals(Long.valueOf(minScn), seenScns.get(0));
    System.out.printf("Clientstats: minScn=%d , maxScn=%d , timespan=%d timeSinceLastEvent=%d\n",
                      clientStats.getTotalStats().getMinScn(),
                      clientStats.getTotalStats().getMaxScn(),
                      clientStats.getTotalStats().getTimeSpan(),
                      clientStats.getTotalStats().getTimeSinceLastEvent());
    assertEquals(maxScn, clientStats.getTotalStats().getMaxScn());
    assertEquals(numEvents - 1, clientStats.getTotalStats().getTimeSpan()/MILLISECONDS);
    assertEquals(clientStats.getTotalStats().getTimestampMaxScnEvent(),
                 emitterStats.getTotalStats().getTimestampMaxScnEvent());
    assertEquals(minScn, clientStats.getTotalStats().getMinScn());


    //stream with scn < prevScn
    cp.setWindowScn(prevScn-1);
    cp.setWindowOffset(-1);
    readEventBuffer.clear();

View Full Code Here

      .setBatchSize(bufferSize)
      .setProducerBufferSize(bufferSize)
      .setPayloadSize(10)
      .setDeleteInterval(1)
      .setProdQueuePolicy(QueuePolicy.OVERWRITE_ON_WRITE);
      DbusEventsStatisticsCollector emitterStats = new DbusEventsStatisticsCollector(1,"appenderStats",true,true,null);


      if (srcTestEvents.isEmpty())
      {
        assertTrue(evGen.generateEvents(numEvents,input.getWindowSize(),512,input.getPayloadSize(),false,srcTestEvents) > 0);
      }
      int eventSize= 0;
      for (DbusEvent e: srcTestEvents)
      {
        if (!e.isControlMessage())
        {
          eventSize = e.size();
          break;
        }
      }
      LOG.info("event size="+ eventSize);
      int indexSize = (input.getIndexSize()*eventSize/100);
      LOG.info("indexSize=" + indexSize);
      DbusEventBuffer prodEventBuffer =
          new DbusEventBuffer(getConfig(input.getProducerBufferSize()*eventSize+i,
                                        input.getIndividualBufferSize()*eventSize,
                                        indexSize, input.getStagingBufferSize()*eventSize,
                                        AllocationPolicy.HEAP_MEMORY, input.getProdQueuePolicy(),
                                        input.getProdBufferAssertLevel()));
      LOG.info("Allocated buffer size=" + prodEventBuffer.getAllocatedSize());
      DbusEventAppender eventProducer = new DbusEventAppender(srcTestEvents, prodEventBuffer,emitterStats,0.600);
      Thread t = new Thread(eventProducer);
      t.start();
      t.join();
      DbusEventsTotalStats stats = emitterStats.getTotalStats();
      LOG.info("minScn=" + prodEventBuffer.getMinScn() + "totalEvents=" + stats.getNumDataEvents());
      //weak assertion ! just check it's not -1 for now
      //TODO! get this assertion to succeed
      //assertTrue(prodEventBuffer.getMinScn()!=-1);
    }

View Full Code Here

    confBuilder.setScnIndexSize(80);
    confBuilder.setAverageEventSize(80);


    DbusEventBuffer buf = new DbusEventBuffer(confBuilder);


    DbusEventsStatisticsCollector statsCollector = new DbusEventsStatisticsCollector(1, "in", true, false, null);
    //generate some data in the buffer
    buf.start(0);
    final int W = 2;
    final int E = 2;
    for (int w = 1; w <= W; ++w)
    {
      buf.startEvents();
      int seq = w * 100;
      for (int i = 0; i < E; ++i)
      {
        buf.appendEvent(new DbusEventKey(i), (short)1, (short)1, System.nanoTime(), (short)1,
                        new byte[16], new byte[1], false, statsCollector);
      }
      buf.endEvents(seq, statsCollector);
    }


    //sanity check for the data
    Assert.assertEquals(W, statsCollector.getTotalStats().getNumSysEvents());
    Assert.assertEquals(W * E, statsCollector.getTotalStats().getNumDataEvents());


    File tmpFile= File.createTempFile(TestDbusEventBuffer.class.getSimpleName(), ".tmp");
    tmpFile.deleteOnExit();
    OutputStream outStream = new FileOutputStream(tmpFile);
    WritableByteChannel outChannel = Channels.newChannel(outStream);


    //stream events
    DbusEventsStatisticsCollector outStatsCollector = new DbusEventsStatisticsCollector(1, "out", true, false, null);
    Checkpoint cp = new Checkpoint();
    cp.setFlexible();
    cp.setConsumptionMode(DbusClientMode.ONLINE_CONSUMPTION);
    ErrorCountingAppender errorCountAppender = new ErrorCountingAppender();
    errorCountAppender.setName("errorCountAppender");

View Full Code Here

0 1 2 3

TOP

Related Classes of com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector

com.linkedin.databus.bootstrap.server.BootstrapRequestProcessor

com.linkedin.databus.client.DatabusHttpClientImpl

com.linkedin.databus.client.registration.DatabusV2RegistrationImpl

com.linkedin.databus.client.RelayPullThread

com.linkedin.databus.client.TestDatabusHttpClient

com.linkedin.databus.client.TestGenericDispatcher

com.linkedin.databus.container.netty.HttpRelay

com.linkedin.databus.container.request.GenerateDataEventsRequestProcessor

com.linkedin.databus.container.request.LoadDataEventsRequestProcessor

com.linkedin.databus.container.request.RelayContainerStatsRequestProcessor

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.