Examples of DbusEventsStatisticsCollector


Examples of com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector

    String category = request.getParams().getProperty(DatabusRequest.PATH_PARAM_NAME);
    String sourceIdStr = category.substring(prefix.length());
    //allow DBNAME/partitionid for REST api
    sourceIdStr = sourceIdStr.replace('/', ':');

    DbusEventsStatisticsCollector s = statsCollectors.getStatsCollector(sourceIdStr);
    
    DbusEventsTotalStats sourceStats = (s==null? null : s.getTotalStats();
    if (null == sourceStats)
    {
      LOG.warn("no stats for this srcId: " + request.getName() + "prefix=" +  prefix + "source ids " + sourceIdStr);
      sourceStats = new DbusEventsTotalStats(0, sourceIdStr, false, false, null);
    }
View Full Code Here

Examples of com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector

  void runAggregateTestStats(int n)
  {
    try
    {
      DbusEventsStatisticsCollector aggregateEventStatsCollectors =
          new AggregatedDbusEventsStatisticsCollector(0, "eventsInbound", true, true, null);

      //collection of n+1 stats collectors;
      StatsCollectors<DbusEventsStatisticsCollector> eventStatsCollectors =
          new StatsCollectors<DbusEventsStatisticsCollector>(aggregateEventStatsCollectors);

      //add new individual stats collectors
      int maxEventsInWindow=10;
      StatsWriter[] nStatsWriters = createStatsWriters(n, maxEventsInWindow);
      for (StatsWriter sw : nStatsWriters)
      {
        eventStatsCollectors.addStatsCollector(sw.getStatsName(), sw.getEventsStatsCollector());
      }

      //aggregator thread; 250 ms poll time
      GlobalStatsCalc agg = new GlobalStatsCalc(10);
      agg.registerStatsCollector(eventStatsCollectors);
      Thread aggThread = new Thread(agg);
      aggThread.start();

      //start writers
      for (StatsWriter sw : nStatsWriters)
      {
        sw.start();
      }

      //Let the writers start
      Thread.sleep(1000);

      long startTimeMs = System.currentTimeMillis();
      long durationInMs = 5*1000; //5s
      DbusEventsTotalStats globalStats = aggregateEventStatsCollectors.getTotalStats();
      long prevValue = 0, prevSize =0;
      while (System.currentTimeMillis() < (startTimeMs+durationInMs))
      {
        //constraint checks;

View Full Code Here

Examples of com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector

    public StatsWriter(int id, int maxWindowSize)
    {
      _maxWindowSize = maxWindowSize;
      _name = "stats_" + id;
      _stats = new DbusEventsStatisticsCollector(OWNERID,_name,true,true,null);
    }
View Full Code Here

Examples of com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector

    try
    {
      try
      {
        String threadName = Thread.currentThread().getName();
        DbusEventsStatisticsCollector threadCollector = _bootstrapServer.getOutBoundStatsCollectors().getStatsCollector(threadName);
        if (null == threadCollector)
        {
            threadCollector = new DbusEventsStatisticsCollector(_bootstrapServer.getContainerStaticConfig().getId(),
                    threadName,
                    true,
                    false,
                    _bootstrapServer.getMbeanServer());
          StatsCollectors<DbusEventsStatisticsCollector> ds = _bootstrapServer.getOutBoundStatsCollectors();
View Full Code Here

Examples of com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector

      //Event writer - Relay in the real world
      Checkpoint cp = Checkpoint.createFlexibleCheckpoint();

      //Event readers - Clients in the real world
      //Checkpoint pullerCheckpoint = Checkpoint.createFlexibleCheckpoint();
      DbusEventsStatisticsCollector clientStats = new DbusEventsStatisticsCollector(0, "client", true, false, null);
      DbusEventBufferReader reader = new DbusEventBufferReader(destEventsBuf, readerStream, null, clientStats);
      UncaughtExceptionTrackingThread tReader = new UncaughtExceptionTrackingThread(reader,"Reader");
      tReader.setDaemon(true);
      tReader.start();
      try
View Full Code Here

Examples of com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector

      //Event writer - Relay in the real world
      Checkpoint cp = Checkpoint.createFlexibleCheckpoint();

      //Event readers - Clients in the real world
      //Checkpoint pullerCheckpoint = Checkpoint.createFlexibleCheckpoint();
      DbusEventsStatisticsCollector clientStats = new DbusEventsStatisticsCollector(0, "client", true, false, null);
      DbusEventBufferReader reader = new DbusEventBufferReader(destEventsBuf, readerStream, null, clientStats);
      UncaughtExceptionTrackingThread tReader = new UncaughtExceptionTrackingThread(reader,"Reader");
      tReader.setDaemon(true);
      tReader.start();

      try
      {
        log.info("send first window -- that one should be OK");
        StreamEventsResult streamRes = srcEventsBuf.streamEvents(cp, writerStream, new StreamEventsArgs(win1Size));
        Assert.assertEquals(numEventsPerWindow + 1, streamRes.getNumEventsStreamed());

        TestUtil.assertWithBackoff(new ConditionCheck()
        {
          @Override
          public boolean check()
          {
            return 1 == callbackStats.getNumSysEventsProcessed();
          }
        }, "first window processed", 5000, log);

        log.info("send the second partial window -- that one should cause an error");
        streamRes = srcEventsBuf.streamEvents(cp, writerStream, new StreamEventsArgs(win2Size));
        Assert.assertEquals(numOfFailureEvent - numEventsPerWindow, streamRes.getNumEventsStreamed());

        log.info("wait for dispatcher to finish");
        TestUtil.assertWithBackoff(new ConditionCheck()
        {
          @Override
          public boolean check()
          {
            log.info("events received: " + callbackStats.getNumDataEventsReceived());
            return numOfFailureEvent <= callbackStats.getNumDataEventsProcessed();
          }
        }, "all events until the error processed", 5000, log);

        log.info("all data events have been received but no EOW");
        Assert.assertEquals(numOfFailureEvent, clientStats.getTotalStats().getNumDataEvents());
        Assert.assertEquals(1, clientStats.getTotalStats().getNumSysEvents());
        //at least one failing event therefore < numOfFailureEvent events can be processed
        Assert.assertTrue(numOfFailureEvent <= callbackStats.getNumDataEventsProcessed());
        //onDataEvent callbacks for e2_1 and e2_2 get cancelled
        Assert.assertEquals(2, callbackStats.getNumDataErrorsProcessed());
        //only one EOW
View Full Code Here

Examples of com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector

        /*DispatcherState dispatchState = curState.getDispatcherState();
          dispatchState.switchToDispatchEvents();
          _dispatcherThread.addNewStateBlocking(dispatchState);*/
        if (debugEnabled) _log.debug("Sending events to buffer");

        DbusEventsStatisticsCollector connCollector = _sourcesConn.getInboundEventsStatsCollector();

        if (curState.isSCNRegress())
        {
          _log.info("SCN Regress requested !! Sending a SCN Regress Message to dispatcher. Curr Ckpt :" + curState.getCheckpoint());

View Full Code Here

Examples of com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector

        relayBuffer.start(0);
        writeEventsToBuffer(relayBuffer, eventInfos, 4);

        //prepare stream response
        Checkpoint cp = Checkpoint.createFlexibleCheckpoint();
        final DbusEventsStatisticsCollector stats =
            new DbusEventsStatisticsCollector(1, "test1", true, false, null);
        ChannelBuffer streamResPrefix =
            NettyTestUtils.streamToChannelBuffer(relayBuffer, cp, 20000, stats);
        final StringBuilder alotOfDeadbeef = new StringBuilder();
        for (int i = 0; i < 1024 * 1024; ++i) {
          alotOfDeadbeef.append("DEADBEEF ");
        }
        ChannelBuffer streamResSuf =
            ChannelBuffers.wrappedBuffer(alotOfDeadbeef.toString().getBytes("UTF-8"));
        final ChannelBuffer streamRes = ChannelBuffers.wrappedBuffer(streamResPrefix, streamResSuf);

        //create client
        _stdClientCfgBuilder.getContainer().setReadTimeoutMs(DEFAULT_READ_TIMEOUT_MS);

        final DatabusHttpClientImpl client = new DatabusHttpClientImpl(_stdClientCfgBuilder.build());

        final TestConsumer consumer = new TestConsumer();
        client.registerDatabusStreamListener(consumer, null, SOURCE1_NAME);

        client.start();
        try
        {
          TestUtil.assertWithBackoff(new ConditionCheck()
          {
            @Override
            public boolean check()
            {
              return client._relayConnections.size() == 1;
            }
          }, "sources connection present", 100, log);

          final DatabusSourcesConnection clientConn = client._relayConnections.get(0);
          TestUtil.assertWithBackoff(new ConditionCheck()
          {
            @Override
            public boolean check()
            {
              return null != clientConn.getRelayPullThread().getLastOpenConnection();
            }
          }, "relay connection present", 100, log);

          final NettyHttpDatabusRelayConnection relayConn =
              (NettyHttpDatabusRelayConnection)clientConn.getRelayPullThread().getLastOpenConnection();
          final NettyHttpDatabusRelayConnectionInspector relayConnInsp =
              new NettyHttpDatabusRelayConnectionInspector(relayConn);

          TestUtil.assertWithBackoff(new ConditionCheck()
          {
            @Override
            public boolean check()
            {
              return null != relayConnInsp.getChannel() && relayConnInsp.getChannel().isConnected();
            }
          }, "client connected", 200, log);

          //figure out the connection to the relay
          Channel clientChannel = relayConnInsp.getChannel();
          InetSocketAddress relayAddr = (InetSocketAddress)clientChannel.getRemoteAddress();
          SocketAddress clientAddr = clientChannel.getLocalAddress();
          int relayPort = relayAddr.getPort();
          log.info("relay selected: " + relayPort);

          SimpleTestServerConnection relay = null;
          for (int i = 0; i < RELAY_PORT.length; ++i)
          {
            if (relayPort == RELAY_PORT[i]) relay = _dummyServer[i];
          }
          assertTrue(null != relay);

          final SocketAddress testClientAddr = clientAddr;
          final SimpleTestServerConnection testRelay = relay;
          TestUtil.assertWithBackoff(new ConditionCheck()
          {
            @Override
            public boolean check()
            {
              return null != testRelay.getChildChannel(testClientAddr);
            }
          }, "relay detects new connection", 1000, log);

          Channel serverChannel = relay.getChildChannel(clientAddr);
          assertTrue(null != serverChannel);
          ChannelPipeline serverPipeline = serverChannel.getPipeline();
          SimpleObjectCaptureHandler objCapture = (SimpleObjectCaptureHandler)serverPipeline.get("3");

          //process the /sources request
          NettyTestUtils.waitForHttpRequest(objCapture, SOURCES_REQUEST_REGEX, 1000);
          objCapture.clear();

          //send back the /sources response
          HttpResponse sourcesResp = new DefaultHttpResponse(HttpVersion.HTTP_1_1,
                                                             HttpResponseStatus.OK);
          sourcesResp.setHeader(HttpHeaders.Names.CONNECTION, HttpHeaders.Values.KEEP_ALIVE);
          sourcesResp.setHeader(HttpHeaders.Names.TRANSFER_ENCODING, HttpHeaders.Values.CHUNKED);
          HttpChunk body =
              new DefaultHttpChunk(ChannelBuffers.wrappedBuffer(("[{\"id\":1,\"name\":\"" +
                                   SOURCE1_NAME + "\"}]").getBytes(Charset.defaultCharset())));
          NettyTestUtils.sendServerResponses(relay, clientAddr, sourcesResp, body);

          //make sure the client processes the response correctly
          TestUtil.assertWithBackoff(new ConditionCheck()
          {
            @Override
            public boolean check()
            {
              String idListString = clientConn.getRelayPullThread()._currentState.getSourcesIdListString();
              return "1".equals(idListString);
            }
          }, "client processes /sources response", 100, log);

          log.debug("process the /register request");
          NettyTestUtils.waitForHttpRequest(objCapture, "/register.*", 1000);
          objCapture.clear();

          log.debug("send back the /register response");
          RegisterResponseEntry entry = new RegisterResponseEntry(1L, (short)1, SOURCE1_SCHEMA_STR);
          String responseStr = NettyTestUtils.generateRegisterResponse(entry);
          body = new DefaultHttpChunk(
              ChannelBuffers.wrappedBuffer(responseStr.getBytes(Charset.defaultCharset())));
          NettyTestUtils.sendServerResponses(relay, clientAddr, sourcesResp, body);

          log.debug("make sure the client processes the response /register correctly");
          TestUtil.assertWithBackoff(new ConditionCheck()
          {
            @Override
            public boolean check()
            {
              DispatcherState dispState = clientConn.getRelayDispatcher().getDispatcherState();
              return null != dispState.getSchemaMap() && 1 == dispState.getSchemaMap().size();
            }
          }, "client processes /register response", 100, log);

          log.debug("process /stream call and return a response with garbled suffix");
          NettyTestUtils.waitForHttpRequest(objCapture, "/stream.*", 1000);
          objCapture.clear();

          final HttpResponse streamResp =
              new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK);
          streamResp.setHeader(HttpHeaders.Names.CONNECTION, HttpHeaders.Values.KEEP_ALIVE);
          streamResp.setHeader(HttpHeaders.Names.TRANSFER_ENCODING, HttpHeaders.Values.CHUNKED);

          //send the response asynchronously in case the client blocks
          final Thread streamRespThread = new Thread(new Runnable()
          {

            @Override
            public void run()
            {
              NettyTestUtils.sendServerResponses(testRelay, testClientAddr, streamResp,
                                                 new DefaultHttpChunk(streamRes),
                                                 60000);
            }
          }, "send /stream resp");
          streamRespThread.setDaemon(true);
          streamRespThread.start();

          log.debug("make sure the client disconnects and recovers from the /stream response");
          TestUtil.assertWithBackoff(new ConditionCheck()
          {
            @Override
            public boolean check()
            {
              return null != relayConnInsp.getChannel() && !relayConnInsp.getChannel().isConnected();
            }
          }, "client disconnected", 30000, log);

          TestUtil.assertWithBackoff(new ConditionCheck()
          {
            @Override
            public boolean check()
            {
              return clientConn.getRelayPullThread().getLastOpenConnection() != relayConn;
            }
          }, "new netty connection", 30000, log);

          //make sure the relay send thread is dead
          streamRespThread.join(100);
          Assert.assertTrue(!streamRespThread.isAlive());

          log.debug("PHASE 2: make sure the client has fully recovered and able to connect to another relay");

          final NettyHttpDatabusRelayConnection newRelayConn =
              (NettyHttpDatabusRelayConnection)clientConn.getRelayPullThread().getLastOpenConnection();
          final NettyHttpDatabusRelayConnectionInspector newRelayConnInsp =
              new NettyHttpDatabusRelayConnectionInspector(newRelayConn);

          TestUtil.assertWithBackoff(new ConditionCheck()
          {
            @Override
            public boolean check()
            {
              return null != newRelayConnInsp.getChannel() && newRelayConnInsp.getChannel().isConnected();
            }
          }, "client connected to new relay", 200, log);

          //figure out the connection to the relay
          clientChannel = newRelayConnInsp.getChannel();
          relayAddr = (InetSocketAddress)clientChannel.getRemoteAddress();
          clientAddr = clientChannel.getLocalAddress();
          relayPort = relayAddr.getPort();
          log.info("new relay selected: " + relayPort);

          relay = null;
          int relayIdx = 0;
          for (; relayIdx < RELAY_PORT.length; ++relayIdx)
          {
            if (relayPort == RELAY_PORT[relayIdx]) relay = _dummyServer[relayIdx];
          }
          assertTrue(null != relay);

          serverChannel = relay.getChildChannel(clientAddr);
          assertTrue(null != serverChannel);
          serverPipeline = serverChannel.getPipeline();
          objCapture = (SimpleObjectCaptureHandler)serverPipeline.get("3");

          //process the /sources request
          NettyTestUtils.waitForHttpRequest(objCapture, SOURCES_REQUEST_REGEX, 1000);
          objCapture.clear();

          //send back the /sources response
          body = new DefaultHttpChunk(ChannelBuffers.wrappedBuffer(("[{\"id\":1,\"name\":\"" +
                                      SOURCE1_NAME + "\"}]").getBytes(Charset.defaultCharset())));
          NettyTestUtils.sendServerResponses(relay, clientAddr, sourcesResp, body);

          //make sure the client processes the response correctly
          TestUtil.assertWithBackoff(new ConditionCheck()
          {
            @Override
            public boolean check()
            {
              String idListString = clientConn.getRelayPullThread()._currentState.getSourcesIdListString();
              return "1".equals(idListString);
            }
          }, "client processes /sources response", 100, log);

          //process the /register request
          NettyTestUtils.waitForHttpRequest(objCapture, "/register.*", 1000);
          objCapture.clear();

          //send back the /register response
          body = new DefaultHttpChunk(
              ChannelBuffers.wrappedBuffer(responseStr.getBytes(Charset.defaultCharset())));
          NettyTestUtils.sendServerResponses(relay, clientAddr, sourcesResp, body);

          //make sure the client processes the response correctly
          TestUtil.assertWithBackoff(new ConditionCheck()
          {
            @Override
            public boolean check()
            {
              DispatcherState dispState = clientConn.getRelayDispatcher().getDispatcherState();
              return null != dispState.getSchemaMap() && 1 == dispState.getSchemaMap().size();
            }
          }, "client processes /register response", 100, log);

          //process /stream call and return a partial window
          Matcher streamMatcher =
              NettyTestUtils.waitForHttpRequest(objCapture, "/stream.*checkPoint=([^&]*)&.*",
                                                1000);
          String cpString = streamMatcher.group(1);
          log.debug("/stream checkpoint: " + cpString);
          objCapture.clear();

          NettyTestUtils.sendServerResponses(relay, clientAddr, streamResp,
                                             new DefaultHttpChunk(streamResPrefix));

          //make sure the client processes the response correctly
          TestUtil.assertWithBackoff(new ConditionCheck()
          {
            @Override
            public boolean check()
            {
              log.debug("lastWrittenScn=" + clientConn.getDataEventsBuffer().lastWrittenScn());
              return clientConn.getDataEventsBuffer().lastWrittenScn() == 20;
            }
          }, "client receives /stream response", 1100, log);


          TestUtil.assertWithBackoff(new ConditionCheck()
          {
            @Override
            public boolean check()
            {
              log.debug("events num=" + consumer.getEventNum());
              return stats.getTotalStats().getNumDataEvents() == consumer.getEventNum();
            }
          }, "client processes /stream response", 11000, log);
        }
        finally {
          client.shutdown();
View Full Code Here

Examples of com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector

        relayBuffer.start(0);
        writeEventsToBuffer(relayBuffer, eventInfos, 4);

        //prepare stream response
        Checkpoint cp = Checkpoint.createFlexibleCheckpoint();
        final DbusEventsStatisticsCollector stats =
            new DbusEventsStatisticsCollector(1, "test1", true, false, null);

        // create ChunnelBuffer and fill it with events from relayBuffer
        ChannelBuffer streamResPrefix =
            NettyTestUtils.streamToChannelBuffer(relayBuffer, cp, 20000, stats);
View Full Code Here

Examples of com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector

        relayBuffer.start(0);
        writeEventsToBuffer(relayBuffer, eventInfos, 4);

        //prepare stream response ??????????????//
        Checkpoint cp = Checkpoint.createFlexibleCheckpoint();
        final DbusEventsStatisticsCollector stats =
            new DbusEventsStatisticsCollector(1, "test1", true, false, null);

        // create ChunnelBuffer and fill it with events from relayBuffer
        ChannelBuffer streamResPrefix =
            NettyTestUtils.streamToChannelBuffer(relayBuffer, cp, 20000, stats);
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.