Examples of com.linkedin.databus.core.test.DbusEventAppender

com.linkedin.databus.core.test.DbusEventAppender
@author snagarajHelper class to test concurrent appends into EventBuffer Creates random window sizes in the list of events available

    DbusEventGenerator generator = new DbusEventGenerator();
    Vector<DbusEvent> events = new Vector<DbusEvent>();
    generator.generateEvents(7, 1, 120, 39, events);


    // Add events to the EventBuffer. Now the buffer is full
    DbusEventAppender appender = new DbusEventAppender(events,dbusBuf,null);
    appender.run(); // running in the same thread
    log.info("Head:" + parser.toString(dbusBuf.getHead()) + ", Tail:" + parser.toString(dbusBuf.getTail()));
    log.info("Num buffers: " + dbusBuf.getBuffer().length);
    log.info("Buffer: " + Arrays.toString(dbusBuf.getBuffer()));


    long headPos = dbusBuf.getHead();
    long tailPos = dbusBuf.getTail();
    long headGenId = parser.bufferGenId(headPos);
    long headIndexId = parser.bufferIndex(headPos);
    long headOffset = parser.bufferOffset(headPos);
    long tailGenId = parser.bufferGenId(tailPos);
    long tailIndexId = parser.bufferIndex(tailPos);
    long tailOffset = parser.bufferOffset(tailPos);


    assertEquals(0, headGenId);
    assertEquals(0, headIndexId);
    assertEquals(222, headOffset);
    assertEquals(1, tailGenId);
    assertEquals(0, tailIndexId);
    assertEquals(61, tailOffset);


    log.info("append windows with one small and one big event");
    generator = new DbusEventGenerator(100);
    events = new Vector<DbusEvent>();
    generator.generateEvents(1, 1, 280, 139, events);
    generator.generateEvents(1, 1, 480, 339, events);


    // Add events to the EventBuffer. Now the buffer is full
    appender = new DbusEventAppender(events,dbusBuf,null);
    appender.run(); // running in the same thread
    log.info("Head:" + parser.toString(dbusBuf.getHead()) + ", Tail:" + parser.toString(dbusBuf.getTail()));
    log.info("Num buffers: " + dbusBuf.getBuffer().length);
    log.info("Buffer: " + Arrays.toString(dbusBuf.getBuffer()));


    headPos = dbusBuf.getHead();

View Full Code Here

    DbusEventGenerator generator = new DbusEventGenerator();
    Vector<DbusEvent> events = new Vector<DbusEvent>();
    generator.generateEvents(11, 1, 120, 39, events);


    // Add events to the EventBuffer. Now the buffer is full
    DbusEventAppender appender = new DbusEventAppender(events,dbusBuf,null);
    appender.run(); // running in the same thread
    log.info("Head:" + parser.toString(dbusBuf.getHead()) + ",Tail:" + parser.toString(dbusBuf.getTail()));
    log.info("Num buffers :" + dbusBuf.getBuffer().length);
    log.info("Buffer :" + Arrays.toString(dbusBuf.getBuffer()));


    long headPos = dbusBuf.getHead();
    long tailPos = dbusBuf.getTail();
    long headGenId = parser.bufferGenId(headPos);
    long headIndexId = parser.bufferIndex(headPos);
    long headOffset = parser.bufferOffset(headPos);
    long tailGenId = parser.bufferGenId(tailPos);
    long tailIndexId = parser.bufferIndex(tailPos);
    long tailOffset = parser.bufferOffset(tailPos);


    // current writing position should be 222 (id=1)
    assertEquals(0, headGenId);
    assertEquals(1, headIndexId);
    assertEquals(383, headOffset);
    assertEquals(1, tailGenId);
    assertEquals(1, tailIndexId);
    assertEquals(222, tailOffset);




    log.info("append event to stretch beyond limit but less than capacity");
    generator = new DbusEventGenerator(100);
    events = new Vector<DbusEvent>();
    generator.generateEvents(1, 1, 400, 150, events); // will add two event 61 + 150


    // Add events. this will cause limit increased
    appender = new DbusEventAppender(events,dbusBuf,null);
    appender.run();
    log.info("Head:" + parser.toString(dbusBuf.getHead()) + ",Tail:" + parser.toString(dbusBuf.getTail()));
    log.info("Num buffers :" + dbusBuf.getBuffer().length);
    log.info("Buffer :" + Arrays.toString(dbusBuf.getBuffer()));

View Full Code Here

    DbusEventGenerator generator = new DbusEventGenerator();
    Vector<DbusEvent> events = new Vector<DbusEvent>();
    generator.generateEvents(70000, 1, 120, 39, events);


    // Add events to the EventBuffer. Now the buffer is full
    DbusEventAppender appender = new DbusEventAppender(events, dbusBuf, null);
    appender.run(); // running in the same thread
    genComplete.set(true);


    streamThread.join(10000);


    Assert.assertFalse(streamThread.isAlive());

View Full Code Here

                                      input.getIndexSize()*eventSize,
                                      input.getStagingBufferSize()*eventSize,
                                      AllocationPolicy.HEAP_MEMORY,
                                      input.getProdQueuePolicy(),
                                      input.getProdBufferAssertLevel()));
    DbusEventAppender eventProducer = new DbusEventAppender(srcTestEvents, prodEventBuffer,emitterStats);
    Thread tEmitter = new Thread(eventProducer);
    tEmitter.start();
    tEmitter.join();


    //sleep 10 ms;

View Full Code Here

        new DbusEventBuffer(getConfig(input.getProducerBufferSize()*eventSize,
                                      input.getIndividualBufferSize()*eventSize, input.getIndexSize()*eventSize,
                                      input.getStagingBufferSize()*eventSize,
                                      AllocationPolicy.HEAP_MEMORY, input.getProdQueuePolicy(),
                                      input.getProdBufferAssertLevel()));
    DbusEventAppender eventProducer = new DbusEventAppender(srcTestEvents, prodEventBuffer,emitterStats);


    Vector<Long> seenScns = new Vector<Long>();
    Checkpoint cp = new Checkpoint();
    cp.setConsumptionMode(DbusClientMode.ONLINE_CONSUMPTION);
    boolean origEmptyValue  = prodEventBuffer.empty();

View Full Code Here

                                        input.getIndividualBufferSize()*eventSize,
                                        indexSize, input.getStagingBufferSize()*eventSize,
                                        AllocationPolicy.HEAP_MEMORY, input.getProdQueuePolicy(),
                                        input.getProdBufferAssertLevel()));
      LOG.info("Allocated buffer size=" + prodEventBuffer.getAllocatedSize());
      DbusEventAppender eventProducer = new DbusEventAppender(srcTestEvents, prodEventBuffer,emitterStats,0.600);
      Thread t = new Thread(eventProducer);
      t.start();
      t.join();
      DbusEventsTotalStats stats = emitterStats.getTotalStats();
      LOG.info("minScn=" + prodEventBuffer.getMinScn() + "totalEvents=" + stats.getNumDataEvents());

View Full Code Here

                                      stagingBufferSize, AllocationPolicy.HEAP_MEMORY,
                                      consQueuePolicy,
                                      input.getConsBufferAssertLevel()));


    //Producer of events, a.k.a. "emitter"
    DbusEventAppender eventProducer = new DbusEventAppender(srcTestEvents,
                                                            prodEventBuffer,
                                                            emitterStats,
                                                            autoStartBuffer);


    //commn channels between reader and writer
    Pipe pipe = Pipe.open();
    Pipe.SinkChannel writerStream = pipe.sink();
    Pipe.SourceChannel readerStream = pipe.source();
    writerStream.configureBlocking(true);
    readerStream.configureBlocking(false);


    //Event writer - Relay in the real world
    int batchSize = input.getBatchSize() * eventSize;
    DbusEventBufferWriter writer = new DbusEventBufferWriter(prodEventBuffer, writerStream,
                                                             batchSize, streamStats);


    //Event readers - Clients in the real world
    DbusEventBufferConsumer consumer = new DbusEventBufferConsumer(consEventBuffer, numEvents,
                                                                   input.getDeleteInterval(),
                                                                   dstTestEvents);
    Vector<EventBufferConsumer> consList = new Vector<EventBufferConsumer>();
    consList.add(consumer);
    //Event readers - Clients in the real world
    DbusEventBufferReader reader = new DbusEventBufferReader(consEventBuffer, readerStream,
                                                             consList, clientStats);


    UncaughtExceptionTrackingThread tEmitter = new UncaughtExceptionTrackingThread(eventProducer,"EventProducer");
    UncaughtExceptionTrackingThread tWriter = new UncaughtExceptionTrackingThread(writer,"Writer");
    UncaughtExceptionTrackingThread tReader = new UncaughtExceptionTrackingThread(reader,"Reader");
    UncaughtExceptionTrackingThread tConsumer = new UncaughtExceptionTrackingThread(consumer,"Consumer");


    long emitterWaitms = 20000;
    long writerWaitms = 10000;
    long readerWaitms = 10000;
    long consumerWaitms = readerWaitms;


    //start emitter;
    tEmitter.start();


    //tarnish events written to buffer;
    int [] corruptIndexList = input.getCorruptIndexList();
    if (corruptIndexList.length > 0)
    {
      tEmitter.join(emitterWaitms);
      EventCorruptionType corruptionType = input.getCorruptionType();
      eventProducer.tarnishEventsInBuffer(corruptIndexList, corruptionType);
    }


    //start  consumer / reader /writer
    tConsumer.start();
    tWriter.start();
    tReader.start();


    //wait until all events have been written;
    dumpEmitterWriterReaderConsumerState(eventProducer, writer, reader, consumer, emitterStats, streamStats, clientStats, dstTestEvents, prodEventBuffer, consEventBuffer);
    LOG.info("runConstEventsReaderWriter(): waiting up to " + (emitterWaitms/1000) + " sec for appender/producer/emitter thread to terminate");
    tEmitter.join(emitterWaitms);
    //try and set a finish for writer
    long eventsEmitted = eventProducer.eventsEmitted();
    writer.setExpectedEvents(eventsEmitted);


    //wait for writer to finish;
    dumpEmitterWriterReaderConsumerState(eventProducer, writer, reader, consumer, emitterStats, streamStats, clientStats, dstTestEvents, prodEventBuffer, consEventBuffer);
    LOG.info("runConstEventsReaderWriter(): waiting up to " + (writerWaitms/1000) + " sec for writer thread to terminate");

View Full Code Here

    generator.generateEvents(6, 2, 120, 38, events);


    log.debug(dbusBuf.toShortString());
    dbusBuf.assertBuffersLimits();


    DbusEventAppender appender = new DbusEventAppender(events, dbusBuf, null, 1.0, false, -1);
    appender.run();


    log.info("verify new iterator");
    DbusEventIterator iter1 =
        dbusBuf.acquireIterator("testInternalIteratorSingleBufFull");


    log.debug("it1=" + iter1);
    Assert.assertEquals(iter1.getCurrentPosition(), dbusBuf.getHead());
    Assert.assertEquals(iter1._iteratorTail.getPosition(), dbusBuf.getTail());
    Assert.assertEquals(dbusBuf._busyIteratorPool.size(), 1);
    Assert.assertTrue(iter1.hasNext());
    DbusEvent e = iter1.next();
    Assert.assertTrue(e.isEndOfPeriodMarker());
    Assert.assertTrue(iter1.hasNext());
    dbusBuf.assertBuffersLimits();


    log.info("make sure we can read some events");
    readAndCompareIteratorEvents(iter1, events, 0, 6, true, true, true);
    log.debug("after read: " + dbusBuf.toShortString());
    log.debug(iter1);


    log.info("append more windows");
    final Vector<DbusEvent> events2 = new Vector<DbusEvent>();
    generator = new DbusEventGenerator(200);
    generator.generateEvents(2, 1, 120, 39, events2);
    appender = new DbusEventAppender(events2, dbusBuf, null, 1.0, false, -1);
    appender.run();
    log.debug("after 2 more events added: " + dbusBuf.toShortString());
    log.debug(iter1);


    readAndCompareIteratorEvents(iter1, events2, 0, 2, true, false, true);
    log.debug("after 2 more events read: " + dbusBuf.toShortString());
    log.debug(iter1);
    dbusBuf.assertBuffersLimits();


    // create another iterator - make sure it can read too
    DbusEventIterator iter2 = dbusBuf.acquireIterator("testInternalIteratorSingleBufFull2");


    long iCWP = iter2.getCurrentPosition();
    long head = dbusBuf.getBufferPositionParser().sanitize(dbusBuf.getHead(), dbusBuf.getBuffer());
    Assert.assertEquals(iCWP, head);
    Assert.assertEquals(iter2._iteratorTail.getPosition(), dbusBuf.getTail());
    Assert.assertEquals(dbusBuf._busyIteratorPool.size(), 2);
    Assert.assertTrue(iter2.hasNext());


    log.debug("iter2=" + iter2);
    readAndCompareIteratorEvents(iter2, events2, 0, 2, true, false, true); // read same events and don't remove
    dbusBuf.releaseIterator(iter2);
    dbusBuf.assertBuffersLimits();


    log.debug("iter1=" + iter1);
    iter1.remove();


    log.debug("buf (after read)=" + dbusBuf);


    generator = new DbusEventGenerator(300);
    final Vector<DbusEvent> events3 = new Vector<DbusEvent>();
    generator.generateEvents(4, 2, 120, 39, events3);
    appender = new DbusEventAppender(events3, dbusBuf, null, 1.0, false, -1);
    appender.run();
    dbusBuf.assertBuffersLimits();


    log.info("make sure we can read remainder of events");
    readAndCompareIteratorEvents(iter1, events3, 0, 4, false, true, true);

View Full Code Here


    // Generate one event that equals the size of the first buffer. We won't be able to
    // append that to the buffer.
    Vector<DbusEvent> events = new Vector<DbusEvent>();
    generator.generateEvents(1, 2, 500, 439, events);
    DbusEventAppender appender = new DbusEventAppender(events,dbusBuf,null, false);
    int eventCount = 0;
    boolean exceptionCaught = false;


    dbusBuf.startEvents();
    try
    {
      eventCount = appender.addEventToBuffer(events.get(0), eventCount);
    }
    catch(DatabusRuntimeException e)
    {
      exceptionCaught = true;
    }
    Assert.assertTrue(exceptionCaught);
    Assert.assertEquals(0, eventCount);
    DbusEventBufferReflector reflector = appender.getDbusEventReflector();
    Assert.assertEquals(0, reflector.getBuffer(0).limit());
    Assert.assertEquals(0, reflector.getBuffer(1).limit());
    Assert.assertEquals(0, reflector.getBuffer(2).limit());


    events.clear();
    dbusBuf =
        new DbusEventBuffer(getConfig(1144, 500, 100, 500, AllocationPolicy.HEAP_MEMORY,
                                      QueuePolicy.OVERWRITE_ON_WRITE, AssertLevel.ALL));
    appender = new DbusEventAppender(events,dbusBuf,null, false);
    generator.generateEvents(1, 20, 500, 438, events);  // total event size = 499
    generator.generateEvents(1, 20, 500, 10, events);   // total event size = 71
    generator.generateEvents(1, 20, 500, 367, events);   // total event size = 428
    generator.generateEvents(1, 1, 500, 10, events);   // event + EOP = 132.


    // We should be able to append the first three events above, filling the two byte buffers completely.
    // And then the last event along with EOP marker in the last byte buffer, filling that to complete
    // one window.
    appender.run();


    // Now try to add one event. Because ScnIndex still has head as -1 for this buffer,
    // we end up getting a DatabusRuntimeException and not being able to add an event.
    // Since this is a rare case, it is TBD whether we need to support this case or not.
    events.clear();
    generator.generateEvents(1, 20, 500, 10, events);   // event size = 71, can append.
    exceptionCaught = false;
    try
    {
      appender.run();
    }
    catch(DatabusRuntimeException e)
    {
      exceptionCaught = true;
    }

View Full Code Here

    // We have 7 byte buffers now, the last one having 200 bytes, others with 500 bytes.
    DbusEventBuffer dbusBuf = new DbusEventBuffer(getConfig(3200, 500, 100, 500, AllocationPolicy.HEAP_MEMORY,
                                      QueuePolicy.OVERWRITE_ON_WRITE, AssertLevel.ALL));
    DbusEventGenerator generator = new DbusEventGenerator();
    Vector<DbusEvent> events = new Vector<DbusEvent>();
    DbusEventAppender appender = new DbusEventAppender(events,dbusBuf,null, false);
    events.clear();


    // Same test case as before, with the first 3 byte buffers having data now.
    generator.generateEvents(1, 20, 500, 438, events);  // total event size = 499
    generator.generateEvents(1, 20, 500, 10, events);   // total event size = 71
    generator.generateEvents(1, 20, 500, 367, events);   // total event size = 428
    generator.generateEvents(1, 1, 500, 10, events);   // event + EOP = 132.
    appender.run();
    DbusEventBufferReflector reflector = appender.getDbusEventReflector();


    // Now we should be able to add an event and roll it back as well.
    events.clear();
    generator.generateEvents(1, 20, 500, 10, events);   // event size = 71, can append.


    long cwp1 = reflector.getCurrentWritePosition().getPosition();
    dbusBuf.startEvents();
    int evCount = 0;
    evCount = appender.addEventToBuffer(events.firstElement(), evCount);
    Assert.assertEquals(1, evCount);
    long cwp2 = reflector.getCurrentWritePosition().getPosition();
    Assert.assertTrue("Cwp should differ event write", cwp1 != cwp2);


    // Roll back.

View Full Code Here

0 1 2

TOP

Related Classes of com.linkedin.databus.core.test.DbusEventAppender

com.linkedin.databus.client.TestGenericDispatcher

com.linkedin.databus.core.Checkpoint

com.linkedin.databus.core.DbusEventInternalReadable

com.linkedin.databus.core.DbusEventInternalWritable

com.linkedin.databus.core.DbusEventKey

com.linkedin.databus.core.DbusEventV2Factory

com.linkedin.databus.core.ReadEventsTestParams

com.linkedin.databus.core.TestDbusEventBuffer

com.linkedin.databus.core.TestDbusEventBufferAppendEvents

com.linkedin.databus.core.TestDbusEventBufferIterators

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.