Package com.linkedin.databus.core.DbusEventBuffer

Examples of com.linkedin.databus.core.DbusEventBuffer.DbusEventIterator

224
225
226
227
228
229
230
231
232
233
234
235
    Checkpoint cp = logReader.read();
    assertTrue(cp.getWindowOffset() == -1);
    assertTrue(cp.getWindowScn() == lastWindowScn);

    // Read and Write Buffers should be the same
    DbusEventIterator writeIter = dbuf.acquireIterator("testIterator");
    DbusEventIterator readIter = readDbuf.acquireIterator("testIterator");
    //LOG.setLevel(Level.DEBUG);
    checkIterEquals(writeIter, readIter);
    dbuf.releaseIterator(writeIter);
    readDbuf.releaseIterator(readIter);
  }
View Full Code Here
253
254
255
256
257
258
259
260
261
262
263
264
    Checkpoint cp = logReader.read();
    assertTrue(cp.getWindowOffset() == -1);
    assertTrue(cp.getWindowScn() == lastWindowScn);

    // Read and Write Buffers should be the same
    DbusEventIterator writeIter = dbuf.acquireIterator("testIterator");
    DbusEventIterator readIter = readDbuf.acquireIterator("testIterator");
    //LOG.setLevel(Level.DEBUG);
    checkIterEquals(writeIter, readIter);
    dbuf.releaseIterator(writeIter);
    readDbuf.releaseIterator(readIter);
  }
View Full Code Here
282
283
284
285
286
287
288
289
290
291
292
293
    Checkpoint cp = logReader.read();
    assertTrue(cp.getWindowOffset() == -1);
    assertTrue(cp.getWindowScn() == lastWindowScn);

    // Read and Write Buffers should be the same
    DbusEventIterator writeIter = dbuf.acquireIterator("testIterator");
    DbusEventIterator readIter = readDbuf.acquireIterator("testIterator");
    // LOG.info("First event = " + readIter.next());
    // LOG.info("Second event = " + readIter.next());
    // LOG.info("Third event = "  + readIter.next());

    checkIterProperSubset(writeIter, readIter);
View Full Code Here
313
314
315
316
317
318
319
320
321
322
323
324
    Checkpoint cp = logReader.read();
    assertTrue(cp.getWindowOffset() == -1);
    assertTrue(cp.getWindowScn() == lastWindowScn);

    // Read and Write Buffers should be the same
    DbusEventIterator writeIter = dbuf.acquireIterator("testIterator");
    DbusEventIterator readIter = readDbuf.acquireIterator("testIterator");
    // LOG.info("First event = " + readIter.next());
    // LOG.info("Second event = " + readIter.next());
    // LOG.info("Third event = "  + readIter.next());

    checkIterProperSubset(writeIter, readIter);
View Full Code Here
344
345
346
347
348
349
350
351
352
353
354
355
    Checkpoint cp = logReader.read();
    assertTrue(cp.getWindowOffset() == -1);
    assertTrue(cp.getWindowScn() == lastWindowScn);

    // Read and Write Buffers should be the same
    DbusEventIterator writeIter = dbuf.acquireIterator("testIterator");
    DbusEventIterator readIter = readDbuf.acquireIterator("testIterator");
    // LOG.info("First event = " + readIter.next());
    // LOG.info("Second event = " + readIter.next());
    // LOG.info("Third event = "  + readIter.next());

    checkIterProperSubset(writeIter, readIter);
View Full Code Here
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
    dbUpdates.add(dbUpdate);

    long timestamp = System.nanoTime();
    gg.addEventToBuffer(dbUpdates, new TransactionInfo(0, 0, timestamp, scn));
    Assert.assertEquals(gg.getRateControl().getNumSleeps(), 0);
    DbusEventIterator iter  = mb.acquireIterator("test");
    int count = 0;
    long eventTs = 0;
    while(iter.hasNext()) {
      DbusEvent e = iter.next();
      if(count==1) { // first event prev control event
        eventTs = e.timestampInNanos();
      }

      count ++;
View Full Code Here
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
                                                                                      eventBuffer1,
                                                                                      null,
                                                                                      false));
    eventBuffer1.endEvents(2);

    DbusEventIterator it1 = eventBuffer1.acquireIterator("it1");
    assertTrue("buffer has event", it1.hasNext());

    DbusEvent testEvent = it1.next();
    assertEquals("key correct", 1L, testEvent.key());
    assertEquals("sequence correct", 2L, testEvent.sequence());
    assertEquals("partitionId correct", 3, testEvent.logicalPartitionId());
    assertEquals("timestamp correct", 4L, testEvent.timestampInNanos());
    assertEquals("srcId correct", 5, testEvent.srcId());
    assertEquals("schemaId correct", new String(schemaId), new String(testEvent.schemaId()));
    assertEquals("value correct", valueStr, Utils.byteBufferToString(testEvent.value()));
    assertEquals("Get DbusEventKey", 1L, ((DbusEventInternalReadable)testEvent).getDbusEventKey().getLongKey().longValue());

    //test JSON
    jsonOut = new ByteArrayOutputStream();
    jsonOutChannel = Channels.newChannel(jsonOut);
    event1.writeTo(jsonOutChannel, Encoding.JSON);

    jsonBytes = jsonOut.toByteArray();
    jsonString = new String(jsonBytes);

    jsonMap = mapper.readValue(jsonString, new TypeReference<Map<String, Object>>(){});
    assertEquals("key correct", 1L, ((Number)jsonMap.get("key")).longValue());
    assertEquals("sequence correct", 2L, ((Number)jsonMap.get("sequence")).longValue());
    assertEquals("logicalPartitionId correct", 3, ((Number)jsonMap.get("logicalPartitionId")).shortValue());
    assertEquals("timestampInNanos correct", 4L, ((Number)jsonMap.get("timestampInNanos")).longValue());
    assertEquals("srcId correct", 5, ((Number)jsonMap.get("srcId")).longValue());
    assertEquals("schemaId correct", Base64.encodeBytes(schemaId), jsonMap.get("schemaId"));
    assertEquals("valueEnc correct", Encoding.JSON.toString(), jsonMap.get("valueEnc"));
    assertEquals("value correct", Base64.encodeBytes(valueStr.getBytes(Charset.defaultCharset())), jsonMap.get("value"));

    assertTrue("buffer has event", it1.hasNext());
    testEvent = it1.next();
    assertTrue("end of window", testEvent.isEndOfPeriodMarker());

    DbusEventBuffer eventBuffer2 =
        new DbusEventBuffer(getConfig(100000, DbusEventBuffer.Config.DEFAULT_INDIVIDUAL_BUFFER_SIZE, 10000, 1000,
                                      AllocationPolicy.HEAP_MEMORY, QueuePolicy.OVERWRITE_ON_WRITE));
    eventBuffer2.startEvents();
    assertTrue("json deserialization", (DbusEventSerializable.appendToEventBuffer(jsonString, eventBuffer2, null, false) > 0));
    eventBuffer2.endEvents(2);

    DbusEventIterator it2 = eventBuffer2.acquireIterator("it2");
    assertTrue("buffer has event", it2.hasNext());

    testEvent = it2.next();
    assertEquals("key correct", 1L, testEvent.key());
    assertEquals("partitionId correct", 3, testEvent.logicalPartitionId());
    assertEquals("timestamp correct", 4L, testEvent.timestampInNanos());
    assertEquals("srcId correct", 5, testEvent.srcId());
    assertEquals("schemaId correct", new String(schemaId), new String(testEvent.schemaId()));
    assertEquals("value correct", valueStr, Utils.byteBufferToString(testEvent.value()));
    assertEquals("Get DbusEventKey", 1L, ((DbusEventInternalReadable)testEvent).getDbusEventKey().getLongKey().longValue());

    assertTrue("buffer has event", it2.hasNext());
    testEvent = it2.next();
    assertTrue("end of window", testEvent.isEndOfPeriodMarker());
  }
View Full Code Here
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
                                      AllocationPolicy.HEAP_MEMORY, QueuePolicy.OVERWRITE_ON_WRITE));
    eventBuffer1.startEvents();
    assertEquals("jsonDeserialization", 10, DbusEventSerializable.appendToEventBuffer(jsonIn, eventBuffer1, null, false));
    eventBuffer1.endEvents(2);

    DbusEventIterator eventIter = eventBuffer1.acquireIterator("it1");
    for (int i = 0; i < 10; ++i)
    {
      String valueStr = "eventValue" + i;
      assertTrue("buffer has event " + i, eventIter.hasNext());
      DbusEvent testEvent = eventIter.next();

      assertEquals("key correct for event " + i, 1L + i, testEvent.key());
      assertEquals("partitionId correct for event " + i, 3 + i, testEvent.logicalPartitionId());
      assertEquals("timestamp correct for event " + i, 4L + i, testEvent.timestampInNanos());
      assertEquals("srcId correct for event " + i, 5 + i, testEvent.srcId());
      assertEquals("schemaId correct for event " + i, new String(schemaId), new String(testEvent.schemaId()));
      assertEquals("value correct for event " + i, valueStr, Utils.byteBufferToString(testEvent.value()));
    }

    assertTrue("buffer has event", eventIter.hasNext());
    DbusEvent testEvent = eventIter.next();
    assertTrue("end of window", testEvent.isEndOfPeriodMarker());
  }
View Full Code Here
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
    DbusEventBuffer dbusBuf = new DbusEventBuffer(conf);

    pushEventsToBuffer(dbusBuf, numEvents);

    // verify events are in the buffer
    DbusEventIterator it = dbusBuf.acquireIterator("allevents");
    int count=-1; // first event is "prev" event
    while(it.hasNext()) {
      DbusEvent e = it.next();
      Assert.assertTrue(e.isValid());
      count ++;
    }
    dbusBuf.releaseIterator(it);
    Assert.assertEquals(count, numEvents);

    // save meta data
    dbusBuf.saveBufferMetaInfo(false);

    // create another similar buffer and see if has the events
    dbusBuf = new DbusEventBuffer(conf);
    dbusBuf.validateEventsInBuffer();
    it = dbusBuf.acquireIterator("alleventsNew");
    count=-1; // first event is "prev" event
    while(it.hasNext()) {
      DbusEvent e = it.next();
      Assert.assertTrue(e.isValid());
      count ++;
    }
    Assert.assertEquals(count, numEvents);
    dbusBuf.releaseIterator(it);

    // meta file should be gone by now
    File metaFile = new File(_mmapDir, dbusBuf.metaFileName());
    Assert.assertFalse(metaFile.exists());

    // we can start a buffer and would be empty
    dbusBuf = new DbusEventBuffer(conf);
    it = dbusBuf.acquireIterator("alleventsNewEmpty");
    Assert.assertFalse(it.hasNext());
    dbusBuf.releaseIterator(it);
  }
View Full Code Here
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
    try
    {
      int totalEvents = 0;
      long  allTotalEvents = 0;
      DbusEventIterator iDbusEvent = _buffer.acquireIterator("Test_DbusEventBufferConsumer");
      do {
        if (!iDbusEvent.hasNext()) {
          if (!_invalidEvent) {
            if (_deletionInterval>0) {
              iDbusEvent.remove();
            }
            try
            {
              iDbusEvent.await();
            }
            catch (InterruptedException e)
            {
              // TODO Auto-generated catch block
              e.printStackTrace();
              return;
            }
          } else {
            //try and consume as many events as possible
            if (allTotalEvents >= eventsReadTillInvalidEvent()) {
              LOG.info("total events read until invalid event=" + allTotalEvents + "; terminating");
              stop();
            }
          }
        }

        while (iDbusEvent.hasNext()) {
          DbusEventInternalReadable e = iDbusEvent.next();
          ++allTotalEvents;
          if (!e.isCheckpointMessage() && !e.isControlMessage() && !e.isEndOfPeriodMarker()) {
            //needs to be idempotent; so - ensure that duplicates are dropped;
            if (!_seenKeys.contains(e.key())) {
              //deep copy
              _out.add(e.createCopy());
              _seenKeys.add(e.key());
              ++totalEvents;
            }
          }
          if ((_deletionInterval>0) && allTotalEvents % _deletionInterval==0) {
            iDbusEvent.remove();
          }
        }
      }
      while (totalEvents < _maxEvents && !_stop);
      iDbusEvent.remove();
    }
    catch (RuntimeException e)
    {
      _exceptionThrown = e;
      LOG.error("consumer exception:" + e.getMessage(), e);
View Full Code Here
TOP

Related Classes of com.linkedin.databus.core.DbusEventBuffer.DbusEventIterator

  • com.linkedin.databus.core.EventLogReader
  • com.linkedin.databus.core.test.DbusEventBufferConsumer
  • com.linkedin.databus.core.TestDbusEvent
  • com.linkedin.databus.core.TestDbusEventBuffer
  • com.linkedin.databus.core.TestDbusEventBufferPersistence
  • com.linkedin.databus.core.TestEventLogWriter
  • com.linkedin.databus2.relay.TestGoldenGateEventProducer
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.