@Test
public void testConfirmDFOChain() throws FlumeSpecException, IOException,
InterruptedException {
// create sources
String c1 = "rpcSource(1236)";
ThriftEventSource c1Src = (ThriftEventSource) FlumeBuilder.buildSource(
LogicalNodeContext.testingContext()
, c1);
c1Src.open();
String c2 = "rpcSource(1237)";
ThriftEventSource c2Src = (ThriftEventSource) FlumeBuilder.buildSource(
LogicalNodeContext.testingContext(), c2);
c2Src.open();
// create agentDFOChain sink
File tmpDir = FileUtil.mktempdir();
String spec = "agentDFOChain(\"localhost:1236\", \"localhost:1237\")";
CompositeSink snk = new CompositeSink(new LogicalNodeContext(
tmpDir.getName(), tmpDir.getName()), spec);
snk.open();
Event e1 = new EventImpl("test 1".getBytes());
Event e2 = new EventImpl("test 2".getBytes());
Event e3 = new EventImpl("test 3".getBytes());
Event e4 = new EventImpl("test 4".getBytes());
// Everything is on and we send some messages.
snk.append(e1);
Clock.sleep(100);
LOG.info("DFO failchain all on metrics {}", c1Src.getMetrics().toString());
assertEquals(1,
(long) c1Src.getMetrics().getLongMetric(ThriftEventSource.A_ENQUEUED));
// it got through, yay.
c1Src.next();
c1Src.close();
// Killed the first of the chain, should go to backup
// the number of events lost here is not consistent after close. this
// seems time based, and the first two seem to be lost
snk.append(e1);
Clock.sleep(20);
snk.append(e2);
Clock.sleep(20);
snk.append(e3);
Clock.sleep(20);
snk.append(e4);
Clock.sleep(20);
LOG.info("DFO failchain kill primary, show 2ndary metrics {}", c2Src
.getMetrics().toString());
long enqueued = (long) c2Src.getMetrics().getLongMetric(
ThriftEventSource.A_ENQUEUED);
assertTrue(
"Expected at least 2 events still enqueueed but got " + enqueued,
2 <= enqueued);
// 2 lost in network buffer, but two received in backup. yay.
c2Src.next();
c2Src.next();
c2Src.close();
e1 = new EventImpl("test 1 b".getBytes());
e2 = new EventImpl("test 2 b".getBytes());
e3 = new EventImpl("test 3 b".getBytes());
e4 = new EventImpl("test 4 b".getBytes());
// all thrift sinks are closed now, we should end up in dfo
snk.append(e1); // lost in thrift sink buffer
Clock.sleep(20);
snk.append(e2); // lost in thrift sink buffer
Clock.sleep(20);
snk.append(e3); // written
Clock.sleep(20);
snk.append(e4); // written
Clock.sleep(20);
LOG.info("DFO Primary returns metrics: {}", snk.getMetrics().toText());
long writingEvts = (long) ReportUtil.getFlattenedReport(snk).getLongMetric(
"backup.DiskFailover.NaiveDiskFailover.writingEvts");
assertTrue("Expected at least 2 writing evetns but had " + writingEvts,
2 <= writingEvts);
e1 = new EventImpl("test 1 c".getBytes());
e2 = new EventImpl("test 2 c".getBytes());
e3 = new EventImpl("test 3 c".getBytes());
e4 = new EventImpl("test 4 c".getBytes());
// re-open destination 1
c1Src.open();
snk.append(e1);
Clock.sleep(20);
snk.append(e2);
Clock.sleep(20);
c1Src.next();
// get handle to roller in dfo log roller to provide data
AgentFailChainSink afcs = (AgentFailChainSink) snk.getSink();
BackOffFailOverSink bofos = (BackOffFailOverSink) ((CompositeSink) afcs.snk)
.getSink();
DiskFailoverDeco dfo = (DiskFailoverDeco) bofos.getBackup();
RollSink dfoWriter = dfo.getDFOWriter();
dfoWriter.rotate(); // allow dfo retry thread to go.
// give data some time to show up.
Clock.sleep(1000);
c1Src.next();
c1Src.next();
c1Src.next();
c1Src.next();
c1Src.close();
ReportEvent rpt = ReportUtil.getFlattenedReport(snk);
LOG.info("DFO Failchain final report {}", rpt.toString());
String written = "backup.DiskFailover.NaiveDiskFailover.writingEvts";
long writtenEvents = (long) rpt.getLongMetric(written);
assertTrue("Exepected at least 4 written events but had " + writtenEvents,
4 <= writtenEvents);
// yay. all four events written to dfo log
String primary = "backup.DiskFailover."
+ "LazyOpenDecorator.InsistentAppend.StubbornAppend."
+ "InsistentOpen.FailoverChainSink.sentPrimary";
long primaryEvts = (long) rpt.getLongMetric(primary);
assertTrue("Expected at least 4 primary events but had " + primaryEvts,
4 <= primaryEvts);
// yay all four go through to the path we wanted. (the primary after the
// disk failover)
// data from DFO log was sent.
long c1Enq = (long) c1Src.getMetrics().getLongMetric(
ThriftEventSource.A_ENQUEUED);
assertTrue("Expected at least 2+4 c1 equeued but had " + c1Enq,
2 + 4 <= c1Enq);
// first one fails on reopen but next succeeds
long c2Enq = (long) c2Src.getMetrics().getLongMetric(
ThriftEventSource.A_ENQUEUED);
assertTrue("Expected at least 2+1 c2 equeued but had " + c2Enq,
2 + 1 <= c2Enq);
}