Callable<Void> producerThread = new Callable<Void>() {
@Override
public Void call() throws Exception {
try {
for (int i = 0; i < numEdits; i++) {
FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
// Set an increasing transaction id to verify correctness
op.setTransactionId(i);
if (LOG.isDebugEnabled()) {
LOG.debug("Writing " + op);
}
FSEditLogTestUtil.writeToStreams(op, out);
if (i % 50 == 0) {
Thread.sleep(100);
FSEditLogTestUtil.flushStreams(out);
}
}
FSEditLogTestUtil.flushStreams(out);
FSEditLogTestUtil.closeStreams(out);
} finally {
// Let the producer know that we've reached the end.
finishedProducing.set(true);
}
return null;
}
};
Callable<Void> consumerThread = new Callable<Void>() {
@Override
public Void call() throws Exception {
List<EditLogInputStream> streams = Lists.newArrayList();
qjm.selectInputStreams(streams, 0, true, false);
EditLogInputStream in = streams.get(0);
long numOps = 0;
long maxTxId = -1;
FSEditLogOp op;
long lastPos = in.getPosition();
do {
op = in.readOp();
if (op == null) { // If we've reached the end prematurely...
Thread.sleep(200);
LOG.info("Refreshing to " + lastPos);
in.refresh(lastPos, maxTxId); // Then refresh to last known good position
} else {
long txId = op.getTransactionId();
if (txId > maxTxId) {
// Standby ingest contains similar logic: transactions
// with ids lower than what is already read are ignored.
numOps++;
maxTxId = txId;
}
// Remember the last known safe position that we can refresh to
lastPos = in.getPosition();
}
} while (op != null || !finishedProducing.get());
Thread.sleep(1000);
// finalize the segment, so we can read to the end
qjm.finalizeLogSegment(0, numEdits - 1);
// Once producer is shutdown, scan again from last known good position
// until the end of the ledger. This mirrors the Ingest logic (last
// read when being quiesced).
in.refresh(lastPos, maxTxId);
do {
op = in.readOp();
if (op != null) {
long txId = op.getTransactionId();
if (txId > maxTxId) {
numOps++;
maxTxId = txId;
}
}