long timeStart = timestamp;
long lastTime = timestamp;
long commitInterval = _config.getCommitInterval();
long totLatency = 0;
GenericRecord record = null;
RateMonitor seedingRate = new RateMonitor("Seeding Rate");
seedingRate.start();
seedingRate.suspend();
long startRowId = _lastRows.get(sourceInfo.getEventView());
LOG.info("Last Known Row Id is :" + startRowId);
boolean resumeSeedingRate = true;
for (File avroSeedFile : files)
{
if (! avroSeedFile.isFile())
continue;
LOG.info("Seeding from File : " + avroSeedFile);
try {
reader = new DataFileReader<GenericRecord>(avroSeedFile, new GenericDatumReader<GenericRecord>());
} catch (IOException e) {
LOG.fatal("Failed to bootstrap from file " + avroSeedFile.getAbsolutePath(), e);
throw new RuntimeException("Failed to bootstrap from file " + avroSeedFile.getAbsolutePath(), e);
}
try
{
boolean committed = false;
for (GenericRecord hdfsRecord : reader)
{
record = hdfsRecord;
committed = false;
numRead++;
if (numRead < startRowId)
continue;
if (resumeSeedingRate)
{
seedingRate.resume();
resumeSeedingRate = false;
}
seedingRate.tick();
//LOG.info("Read record :" + record);
long start = System.nanoTime();
long eventSize = sourceInfo.getFactory().createAndAppendEvent(windowSCN, timestamp, hdfsRecord,
_bootstrapEventBuffer, false, null);
numBytes+=eventSize;
long latency = System.nanoTime() - start;
totLatency += latency;
if (numRead%commitInterval == 0)
{
_bootstrapEventBuffer.endEvents(numRead,timestamp,null);
_bootstrapEventBuffer.startEvents();
long procTime = totLatency/1000000000;
long currTime = System.currentTimeMillis();
long diff = (currTime - lastTime)/1000;
long timeSinceStart = (currTime - timeStart)/1000;
LOG.info("Processed " + commitInterval + " rows in " + diff
+ " seconds, Avro Processing Time (seconds) so far :" + (procTime)
+ ",Seconds elapsed since start :" + (timeSinceStart)
+ ",Overall Row Rate:" + seedingRate.getRate() +
", NumRows Fetched so far:" + numRead +
". TotalEventSize :" + numBytes);
lastTime = currTime;
seedingRate.resume();
committed = true;
}
}
if ( ! committed)
{
_bootstrapEventBuffer.endEvents(numRead,timestamp,null);
_bootstrapEventBuffer.startEvents();
long procTime = totLatency/1000000000;
long currTime = System.currentTimeMillis();
long diff = (currTime - lastTime)/1000;
long timeSinceStart = (currTime - timeStart)/1000;
LOG.info("Completed Seeding from : " + avroSeedFile + ", Processed " + commitInterval + " rows in " + diff
+ " seconds, Avro Processing Time (seconds) so far :" + (procTime)
+ ",Seconds elapsed since start :" + (timeSinceStart)
+ ",Overall Row Rate:" + seedingRate.getRate() +
", NumRows Fetched so far:" + numRead +
". TotalEventSize :" + numBytes);
lastTime = currTime;
seedingRate.resume();
}
} catch (Exception e) {
LOG.fatal("NumRead :" + numRead + ", Got Exception while processing generic record :" + record, e);
throw new RuntimeException(e);
}