waitForRecoveriesToFinish(false);
// load avro records via morphline and zk into solr
morphline = parse("test-morphlines/tutorialReadAvroContainer");
Record record = new Record();
byte[] body = Files.toByteArray(file);
record.put(Fields.ATTACHMENT_BODY, body);
startSession();
Notifications.notifyBeginTransaction(morphline);
assertTrue(morphline.process(record));
assertEquals(1, collector.getNumStartEvents());
commit();
// fetch sorted result set from solr
QueryResponse rsp = cloudClient.query(new SolrQuery("*:*").setRows(100000).addSort("id", SolrQuery.ORDER.asc));
assertEquals(2104, collector.getRecords().size());
assertEquals(collector.getRecords().size(), rsp.getResults().size());
Collections.sort(collector.getRecords(), new Comparator<Record>() {
@Override
public int compare(Record r1, Record r2) {
return r1.get("id").toString().compareTo(r2.get("id").toString());
}
});
// fetch test input data and sort like solr result set
List<GenericData.Record> records = new ArrayList();
FileReader<GenericData.Record> reader = new DataFileReader(file, new GenericDatumReader());
while (reader.hasNext()) {
GenericData.Record expected = reader.next();
records.add(expected);
}
assertEquals(collector.getRecords().size(), records.size());
Collections.sort(records, new Comparator<GenericData.Record>() {
@Override
public int compare(GenericData.Record r1, GenericData.Record r2) {
return r1.get("id").toString().compareTo(r2.get("id").toString());
}
});
Object lastId = null;
for (int i = 0; i < records.size(); i++) {
//System.out.println("myrec" + i + ":" + records.get(i));
Object id = records.get(i);
if (id != null && id.equals(lastId)) {
throw new IllegalStateException("Detected duplicate id. Test input data must not contain duplicate ids!");
}
lastId = id;
}
for (int i = 0; i < records.size(); i++) {
//System.out.println("myrsp" + i + ":" + rsp.getResults().get(i));
}
Iterator<SolrDocument> rspIter = rsp.getResults().iterator();
for (int i = 0; i < records.size(); i++) {
// verify morphline spat out expected data
Record actual = collector.getRecords().get(i);
GenericData.Record expected = records.get(i);
Preconditions.checkNotNull(expected);
assertTweetEquals(expected, actual, i);
// verify Solr result set contains expected data
actual = new Record();
actual.getFields().putAll(next(rspIter));
assertTweetEquals(expected, actual, i);
}
Notifications.notifyRollbackTransaction(morphline);
Notifications.notifyShutdown(morphline);