runner.update("delete from report_observation where reportId = ?", report.getId());
for(String id : bills.keySet()) {
//logger.info("checking bill "+id);
String billNo = id+"-2013";
Bill jsonBill = (Bill)storage.get("2013/bill/"+billNo, Bill.class);
if (jsonBill == null) {
logger.error("Missing bill "+"2013/bill/"+billNo);
continue;
}
if (!jsonBill.isPublished()) {
logger.error("Bill Unpublished: "+billNo);
continue;
}
// Compare the titles, ignore white space differences
String jsonTitle = unescapeHTML(jsonBill.getTitle());
String lbdcTitle = bills.get(id).getTitle();
if (!lbdcTitle.isEmpty() && !stringEquals(jsonTitle, lbdcTitle, true, true)) {
// What is this D?
if (!id.startsWith("D")) {
logger.error("Title: "+billNo);
logger.error(" LBDC: "+lbdcTitle);
logger.error(" JSON: "+jsonTitle);
observations.add(new ReportObservation(report.getId(), billNo, "BILL_TITLE", lbdcTitle, jsonTitle));
errorTotals.put("title", errorTotals.get("title")+1);
}
}
// Compare the summaries. LBDC reports summary and law changes together
String jsonLaw = jsonBill.getLaw();
String jsonSummary = unescapeHTML(jsonBill.getSummary());
String lbdcSummary = bills.get(id).getSummary().replaceAll("\\s+", " ");
if( jsonLaw != null && jsonLaw != "" && jsonLaw != "null") {
jsonSummary = unescapeHTML(jsonLaw)+" "+jsonSummary;
}
if (lbdcSummary.equals("BILL SUMMARY NOT FOUND")) {
lbdcSummary = "";
}
jsonSummary = jsonSummary.replace('§', 'S').replace('¶', 'P');
if (!lbdcSummary.isEmpty() && !jsonSummary.replace(" ","").equals(lbdcSummary.replace(" ", "")) ) {
if (!id.startsWith("D")) {
logger.error("Summary: "+billNo);
logger.error(" LBDC: "+lbdcSummary);
logger.error(" JSON: "+jsonSummary);
observations.add(new ReportObservation(report.getId(), billNo, "BILL_SUMMARY", lbdcSummary, jsonSummary));
errorTotals.put("summary", errorTotals.get("summary")+1);
}
}
String jsonSponsor = "";
if (jsonBill.getSponsor() != null) {
jsonSponsor = unescapeHTML(jsonBill.getSponsor().getFullname()).toUpperCase().replace(" (MS)","").replace("BILL", "").replace("COM", "");
}
String lbdcSponsor = bills.get(id).getSponsor().toUpperCase().replace("BILL", "").replace("COM", "");
if (lbdcSponsor.startsWith("RULES ") && lbdcSponsor.contains("(") && !lbdcSponsor.contains(")")){
lbdcSponsor = lbdcSponsor.concat(")");
}
if (!lbdcSponsor.isEmpty() && !jsonSponsor.replace(" ","").equals(lbdcSponsor.replace(" ", "")) ) {
if (!id.startsWith("D")) {
logger.error("Sponsor: "+billNo);
logger.error(" LBDC: "+lbdcSponsor);
logger.error(" JSON: "+jsonSponsor);
observations.add(new ReportObservation(report.getId(), billNo, "BILL_SPONSOR", lbdcSponsor, jsonSponsor));
errorTotals.put("sponsor", errorTotals.get("sponsor")+1);
}
}
TreeSet<String> lbdcCosponsors = new TreeSet<String>();
TreeSet<String> jsonCosponsors = new TreeSet<String>();
if ( jsonBill.getCoSponsors() != null ) {
List<Person> cosponsors = jsonBill.getCoSponsors();
for(Person cosponsor : cosponsors) {
// store all names as Capitalized without () for comparison
jsonCosponsors.add(WordUtils.capitalizeFully(cosponsor.getFullname().replaceAll("[\\(\\)]+", "")));
}
}
// Capitalize and remove () for lbdc too
for(String cosponsor : bills.get(id).getCosponsors()){
lbdcCosponsors.add(WordUtils.capitalizeFully(cosponsor.replaceAll("[\\(\\)]+", "")));
}
if (!lbdcCosponsors.isEmpty() && (lbdcCosponsors.size() != jsonCosponsors.size() || (!lbdcCosponsors.isEmpty() && !lbdcCosponsors.containsAll(jsonCosponsors))) ) {
if (!id.startsWith("D")) {
logger.error("Cosponsors: "+billNo);
logger.error(" LBDC: "+lbdcCosponsors);
logger.error(" JSON: "+jsonCosponsors);
observations.add(new ReportObservation(report.getId(), billNo, "BILL_COSPONSOR", StringUtils.join(lbdcCosponsors, " "), StringUtils.join(jsonCosponsors, " ")));
errorTotals.put("cosponsors", errorTotals.get("cosponsors")+1);
}
}
ArrayList<String> lbdcEvents = bills.get(id).getActions();
ArrayList<String> jsonEvents = new ArrayList<String>();
SimpleDateFormat dateFormat = new SimpleDateFormat("MM/dd/yy");
for (Action action : jsonBill.getActions()) {
jsonEvents.add(dateFormat.format(action.getDate())+" "+action.getText());
}
if (!lbdcEvents.isEmpty() && (lbdcEvents.size() != jsonEvents.size() || (!lbdcEvents.isEmpty() && !lbdcEvents.containsAll(jsonEvents))) ) {
boolean substituted = StringUtils.join(jsonEvents, " ").toLowerCase().contains(" substituted ");
boolean delivered = !jsonEvents.isEmpty() ? jsonEvents.get(jsonEvents.size()-1).toLowerCase().contains(" delivered to ") : false;
if (!id.startsWith("D") && !substituted && !delivered) {
logger.error("Events: "+billNo);
logger.error(" LBDC: "+lbdcEvents);
logger.error(" JSON: "+jsonEvents);
observations.add(new ReportObservation(report.getId(), billNo, "BILL_ACTION", StringUtils.join(lbdcEvents,"\n"), StringUtils.join(jsonEvents,"\n")));
errorTotals.put("events", errorTotals.get("events")+1);
}
}
int lbdcPages = bills.get(id).pages;
int jsonPages = TextFormatter.pdfPrintablePages(jsonBill).size();
if (jsonBill.getFulltext().equals("")) {
jsonPages = 0;
}
if (jsonPages != lbdcPages) {
logger.error("Pages: "+billNo);
logger.error(" LBDC: "+lbdcPages);
logger.error(" JSON: "+jsonPages);
observations.add(new ReportObservation(report.getId(), billNo, "BILL_TEXT_PAGE", String.valueOf(lbdcPages), String.valueOf(jsonPages)));
errorTotals.put("pages", errorTotals.get("pages")+1);
}
ArrayList<String> lbdcAmendments = bills.get(id).getAmendments();
ArrayList<String> jsonAmendments = (ArrayList) jsonBill.getAmendments();
// Bill.getAmendments() does not include Bill itself, SpotCheckBill.getAmendments() does.
jsonAmendments.add(jsonBill.getBillId());
if (!amendmentsEqual(lbdcAmendments, jsonAmendments)) {
logger.error("Amendments: " + billNo);
logger.error("This Bill amendment : " + jsonBill.getBillId());
logger.error(" LBDC: " + lbdcAmendments);
logger.error(" JSON: " + jsonAmendments);
observations.add(new ReportObservation(report.getId(), billNo, "BILL_AMENDMENT", StringUtils.join(lbdcAmendments, "\n"), StringUtils.join(jsonAmendments, "\n")));
errorTotals.put("amendments", errorTotals.get("amendments") + 1);
}