Package gov.nysenate.openleg.model

Examples of gov.nysenate.openleg.model.Bill


            // Normally it is always sent to the base bill and broadcasted to amendments
            // In our 2009 data set we are missing tons of base amendments and it actually
            // needs to be broadcasted backwards to the original bill.
            // Make sure we are in the amendment list and that if we are active, that no one else is
            for (String versionKey : bill.getAmendments()) {
                Bill billVersion = storage.getBill(versionKey);
                if (!billVersion.getAmendments().contains(bill.getBillId())) {
                    billVersion.addAmendment(bill.getBillId());
                }
                if (bill.isActive()) {
                    billVersion.setActive(false);
                }
                billVersion.setSponsor(bill.getSponsor());
                billVersion.setCoSponsors(bill.getCoSponsors());
                billVersion.setOtherSponsors(bill.getOtherSponsors());
                billVersion.setMultiSponsors(bill.getMultiSponsors());
                billVersion.setLawSection(bill.getLawSection());
                billVersion.setLaw(bill.getLaw());
                billVersion.setSummary(bill.getSummary());
                storage.set(billVersion);
                ChangeLogger.record(storage.key(billVersion), storage);
            }
            if (bill.isUniBill()) {
                // Uni bills share text, always sent to the senate bill.
                Bill uniBill = storage.getBill(bill.getSameAs());
                if (uniBill != null) {
                    String billText = bill.getFulltext();
                    String uniBillText = uniBill.getFulltext();

                    if (billText.isEmpty()) {
                        if (!uniBillText.isEmpty()) {
                            // if we are empty then we must need their text
                            bill.setFulltext(uniBillText);
                        }
                    }
                    else if (!billText.equals(uniBillText)) {
                        // If we differ, then we must have just changed, share the text
                        uniBill.setFulltext(bill.getFulltext());
                    }
                    storage.set(uniBill);
                }
            }

            storage.set(bill);
            ChangeLogger.record(storage.key(bill), storage);
        }
        else {
            // When saving an unpublished bill make sure that references to it are removed
            // from other bills and find a new bill to make active.
            List<String> amendments = bill.getAmendments();
            if (amendments.size() > 0) {
                // This assumes that the last amendment on the list is the most recent one.
                // TODO: In rare cases with multiple substitutions this might not be the right thing to do!
                String newActiveBill = amendments.get(amendments.size()-1);

                // Remove all references to the unpublished bill from other versions.
                for (String versionKey : bill.getAmendments()) {
                    Bill billVersion = storage.getBill(versionKey);
                    billVersion.removeAmendment(bill.getBillId());
                    if (bill.isActive() && versionKey.equals(newActiveBill)) {
                        billVersion.setActive(true);
                    }
                    storage.set(billVersion);
                    ChangeLogger.record(storage.key(billVersion), storage);
                }
            }
View Full Code Here


        // This is a crappy situation, all bills on calendars should already exist but sometimes they won't.
        // This almost exclusively because we are missing sobi files. It shouldn't happen in production but
        // does frequently in development.
        BillProcessor processor = new BillProcessor();
        SOBIBlock mockBlock = new SOBIBlock(year+billId+(billId.matches("[A-Z]$") ? "" : " ")+1+"     ");
        Bill bill = processor.getOrCreateBill(mockBlock, modifiedDate, storage);

        if (sponsorName != null) {
            String[] sponsors = sponsorName.trim().split(",");
            bill.setSponsor(new Person(sponsors[0].trim()));

            // Other sponsors are removed when a calendar/agenda is resent without
            // The other sponsor included in the sponsors list.
            ArrayList<Person> otherSponsors = new ArrayList<Person>();
            for (int i = 1; i < sponsors.length; i++) {
                otherSponsors.add(new Person(sponsors[i].trim()));
            }

            if (!bill.getOtherSponsors().equals(otherSponsors)) {
                bill.setOtherSponsors(otherSponsors);
                processor.saveBill(bill, storage);
            }
        }

        if (!bill.isPublished()) {
            // It must be published if it is on the calendar
            bill.setPublishDate(modifiedDate);
            bill.setActive(true);
            processor.saveBill(bill, storage);
        }

        return bill;
    }
View Full Code Here

        // Catch exceptions on a per-block basis so that a single error won't corrupt the whole file.
        for (SOBIBlock block : getBlocks(sobiFile)) {
            logger.info("Processing "+block);
            try {
                String data = block.getData().toString();
                Bill bill = getOrCreateBill(block, date, storage);
                switch (block.getType()) {
                    case '1': applyBillInfo(data, bill, date); break;
                    case '2': applyLawSection(data, bill, date); break;
                    case '3': applyTitle(data, bill, date); break;
                    case '4': applyBillEvent(data, bill, date); break;
                    case '5': applySameAs(data, bill, date); break;
                    case '6': applySponsor(data, bill, date); break;
                    case '7': applyCosponsors(data, bill, date); break;
                    case '8': applyMultisponsors(data, bill, date); break;
                    case '9': applyProgramInfo(data, bill, date); break;
                    case 'A': applyActClause(data, bill, date); break;
                    case 'B': applyLaw(data, bill, date); break;
                    case 'C': applySummary(data, bill, date); break;
                    case 'M': // Sponsor memo text - handled by applyText
                    case 'R': // Resolution text - handled by applyText
                    case 'T': applyText(data, bill, date); break;
                    case 'V': applyVoteMemo(data, bill, date); break;
                    default: throw new ParseError("Invalid Line Code "+block.getType() );
                }
                bill.addDataSource(sobiFile.getName());
                saveBill(bill, storage);
            }
            catch (ParseError e) {
                logger.error("ParseError at "+block.getLocation(), e);
            }
View Full Code Here

     * @throws ParseError
     */
    public Bill getOrCreateBill(SOBIBlock block, Date date, Storage storage)
    {
        String billId = block.getPrintNo()+block.getAmendment()+"-"+block.getYear();
        Bill bill = storage.getBill(block.getPrintNo()+block.getAmendment(), block.getYear());

        if (bill == null) {
            bill = new Bill(billId, block.getYear());
            bill.setModifiedDate(date);
        }

        // If this bill is currently published or it is a new base bill
        // then we don't need to worry about syncing up all the versions.
        if (bill.isPublished() || block.getAmendment().isEmpty()) {
            return bill;
        }

        // All amendments are based on the original bill. New and unpublished
        // bill amendments need to sync certain data with the base bill.
        String baseBillId = block.getPrintNo()+"-"+block.getYear();
        Bill baseBill = storage.getBill(block.getPrintNo(), block.getYear());
        if (baseBill == null) {
            // Amendments should always have original bills already made, make it happen
            logger.warn("Bill Amendment filed without initial bill at "+block.getLocation()+" - "+block.getHeader());
            baseBill = new Bill(baseBillId, block.getYear());
            baseBill.setModifiedDate(date);
            // TODO: This isn't quite right since it isn't published..
            // ??/
            storage.set(baseBill);
        }

        // Base bills are always the first "amendment" to a bill.
        // Grab any other amendments that the base bill knows about.
        bill.setAmendments(new ArrayList<String>(Arrays.asList(baseBillId)));
        bill.addAmendments(baseBill.getAmendments());

        // Pull shared information up from the base bill
        bill.setSponsor(baseBill.getSponsor());
        bill.setCoSponsors(baseBill.getCoSponsors());
        bill.setOtherSponsors(baseBill.getOtherSponsors());
        bill.setMultiSponsors(baseBill.getMultiSponsors());
        bill.setSummary(baseBill.getSummary());
        bill.setLaw(baseBill.getLaw());

        // Some information isn't exactly shared but should be pulled
        // up from the previously active version of the bill until a
        // proper update comes through for the data.
        for (String versionKey : bill.getAmendments()) {
            // Sometimes a re-published bill version was more recently updated than the
            // currently active bill version. In these cases, don't update values.
            Bill billVersion = storage.getBill(versionKey);
            logger.info(versionKey + " - "+billVersion.getModifiedDate()+"; "+bill.getBillId() + " - "+bill.getModifiedDate());
            if (billVersion.isActive() && billVersion.getModifiedDate().getTime() > bill.getModifiedDate().getTime()) {
                bill.setTitle(billVersion.getTitle());
                bill.setActClause(billVersion.getActClause());
                bill.setLawSection(billVersion.getLawSection());
            }
        }

        return bill;
    }
View Full Code Here

        BillProcessor bp = new BillProcessor();
        Storage storage = Application.getStorage();
        List<SOBIBlock> blocks = bp.getBlocks(blockFile);
        for (SOBIBlock block : blocks) {
            String billNo = block.getPrintNo()+block.getAmendment()+"-"+block.getYear();
            Bill jsonBill = storage.getBill(block.getPrintNo()+block.getAmendment(), block.getYear());
            Bill lbdcBill = new Bill(billNo, block.getYear());
            bp.applyText(new String(block.getData().getBytes(), "utf-8"), lbdcBill, new Date());
            String jsonMemo = StringUtils.normalizeSpace(jsonBill.getMemo().replaceAll("-\n+ *", "").replaceAll("\n *", " ").replaceAll(" *([:,]) *", "$1").replaceAll(" *([()!\\\"]) *", " $1 ").replaceAll("([A-Za-z])- ?([A-Za-z])","$1$2").replaceAll("-", "").trim()).toLowerCase();
            String lbdcMemo = StringUtils.normalizeSpace(lbdcBill.getMemo().replaceAll("�", "§").replaceAll("-\n+ *", "").replaceAll("\n *", " ").replaceAll(" *([:,]) *", "$1").replaceAll(" *([()!\\\"]) *", " $1 ").replaceAll("([A-Za-z])- ?([A-Za-z])","$1$2").replaceAll("-", "").trim()).toLowerCase();

            if(jsonMemo.isEmpty()) {
                logger.error(billNo+": MISSING");
                errors.put("missing", errors.get("missing")+1);
            }
View Full Code Here

        runner.update("delete from report_observation where reportId = ?", report.getId());

        for(String id : bills.keySet()) {
            //logger.info("checking bill "+id);
            String billNo = id+"-2013";
            Bill jsonBill = (Bill)storage.get("2013/bill/"+billNo, Bill.class);

            if (jsonBill == null) {
                logger.error("Missing bill "+"2013/bill/"+billNo);
                continue;
            }

            if (!jsonBill.isPublished()) {
                logger.error("Bill Unpublished: "+billNo);
                continue;
            }

            // Compare the titles, ignore white space differences
            String jsonTitle = unescapeHTML(jsonBill.getTitle());
            String lbdcTitle = bills.get(id).getTitle();
            if (!lbdcTitle.isEmpty() && !stringEquals(jsonTitle, lbdcTitle, true, true)) {
                // What is this D?
                if (!id.startsWith("D")) {
                    logger.error("Title: "+billNo);
                    logger.error("  LBDC: "+lbdcTitle);
                    logger.error("  JSON: "+jsonTitle);
                    observations.add(new ReportObservation(report.getId(), billNo, "BILL_TITLE", lbdcTitle, jsonTitle));
                    errorTotals.put("title", errorTotals.get("title")+1);
                }
            }

            // Compare the summaries. LBDC reports summary and law changes together
            String jsonLaw = jsonBill.getLaw();
            String jsonSummary = unescapeHTML(jsonBill.getSummary());
            String lbdcSummary = bills.get(id).getSummary().replaceAll("\\s+", " ");

            if( jsonLaw != null && jsonLaw != "" && jsonLaw != "null") {
                jsonSummary = unescapeHTML(jsonLaw)+" "+jsonSummary;
            }

            if (lbdcSummary.equals("BILL SUMMARY NOT FOUND")) {
                lbdcSummary = "";
            }

            jsonSummary = jsonSummary.replace('§', 'S').replace('¶', 'P');
            if (!lbdcSummary.isEmpty() && !jsonSummary.replace(" ","").equals(lbdcSummary.replace(" ", "")) ) {
                if (!id.startsWith("D")) {
                    logger.error("Summary: "+billNo);
                    logger.error("  LBDC: "+lbdcSummary);
                    logger.error("  JSON: "+jsonSummary);
                    observations.add(new ReportObservation(report.getId(), billNo, "BILL_SUMMARY", lbdcSummary, jsonSummary));
                    errorTotals.put("summary", errorTotals.get("summary")+1);
                }
            }

            String jsonSponsor = "";
            if (jsonBill.getSponsor() != null) {
                jsonSponsor = unescapeHTML(jsonBill.getSponsor().getFullname()).toUpperCase().replace(" (MS)","").replace("BILL", "").replace("COM", "");
            }
            String lbdcSponsor = bills.get(id).getSponsor().toUpperCase().replace("BILL", "").replace("COM", "");
            if (lbdcSponsor.startsWith("RULES ") && lbdcSponsor.contains("(") && !lbdcSponsor.contains(")")){
                lbdcSponsor = lbdcSponsor.concat(")");
            }
            if (!lbdcSponsor.isEmpty() && !jsonSponsor.replace(" ","").equals(lbdcSponsor.replace(" ", "")) ) {
                if (!id.startsWith("D")) {
                    logger.error("Sponsor: "+billNo);
                    logger.error("  LBDC: "+lbdcSponsor);
                    logger.error("  JSON: "+jsonSponsor);
                    observations.add(new ReportObservation(report.getId(), billNo, "BILL_SPONSOR", lbdcSponsor, jsonSponsor));
                    errorTotals.put("sponsor", errorTotals.get("sponsor")+1);
                }
            }


            TreeSet<String> lbdcCosponsors = new TreeSet<String>();
            TreeSet<String> jsonCosponsors = new TreeSet<String>();
            if ( jsonBill.getCoSponsors() != null ) {
                List<Person> cosponsors = jsonBill.getCoSponsors();
                for(Person cosponsor : cosponsors) {
                    // store all names as Capitalized without () for comparison
                    jsonCosponsors.add(WordUtils.capitalizeFully(cosponsor.getFullname().replaceAll("[\\(\\)]+", "")));
                }
            }
            // Capitalize and remove () for lbdc too
            for(String cosponsor : bills.get(id).getCosponsors()){
                lbdcCosponsors.add(WordUtils.capitalizeFully(cosponsor.replaceAll("[\\(\\)]+", "")));
            }


            if (!lbdcCosponsors.isEmpty() && (lbdcCosponsors.size() != jsonCosponsors.size() || (!lbdcCosponsors.isEmpty() && !lbdcCosponsors.containsAll(jsonCosponsors))) ) {
                if (!id.startsWith("D")) {
                    logger.error("Cosponsors: "+billNo);
                    logger.error("  LBDC: "+lbdcCosponsors);
                    logger.error("  JSON: "+jsonCosponsors);
                    observations.add(new ReportObservation(report.getId(), billNo, "BILL_COSPONSOR", StringUtils.join(lbdcCosponsors, " "), StringUtils.join(jsonCosponsors, " ")));
                    errorTotals.put("cosponsors", errorTotals.get("cosponsors")+1);
                }
            }

            ArrayList<String> lbdcEvents = bills.get(id).getActions();
            ArrayList<String> jsonEvents = new ArrayList<String>();
            SimpleDateFormat dateFormat = new SimpleDateFormat("MM/dd/yy");

            for (Action action : jsonBill.getActions()) {
                jsonEvents.add(dateFormat.format(action.getDate())+" "+action.getText());
            }

            if (!lbdcEvents.isEmpty() &&  (lbdcEvents.size() != jsonEvents.size() || (!lbdcEvents.isEmpty() && !lbdcEvents.containsAll(jsonEvents))) ) {
                boolean substituted = StringUtils.join(jsonEvents, " ").toLowerCase().contains(" substituted ");
                boolean delivered =  !jsonEvents.isEmpty() ? jsonEvents.get(jsonEvents.size()-1).toLowerCase().contains(" delivered to ") : false;
                if (!id.startsWith("D") && !substituted && !delivered) {
                    logger.error("Events: "+billNo);
                    logger.error("  LBDC: "+lbdcEvents);
                    logger.error("  JSON: "+jsonEvents);
                    observations.add(new ReportObservation(report.getId(), billNo, "BILL_ACTION", StringUtils.join(lbdcEvents,"\n"), StringUtils.join(jsonEvents,"\n")));
                    errorTotals.put("events", errorTotals.get("events")+1);
                }
            }

            int lbdcPages = bills.get(id).pages;
            int jsonPages = TextFormatter.pdfPrintablePages(jsonBill).size();
            if (jsonBill.getFulltext().equals("")) {
                jsonPages = 0;
            }

            if (jsonPages != lbdcPages) {
                logger.error("Pages: "+billNo);
                logger.error("  LBDC: "+lbdcPages);
                logger.error("  JSON: "+jsonPages);
                observations.add(new ReportObservation(report.getId(), billNo, "BILL_TEXT_PAGE", String.valueOf(lbdcPages), String.valueOf(jsonPages)));
                errorTotals.put("pages", errorTotals.get("pages")+1);
            }

            ArrayList<String> lbdcAmendments = bills.get(id).getAmendments();
            ArrayList<String> jsonAmendments = (ArrayList) jsonBill.getAmendments();

            // Bill.getAmendments() does not include Bill itself, SpotCheckBill.getAmendments() does.
            jsonAmendments.add(jsonBill.getBillId());

            if (!amendmentsEqual(lbdcAmendments, jsonAmendments)) {
                logger.error("Amendments: " + billNo);
                logger.error("This Bill amendment : " + jsonBill.getBillId());
                logger.error("  LBDC: " + lbdcAmendments);
                logger.error("  JSON: " + jsonAmendments);
                observations.add(new ReportObservation(report.getId(), billNo, "BILL_AMENDMENT", StringUtils.join(lbdcAmendments, "\n"), StringUtils.join(jsonAmendments, "\n")));
                errorTotals.put("amendments", errorTotals.get("amendments") + 1);
            }
View Full Code Here

        Storage storage = Application.getStorage();
        String billId = opts.getOptionValue("b");
        int sessionYear = Integer.valueOf(opts.getOptionValue("y"));
        boolean wasRestored = opts.hasOption("r");

        Bill bill = storage.getBill(billId, sessionYear);
        removeAmendmentFromOtherVersions(storage, bill);

        // Deactivate ourselves.
        bill.setActive(false);
        storage.set(bill);

        if (wasRestored) {
            ChangeLogger.record(storage.key(bill), storage);
        }
        else {
            bill.setPublishDate(null); // un publish.
            ChangeLogger.delete(storage.key(bill), storage);
        }

        storage.flush();
View Full Code Here

        if (amendments.size() > 0) {
            String newActiveBill = amendments.get(amendments.size() - 1);

            // Remove all references to the unpublished bill from other versions.
            for (String versionKey : bill.getAmendments()) {
                Bill billVersion = storage.getBill(versionKey);
                billVersion.removeAmendment(bill.getBillId());

                if (bill.isActive() && versionKey.equals(newActiveBill)) {
                    billVersion.setActive(true);
                }
                storage.set(billVersion);
                ChangeLogger.record(storage.key(billVersion), storage);
            }
        }
View Full Code Here

                            // Regenerate all the bill sub-documents. Delete them all first to
                            // account for any removals or changes in the document oid. Add back
                            // references so that the document can be properly constructed.
                            //
                            // TODO: Should we really be pulling the modified date from the bill?
                            Bill bill = (Bill)obj;

                            lucene.deleteDocumentsByQuery("otype:action AND billno:" + bill.getBillId());
                            for(Action billEvent:bill.getActions()) {
                                try {
                                    billEvent.setBill(bill);
                                    lucene.updateDocument(DocumentBuilder.build(billEvent));
                                }
                                catch (IOException e) {
                                    logger.error("Error indexing: "+key, e);
                                }
                            }

                            lucene.deleteDocumentsByQuery("otype:vote AND billno:" + bill.getBillId());
                            for(Vote vote: bill.getVotes()) {
                                try {
                                    vote.setBill(bill);
                                    lucene.updateDocument(DocumentBuilder.build(vote));
                                }
                                catch(IOException e) {
View Full Code Here

            }
        }
    }

    public Bill handleXMLBill(Storage storage, Meeting meeting, XMLBill xmlBill, int sessionYear) {
        Bill bill = getBill(storage, xmlBill.getNo(), sessionYear, xmlBill.getSponsor().getContent());

        if (xmlBill.getTitle() != null && bill.getActClause().isEmpty()) {
            bill.setActClause(xmlBill.getTitle().getContent());
        }

        if (xmlBill.getVotes() != null) {
            Date voteDate = meeting.getMeetingDateTime();
            Vote vote = new Vote(bill, voteDate, Vote.VOTE_TYPE_COMMITTEE, "1");
            vote.setPublishDate(modifiedDate);
            vote.setModifiedDate(modifiedDate);

            // remove the old vote
            // TODO: will this ever actually work with aye/nay counts at -1?
            //    I suppose is will now that I'm using sequence numbers instead
            bill.removeVote(vote);

            vote.setVoteType(Vote.VOTE_TYPE_COMMITTEE);
            vote.setDescription(meeting.getCommitteeName());
            for( XMLMember member : xmlBill.getVotes().getMember()) {
                Person person = new Person(member.getName().getContent());
                String voteType = member.getVote().getContent().toLowerCase();

                if (voteType.startsWith("abstain"))
                    vote.addAbstain(person);
                else if (voteType.startsWith("aye w/r"))
                    vote.addAyeWR(person);
                else if (voteType.startsWith("aye"))
                    vote.addAye(person);
                else if (voteType.startsWith("excused"))
                    vote.addExcused(person);
                else if (voteType.startsWith("nay"))
                    vote.addNay(person);
                else if (voteType.startsWith("absent"))
                    vote.addAbsent(person);
            }

            // Add the new vote, effectively replacing an older copy
            bill.updateVote(vote);

            // Make sure the bill gets updated on disc
            storage.set(bill);
            ChangeLogger.record(storage.key(bill), storage);
        }
View Full Code Here

TOP

Related Classes of gov.nysenate.openleg.model.Bill

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.