Examples of gov.nysenate.openleg.util.Storage

gov.nysenate.openleg.util.Storage
Simple file backed key-value store with that supports both published and unpublished file modes. Buffers changes in memory to reduce file system access and increase performance. @author GraylinKim

    }


    public HashMap<String, Change> ingestFiles(Collection<File> files)
    {
        Timer timer = new Timer();
        Storage storage = new Storage(getStorageDirectory());
        BillProcessor billProcessor = new BillProcessor();
        CalendarProcessor calendarProcessor = new CalendarProcessor();
        AgendaProcessor agendaProcessor = new AgendaProcessor();
        TranscriptProcessor transcriptProcessor = new TranscriptProcessor();


        Collections.sort((List<File>)files, new FileNameComparator());


        // Process each file individually, flushing changes to storage as necessary
        // Each file processor should produce a change log indicating what happened
        timer.start();
        for(File file : files) {
            try {
                logger.debug("Ingesting: "+file);
                String type = file.getParentFile().getName();
                if (type.equals("bills")) {
                    billProcessor.process(file, storage);
                }
                else if (type.equals("calendars")) {
                    calendarProcessor.process(file, storage);
                }
                else if (type.equals("agendas")) {
                    agendaProcessor.process(file, storage);
                }
                else if (type.equals("annotations")) {
                    continue;
                }
                else if (type.equals("transcripts")) {
                    transcriptProcessor.process(file, storage);
                }


                // To avoid memory issues, occasionally flush changes to file-system and truncate memory
                if (storage.memory.size() > 4000) {
                    storage.flush();
                    storage.clear();
                }


            }
            catch (IOException e) {
                logger.error("Issue with "+file.getName(), e);
            }
            catch (UnmarshalException e) {
                logger.error("Issue with "+file.getName(), e);
            }
            catch (JAXBException e) {
                logger.error("Unable to parse xml "+file.getName(), e);
            }
        }
        storage.flush();
        logger.info(timer.stop()+" seconds to injest "+files.size()+" files.");
        return ChangeLogger.getChangeLog();
    }

View Full Code Here

        HashMap<String, Integer> errors = new HashMap<String, Integer>();
        errors.put("missing",0);
        errors.put("mismatch", 0);
        File blockFile = new File("/data/openleg/PALMER.SEN.ALL.MEMO2011.TXT");
        BillProcessor bp = new BillProcessor();
        Storage storage = Application.getStorage();
        List<SOBIBlock> blocks = bp.getBlocks(blockFile);
        for (SOBIBlock block : blocks) {
            String billNo = block.getPrintNo()+block.getAmendment()+"-"+block.getYear();
            Bill jsonBill = storage.getBill(block.getPrintNo()+block.getAmendment(), block.getYear());
            Bill lbdcBill = new Bill(billNo, block.getYear());
            bp.applyText(new String(block.getData().getBytes(), "utf-8"), lbdcBill, new Date());
            String jsonMemo = StringUtils.normalizeSpace(jsonBill.getMemo().replaceAll("-\n+ *", "").replaceAll("\n *", " ").replaceAll(" *([:,]) *", "$1").replaceAll(" *([()!\\\"]) *", " $1 ").replaceAll("([A-Za-z])- ?([A-Za-z])","$1$2").replaceAll("-", "").trim()).toLowerCase();
            String lbdcMemo = StringUtils.normalizeSpace(lbdcBill.getMemo().replaceAll("�", "§").replaceAll("-\n+ *", "").replaceAll("\n *", " ").replaceAll(" *([:,]) *", "$1").replaceAll(" *([()!\\\"]) *", " $1 ").replaceAll("([A-Za-z])- ?([A-Za-z])","$1$2").replaceAll("-", "").trim()).toLowerCase();

View Full Code Here

    public void execute(CommandLine opts) throws IOException, ParseException, SQLException
    {
        QueryRunner runner = new QueryRunner(Application.getDB().getDataSource());


        String[] args = opts.getArgs();
        Storage storage = Application.getStorage();


        Config config = Application.getConfig();


        List<ReportObservation> observations = new ArrayList<ReportObservation>();
        HashMap<String, Integer> errorTotals = new HashMap<String, Integer>();
        for (String error_type : new String[] {"title", "summary", "sponsor", "cosponsors", "events", "pages", "amendments"}) {
            errorTotals.put(error_type, 0);
        }


        String prefix = args[0];
        Date date = dateFormat.parse(prefix);
        logger.info("Processing daybreak files for: "+date);
        File directory = new File(config.getValue("checkmail.lrsFileDir"));
        HashMap<String, SpotCheckBill> bills = new HashMap<String, SpotCheckBill>();
        logger.info("Reading " + prefix+".senate.low.html");
        bills.putAll(readDaybreak(new File(directory, prefix+".senate.low.html")));
        logger.info("Reading " + prefix+".senate.high.html");
        bills.putAll(readDaybreak(new File(directory, prefix+".senate.high.html")));
        logger.info("Reading " + prefix+".assembly.low.html");
        bills.putAll(readDaybreak(new File(directory, prefix+".assembly.low.html")));
        logger.info("Reading " + prefix+".assembly.high.html");
        bills.putAll(readDaybreak(new File(directory, prefix+".assembly.high.html")));
        logger.info("Reading " + prefix+".page_file.txt");
        loadPageFile(new File(directory, prefix+".page_file.txt"), bills);


        SpotCheckBill testbill = bills.get("S1743A");


        runner.update("insert ignore into report(time) values(?)", date);
        Report report = runner.query("select * from report where time = ?", new BeanHandler<Report>(Report.class), date);
        runner.update("delete from report_observation where reportId = ?", report.getId());


        for(String id : bills.keySet()) {
            //logger.info("checking bill "+id);
            String billNo = id+"-2013";
            Bill jsonBill = (Bill)storage.get("2013/bill/"+billNo, Bill.class);


            if (jsonBill == null) {
                logger.error("Missing bill "+"2013/bill/"+billNo);
                continue;
            }

View Full Code Here

        return options;
    }


    @Override
    protected void execute(CommandLine opts) throws Exception {
        Storage storage = Application.getStorage();
        String billId = opts.getOptionValue("b");
        int sessionYear = Integer.valueOf(opts.getOptionValue("y"));
        boolean wasRestored = opts.hasOption("r");


        Bill bill = storage.getBill(billId, sessionYear);
        removeAmendmentFromOtherVersions(storage, bill);


        // Deactivate ourselves.
        bill.setActive(false);
        storage.set(bill);


        if (wasRestored) {
            ChangeLogger.record(storage.key(bill), storage);
        }
        else {
            bill.setPublishDate(null); // un publish.
            ChangeLogger.delete(storage.key(bill), storage);
        }


        storage.flush();


        // push changes to lucene and varnish
        ArrayList<ServiceBase> services = new ArrayList<ServiceBase>();
        services.add(new Lucene());
        services.add(new Varnish("127.0.0.1", 80));

View Full Code Here


  @Test
  public void testDeleteWhole()
  {
    File sobiDirectory = new File("/home/shweta/OpenLegislation/src/test/resources/sobi");
      Storage storage = new Storage(env.getStorageDirectory());
    File[] initialCommit = TestHelper.getFilesByName(sobiDirectory, "SOBI.D120311.T202049.TXT");
      TestHelper.processFile(env, initialCommit);
      File[] finalCommit = TestHelper.getFilesByName(sobiDirectory, "SOBI.D120311.T202549.TXT");
      TestHelper.processFile(env, finalCommit);
      Bill theBill = (Bill)storage.get("2011/bill/S6696-2011", Bill.class);
      assertThat(theBill,nullValue());




  }

View Full Code Here

  @Test
  public void verifyBill2013S2230() throws IOException
  {
    Bill theBill;
    File sobiDirectory = new File("/home/shweta/test/processed/2013/bills/");
      Storage storage = new Storage(env.getStorageDirectory());
      ArrayList<File> file = (ArrayList<File>) TestHelper.getFilesByNameCollection(sobiDirectory, "SOBI.D130114.T213149.TXT",
      "SOBI.D130115.T124554.TXT", "SOBI.D130114.T212649.TXT", "SOBI.D130115.T095048.TXT", "SOBI.D130115.T124054.TXT",
      "SOBI.D130115.T093046.TXT", "SOBI.D130116.T122729.TXT", "SOBI.D130114.T225652.TXT", "SOBI.D130114.T233653.TXT",
      "SOBI.D130115.T093547.TXT", "SOBI.D130116.T120230.TXT", "SOBI.D130115.T162604.TXT", "SOBI.D130114.T232653.TXT",
      "SOBI.D130116.T115728.TXT", "SOBI.D130114.T223651.TXT", "SOBI.D130115.T162104.TXT", "SOBI.D130115.T132056.TXT",
      "SOBI.D130115.T094047.TXT", "SOBI.D130115.T101048.TXT");




     // Tests for SOBI.D130114.T213149.TXT
      TestHelper.processFileC(env ,file.get(0));
      theBill=(Bill)storage.get("2013/bill/S2230-2013", Bill.class);
      assertEquals(theBill.getSameAs(),"A2388");




     // Tests for SOBI.D130114.T212649.TXT
     TestHelper.processFileC(env ,file.get(2));
     theBill=(Bill)storage.get("2013/bill/S2230-2013", Bill.class);
     ArrayList<Person> list=(ArrayList<Person>)theBill.getCoSponsors();
     assertEquals(theBill.getSponsor().getFullname(),"KLEIN");
     assertEquals(list.get(0).toString(),"SMITH");


      // Tests for the SOBI.D130114.T223651.TXT
      TestHelper.processFileC(env ,file.get(14));
      theBill=(Bill)storage.get("2013/bill/S2230-2013", Bill.class);
      //Should have a title:It is left empty (Memo) Dont Care
      //assertNull("The title should not be null",theBill.getTitle());


      // Tests for the SOBI.D130114.T225652.TXT
      TestHelper.processFileC(env ,file.get(7));
      theBill=(Bill)storage.get("2013/bill/S2230-2013", Bill.class);
      assertNotNull("The title should not be null",theBill.getTitle());
      assertEquals(theBill.getSponsor().getFullname(),"KLEIN");






      // Tests for the SOBI.D130114.T232653.TXT
      TestHelper.processFileC(env ,file.get(12));


      theBill=(Bill)storage.get("2013/bill/S2230-2013", Bill.class);
      assertNotNull("The title should not be null",theBill.getTitle());
      assertEquals(theBill.getSponsor().getFullname(),"KLEIN");


      // Tests for the SOBI.D130114.T233653.TXT
      TestHelper.processFileC(env ,file.get(8));
      theBill=(Bill)storage.get("2013/bill/S2230-2013", Bill.class);
      boolean var=voteList2013S2230(theBill);
      assertEquals(var,true);




   // Tests for the SOBI.D130115.T93046.TXT
      TestHelper.processFileC(env ,file.get(5));
       theBill=(Bill)storage.get("2013/bill/S2230-2013", Bill.class);
       assertEquals(theBill.getLawSection().equalsIgnoreCase("Criminal Procedure Law"),true);




     /*  Nothing to test in SOBI.D130115.T093547.TXT,SOBI.D130115.T095048.TXT,SOBI.D130115.T101048.TXT,
       SOBI.D130115.T124554.TXT, SOBI.D130115.T132056.TXT,SOBI.D130116.T120230.TXT(Memo), SOBI.D130116.T122729.TXT */


     //Tests for the SOBI.D130115.T094047.TXT,SOBI.D130115.T124054.TXT,SOBI.D130115.T162104.TXT,SOBI.D130116.T115728.TXT
        TestHelper.processFileC(env ,file.get(17));
      theBill=(Bill)storage.get("2013/bill/S2230-2013", Bill.class);
      assertNotNull("The title should not be null",theBill.getTitle());
      assertEquals(theBill.getSponsor().getFullname(),"KLEIN");
      TestHelper.processFileC(env ,file.get(4));
      theBill=(Bill)storage.get("2013/bill/S2230-2013", Bill.class);
      assertNotNull("The title should not be null",theBill.getTitle());
      assertEquals(theBill.getSponsor().getFullname(),"KLEIN");
      TestHelper.processFileC(env ,file.get(15));
      theBill=(Bill)storage.get("2013/bill/S2230-2013", Bill.class);
      assertNotNull("The title should not be null",theBill.getTitle());
      assertEquals(theBill.getSponsor().getFullname(),"KLEIN");
      TestHelper.processFileC(env ,file.get(13));
      theBill=(Bill)storage.get("2013/bill/S2230-2013", Bill.class);
      assertNotNull("The title should not be null",theBill.getTitle());
      assertEquals(theBill.getSponsor().getFullname(),"KLEIN");


      }

View Full Code Here

  @Test
     public void verifyWholeBill2011S1462() throws IOException
    {
    Bill theBill;
    File sobiDirectory = new File("/home/shweta/test/processed/2013/bills/");
      Storage storage = new Storage(env.getStorageDirectory());
      ArrayList<File> file = (ArrayList<File>) TestHelper.getFilesByNameCollection(sobiDirectory,"SOBI.D110110.T142112.TXT",
     "SOBI.D110107.T144910.TXT",   "SOBI.D110112.T175532.TXT", "SOBI.D110107.T150914.TXT", "SOBI.D110107.T141407.TXT");


      // Test for SOBI.D110107.T141407.TXT
      TestHelper.processFileC(env ,file.get(4));
      theBill=(Bill)storage.get("2011/bill/S1462-2011", Bill.class);
      assertNotNull("The title should not be null",theBill.getTitle());
    assertEquals(theBill.getSponsor().getFullname(),"LAVALLE");


    //Test for SOBI.D110107.T144910.TXT
     TestHelper.processFileC(env ,file.get(1));
     theBill=(Bill)storage.get("2011/bill/S1462-2011", Bill.class);
     assertEquals(theBill.getSameAs(),"A1415");


    // No Test for SOBI.D110107.T150914.TXT,SOBI.D110110.T142112.TXT,SOBI.D110112.T175532.TXT(Memo)
    }

View Full Code Here

  @Test
     public void verifyWholeBill2011S1462A() throws IOException  // Testing Amendment A of the bill 2011S1462
    {
    Bill theBill;
    File sobiDirectory = new File("/home/shweta/test/processed/2013/bills/");
      Storage storage = new Storage(env.getStorageDirectory());
      ArrayList<File> file = (ArrayList<File>) TestHelper.getFilesByNameCollection(sobiDirectory,"SOBI.D110110.T142112.TXT",
          "SOBI.D110613.T222123.TXT", "SOBI.D110614.T192241.TXT", "SOBI.D110210.T221519.TXT", "SOBI.D120130.T202912.TXT",
          "SOBI.D110209.T171647.TXT", "SOBI.D120126.T103841.TXT", "SOBI.D110614.T195743.TXT", "SOBI.D120104.T223233.TXT",
          "SOBI.D110215.T151615.TXT", "SOBI.D110209.T105617.TXT", "SOBI.D110209.T110618.TXT", "SOBI.D110614.T152729.TXT",
          "SOBI.D110210.T102843.TXT", "SOBI.D110112.T175532.TXT", "SOBI.D110107.T144910.TXT", "SOBI.D110614.T185240.TXT",
          "SOBI.D120125.T130337.TXT", "SOBI.D110209.T171147.TXT", "SOBI.D110107.T150914.TXT", "SOBI.D110209.T170214.TXT",
          "SOBI.D110107.T141407.TXT");






        // Test for SOBI.D110209.T105617.TXT
     TestHelper.processFileC(env ,file.get(10));
     theBill=(Bill)storage.get("2011/bill/S1462A-2011", Bill.class);
     assertEquals(theBill.getSponsor().getFullname(),"LAVALLE");
     assertNotNull("The title should not be null",theBill.getTitle());


      // Test for SOBI.D110209.T110618.TXT
      TestHelper.processFileC(env ,file.get(11));
        theBill=(Bill)storage.get("2011/bill/S1462A-2011", Bill.class);
        assertEquals(theBill.getSponsor().getFullname(),"LAVALLE");
        assertNotNull("The title should not be null",theBill.getTitle());


        // Test for SOBI.D110215.T151615.TXT
        TestHelper.processFileC(env ,file.get(9));
        theBill=(Bill)storage.get("2011/bill/S1462A-2011", Bill.class);
        assertEquals(theBill.getSponsor().getFullname(),"LAVALLE");
        assertNotNull("The title should not be null",theBill.getTitle());


        // Test for SOBI.D110613.T222123.TXT
        TestHelper.processFileC(env ,file.get(1));
        theBill=(Bill)storage.get("2011/bill/S1462A-2011", Bill.class);
        assertEquals(theBill.getSponsor().getFullname(),"LAVALLE");
        assertNotNull("The title should not be null",theBill.getTitle());


        // Test for SOBI.D110614.T152729.TXT
        TestHelper.processFileC(env ,file.get(12));
        theBill=(Bill)storage.get("2011/bill/S1462A-2011", Bill.class);
        assertEquals(theBill.getSponsor().getFullname(),"LAVALLE");
        assertNotNull("The title should not be null",theBill.getTitle());


        // Test for SOBI.D110209.T170214.TXT,
        TestHelper.processFileC(env ,file.get(20));
        theBill=(Bill)storage.get("2011/bill/S1462A-2011", Bill.class);
        assertEquals(theBill.getSponsor().getFullname(),"LAVALLE");
        assertNotNull("The title should not be null",theBill.getTitle());


        // Test for SOBI.D110209.T171147.TXT
        TestHelper.processFileC(env ,file.get(18));
        theBill=(Bill)storage.get("2011/bill/S1462A-2011", Bill.class);
        assertEquals(theBill.getSameAs(),"A1415A");


        // NO test for SOBI.D110210.T102843.TXT ,SOBI.D110210.T221519.TXT (Memo)

View Full Code Here

  public static void initalSetup()
  {
    loadProperties();
    sobiDirectory = new File("src/test/resources/sobi");
    
    storage = new Storage(env.getStorageDirectory());
  }

View Full Code Here

  @Test
     public void verifyWholeBill2011S1462A() throws IOException  // Testing Amendment A of the bill 2011S1462
    {
    Bill theBill;
    File sobiDirectory = new File("/home/shweta/test/processed/2013/bills/");
      Storage storage = new Storage(env.getStorageDirectory());
      ArrayList<File> file = (ArrayList<File>) TestHelper.getFilesByNameCollection(sobiDirectory,"SOBI.D110110.T142112.TXT",
          "SOBI.D110613.T222123.TXT", "SOBI.D110614.T192241.TXT","SOBI.D120130.T202912.TXT", "SOBI.D120126.T103841.TXT",
          "SOBI.D110614.T195743.TXT", "SOBI.D120104.T223233.TXT", "SOBI.D110215.T151615.TXT", "SOBI.D110209.T105617.TXT",
          "SOBI.D110209.T110618.TXT", "SOBI.D110614.T152729.TXT", "SOBI.D110112.T175532.TXT", "SOBI.D110107.T144910.TXT",
          "SOBI.D110614.T185240.TXT", "SOBI.D120125.T130337.TXT", "SOBI.D110209.T171147.TXT", "SOBI.D110107.T150914.TXT",
          "SOBI.D110209.T170214.TXT", "SOBI.D110107.T141407.TXT");
      // NO test for SOBI.D110210.T102843.TXT ,SOBI.D110210.T221519.TXT (Memo),SOBI.D110209.T171647.TXT




        // Test for SOBI.D110209.T105617.TXT
     TestHelper.processFileC(env ,file.get(8));
     theBill=(Bill)storage.get("2011/bill/S1462A-2011", Bill.class);
     assertEquals(theBill.getSponsor().getFullname(),"LAVALLE");
     assertNotNull("The title should not be null",theBill.getTitle());


      // Test for SOBI.D110209.T110618.TXT
      TestHelper.processFileC(env ,file.get(9));
        theBill=(Bill)storage.get("2011/bill/S1462A-2011", Bill.class);
        assertEquals(theBill.getSponsor().getFullname(),"LAVALLE");
        assertNotNull("The title should not be null",theBill.getTitle());


        // Test for SOBI.D110215.T151615.TXT
        TestHelper.processFileC(env ,file.get(7));
        theBill=(Bill)storage.get("2011/bill/S1462A-2011", Bill.class);
        assertEquals(theBill.getSponsor().getFullname(),"LAVALLE");
        assertNotNull("The title should not be null",theBill.getTitle());


        // Test for SOBI.D110613.T222123.TXT
        TestHelper.processFileC(env ,file.get(1));
        theBill=(Bill)storage.get("2011/bill/S1462A-2011", Bill.class);
        assertEquals(theBill.getSponsor().getFullname(),"LAVALLE");
        assertNotNull("The title should not be null",theBill.getTitle());


        // Test for SOBI.D110614.T152729.TXT
        TestHelper.processFileC(env ,file.get(10));
        theBill=(Bill)storage.get("2011/bill/S1462A-2011", Bill.class);
        assertEquals(theBill.getSponsor().getFullname(),"LAVALLE");
        assertNotNull("The title should not be null",theBill.getTitle());


        // Test for SOBI.D110209.T170214.TXT,
        TestHelper.processFileC(env ,file.get(17));
        theBill=(Bill)storage.get("2011/bill/S1462A-2011", Bill.class);
        assertEquals(theBill.getSponsor().getFullname(),"LAVALLE");
        assertNotNull("The title should not be null",theBill.getTitle());


        // Test for SOBI.D110209.T171147.TXT
        TestHelper.processFileC(env ,file.get(15));
        theBill=(Bill)storage.get("2011/bill/S1462A-2011", Bill.class);
       //assertEquals(theBill.getSameAs(),"A1415A");  ?? Have to check why its failing? Not written in json file ..why?


        // Test for SOBI.D110614.T195743.TXT
        TestHelper.processFileC(env ,file.get(5));
        theBill=(Bill)storage.get("2011/bill/S1462A-2011", Bill.class);
        String[] voteDate=(theBill.getVotes().get(0).getOid()).split("-");
      String date=voteDate[2];
        // Checking vote Date
      assertEquals(date.equals("2011/06/14"),true);
        Vote vote=processExpectedVote2011S1462A();
      boolean flag=VoteTests.voteCheck(theBill,vote);
        assertEquals(flag,true);


     // Test for SOBI.D110614.T185240.TXT
        TestHelper.processFileC(env ,file.get(13));
        theBill=(Bill)storage.get("2011/bill/S1462A-2011", Bill.class);
        assertEquals(theBill.getSponsor().getFullname(),"LAVALLE");
        assertNotNull("The title should not be null",theBill.getTitle());


        // Test for SOBI.D110614.T192241.TXT
        TestHelper.processFileC(env ,file.get(2));
        theBill=(Bill)storage.get("2011/bill/S1462A-2011", Bill.class);
        assertEquals(theBill.getSponsor().getFullname(),"LAVALLE");
        assertNotNull("The title should not be null",theBill.getTitle());


        // Test for SOBI.D120104.T223233.TXT
        TestHelper.processFileC(env ,file.get(6));
        theBill=(Bill)storage.get("2011/bill/S1462A-2011", Bill.class);
        assertEquals(theBill.getSponsor().getFullname(),"LAVALLE");
        assertNotNull("The title should not be null",theBill.getTitle());
       // assertEquals(theBill.getSameAs(),"A1415A");   ( In json file we dont have sameAs in amendment A??






     // Tests for SOBI.D120125.T130337.TXT
        TestHelper.processFileC(env ,file.get(14));
        theBill=(Bill)storage.get("2011/bill/S1462A-2011", Bill.class);
        assertEquals(theBill.getSponsor().getFullname(),"LAVALLE");
        ArrayList<Person> list=(ArrayList<Person>)theBill.getCoSponsors();
        assertEquals(list.get(0).toString(),"DUANE");




        // Tests for SOBI.D120126.T103841.TXT
        TestHelper.processFileC(env ,file.get(4));
        theBill=(Bill)storage.get("2011/bill/S1462A-2011", Bill.class);
        assertEquals(theBill.getSponsor().getFullname(),"LAVALLE");
        ArrayList<Person> list1=(ArrayList<Person>)theBill.getCoSponsors();
        assertEquals(list1.get(0).toString(),"DUANE");
        assertEquals(list1.get(1).toString(),"MONTGOMERY");

View Full Code Here

TOP

Related Classes of gov.nysenate.openleg.util.Storage

gov.nysenate.openleg.converter.StorageJsonConverter

gov.nysenate.openleg.Environment

gov.nysenate.openleg.model.BaseObject

gov.nysenate.openleg.scripts.admin.RemoveAmendment

gov.nysenate.openleg.scripts.admin.SpotCheck

gov.nysenate.openleg.scripts.MemoCheck

gov.nysenate.openleg.tests.AllTests

gov.nysenate.openleg.tests.Bill2011S1462

gov.nysenate.openleg.tests.TestSetup

java.io.File

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.