Examples of org.apache.oozie.service.EventHandlerService$EventWorker

org.apache.oozie.service.EventHandlerService
Service class that handles the events system - creating events queue, managing configured properties and managing and invoking various event listeners via worker threads

    }


    @Test
    public void testSLAEvents1() throws Exception {
        SLACalculatorMemory slaCalcMemory = new SLACalculatorMemory();
        EventHandlerService ehs = Services.get().get(EventHandlerService.class);
        slaCalcMemory.init(Services.get().getConf());
        WorkflowJobBean job1 = addRecordToWfJobTable(WorkflowJob.Status.PREP, WorkflowInstance.Status.PREP);
        SLARegistrationBean slaRegBean = _createSLARegistration(job1.getId(), AppType.WORKFLOW_JOB);
        slaRegBean.setExpectedStart(new Date(System.currentTimeMillis() - 1 * 1 * 3600 * 1000)); // 1 hour
        slaRegBean.setExpectedDuration(2 * 3600 * 1000);
        slaRegBean.setExpectedEnd(new Date(System.currentTimeMillis() - 1 * 1 * 3600 * 1000)); // 1 hour
        String jobId = slaRegBean.getId();
        slaCalcMemory.addRegistration(jobId, slaRegBean);
        assertEquals(1, slaCalcMemory.size());
        SLASummaryBean slaSummary = SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, jobId);
        assertEquals(SLAStatus.NOT_STARTED, slaSummary.getSLAStatus());
        assertEquals("PREP", slaSummary.getJobStatus());
        slaCalcMemory.updateJobSla(jobId);
        assertEquals(2, ehs.getEventQueue().size());
        slaSummary = SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, jobId);
        // both start miss and end miss (101)
        assertEquals(5, slaSummary.getEventProcessed());
        assertEquals(SLAEvent.EventStatus.END_MISS, slaSummary.getEventStatus());
        assertEquals(SLAStatus.MISS, slaSummary.getSLAStatus());


        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");


        assertEquals(SLAStatus.MISS, slaSummary.getSLAStatus());
        slaCalcMemory.addJobStatus(jobId, WorkflowJob.Status.RUNNING.toString(), EventStatus.STARTED,
                sdf.parse("2012-01-01"), null);
        slaCalcMemory.addJobStatus(jobId, WorkflowJob.Status.SUSPENDED.toString(), EventStatus.SUSPEND,
                sdf.parse("2012-01-01"), null);
        slaSummary = SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, jobId);
        assertEquals(WorkflowJob.Status.SUSPENDED.toString(), slaSummary.getJobStatus());
        assertEquals(5, slaSummary.getEventProcessed());
        slaCalcMemory.addJobStatus(jobId, WorkflowJob.Status.SUCCEEDED.toString(), EventStatus.SUCCESS,
                sdf.parse("2012-01-01"), sdf.parse("2012-01-02"));


        assertEquals(3, ehs.getEventQueue().size());
        slaSummary = SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, jobId);
        // All events processed and actual times stored (1000)
        assertEquals(8, slaSummary.getEventProcessed());
        assertEquals(SLAStatus.MISS, slaSummary.getSLAStatus());
        assertEquals(WorkflowJob.Status.SUCCEEDED.toString(), slaSummary.getJobStatus());

View Full Code Here

    }


    @Test
    public void testSLAEvents2() throws Exception {
        SLACalculatorMemory slaCalcMemory = new SLACalculatorMemory();
        EventHandlerService ehs = Services.get().get(EventHandlerService.class);
        slaCalcMemory.init(Services.get().getConf());


        WorkflowJobBean job1 = addRecordToWfJobTable(WorkflowJob.Status.PREP, WorkflowInstance.Status.PREP);
        SLARegistrationBean slaRegBean = _createSLARegistration(job1.getId(), AppType.WORKFLOW_JOB);
        slaRegBean.setExpectedStart(new Date(System.currentTimeMillis() - 1 * 1 * 3600 * 1000));
        slaRegBean.setExpectedEnd(new Date(System.currentTimeMillis() + 2 * 1 * 3600 * 1000));


        String jobId = slaRegBean.getId();
        slaCalcMemory.addRegistration(jobId, slaRegBean);
        assertEquals(1, slaCalcMemory.size());
        slaCalcMemory.updateJobSla(jobId);
        SLASummaryBean slaSummary = SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, jobId);
        // Duration bit should be processed as expected duration is not set
        assertEquals(3, slaSummary.getEventProcessed());
        // check only start event in queue
        assertEquals(1, ehs.getEventQueue().size());
        ehs.getEventQueue().clear();


        // set back to 1, to make duration event not processed
        slaSummary.setEventProcessed(1);
        SLASummaryQueryExecutor.getInstance().executeUpdate(SLASummaryQuery.UPDATE_SLA_SUMMARY_ALL, slaSummary);


        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
        slaCalcMemory.addJobStatus(jobId, WorkflowJob.Status.SUCCEEDED.toString(), EventStatus.SUCCESS,
                sdf.parse("2012-01-01"), sdf.parse("2012-01-02"));
        slaSummary = SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, jobId);
        // all should be processed
        assertEquals(8, slaSummary.getEventProcessed());
        // check only end event is in queue
        assertEquals(1, ehs.getEventQueue().size());
        ehs.getEventQueue().clear();


        slaSummary.setEventProcessed(1);
        SLASummaryQueryExecutor.getInstance().executeUpdate(SLASummaryQuery.UPDATE_SLA_SUMMARY_ALL, slaSummary);


        slaRegBean = _createSLARegistration("job-2", AppType.WORKFLOW_JOB);
        slaRegBean.setExpectedStart(new Date(System.currentTimeMillis() - 1 * 1 * 3600 * 1000));
        slaRegBean.setExpectedEnd(new Date(System.currentTimeMillis() + 2 * 1 * 3600 * 1000));


        jobId = slaRegBean.getId();
        slaCalcMemory.addRegistration(jobId, slaRegBean);
        assertEquals(1, slaCalcMemory.size());


        slaCalcMemory.addJobStatus(jobId, WorkflowJob.Status.KILLED.toString(), EventStatus.FAILURE, null,
                sdf.parse("2012-01-02"));
        slaSummary = SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, jobId);
        // Actual start null, so all events processed
        assertEquals(8, slaSummary.getEventProcessed());
        assertEquals(1, ehs.getEventQueue().size());
        assertNull(slaSummary.getActualStart());
        assertEquals(sdf.parse("2012-01-02"), slaSummary.getActualEnd());
        assertEquals(SLAStatus.MISS, slaSummary.getSLAStatus());
        assertEquals(SLAEvent.EventStatus.END_MISS, slaSummary.getEventStatus());
    }

View Full Code Here

    }


    @Test
    public void testDuplicateStartMiss() throws Exception {
        // test start-miss
        EventHandlerService ehs = Services.get().get(EventHandlerService.class);
        SLACalculatorMemory slaCalcMemory = new SLACalculatorMemory();
        slaCalcMemory.init(Services.get().getConf());
        WorkflowJobBean job1 = addRecordToWfJobTable(WorkflowJob.Status.PREP, WorkflowInstance.Status.PREP);
        SLARegistrationBean slaRegBean = _createSLARegistration(job1.getId(), AppType.WORKFLOW_JOB);
        Date startTime = new Date(System.currentTimeMillis() - 1 * 1 * 3600 * 1000); // 1 hour back
        slaRegBean.setExpectedStart(startTime);
        slaRegBean.setExpectedDuration(3600 * 1000);
        slaRegBean.setExpectedEnd(new Date(System.currentTimeMillis() + 1 * 1 * 3600 * 1000)); // 1 hour ahead
        String jobId = slaRegBean.getId();
        slaCalcMemory.addRegistration(slaRegBean.getId(), slaRegBean);
        slaCalcMemory.updateJobSla(jobId);
        SLASummaryBean slaSummary = SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, jobId);


        assertEquals(1, slaSummary.getEventProcessed());
        assertEquals(SLAStatus.NOT_STARTED, slaSummary.getSLAStatus());
        slaCalcMemory.addJobStatus(jobId, WorkflowJob.Status.RUNNING.toString(), EventStatus.STARTED,
                new Date(System.currentTimeMillis()), null);
        slaCalcMemory.updateJobSla(jobId);
        slaSummary = SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, jobId);


        assertEquals(1, slaSummary.getEventProcessed());
        assertEquals(SLAStatus.IN_PROCESS, slaSummary.getSLAStatus());
        assertEquals(WorkflowJob.Status.RUNNING.toString(), slaSummary.getJobStatus());
        assertEquals(1, ehs.getEventQueue().size());
    }

View Full Code Here

        assertEquals(1, ehs.getEventQueue().size());
    }


    @Test
    public void testDuplicateEndMiss() throws Exception {
        EventHandlerService ehs = Services.get().get(EventHandlerService.class);
        SLACalculatorMemory slaCalcMemory = new SLACalculatorMemory();
        slaCalcMemory.init(Services.get().getConf());
        WorkflowJobBean job1 = addRecordToWfJobTable(WorkflowJob.Status.RUNNING, WorkflowInstance.Status.RUNNING);
        SLARegistrationBean slaRegBean = _createSLARegistration(job1.getId(), AppType.WORKFLOW_JOB);
        Date startTime = new Date(System.currentTimeMillis() + 1 * 1 * 3600 * 1000); // 1 hour ahead
        slaRegBean.setExpectedStart(startTime);
        slaRegBean.setExpectedDuration(3600 * 1000);
        slaRegBean.setExpectedEnd(new Date(System.currentTimeMillis() - 1 * 1 * 3600 * 1000)); // 1 hour back
        String jobId = slaRegBean.getId();
        slaCalcMemory.addRegistration(slaRegBean.getId(), slaRegBean);
        slaCalcMemory.updateJobSla(jobId);
        SLASummaryBean slaSummary = SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, jobId);
        slaRegBean = SLARegistrationQueryExecutor.getInstance().get(SLARegQuery.GET_SLA_REG_ALL, jobId);
        assertNotNull(slaRegBean.getCreatedTimestamp());
        assertEquals(slaRegBean.getCreatedTimestamp(), slaSummary.getCreatedTimestamp());
        // Only end sla should be processed (100)
        assertEquals(4, slaSummary.getEventProcessed());
        slaCalcMemory.updateJobSla(jobId);
        slaSummary = SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, jobId);
        assertEquals(4, slaSummary.getEventProcessed());
        assertEquals(SLAStatus.MISS, slaSummary.getSLAStatus());
        slaCalcMemory.addJobStatus(jobId, WorkflowJob.Status.SUCCEEDED.toString(), EventStatus.SUCCESS,
                new Date(System.currentTimeMillis()), new Date(System.currentTimeMillis() + 1 * 1 * 3600 * 1000));
        slaSummary = SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, jobId);


        // Only Duration sla should be processed as end is already processed
        // (110)
        assertEquals(6, slaSummary.getEventProcessed());
        assertEquals(SLAStatus.MISS, slaSummary.getSLAStatus());
        // Recieve start event
        assertTrue(slaCalcMemory.addJobStatus(jobId, WorkflowJob.Status.RUNNING.toString(), EventStatus.STARTED,
                new Date(System.currentTimeMillis()), new Date(System.currentTimeMillis() + 1 * 1 * 3600 * 1000)));
        slaSummary = SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, jobId);
        // Start event received so all bits should be processed (111)
        assertEquals(8, slaSummary.getEventProcessed());
        assertEquals(SLAStatus.MISS, slaSummary.getSLAStatus());
        assertEquals(0, slaCalcMemory.size());
        assertEquals(3, ehs.getEventQueue().size());


    }

View Full Code Here

        assertEquals(3, ehs.getEventQueue().size());


    }


    public void testSLAHistorySet() throws Exception {
            EventHandlerService ehs = Services.get().get(EventHandlerService.class);
            SLACalculatorMemory slaCalcMemory = new SLACalculatorMemory();
            slaCalcMemory.init(Services.get().getConf());
            WorkflowJobBean job1 = addRecordToWfJobTable(WorkflowJob.Status.PREP, WorkflowInstance.Status.PREP);
            SLARegistrationBean slaRegBean = _createSLARegistration(job1.getId(), AppType.WORKFLOW_JOB);
            Date startTime = new Date(System.currentTimeMillis() - 1 * 1 * 3600 * 1000);
            slaRegBean.setExpectedStart(startTime); // 1 hour back
            slaRegBean.setExpectedDuration(1000);
            slaRegBean.setExpectedEnd(new Date(System.currentTimeMillis() - 1 * 1 * 3600 * 1000));
            String jobId = slaRegBean.getId();
            slaCalcMemory.addRegistration(slaRegBean.getId(), slaRegBean);
            slaCalcMemory.updateJobSla(jobId);
            slaCalcMemory.addJobStatus(jobId, WorkflowJob.Status.RUNNING.toString(), EventStatus.STARTED, new Date(
                    System.currentTimeMillis() - 3600 * 1000), null);
            slaCalcMemory.updateJobSla(jobId);
            SLASummaryBean slaSummary = SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, jobId);
            // The actual end times are not stored, but sla's processed so (111)
            assertEquals(7, slaSummary.getEventProcessed());
            // Moved from map to history set
            assertEquals(0, slaCalcMemory.size());
            // Add terminal state event so actual end time is stored
            slaCalcMemory.addJobStatus(jobId, WorkflowJob.Status.SUCCEEDED.toString(), EventStatus.SUCCESS, new Date(
                    System.currentTimeMillis() - 3600 * 1000), new Date(System.currentTimeMillis()));
            slaSummary = SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, jobId);
            // The actual times are stored, so event processed(1000)
            assertEquals(8, slaSummary.getEventProcessed());
            assertEquals(3, ehs.getEventQueue().size());
    }

View Full Code Here

        bean.setAppType(appType);
        return bean;
    }


    public void testHistoryPurge() throws Exception{
        EventHandlerService ehs = Services.get().get(EventHandlerService.class);
        SLACalculatorMemory slaCalcMemory = new SLACalculatorMemory();
        slaCalcMemory.init(Services.get().getConf());
        WorkflowJobBean job1 = addRecordToWfJobTable(WorkflowJob.Status.PREP, WorkflowInstance.Status.PREP);
        SLARegistrationBean slaRegBean = _createSLARegistration(job1.getId(), AppType.WORKFLOW_JOB);
        Date startTime = new Date(System.currentTimeMillis() - 1 * 1 * 3600 * 1000);

View Full Code Here

        super.tearDown();
    }


    @Test
    public void testMemoryEventQueueBasic() throws Exception {
        EventHandlerService ehs = Services.get().get(EventHandlerService.class);
        assertNotNull(ehs);
        EventQueue eventQ = ehs.getEventQueue();
        assertNotNull(eventQ);
        assertTrue(eventQ instanceof MemoryEventQueue); //default
    }

View Full Code Here

        assertTrue(eventQ instanceof MemoryEventQueue); //default
    }


    @Test
    public void testQueueOperations() throws Exception {
        EventHandlerService ehs = Services.get().get(EventHandlerService.class);
        EventQueue eventQ = ehs.getEventQueue();
        assertEquals(eventQ.size(), 0);
        assertEquals(eventQ.getBatchSize(), 3);


        // create some events to enqueue
        WorkflowJobEvent wfEvent = new WorkflowJobEvent("1234-W", "1234-C", WorkflowJob.Status.RUNNING, getTestUser(),
                "myapp", null, null);
        for (int i = 0; i < 10; i++) {
            ehs.queueEvent(wfEvent);
        }
        assertEquals(eventQ.size(), 10);


        // test single threads polling from queue
        int numThreads = 1;
        Thread[] thread = new Thread[numThreads];
        for (int i = 0; i < numThreads; i++) {
            thread[i] = new Thread(ehs.new EventWorker());
            thread[i].run();
        }
        assertEquals(eventQ.size(), 7); // n(events) - n(batch) i.e.
                                                  // 10-3 = 7


        // restore events count to 10
        for (int i = 0; i < 3; i++) {
            ehs.queueEvent(wfEvent);
        }
        assertEquals(eventQ.size(), 10);
        // test two threads polling concurrently from queue
        numThreads = 2;
        thread = new Thread[numThreads];
        for (int i = 0; i < numThreads; i++) {
            thread[i] = new Thread(ehs.new EventWorker());
            thread[i].run();
        }
        assertEquals(eventQ.size(), 4); // n(events) - n(batch)*n(threads)
                                                  // i.e. 10 - 3*2 = 4


        // enqueue events again
        for (int i = 0; i < 6; i++) {
            ehs.queueEvent(wfEvent);
        }
        assertEquals(eventQ.size(), 10);
        // test the 2 threads draining repeatedly (mimicking SchedulerService)
        // from queue
        int repetition = 3;

View Full Code Here

    public void testUpdateSLA() throws Exception {
        SLAService slas = Services.get().get(SLAService.class);
        assertNotNull(slas);
        assertTrue(SLAService.isEnabled());


        EventHandlerService ehs = Services.get().get(EventHandlerService.class);
        // test start-miss
        SLARegistrationBean sla1 = _createSLARegistration("job-1", AppType.WORKFLOW_JOB);
        sla1.setExpectedStart(new Date(System.currentTimeMillis() - 1 * 1 * 3600 * 1000)); //1 hour back
        sla1.setExpectedEnd(new Date(System.currentTimeMillis() - 1 * 1 * 3600 * 1000)); //1 hour back
        sla1.setExpectedDuration(10 * 60 * 1000); //10 mins
        slas.addRegistrationEvent(sla1);
        assertEquals(1, slas.getSLACalculator().size());
        slas.runSLAWorker();
        ehs.new EventWorker().run();
        assertEventNoDuplicates(output.toString(), "Sla START - MISS!!!");
        assertEventNoDuplicates(output.toString(), "Sla END - MISS!!!");
        output.setLength(0);


        // test different jobs and events start-met and end-miss
        sla1 = _createSLARegistration("job-2", AppType.WORKFLOW_JOB);
        sla1.setExpectedStart(new Date(System.currentTimeMillis() + 1 * 3600 * 1000)); //1 hour ahead
        sla1.setExpectedEnd(new Date(System.currentTimeMillis() + 2 * 3600 * 1000)); //2 hours ahead
        slas.addRegistrationEvent(sla1);
        slas.addStatusEvent(sla1.getId(), WorkflowJob.Status.RUNNING.name(), EventStatus.STARTED, new Date(),
                new Date());
        SLARegistrationBean sla2 = _createSLARegistration("job-3", AppType.COORDINATOR_JOB);
        sla2.setExpectedStart(new Date(System.currentTimeMillis() + 1 * 3600 * 1000)); //1 hour ahead only for testing
        sla2.setExpectedEnd(new Date(System.currentTimeMillis() - 2 * 3600 * 1000)); //2 hours back
        sla2.setExpectedDuration(10); //to process duration too
        slas.addRegistrationEvent(sla2);
        assertEquals(3, slas.getSLACalculator().size());
        Date startTime = new Date();
        slas.addStatusEvent(sla2.getId(), CoordinatorJob.Status.RUNNING.name(), EventStatus.STARTED, startTime,
                null);
        slas.addStatusEvent(sla2.getId(), CoordinatorJob.Status.SUCCEEDED.name(), EventStatus.SUCCESS, startTime,
                new Date());
        slas.runSLAWorker();
        ehs.new EventWorker().run();
        assertTrue(output.toString().contains(sla1.getId() + " Sla START - MET!!!"));
        assertTrue(output.toString().contains(sla2.getId() + " Sla END - MISS!!!"));
        assertTrue(output.toString().contains(sla2.getId() + " Sla DURATION - MET!!!"));
        output.setLength(0);


        // test same job multiple events (start-miss, end-miss) through regular check
        WorkflowJobBean job4 = addRecordToWfJobTable(WorkflowJob.Status.KILLED, WorkflowInstance.Status.KILLED);
        sla2 = _createSLARegistration(job4.getId(), AppType.WORKFLOW_JOB);
        sla2.setExpectedStart(new Date(System.currentTimeMillis() - 2 * 3600 * 1000)); //2 hours back
        sla2.setExpectedEnd(new Date(System.currentTimeMillis() - 1 * 3600 * 1000)); //1 hour back
        slas.addRegistrationEvent(sla2);
        assertEquals(3, slas.getSLACalculator().size()); // tests job slaProcessed == 7 removed from map
        slas.runSLAWorker();
        ehs.new EventWorker().run();
        assertTrue(output.toString().contains(sla2.getId() + " Sla START - MISS!!!"));
        assertTrue(output.toString().contains(sla2.getId() + " Sla END - MISS!!!"));
        output.setLength(0);
        // As expected duration is not set, duration shall be processed and job removed from map
        assertEquals(2, slas.getSLACalculator().size());
        // test same job multiple events (start-met, end-met) through job status event
        sla1 = _createSLARegistration("action@1", AppType.COORDINATOR_ACTION);
        sla1.setExpectedStart(new Date(System.currentTimeMillis() + 1 * 3600 * 1000)); //1 hour ahead
        sla1.setExpectedEnd(new Date(System.currentTimeMillis() + 2 * 3600 * 1000)); //2 hours ahead
        slas.addRegistrationEvent(sla1);
        assertEquals(3, slas.getSLACalculator().size());
        slas.addStatusEvent(sla1.getId(), CoordinatorAction.Status.RUNNING.name(), EventStatus.STARTED, new Date(),
                new Date());
        slas.addStatusEvent(sla1.getId(), CoordinatorAction.Status.SUCCEEDED.name(), EventStatus.SUCCESS,
                new Date(), new Date());
        slas.runSLAWorker();
        assertEquals(2, ehs.getEventQueue().size());
        ehs.new EventWorker().run();
        assertTrue(output.toString().contains(sla1.getId() + " Sla START - MET!!!"));
        assertTrue(output.toString().contains(sla1.getId() + " Sla END - MET!!!"));
    }

View Full Code Here

    }


    @Test
    public void testEndMissDBConfirm() throws Exception {
        SLAService slas = Services.get().get(SLAService.class);
        EventHandlerService ehs = Services.get().get(EventHandlerService.class);
        JPAService jpaService = Services.get().get(JPAService.class);


        // CASE 1: positive test WF job
        WorkflowJobBean job1 = addRecordToWfJobTable(WorkflowJob.Status.PREP, WorkflowInstance.Status.PREP);
        SLARegistrationBean sla = _createSLARegistration(job1.getId(), AppType.WORKFLOW_JOB);

View Full Code Here

0 1 2

TOP

Related Classes of org.apache.oozie.service.EventHandlerService$EventWorker

com.ngdata.hbaseindexer.model.api.IndexerDefinition

com.ngdata.hbaseindexer.model.api.IndexerModelEvent

org.apache.hadoop.conf.Configuration

org.apache.oozie.client.event.Event.MessageType

org.apache.oozie.event.MemoryEventQueue

org.apache.oozie.event.TestEventQueue

org.apache.oozie.sla.TestSLACalculatorMemory

org.apache.oozie.sla.TestSLAJobEventListener

org.apache.oozie.sla.TestSLAService

org.lilyproject.repository.model.api.RepositoryDefinition

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.