Package azkaban.common.jobs

Examples of azkaban.common.jobs.Job


        // Check if we have already loaded this job
        if(loadedJobs.containsKey(jobName))
            return loadedJobs.get(jobName);

        Job job = _factory.apply(desc);

        if(ignoreDependencies || !desc.hasDependencies()) {
            loadedJobs.put(jobName, job);
            return job;
        } else {
View Full Code Here


    @Override
    public Job apply(JobDescriptor jobDescriptor)
    {
     
      Job job;
      try {
        String jobType = jobDescriptor.getJobType();
        if (jobType == null || jobType.length() == 0) {
           /*throw an exception when job name is null or empty*/
          throw new JobExecutionException (
                                           String.format("The 'type' parameter for job[%s] is null or empty", jobDescriptor));
        }
        Class<? extends Object> executorClass = _jobToClass.get(jobType);

        if (executorClass == null) {
            throw new JobExecutionException(
                    String.format(
                            "Could not construct job[%s] of type[%s].",
                            jobDescriptor,
                            jobType
                    ));
        }
       
        job = (Job)Utils.callConstructor(executorClass, jobDescriptor);

        // wrap up job in retrying proxy if necessary
        if(jobDescriptor.getRetries() > 0)
            job = new RetryingJob(job, jobDescriptor.getRetries(), jobDescriptor.getRetryBackoffMs());

        // Group Lock List
        ArrayList<JobLock> jobLocks = new ArrayList<JobLock>();

        // If this job requires work permits wrap it in a resource throttler
        if(jobDescriptor.getNumRequiredPermits() > 0) {
            PermitLock permits = _permitManager.getNamedPermit("default",
                                                               jobDescriptor.getNumRequiredPermits());
            if(permits == null) {
                throw new RuntimeException("Job " + jobDescriptor.getId() + " requires non-existant default");
            } else if(permits.getDesiredNumPermits() > permits.getTotalNumberOfPermits()) {
                throw new RuntimeException(
                        String.format(
                                "Job %s requires %s but azkaban only has a total of %s permits, so this job cannot ever run.",
                                jobDescriptor.getId(),
                                permits.getTotalNumberOfPermits(),
                                permits.getDesiredNumPermits()
                        )
                );
            } else {
                jobLocks.add(permits);
            }
        }

        if(jobDescriptor.getReadResourceLocks() != null) {
            List<String> readLocks = jobDescriptor.getReadResourceLocks();
            for(String resource: readLocks) {
                jobLocks.add(_readWriteLockManager.getReadLock(resource));
            }
        }
        if(jobDescriptor.getWriteResourceLocks() != null) {
            List<String> writeLocks = jobDescriptor.getWriteResourceLocks();
            for(String resource: writeLocks) {
                jobLocks.add(_readWriteLockManager.getWriteLock(resource));
            }
        }

        if(jobLocks.size() > 0) {
            // Group lock
            GroupLock groupLock = new GroupLock(jobLocks);
            job = new ResourceThrottledJob(job, groupLock);
        }

      }
      catch (Exception e) {
          job = new InitErrorJob(jobDescriptor.getId(), e);
      }

        // wrap up job in logging proxy
        if (jobDescriptor.getLoggerPattern() != null) {
          job = new LoggingJob(_logDir, job, job.getId(), jobDescriptor.getLoggerPattern())
        }
        else {
          job = new LoggingJob(_logDir, job, job.getId())
        }
       
        return job;
    }
View Full Code Here

                    }
                },
                String.format("%s thread-%s", job.getId(), threadCounter.getAndIncrement())
        );

        Job currJob = job;
        while (true) {
            if (currJob instanceof DelegatingJob) {
                currJob = ((DelegatingJob) currJob).getInnerJob();
            }
            else {
View Full Code Here

    @Test
    public void testSanity() throws Throwable
    {
        final CountDownLatch completionLatch = new CountDownLatch(1);

        final Job mockJob = EasyMock.createMock(Job.class);
        final Props overrideProps = new Props();
        final IndividualJobExecutableFlow executableFlow = new IndividualJobExecutableFlow("blah", "blah", jobManager);

        EasyMock.expect(jobManager.loadJob("blah", overrideProps, true)).andReturn(mockJob).once();
        EasyMock.expect(mockJob.getId()).andReturn("success Job").once();

        mockJob.run();
        EasyMock.expectLastCall().andAnswer(new IAnswer<Void>()
        {
            @Override
            public Void answer() throws Throwable
            {
                Assert.assertEquals(Status.RUNNING, executableFlow.getStatus());

                return null;
            }
        }).once();

        final Props returnProps = new Props();
        EasyMock.expect(mockJob.getJobGeneratedProperties()).andReturn(returnProps).once();

        EasyMock.replay(mockJob, jobManager);

        Assert.assertEquals(Status.READY, executableFlow.getStatus());
View Full Code Here

    public void testFailure() throws Throwable
    {
               
        final CountDownLatch completionLatch = new CountDownLatch(1);

        final Job mockJob = EasyMock.createMock(Job.class);
        final Props overrideProps = new Props();
        final IndividualJobExecutableFlow executableFlow = new IndividualJobExecutableFlow("blah", "blah", jobManager);

        EasyMock.expect(jobManager.loadJob("blah", overrideProps, true)).andReturn(mockJob).once();
        EasyMock.expect(mockJob.getId()).andReturn("blah").times(1);
       
       
        mockJob.run();
        EasyMock.expectLastCall().andAnswer(new IAnswer<Void>()
        {
            @Override
            public Void answer() throws Throwable
            {
View Full Code Here

    public void testAllExecuteCallbacksCalledOnSuccess() throws Throwable
    {
        final CountDownLatch firstCallbackLatch = new CountDownLatch(1);
        final CountDownLatch secondCallbackLatch = new CountDownLatch(1);

        final Job mockJob = EasyMock.createMock(Job.class);
        final Props overrideProps = new Props();
        final IndividualJobExecutableFlow executableFlow = new IndividualJobExecutableFlow("blah", "blah", jobManager);

        EasyMock.expect(jobManager.loadJob("blah", overrideProps, true)).andReturn(mockJob).once();
        EasyMock.expect(mockJob.getId()).andReturn("success Job").once();

        mockJob.run();
        EasyMock.expectLastCall().andAnswer(new IAnswer<Void>()
        {
            @Override
            public Void answer() throws Throwable
            {
                Assert.assertEquals(Status.RUNNING, executableFlow.getStatus());

                return null;
            }
        }).once();

        final Props returnProps = new Props();
        EasyMock.expect(mockJob.getJobGeneratedProperties()).andReturn(returnProps).once();

        EasyMock.replay(mockJob, jobManager);

        Assert.assertEquals(Status.READY, executableFlow.getStatus());
View Full Code Here

    public void testAllExecuteCallbacksCalledOnFailure() throws Throwable
    {
        final CountDownLatch firstCallbackLatch = new CountDownLatch(1);
        final CountDownLatch secondCallbackLatch = new CountDownLatch(1);

        final Job mockJob = EasyMock.createMock(Job.class);
        final Props overrideProps = new Props();
        final IndividualJobExecutableFlow executableFlow = new IndividualJobExecutableFlow("blah", "blah", jobManager);

        EasyMock.expect(jobManager.loadJob("blah", overrideProps, true)).andReturn(mockJob).once();
        EasyMock.expect(mockJob.getId()).andReturn("blah").times(1);

        mockJob.run();
        EasyMock.expectLastCall().andThrow(theException).once();

        EasyMock.replay(mockJob, jobManager);

        Assert.assertEquals(Status.READY, executableFlow.getStatus());
View Full Code Here

    @Test
    public void testReset() throws Exception
    {
        final CountDownLatch completionLatch = new CountDownLatch(1);

        final Job mockJob = EasyMock.createMock(Job.class);
        final Props overrideProps = new Props();
        final IndividualJobExecutableFlow executableFlow = new IndividualJobExecutableFlow("blah", "blah", jobManager);

        final Props firstProps = new Props();
        final Props secondProps = new Props();

        EasyMock.expect(jobManager.loadJob("blah", overrideProps, true)).andReturn(mockJob).once();
        EasyMock.expect(mockJob.getId()).andReturn("success Job").once();
        EasyMock.expect(mockJob.getJobGeneratedProperties()).andReturn(firstProps).once();

        mockJob.run();
        EasyMock.expectLastCall().andAnswer(new IAnswer<Void>()
        {
            @Override
            public Void answer() throws Throwable
            {
                Assert.assertEquals(Status.RUNNING, executableFlow.getStatus());

                return null;
            }
        }).once();

        EasyMock.replay(mockJob, jobManager);

        Assert.assertEquals(Status.READY, executableFlow.getStatus());

        executableFlow.execute(
                overrideProps,
                new FlowCallback()
                {
                    @Override
                    public void progressMade()
                    {
                        assertionViolated.set(true);
                        reason = String.format("progressMade() shouldn't actually be called.");
                    }

                    @Override
                    public void completed(Status status)
                    {
                        completionLatch.countDown();
                        if (Status.SUCCEEDED != status) {
                            assertionViolated.set(true);
                            reason = String.format("In executableFlow Callback: status[%s] != Status.SUCCEEDED", status);
                        }
                    }
                });

        completionLatch.await(1000, TimeUnit.MILLISECONDS);
        Assert.assertEquals(Status.SUCCEEDED, executableFlow.getStatus());
        Assert.assertEquals(emptyExceptions, executableFlow.getExceptions());
        Assert.assertEquals(firstProps, executableFlow.getReturnProps());

        EasyMock.verify(mockJob, jobManager);
        EasyMock.reset(mockJob, jobManager);

        final CountDownLatch completionLatch2 = new CountDownLatch(1);

        Assert.assertTrue("Expected to be able to reset the executableFlow.", executableFlow.reset());
        Assert.assertEquals(Status.READY, executableFlow.getStatus());
        Assert.assertEquals(null, executableFlow.getParentProps());
        Assert.assertEquals(null, executableFlow.getReturnProps());

        EasyMock.expect(jobManager.loadJob("blah", overrideProps, true)).andReturn(mockJob).once();
        EasyMock.expect(mockJob.getId()).andReturn("success Job").once();
        EasyMock.expect(mockJob.getJobGeneratedProperties()).andReturn(secondProps).once();

        mockJob.run();
        EasyMock.expectLastCall().andAnswer(new IAnswer<Void>()
        {
            @Override
            public Void answer() throws Throwable
            {
View Full Code Here

    @Test
    public void testResetWithFailedJob() throws Exception
    {
        final CountDownLatch completionLatch = new CountDownLatch(1);

        final Job mockJob = EasyMock.createMock(Job.class);
        final Props overrideProps = new Props();
        final IndividualJobExecutableFlow executableFlow = new IndividualJobExecutableFlow("blah", "blah", jobManager);
        executableFlow.setStatus(Status.FAILED);

        Assert.assertTrue("Should be able to reset the flow.", executableFlow.reset());

        EasyMock.expect(jobManager.loadJob("blah", overrideProps, true)).andReturn(mockJob).once();
        EasyMock.expect(mockJob.getId()).andReturn("success Job").once();

        mockJob.run();
        EasyMock.expectLastCall().andAnswer(new IAnswer<Void>()
        {
            @Override
            public Void answer() throws Throwable
            {
                Assert.assertEquals(Status.RUNNING, executableFlow.getStatus());

                return null;
            }
        }).once();

        EasyMock.expect(mockJob.getJobGeneratedProperties()).andReturn(new Props()).once();

        EasyMock.replay(mockJob, jobManager);

        Assert.assertEquals(Status.READY, executableFlow.getStatus());
View Full Code Here

    public void testCancel() throws Exception
    {
        final CountDownLatch cancelLatch = new CountDownLatch(1);
        final CountDownLatch runLatch = new CountDownLatch(1);

        final Job mockJob = EasyMock.createMock(Job.class);
        final Props overrideProps = new Props();
        final IndividualJobExecutableFlow executableFlow = new IndividualJobExecutableFlow("blah", "blah", jobManager);

        Assert.assertTrue("Should be able to reset the flow.", executableFlow.reset());

        EasyMock.expect(mockJob.getId()).andReturn("blah").once();
        mockJob.run();
        EasyMock.expect(jobManager.loadJob("blah", overrideProps, true)).andAnswer(new IAnswer<Job>()
        {
            @Override
            public Job answer() throws Throwable
            {
                cancelLatch.countDown();
                runLatch.await();

                return mockJob;
            }
        }).once();

        EasyMock.expect(mockJob.getJobGeneratedProperties()).andReturn(new Props()).once();
       
        EasyMock.replay(mockJob, jobManager);

        new Thread(new Runnable()
        {
View Full Code Here

TOP

Related Classes of azkaban.common.jobs.Job

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.