Examples of org.apache.oozie.service.CallableQueueService$CompositeCallable

org.apache.oozie.service.CallableQueueService
The callable queue service queues {@link XCallable}s for asynchronous execution.
Callables can be queued for immediate execution or for delayed execution (some time in the future).
Callables are consumed from the queue for execution based on their priority.
When the queues (for immediate execution and for delayed execution) are full, the callable queue service stops queuing callables.
A thread-pool is used to execute the callables asynchronously.
The following configuration parameters control the callable queue service:
{@link #CONF_QUEUE_SIZE} size of the immediate execution queue. Defaulf value is 10000.
{@link #CONF_THREADS} number of threads in the thread-pool used for asynchronous command execution. When this numberof threads is reached, commands remain the queue until threads become available. Sets up a priority queue for the execution of Commands via a ThreadPool. Sets up a Delayed Queue to handle actions which will be ready for execution sometime in the future.

        final MyCoordKillXCommand callable2 = new MyCoordKillXCommand(job.getId(), 100);
        final MyCoordKillXCommand callable3 = new MyCoordKillXCommand(job.getId(), 100);


        List<MyCoordKillXCommand> callables = Arrays.asList(callable1, callable2, callable3);


        CallableQueueService queueservice = services.get(CallableQueueService.class);


        for (MyCoordKillXCommand c : callables) {
            queueservice.queue(c);
        }


        waitFor(500, new Predicate() {
            public boolean evaluate() throws Exception {
                return callable1.executed != 0 && callable2.executed == 0 && callable3.executed == 0;

View Full Code Here

                    ret = execute();
                    executeCron.stop();
                    instrumentation.addCron(INSTRUMENTATION_GROUP, getName() + ".execute", executeCron);
                }
                if (commandQueue != null) {
                    CallableQueueService callableQueueService = Services.get().get(CallableQueueService.class);
                    for (Map.Entry<Long, List<XCommand<?>>> entry : commandQueue.entrySet()) {
                        LOG.debug("Queuing [{0}] commands with delay [{1}]ms", entry.getValue().size(), entry.getKey());
                        if (!callableQueueService.queueSerial(entry.getValue(), entry.getKey())) {
                            LOG.warn("Could not queue [{0}] commands with delay [{1}]ms, queue full", entry.getValue()
                                    .size(), entry.getKey());
                        }
                    }
                }

View Full Code Here

        final MyCoordActionInputCheckXCommand callable2 = new MyCoordActionInputCheckXCommand(action1.getId(), 100);
        final MyCoordActionInputCheckXCommand callable3 = new MyCoordActionInputCheckXCommand(action1.getId(), 100);


        List<MyCoordActionInputCheckXCommand> callables = Arrays.asList(callable1, callable2, callable3);


        CallableQueueService queueservice = services.get(CallableQueueService.class);


        for (MyCoordActionInputCheckXCommand c : callables) {
            queueservice.queue(c);
        }


        waitFor(200, new Predicate() {
            public boolean evaluate() throws Exception {
                return callable1.executed != 0 && callable2.executed == 0 && callable3.executed == 0;

View Full Code Here

        final MyCoordKillXCommand callable2 = new MyCoordKillXCommand(job.getId(), 100);
        final MyCoordKillXCommand callable3 = new MyCoordKillXCommand(job.getId(), 100);


        List<MyCoordKillXCommand> callables = Arrays.asList(callable1, callable2, callable3);


        CallableQueueService queueservice = services.get(CallableQueueService.class);


        for (MyCoordKillXCommand c : callables) {
            queueservice.queue(c);
        }


        waitFor(1000, new Predicate() {
            public boolean evaluate() throws Exception {
                return callable1.executed != 0 && callable2.executed == 0 && callable3.executed == 0;

View Full Code Here

        final MyCoordActionInputCheckXCommand callable2 = new MyCoordActionInputCheckXCommand(action1.getId(), 100, "2");
        final MyCoordActionInputCheckXCommand callable3 = new MyCoordActionInputCheckXCommand(action1.getId(), 100, "3");


        List<MyCoordActionInputCheckXCommand> callables = Arrays.asList(callable1, callable2, callable3);


        CallableQueueService queueservice = services.get(CallableQueueService.class);


        for (MyCoordActionInputCheckXCommand c : callables) {
            queueservice.queue(c);
        }


        waitFor(200, new Predicate() {
            public boolean evaluate() throws Exception {
                return callable1.executed != 0 && callable2.executed == 0 && callable3.executed == 0;

View Full Code Here

        final MyCoordActionInputCheckXCommand callable2 = new MyCoordActionInputCheckXCommand(action1.getId(), 100, "2");
        final MyCoordActionInputCheckXCommand callable3 = new MyCoordActionInputCheckXCommand(action1.getId(), 100, "3");


        List<MyCoordActionInputCheckXCommand> callables = Arrays.asList(callable1, callable2, callable3);


        CallableQueueService queueservice = services.get(CallableQueueService.class);


        for (MyCoordActionInputCheckXCommand c : callables) {
            queueservice.queue(c);
        }


        waitFor(200, new Predicate() {
            public boolean evaluate() throws Exception {
                return callable1.executed != 0 && callable2.executed == 0 && callable3.executed == 0;

View Full Code Here

                    ret = execute();
                    executeCron.stop();
                    instrumentation.addCron(INSTRUMENTATION_GROUP, getName() + ".execute", executeCron);
                }
                if (commandQueue != null) {
                    CallableQueueService callableQueueService = Services.get().get(CallableQueueService.class);
                    for (Map.Entry<Long, List<XCommand<?>>> entry : commandQueue.entrySet()) {
                        LOG.debug("Queuing [{0}] commands with delay [{1}]ms", entry.getValue().size(), entry.getKey());
                        if (!callableQueueService.queueSerial(entry.getValue(), entry.getKey())) {
                            LOG.warn("Could not queue [{0}] commands with delay [{1}]ms, queue full", entry.getValue()
                                    .size(), entry.getKey());
                        }
                    }
                }

View Full Code Here

     * Check for the existence of interrupts for the same lock key
     * Execute them if exist.
     *
     */
    protected void executeInterrupts() {
        CallableQueueService callableQueueService = Services.get().get(CallableQueueService.class);
        // getting all the list of interrupts to be executed
        Set<XCallable<?>> callables = callableQueueService.checkInterrupts(this.getEntityKey());


        if (callables != null) {
            // executing the list of interrupts in the given order of insertion
            // in the list
            for (XCallable<?> callable : callables) {

View Full Code Here

                if (timeout) {
                    unregisterMissingDependencies(actionDep.getMissingDependencies(), actionId);
                }
            }
            catch (Exception e) {
                final CallableQueueService callableQueueService = Services.get().get(CallableQueueService.class);
                if (isTimeout()) {
                    LOG.debug("Queueing timeout command");
                    // XCommand.queue() will not work when there is a Exception
                    callableQueueService.queue(new CoordActionTimeOutXCommand(coordAction, coordJob.getUser(), coordJob.getAppName()));
                    unregisterMissingDependencies(Arrays.asList(missingDepsArray), actionId);
                }
                else if (coordAction.getMissingDependencies() != null
                        && coordAction.getMissingDependencies().length() > 0) {
                    // Queue again on exception as RecoveryService will not queue this again with
                    // the action being updated regularly by CoordActionInputCheckXCommand
                    callableQueueService.queue(new CoordPushDependencyCheckXCommand(coordAction.getId(),
                            registerForNotification, removeAvailDependencies),
                            Services.get().getConf().getInt(RecoveryService.CONF_COORD_OLDER_THAN, 600) * 1000);
                }
                throw new CommandException(ErrorCode.E1021, e.getMessage(), e);
            }

View Full Code Here

        }
        catch (Exception e) {
            assertTrue(e.getMessage().contains("NoSuchObjectException"));
        }
        // Nothing should be queued as there are no pull dependencies
        CallableQueueService callableQueueService = Services.get().get(CallableQueueService.class);
        assertEquals(0, callableQueueService.getQueueDump().size());


        // Nothing should be queued as there are no pull missing dependencies
        // but only push missing deps are there
        new CoordActionInputCheckXCommand(actionId, job.getId()).call();
        callableQueueService = Services.get().get(CallableQueueService.class);
        assertEquals(0, callableQueueService.getQueueDump().size());


        setMissingDependencies(actionId, newHCatDependency1);
        try {
            new CoordPushDependencyCheckXCommand(actionId, true).call();
            fail();
        }
        catch (Exception e) {
            assertTrue(e.getMessage().contains("NoSuchObjectException"));
        }
        // Should be requeued at the recovery service interval
        final List<String> queueDump = callableQueueService.getQueueDump();
        assertEquals(1, callableQueueService.getQueueDump().size());
        assertTrue(queueDump.get(0).contains("coord_push_dep_check"));
        log.info("Queue dump is " + queueDump.toString());
        // Delay should be something like delay=599999. Ignore last three digits
        assertTrue(queueDump.get(0).matches(".* delay=599[0-9]{3}"));
    }

View Full Code Here

0 1 2 3

TOP

Related Classes of org.apache.oozie.service.CallableQueueService$CompositeCallable

org.apache.hadoop.conf.Configuration

org.apache.oozie.command.coord.CoordPushDependencyCheckXCommand

org.apache.oozie.command.coord.TestCoordActionInputCheckXCommand

org.apache.oozie.command.coord.TestCoordKillXCommand

org.apache.oozie.command.coord.TestCoordPushDependencyCheckXCommand

org.apache.oozie.command.XCommand

org.apache.oozie.util.XLog

java.util.concurrent.ThreadPoolExecutor

java.util.Date

java.util.concurrent.atomic.AtomicInteger

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.