Examples of java.util.concurrent.ExecutorCompletionService

java.util.concurrent.ExecutorCompletionService

A {@link CompletionService} that uses a supplied {@link Executor}to execute tasks. This class arranges that submitted tasks are, upon completion, placed on a queue accessible using take. The class is lightweight enough to be suitable for transient use when processing groups of tasks.

Usage Examples. Suppose you have a set of solvers for a certain problem, each returning a value of some type Result, and would like to run them concurrently, processing the results of each of them that return a non-null value, in some method use(Result r). You could write this as:

 void solve(Executor e, Collection<Callable<Result>> solvers) throws InterruptedException, ExecutionException { CompletionService<Result> ecs = new ExecutorCompletionService<Result>(e); for (Callable<Result> s : solvers) ecs.submit(s); int n = solvers.size(); for (int i = 0; i < n; ++i) { Result r = ecs.take().get(); if (r != null) use(r); } }

Suppose instead that you would like to use the first non-null result of the set of tasks, ignoring any that encounter exceptions, and cancelling all other tasks when the first one is ready:

 void solve(Executor e, Collection<Callable<Result>> solvers) throws InterruptedException { CompletionService<Result> ecs = new ExecutorCompletionService<Result>(e); int n = solvers.size(); List<Future<Result>> futures = new ArrayList<Future<Result>>(n); Result result = null; try { for (Callable<Result> s : solvers) futures.add(ecs.submit(s)); for (int i = 0; i < n; ++i) { try { Result r = ecs.take().get(); if (r != null) { result = r; break; } } catch (ExecutionException ignore) {} } } finally { for (Future<Result> f : futures) f.cancel(true); } if (result != null) use(result); }


            final DocumentFactory documentFactory = new DocumentFactory(basedir, buildMapping(), buildHeaderDefinitions(), encoding, keywords);


            int nThreads = (int) (Runtime.getRuntime().availableProcessors() * concurrencyFactor);
            ExecutorService executorService = Executors.newFixedThreadPool(nThreads);
            CompletionService completionService = new ExecutorCompletionService(executorService);
            int count = 0;
            debug("Number of execution threads: %s", nThreads);


            try {
                for (final String file : listSelectedFiles()) {
                    completionService.submit(new Runnable() {
                        public void run() {
                            Document document = documentFactory.createDocuments(file);
                            debug("Selected file: %s [header style: %s]", document.getFile(), document.getHeaderDefinition());
                            if (document.isNotSupported()) {
                                warn("Unknown file extension: %s", document.getFile());
                            } else if (document.is(h)) {
                                debug("Skipping header file: %s", document.getFile());
                            } else if (document.hasHeader(h, strictCheck)) {
                                callback.onExistingHeader(document, h);
                            } else {
                                boolean headerFound = false;
                                for (Header validHeader : validHeaders) {
                                    if (headerFound = document.hasHeader(validHeader, strictCheck)) {
                                        callback.onExistingHeader(document, h);
                                        break;
                                    }
                                }
                                if (!headerFound)
                                    callback.onHeaderNotFound(document, h);
                            }
                        }
                    }, null);
                    count++;
                }


                while (count-- > 0) {
                    try {
                        completionService.take().get();
                    } catch (InterruptedException e) {
                        Thread.currentThread().interrupt();
                    } catch (ExecutionException e) {
                        Throwable cause = e.getCause();
                        if (cause instanceof Error)

View Full Code Here


            final DocumentFactory documentFactory = new DocumentFactory(basedir, buildMapping(), buildHeaderDefinitions(), encoding, keywords);


            int nThreads = (int) (Runtime.getRuntime().availableProcessors() * concurrencyFactor);
            ExecutorService executorService = Executors.newFixedThreadPool(nThreads);
            CompletionService completionService = new ExecutorCompletionService(executorService);
            int count = 0;
            debug("Number of execution threads: %s", nThreads);


            try {
                for (final String file : listSelectedFiles()) {
                    completionService.submit(new Runnable() {
                        public void run() {
                            Document document = documentFactory.createDocuments(file);
                            debug("Selected file: %s [header style: %s]", document.getFile(), document.getHeaderDefinition());
                            if (document.isNotSupported()) {
                                warn("Unknown file extension: %s", document.getFile());
                            } else if (document.is(h)) {
                                debug("Skipping header file: %s", document.getFile());
                            } else if (document.hasHeader(h, strictCheck)) {
                                callback.onExistingHeader(document, h);
                            } else {
                                boolean headerFound = false;
                                for (Header validHeader : validHeaders) {
                                    if (headerFound = document.hasHeader(validHeader, strictCheck)) {
                                        callback.onExistingHeader(document, h);
                                        break;
                                    }
                                }
                                if (!headerFound)
                                    callback.onHeaderNotFound(document, h);
                            }
                        }
                    }, null);
                    count++;
                }


                while (count-- > 0) {
                    try {
                        completionService.take().get();
                    } catch (InterruptedException e) {
                        Thread.currentThread().interrupt();
                    } catch (ExecutionException e) {
                        Throwable cause = e.getCause();
                        if (cause instanceof Error)

View Full Code Here

        this.logger.fine("Construction proxy to remote endpoint " + address);
        final DiscoveryManager newClient = Proxies.newClient(EXPORT_NAME, address, getClass().getClassLoader(), DiscoveryManager.class);


        // Execute collection asynchronously (TODO: cache pool usage could be improved)
        final ExecutorService cachePool = Executors.newCachedThreadPool(this.threadFactory);
        final ExecutorCompletionService<String> ecs = new ExecutorCompletionService(cachePool);
        final Future<String> future = ecs.submit(new Callable<String>() {


            public String call() throws Exception {
                return AccessController.doPrivileged(new PrivilegedAction<String>() {
                    public String run() {
                        return newClient.ping(667) == 667 ? "OK" : null;

View Full Code Here

    final ExecutorService real;
    final CompletionService completionService;


    CountingExecutorService(ExecutorService real) {
      this.real = real;
      completionService = new ExecutorCompletionService(real);
    }

View Full Code Here

        Pipeline pipeline = getPipeline(dbBatch.getRowBatch().getIdentity().getPipelineId());
        boolean mustDb = pipeline.getParameters().getSyncConsistency().isMedia();
        boolean isRow = pipeline.getParameters().getSyncMode().isRow();// 如果是行记录是必须进行数据库反查
        // 读取一次配置
        adjustPoolSize(pipeline.getParameters().getExtractPoolSize()); // 调整下线程池，Extractor会被池化处理
        ExecutorCompletionService completionService = new ExecutorCompletionService(executor);


        // 进行并发提交
        ExtractException exception = null;
        // 每个表进行处理
        List<DataItem> items = new ArrayList<DataItem>();
        List<Future> futures = new ArrayList<Future>();
        List<EventData> eventDatas = dbBatch.getRowBatch().getDatas();
        for (EventData eventData : eventDatas) {
            DataItem item = new DataItem(eventData);
            // 针对row模式，需要去检查一下当前是否已经包含row记录的所有字段，如果发现字段不足，则执行一次数据库查询
            boolean flag = mustDb
                           || (eventData.getSyncConsistency() != null && eventData.getSyncConsistency().isMedia());


            // 增加一种case, 针对oracle erosa有时侯结果记录只有主键，没有变更字段，需要做一次反查
            if (!flag && CollectionUtils.isEmpty(eventData.getUpdatedColumns())) {
                DataMedia dataMedia = ConfigHelper.findDataMedia(pipeline, eventData.getTableId());
                if (dataMedia.getSource().getType().isOracle()) {
                    flag |= true;
                    eventData.setRemedy(true);// 针对这类数据，也统一视为补救的操作，可能erosa解析时反查数据库也不存在记录
                }
            }


            if (isRow && !flag) {
                // 提前判断一次，避免进入多线程进行竞争
                // 针对view视图的情况，会有后续再判断一次
                flag = checkNeedDbForRowMode(pipeline, eventData);
            }


            if (flag && (eventData.getEventType().isInsert() || eventData.getEventType().isUpdate())) {// 判断是否需要反查
                Future future = completionService.submit(new DatabaseExtractWorker(pipeline, item), null); // 提交进行并行查询
                if (future.isDone()) {
                    // 立即判断一次，因为使用了CallerRun可能当场跑出结果，针对有异常时快速响应，而不是等跑完所有的才抛异常
                    try {
                        future.get();
                    } catch (InterruptedException e) {
                        cancel(futures);// 取消完之后立马退出
                        throw new ExtractException(e);
                    } catch (ExecutionException e) {
                        cancel(futures); // 取消完之后立马退出
                        throw new ExtractException(e);
                    }
                }


                futures.add(future);// 记录一下添加的任务
            }


            items.add(item);// 按顺序添加
        }


        // 开始处理结果
        int index = 0;
        while (index < futures.size()) { // 循环处理发出去的所有任务
            try {
                Future future = completionService.take();// 它也可能被打断
                future.get();
            } catch (InterruptedException e) {
                exception = new ExtractException(e);
                break;// 如何一个future出现了异常，就退出
            } catch (ExecutionException e) {

View Full Code Here

    configureSource();
    context.put(ThriftSource.CONFIG_BIND, "0.0.0.0");
    context.put(ThriftSource.CONFIG_PORT, String.valueOf(port));
    Configurables.configure(source, context);
    source.start();
    ExecutorCompletionService<Void> completionService = new
      ExecutorCompletionService(submitter);
    for (int i = 0; i < 30; i++) {
      completionService.submit(new SubmitHelper(i), null);
    }
    //wait for all threads to be done




    for(int i = 0; i < 30; i++) {
      completionService.take();
    }


    Transaction transaction = channel.getTransaction();
    transaction.begin();
    long after = System.currentTimeMillis();

View Full Code Here

   @SuppressWarnings("unchecked")
   @Override
   public void process(KeyFilter keyFilter, CacheLoaderTask cacheLoaderTask, Executor executor, boolean loadValues, boolean loadMetadata) {


      int batchSize = 100;
      ExecutorCompletionService ecs = new ExecutorCompletionService(executor);
      int tasks = 0;
      final TaskContext taskContext = new TaskContextImpl();


      Set<Object> allKeys = new HashSet<Object>(batchSize);
      Set<Object> batch = new HashSet<Object>();

View Full Code Here

    this.completionService = createAppCallbackCompletionService();
  }


  @VisibleForTesting
  CompletionService createAppCallbackCompletionService() {
    return new ExecutorCompletionService(this.executorService);
  }

View Full Code Here

    this.completionService = craeteAppCallbackCompletionService();
  }


  @VisibleForTesting
  CompletionService craeteAppCallbackCompletionService() {
    return new ExecutorCompletionService(this.executorService);
  }

View Full Code Here

    public void testWithExecutorService() throws Exception {
        ThreadPoolExecutor threadPoolExecutor = 
            new ThreadPoolExecutor(5, 10, 5, TimeUnit.SECONDS,
                                   new ArrayBlockingQueue<Runnable>(10));
        
        ExecutorCompletionService threadPool = 
            new ExecutorCompletionService(threadPoolExecutor);
        
        ContextService service = getContextService();
                
        MyTask task1 = new MyTask();   
        
        String expected = "hello123World";
        TestContextHandler.setCurrentObject(expected);
        
        Callable task2 = (Callable)service.createContextObject(task1, new Class[] {Callable.class});
        
        assertEquals(expected, TestContextHandler.getCurrentObject());
        TestContextHandler.setCurrentObject(null);
        assertNull(TestContextHandler.getCurrentObject());
        
        threadPool.submit(task2);
        
        assertEquals(expected, threadPool.take().get());
        
        threadPoolExecutor.shutdown();
    }

View Full Code Here

0 1 2 3

TOP

Related Classes of java.util.concurrent.ExecutorCompletionService

com.alibaba.otter.manager.biz.monitor.impl.GlobalMonitor

com.alibaba.otter.node.etl.common.pipe.impl.http.archive.ArchiveBean

com.alibaba.otter.node.etl.extract.extractor.DatabaseExtractor

com.alibaba.otter.node.etl.load.loader.db.DataBatchLoader

com.alibaba.otter.shared.common.utils.thread.ExecutorTemplate

com.alibaba.otter.shared.communication.core.impl.DefaultCommunicationClientImpl

com.google.code.mojo.license.AbstractLicenseMojo

com.google.common.util.concurrent.AbstractListeningExecutorService

net.xeoh.plugins.remote.impl.ermi.RemoteAPIImpl

net.xeoh.plugins.remotediscovery.impl.v2.RemoteDiscoveryImpl

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.