Package java.util.concurrent

Examples of java.util.concurrent.ExecutorCompletionService

383
384
385
386
387
388
389
    final ExecutorService real;
    final CompletionService completionService;

    CountingExecutorService(ExecutorService real) {
      this.real = real;
      completionService = new ExecutorCompletionService(real);
    }
View Full Code Here
69
70
71
72
73
74
75
76
77
78
79
   @SuppressWarnings("unchecked")
   @Override
   public void process(KeyFilter keyFilter, CacheLoaderTask cacheLoaderTask, Executor executor, boolean loadValues, boolean loadMetadata) {

      int batchSize = 100;
      ExecutorCompletionService ecs = new ExecutorCompletionService(executor);
      int tasks = 0;
      final TaskContext taskContext = new TaskContextImpl();

      Set<Object> allKeys = new HashSet<Object>(batchSize);
      Set<Object> batch = new HashSet<Object>();
View Full Code Here
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
        Assert.notNull(this.factory, "No factory specified");
        if (addrs == null || addrs.length == 0) {
            throw new IllegalArgumentException("addrs example: 127.0.0.1:1099");
        }

        ExecutorCompletionService completionService = new ExecutorCompletionService(executor);
        List<Future<Object>> futures = new ArrayList<Future<Object>>(addrs.length);
        List result = new ArrayList(10);
        for (final String addr : addrs) {
            futures.add(completionService.submit((new Callable<Object>() {

                @Override
                public Object call() throws Exception {
                    return DefaultCommunicationClientImpl.this.call(addr, event);
                }
            })));
        }

        Exception ex = null;
        int errorIndex = 0;
        while (errorIndex < futures.size()) {
            try {
                Future future = completionService.take();// 它也可能被打断
                future.get();
            } catch (InterruptedException e) {
                Thread.currentThread().interrupt();
                ex = e;
                break;
View Full Code Here
52
53
54
55
56
57
58
59
60
61
62
        this.size = 1;
      }
      executor = Executors.newFixedThreadPool(size);
    }
    if(completion==null) {
      completion = new ExecutorCompletionService(executor);
    }
    try {
      for(int i=0;i<fileList.length;i++) {
        String filename = fileList[i].getPath().toUri().toString();
        log.info("Processing: "+filename);
View Full Code Here
65
66
67
68
69
70
71
72
73
74
75
        this.size = 1;
      }
      executor = Executors.newFixedThreadPool(size);
    }
    if(completion==null) {
      completion = new ExecutorCompletionService(executor);
    }
   
    try {
      // Locate directory output directories of the current demux, and create a unique directory list.
      HashSet<Path> inputPaths = new HashSet<Path>();
View Full Code Here
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
   }

   @Override
   public void process(final KeyFilter filter, final CacheLoaderTask task, Executor executor, boolean fetchValue, boolean fetchMetadata) {
      scanForUnknownDirectories();
      ExecutorCompletionService ecs = new ExecutorCompletionService(executor);

      final TaskContextImpl taskContext = new TaskContextImpl();
      int count = 0;
      for (final DirectoryLoaderAdaptor dir : openDirectories.values()) {
         ecs.submit(new Callable<Void>() {
            @Override
            public Void call() throws Exception {
               final HashSet<MarshalledEntry> allInternalEntries = new HashSet<MarshalledEntry>();
               dir.loadAllEntries(allInternalEntries, Integer.MAX_VALUE, ctx.getMarshaller());
               for (MarshalledEntry me : allInternalEntries) {
View Full Code Here
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
        boolean exist = false;
        ZipOutputStream zipOut = null;
        Set<String> entryNames = new HashSet<String>();
        BlockingQueue<Future<ArchiveEntry>> queue = new LinkedBlockingQueue<Future<ArchiveEntry>>(); // 下载成功的任务列表
        ExecutorCompletionService completionService = new ExecutorCompletionService(executor, queue);

        final File targetDir = new File(targetArchiveFile.getParentFile(),
                                        FilenameUtils.getBaseName(targetArchiveFile.getPath()));
        try {
            // 创建一个临时目录
            FileUtils.forceMkdir(targetDir);

            zipOut = new ZipOutputStream(new BufferedOutputStream(new FileOutputStream(targetArchiveFile)));
            zipOut.setLevel(Deflater.BEST_SPEED);
            // 进行并发压缩处理
            for (final FileData fileData : fileDatas) {
                if (fileData.getEventType().isDelete()) {
                    continue; // 忽略delete类型的数据打包，因为只需直接在目标进行删除
                }

                String namespace = fileData.getNameSpace();
                String path = fileData.getPath();
                boolean isLocal = StringUtils.isBlank(namespace);
                String entryName = null;
                if (true == isLocal) {
                    entryName = FilenameUtils.getPath(path) + FilenameUtils.getName(path);
                } else {
                    entryName = namespace + File.separator + path;
                }

                // 过滤一些重复的文件数据同步
                if (entryNames.contains(entryName) == false) {
                    entryNames.add(entryName);
                } else {
                    continue;
                }

                final String name = entryName;
                if (true == isLocal && !useLocalFileMutliThread) {
                    // 采用串行处理，不走临时文件
                    queue.add(new DummyFuture(new ArchiveEntry(name, callback.retrive(fileData))));
                } else {
                    completionService.submit(new Callable<ArchiveEntry>() {

                        public ArchiveEntry call() throws Exception {
                            // 处理下异常，可能失败
                            InputStream input = null;
                            OutputStream output = null;
View Full Code Here
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
            count += 1;
        }

        WeightController controller = new WeightController(count);
        List<Future> futures = new ArrayList<Future>();
        ExecutorCompletionService completionService = new ExecutorCompletionService(executorService);

        if (existFileBatch) {
            submitFileBatch(futures, completionService, fileBatch, data.getRoot(), controller);
        }

        if (existRowBatch) {
            submitRowBatch(futures, completionService, rowBatchs, controller);
        }

        // 先获取一下异步处理的结果，记录一下出错的index
        List<LoadContext> processedContexts = new ArrayList<LoadContext>();
        int index = 0;
        LoadException exception = null;
        while (index < futures.size()) {
            try {
                Future future = completionService.take();// 它也可能被打断
                future.get();
            } catch (InterruptedException e) {
                exception = new LoadException(e);
                break;
            } catch (ExecutionException e) {
View Full Code Here
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
        Pipeline pipeline = getPipeline(dbBatch.getRowBatch().getIdentity().getPipelineId());
        boolean mustDb = pipeline.getParameters().getSyncConsistency().isMedia();
        boolean isRow = pipeline.getParameters().getSyncMode().isRow();// 如果是行记录是必须进行数据库反查
        // 读取一次配置
        adjustPoolSize(pipeline.getParameters().getExtractPoolSize()); // 调整下线程池，Extractor会被池化处理
        ExecutorCompletionService completionService = new ExecutorCompletionService(executor);

        // 进行并发提交
        ExtractException exception = null;
        // 每个表进行处理
        List<DataItem> items = new ArrayList<DataItem>();
        List<Future> futures = new ArrayList<Future>();
        List<EventData> eventDatas = dbBatch.getRowBatch().getDatas();
        for (EventData eventData : eventDatas) {
            if (eventData.getEventType().isDdl()) {
                continue;
            }

            DataItem item = new DataItem(eventData);
            // 针对row模式，需要去检查一下当前是否已经包含row记录的所有字段，如果发现字段不足，则执行一次数据库查询
            boolean flag = mustDb
                           || (eventData.getSyncConsistency() != null && eventData.getSyncConsistency().isMedia());

            // 增加一种case, 针对oracle erosa有时侯结果记录只有主键，没有变更字段，需要做一次反查
            if (!flag && CollectionUtils.isEmpty(eventData.getUpdatedColumns())) {
                DataMedia dataMedia = ConfigHelper.findDataMedia(pipeline, eventData.getTableId());
                if (dataMedia.getSource().getType().isOracle()) {
                    flag |= true;
                    eventData.setRemedy(true);// 针对这类数据，也统一视为补救的操作，可能erosa解析时反查数据库也不存在记录
                }
            }

            if (isRow && !flag) {
                // 提前判断一次，避免进入多线程进行竞争
                // 针对view视图的情况，会有后续再判断一次
                flag = checkNeedDbForRowMode(pipeline, eventData);
            }

            if (flag && (eventData.getEventType().isInsert() || eventData.getEventType().isUpdate())) {// 判断是否需要反查
                Future future = completionService.submit(new DatabaseExtractWorker(pipeline, item), null); // 提交进行并行查询
                if (future.isDone()) {
                    // 立即判断一次，因为使用了CallerRun可能当场跑出结果，针对有异常时快速响应，而不是等跑完所有的才抛异常
                    try {
                        future.get();
                    } catch (InterruptedException e) {
                        cancel(futures);// 取消完之后立马退出
                        throw new ExtractException(e);
                    } catch (ExecutionException e) {
                        cancel(futures); // 取消完之后立马退出
                        throw new ExtractException(e);
                    }
                }

                futures.add(future);// 记录一下添加的任务
            }

            items.add(item);// 按顺序添加
        }

        // 开始处理结果
        int index = 0;
        while (index < futures.size()) { // 循环处理发出去的所有任务
            try {
                Future future = completionService.take();// 它也可能被打断
                future.get();
            } catch (InterruptedException e) {
                exception = new ExtractException(e);
                break;// 如何一个future出现了异常，就退出
            } catch (ExecutionException e) {
View Full Code Here
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
        Pipeline pipeline = getPipeline(dbBatch.getRowBatch().getIdentity().getPipelineId());
        boolean mustDb = pipeline.getParameters().getSyncConsistency().isMedia();
        boolean isRow = pipeline.getParameters().getSyncMode().isRow();// 如果是行记录是必须进行数据库反查
        // 读取一次配置
        adjustPoolSize(pipeline.getParameters().getExtractPoolSize()); // 调整下线程池，Extractor会被池化处理
        ExecutorCompletionService completionService = new ExecutorCompletionService(executor);

        // 进行并发提交
        ExtractException exception = null;
        // 每个表进行处理
        List<DataItem> items = new ArrayList<DataItem>();
        List<Future> futures = new ArrayList<Future>();
        List<EventData> eventDatas = dbBatch.getRowBatch().getDatas();
        for (EventData eventData : eventDatas) {
            if (eventData.getEventType().isDdl()) {
                continue;
            }

            DataItem item = new DataItem(eventData);
            // 针对row模式，需要去检查一下当前是否已经包含row记录的所有字段，如果发现字段不足，则执行一次数据库查询
            boolean flag = mustDb
                           || (eventData.getSyncConsistency() != null && eventData.getSyncConsistency().isMedia());

            // 增加一种case, 针对oracle erosa有时侯结果记录只有主键，没有变更字段，需要做一次反查
            if (!flag && CollectionUtils.isEmpty(eventData.getUpdatedColumns())) {
                DataMedia dataMedia = ConfigHelper.findDataMedia(pipeline, eventData.getTableId());
                if (dataMedia.getSource().getType().isOracle()) {
                    flag |= true;
                    eventData.setRemedy(true);// 针对这类数据，也统一视为补救的操作，可能erosa解析时反查数据库也不存在记录
                }
            }

            if (isRow && !flag) {
                // 提前判断一次，避免进入多线程进行竞争
                // 针对view视图的情况，会有后续再判断一次
                flag = checkNeedDbForRowMode(pipeline, eventData);
            }

            if (flag && (eventData.getEventType().isInsert() || eventData.getEventType().isUpdate())) {// 判断是否需要反查
                Future future = completionService.submit(new DatabaseExtractWorker(pipeline, item), null); // 提交进行并行查询
                if (future.isDone()) {
                    // 立即判断一次，因为使用了CallerRun可能当场跑出结果，针对有异常时快速响应，而不是等跑完所有的才抛异常
                    try {
                        future.get();
                    } catch (InterruptedException e) {
                        cancel(futures);// 取消完之后立马退出
                        throw new ExtractException(e);
                    } catch (ExecutionException e) {
                        cancel(futures); // 取消完之后立马退出
                        throw new ExtractException(e);
                    }
                }

                futures.add(future);// 记录一下添加的任务
            }

            items.add(item);// 按顺序添加
        }

        // 开始处理结果
        int index = 0;
        while (index < futures.size()) { // 循环处理发出去的所有任务
            try {
                Future future = completionService.take();// 它也可能被打断
                future.get();
            } catch (InterruptedException e) {
                exception = new ExtractException(e);
                break;// 如何一个future出现了异常，就退出
            } catch (ExecutionException e) {
View Full Code Here
TOP

Related Classes of java.util.concurrent.ExecutorCompletionService

  • com.alibaba.otter.manager.biz.monitor.impl.GlobalMonitor
  • com.alibaba.otter.node.etl.common.pipe.impl.http.archive.ArchiveBean
  • com.alibaba.otter.node.etl.extract.extractor.DatabaseExtractor
  • com.alibaba.otter.node.etl.load.loader.db.DataBatchLoader
  • com.alibaba.otter.shared.common.utils.thread.ExecutorTemplate
  • com.alibaba.otter.shared.communication.core.impl.DefaultCommunicationClientImpl
  • com.google.code.mojo.license.AbstractLicenseMojo
  • com.google.common.util.concurrent.AbstractListeningExecutorService
  • net.xeoh.plugins.remote.impl.ermi.RemoteAPIImpl
  • net.xeoh.plugins.remotediscovery.impl.v2.RemoteDiscoveryImpl
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.