private ExtensionFactory extensionFactory;
private DataSourceService dataSourceService;
private ExecutorTemplateGetter executorTemplateGetter;
public void extract(DbBatch param) throws ExtractException {
ExecutorTemplate executorTemplate = null;
try {
RowBatch rowBatch = param.getRowBatch();
final Pipeline pipeline = getPipeline(rowBatch.getIdentity().getPipelineId());
List<EventData> eventDatas = rowBatch.getDatas();
final Set<EventData> removeDatas = Collections.synchronizedSet(new HashSet<EventData>());// 使用set，提升remove时的查找速度
executorTemplate = executorTemplateGetter.get();
executorTemplate.start();
// 重新设置下poolSize
executorTemplate.adjustPoolSize(pipeline.getParameters().getExtractPoolSize());
for (final EventData eventData : eventDatas) {
List<DataMediaPair> dataMediaPairs = ConfigHelper.findDataMediaPairByMediaId(pipeline,
eventData.getTableId());
if (dataMediaPairs == null) {
throw new ExtractException("ERROR ## the dataMediaId = " + eventData.getTableId()
+ " dataMediaPair is null,please check");
}
for (DataMediaPair dataMediaPair : dataMediaPairs) {
if (!dataMediaPair.isExistFilter()) {
continue;
}
final EventProcessor eventProcessor = extensionFactory.getExtension(EventProcessor.class,
dataMediaPair.getFilterData());
if (eventProcessor instanceof DataSourceFetcherAware) {
((DataSourceFetcherAware) eventProcessor).setDataSourceFetcher(new DataSourceFetcher() {
@Override
public DataSource fetch(Long tableId) {
DataMedia dataMedia = ConfigHelper.findDataMedia(pipeline, tableId);
return dataSourceService.getDataSource(pipeline.getId(), dataMedia.getSource());
}
});
executorTemplate.submit(new Runnable() {
@Override
public void run() {
MDC.put(OtterConstants.splitPipelineLogFileKey, String.valueOf(pipeline.getId()));
boolean process = eventProcessor.process(eventData);
if (!process) {
removeDatas.add(eventData);// 添加到删除记录中
}
}
});
} else {
boolean process = eventProcessor.process(eventData);
if (!process) {
removeDatas.add(eventData);// 添加到删除记录中
break;
}
}
}
}
// 等待所有都处理完成
executorTemplate.waitForResult();
if (!CollectionUtils.isEmpty(removeDatas)) {
eventDatas.removeAll(removeDatas);
}
} finally {