* <i>insert:从afterColumns中获取所有的变更数据<br>
* <i>delete:从beforeColumns中获取所有的变更数据<br>
* <i>update:在beforeColumns中存放变更前的所有数据,在afterColumns中存放变更后的所有数据<br>
*/
private EventData internParse(Pipeline pipeline, Entry entry, RowChange rowChange, RowData rowData) {
EventData eventData = new EventData();
eventData.setTableName(entry.getHeader().getTableName());
eventData.setSchemaName(entry.getHeader().getSchemaName());
eventData.setEventType(EventType.valueOf(rowChange.getEventType().name()));
eventData.setExecuteTime(entry.getHeader().getExecuteTime());
boolean useTableTransform = pipeline.getParameters().getUseTableTransform();
EventType eventType = eventData.getEventType();
Table table = null;
TableInfoHolder tableHolder = null;
if (eventType.isQuery()) {
// 直接忽略query事件
return null;
}
// 首先判断是否为系统表
if (StringUtils.equalsIgnoreCase(pipeline.getParameters().getSystemSchema(), eventData.getSchemaName())) {
// do noting
if (eventType.isCreate() || eventType.isAlter() || eventType.isErase()) {
return null;
}
if (StringUtils.equalsIgnoreCase(pipeline.getParameters().getSystemDualTable(), eventData.getTableName())) {
// 心跳表数据直接忽略
return null;
}
} else {
DataMedia dataMedia = ConfigHelper.findSourceDataMedia(pipeline, eventData.getSchemaName(),
eventData.getTableName());
if (dataMedia == null) {
throw new SelectException("the namespace = " + eventData.getSchemaName() + " name = "
+ eventData.getTableName() + " dataMedia is null,please check , entry: "
+ entry.toString() + " and rowdata: " + rowChange.toString());
}
eventData.setTableId(dataMedia.getId());
if (eventType.isCreate() || eventType.isAlter() || eventType.isErase()) {
// 如果EventType是CREATE/ALTER，需要reload DataMediaInfo;并且把CREATE/ALTER类型的事件丢弃掉.
if (eventType.isCreate() || eventType.isAlter()) {
DbDialect dbDialect = dbDialectFactory.getDbDialect(pipeline.getId(),
(DbMediaSource) dataMedia.getSource());
dbDialect.reloadTable(eventData.getSchemaName(), eventData.getTableName());// 更新下meta信息
}
return null;
}
if (useTableTransform || dataMedia.getSource().getType().isOracle()) {// oracle需要反查一次meta
// 如果设置了需要进行table meta转化，则反查一下table信息
// 比如oracle erosa解析时可能使用了非物理主键，需要直接使用，信任erosa的信息
DbDialect dbDialect = dbDialectFactory.getDbDialect(pipeline.getId(),
(DbMediaSource) dataMedia.getSource());
table = dbDialect.findTable(eventData.getSchemaName(), eventData.getTableName());// 查询一下meta信息
if (table == null) {
logger.warn("find table[{}.{}] is null , may be drop table.", eventData.getSchemaName(),
eventData.getTableName());
}
tableHolder = new TableInfoHolder(dbDialect, table, useTableTransform);
}
}
List<Column> beforeColumns = rowData.getBeforeColumnsList();
List<Column> afterColumns = rowData.getAfterColumnsList();
String tableName = eventData.getSchemaName() + "." + eventData.getTableName();
// 判断一下是否需要all columns
boolean isRowMode = pipeline.getParameters().getSyncMode().isRow(); // 如果是rowMode模式，所有字段都需要标记为updated
boolean needAllColumns = isRowMode || checkNeedAllColumns(pipeline);
// 变更后的主键
Map<String, EventColumn> keyColumns = new LinkedHashMap<String, EventColumn>();
// 变更前的主键
Map<String, EventColumn> oldKeyColumns = new LinkedHashMap<String, EventColumn>();
// 有变化的非主键
Map<String, EventColumn> notKeyColumns = new LinkedHashMap<String, EventColumn>();
if (eventType.isInsert()) {
for (Column column : afterColumns) {
if (isKey(tableHolder, tableName, column)) {
keyColumns.put(column.getName(), copyEventColumn(column, true, tableHolder));
} else {
// mysql 有效
notKeyColumns.put(column.getName(), copyEventColumn(column, true, tableHolder));
}
}
} else if (eventType.isDelete()) {
for (Column column : beforeColumns) {
if (isKey(tableHolder, tableName, column)) {
keyColumns.put(column.getName(), copyEventColumn(column, true, tableHolder));
} else {
// mysql 有效
notKeyColumns.put(column.getName(), copyEventColumn(column, true, tableHolder));
}
}
} else if (eventType.isUpdate()) {
// 获取变更前的主键.
for (Column column : beforeColumns) {
if (isKey(tableHolder, tableName, column)) {
oldKeyColumns.put(column.getName(), copyEventColumn(column, true, tableHolder));
} else if (needAllColumns && entry.getHeader().getSourceType() == CanalEntry.Type.ORACLE) {
// 针对行记录同步时，针对oracle记录一下非主键的字段，因为update时针对未变更的字段在aftercolume里没有
notKeyColumns.put(column.getName(), copyEventColumn(column, isRowMode, tableHolder));
}
}
for (Column column : afterColumns) {
if (isKey(tableHolder, tableName, column)) {
// 获取变更后的主键
keyColumns.put(column.getName(), copyEventColumn(column, true, tableHolder));
} else if (needAllColumns || entry.getHeader().getSourceType() == CanalEntry.Type.ORACLE
|| column.getUpdated()) {
// 在update操作时，oracle和mysql存放变更的非主键值的方式不同,oracle只有变更的字段; mysql会把变更前和变更后的字段都发出来，只需要取有变更的字段.
// 如果是oracle库，after里一定为对应的变更字段
boolean isUpdate = true;
if (entry.getHeader().getSourceType() == CanalEntry.Type.MYSQL) { // mysql的after里部分数据为未变更,oracle里after里为变更字段
isUpdate = column.getUpdated();
}
notKeyColumns.put(column.getName(), copyEventColumn(column, isRowMode || isUpdate, tableHolder));// 如果是rowMode，所有字段都为updated
}
}
if (entry.getHeader().getSourceType() == CanalEntry.Type.ORACLE) { // 针对oracle进行特殊处理
checkUpdateKeyColumns(oldKeyColumns, keyColumns);
}
}
if (keyColumns.isEmpty()) {
throw new SelectException("this rowdata has no pks , entry: " + entry.toString() + " and rowData: "
+ rowData);
}
List<EventColumn> keys = new ArrayList<EventColumn>(keyColumns.values());
List<EventColumn> oldKeys = new ArrayList<EventColumn>(oldKeyColumns.values());
List<EventColumn> columns = new ArrayList<EventColumn>(notKeyColumns.values());
Collections.sort(keys, new EventColumnIndexComparable());
Collections.sort(oldKeys, new EventColumnIndexComparable());
Collections.sort(columns, new EventColumnIndexComparable());
eventData.setKeys(keys);
if (eventData.getEventType().isUpdate() && !oldKeys.equals(keys)) { // update类型，如果存在主键不同,则记录下old keys为变更前的主键
eventData.setOldKeys(oldKeys);
}
eventData.setColumns(columns);
return eventData;
}