Package com.alibaba.otter.shared.etl.model

Examples of com.alibaba.otter.shared.etl.model.EventData


     * <i>insert:从afterColumns中获取所有的变更数据<br>
     * <i>delete:从beforeColumns中获取所有的变更数据<br>
     * <i>update:在beforeColumns中存放变更前的所有数据,在afterColumns中存放变更后的所有数据<br>
     */
    private EventData internParse(Pipeline pipeline, Entry entry, RowChange rowChange, RowData rowData) {
        EventData eventData = new EventData();
        eventData.setTableName(entry.getHeader().getTableName());
        eventData.setSchemaName(entry.getHeader().getSchemaName());
        eventData.setEventType(EventType.valueOf(rowChange.getEventType().name()));
        eventData.setExecuteTime(entry.getHeader().getExecuteTime());

        boolean useTableTransform = pipeline.getParameters().getUseTableTransform();
        EventType eventType = eventData.getEventType();
        Table table = null;
        TableInfoHolder tableHolder = null;
        DataMedia dataMedia = ConfigHelper.findSourceDataMedia(pipeline, eventData.getSchemaName(),
                                                               eventData.getTableName());
        if (dataMedia == null) {
            throw new SelectException("the namespace = " + eventData.getSchemaName() + " name = "
                                      + eventData.getTableName() + " dataMedia is null,please check , entry: "
                                      + entry.toString() + " and rowdata: " + rowChange.toString());
        }
        eventData.setTableId(dataMedia.getId());
        if (useTableTransform || dataMedia.getSource().getType().isOracle()) {// oracle需要反查一次meta
            // 如果设置了需要进行table meta转化,则反查一下table信息
            // 比如oracle erosa解析时可能使用了非物理主键,需要直接使用,信任erosa的信息
            DbDialect dbDialect = dbDialectFactory.getDbDialect(pipeline.getId(), (DbMediaSource) dataMedia.getSource());
            table = dbDialect.findTable(eventData.getSchemaName(), eventData.getTableName());// 查询一下meta信息
            if (table == null) {
                logger.warn("find table[{}.{}] is null , may be drop table.", eventData.getSchemaName(),
                            eventData.getTableName());
            }
            tableHolder = new TableInfoHolder(dbDialect, table, useTableTransform);
        }

        List<Column> beforeColumns = rowData.getBeforeColumnsList();
        List<Column> afterColumns = rowData.getAfterColumnsList();
        String tableName = eventData.getSchemaName() + "." + eventData.getTableName();

        // 判断一下是否需要all columns
        boolean isRowMode = pipeline.getParameters().getSyncMode().isRow(); // 如果是rowMode模式,所有字段都需要标记为updated
        boolean needAllColumns = isRowMode || checkNeedAllColumns(pipeline);

        // 变更后的主键
        Map<String, EventColumn> keyColumns = new LinkedHashMap<String, EventColumn>();
        // 变更前的主键
        Map<String, EventColumn> oldKeyColumns = new LinkedHashMap<String, EventColumn>();
        // 有变化的非主键
        Map<String, EventColumn> notKeyColumns = new LinkedHashMap<String, EventColumn>();

        if (eventType.isInsert()) {
            for (Column column : afterColumns) {
                if (isKey(tableHolder, tableName, column)) {
                    keyColumns.put(column.getName(), copyEventColumn(column, true, tableHolder));
                } else {
                    // mysql 有效
                    notKeyColumns.put(column.getName(), copyEventColumn(column, true, tableHolder));
                }
            }
        } else if (eventType.isDelete()) {
            for (Column column : beforeColumns) {
                if (isKey(tableHolder, tableName, column)) {
                    keyColumns.put(column.getName(), copyEventColumn(column, true, tableHolder));
                } else {
                    // mysql 有效
                    notKeyColumns.put(column.getName(), copyEventColumn(column, true, tableHolder));
                }
            }
        } else if (eventType.isUpdate()) {
            // 获取变更前的主键.
            for (Column column : beforeColumns) {
                if (isKey(tableHolder, tableName, column)) {
                    oldKeyColumns.put(column.getName(), copyEventColumn(column, true, tableHolder));
                } else {
                    if (needAllColumns && entry.getHeader().getSourceType() == CanalEntry.Type.ORACLE) {
                        // 针对行记录同步时,针对oracle记录一下非主键的字段,因为update时针对未变更的字段在aftercolume里没有
                        notKeyColumns.put(column.getName(), copyEventColumn(column, isRowMode, tableHolder));
                    }
                }
            }
            for (Column column : afterColumns) {
                if (isKey(tableHolder, tableName, column)) {
                    // 获取变更后的主键
                    keyColumns.put(column.getName(), copyEventColumn(column, true, tableHolder));
                } else if (needAllColumns || entry.getHeader().getSourceType() == CanalEntry.Type.ORACLE
                           || column.getUpdated()) {
                    // 在update操作时,oracle和mysql存放变更的非主键值的方式不同,oracle只有变更的字段; mysql会把变更前和变更后的字段都发出来,只需要取有变更的字段.
                    // 如果是oracle库,after里一定为对应的变更字段

                    boolean isUpdate = true;
                    if (entry.getHeader().getSourceType() == CanalEntry.Type.MYSQL) { // mysql的after里部分数据为未变更,oracle里after里为变更字段
                        isUpdate = column.getUpdated();
                    }
                    notKeyColumns.put(column.getName(), copyEventColumn(column, isRowMode || isUpdate, tableHolder));// 如果是rowMode,所有字段都为updated
                }
            }

            if (entry.getHeader().getSourceType() == CanalEntry.Type.ORACLE) { // 针对oracle进行特殊处理
                checkUpdateKeyColumns(oldKeyColumns, keyColumns);
            }
        }

        List<EventColumn> keys = new ArrayList<EventColumn>(keyColumns.values());
        List<EventColumn> oldKeys = new ArrayList<EventColumn>(oldKeyColumns.values());
        List<EventColumn> columns = new ArrayList<EventColumn>(notKeyColumns.values());

        Collections.sort(keys, new EventColumnIndexComparable());
        Collections.sort(oldKeys, new EventColumnIndexComparable());
        Collections.sort(columns, new EventColumnIndexComparable());
        if (!keyColumns.isEmpty()) {
            eventData.setKeys(keys);
            if (eventData.getEventType().isUpdate() && !oldKeys.equals(keys)) { // update类型,如果存在主键不同,则记录下old keys为变更前的主键
                eventData.setOldKeys(oldKeys);
            }
            eventData.setColumns(columns);
        //} else if (CanalEntry.Type.MYSQL == entry.getHeader().getSourceType()) {
        //    // 只支持mysql无主键同步
        //    if (eventType.isUpdate()) {
        //        List<EventColumn> oldColumns = new ArrayList<EventColumn>();
        //        List<EventColumn> newColumns = new ArrayList<EventColumn>();
View Full Code Here


        eventData.setColumns(columns);
        return eventData;
    }

    private EventData makeUpdateEventData() {
        EventData eventData = new EventData();
        eventData.setEventType(EventType.UPDATE);
        eventData.setSchemaName(SCHEMA_NAME);
        eventData.setTableName(TABLE_NAME);
        eventData.setTableId(TABLE_ID);

        List<EventColumn> keys = new ArrayList<EventColumn>();
        keys.add(makeEventColumn(KEY_NAME, KEY_VALUE, true));
        eventData.setKeys(keys);

        List<EventColumn> columns = new ArrayList<EventColumn>();
        int i = 0;
        for (String columnName : COLUMN_NAMES2) {
            columns.add(makeEventColumn(columnName, columnName + i, false));
            i++;
        }
        eventData.setColumns(columns);
        return eventData;
    }
View Full Code Here

        eventData.setColumns(columns);
        return eventData;
    }

    private EventData makeUpdateEventData(String oldKeyValue, String newKeyValue) {
        EventData eventData = new EventData();
        eventData.setEventType(EventType.UPDATE);
        eventData.setSchemaName(SCHEMA_NAME);
        eventData.setTableName(TABLE_NAME);
        eventData.setTableId(TABLE_ID);

        List<EventColumn> oldKeys = new ArrayList<EventColumn>();
        oldKeys.add(makeEventColumn(KEY_NAME, oldKeyValue, true));
        List<EventColumn> newKeys = new ArrayList<EventColumn>();
        newKeys.add(makeEventColumn(KEY_NAME, newKeyValue, true));
        eventData.setKeys(newKeys);
        eventData.setOldKeys(oldKeys);
        return eventData;
    }
View Full Code Here

        eventData.setOldKeys(oldKeys);
        return eventData;
    }

    private EventData makeDeleteEventData() {
        EventData eventData = new EventData();
        eventData.setEventType(EventType.DELETE);
        eventData.setSchemaName(SCHEMA_NAME);
        eventData.setTableName(TABLE_NAME);
        eventData.setTableId(TABLE_ID);

        List<EventColumn> keys = new ArrayList<EventColumn>();
        keys.add(makeEventColumn(KEY_NAME, KEY_VALUE, true));
        eventData.setKeys(keys);
        return eventData;
    }
View Full Code Here

    private List<EventData> generatorEventDataForOracle(int start, int count, EventType type) {
        List<EventData> eventDatas = new ArrayList<EventData>();
        for (int i = 0; i < count; i++) {
            int index = i + 1 + start;
            EventData eventData = new EventData();
            eventData.setPairId(index);
            eventData.setTableId(1L);
            eventData.setSchemaName("srf");
            eventData.setTableName("columns");
            eventData.setEventType(type);
            eventData.setExecuteTime(100L);
            eventData.getKeys().add(buildColumn("id", Types.NUMERIC, "" + index, true, false));
            eventData.getKeys().add(buildColumn("name", Types.VARCHAR, "ljh_" + index, true, false));
            eventData.getOldKeys().add(buildColumn("id", Types.NUMERIC, "" + index, true, false));
            eventData.getOldKeys().add(buildColumn("name", Types.VARCHAR, "ljh_" + index, true, false));

            eventData.getColumns().add(buildColumn("alias_name", Types.CHAR, "hello_" + index, false, false));
            eventData.getColumns().add(buildColumn("amount", Types.NUMERIC, "100.01", false, false));
            eventData.getColumns().add(buildColumn("text_b", Types.BLOB, "[116,101,120,116,95,98]", false, false));
            eventData.getColumns().add(buildColumn("text_c", Types.CLOB, "中文", false, false));
            eventData.getColumns().add(buildColumn("curr_date", Types.DATE, "2011-01-01", false, false));
            eventData.getColumns().add(buildColumn("gmt_create", Types.DATE, "2011-01-01 11:11:11", false, false));
            eventData.getColumns().add(buildColumn("gmt_modify", Types.DATE, "2011-01-01 11:11:11", false, false));

            // OracleSqlTemplate sqlTemplate = new OracleSqlTemplate();
            // String sql = null;
            // if (type.isInsert()) {
            // sql = sqlTemplate.getMergeSql(eventData.getSchemaName(),
View Full Code Here

    private List<EventData> generatorEventDataForMysql(int start, int count, EventType type) {
        List<EventData> eventDatas = new ArrayList<EventData>();
        for (int i = 0; i < count; i++) {
            int index = i + 1 + start;
            EventData eventData = new EventData();
            eventData.setPairId(index);
            eventData.setTableId(1L);
            eventData.setSchemaName("srf");
            eventData.setTableName("columns");
            eventData.setEventType(type);
            eventData.setExecuteTime(100L);
            eventData.getKeys().add(buildColumn("id", Types.INTEGER, "" + index, true, false));
            eventData.getKeys().add(buildColumn("name", Types.VARCHAR, "ljh_" + index, true, false));

            eventData.getOldKeys().add(buildColumn("id", Types.INTEGER, "" + index, true, false));
            eventData.getOldKeys().add(buildColumn("name", Types.VARCHAR, "ljh_" + index, true, false));

            eventData.getColumns().add(buildColumn("alias_name", Types.CHAR, "hello_" + index, false, false));
            eventData.getColumns().add(buildColumn("amount", Types.DECIMAL, "100.01", false, false));
            eventData.getColumns().add(buildColumn("text_b", Types.BLOB, "[116,101,120,116,95,98]", false, false));
            eventData.getColumns().add(buildColumn("text_c", Types.CLOB, "中文", false, false));
            eventData.getColumns().add(buildColumn("curr_date", Types.DATE, "2011-01-01", false, false));
            eventData.getColumns().add(buildColumn("gmt_create", Types.TIMESTAMP, "2011-01-01 11:11:11", false, false));
            eventData.getColumns().add(buildColumn("gmt_modify", Types.TIMESTAMP, "2011-01-01 11:11:11", false, false));

            // MysqlSqlTemplate sqlTemplate = new MysqlSqlTemplate();
            // String sql = null;
            // if (type.isInsert()) {
            // sql = sqlTemplate.getMergeSql(eventData.getSchemaName(),
View Full Code Here

            RowKey key = entry.getKey();
            EventColumn keyColumn = key.getKeys().get(0);
            Assert.assertEquals(KEY_VALUE, keyColumn.getColumnValue());
            Assert.assertEquals(KEY_NAME, keyColumn.getColumnName());

            EventData eventData = entry.getValue();

            Assert.assertEquals(SCHEMA_NAME, eventData.getSchemaName());
            Assert.assertEquals(TABLE_NAME, eventData.getTableName());
            Assert.assertEquals(TABLE_ID, eventData.getTableId());
            Assert.assertEquals(EventType.INSERT, eventData.getEventType());
            Assert.assertEquals(eventData.getOldKeys().size(), 0); // 不存在oldKeys

            List<EventColumn> columns = eventData.getColumns();
            Assert.assertEquals(3, columns.size());
        }
    }
View Full Code Here

            RowKey key = entry.getKey();
            EventColumn keyColumn = key.getKeys().get(0);
            Assert.assertEquals(KEY_VALUE, keyColumn.getColumnValue());
            Assert.assertEquals(KEY_NAME, keyColumn.getColumnName());

            EventData eventData = entry.getValue();
            Assert.assertEquals(SCHEMA_NAME, eventData.getSchemaName());
            Assert.assertEquals(TABLE_NAME, eventData.getTableName());
            Assert.assertEquals(TABLE_ID, eventData.getTableId());
            Assert.assertEquals(EventType.DELETE, eventData.getEventType());
            Assert.assertEquals(eventData.getOldKeys().size(), 0); // 不存在oldKeys

            List<EventColumn> columns = eventData.getColumns();
            Assert.assertEquals(0, columns.size());
        }
    }
View Full Code Here

            RowKey key = entry.getKey();
            EventColumn keyColumn = key.getKeys().get(0);
            Assert.assertEquals(KEY_VALUE, keyColumn.getColumnValue());
            Assert.assertEquals(KEY_NAME, keyColumn.getColumnName());

            EventData eventData = entry.getValue();
            Assert.assertEquals(SCHEMA_NAME, eventData.getSchemaName());
            Assert.assertEquals(TABLE_NAME, eventData.getTableName());
            Assert.assertEquals(TABLE_ID, eventData.getTableId());
            Assert.assertEquals(EventType.INSERT, eventData.getEventType());
            Assert.assertEquals(eventData.getOldKeys().size(), 0); // 不存在oldKeys

            List<EventColumn> columns = eventData.getColumns();
            Assert.assertEquals(2, columns.size());
        }
    }
View Full Code Here

            RowKey key = entry.getKey();
            EventColumn keyColumn = key.getKeys().get(0);
            Assert.assertEquals(KEY_VALUE_NEW2, keyColumn.getColumnValue());
            Assert.assertEquals(KEY_NAME, keyColumn.getColumnName());

            EventData eventData = entry.getValue();
            Assert.assertEquals(SCHEMA_NAME, eventData.getSchemaName());
            Assert.assertEquals(TABLE_NAME, eventData.getTableName());
            Assert.assertEquals(TABLE_ID, eventData.getTableId());
            Assert.assertEquals(EventType.UPDATE, eventData.getEventType());

            List<EventColumn> oldKeys = eventData.getOldKeys();
            List<EventColumn> keys = eventData.getKeys();
            Assert.assertNotSame(oldKeys, keys);
        }
    }
View Full Code Here

TOP

Related Classes of com.alibaba.otter.shared.etl.model.EventData

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.