Package com.alibaba.otter.shared.etl.model

Examples of com.alibaba.otter.shared.etl.model.EventColumn


        eventData.setSchemaName("otter");
        eventData.setTableName("test");
        eventData.setEventType(EventType.INSERT);
        eventData.setExecuteTime(100L);

        EventColumn primaryKey = new EventColumn();
        primaryKey.setColumnName("id");
        primaryKey.setColumnType(1);
        primaryKey.setColumnValue("1");
        primaryKey.setKey(true);
        primaryKey.setNull(false);
        eventData.getKeys().add(primaryKey);

        EventColumn column = new EventColumn();
        column.setColumnName("name");
        column.setColumnType(1);
        column.setColumnValue("test");
        column.setKey(false);
        column.setNull(false);
        eventData.getColumns().add(column);

        rowBatch.merge(eventData);

        DbBatch dbBatch = new DbBatch();
View Full Code Here


        return eventData;
    }

    private List<EventColumn> getPrimary(int value) {
        EventColumn pk = new EventColumn();
        pk.setColumnName("id");
        pk.setColumnType(java.sql.Types.INTEGER);
        pk.setColumnValue(String.valueOf(value));
        pk.setIndex(1);
        pk.setNull(false);
        return Arrays.asList(pk);
    }
View Full Code Here

        result.add(buildColumn("gmt_modify", Types.TIMESTAMP, "2011-01-01 11:11:11", false, false));
        return result;
    }

    private EventColumn buildColumn(String name, int type, String value, boolean isKey, boolean isNull) {
        EventColumn column = new EventColumn();
        column.setColumnName(name);
        column.setColumnType(type);
        column.setColumnValue(value);
        column.setKey(isKey);
        column.setNull(isNull);
        return column;
    }
View Full Code Here

        return eventData;
    }

    private List<EventColumn> getPrimary(int value) {
        EventColumn pk = new EventColumn();
        pk.setColumnName("id");
        pk.setColumnType(java.sql.Types.INTEGER);
        pk.setColumnValue(String.valueOf(value));
        pk.setIndex(1);
        pk.setNull(false);
        return Arrays.asList(pk);
    }
View Full Code Here

        pk.setNull(false);
        return Arrays.asList(pk);
    }

    private List<EventColumn> getColumn(int value) {
        EventColumn c1 = new EventColumn();
        c1.setColumnName("type");
        c1.setColumnValue("I");

        EventColumn c2 = new EventColumn();
        c2.setColumnName("table_id");
        c2.setColumnValue(String.valueOf(value));

        EventColumn c3 = new EventColumn();
        c3.setColumnName("pk_data");
        c3.setColumnValue(String.valueOf(value) + ((char) 1) + "hello");
        return Arrays.asList(c1, c2, c3);
    }
View Full Code Here

                    columns.addAll(data.getOldKeys());
                }
            }
            for (int i = 0; i < columns.size(); i++) {
                int paramIndex = i + 1;
                EventColumn column = columns.get(i);
                int sqlType = column.getColumnType();

                // 获取一下当前字段名的数据是否必填
                Table table = dbDialect.findTable(data.getSchemaName(), data.getTableName());
                Map<String, Boolean> isRequiredMap = new HashMap<String, Boolean>();
                for (Column tableColumn : table.getColumns()) {
                    isRequiredMap.put(StringUtils.lowerCase(tableColumn.getName()), tableColumn.isRequired());
                }

                Boolean isRequired = isRequiredMap.get(StringUtils.lowerCase(column.getColumnName()));
                if (isRequired == null) {
                    throw new LoadException(String.format("column name %s is not found in Table[%s]",
                                                          column.getColumnName(), table.toString()));
                }

                Object param = SqlUtils.stringToSqlValue(column.getColumnValue(), sqlType, isRequired,
                                                         dbDialect.isEmptyStringNulled());
                try {
                    switch (sqlType) {
                        case Types.CLOB:
                            lobCreator.setClobAsString(ps, paramIndex, (String) param);
                            break;

                        case Types.BLOB:
                            lobCreator.setBlobAsBytes(ps, paramIndex, (byte[]) param);
                            break;
                        case Types.TIME:
                        case Types.TIMESTAMP:
                        case Types.DATE:
                            // 只处理mysql的时间类型,oracle的进行转化处理
                            if (dbDialect instanceof MysqlDialect) {
                                // 解决mysql的0000-00-00 00:00:00问题,直接依赖mysql driver进行处理,如果转化为Timestamp会出错
                                ps.setObject(paramIndex, column.getColumnValue());
                                break;
                            }
                        default:
                            StatementCreatorUtils.setParameterValue(ps, paramIndex, sqlType, null, param);
                            break;
View Full Code Here

    private List<EventColumn> translateColumns(EventData data, List<EventColumn> columns, DataMediaPair dataMediaPair,
                                               Multimap<String, String> translateColumnNames,
                                               TableInfoHolder tableHolder) {
        List<EventColumn> tcolumns = new ArrayList<EventColumn>();
        for (EventColumn scolumn : columns) {
            EventColumn tcolumn = translateColumn(data, scolumn, tableHolder, dataMediaPair, translateColumnNames);
            if (tcolumn != null) {
                tcolumns.add(tcolumn);
            }
        }
        return tcolumns;
View Full Code Here

                                   List<EventColumn> columns, DataMediaPair dataMediaPair,
                                   Multimap<String, String> translateColumnNames, TableInfoHolder tableHolder) {
        if (CollectionUtils.isEmpty(oldPks)) { // 如果不存在主键变更
            List<EventColumn> tpks = new ArrayList<EventColumn>();
            for (EventColumn scolumn : pks) {
                EventColumn tcolumn = translateColumn(data, scolumn, tableHolder, dataMediaPair, translateColumnNames);
                if (tcolumn != null) {
                    tpks.add(tcolumn);
                }
            }

            data.setKeys(tpks);
        } else { // 存在主键变更
            // modify by ljh at 2012-11-07 , 只做view视图映射的转化,不再做update table xxx set pk = newPK where pk = oldPk的处理
            List<EventColumn> tnewPks = new ArrayList<EventColumn>();
            List<EventColumn> toldPks = new ArrayList<EventColumn>();
            for (int i = 0; i < pks.size(); i++) {
                EventColumn newPk = pks.get(i);
                EventColumn oldPk = oldPks.get(i);
                // 转化new pk
                EventColumn tnewPk = translateColumn(data, newPk, tableHolder, dataMediaPair, translateColumnNames);
                if (tnewPk != null) {
                    tnewPks.add(tnewPk);
                    // 转化old pk,这里不能再用translateColumnNames了,因为转化new pk已经remove过一次view name了
                    toldPks.add(translateColumn(tnewPk, oldPk.getColumnValue(), dataMediaPair));
                }
View Full Code Here

    }

    private EventColumn translateColumn(EventData data, EventColumn scolumn, TableInfoHolder tableHolder,
                                        DataMediaPair dataMediaPair, Multimap<String, String> translateColumnNames) {
        EventType type = data.getEventType();
        EventColumn tcolumn = new EventColumn();
        tcolumn.setNull(scolumn.getColumnValue() == null);
        tcolumn.setKey(scolumn.isKey());// 左右两边的主键值必须保持一样,可以不为物理主键
        tcolumn.setIndex(scolumn.getIndex());
        tcolumn.setUpdate(scolumn.isUpdate());

        String columnName = translateColumnName(scolumn.getColumnName(), dataMediaPair, translateColumnNames);
        if (StringUtils.isBlank(columnName)) {
            throw new TransformException("can't translate column name:" + scolumn.getColumnName() + "in pair:"
                                         + dataMediaPair.toString());
        }

        // 特殊处理
        // columnName = StringUtils.remove(columnName, "`"); // 处理下特殊字符,eromanga给了错误的字段名
        tcolumn.setColumnName(columnName);
        tcolumn.setColumnType(scolumn.getColumnType());// 不反查,直接使用源库的类型
        if (tableHolder != null) {
            // modify by ljh at 2013-01-23
            // 双向同步新增字段,在一边加了字段后,虽然新的字段没有产生业务变化,但会因为某些原因导致传递了新的字段到T模块
            // 此时在目标库并不存在这个字段,导致一直挂起。ps. mysql新增字段时间不是一般的长
            // 所以,做了一个容错处理,针对目标库不存在的字段,如果变更记录在源库不存在变更,并且是null值的,允许丢弃该字段(其实最好还是要判断源库的column的defaultValue和当前值是否一致)
            boolean canColumnsNotExist = tableHolder.isEnableCompatibleMissColumn();
            if (type == EventType.UPDATE) {
                // 非变更字段,且当前值为null
                canColumnsNotExist &= !scolumn.isUpdate() && scolumn.isNull();
            } else if (type == EventType.INSERT) {
                // 当前值为null
                canColumnsNotExist &= scolumn.isNull();
            } else if (type == EventType.DELETE) {
                canColumnsNotExist &= !scolumn.isKey(); // 主键不允许不存在
            }

            Column matchDbColumn = getMatchColumn(tableHolder.getTable().getColumns(), tcolumn.getColumnName());
            // 匹配字段为空,可能源库发生过DDL操作,目标库重新载入一下meta信息
            if (matchDbColumn == null) { // 尝试reload一下table meta
                // 获取目标库的表信息
                DbMediaSource dbMediaSource = (DbMediaSource) dataMediaPair.getTarget().getSource();
                DbDialect dbDialect = dbDialectFactory.getDbDialect(dataMediaPair.getPipelineId(), dbMediaSource);
                String schemaName = tableHolder.getTable().getSchema();
                if (StringUtils.isEmpty(schemaName)) {
                    schemaName = tableHolder.getTable().getCatalog();
                }
                Table table = dbDialect.findTable(schemaName, tableHolder.getTable().getName(), false); // 强制反查一次,并放入cache

                tableHolder.setTable(table);
                matchDbColumn = getMatchColumn(tableHolder.getTable().getColumns(), tcolumn.getColumnName());
                if (matchDbColumn == null) {
                    if (canColumnsNotExist) {
                        return null;
                    } else {
                        throw new TransformException(scolumn.getColumnName() + " is not found in " + table.toString()
                                                     + " and source : " + dataMediaPair.getTarget().getNamespace()
                                                     + "." + dataMediaPair.getTarget().getName());
                    }
                }
            }

            if (tableHolder.isUseTableTransform()) {
                int sqlType = matchDbColumn.getTypeCode();
                tcolumn.setColumnType(sqlType);
            }
        }

        if (dataMediaPair.getTarget().getSource().getType().isOracle()) {
            // 特殊处理下oracle编码
            String encodeValue = SqlUtils.encoding(scolumn.getColumnValue(), scolumn.getColumnType(),
                                                   dataMediaPair.getSource().getSource().getEncode(),
                                                   dataMediaPair.getTarget().getSource().getEncode());
            tcolumn.setColumnValue(encodeValue);
        } else {
            // mysql编码转化已经在驱动层面上完成
            tcolumn.setColumnValue(scolumn.getColumnValue());
        }
        translateColumnNames.remove(scolumn.getColumnName(), columnName);// 删除映射关系,避免下次重复转换
        return tcolumn;
    }
View Full Code Here

        return tcolumn;
    }

    // 根据pk的值 + oldPk的value重新构造一个column对象,用于where pk = oldValue
    private EventColumn translateColumn(EventColumn scolumn, String newValue, DataMediaPair dataMediaPair) {
        EventColumn tcolumn = new EventColumn();
        tcolumn.setNull(newValue == null);
        tcolumn.setKey(scolumn.isKey());// 左右两边的主键值必须保持一样,可以不为物理主键
        tcolumn.setIndex(scolumn.getIndex());
        tcolumn.setColumnName(scolumn.getColumnName());
        tcolumn.setColumnType(scolumn.getColumnType());
        tcolumn.setUpdate(scolumn.isUpdate());
        if (dataMediaPair.getTarget().getSource().getType().isOracle()) {
            // 特殊处理下oracle编码
            String encodeValue = SqlUtils.encoding(newValue, scolumn.getColumnType(),
                                                   dataMediaPair.getSource().getSource().getEncode(),
                                                   dataMediaPair.getTarget().getSource().getEncode());
            tcolumn.setColumnValue(encodeValue);
        } else {
            tcolumn.setColumnValue(newValue);
        }
        return tcolumn;
    }
View Full Code Here

TOP

Related Classes of com.alibaba.otter.shared.etl.model.EventColumn

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.