Package com.alibaba.otter.shared.etl.model

Examples of com.alibaba.otter.shared.etl.model.EventColumn


    private List<EventColumn> translateColumns(EventData data, List<EventColumn> columns, DataMediaPair dataMediaPair,
                                               Multimap<String, String> translateColumnNames,
                                               TableInfoHolder tableHolder) {
        List<EventColumn> tcolumns = new ArrayList<EventColumn>();
        for (EventColumn scolumn : columns) {
            EventColumn tcolumn = translateColumn(data, scolumn, tableHolder, dataMediaPair, translateColumnNames);
            if (tcolumn != null) {
                tcolumns.add(tcolumn);
            }
        }
        return tcolumns;
View Full Code Here


                                   List<EventColumn> columns, DataMediaPair dataMediaPair,
                                   Multimap<String, String> translateColumnNames, TableInfoHolder tableHolder) {
        if (CollectionUtils.isEmpty(oldPks)) { // 如果不存在主键变更
            List<EventColumn> tpks = new ArrayList<EventColumn>();
            for (EventColumn scolumn : pks) {
                EventColumn tcolumn = translateColumn(data, scolumn, tableHolder, dataMediaPair, translateColumnNames);
                if (tcolumn != null) {
                    tpks.add(tcolumn);
                }
            }

            data.setKeys(tpks);
        } else { // 存在主键变更
            // modify by ljh at 2012-11-07 , 只做view视图映射的转化,不再做update table xxx
            // set pk = newPK where pk = oldPk的处理
            List<EventColumn> tnewPks = new ArrayList<EventColumn>();
            List<EventColumn> toldPks = new ArrayList<EventColumn>();
            for (int i = 0; i < pks.size(); i++) {
                EventColumn newPk = pks.get(i);
                EventColumn oldPk = oldPks.get(i);
                // 转化new pk
                EventColumn tnewPk = translateColumn(data, newPk, tableHolder, dataMediaPair, translateColumnNames);
                if (tnewPk != null) {
                    tnewPks.add(tnewPk);
                    // 转化old pk,这里不能再用translateColumnNames了,因为转化new
                    // pk已经remove过一次view name了
                    toldPks.add(translateColumn(tnewPk, oldPk.getColumnValue(), dataMediaPair));
View Full Code Here

    }

    private EventColumn translateColumn(EventData data, EventColumn scolumn, TableInfoHolder tableHolder,
                                        DataMediaPair dataMediaPair, Multimap<String, String> translateColumnNames) {
        EventType type = data.getEventType();
        EventColumn tcolumn = new EventColumn();
        tcolumn.setNull(scolumn.getColumnValue() == null);
        tcolumn.setKey(scolumn.isKey());// 左右两边的主键值必须保持一样,可以不为物理主键
        tcolumn.setIndex(scolumn.getIndex());
        tcolumn.setUpdate(scolumn.isUpdate());

        String columnName = translateColumnName(scolumn.getColumnName(), dataMediaPair, translateColumnNames);
        if (StringUtils.isBlank(columnName)) {
            throw new TransformException("can't translate column name:" + scolumn.getColumnName() + "in pair:"
                                         + dataMediaPair.toString());
        }

        // 特殊处理
        // columnName = StringUtils.remove(columnName, "`"); //
        // 处理下特殊字符,eromanga给了错误的字段名
        tcolumn.setColumnName(columnName);
        tcolumn.setColumnType(scolumn.getColumnType());// 不反查,直接使用源库的类型
        if (tableHolder != null) {
            // modify by ljh at 2013-01-23
            // 双向同步新增字段,在一边加了字段后,虽然新的字段没有产生业务变化,但会因为某些原因导致传递了新的字段到T模块
            // 此时在目标库并不存在这个字段,导致一直挂起。ps. mysql新增字段时间不是一般的长
            // 所以,做了一个容错处理,针对目标库不存在的字段,如果变更记录在源库不存在变更,并且是null值的,允许丢弃该字段(其实最好还是要判断源库的column的defaultValue和当前值是否一致)
            boolean canColumnsNotExist = tableHolder.isEnableCompatibleMissColumn();
            if (type == EventType.UPDATE) {
                // 非变更字段,且当前值为null
                canColumnsNotExist &= !scolumn.isUpdate() && scolumn.isNull();
            } else if (type == EventType.INSERT) {
                // 当前值为null
                canColumnsNotExist &= scolumn.isNull();
            } else if (type == EventType.DELETE) {
                canColumnsNotExist &= !scolumn.isKey(); // 主键不允许不存在
            }

            Column matchDbColumn = getMatchColumn(tableHolder.getTable().getColumns(), tcolumn.getColumnName());
            // 匹配字段为空,可能源库发生过DDL操作,目标库重新载入一下meta信息
            if (matchDbColumn == null) { // 尝试reload一下table meta
                // 获取目标库的表信息
                DbMediaSource dbMediaSource = (DbMediaSource) dataMediaPair.getTarget().getSource();
                DbDialect dbDialect = dbDialectFactory.getDbDialect(dataMediaPair.getPipelineId(), dbMediaSource);
                String schemaName = tableHolder.getTable().getSchema();
                if (StringUtils.isEmpty(schemaName)) {
                    schemaName = tableHolder.getTable().getCatalog();
                }
                Table table = dbDialect.findTable(schemaName, tableHolder.getTable().getName(), false); // 强制反查一次,并放入cache

                tableHolder.setTable(table);
                matchDbColumn = getMatchColumn(tableHolder.getTable().getColumns(), tcolumn.getColumnName());
                if (matchDbColumn == null) {
                    if (canColumnsNotExist) {
                        return null;
                    } else {
                        throw new TransformException(scolumn.getColumnName() + " is not found in " + table.toString()
                                                     + " and source : " + dataMediaPair.getTarget().getNamespace()
                                                     + "." + dataMediaPair.getTarget().getName());
                    }
                }
            }

            if (tableHolder.isUseTableTransform()) {
                int sqlType = matchDbColumn.getTypeCode();
                tcolumn.setColumnType(sqlType);
            }
        }

        // if (dataMediaPair.getTarget().getSource().getType().isOracle()) {
        // // 特殊处理下oracle编码
        // String encodeValue = SqlUtils.encoding(scolumn.getColumnValue(),
        // scolumn.getColumnType(),
        // dataMediaPair.getSource().getSource().getEncode(),
        // dataMediaPair.getTarget().getSource().getEncode());
        // tcolumn.setColumnValue(encodeValue);
        // } else {
        // mysql编码转化已经在驱动层面上完成
        tcolumn.setColumnValue(scolumn.getColumnValue());
        // }
        translateColumnNames.remove(scolumn.getColumnName(), columnName);// 删除映射关系,避免下次重复转换
        return tcolumn;
    }
View Full Code Here

        return tcolumn;
    }

    // 根据pk的值 + oldPk的value重新构造一个column对象,用于where pk = oldValue
    private EventColumn translateColumn(EventColumn scolumn, String newValue, DataMediaPair dataMediaPair) {
        EventColumn tcolumn = new EventColumn();
        tcolumn.setNull(newValue == null);
        tcolumn.setKey(scolumn.isKey());// 左右两边的主键值必须保持一样,可以不为物理主键
        tcolumn.setIndex(scolumn.getIndex());
        tcolumn.setColumnName(scolumn.getColumnName());
        tcolumn.setColumnType(scolumn.getColumnType());
        tcolumn.setUpdate(scolumn.isUpdate());
        if (dataMediaPair.getTarget().getSource().getType().isOracle()) {
            // 特殊处理下oracle编码
            String encodeValue = SqlUtils.encoding(newValue, scolumn.getColumnType(), dataMediaPair.getSource()
                .getSource()
                .getEncode(), dataMediaPair.getTarget().getSource().getEncode());
            tcolumn.setColumnValue(encodeValue);
        } else {
            tcolumn.setColumnValue(newValue);
        }
        return tcolumn;
    }
View Full Code Here

     *
     * @param column
     * @return
     */
    private EventColumn copyEventColumn(Column column, boolean isUpdate, TableInfoHolder tableHolder) {
        EventColumn eventColumn = new EventColumn();
        eventColumn.setIndex(column.getIndex());
        eventColumn.setKey(column.getIsKey());
        eventColumn.setNull(column.getIsNull());
        eventColumn.setColumnName(column.getName());
        eventColumn.setColumnValue(column.getValue());
        eventColumn.setUpdate(isUpdate);
        eventColumn.setColumnType(column.getSqlType());

        if (tableHolder != null && tableHolder.getTable() != null
            && (tableHolder.isUseTableTransform() || tableHolder.isOracle())) {
            org.apache.ddlutils.model.Column dbColumn = tableHolder.getTable().findColumn(column.getName(), false);
            if (dbColumn == null) {
                // 可能存在ddl,重新reload一下table
                tableHolder.reload();
                dbColumn = tableHolder.getTable().findColumn(column.getName(), false);
            }

            if (dbColumn != null) {
                int sqlType = dbColumn.getTypeCode();
                if (sqlType != column.getSqlType()) {
                    // 针对oracle的erosa给出的字段为非标准的jdbc,需要做一次类型反查
                    eventColumn.setColumnType(sqlType);
                    logger.info("table [{}] column [{}] is not match , MeType: {}, EType {}", new Object[] {
                            tableHolder.getTable().getName(), column.getName(), sqlType, column.getSqlType() });
                }
            }
        }
View Full Code Here

            IOUtils.closeQuietly(reader);
        }
    }

    private EventColumn buildColumn(BatchProto.Column columnProto) {
        EventColumn column = new EventColumn();
        column.setColumnName(columnProto.getName());
        column.setNull(columnProto.getIsNull());
        column.setColumnType(columnProto.getType());
        column.setColumnValue(columnProto.getValue());
        column.setKey(columnProto.getIsPrimaryKey());
        column.setIndex(columnProto.getIndex());
        column.setUpdate(columnProto.getIsUpdate());// add by ljh 2012-08-30,标记变更字段
        return column;
    }
View Full Code Here

                    columns.addAll(data.getOldKeys());
                }
            }
            for (int i = 0; i < columns.size(); i++) {
                int paramIndex = i + 1;
                EventColumn column = columns.get(i);
                int sqlType = column.getColumnType();

                // 获取一下当前字段名的数据是否必填
                Table table = dbDialect.findTable(data.getSchemaName(), data.getTableName());
                Map<String, Boolean> isRequiredMap = new HashMap<String, Boolean>();
                for (Column tableColumn : table.getColumns()) {
                    isRequiredMap.put(StringUtils.lowerCase(tableColumn.getName()), tableColumn.isRequired());
                }

                Boolean isRequired = isRequiredMap.get(StringUtils.lowerCase(column.getColumnName()));
                if (isRequired == null) {
                    throw new LoadException(String.format("column name %s is not found in Table[%s]",
                                                          column.getColumnName(), table.toString()));
                }

                Object param = SqlUtils.stringToSqlValue(column.getColumnValue(), sqlType, isRequired,
                                                         dbDialect.isEmptyStringNulled());
                try {
                    switch (sqlType) {
                        case Types.CLOB:
                            lobCreator.setClobAsString(ps, paramIndex, (String) param);
                            break;

                        case Types.BLOB:
                            lobCreator.setBlobAsBytes(ps, paramIndex, (byte[]) param);
                            break;
                        case Types.TIME:
                        case Types.TIMESTAMP:
                        case Types.DATE:
                            // 只处理mysql的时间类型,oracle的进行转化处理
                            if (dbDialect instanceof MysqlDialect) {
                                // 解决mysql的0000-00-00 00:00:00问题,直接依赖mysql driver进行处理,如果转化为Timestamp会出错
                                ps.setObject(paramIndex, column.getColumnValue());
                                break;
                            }
                        default:
                            StatementCreatorUtils.setParameterValue(ps, paramIndex, sqlType, null, param);
                            break;
View Full Code Here

    private List<EventColumn> translateColumns(EventData data, List<EventColumn> columns, DataMediaPair dataMediaPair,
                                               Multimap<String, String> translateColumnNames,
                                               TableInfoHolder tableHolder) {
        List<EventColumn> tcolumns = new ArrayList<EventColumn>();
        for (EventColumn scolumn : columns) {
            EventColumn tcolumn = translateColumn(data, scolumn, tableHolder, dataMediaPair, translateColumnNames);
            if (tcolumn != null) {
                tcolumns.add(tcolumn);
            }
        }
        return tcolumns;
View Full Code Here

                                   List<EventColumn> columns, DataMediaPair dataMediaPair,
                                   Multimap<String, String> translateColumnNames, TableInfoHolder tableHolder) {
        if (CollectionUtils.isEmpty(oldPks)) { // 如果不存在主键变更
            List<EventColumn> tpks = new ArrayList<EventColumn>();
            for (EventColumn scolumn : pks) {
                EventColumn tcolumn = translateColumn(data, scolumn, tableHolder, dataMediaPair, translateColumnNames);
                if (tcolumn != null) {
                    tpks.add(tcolumn);
                }
            }

            data.setKeys(tpks);
        } else { // 存在主键变更
            // modify by ljh at 2012-11-07 , 只做view视图映射的转化,不再做update table xxx set pk = newPK where pk = oldPk的处理
            List<EventColumn> tnewPks = new ArrayList<EventColumn>();
            List<EventColumn> toldPks = new ArrayList<EventColumn>();
            for (int i = 0; i < pks.size(); i++) {
                EventColumn newPk = pks.get(i);
                EventColumn oldPk = oldPks.get(i);
                // 转化new pk
                EventColumn tnewPk = translateColumn(data, newPk, tableHolder, dataMediaPair, translateColumnNames);
                if (tnewPk != null) {
                    tnewPks.add(tnewPk);
                    // 转化old pk,这里不能再用translateColumnNames了,因为转化new pk已经remove过一次view name了
                    toldPks.add(translateColumn(tnewPk, oldPk.getColumnValue(), dataMediaPair));
                }
View Full Code Here

    }

    private EventColumn translateColumn(EventData data, EventColumn scolumn, TableInfoHolder tableHolder,
                                        DataMediaPair dataMediaPair, Multimap<String, String> translateColumnNames) {
        EventType type = data.getEventType();
        EventColumn tcolumn = new EventColumn();
        tcolumn.setNull(scolumn.getColumnValue() == null);
        tcolumn.setKey(scolumn.isKey());// 左右两边的主键值必须保持一样,可以不为物理主键
        tcolumn.setIndex(scolumn.getIndex());
        tcolumn.setUpdate(scolumn.isUpdate());

        String columnName = translateColumnName(scolumn.getColumnName(), dataMediaPair, translateColumnNames);
        if (StringUtils.isBlank(columnName)) {
            throw new TransformException("can't translate column name:" + scolumn.getColumnName() + "in pair:"
                                         + dataMediaPair.toString());
        }

        // 特殊处理
        // columnName = StringUtils.remove(columnName, "`"); // 处理下特殊字符,eromanga给了错误的字段名
        tcolumn.setColumnName(columnName);
        tcolumn.setColumnType(scolumn.getColumnType());// 不反查,直接使用源库的类型
        if (tableHolder != null) {
            // modify by ljh at 2013-01-23
            // 双向同步新增字段,在一边加了字段后,虽然新的字段没有产生业务变化,但会因为某些原因导致传递了新的字段到T模块
            // 此时在目标库并不存在这个字段,导致一直挂起。ps. mysql新增字段时间不是一般的长
            // 所以,做了一个容错处理,针对目标库不存在的字段,如果变更记录在源库不存在变更,并且是null值的,允许丢弃该字段(其实最好还是要判断源库的column的defaultValue和当前值是否一致)
            boolean canColumnsNotExist = tableHolder.isEnableCompatibleMissColumn();
            if (type == EventType.UPDATE) {
                // 非变更字段,且当前值为null
                canColumnsNotExist &= !scolumn.isUpdate() && scolumn.isNull();
            } else if (type == EventType.INSERT) {
                // 当前值为null
                canColumnsNotExist &= scolumn.isNull();
            } else if (type == EventType.DELETE) {
                canColumnsNotExist &= !scolumn.isKey(); // 主键不允许不存在
            }

            Column matchDbColumn = getMatchColumn(tableHolder.getTable().getColumns(), tcolumn.getColumnName());
            // 匹配字段为空,可能源库发生过DDL操作,目标库重新载入一下meta信息
            if (matchDbColumn == null) { // 尝试reload一下table meta
                // 获取目标库的表信息
                DbMediaSource dbMediaSource = (DbMediaSource) dataMediaPair.getTarget().getSource();
                DbDialect dbDialect = dbDialectFactory.getDbDialect(dataMediaPair.getPipelineId(), dbMediaSource);
                String schemaName = tableHolder.getTable().getSchema();
                if (StringUtils.isEmpty(schemaName)) {
                    schemaName = tableHolder.getTable().getCatalog();
                }
                Table table = dbDialect.findTable(schemaName, tableHolder.getTable().getName(), false); // 强制反查一次,并放入cache

                tableHolder.setTable(table);
                matchDbColumn = getMatchColumn(tableHolder.getTable().getColumns(), tcolumn.getColumnName());
                if (matchDbColumn == null) {
                    if (canColumnsNotExist) {
                        return null;
                    } else {
                        throw new TransformException(scolumn.getColumnName() + " is not found in " + table.toString()
                                                     + " and source : " + dataMediaPair.getTarget().getNamespace()
                                                     + "." + dataMediaPair.getTarget().getName());
                    }
                }
            }

            if (tableHolder.isUseTableTransform()) {
                int sqlType = matchDbColumn.getTypeCode();
                tcolumn.setColumnType(sqlType);
            }
        }

        if (dataMediaPair.getTarget().getSource().getType().isOracle()) {
            // 特殊处理下oracle编码
            String encodeValue = SqlUtils.encoding(scolumn.getColumnValue(), scolumn.getColumnType(),
                                                   dataMediaPair.getSource().getSource().getEncode(),
                                                   dataMediaPair.getTarget().getSource().getEncode());
            tcolumn.setColumnValue(encodeValue);
        } else {
            // mysql编码转化已经在驱动层面上完成
            tcolumn.setColumnValue(scolumn.getColumnValue());
        }
        translateColumnNames.remove(scolumn.getColumnName(), columnName);// 删除映射关系,避免下次重复转换
        return tcolumn;
    }
View Full Code Here

TOP

Related Classes of com.alibaba.otter.shared.etl.model.EventColumn

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.