Package com.alibaba.otter.node.etl.common.db.dialect

Examples of com.alibaba.otter.node.etl.common.db.dialect.DbDialect


    }

    @Test(expectedExceptions = RuntimeException.class)
    public void test_oracle() {
        DbDataMedia media = getOracleMedia();
        final DbDialect dbDialect = dbDialectFactory.getDbDialect(1L, media.getSource());

        want.object(dbDialect).clazIs(OracleDialect.class);
        final SqlTemplate sqlTemplate = dbDialect.getSqlTemplate();
        final JdbcTemplate jdbcTemplate = dbDialect.getJdbcTemplate();
        final TransactionTemplate transactionTemplate = dbDialect.getTransactionTemplate();
        final int[] pkColumnTypes = { Types.NUMERIC, Types.VARCHAR };
        final int[] columnTypes = { Types.CHAR, Types.NUMERIC, Types.BLOB, Types.CLOB, Types.DATE, Types.DATE,
                Types.DATE };
        transactionTemplate.execute(new TransactionCallback() {
View Full Code Here


     * @param eventDatas
     */
    private void doDdl(DbLoadContext context, List<EventData> eventDatas) {
        for (final EventData data : eventDatas) {
            DataMedia dataMedia = ConfigHelper.findDataMedia(context.getPipeline(), data.getTableId());
            final DbDialect dbDialect = dbDialectFactory.getDbDialect(context.getIdentity().getPipelineId(),
                                                                      (DbMediaSource) dataMedia.getSource());
            Boolean skipDdlException = context.getPipeline().getParameters().getSkipDdlException();
            try {
                Boolean result = dbDialect.getJdbcTemplate().execute(new StatementCallback<Boolean>() {

                    public Boolean doInStatement(Statement stmt) throws SQLException, DataAccessException {
                        Boolean result = false;
                        if (dbDialect instanceof MysqlDialect && StringUtils.isNotEmpty(data.getDdlSchemaName())) {
                            //如果mysql,执行ddl时,切换到在源库执行的schema上
View Full Code Here

    private LoadInterceptor[] getIntercetptor(DbLoadContext context, List<EventData> currentData) {
        if (currentData == null || currentData.size() == 0) {
            return empty;
        }
        DataMedia dataMedia = ConfigHelper.findDataMedia(context.getPipeline(), currentData.get(0).getTableId());
        DbDialect dbDialect = dbDialectFactory.getDbDialect(context.getIdentity().getPipelineId(),
                                                            (DbMediaSource) dataMedia.getSource());

        if (dbDialect instanceof MysqlDialect) {
            return mysqlInterceptors;
        } else if (dbDialect instanceof OracleDialect) {
View Full Code Here

        boolean useTableTransform = context.getPipeline().getParameters().getUseTableTransform();
        boolean enableCompatibleMissColumn = context.getPipeline().getParameters().getEnableCompatibleMissColumn();
        TableInfoHolder tableHolder = null;
        if (useTableTransform || enableCompatibleMissColumn) {// 控制一下是否需要反查table meta信息,如果同构数据库,完全没必要反查
            // 获取目标库的表信息
            DbDialect dbDialect = dbDialectFactory.getDbDialect(dataMediaPair.getPipelineId(),
                                                                (DbMediaSource) dataMedia.getSource());

            Table table = dbDialect.findTable(result.getSchemaName(), result.getTableName());
            tableHolder = new TableInfoHolder(table, useTableTransform, enableCompatibleMissColumn);
        }

        // 处理column转化
        List<EventColumn> otherColumns = translateColumns(result, data.getColumns(), context.getDataMediaPair(),
View Full Code Here

            Column matchDbColumn = getMatchColumn(tableHolder.getTable().getColumns(), tcolumn.getColumnName());
            // 匹配字段为空,可能源库发生过DDL操作,目标库重新载入一下meta信息
            if (matchDbColumn == null) { // 尝试reload一下table meta
                // 获取目标库的表信息
                DbMediaSource dbMediaSource = (DbMediaSource) dataMediaPair.getTarget().getSource();
                DbDialect dbDialect = dbDialectFactory.getDbDialect(dataMediaPair.getPipelineId(), dbMediaSource);
                String schemaName = tableHolder.getTable().getSchema();
                if (StringUtils.isEmpty(schemaName)) {
                    schemaName = tableHolder.getTable().getCatalog();
                }
                Table table = dbDialect.findTable(schemaName, tableHolder.getTable().getName(), false); // 强制反查一次,并放入cache

                tableHolder.setTable(table);
                matchDbColumn = getMatchColumn(tableHolder.getTable().getColumns(), tcolumn.getColumnName());
                if (matchDbColumn == null) {
                    if (canColumnsNotExist) {
View Full Code Here

                            // 如果指定了tableId,需要按照tableId进行严格查找,如果没找到,那说明不需要进行同步
                            dataMedia = ConfigHelper.findDataMedia(pipeline,
                                                                   Long.valueOf(tableIdColumn.getColumnValue()));
                        }

                        DbDialect dbDialect = dbDialectFactory.getDbDialect(pipeline.getId(),
                                                                            (DbMediaSource) dataMedia.getSource());
                        // 考虑offer[1-128]的配置模式
                        if (!dataMedia.getNameMode().getMode().isSingle()
                            || !dataMedia.getNamespaceMode().getMode().isSingle()) {
                            boolean hasError = true;
                            EventColumn fullNameColumn = getMatchColumn(eventData.getColumns(), FULL_NAME);
                            if (fullNameColumn != null) {
                                String[] names = StringUtils.split(fullNameColumn.getColumnValue(), ".");
                                if (names.length >= 2) {
                                    eventData.setSchemaName(names[0]);
                                    eventData.setTableName(names[1]);
                                    hasError = false;
                                }
                            }

                            if (hasError) {
                                // 出现异常,需要记录一下
                                logger.warn("dataMedia mode:{} , fullname:{} ", dataMedia.getMode(),
                                            fullNameColumn == null ? null : fullNameColumn.getColumnValue());
                                removeDatas.add(eventData);
                                // 跳过这条记录
                                continue;
                            }
                        } else {
                            eventData.setSchemaName(dataMedia.getNamespace());
                            eventData.setTableName(dataMedia.getName());
                        }

                        // 更新业务类型
                        EventColumn typeColumn = getMatchColumn(eventData.getColumns(), TYPE);
                        EventType eventType = EventType.valuesOf(typeColumn.getColumnValue());
                        eventData.setEventType(eventType);
                        if (eventType.isUpdate()) {// 如果是update强制修改为insert,这样可以在目标端执行merge sql
                            eventData.setEventType(EventType.INSERT);
                        } else if (eventType.isDdl()) {
                            dbDialect.reloadTable(eventData.getSchemaName(), eventData.getTableName());
                            removeDatas.add(eventData);// 删除当前记录
                            continue;
                        }
                        // 重新构建新的业务主键字段
                        EventColumn pkDataColumn = getMatchColumn(eventData.getColumns(), PK_DATA);
                        String pkData = pkDataColumn.getColumnValue();
                        String[] pks = StringUtils.split(pkData, PK_SPLIT);

                        Table table = dbDialect.findTable(eventData.getSchemaName(), eventData.getTableName());
                        List<EventColumn> newColumns = new ArrayList<EventColumn>();
                        Column[] primaryKeyColumns = table.getPrimaryKeyColumns();
                        if (primaryKeyColumns.length > pks.length) {
                            throw new ExtractException("data pk column size not match , data:" + eventData.toString());
                        }
View Full Code Here

    private DbDialectFactory dbDialectFactory;

    public boolean before(DbLoadContext context, EventData currentData) {
        // 初步构建sql
        DbDialect dbDialect = dbDialectFactory.getDbDialect(context.getIdentity().getPipelineId(),
            (DbMediaSource) context.getDataMediaSource());
        SqlTemplate sqlTemplate = dbDialect.getSqlTemplate();
        EventType type = currentData.getEventType();
        String sql = null;

        // 注意insert/update语句对应的字段数序都是将主键排在后面
        if (type.isInsert()) {
View Full Code Here

     * @param eventDatas
     */
    private void doDdl(DbLoadContext context, List<EventData> eventDatas) {
        for (final EventData data : eventDatas) {
            DataMedia dataMedia = ConfigHelper.findDataMedia(context.getPipeline(), data.getTableId());
            final DbDialect dbDialect = dbDialectFactory.getDbDialect(context.getIdentity().getPipelineId(),
                (DbMediaSource) dataMedia.getSource());
            Boolean skipDdlException = context.getPipeline().getParameters().getSkipDdlException();
            try {
                Boolean result = dbDialect.getJdbcTemplate().execute(new StatementCallback<Boolean>() {

                    public Boolean doInStatement(Statement stmt) throws SQLException, DataAccessException {
                        Boolean result = false;
                        if (dbDialect instanceof MysqlDialect && StringUtils.isNotEmpty(data.getDdlSchemaName())) {
                            // 如果mysql,执行ddl时,切换到在源库执行的schema上
View Full Code Here

            if (eventType.isDdl()) {
                DataMedia dataMedia = ConfigHelper.findSourceDataMedia(pipeline, schemaName, tableName);
                // 如果EventType是CREATE/ALTER,需要reload
                // DataMediaInfo;并且把CREATE/ALTER类型的事件丢弃掉.
                if (eventType.isCreate() || eventType.isAlter() || eventType.isRename()) {
                    DbDialect dbDialect = dbDialectFactory.getDbDialect(pipeline.getId(),
                        (DbMediaSource) dataMedia.getSource());
                    dbDialect.reloadTable(schemaName, tableName);// 更新下meta信息
                }

                boolean ddlSync = pipeline.getParameters().getDdlSync();
                if (ddlSync) {
                    // 处理下ddl操作
View Full Code Here

            eventData.getTableName());
        eventData.setTableId(dataMedia.getId());
        if (useTableTransform || dataMedia.getSource().getType().isOracle()) {// oracle需要反查一次meta
            // 如果设置了需要进行table meta转化,则反查一下table信息
            // 比如oracle erosa解析时可能使用了非物理主键,需要直接使用,信任erosa的信息
            DbDialect dbDialect = dbDialectFactory.getDbDialect(pipeline.getId(), (DbMediaSource) dataMedia.getSource());
            table = dbDialect.findTable(eventData.getSchemaName(), eventData.getTableName());// 查询一下meta信息
            if (table == null) {
                logger.warn("find table[{}.{}] is null , may be drop table.",
                    eventData.getSchemaName(),
                    eventData.getTableName());
            }
View Full Code Here

TOP

Related Classes of com.alibaba.otter.node.etl.common.db.dialect.DbDialect

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.