Package com.alibaba.otter.shared.common.model.config.data

Examples of com.alibaba.otter.shared.common.model.config.data.DataMedia


    private DbDialectFactory dbDialectFactory;

    public EventData transform(EventData data, OtterTransformerContext context) {
        EventData result = new EventData();
        // 处理Table转化
        DataMedia dataMedia = context.getDataMediaPair().getTarget();
        result.setPairId(context.getDataMediaPair().getId());
        result.setTableId(dataMedia.getId());
        // 需要特殊处理下multi场景
        buildName(data, result, context.getDataMediaPair());
        result.setEventType(data.getEventType());
        result.setExecuteTime(data.getExecuteTime());
        result.setSyncConsistency(data.getSyncConsistency());
        result.setRemedy(data.isRemedy());
        result.setSyncMode(data.getSyncMode());
        result.setSize(data.getSize());
        result.setHint(data.getHint());
        result.setWithoutSchema(data.isWithoutSchema());
        if (data.getEventType().isDdl()) {
            // ddl不需要处理字段
            if (StringUtils.equalsIgnoreCase(result.getSchemaName(), data.getSchemaName())
                && StringUtils.equalsIgnoreCase(result.getTableName(), data.getTableName())) {
                // 是否需要对ddl sql进行转化,暂时不支持异构,必须保证源表和目标表的名字相同
                result.setDdlSchemaName(data.getDdlSchemaName());
                result.setSql(data.getSql());
                return result;
            } else {
                throw new TransformException("no support ddl for [" + data.getSchemaName() + "." + data.getTableName()
                                             + "] to [" + result.getSchemaName() + "." + result.getTableName()
                                             + "] , sql :" + data.getSql());
            }
        }

        Multimap<String, String> translateColumnNames = HashMultimap.create();
        if (context.getDataMediaPair().getColumnPairMode().isInclude()) { // 只针对正向匹配进行名字映射,exclude不做处理
            List<ColumnPair> columnPairs = context.getDataMediaPair().getColumnPairs();
            for (ColumnPair columnPair : columnPairs) {
                translateColumnNames.put(columnPair.getSourceColumn().getName(), columnPair.getTargetColumn().getName());
            }
        }
        // 准备一下table meta
        DataMediaPair dataMediaPair = context.getDataMediaPair();
        boolean useTableTransform = context.getPipeline().getParameters().getUseTableTransform();
        boolean enableCompatibleMissColumn = context.getPipeline().getParameters().getEnableCompatibleMissColumn();
        TableInfoHolder tableHolder = null;
        if (useTableTransform || enableCompatibleMissColumn) {// 控制一下是否需要反查table
                                                              // meta信息,如果同构数据库,完全没必要反查
            // 获取目标库的表信息
            DbDialect dbDialect = dbDialectFactory.getDbDialect(dataMediaPair.getPipelineId(),
                (DbMediaSource) dataMedia.getSource());

            Table table = dbDialect.findTable(result.getSchemaName(), result.getTableName());
            tableHolder = new TableInfoHolder(table, useTableTransform, enableCompatibleMissColumn);
        }
View Full Code Here


     * 2. 源:offer[1-128] , 目:offer
     * 3. 源:offer[1-128] , 目:offer[1-128]
     * 4. 源:offer , 目:offer[1-128] 不支持,会报错
     */
    private void buildName(EventData data, EventData result, DataMediaPair pair) {
        DataMedia targetDataMedia = pair.getTarget();
        DataMedia sourceDataMedia = pair.getSource();
        String schemaName = buildName(data.getSchemaName(),
            sourceDataMedia.getNamespaceMode(),
            targetDataMedia.getNamespaceMode());
        String tableName = buildName(data.getTableName(), sourceDataMedia.getNameMode(), targetDataMedia.getNameMode());
        result.setSchemaName(schemaName);
        result.setTableName(tableName);
    }
View Full Code Here

                boolean notExistReturnNull = false;
                if (eventType.isRename()) {
                    notExistReturnNull = true;
                }

                DataMedia dataMedia = ConfigHelper.findSourceDataMedia(pipeline,
                    schemaName,
                    tableName,
                    notExistReturnNull);
                // 如果EventType是CREATE/ALTER,需要reload
                // DataMediaInfo;并且把CREATE/ALTER类型的事件丢弃掉.
                if (dataMedia != null && (eventType.isCreate() || eventType.isAlter() || eventType.isRename())) {
                    DbDialect dbDialect = dbDialectFactory.getDbDialect(pipeline.getId(),
                        (DbMediaSource) dataMedia.getSource());
                    dbDialect.reloadTable(schemaName, tableName);// 更新下meta信息
                }

                boolean ddlSync = pipeline.getParameters().getDdlSync();
                if (ddlSync) {
                    // 处理下ddl操作
                    EventData eventData = new EventData();
                    eventData.setSchemaName(schemaName);
                    eventData.setTableName(tableName);
                    eventData.setEventType(eventType);
                    eventData.setExecuteTime(entry.getHeader().getExecuteTime());
                    eventData.setSql(rowChange.getSql());
                    eventData.setDdlSchemaName(rowChange.getDdlSchemaName());
                    eventData.setTableId(dataMedia.getId());
                    return Arrays.asList(eventData);
                } else {
                    return null;
                }
            }
View Full Code Here

        TableInfoHolder tableHolder = null;

        if (!StringUtils.equalsIgnoreCase(pipeline.getParameters().getSystemSchema(), eventData.getSchemaName())) {
            boolean useTableTransform = pipeline.getParameters().getUseTableTransform();
            Table table = null;
            DataMedia dataMedia = ConfigHelper.findSourceDataMedia(pipeline,
                eventData.getSchemaName(),
                eventData.getTableName());
            eventData.setTableId(dataMedia.getId());
            if (useTableTransform || dataMedia.getSource().getType().isOracle()) {// oracle需要反查一次meta
                // 如果设置了需要进行table meta转化,则反查一下table信息
                // 比如oracle erosa解析时可能使用了非物理主键,需要直接使用,信任erosa的信息
                DbDialect dbDialect = dbDialectFactory.getDbDialect(pipeline.getId(),
                    (DbMediaSource) dataMedia.getSource());
                table = dbDialect.findTable(eventData.getSchemaName(), eventData.getTableName());// 查询一下meta信息
                if (table == null) {
                    logger.warn("find table[{}.{}] is null , may be drop table.",
                        eventData.getSchemaName(),
                        eventData.getTableName());
View Full Code Here

            this.context = context;
            this.datas = datas;
            this.canBatch = canBatch;

            EventData data = datas.get(0); // eventData为同一数据库的记录,只取第一条即可
            DataMedia dataMedia = ConfigHelper.findDataMedia(context.getPipeline(), data.getTableId());
            dbDialect = dbDialectFactory.getDbDialect(context.getIdentity().getPipelineId(),
                                                      (DbMediaSource) dataMedia.getSource());

        }
View Full Code Here

    private DbDialectFactory dbDialectFactory;

    public EventData transform(EventData data, OtterTransformerContext context) {
        EventData result = new EventData();
        // 处理Table转化
        DataMedia dataMedia = context.getDataMediaPair().getTarget();
        result.setPairId(context.getDataMediaPair().getId());
        result.setTableId(dataMedia.getId());
        // 需要特殊处理下multi场景
        buildName(data, result, context.getDataMediaPair());
        result.setEventType(data.getEventType());
        result.setExecuteTime(data.getExecuteTime());
        result.setSyncConsistency(data.getSyncConsistency());
        result.setRemedy(data.isRemedy());
        result.setSyncMode(data.getSyncMode());
        result.setSize(data.getSize());

        Multimap<String, String> translateColumnNames = HashMultimap.create();
        if (context.getDataMediaPair().getColumnPairMode().isInclude()) { // 只针对正向匹配进行名字映射,exclude不做处理
            List<ColumnPair> columnPairs = context.getDataMediaPair().getColumnPairs();
            for (ColumnPair columnPair : columnPairs) {
                translateColumnNames.put(columnPair.getSourceColumn().getName(), columnPair.getTargetColumn().getName());
            }
        }
        // 准备一下table meta
        DataMediaPair dataMediaPair = context.getDataMediaPair();
        boolean useTableTransform = context.getPipeline().getParameters().getUseTableTransform();
        boolean enableCompatibleMissColumn = context.getPipeline().getParameters().getEnableCompatibleMissColumn();
        TableInfoHolder tableHolder = null;
        if (useTableTransform || enableCompatibleMissColumn) {// 控制一下是否需要反查table meta信息,如果同构数据库,完全没必要反查
            // 获取目标库的表信息
            DbDialect dbDialect = dbDialectFactory.getDbDialect(dataMediaPair.getPipelineId(),
                                                                (DbMediaSource) dataMedia.getSource());

            Table table = dbDialect.findTable(result.getSchemaName(), result.getTableName());
            tableHolder = new TableInfoHolder(table, useTableTransform, enableCompatibleMissColumn);
        }
View Full Code Here

     * 2. 源:offer[1-128] , 目:offer
     * 3. 源:offer[1-128] , 目:offer[1-128]
     * 4. 源:offer , 目:offer[1-128] 不支持,会报错
     */
    private void buildName(EventData data, EventData result, DataMediaPair pair) {
        DataMedia targetDataMedia = pair.getTarget();
        DataMedia sourceDataMedia = pair.getSource();
        String schemaName = buildName(data.getSchemaName(), sourceDataMedia.getNamespaceMode(),
                                      targetDataMedia.getNamespaceMode());
        String tableName = buildName(data.getTableName(), sourceDataMedia.getNameMode(), targetDataMedia.getNameMode());
        result.setSchemaName(schemaName);
        result.setTableName(tableName);
    }
View Full Code Here

            if (StringUtils.equalsIgnoreCase(pipeline.getParameters().getSystemDualTable(), eventData.getTableName())) {
                // 心跳表数据直接忽略
                return null;
            }
        } else {
            DataMedia dataMedia = ConfigHelper.findSourceDataMedia(pipeline, eventData.getSchemaName(),
                                                                   eventData.getTableName());
            if (dataMedia == null) {
                throw new SelectException("the namespace = " + eventData.getSchemaName() + " name = "
                                          + eventData.getTableName() + " dataMedia is null,please check , entry: "
                                          + entry.toString() + " and rowdata: " + rowChange.toString());
            }
            eventData.setTableId(dataMedia.getId());
            if (eventType.isCreate() || eventType.isAlter() || eventType.isErase()) {
                // 如果EventType是CREATE/ALTER,需要reload DataMediaInfo;并且把CREATE/ALTER类型的事件丢弃掉.
                if (eventType.isCreate() || eventType.isAlter()) {
                    DbDialect dbDialect = dbDialectFactory.getDbDialect(pipeline.getId(),
                                                                        (DbMediaSource) dataMedia.getSource());
                    dbDialect.reloadTable(eventData.getSchemaName(), eventData.getTableName());// 更新下meta信息
                }
                return null;
            }

            if (useTableTransform || dataMedia.getSource().getType().isOracle()) {// oracle需要反查一次meta
                // 如果设置了需要进行table meta转化,则反查一下table信息
                // 比如oracle erosa解析时可能使用了非物理主键,需要直接使用,信任erosa的信息
                DbDialect dbDialect = dbDialectFactory.getDbDialect(pipeline.getId(),
                                                                    (DbMediaSource) dataMedia.getSource());
                table = dbDialect.findTable(eventData.getSchemaName(), eventData.getTableName());// 查询一下meta信息
                if (table == null) {
                    logger.warn("find table[{}.{}] is null , may be drop table.", eventData.getSchemaName(),
                                eventData.getTableName());
                }
View Full Code Here

        public void run() {
            try {
                MDC.put(OtterConstants.splitPipelineLogFileKey, String.valueOf(pipeline.getId()));
                Thread.currentThread().setName(String.format(WORKER_NAME_FORMAT, pipeline.getId(), pipeline.getName()));
                // 获取数据表信息
                DataMedia dataMedia = ConfigHelper.findDataMedia(pipeline, eventData.getTableId());
                DbDialect dbDialect = dbDialectFactory.getDbDialect(pipeline.getId(),
                                                                    (DbMediaSource) dataMedia.getSource());
                Table table = dbDialect.findTable(eventData.getSchemaName(), eventData.getTableName());
                TableData keyTableData = buildTableData(table, eventData.getKeys());

                // oracle类型特殊处理下
                if (dbDialect instanceof OracleDialect) {
                    keyTableData.columnTypes = getOraclePkTypes(table, keyTableData.columnNames);
                }

                boolean needAll = pipeline.getParameters().getSyncMode().isRow()
                                  || (eventData.getSyncMode() != null && eventData.getSyncMode().isRow());

                // 增加一种case, 针对oracle erosa有时侯结果记录只有主键,没有变更字段,需要做一次反查,获取所有字段
                needAll |= CollectionUtils.isEmpty(eventData.getUpdatedColumns())
                           && dataMedia.getSource().getType().isOracle();

                List<DataMediaPair> mediaParis = ConfigHelper.findDataMediaPairByMediaId(pipeline, dataMedia.getId());
                List<String> viewColumnNames = buildMaxColumnsFromColumnPairs(mediaParis, eventData.getKeys());

                // TODO 后续版本测试下
                // if (needAll) {
                // boolean needDb = checkNeedDbForRowMode(table, viewColumnNames, eventData);
View Full Code Here

                if (eventData.getEventType().isInsert() || eventData.getEventType().isUpdate()) {
                    // 重新改写一下EventData的数据,根据系统表的定义
                    EventColumn tableIdColumn = getMatchColumn(eventData.getColumns(), TABLE_ID);
                    // 获取到对应tableId的media信息
                    try {
                        DataMedia dataMedia = null;
                        Long tableId = Long.valueOf(tableIdColumn.getColumnValue());
                        eventData.setTableId(tableId);
                        if (tableId <= 0) { //直接按照full_name进行查找
                            //尝试直接根据schema+table name进行查找
                            EventColumn fullNameColumn = getMatchColumn(eventData.getColumns(), FULL_NAME);
                            if (fullNameColumn != null) {
                                String[] names = StringUtils.split(fullNameColumn.getColumnValue(), ".");
                                if (names.length >= 2) {
                                    dataMedia = ConfigHelper.findSourceDataMedia(pipeline, names[0], names[1]);
                                    eventData.setTableId(dataMedia.getId());
                                } else {
                                    throw new ConfigException("no such DataMedia " + names);
                                }
                            }
                        } else {
                            // 如果指定了tableId,需要按照tableId进行严格查找,如果没找到,那说明不需要进行同步
                            dataMedia = ConfigHelper.findDataMedia(pipeline,
                                                                   Long.valueOf(tableIdColumn.getColumnValue()));
                        }

                        DbDialect dbDialect = dbDialectFactory.getDbDialect(pipeline.getId(),
                                                                            (DbMediaSource) dataMedia.getSource());
                        // 考虑offer[1-128]的配置模式
                        if (!dataMedia.getNameMode().getMode().isSingle()
                            || !dataMedia.getNamespaceMode().getMode().isSingle()) {
                            boolean hasError = true;
                            EventColumn fullNameColumn = getMatchColumn(eventData.getColumns(), FULL_NAME);
                            if (fullNameColumn != null) {
                                String[] names = StringUtils.split(fullNameColumn.getColumnValue(), ".");
                                if (names.length >= 2) {
                                    eventData.setSchemaName(names[0]);
                                    eventData.setTableName(names[1]);
                                    hasError = false;
                                }
                            }

                            if (hasError) {
                                // 出现异常,需要记录一下
                                logger.warn("dataMedia mode:{} , fullname:{} ", dataMedia.getMode(),
                                            fullNameColumn == null ? null : fullNameColumn.getColumnValue());
                                removeDatas.add(eventData);
                                // 跳过这条记录
                                continue;
                            }
                        } else {
                            eventData.setSchemaName(dataMedia.getNamespace());
                            eventData.setTableName(dataMedia.getName());
                        }

                        // 更新业务类型
                        EventColumn typeColumn = getMatchColumn(eventData.getColumns(), TYPE);
                        EventType eventType = EventType.valuesOf(typeColumn.getColumnValue());
View Full Code Here

TOP

Related Classes of com.alibaba.otter.shared.common.model.config.data.DataMedia

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.