Package com.alibaba.otter.shared.etl.model

Examples of com.alibaba.otter.shared.etl.model.EventData


        }
        return pairs;
    }

    private EventData getEventData(long tableId, int value) {
        EventData eventData = new EventData();
        eventData.setTableId(tableId);
        eventData.setEventType(EventType.INSERT);
        eventData.setExecuteTime(new Date().getTime());
        eventData.setKeys(getPrimary(value));
        eventData.setColumns(getColumn(value));

        return eventData;
    }
View Full Code Here


        // 构造数据
        RowBatch rowBatch = new RowBatch();
        rowBatch.setIdentity(identity);
        for (int tableId = start; tableId < start + count; tableId++) {
            for (int i = start; i < start + count; i++) {
                EventData eventData = getEventData(tableId, i);
                eventData.setSchemaName("retl");
                eventData.setTableName("retl_buffer");
                rowBatch.merge(eventData);
            }
        }

        DbBatch dbBatch = new DbBatch(rowBatch);
View Full Code Here

        // 构造数据
        RowBatch rowBatch = new RowBatch();
        rowBatch.setIdentity(identity);
        for (int tableId = start; tableId < start + count; tableId++) {
            for (int i = start; i < start + count; i++) {
                EventData eventData = getEventData(tableId, i);
                eventData.setSchemaName("retl");
                eventData.setTableName("retl_buffer");
                rowBatch.merge(eventData);
            }
        }

        DbBatch dbBatch = new DbBatch(rowBatch);
View Full Code Here

        freedomExtractor.extract(dbBatch);
        want.collection(dbBatch.getRowBatch().getDatas()).sizeEq(count * count);
    }

    private EventData getEventData(long tableId, int value) {
        EventData eventData = new EventData();
        eventData.setTableId(tableId);
        eventData.setEventType(EventType.INSERT);
        eventData.setExecuteTime(new Date().getTime());
        eventData.setKeys(getPrimary(value));
        eventData.setColumns(getColumn(value));

        return eventData;
    }
View Full Code Here

                            // 更新统计信息
                            for (int i = 0; i < splitDatas.size(); i++) {
                                processStat(splitDatas.get(i), affects[i], true);
                            }
                        } else {
                            final EventData data = splitDatas.get(0);// 直接取第一条
                            int affect = 0;
                            affect = (Integer) dbDialect.getTransactionTemplate().execute(new TransactionCallback() {

                                public Object doInTransaction(TransactionStatus status) {
                                    try {
                                        failedDatas.clear(); // 先清理
                                        processedDatas.clear();
                                        interceptor.transactionBegin(context, Arrays.asList(data), dbDialect);
                                        JdbcTemplate template = dbDialect.getJdbcTemplate();
                                        int affect = template.update(data.getSql(), new PreparedStatementSetter() {

                                            public void setValues(PreparedStatement ps) throws SQLException {
                                                doPreparedStatement(ps, dbDialect, lobCreator, data);
                                            }
                                        });
View Full Code Here

        public DbLoadWorker(DbLoadContext context, List<EventData> datas, boolean canBatch){
            this.context = context;
            this.datas = datas;
            this.canBatch = canBatch;

            EventData data = datas.get(0); // eventData为同一数据库的记录,只取第一条即可
            DataMedia dataMedia = ConfigHelper.findDataMedia(context.getPipeline(), data.getTableId());
            dbDialect = dbDialectFactory.getDbDialect(context.getIdentity().getPipelineId(),
                                                      (DbMediaSource) dataMedia.getSource());

        }
View Full Code Here

        RowKey rowKey = new RowKey(eventData.getTableId(), eventData.getSchemaName(), eventData.getTableName(),
                                   eventData.getKeys());
        if (!result.containsKey(rowKey)) {
            result.put(rowKey, eventData);
        } else {
            EventData oldEventData = result.get(rowKey);
            eventData.setSize(oldEventData.getSize() + eventData.getSize());
            if (!CollectionUtils.isEmpty(oldEventData.getOldKeys())) {// 存在主键变更
                // insert/update -> delete记录组合时,delete的对应的pk为上一条记录的pk
                eventData.setKeys(oldEventData.getOldKeys());
                eventData.getOldKeys().clear();// 清除oldKeys

                result.remove(rowKey);// 删除老的对象
                result.put(new RowKey(eventData.getTableId(), eventData.getSchemaName(), eventData.getTableName(),
                                      eventData.getKeys()), eventData); // key发生变化,需要重新构造一个RowKey
View Full Code Here

public class RowDataTransformer extends AbstractOtterTransformer<EventData, EventData> {

    private DbDialectFactory dbDialectFactory;

    public EventData transform(EventData data, OtterTransformerContext context) {
        EventData result = new EventData();
        // 处理Table转化
        DataMedia dataMedia = context.getDataMediaPair().getTarget();
        result.setPairId(context.getDataMediaPair().getId());
        result.setTableId(dataMedia.getId());
        // 需要特殊处理下multi场景
        buildName(data, result, context.getDataMediaPair());
        result.setEventType(data.getEventType());
        result.setExecuteTime(data.getExecuteTime());
        result.setSyncConsistency(data.getSyncConsistency());
        result.setRemedy(data.isRemedy());
        result.setSyncMode(data.getSyncMode());
        result.setSize(data.getSize());
        if (data.getEventType().isDdl()) {
            // ddl不需要处理字段
            if (StringUtils.equalsIgnoreCase(result.getSchemaName(), data.getSchemaName())
                && StringUtils.equalsIgnoreCase(result.getTableName(), data.getTableName())) {
                // 是否需要对ddl sql进行转化,暂时不支持异构,必须保证源表和目标表的名字相同
                result.setDdlSchemaName(data.getDdlSchemaName());
                result.setSql(data.getSql());
                return result;
            } else {
                throw new TransformException("no support ddl for [" + data.getSchemaName() + "." + data.getTableName()
                                             + "] to [" + result.getSchemaName() + "." + result.getTableName()
                                             + "] , sql :" + data.getSql());
            }
        }

        Multimap<String, String> translateColumnNames = HashMultimap.create();
        if (context.getDataMediaPair().getColumnPairMode().isInclude()) { // 只针对正向匹配进行名字映射,exclude不做处理
            List<ColumnPair> columnPairs = context.getDataMediaPair().getColumnPairs();
            for (ColumnPair columnPair : columnPairs) {
                translateColumnNames.put(columnPair.getSourceColumn().getName(), columnPair.getTargetColumn().getName());
            }
        }
        // 准备一下table meta
        DataMediaPair dataMediaPair = context.getDataMediaPair();
        boolean useTableTransform = context.getPipeline().getParameters().getUseTableTransform();
        boolean enableCompatibleMissColumn = context.getPipeline().getParameters().getEnableCompatibleMissColumn();
        TableInfoHolder tableHolder = null;
        if (useTableTransform || enableCompatibleMissColumn) {// 控制一下是否需要反查table meta信息,如果同构数据库,完全没必要反查
            // 获取目标库的表信息
            DbDialect dbDialect = dbDialectFactory.getDbDialect(dataMediaPair.getPipelineId(),
                                                                (DbMediaSource) dataMedia.getSource());

            Table table = dbDialect.findTable(result.getSchemaName(), result.getTableName());
            tableHolder = new TableInfoHolder(table, useTableTransform, enableCompatibleMissColumn);
        }

        // 处理column转化
        List<EventColumn> otherColumns = translateColumns(result, data.getColumns(), context.getDataMediaPair(),
                                                          translateColumnNames, tableHolder);
        translatePkColumn(result, data.getKeys(), data.getOldKeys(), otherColumns, context.getDataMediaPair(),
                          translateColumnNames, tableHolder);

        result.setColumns(otherColumns);
        return result;
    }
View Full Code Here

            BatchProto.RowBatch rowbatchProto = BatchProto.RowBatch.parseFrom(new LimitedInputStream(input, length));
            // 构造原始的model对象
            RowBatch rowBatch = new RowBatch();
            rowBatch.setIdentity(build(rowbatchProto.getIdentity()));
            for (BatchProto.RowData rowDataProto : rowbatchProto.getRowsList()) {
                EventData eventData = new EventData();
                eventData.setPairId(rowDataProto.getPairId());
                eventData.setTableId(rowDataProto.getTableId());
                eventData.setTableName(rowDataProto.getTableName());
                eventData.setSchemaName(rowDataProto.getSchemaName());
                eventData.setEventType(EventType.valuesOf(rowDataProto.getEventType()));
                eventData.setExecuteTime(rowDataProto.getExecuteTime());
                // add by ljh at 2012-10-31
                if (StringUtils.isNotEmpty(rowDataProto.getSyncMode())) {
                    eventData.setSyncMode(SyncMode.valuesOf(rowDataProto.getSyncMode()));
                }
                if (StringUtils.isNotEmpty(rowDataProto.getSyncConsistency())) {
                    eventData.setSyncConsistency(SyncConsistency.valuesOf(rowDataProto.getSyncConsistency()));
                }
                // 处理主键
                List<EventColumn> keys = new ArrayList<EventColumn>();
                for (BatchProto.Column columnProto : rowDataProto.getKeysList()) {
                    keys.add(buildColumn(columnProto));
                }
                eventData.setKeys(keys);
                // 处理old主键
                if (CollectionUtils.isEmpty(rowDataProto.getOldKeysList()) == false) {
                    List<EventColumn> oldKeys = new ArrayList<EventColumn>();
                    for (BatchProto.Column columnProto : rowDataProto.getOldKeysList()) {
                        oldKeys.add(buildColumn(columnProto));
                    }
                    eventData.setOldKeys(oldKeys);
                }
                // 处理具体的column value
                List<EventColumn> columns = new ArrayList<EventColumn>();
                for (BatchProto.Column columnProto : rowDataProto.getColumnsList()) {
                    columns.add(buildColumn(columnProto));
                }
                eventData.setColumns(columns);

                eventData.setRemedy(rowDataProto.getRemedy());
                eventData.setSize(rowDataProto.getSize());
                eventData.setSql(rowDataProto.getSql());
                eventData.setDdlSchemaName(rowDataProto.getDdlSchemaName());
                // 添加到总记录
                rowBatch.merge(eventData);
            }
            dbBatch.setRowBatch(rowBatch);
View Full Code Here

                }

                boolean ddlSync = pipeline.getParameters().getDdlSync();
                if (ddlSync) {
                    // 处理下ddl操作
                    EventData eventData = new EventData();
                    eventData.setSchemaName(schemaName);
                    eventData.setTableName(tableName);
                    eventData.setEventType(eventType);
                    eventData.setExecuteTime(entry.getHeader().getExecuteTime());
                    eventData.setSql(rowChange.getSql());
                    eventData.setDdlSchemaName(rowChange.getDdlSchemaName());
                    eventData.setTableId(dataMedia.getId());
                    return Arrays.asList(eventData);
                } else {
                    return null;
                }
            }
        }

        List<EventData> eventDatas = new ArrayList<EventData>();
        for (RowData rowData : rowChange.getRowDatasList()) {
            EventData eventData = internParse(pipeline, entry, rowChange, rowData);
            if (eventData != null) {
                eventDatas.add(eventData);
            }
        }
View Full Code Here

TOP

Related Classes of com.alibaba.otter.shared.etl.model.EventData

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.