Package com.alibaba.otter.shared.common.model.config.data

Examples of com.alibaba.otter.shared.common.model.config.data.DataMediaPair

88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
            for (ColumnPair columnPair : columnPairs) {
                translateColumnNames.put(columnPair.getSourceColumn().getName(), columnPair.getTargetColumn().getName());
            }
        }
        // 准备一下table meta
        DataMediaPair dataMediaPair = context.getDataMediaPair();
        boolean useTableTransform = context.getPipeline().getParameters().getUseTableTransform();
        boolean enableCompatibleMissColumn = context.getPipeline().getParameters().getEnableCompatibleMissColumn();
        TableInfoHolder tableHolder = null;
        if (useTableTransform || enableCompatibleMissColumn) {// 控制一下是否需要反查table
                                                              // meta信息，如果同构数据库，完全没必要反查
            // 获取目标库的表信息
            DbDialect dbDialect = dbDialectFactory.getDbDialect(dataMediaPair.getPipelineId(),
                (DbMediaSource) dataMedia.getSource());

            Table table = dbDialect.findTable(result.getSchemaName(), result.getTableName());
            tableHolder = new TableInfoHolder(table, useTableTransform, enableCompatibleMissColumn);
        }
View Full Code Here
181
182
183
184
185
186
187
188
189
190
191
     */
    private WeightBuckets<EventData> buildWeightBuckets(DbLoadContext context, List<EventData> datas) {
        WeightBuckets<EventData> buckets = new WeightBuckets<EventData>();
        for (EventData data : datas) {
            // 获取对应的weight
            DataMediaPair pair = ConfigHelper.findDataMediaPair(context.getPipeline(), data.getPairId());
            buckets.addItem(pair.getPushWeight(), data);
        }

        return buckets;
    }
View Full Code Here
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
            for (ColumnPair columnPair : columnPairs) {
                translateColumnNames.put(columnPair.getSourceColumn().getName(), columnPair.getTargetColumn().getName());
            }
        }
        // 准备一下table meta
        DataMediaPair dataMediaPair = context.getDataMediaPair();
        boolean useTableTransform = context.getPipeline().getParameters().getUseTableTransform();
        boolean enableCompatibleMissColumn = context.getPipeline().getParameters().getEnableCompatibleMissColumn();
        TableInfoHolder tableHolder = null;
        if (useTableTransform || enableCompatibleMissColumn) {// 控制一下是否需要反查table meta信息，如果同构数据库，完全没必要反查
            // 获取目标库的表信息
            DbDialect dbDialect = dbDialectFactory.getDbDialect(dataMediaPair.getPipelineId(),
                                                                (DbMediaSource) dataMedia.getSource());

            Table table = dbDialect.findTable(result.getSchemaName(), result.getTableName());
            tableHolder = new TableInfoHolder(table, useTableTransform, enableCompatibleMissColumn);
        }
View Full Code Here
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
    /*-------------------------------------------------------------*/

    private DataMediaPair doToModel(DataMediaPairDO dataMediaPairDo, List<ColumnPair> columnPairs,
                                    List<ColumnGroup> columnGroups) {
        DataMediaPair dataMediaPair = new DataMediaPair();
        try {
            dataMediaPair.setId(dataMediaPairDo.getId());
            dataMediaPair.setPipelineId(dataMediaPairDo.getPipelineId());
            dataMediaPair.setPullWeight(dataMediaPairDo.getPullWeight());
            dataMediaPair.setPushWeight(dataMediaPairDo.getPushWeight());
            if (StringUtils.isNotBlank(dataMediaPairDo.getFilter())) {
                dataMediaPair.setFilterData(JsonUtils.unmarshalFromString(dataMediaPairDo.getFilter(),
                                                                          ExtensionData.class));
            }

            if (StringUtils.isNotBlank(dataMediaPairDo.getResolver())) {
                dataMediaPair.setResolverData(JsonUtils.unmarshalFromString(dataMediaPairDo.getResolver(),
                                                                            ExtensionData.class));
            }
            dataMediaPair.setColumnPairs(columnPairs);
            dataMediaPair.setColumnGroups(columnGroups);
            dataMediaPair.setColumnPairMode(dataMediaPairDo.getColumnPairMode());
            dataMediaPair.setGmtCreate(dataMediaPairDo.getGmtCreate());
            dataMediaPair.setGmtModified(dataMediaPairDo.getGmtModified());

            // 组装DataMedia
            List<DataMedia> dataMedias = dataMediaService.listByIds(dataMediaPairDo.getSourceDataMediaId(),
                                                                    dataMediaPairDo.getTargetDataMediaId());
            if (null == dataMedias || dataMedias.size() != 2) {
                // 抛出异常
                return dataMediaPair;
            }

            for (DataMedia dataMedia : dataMedias) {
                if (dataMedia.getId().equals(dataMediaPairDo.getSourceDataMediaId())) {
                    dataMediaPair.setSource(dataMedia);
                } else if (dataMedia.getId().equals(dataMediaPairDo.getTargetDataMediaId())) {
                    dataMediaPair.setTarget(dataMedia);
                }
            }
        } catch (Exception e) {
            logger.error("ERROR ## change the dataMediaPair Do to Model has an exception", e);
            throw new ManagerException(e);
View Full Code Here
134
135
136
137
138
139
140
141
142
143
        pipeline.setParameters(new PipelineParameter());

        int dataMediaPairCount = fileDataCount / NUMBER_OF_FILE_DATA_COPIES;
        pipeline.setPairs(new ArrayList<DataMediaPair>(dataMediaPairCount));
        for (int i = fileDataStartIndex; i < dataMediaPairCount; i++) {
            DataMediaPair dataMediaPair = buildDataMediaPair(i, i);
            pipeline.getPairs().add(dataMediaPair);
        }
        return pipeline;
    }
View Full Code Here
141
142
143
144
145
146
147
148
149
150
        }
        return pipeline;
    }

    protected DataMediaPair buildDataMediaPair(long id, long pushWeight) {
        DataMediaPair result = new DataMediaPair();
        result.setId(id);
        result.setPushWeight(pushWeight);
        return result;
    }
View Full Code Here
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
    }

    private List<DataMediaPair> generatorDataMediaPairForOracle(int count) {
        List<DataMediaPair> pairs = new ArrayList<DataMediaPair>();
        for (int i = 0; i < count; i++) {
            DataMediaPair pair = new DataMediaPair();
            int index = i + 1;
            pair.setId(Long.valueOf(index));
            pair.setPullWeight(count - Long.valueOf(index));
            pair.setPushWeight(count - Long.valueOf(index));

            DbDataMedia mysqlMedia = getMysqlMedia();
            mysqlMedia.setId(2L);
            pair.setSource(mysqlMedia);

            DbDataMedia oracleMedia = getOracleMedia();
            oracleMedia.setId(1L);
            pair.setTarget(oracleMedia);
            pairs.add(pair);
        }
        return pairs;
    }
View Full Code Here
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
    }

    private List<DataMediaPair> generatorDataMediaPairForMysql(int count) {
        List<DataMediaPair> pairs = new ArrayList<DataMediaPair>();
        for (int i = 0; i < count; i++) {
            DataMediaPair pair = new DataMediaPair();
            int index = i + 1;
            pair.setId(Long.valueOf(index));
            pair.setPullWeight(count - Long.valueOf(index));
            pair.setPushWeight(count - Long.valueOf(index));

            DbDataMedia oracleMedia = getOracleMedia();
            oracleMedia.setId(2L);
            pair.setSource(oracleMedia);

            DbDataMedia mysqlMedia = getMysqlMedia();
            mysqlMedia.setId(1L);
            pair.setTarget(mysqlMedia);
            pairs.add(pair);
        }
        return pairs;
    }
View Full Code Here
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
    }

    private List<DataMediaPair> generatorDataMediaPair(int count) {
        List<DataMediaPair> pairs = new ArrayList<DataMediaPair>();
        for (int i = 0; i < count; i++) {
            DataMediaPair pair = new DataMediaPair();
            pair.setId(Long.valueOf(i));
            pair.setPullWeight(Long.valueOf(i));
            pair.setPushWeight(Long.valueOf(i));

            DbDataMedia mysqlMedia = getMysqlMedia();
            mysqlMedia.setId(Long.valueOf(count + 1));
            pair.setSource(mysqlMedia);

            DbDataMedia oracleMedia = getOracleMedia();
            oracleMedia.setId(Long.valueOf(i));
            pair.setTarget(oracleMedia);
            pairs.add(pair);
        }
        return pairs;
    }
View Full Code Here
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
    public void test_rowData_mysql_oracle() {
        final Pipeline pipeline = new Pipeline();
        pipeline.setId(100L);

        List<DataMediaPair> pairs = new ArrayList<DataMediaPair>();
        DataMediaPair pair1 = new DataMediaPair();
        pair1.setId(1L);
        pair1.setPipelineId(pipeline.getId());
        pair1.setPullWeight(1L);
        pair1.setPushWeight(1L);

        DbDataMedia mysqlMedia = getMysqlMedia();
        mysqlMedia.setId(1L);
        pair1.setSource(mysqlMedia);

        DbDataMedia oracleMedia = getOracleMedia();
        pair1.setTarget(oracleMedia);
        pairs.add(pair1);
        pipeline.setPairs(pairs);
        PipelineParameter param = new PipelineParameter();
        param.setSyncMode(SyncMode.ROW);
View Full Code Here
TOP

Related Classes of com.alibaba.otter.shared.common.model.config.data.DataMediaPair

  • com.alibaba.otter.manager.biz.config.datamediapair.impl.DataMediaPairServiceImpl
  • com.alibaba.otter.manager.web.home.module.action.DataMediaPairAction
  • com.alibaba.otter.manager.web.home.module.screen.AddColumnPairGroup
  • com.alibaba.otter.manager.web.home.module.screen.BehaviorHistoryCurve
  • com.alibaba.otter.manager.web.home.module.screen.DataMediaPairInfo
  • com.alibaba.otter.manager.web.home.module.screen.EditDataMediaPair
  • com.alibaba.otter.node.etl.extract.DatabaseExtractorTest
  • com.alibaba.otter.node.etl.extract.FreedomExtractorTest
  • com.alibaba.otter.node.etl.load.DbLoadActionTest
  • com.alibaba.otter.node.etl.load.FileLoadActionTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.