Package com.alibaba.otter.shared.etl.model

Examples of com.alibaba.otter.shared.etl.model.Identity

56
57
58
59
60
61
62
63
64
65
    private Identity            identity = null;

    @BeforeMethod
    public void setUp() {
        identity = new Identity();
        identity.setChannelId(100L);
        identity.setPipelineId(100L);
        identity.setProcessId(100L);
    }
View Full Code Here
53
54
55
56
57
58
59
60
61
62
    private Identity            identity = null;

    @BeforeMethod
    public void setUp() {
        identity = new Identity();
        identity.setChannelId(100L);
        identity.setPipelineId(100L);
        identity.setProcessId(100L);
    }
View Full Code Here
43
44
45
46
47
48
49
50
51
52
53
54
55
56
        System.setProperty("nid", "1");
    }

    @Test
    public void test_simple() {
        Identity identity = new Identity();
        identity.setChannelId(100L);
        identity.setPipelineId(100L);
        identity.setProcessId(100L);

        RowBatch rowBatch = new RowBatch();
        rowBatch.setIdentity(identity);

        FileBatch fileBatch = new FileBatch();
View Full Code Here
107
108
109
110
111
112
113
114
115
116
117
    /**
     * 返回结果为已处理成功的记录
     */
    public DbLoadContext load(RowBatch rowBatch, WeightController controller) {
        Assert.notNull(rowBatch);
        Identity identity = rowBatch.getIdentity();
        DbLoadContext context = buildContext(identity);

        try {
            List<EventData> datas = rowBatch.getDatas();
            context.setPrepareDatas(datas);
View Full Code Here
130
131
132
133
134
135
136
137
138
139
140
    /**
     * 更新一下事务标记
     */
    private void updateMark(DbLoadContext context, DbDialect dialect, int threadId, String sql, boolean needInfo) {
        Identity identity = context.getIdentity();
        Channel channel = context.getChannel();
        // 获取dbDialect
        String markTableName = context.getPipeline().getParameters().getSystemSchema() + "."
                               + context.getPipeline().getParameters().getSystemMarkTable();
        String markTableColumn = context.getPipeline().getParameters().getSystemMarkTableColumn();
View Full Code Here
291
292
293
294
295
296
297
298
299
300
301
302
303
304
                            }

                            Channel channel = configClientService.findChannelByPipelineId(pipelineId);
                            RowBatch rowBatch = new RowBatch();
                            // 构造唯一标识
                            Identity identity = new Identity();
                            identity.setChannelId(channel.getId());
                            identity.setPipelineId(pipelineId);
                            identity.setProcessId(etlEventData.getProcessId());
                            rowBatch.setIdentity(identity);
                            // 进行数据合并
                            for (EventData data : eventData) {
                                rowBatch.merge(data);
                            }
View Full Code Here
63
64
65
66
67
68
69
70
71
72
73
74
    public void extract(DbBatch dbBatch) throws ExtractException {
        List<FileData> fileDatas = doFileExtract(dbBatch.getRowBatch());
        FileBatch fileBatch = new FileBatch();
        fileBatch.setFiles(fileDatas);
        Identity identity = new Identity();
        identity.setChannelId(dbBatch.getRowBatch().getIdentity().getChannelId());
        identity.setPipelineId(dbBatch.getRowBatch().getIdentity().getPipelineId());
        identity.setProcessId(dbBatch.getRowBatch().getIdentity().getProcessId());
        fileBatch.setIdentity(identity);
        dbBatch.setFileBatch(fileBatch);
    }
View Full Code Here
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
    /**
     * 将rowBatch中的记录，按找载入的目标数据源进行分类
     */
    private List<RowBatch> split(RowBatch rowBatch) {
        final Identity identity = rowBatch.getIdentity();
        Map<DataMediaSource, RowBatch> result = new MapMaker().makeComputingMap(new Function<DataMediaSource, RowBatch>() {

            public RowBatch apply(DataMediaSource input) {
                RowBatch rowBatch = new RowBatch();
                rowBatch.setIdentity(identity);
                return rowBatch;
            }
        });

        for (EventData eventData : rowBatch.getDatas()) {
            // 获取介质信息
            DataMedia media = ConfigHelper.findDataMedia(configClientService.findPipeline(identity.getPipelineId()),
                                                         eventData.getTableId());
            result.get(media.getSource()).merge(eventData); // 归类
        }

        return new ArrayList<RowBatch>(result.values());
View Full Code Here
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
     * key : Class对象，代表生成的目标数据对象
     * value : 每种目标数据对象的集合数据
     * </pre>
     */
    public Map<Class, BatchObject> transform(RowBatch rowBatch) {
        final Identity identity = translateIdentity(rowBatch.getIdentity());
        Map<Class, BatchObject> result = new HashMap<Class, BatchObject>();
        // 初始化默认值
        result.put(EventData.class, initBatchObject(identity, EventData.class));

        for (EventData eventData : rowBatch.getDatas()) {
            // 处理eventData
            Long tableId = eventData.getTableId();
            Pipeline pipeline = configClientService.findPipeline(identity.getPipelineId());
            // 针对每个同步数据，可能会存在多路复制的情况
            List<DataMediaPair> dataMediaPairs = ConfigHelper.findDataMediaPairByMediaId(pipeline, tableId);
            for (DataMediaPair pair : dataMediaPairs) {
                if (!pair.getSource().getId().equals(tableId)) { // 过滤tableID不为源的同步
                    continue;
View Full Code Here
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
    /**
     * 转化FileBatch对象
     */
    public Map<Class, BatchObject> transform(FileBatch fileBatch) {
        final Identity identity = translateIdentity(fileBatch.getIdentity());
        List<FileData> fileDatas = fileBatch.getFiles();
        Map<Class, BatchObject> result = new HashMap<Class, BatchObject>();
        // 初始化默认值
        result.put(FileData.class, initBatchObject(identity, FileData.class));

        for (FileData fileData : fileDatas) {
            // 进行转化
            Long tableId = fileData.getTableId();
            Pipeline pipeline = configClientService.findPipeline(identity.getPipelineId());
            // 针对每个同步数据，可能会存在多路复制的情况
            List<DataMediaPair> dataMediaPairs = ConfigHelper.findDataMediaPairByMediaId(pipeline, tableId);
            for (DataMediaPair pair : dataMediaPairs) {
                if (!pair.getSource().getId().equals(tableId)) { // 过滤tableID不为源的同步
                    continue;
View Full Code Here
TOP

Related Classes of com.alibaba.otter.shared.etl.model.Identity

  • com.alibaba.otter.node.etl.common.pipe.HttpPipeIntegration
  • com.alibaba.otter.node.etl.common.pipe.impl.http.RowDataHttpPipe
  • com.alibaba.otter.node.etl.common.pipe.MemoryPipeTest
  • com.alibaba.otter.node.etl.common.pipe.RpcPipeTest
  • com.alibaba.otter.node.etl.conflict.FileBatchConflictDetectServiceIntegration
  • com.alibaba.otter.node.etl.extract.DatabaseExtractorTest
  • com.alibaba.otter.node.etl.extract.extractor.FileExtractor
  • com.alibaba.otter.node.etl.extract.FreedomExtractorTest
  • com.alibaba.otter.node.etl.launcher.OtterLoaderFactoryIntegration
  • com.alibaba.otter.node.etl.load.DbLoadActionTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.