Package com.alibaba.otter.shared.etl.model

Examples of com.alibaba.otter.shared.etl.model.Identity

105
106
107
108
109
110
111
112
113
114
115
    /**
     * 返回结果为已处理成功的记录
     */
    public DbLoadContext load(RowBatch rowBatch, WeightController controller) {
        Assert.notNull(rowBatch);
        Identity identity = rowBatch.getIdentity();
        DbLoadContext context = buildContext(identity);

        try {
            List<EventData> datas = rowBatch.getDatas();
            context.setPrepareDatas(datas);
View Full Code Here
291
292
293
294
295
296
297
298
299
300
301
302
303
304
                            }

                            Channel channel = configClientService.findChannelByPipelineId(pipelineId);
                            RowBatch rowBatch = new RowBatch();
                            // 构造唯一标识
                            Identity identity = new Identity();
                            identity.setChannelId(channel.getId());
                            identity.setPipelineId(pipelineId);
                            identity.setProcessId(etlEventData.getProcessId());
                            rowBatch.setIdentity(identity);
                            // 进行数据合并
                            for (EventData data : eventData) {
                                rowBatch.merge(data);
                            }
View Full Code Here
63
64
65
66
67
68
69
70
71
72
73
74
    public void extract(DbBatch dbBatch) throws ExtractException {
        List<FileData> fileDatas = doFileExtract(dbBatch.getRowBatch());
        FileBatch fileBatch = new FileBatch();
        fileBatch.setFiles(fileDatas);
        Identity identity = new Identity();
        identity.setChannelId(dbBatch.getRowBatch().getIdentity().getChannelId());
        identity.setPipelineId(dbBatch.getRowBatch().getIdentity().getPipelineId());
        identity.setProcessId(dbBatch.getRowBatch().getIdentity().getProcessId());
        fileBatch.setIdentity(identity);
        dbBatch.setFileBatch(fileBatch);
    }
View Full Code Here
324
325
326
327
328
329
330
331
332
333
334
        return identityBuilder.build();
    }

    // 从proto对象构造回object
    private Identity build(BatchProto.Identity identityProto) {
        Identity identity = new Identity();
        identity.setChannelId(identityProto.getChannelId());
        identity.setPipelineId(identityProto.getPipelineId());
        identity.setProcessId(identityProto.getProcessId());
        return identity;
    }
View Full Code Here
63
64
65
66
67
68
69
70
71
72
73
    private ConfigClientService configClientService;

    @Test
    public void testWithOutRootDir() throws Exception {
        File rootDir = new File("/null");
        Identity id = buildIdentity(1L, 2L, 3L);
        FileBatch fileBatch = buildFileBatch(id);
        fileBatch.getFiles().addAll(buildFileDatas("ns_", EventType.INSERT, 0, 20, false));

        try {
            fileLoadAction.load(fileBatch, rootDir, null);
View Full Code Here
105
106
107
108
109
110
111
112
113
114
115
                returns(pipeline);

            }
        };

        Identity id = buildIdentity(1L, 2L, 3L);
        FileBatch fileBatch = buildFileBatch(id);
        fileBatch.getFiles().addAll(buildFileDatas(null, EventType.INSERT, fileDataStartIndex, fileDataCount, true));

        WeightController controller = new WeightController(1);
        FileLoadContext context = fileLoadAction.load(fileBatch, ROOT_DIR, controller);
View Full Code Here
185
186
187
188
189
190
191
192
193
194
195
        return files;
    }

    protected Identity buildIdentity(long channelId, long pipelineId, long processId) {
        Identity identity = new Identity();
        identity.setChannelId(channelId);
        identity.setPipelineId(pipelineId);
        identity.setProcessId(processId);
        return identity;
    }
View Full Code Here
84
85
86
87
88
89
90
91
92
93
94
95
96
97
                configClientService.currentNode();
                returns(currentNode);
            }
        };

        Identity identity = new Identity();
        identity.setChannelId(100L);
        identity.setPipelineId(100L);
        identity.setProcessId(100L);

        RowBatch rowBatch = new RowBatch();
        rowBatch.setIdentity(identity);
        List<EventData> eventDatas = generatorEventDataForOracle(0, 20, EventType.INSERT);
        for (EventData eventData : eventDatas) {
View Full Code Here
142
143
144
145
146
147
148
149
150
151
152
153
154
155
                configClientService.currentNode();
                returns(currentNode);
            }
        };

        Identity identity = new Identity();
        identity.setChannelId(100L);
        identity.setPipelineId(100L);
        identity.setProcessId(100L);

        RowBatch rowBatch = new RowBatch();
        rowBatch.setIdentity(identity);
        List<EventData> eventDatas = generatorEventDataForMysql(0, 20, EventType.INSERT);
        for (EventData eventData : eventDatas) {
View Full Code Here
66
67
68
69
70
71
72
73
74
75
76
77
78
79
                configClientService.findPipeline(anyLong);
                returns(pipeline);
            }
        };

        Identity identity = new Identity();
        identity.setChannelId(100L);
        identity.setPipelineId(100L);
        identity.setProcessId(100L);

        FileBatch fileBatch = new FileBatch();
        fileBatch.setIdentity(identity);
        fileBatch.getFiles().addAll(generatorLocalFileData("fileLoad", 10));
View Full Code Here
TOP

Related Classes of com.alibaba.otter.shared.etl.model.Identity

  • com.alibaba.otter.node.etl.common.pipe.HttpPipeIntegration
  • com.alibaba.otter.node.etl.common.pipe.impl.http.RowDataHttpPipe
  • com.alibaba.otter.node.etl.common.pipe.MemoryPipeTest
  • com.alibaba.otter.node.etl.common.pipe.RpcPipeTest
  • com.alibaba.otter.node.etl.conflict.FileBatchConflictDetectServiceIntegration
  • com.alibaba.otter.node.etl.extract.DatabaseExtractorTest
  • com.alibaba.otter.node.etl.extract.extractor.FileExtractor
  • com.alibaba.otter.node.etl.extract.FreedomExtractorTest
  • com.alibaba.otter.node.etl.launcher.OtterLoaderFactoryIntegration
  • com.alibaba.otter.node.etl.load.DbLoadActionTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.