Package com.alibaba.otter.shared.etl.model

Examples of com.alibaba.otter.shared.etl.model.FileBatch

61
62
63
64
65
66
67
68
69
70
71
72
73
74
    private int                    retry  = 3;
    private ExecutorTemplateGetter executorTemplateGetter;

    public void extract(DbBatch dbBatch) throws ExtractException {
        List<FileData> fileDatas = doFileExtract(dbBatch.getRowBatch());
        FileBatch fileBatch = new FileBatch();
        fileBatch.setFiles(fileDatas);
        Identity identity = new Identity();
        identity.setChannelId(dbBatch.getRowBatch().getIdentity().getChannelId());
        identity.setPipelineId(dbBatch.getRowBatch().getIdentity().getPipelineId());
        identity.setProcessId(dbBatch.getRowBatch().getIdentity().getProcessId());
        fileBatch.setIdentity(identity);
        dbBatch.setFileBatch(fileBatch);
    }
View Full Code Here
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
            rowDataBuilder.setSize(eventData.getSize());
            rowBatchBuilder.addRows(rowDataBuilder.build());// 添加一条rowData记录
        }

        // 处理下FileBatch
        FileBatch fileBatch = dbBatch.getFileBatch();
        BatchProto.FileBatch.Builder fileBatchBuilder = null;
        fileBatchBuilder = BatchProto.FileBatch.newBuilder();
        fileBatchBuilder.setIdentity(build(fileBatch.getIdentity()));
        // 构造对应的proto对象
        for (FileData fileData : fileBatch.getFiles()) {
            BatchProto.FileData.Builder fileDataBuilder = BatchProto.FileData.newBuilder();
            fileDataBuilder.setPairId(fileData.getPairId());
            fileDataBuilder.setTableId(fileData.getTableId());
            if (fileData.getNameSpace() != null) {
                fileDataBuilder.setNamespace(fileData.getNameSpace());
View Full Code Here
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
            input.read(lengthBytes);
            length = ByteUtils.bytes2int(lengthBytes);
            BatchProto.FileBatch filebatchProto = BatchProto.FileBatch.parseFrom(new LimitedInputStream(input, length));
            // 构造原始的model对象
            FileBatch fileBatch = new FileBatch();
            fileBatch.setIdentity(build(filebatchProto.getIdentity()));
            for (BatchProto.FileData fileDataProto : filebatchProto.getFilesList()) {
                FileData fileData = new FileData();
                fileData.setPairId(fileDataProto.getPairId());
                fileData.setTableId(fileDataProto.getTableId());
                fileData.setEventType(EventType.valuesOf(fileDataProto.getEventType()));
                fileData.setLastModifiedTime(fileDataProto.getLastModifiedTime());
                fileData.setNameSpace(fileDataProto.getNamespace());
                fileData.setPath(fileDataProto.getPath());
                fileData.setSize(fileDataProto.getSize());
                // 添加到filebatch中
                fileBatch.getFiles().add(fileData);
            }
            dbBatch.setFileBatch(fileBatch);
            return dbBatch;
        } catch (IOException e) {
            throw new PipeException("deserial_error", e);
View Full Code Here
64
65
66
67
68
69
70
71
72
73
74
75
    @Test
    public void testWithOutRootDir() throws Exception {
        File rootDir = new File("/null");
        Identity id = buildIdentity(1L, 2L, 3L);
        FileBatch fileBatch = buildFileBatch(id);
        fileBatch.getFiles().addAll(buildFileDatas("ns_", EventType.INSERT, 0, 20, false));

        try {
            fileLoadAction.load(fileBatch, rootDir, null);
        } catch (Exception e) {
            // expect for LoadException
View Full Code Here
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
            }
        };

        Identity id = buildIdentity(1L, 2L, 3L);
        FileBatch fileBatch = buildFileBatch(id);
        fileBatch.getFiles().addAll(buildFileDatas(null, EventType.INSERT, fileDataStartIndex, fileDataCount, true));

        WeightController controller = new WeightController(1);
        FileLoadContext context = fileLoadAction.load(fileBatch, ROOT_DIR, controller);
        want.object(context.getChannel()).isEqualTo(channel);
        want.object(context.getPipeline()).isEqualTo(pipeline);
        want.object(context.getPrepareDatas()).isEqualTo(fileBatch.getFiles());
        want.number(context.getProcessedDatas().size()).isEqualTo(fileBatch.getFiles().size());
    }
View Full Code Here
148
149
150
151
152
153
154
155
156
        result.setPushWeight(pushWeight);
        return result;
    }

    protected FileBatch buildFileBatch(Identity identity) {
        FileBatch fileBatch = new FileBatch();
        fileBatch.setIdentity(identity);
        return fileBatch;
    }
View Full Code Here
71
72
73
74
75
76
77
78
79
80
81
82
83
        Identity identity = new Identity();
        identity.setChannelId(100L);
        identity.setPipelineId(100L);
        identity.setProcessId(100L);

        FileBatch fileBatch = new FileBatch();
        fileBatch.setIdentity(identity);
        fileBatch.getFiles().addAll(generatorLocalFileData("fileLoad", 10));

        WeightController controller = new WeightController(1);
        fileLoadAction.load(fileBatch, new File(tmp + File.separator + OTTERLOAD), controller);

        File target = new File(tmp + File.separator + OTTERLOAD + "_loaded/");
View Full Code Here
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
        Identity identity = new Identity();
        identity.setChannelId(100L);
        identity.setPipelineId(100L);
        identity.setProcessId(100L);

        FileBatch fileBatch = new FileBatch();
        fileBatch.setIdentity(identity);
        File localFile = new File("/tmp", "httpPipeTest.jpg");
        FileData localFileData = new FileData();
        localFileData.setTableId(1L);
        localFileData.setPairId(1L);
        localFileData.setPath(localFile.getPath());
        fileBatch.getFiles().add(localFileData);

        Map<Class, BatchObject> batchs = otterTransformFactory.transform(fileBatch);
        FileBatch result = (FileBatch) batchs.get(FileData.class);
        want.number(result.getFiles().size()).isEqualTo(1);
    }
View Full Code Here
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
        Identity identity = new Identity();
        identity.setChannelId(100L);
        identity.setPipelineId(100L);
        identity.setProcessId(100L);

        FileBatch fileBatch = new FileBatch();
        fileBatch.setIdentity(identity);
        File localFile = new File(tmp, "httpPipeTest.jpg");
        FileData localFileData = new FileData();
        localFileData.setEventType(EventType.INSERT);
        localFileData.setPath(localFile.getPath());
        fileBatch.getFiles().add(localFileData);
        try {
            byte[] data = getBlock(10 * 1024);
            NioUtils.write(data, localFile);
            HttpPipeKey key = attachmentHttpPipe.put(fileBatch);
            File target = attachmentHttpPipe.get(key);
View Full Code Here
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
        Identity identity = new Identity();
        identity.setChannelId(100L);
        identity.setPipelineId(100L);
        identity.setProcessId(100L);

        FileBatch fileBatch = new FileBatch();
        fileBatch.setIdentity(identity);
        File localFile = new File(tmp, "httpPipeTest.jpg");
        FileData localFileData = new FileData();
        localFileData.setPath(localFile.getPath());
        localFileData.setEventType(EventType.INSERT);
        localFileData.setLastModifiedTime(new Date().getTime());
        localFileData.setSize(100L);
        localFileData.setTableId(1L);
        fileBatch.getFiles().add(localFileData);

        RowBatch rowBatch = new RowBatch();
        rowBatch.setIdentity(identity);
        EventData eventData = new EventData();
        eventData.setTableId(1L);
View Full Code Here
TOP

Related Classes of com.alibaba.otter.shared.etl.model.FileBatch

  • com.alibaba.otter.node.etl.common.pipe.HttpPipeIntegration
  • com.alibaba.otter.node.etl.common.pipe.impl.http.RowDataHttpPipe
  • com.alibaba.otter.node.etl.conflict.FileBatchConflictDetectServiceIntegration
  • com.alibaba.otter.node.etl.conflict.impl.FileBatchConflictDetectServiceImpl
  • com.alibaba.otter.node.etl.extract.extractor.FileExtractor
  • com.alibaba.otter.node.etl.extract.ExtractTask
  • com.alibaba.otter.node.etl.launcher.OtterLoaderFactoryIntegration
  • com.alibaba.otter.node.etl.load.FileLoadActionTest
  • com.alibaba.otter.node.etl.load.loader.db.DataBatchLoader
  • com.alibaba.otter.node.etl.load.LocalFileLoaderActionTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.