Package com.alibaba.otter.shared.etl.model

Examples of com.alibaba.otter.shared.etl.model.DbBatch


                eventData.setTableName("retl_buffer");
                rowBatch.merge(eventData);
            }
        }

        DbBatch dbBatch = new DbBatch(rowBatch);
        freedomExtractor.extract(dbBatch);
        want.collection(dbBatch.getRowBatch().getDatas()).sizeEq(count * count);
    }
View Full Code Here


        rowBatch.setIdentity(identity);

        FileBatch fileBatch = new FileBatch();
        fileBatch.setIdentity(identity);

        final DbBatch dbBatch = new DbBatch();
        dbBatch.setRowBatch(rowBatch);
        dbBatch.setFileBatch(fileBatch);
        final CountDownLatch latch = new CountDownLatch(1);
        executorService.submit(new Runnable() {

            public void run() {
                System.out.println("first run!!!!!!");
View Full Code Here

                        Thread.currentThread().setName(createTaskName(pipelineId, "LoadWorker"));
                        List<LoadContext> processedContexts = null;
                        try {
                            // 后续可判断同步数据是否为rowData
                            List<PipeKey> keys = (List<PipeKey>) etlEventData.getDesc();
                            DbBatch dbBatch = rowDataPipeDelegate.get(keys);

                            // 可能拿到为null,因为内存不足或者网络异常,长时间阻塞时,导致从pipe拿数据出现异常,数据可能被上一个节点已经删除
                            if (dbBatch == null) {
                                processMissData(pipelineId, "load miss data with keys:" + keys.toString());
                                return;
                            }

                            // 进行数据load处理
                            otterLoaderFactory.setStartTime(dbBatch.getRowBatch().getIdentity(),
                                                            etlEventData.getStartTime());

                            processedContexts = otterLoaderFactory.load(dbBatch);

                            if (profiling) {
View Full Code Here

                            for (EventData data : eventData) {
                                rowBatch.merge(data);
                            }

                            long nextNodeId = etlEventData.getNextNid();
                            List<PipeKey> pipeKeys = rowDataPipeDelegate.put(new DbBatch(rowBatch), nextNodeId);
                            etlEventData.setDesc(pipeKeys);
                            etlEventData.setNumber((long) eventData.size());
                            etlEventData.setFirstTime(startTime); // 使用原始数据的第一条
                            etlEventData.setBatchId(message.getId());
View Full Code Here

                        Thread.currentThread().setName(createTaskName(pipelineId, "ExtractWorker"));
                        try {
                            pipeline = configClientService.findPipeline(pipelineId);
                            List<PipeKey> keys = (List<PipeKey>) etlEventData.getDesc();
                            long nextNodeId = etlEventData.getNextNid();
                            DbBatch dbBatch = rowDataPipeDelegate.get(keys);

                            // 可能拿到为null,因为内存不足或者网络异常,长时间阻塞时,导致从pipe拿数据出现异常,数据可能被上一个节点已经删除
                            if (dbBatch == null) {
                                processMissData(pipelineId, "extract miss data with keys:" + keys.toString());
                                return;
                            }

                            otterExtractorFactory.extract(dbBatch);// 重新装配一下数据
                            if (dbBatch.getFileBatch() != null
                                && !CollectionUtils.isEmpty(dbBatch.getFileBatch().getFiles())
                                && pipeline.getParameters().getFileDetect()) { // 判断一下是否有文件同步,并且需要进行文件对比
                                // 对比一下中美图片是否有变化
                                FileBatch fileBatch = fileBatchConflictDetectService.detect(dbBatch.getFileBatch(),
                                                                                            nextNodeId);
                                dbBatch.setFileBatch(fileBatch);
                            }

                            List<PipeKey> pipeKeys = rowDataPipeDelegate.put(dbBatch, nextNodeId);
                            etlEventData.setDesc(pipeKeys);
View Full Code Here

        InputStream input = null;
        JSONReader reader = null;
        try {
            input = new BufferedInputStream(new FileInputStream(archiveFile));
            DbBatch dbBatch = new DbBatch();
            byte[] lengthBytes = new byte[4];
            input.read(lengthBytes);
            int length = ByteUtils.bytes2int(lengthBytes);
            BatchProto.RowBatch rowbatchProto = BatchProto.RowBatch.parseFrom(new LimitedInputStream(input, length));
            // 构造原始的model对象
            RowBatch rowBatch = new RowBatch();
            rowBatch.setIdentity(build(rowbatchProto.getIdentity()));
            for (BatchProto.RowData rowDataProto : rowbatchProto.getRowsList()) {
                EventData eventData = new EventData();
                eventData.setPairId(rowDataProto.getPairId());
                eventData.setTableId(rowDataProto.getTableId());
                eventData.setTableName(rowDataProto.getTableName());
                eventData.setSchemaName(rowDataProto.getSchemaName());
                eventData.setEventType(EventType.valuesOf(rowDataProto.getEventType()));
                eventData.setExecuteTime(rowDataProto.getExecuteTime());
                // add by ljh at 2012-10-31
                if (StringUtils.isNotEmpty(rowDataProto.getSyncMode())) {
                    eventData.setSyncMode(SyncMode.valuesOf(rowDataProto.getSyncMode()));
                }
                if (StringUtils.isNotEmpty(rowDataProto.getSyncConsistency())) {
                    eventData.setSyncConsistency(SyncConsistency.valuesOf(rowDataProto.getSyncConsistency()));
                }
                // 处理主键
                List<EventColumn> keys = new ArrayList<EventColumn>();
                for (BatchProto.Column columnProto : rowDataProto.getKeysList()) {
                    keys.add(buildColumn(columnProto));
                }
                eventData.setKeys(keys);
                // 处理old主键
                if (CollectionUtils.isEmpty(rowDataProto.getOldKeysList()) == false) {
                    List<EventColumn> oldKeys = new ArrayList<EventColumn>();
                    for (BatchProto.Column columnProto : rowDataProto.getOldKeysList()) {
                        oldKeys.add(buildColumn(columnProto));
                    }
                    eventData.setOldKeys(oldKeys);
                }
                // 处理具体的column value
                List<EventColumn> columns = new ArrayList<EventColumn>();
                for (BatchProto.Column columnProto : rowDataProto.getColumnsList()) {
                    columns.add(buildColumn(columnProto));
                }
                eventData.setColumns(columns);

                eventData.setRemedy(rowDataProto.getRemedy());
                eventData.setSize(rowDataProto.getSize());
                eventData.setSql(rowDataProto.getSql());
                eventData.setDdlSchemaName(rowDataProto.getDdlSchemaName());
                // 添加到总记录
                rowBatch.merge(eventData);
            }
            dbBatch.setRowBatch(rowBatch);

            input.read(lengthBytes);
            length = ByteUtils.bytes2int(lengthBytes);
            BatchProto.FileBatch filebatchProto = BatchProto.FileBatch.parseFrom(new LimitedInputStream(input, length));
            // 构造原始的model对象
            FileBatch fileBatch = new FileBatch();
            fileBatch.setIdentity(build(filebatchProto.getIdentity()));
            for (BatchProto.FileData fileDataProto : filebatchProto.getFilesList()) {
                FileData fileData = new FileData();
                fileData.setPairId(fileDataProto.getPairId());
                fileData.setTableId(fileDataProto.getTableId());
                fileData.setEventType(EventType.valuesOf(fileDataProto.getEventType()));
                fileData.setLastModifiedTime(fileDataProto.getLastModifiedTime());
                fileData.setNameSpace(fileDataProto.getNamespace());
                fileData.setPath(fileDataProto.getPath());
                fileData.setSize(fileDataProto.getSize());
                // 添加到filebatch中
                fileBatch.getFiles().add(fileData);
            }
            dbBatch.setFileBatch(fileBatch);
            return dbBatch;
        } catch (IOException e) {
            throw new PipeException("deserial_error", e);
        } finally {
            IOUtils.closeQuietly(reader);
View Full Code Here

        return keys;
    }

    public DbBatch get(List<PipeKey> keys) {
        Assert.notNull(keys);
        DbBatch dbBatch = new DbBatch();
        Future<File> future = null;
        for (final PipeKey key : keys) {
            if (key == null) {
                continue; // 忽略空的key
            }

            if (key instanceof MemoryPipeKey) {
                dbBatch = rowDataMemoryPipe.get((MemoryPipeKey) key);
                return dbBatch;// 直接返回
            } else if (key instanceof HttpPipeKey) {
                if (key.getDataType().isDbBatch()) { // 区分一下数据下载
                    dbBatch = rowDataHttpPipe.get((HttpPipeKey) key);
                } else {
                    future = executorService.submit(new Callable<File>() {

                        public File call() throws Exception {
                            try {
                                HttpPipeKey pipeKey = (HttpPipeKey) key;
                                MDC.put(OtterConstants.splitPipelineLogFileKey,
                                        String.valueOf(pipeKey.getIdentity().getPipelineId()));
                                return attachmentHttpPipe.get(pipeKey);
                            } finally {
                                MDC.remove(OtterConstants.splitPipelineLogFileKey);
                            }

                        }
                    });
                }
            } else if (key instanceof RpcPipeKey) {
                dbBatch = rowDataRpcPipe.get((RpcPipeKey) key);
            } else {
                throw new PipeException("unknow_PipeKey", key.toString());
            }
        }

        if (future != null && dbBatch != null) {
            try {
                dbBatch.setRoot(future.get());
            } catch (Exception e) {
                throw new PipeException(e);
            }
        }
        return dbBatch;
View Full Code Here

                        Thread.currentThread().setName(createTaskName(pipelineId, "transformWorker"));

                        try {
                            // 后续可判断同步数据是否为rowData
                            List<PipeKey> keys = (List<PipeKey>) etlEventData.getDesc();
                            DbBatch dbBatch = rowDataPipeDelegate.get(keys);

                            // 可能拿到为null,因为内存不足或者网络异常,长时间阻塞时,导致从pipe拿数据出现异常,数据可能被上一个节点已经删除
                            if (dbBatch == null) {
                                processMissData(pipelineId, "transform miss data with keys:" + keys.toString());
                                return;
                            }

                            // 根据对应的tid,转化为目标端的tid。后续可进行字段的加工处理
                            // 暂时认为rowBatchs和fileBatchs不会有异构数据的转化
                            Map<Class, BatchObject> dataBatchs = otterTransformerFactory.transform(dbBatch.getRowBatch());

                            // 可能存在同一个Pipeline下有Mq和Db两种同步类型
                            dbBatch.setRowBatch((RowBatch) dataBatchs.get(EventData.class));

                            if (dbBatch.getFileBatch() != null) {
                                Map<Class, BatchObject> fileBatchs = otterTransformerFactory.transform(dbBatch.getFileBatch());
                                dbBatch.setFileBatch((FileBatch) fileBatchs.get(FileData.class));
                            }
                            // 传递给下一个流程
                            List<PipeKey> nextKeys = rowDataPipeDelegate.put(dbBatch, etlEventData.getNextNid());
                            etlEventData.setDesc(nextKeys);
View Full Code Here

        InputStream input = null;
        JSONReader reader = null;
        try {
            input = new BufferedInputStream(new FileInputStream(archiveFile));
            DbBatch dbBatch = new DbBatch();
            byte[] lengthBytes = new byte[4];
            input.read(lengthBytes);
            int length = ByteUtils.bytes2int(lengthBytes);
            BatchProto.RowBatch rowbatchProto = BatchProto.RowBatch.parseFrom(new LimitedInputStream(input, length));
            // 构造原始的model对象
            RowBatch rowBatch = new RowBatch();
            rowBatch.setIdentity(build(rowbatchProto.getIdentity()));
            for (BatchProto.RowData rowDataProto : rowbatchProto.getRowsList()) {
                EventData eventData = new EventData();
                eventData.setPairId(rowDataProto.getPairId());
                eventData.setTableId(rowDataProto.getTableId());
                eventData.setTableName(rowDataProto.getTableName());
                eventData.setSchemaName(rowDataProto.getSchemaName());
                eventData.setEventType(EventType.valuesOf(rowDataProto.getEventType()));
                eventData.setExecuteTime(rowDataProto.getExecuteTime());
                // add by ljh at 2012-10-31
                if (StringUtils.isNotEmpty(rowDataProto.getSyncMode())) {
                    eventData.setSyncMode(SyncMode.valuesOf(rowDataProto.getSyncMode()));
                }
                if (StringUtils.isNotEmpty(rowDataProto.getSyncConsistency())) {
                    eventData.setSyncConsistency(SyncConsistency.valuesOf(rowDataProto.getSyncConsistency()));
                }
                // 处理主键
                List<EventColumn> keys = new ArrayList<EventColumn>();
                for (BatchProto.Column columnProto : rowDataProto.getKeysList()) {
                    keys.add(buildColumn(columnProto));
                }
                eventData.setKeys(keys);
                // 处理old主键
                if (CollectionUtils.isEmpty(rowDataProto.getOldKeysList()) == false) {
                    List<EventColumn> oldKeys = new ArrayList<EventColumn>();
                    for (BatchProto.Column columnProto : rowDataProto.getOldKeysList()) {
                        oldKeys.add(buildColumn(columnProto));
                    }
                    eventData.setOldKeys(oldKeys);
                }
                // 处理具体的column value
                List<EventColumn> columns = new ArrayList<EventColumn>();
                for (BatchProto.Column columnProto : rowDataProto.getColumnsList()) {
                    columns.add(buildColumn(columnProto));
                }
                eventData.setColumns(columns);

                eventData.setRemedy(rowDataProto.getRemedy());
                eventData.setSize(rowDataProto.getSize());
                eventData.setSql(rowDataProto.getSql());
                eventData.setDdlSchemaName(rowDataProto.getDdlSchemaName());
                eventData.setHint(rowDataProto.getHint());
                eventData.setWithoutSchema(rowDataProto.getWithoutSchema());
                // 添加到总记录
                rowBatch.merge(eventData);
            }
            dbBatch.setRowBatch(rowBatch);

            input.read(lengthBytes);
            length = ByteUtils.bytes2int(lengthBytes);
            BatchProto.FileBatch filebatchProto = BatchProto.FileBatch.parseFrom(new LimitedInputStream(input, length));
            // 构造原始的model对象
            FileBatch fileBatch = new FileBatch();
            fileBatch.setIdentity(build(filebatchProto.getIdentity()));
            for (BatchProto.FileData fileDataProto : filebatchProto.getFilesList()) {
                FileData fileData = new FileData();
                fileData.setPairId(fileDataProto.getPairId());
                fileData.setTableId(fileDataProto.getTableId());
                fileData.setEventType(EventType.valuesOf(fileDataProto.getEventType()));
                fileData.setLastModifiedTime(fileDataProto.getLastModifiedTime());
                fileData.setNameSpace(fileDataProto.getNamespace());
                fileData.setPath(fileDataProto.getPath());
                fileData.setSize(fileDataProto.getSize());
                // 添加到filebatch中
                fileBatch.getFiles().add(fileData);
            }
            dbBatch.setFileBatch(fileBatch);
            return dbBatch;
        } catch (IOException e) {
            throw new PipeException("deserial_error", e);
        } finally {
            IOUtils.closeQuietly(reader);
View Full Code Here

                            for (EventData data : eventData) {
                                rowBatch.merge(data);
                            }

                            long nextNodeId = etlEventData.getNextNid();
                            List<PipeKey> pipeKeys = rowDataPipeDelegate.put(new DbBatch(rowBatch), nextNodeId);
                            etlEventData.setDesc(pipeKeys);
                            etlEventData.setNumber((long) eventData.size());
                            etlEventData.setFirstTime(startTime); // 使用原始数据的第一条
                            etlEventData.setBatchId(message.getId());
View Full Code Here

TOP

Related Classes of com.alibaba.otter.shared.etl.model.DbBatch

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.