Package com.alibaba.otter.shared.common.model.config.pipeline

Examples of com.alibaba.otter.shared.common.model.config.pipeline.Pipeline


        Long channelId = 100L;
        Long pipelineId = 100L;
        Long oppositePipelineId = 101L;
        final Channel channel = new Channel();
        channel.setId(channelId);
        Pipeline pipeline1 = new Pipeline();
        pipeline1.setChannelId(channelId);
        pipeline1.setId(pipelineId);

        Pipeline pipeline2 = new Pipeline();
        pipeline2.setChannelId(channelId);
        pipeline2.setId(oppositePipelineId);
        channel.setPipelines(Arrays.asList(pipeline1, pipeline2));

        Mockit.setUpMock(NodeCommmunicationClient.class, new Object() {

            @Mock
            public Object callManager(final Event event) {
                return channel;
            }
        });

        Pipeline pipeline = configClientService.findPipeline(pipelineId);
        want.bool(pipeline.getId() == pipelineId);

        pipeline = configClientService.findOppositePipeline(pipelineId);
        want.bool(pipeline.getId() == oppositePipelineId);

        Channel channel1 = configClientService.findChannel(channelId);
        want.bool(channel1.getId() == channelId);

        channel1 = configClientService.findChannelByPipelineId(pipelineId);
View Full Code Here


    public void start() {
        if (running) {
            return;
        }
        // 获取destination/filter参数
        Pipeline pipeline = configClientService.findPipeline(pipelineId);
        filter = makeFilterExpression(pipeline);
        destination = pipeline.getParameters().getDestinationName();
        batchSize = pipeline.getParameters().getMainstemBatchsize();
        batchTimeout = pipeline.getParameters().getBatchTimeout();
        ddlSync = pipeline.getParameters().getDdlSync();
        if (pipeline.getParameters().getDumpSelector() != null) {
            dump = pipeline.getParameters().getDumpSelector();
        }

        if (pipeline.getParameters().getDumpSelectorDetail() != null) {
            dumpDetail = pipeline.getParameters().getDumpSelectorDetail();
        }

        canalServer.setCanalInstanceGenerator(new CanalInstanceGenerator() {

            public CanalInstance generate(String destination) {
                Canal canal = canalConfigClient.findCanal(destination);
                final OtterAlarmHandler otterAlarmHandler = new OtterAlarmHandler();
                otterAlarmHandler.setPipelineId(pipelineId);
                OtterContextLocator.autowire(otterAlarmHandler); // 注入一下spring资源
                //设置下slaveId,保证多个piplineId下重复引用时不重复
                long slaveId = 10000;//默认基数
                if (canal.getCanalParameter().getSlaveId() != null) {
                    slaveId = canal.getCanalParameter().getSlaveId();
                }
                canal.getCanalParameter().setSlaveId(slaveId + pipelineId);
                canal.getCanalParameter().setDdlIsolation(ddlSync);

                CanalInstanceWithManager instance = new CanalInstanceWithManager(canal, filter) {

                    protected CanalHAController initHaController() {
                        HAMode haMode = parameters.getHaMode();
                        if (haMode.isMedia()) {
                            return new MediaHAController(parameters.getMediaGroup(), parameters.getDbUsername(),
                                                         parameters.getDbPassword(),
                                                         parameters.getDefaultDatabaseName());
                        } else {
                            return super.initHaController();
                        }
                    }

                    @Override
                    protected void startEventParserInternal(CanalEventParser parser) {
                        super.startEventParserInternal(parser);

                        if (eventParser instanceof MysqlEventParser) {
                            MysqlEventParser mysqlEventParser = (MysqlEventParser) eventParser;
                            CanalHAController haController = mysqlEventParser.getHaController();

                            if (haController instanceof MediaHAController) {
                                ((MediaHAController) haController).setCanalHASwitchable(mysqlEventParser);
                            }

                            if (!haController.isStart()) {
                                haController.start();
                            }

                            // 基于media的Ha,直接从tddl中获取数据库信息
                            if (haController instanceof MediaHAController) {
                                AuthenticationInfo authenticationInfo = ((MediaHAController) haController).getAvailableAuthenticationInfo();
                                ((MysqlEventParser) eventParser).setMasterInfo(authenticationInfo);
                            }
                        }
                    }

                };
                instance.setAlarmHandler(otterAlarmHandler);

                CanalEventSink eventSink = instance.getEventSink();
                if (eventSink instanceof AbstractCanalEventSink) {
                    handler = new OtterDownStreamHandler();
                    handler.setPipelineId(pipelineId);
                    handler.setDetectingEnable(canal.getCanalParameter().getDetectingEnable());
                    handler.setDetectingIntervalInSeconds(canal.getCanalParameter().getDetectingIntervalInSeconds());
                    OtterContextLocator.autowire(handler); // 注入一下spring资源
                    ((AbstractCanalEventSink) eventSink).addHandler(handler);
                    handler.start();
                }

                return instance;
            }
        });
        canalServer.start();

        canalServer.start(destination);
        this.clientIdentity = new ClientIdentity(destination, pipeline.getParameters().getMainstemClientId(), filter);
        canalServer.subscribe(clientIdentity);// 发起一次订阅

        running = true;
    }
View Full Code Here

    private DbLoadContext buildContext(Identity identity) {
        DbLoadContext context = new DbLoadContext();
        context.setIdentity(identity);
        Channel channel = configClientService.findChannel(identity.getChannelId());
        Pipeline pipeline = configClientService.findPipeline(identity.getPipelineId());
        context.setChannel(channel);
        context.setPipeline(pipeline);
        return context;
    }
View Full Code Here

    }

    // 调整一下线程池
    private void adjustPoolSize(DbLoadContext context) {
        Pipeline pipeline = context.getPipeline();
        int newPoolSize = pipeline.getParameters().getLoadPoolSize();
        if (newPoolSize != poolSize) {
            poolSize = newPoolSize;
            if (executor instanceof ThreadPoolExecutor) {
                ThreadPoolExecutor pool = (ThreadPoolExecutor) executor;
                pool.setCorePoolSize(newPoolSize);
View Full Code Here

            }
        }
    }

    private void adjustConfig(DbLoadContext context) {
        Pipeline pipeline = context.getPipeline();
        this.useBatch = pipeline.getParameters().isUseBatch();
    }
View Full Code Here

        ExecutorTemplate executorTemplate = executorTemplateGetter.get();
        try {
            MDC.put(OtterConstants.splitPipelineLoadLogFileKey, String.valueOf(fileBatch.getIdentity().getPipelineId()));
            executorTemplate.start();
            // 重新设置下poolSize
            Pipeline pipeline = configClientService.findPipeline(fileBatch.getIdentity().getPipelineId());
            executorTemplate.adjustPoolSize(pipeline.getParameters().getFileLoadPoolSize());
            // 启动
            final List<FileData> result = Collections.synchronizedList(new ArrayList<FileData>());
            final List<FileData> filter = Collections.synchronizedList(new ArrayList<FileData>());
            for (final FileData source : fileBatch.getFiles()) {
                EventType type = source.getEventType();
                if (type.isDelete()) {
                    result.add(source);
                } else {
                    executorTemplate.submit(new Runnable() {

                        public void run() {
                            MDC.put(OtterConstants.splitPipelineLoadLogFileKey,
                                    String.valueOf(fileBatch.getIdentity().getPipelineId()));
                            // 处理更新类型
                            String namespace = source.getNameSpace();
                            String path = source.getPath();
                            FileData target = null;

                            int count = 0;
                            while (count++ < retry) {// 进行重试处理
                                try {
                                    if (true == StringUtils.isBlank(namespace)) {
                                        // local file
                                        java.io.File targetFile = new java.io.File(path);
                                        if (true == targetFile.exists()) {
                                            // modified time cost
                                            long lastModified = targetFile.lastModified();
                                            long size = targetFile.length();
                                            // 更新数据
                                            target = new FileData();
                                            target.setLastModifiedTime(lastModified);
                                            target.setSize(size);
                                        }
                                    } else {
                                        // remote file
                                        throw new RuntimeException(source + " is not support!");
                                    }

                                    break; // 不出异常就跳出
                                } catch (Exception ex) {
                                    target = null;
                                }
                            }

                            boolean shouldSync = false;
                            if (target != null) {
                                if (true == accept(target, source)) {
                                    shouldSync = true;
                                }
                            } else {
                                shouldSync = true;
                            }

                            if (true == shouldSync) {
                                result.add(source);
                            } else {
                                filter.add(source);
                            }
                        }
                    });
                }
            }
            // 等待所有都处理完成
            executorTemplate.waitForResult();

            if (pipeline.getParameters().getDumpEvent() && logger.isInfoEnabled()) {
                logger.info(FileloadDumper.dumpFilterFileDatas(fileBatch.getIdentity(), fileBatch.getFiles().size(),
                                                               result.size(), filter));
            }

            // 构造返回结果
View Full Code Here

        Assert.notNull(urlFormat);
    }

    public String getUrl(Long pipelineId, String filePath) {
        Node node = configClientService.currentNode();
        Pipeline pipeline = configClientService.findPipeline(pipelineId);
        String ip = node.getIp();
        if (node.getParameters().getUseExternalIp() || pipeline.getParameters().getUseExternalIp()) {
            ip = node.getParameters().getExternalIp();

            if (StringUtils.isEmpty(ip)) {
                throw new ArchiveException(String.format("pipelineId:%s useExternalIp by nid[%s] has no external ip",
                    String.valueOf(pipelineId),
View Full Code Here

        return context;
    }

    private void adjustPoolSize(FileLoadContext context) {
        Pipeline pipeline = context.getPipeline();
        int newPoolSize = pipeline.getParameters().getFileLoadPoolSize();
        if (newPoolSize != poolSize) {
            poolSize = newPoolSize;
            if (executor instanceof ThreadPoolExecutor) {
                ThreadPoolExecutor pool = (ThreadPoolExecutor) executor;
                pool.setCorePoolSize(newPoolSize);
View Full Code Here

    private FileLoadContext buildContext(Identity identity) {
        FileLoadContext context = new FileLoadContext();
        context.setIdentity(identity);
        Channel channel = configClientService.findChannel(identity.getChannelId());
        Pipeline pipeline = configClientService.findPipeline(identity.getPipelineId());
        context.setChannel(channel);
        context.setPipeline(pipeline);
        return context;
    }
View Full Code Here

        // 处理构造对应的文件url
        String filename = buildFileName(fileBatch.getIdentity(), ClassUtils.getShortClassName(fileBatch.getClass()));
        File file = new File(htdocsDir, filename);
        // 压缩对应的文件数据
        List<FileData> fileDatas = fileBatch.getFiles();
        Pipeline pipeline = configClientService.findPipeline(fileBatch.getIdentity().getPipelineId());
        int poolSize = pipeline.getParameters().getFileLoadPoolSize();
        boolean useLocalFileMutliThread = pipeline.getParameters().getUseLocalFileMutliThread();
        ArchiveBean archiveBean = getArchiveBean();
        archiveBean.adjustPoolSize(poolSize);// 调整线程池大小
        archiveBean.setUseLocalFileMutliThread(useLocalFileMutliThread);// 设置是否启用local多线程同步
        boolean done = archiveBean.pack(file, fileDatas, new ArchiveRetriverCallback<FileData>() {

            public InputStream retrive(FileData fileData) {
                boolean miss = false;
                try {
                    if (StringUtils.isNotEmpty(fileData.getNameSpace())) {
                        throw new RuntimeException(fileData + " is not support!");
                    } else {
                        File source = new File(fileData.getPath());
                        if (source.exists() && source.isFile()) {
                            return new LazyFileInputStream(source);
                        } else {
                            miss = true;
                            return null;
                        }
                    }
                } finally {
                    if (miss && logger.isInfoEnabled()) {
                        MDC.put(OtterConstants.splitPipelineLoadLogFileKey,
                                String.valueOf(fileBatch.getIdentity().getPipelineId()));
                        logger.info(FileloadDumper.dumpMissFileDatas(fileBatch.getIdentity(), fileData));
                    }
                }

            }
        });

        if (done == false) {
            return null; // 直接返回
        }

        HttpPipeKey key = new HttpPipeKey();
        key.setUrl(remoteUrlBuilder.getUrl(fileBatch.getIdentity().getPipelineId(), filename));
        key.setDataType(PipeDataType.FILE_BATCH);
        key.setIdentity(fileBatch.getIdentity());
        if (encrypt || pipeline.getParameters().getUseFileEncrypt()) {
            // 加密处理
            EncryptedData encryptedData = encryptFile(file);
            key.setKey(encryptedData.getKey());
            key.setCrc(encryptedData.getCrc());
        }
View Full Code Here

TOP

Related Classes of com.alibaba.otter.shared.common.model.config.pipeline.Pipeline

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.