Package com.alibaba.otter.shared.common.model.config.pipeline

Examples of com.alibaba.otter.shared.common.model.config.pipeline.Pipeline


    public void start() {
        if (running) {
            return;
        }
        // 获取destination/filter参数
        Pipeline pipeline = configClientService.findPipeline(pipelineId);
        filter = makeFilterExpression(pipeline);
        destination = pipeline.getParameters().getDestinationName();
        batchSize = pipeline.getParameters().getMainstemBatchsize();
        batchTimeout = pipeline.getParameters().getBatchTimeout();
        ddlSync = pipeline.getParameters().getDdlSync();
        if (pipeline.getParameters().getDumpSelector() != null) {
            dump = pipeline.getParameters().getDumpSelector();
        }

        if (pipeline.getParameters().getDumpSelectorDetail() != null) {
            dumpDetail = pipeline.getParameters().getDumpSelectorDetail();
        }

        canalServer.setCanalInstanceGenerator(new CanalInstanceGenerator() {

            public CanalInstance generate(String destination) {
                Canal canal = canalConfigClient.findCanal(destination);
                final OtterAlarmHandler otterAlarmHandler = new OtterAlarmHandler();
                otterAlarmHandler.setPipelineId(pipelineId);
                OtterContextLocator.autowire(otterAlarmHandler); // 注入一下spring资源
                // 设置下slaveId,保证多个piplineId下重复引用时不重复
                long slaveId = 10000;// 默认基数
                if (canal.getCanalParameter().getSlaveId() != null) {
                    slaveId = canal.getCanalParameter().getSlaveId();
                }
                canal.getCanalParameter().setSlaveId(slaveId + pipelineId);
                canal.getCanalParameter().setDdlIsolation(ddlSync);

                CanalInstanceWithManager instance = new CanalInstanceWithManager(canal, filter) {

                    protected CanalHAController initHaController() {
                        HAMode haMode = parameters.getHaMode();
                        if (haMode.isMedia()) {
                            return new MediaHAController(parameters.getMediaGroup(),
                                parameters.getDbUsername(),
                                parameters.getDbPassword(),
                                parameters.getDefaultDatabaseName());
                        } else {
                            return super.initHaController();
                        }
                    }

                    protected void startEventParserInternal(CanalEventParser parser) {
                        super.startEventParserInternal(parser);

                        if (eventParser instanceof MysqlEventParser) {
                            MysqlEventParser mysqlEventParser = (MysqlEventParser) eventParser;
                            CanalHAController haController = mysqlEventParser.getHaController();

                            if (haController instanceof MediaHAController) {
                                ((MediaHAController) haController).setCanalHASwitchable(mysqlEventParser);
                            }

                            if (!haController.isStart()) {
                                haController.start();
                            }

                            // 基于media的Ha,直接从tddl中获取数据库信息
                            if (haController instanceof MediaHAController) {
                                AuthenticationInfo authenticationInfo = ((MediaHAController) haController).getAvailableAuthenticationInfo();
                                ((MysqlEventParser) eventParser).setMasterInfo(authenticationInfo);
                            }
                        }
                    }

                };
                instance.setAlarmHandler(otterAlarmHandler);

                CanalEventSink eventSink = instance.getEventSink();
                if (eventSink instanceof AbstractCanalEventSink) {
                    handler = new OtterDownStreamHandler();
                    handler.setPipelineId(pipelineId);
                    handler.setDetectingIntervalInSeconds(canal.getCanalParameter().getDetectingIntervalInSeconds());
                    OtterContextLocator.autowire(handler); // 注入一下spring资源
                    ((AbstractCanalEventSink) eventSink).addHandler(handler, 0); // 添加到开头
                    handler.start();
                }

                return instance;
            }
        });
        canalServer.start();

        canalServer.start(destination);
        this.clientIdentity = new ClientIdentity(destination, pipeline.getParameters().getMainstemClientId(), filter);
        canalServer.subscribe(clientIdentity);// 发起一次订阅

        running = true;
    }
View Full Code Here


    private DbLoadContext buildContext(Identity identity) {
        DbLoadContext context = new DbLoadContext();
        context.setIdentity(identity);
        Channel channel = configClientService.findChannel(identity.getChannelId());
        Pipeline pipeline = configClientService.findPipeline(identity.getPipelineId());
        context.setChannel(channel);
        context.setPipeline(pipeline);
        return context;
    }
View Full Code Here

    }

    // 调整一下线程池
    private void adjustPoolSize(DbLoadContext context) {
        Pipeline pipeline = context.getPipeline();
        int newPoolSize = pipeline.getParameters().getLoadPoolSize();
        if (newPoolSize != poolSize) {
            poolSize = newPoolSize;
            if (executor instanceof ThreadPoolExecutor) {
                ThreadPoolExecutor pool = (ThreadPoolExecutor) executor;
                pool.setCorePoolSize(newPoolSize);
View Full Code Here

            }
        }
    }

    private void adjustConfig(DbLoadContext context) {
        Pipeline pipeline = context.getPipeline();
        this.useBatch = pipeline.getParameters().isUseBatch();
    }
View Full Code Here

    @Override
    public void extract(DbBatch dbBatch) throws ExtractException {
        Assert.notNull(dbBatch);
        Assert.notNull(dbBatch.getRowBatch());
        // 读取配置
        Pipeline pipeline = getPipeline(dbBatch.getRowBatch().getIdentity().getPipelineId());
        boolean mustDb = pipeline.getParameters().getSyncConsistency().isMedia();
        boolean isRow = pipeline.getParameters().getSyncMode().isRow();// 如果是行记录是必须进行数据库反查
        // 读取一次配置
        adjustPoolSize(pipeline.getParameters().getExtractPoolSize()); // 调整下线程池,Extractor会被池化处理
        ExecutorCompletionService completionService = new ExecutorCompletionService(executor);

        // 进行并发提交
        ExtractException exception = null;
        // 每个表进行处理
View Full Code Here

    @Override
    public void extract(DbBatch dbBatch) throws ExtractException {
        Assert.notNull(dbBatch);
        Assert.notNull(dbBatch.getRowBatch());

        Pipeline pipeline = getPipeline(dbBatch.getRowBatch().getIdentity().getPipelineId());
        List<DataMediaPair> dataMediaPairs = pipeline.getPairs();

        /**
         * Key = TableId<br>
         * Value = a List of this tableId's column need to sync<br>
         */
 
View Full Code Here

        List<PipeKey> keys = new ArrayList<PipeKey>();
        if (isLocal(nid)) {
            keys.add(rowDataMemoryPipe.put(data));
        } else {
            Future<PipeKey> future = null;
            Pipeline pipeline = configClientService.findPipeline(data.getRowBatch().getIdentity().getPipelineId());
            if (data.getFileBatch() != null && !CollectionUtils.isEmpty(data.getFileBatch().getFiles())) {
                future = executorService.submit(new Callable<PipeKey>() {

                    public PipeKey call() throws Exception {
                        try {
                            MDC.put(OtterConstants.splitPipelineLogFileKey,
                                    String.valueOf(data.getFileBatch().getIdentity().getPipelineId()));
                            return attachmentHttpPipe.put(data.getFileBatch());
                        } finally {
                            MDC.remove(OtterConstants.splitPipelineLogFileKey);
                        }
                    }
                });
            }
            try {
                PipeChooseMode pipeChooseMode = pipeline.getParameters().getPipeChooseType();
                if (pipeChooseMode.isAutomatic()) {
                    if (calculateSize(data) <= sizeThresold) {
                        keys.add(rowDataRpcPipe.put(data));
                    } else {
                        keys.add(rowDataHttpPipe.put(data));
View Full Code Here

    @Resource(name = "nodeService")
    private NodeService     nodeService;

    public void execute(@Param("pipelineId") Long pipelineId, Context context) throws Exception {
        Pipeline pipeline = pipelineService.findById(pipelineId);

        context.put("pipeline", pipeline);
        context.put("nodes", nodeService.listAll());
    }
View Full Code Here

    @Resource(name = "arbitrateViewService")
    private ArbitrateViewService arbitrateViewService;

    public void doRemove(@Param("pipelineId") Long pipelineId, Navigator nav) throws Exception {
        Pipeline pipeline = pipelineService.findById(pipelineId);
        String destination = pipeline.getParameters().getDestinationName();
        short clientId = pipeline.getParameters().getMainstemClientId();
        PositionEventData position = arbitrateViewService.getCanalCursor(destination, clientId);
        logger.warn("remove pipelineId[{}] position \n {}", pipelineId, position); // 记录一下日志
        arbitrateViewService.removeCanalCursor(destination, clientId);
        nav.redirectToLocation("analysisStageStat.htm?pipelineId=" + pipelineId);
    }
View Full Code Here

    private ChannelArbitrateEvent channelArbitrateEvent;

    public void execute(@Param("pipelineId") Long pipelineId, Context context) throws Exception {

        List<ProcessStat> processStats = new ArrayList<ProcessStat>();
        Pipeline pipeline = pipelineService.findById(pipelineId);
        processStats = processStatService.listRealtimeProcessStat(pipelineId);
        // Map ext = new HashMap<Long, String>();
        // // ext.put(145456451, "asdf");
        // for (Long i = 1L; i <= 3; i++) {
        // List<StageStat> stageStats = new ArrayList<StageStat>();
        // ProcessStat processStat = new ProcessStat();
        // processStat.setPipelineId(1L);
        // processStat.setProcessId(i);
        // StageStat stage = new StageStat();
        // stage.setStage(StageType.SELECT);
        // stage.setStartTime(((new Date()).getTime() + i * 10 * 1000));
        // stage.setEndTime(((new Date()).getTime() + i * 200 * 1000));
        // stage.setNumber(11231230L);
        // stage.setSize(14545645640L);
        // // stage.setExts(ext);
        // stageStats.add(stage);
        // stage = new StageStat();
        // stage.setStage(StageType.EXTRACT);
        // stage.setStartTime(((new Date()).getTime() + i * 2000 * 1000));
        // stage.setEndTime(((new Date()).getTime() + i * 3000 * 1000));
        // stage.setExts(ext);
        // // stage.setNumber(10L);
        // // stage.setSize(10L);
        // stageStats.add(stage);
        // stage = new StageStat();
        // stage.setStage(StageType.TRANSFORM);
        // stage.setStartTime(((new Date()).getTime() + i * 5000 * 1000));
        // stage.setEndTime(((new Date()).getTime() + i * 6000 * 1000));
        // stage.setNumber(154640L);
        // stage.setExts(ext);
        // // stage.setSize(10L);
        // stageStats.add(stage);
        // stage = new StageStat();
        // stage.setStage(StageType.LOAD);
        // stage.setStartTime(((new Date()).getTime() + i * 70000 * 1000));
        // stage.setEndTime(((new Date()).getTime() + i * 80000 * 1000));
        // // stage.setNumber(10L);
        // stage.setSize(101445L);
        // // stage.setExts(ext);
        // stageStats.add(stage);
        // processStat.setStageStats(stageStats);
        // processStats.add(processStat);
        // }

        Long stageStart = 0L;
        // Long stageEnd = new Date().getTime() + 3 * 80000 * 1000;
        Long stageEnd = new Date().getTime();
        Long interval = 0L;
        double offset = 0L;
        // 找出最先开始的process的select阶段的开始时间作为起始时间
        if (processStats.size() > 0) {
            if (processStats.get(0).getStageStats().size() > 0) {
                stageStart = processStats.get(0).getStageStats().get(0).getStartTime();
            }
        }

        // 动态计算每个阶段的长度比例
        if (stageStart > 0) {
            interval = stageEnd - stageStart;
        }
        if (interval > 0) {
            offset = 800.0 / interval;
        }

        // 计算每个process当前任务所做的时间总和
        Map<Long, Long> processTime = new HashMap<Long, Long>();
        for (ProcessStat processStat : processStats) {
            Long timeout = 0L;
            if (processStat.getStageStats().size() > 0) {
                timeout = stageEnd - processStat.getStageStats().get(0).getStartTime();
            }
            processTime.put(processStat.getProcessId(), timeout);
        }

        // 获取下mainstem状态信息
        MainStemEventData mainstemData = arbitrateViewService.mainstemData(pipeline.getChannelId(), pipelineId);

        PositionEventData positionData = arbitrateViewService.getCanalCursor(pipeline.getParameters().getDestinationName(),
                                                                             pipeline.getParameters().getMainstemClientId());

        ChannelStatus status = channelArbitrateEvent.status(pipeline.getChannelId());

        context.put("pipeline", pipeline);
        context.put("pipelineId", pipelineId);
        context.put("processStats", processStats);
        context.put("offset", offset);
View Full Code Here

TOP

Related Classes of com.alibaba.otter.shared.common.model.config.pipeline.Pipeline

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.