Package com.alibaba.otter.shared.common.model.config.pipeline

Examples of com.alibaba.otter.shared.common.model.config.pipeline.Pipeline


    private ArbitrateViewService arbitrateViewService;

    public void execute(@Param("pipelineId") Long pipelineId, Context context) throws Exception {

        List<ProcessStat> processStats = new ArrayList<ProcessStat>();
        Pipeline pipeline = pipelineService.findById(pipelineId);
        processStats = processStatService.listRealtimeProcessStat(pipelineId);
        // Map ext = new HashMap<Long, String>();
        // // ext.put(145456451, "asdf");
        // for (Long i = 1L; i <= 3; i++) {
        // List<StageStat> stageStats = new ArrayList<StageStat>();
        // ProcessStat processStat = new ProcessStat();
        // processStat.setPipelineId(1L);
        // processStat.setProcessId(i);
        // StageStat stage = new StageStat();
        // stage.setStage(StageType.SELECT);
        // stage.setStartTime(((new Date()).getTime() + i * 10 * 1000));
        // stage.setEndTime(((new Date()).getTime() + i * 200 * 1000));
        // stage.setNumber(11231230L);
        // stage.setSize(14545645640L);
        // // stage.setExts(ext);
        // stageStats.add(stage);
        // stage = new StageStat();
        // stage.setStage(StageType.EXTRACT);
        // stage.setStartTime(((new Date()).getTime() + i * 2000 * 1000));
        // stage.setEndTime(((new Date()).getTime() + i * 3000 * 1000));
        // stage.setExts(ext);
        // // stage.setNumber(10L);
        // // stage.setSize(10L);
        // stageStats.add(stage);
        // stage = new StageStat();
        // stage.setStage(StageType.TRANSFORM);
        // stage.setStartTime(((new Date()).getTime() + i * 5000 * 1000));
        // stage.setEndTime(((new Date()).getTime() + i * 6000 * 1000));
        // stage.setNumber(154640L);
        // stage.setExts(ext);
        // // stage.setSize(10L);
        // stageStats.add(stage);
        // stage = new StageStat();
        // stage.setStage(StageType.LOAD);
        // stage.setStartTime(((new Date()).getTime() + i * 70000 * 1000));
        // stage.setEndTime(((new Date()).getTime() + i * 80000 * 1000));
        // // stage.setNumber(10L);
        // stage.setSize(101445L);
        // // stage.setExts(ext);
        // stageStats.add(stage);
        // processStat.setStageStats(stageStats);
        // processStats.add(processStat);
        // }

        Long stageStart = 0L;
        // Long stageEnd = new Date().getTime() + 3 * 80000 * 1000;
        Long stageEnd = new Date().getTime();
        Long interval = 0L;
        double offset = 0L;
        // 找出最先开始的process的select阶段的开始时间作为起始时间
        if (processStats.size() > 0) {
            if (processStats.get(0).getStageStats().size() > 0) {
                stageStart = processStats.get(0).getStageStats().get(0).getStartTime();
            }
        }

        // 动态计算每个阶段的长度比例
        if (stageStart > 0) {
            interval = stageEnd - stageStart;
        }
        if (interval > 0) {
            offset = 800.0 / interval;
        }

        // 计算每个process当前任务所做的时间总和
        Map<Long, Long> processTime = new HashMap<Long, Long>();
        for (ProcessStat processStat : processStats) {
            Long timeout = 0L;
            if (processStat.getStageStats().size() > 0) {
                timeout = stageEnd - processStat.getStageStats().get(0).getStartTime();
            }
            processTime.put(processStat.getProcessId(), timeout);
        }

        // 获取下mainstem状态信息
        MainStemEventData mainstemData = arbitrateViewService.mainstemData(pipeline.getChannelId(), pipelineId);

        PositionEventData positionData = arbitrateViewService.getCanalCursor(pipeline.getParameters().getDestinationName(),
                                                                             pipeline.getParameters().getMainstemClientId());

        context.put("pipeline", pipeline);
        context.put("pipelineId", pipelineId);
        context.put("processStats", processStats);
        context.put("offset", offset);
View Full Code Here


     *
     * @param pipelineDO
     * @return Pipeline
     */
    private Pipeline doToModel(PipelineDO pipelineDo) {
        Pipeline pipeline = new Pipeline();
        try {
            pipeline.setId(pipelineDo.getId());
            pipeline.setName(pipelineDo.getName());
            pipeline.setParameters(pipelineDo.getParameters());
            pipeline.setDescription(pipelineDo.getDescription());
            pipeline.setGmtCreate(pipelineDo.getGmtCreate());
            pipeline.setGmtModified(pipelineDo.getGmtModified());
            pipeline.setChannelId(pipelineDo.getChannelId());
            pipeline.getParameters().setMainstemClientId(pipeline.getId().shortValue());

            // 组装DatamediaPair
            List<DataMediaPair> pairs = dataMediaPairService.listByPipelineId(pipelineDo.getId());
            Collections.sort(pairs, new DataMediaPairComparable());
            pipeline.setPairs(pairs);

            // 组装Node
            List<PipelineNodeRelationDO> relations = pipelineNodeRelationDao.listByPipelineIds(pipelineDo.getId());

            List<Long> totalNodeIds = new ArrayList<Long>();

            for (PipelineNodeRelationDO relation : relations) {
                Long nodeId = relation.getNodeId();
                if (!totalNodeIds.contains(nodeId)) {
                    totalNodeIds.add(nodeId);
                }
            }

            List<Node> totalNodes = nodeService.listByIds(totalNodeIds.toArray(new Long[totalNodeIds.size()]));
            List<Node> selectNodes = new ArrayList<Node>();
            List<Node> extractNodes = new ArrayList<Node>();
            List<Node> loadNodes = new ArrayList<Node>();

            for (Node node : totalNodes) {
                for (PipelineNodeRelationDO relation : relations) {
                    if (node.getId().equals(relation.getNodeId())) {
                        if (relation.getLocation().isSelect()) {
                            selectNodes.add(node);
                        } else if (relation.getLocation().isExtract()) {
                            extractNodes.add(node);
                        } else if (relation.getLocation().isLoad()) {
                            loadNodes.add(node);
                        }
                    }
                }
            }

            pipeline.setSelectNodes(selectNodes);
            pipeline.setExtractNodes(extractNodes);
            pipeline.setLoadNodes(loadNodes);

        } catch (Exception e) {
            logger.error("ERROR ## change the pipeline Do to Model has an exception");
            throw new ManagerException(e);
        }
View Full Code Here

        return pipeline;
    }

    private Pipeline doToModelWithoutColumn(PipelineDO pipelineDo) {
        Pipeline pipeline = new Pipeline();
        try {
            pipeline.setId(pipelineDo.getId());
            pipeline.setName(pipelineDo.getName());
            pipeline.setParameters(pipelineDo.getParameters());
            pipeline.setDescription(pipelineDo.getDescription());
            pipeline.setGmtCreate(pipelineDo.getGmtCreate());
            pipeline.setGmtModified(pipelineDo.getGmtModified());
            pipeline.setChannelId(pipelineDo.getChannelId());
            pipeline.getParameters().setMainstemClientId(pipeline.getId().shortValue());

            // 组装DatamediaPair
            List<DataMediaPair> pairs = dataMediaPairService.listByPipelineIdWithoutColumn(pipelineDo.getId());
            Collections.sort(pairs, new DataMediaPairComparable());
            pipeline.setPairs(pairs);

            // 组装Node
            List<PipelineNodeRelationDO> relations = pipelineNodeRelationDao.listByPipelineIds(pipelineDo.getId());

            List<Long> totalNodeIds = new ArrayList<Long>();

            for (PipelineNodeRelationDO relation : relations) {
                Long nodeId = relation.getNodeId();
                if (!totalNodeIds.contains(nodeId)) {
                    totalNodeIds.add(nodeId);
                }
            }

            List<Node> totalNodes = nodeService.listByIds(totalNodeIds.toArray(new Long[totalNodeIds.size()]));
            List<Node> selectNodes = new ArrayList<Node>();
            List<Node> extractNodes = new ArrayList<Node>();
            List<Node> loadNodes = new ArrayList<Node>();

            for (Node node : totalNodes) {
                for (PipelineNodeRelationDO relation : relations) {
                    if (node.getId().equals(relation.getNodeId())) {
                        if (relation.getLocation().isSelect()) {
                            selectNodes.add(node);
                        } else if (relation.getLocation().isExtract()) {
                            extractNodes.add(node);
                        } else if (relation.getLocation().isLoad()) {
                            loadNodes.add(node);
                        }
                    }
                }
            }

            pipeline.setSelectNodes(selectNodes);
            pipeline.setExtractNodes(extractNodes);
            pipeline.setLoadNodes(loadNodes);

        } catch (Exception e) {
            logger.error("ERROR ## change the pipeline Do to Model has an exception");
            throw new ManagerException(e);
        }
View Full Code Here

        return pipeline;
    }

    private Pipeline doToModelWithoutOther(PipelineDO pipelineDo) {
        Pipeline pipeline = new Pipeline();
        try {
            pipeline.setId(pipelineDo.getId());
            pipeline.setName(pipelineDo.getName());
            pipeline.setParameters(pipelineDo.getParameters());
            pipeline.setDescription(pipelineDo.getDescription());
            pipeline.setGmtCreate(pipelineDo.getGmtCreate());
            pipeline.setGmtModified(pipelineDo.getGmtModified());
            pipeline.setChannelId(pipelineDo.getChannelId());
            pipeline.getParameters().setMainstemClientId(pipeline.getId().shortValue());

        } catch (Exception e) {
            logger.error("ERROR ## change the pipeline Do to Model has an exception");
            throw new ManagerException(e);
        }
View Full Code Here

    public void create(Event event) {
        LogRecord logRecord = new LogRecord();
        if (event instanceof NodeAlarmEvent) {
            NodeAlarmEvent nodeAlarmEvent = (NodeAlarmEvent) event;
            Pipeline tempPipeline = new Pipeline();
            tempPipeline.setId(nodeAlarmEvent.getPipelineId());
            logRecord.setPipeline(tempPipeline);
            logRecord.setNid(nodeAlarmEvent.getNid());
            logRecord.setTitle(nodeAlarmEvent.getTitle());
            logRecord.setMessage(nodeAlarmEvent.getMessage());
        }
View Full Code Here

                } catch (Exception e) {
                    // 可能历史的log记录对应的channel/pipeline已经被删除了,直接忽略吧
                    Channel channel = new Channel();
                    channel.setId(0l);
                    logRecord.setChannel(channel);
                    Pipeline pipeline = new Pipeline();
                    pipeline.setId(0l);
                    logRecord.setPipeline(pipeline);
                }
            } else {
                Channel channel = new Channel();
                channel.setId(-1l);
                logRecord.setChannel(channel);
                Pipeline pipeline = new Pipeline();
                pipeline.setId(-1l);
                logRecord.setPipeline(pipeline);
            }

            logRecord.setTitle(logRecordDo.getTitle());
            logRecord.setNid(logRecordDo.getNid());
View Full Code Here

     * 优化设想:
     *    应该通过变长参数达到后期扩展的方便性
     * </pre>
     */
    public Channel findByPipelineId(Long pipelineId) {
        Pipeline pipeline = pipelineService.findById(pipelineId);
        Channel channel = findById(pipeline.getChannelId());
        return channel;
    }
View Full Code Here

        // 构造fileData使用的参数,fileDataStartIndex 决定着 pipeline 与 fileData 对应的关系(通过
        // dataMediaPair 的 id),
        // 以及 dataMediaPair 的 pushWeight
        final int fileDataStartIndex = 0;
        final int fileDataCount = 50;
        final Pipeline pipeline = buildPipeline(fileDataStartIndex, fileDataCount);

        final Channel channel = new Channel();

        new NonStrictExpectations() {
View Full Code Here

    public void cleanUp() throws IOException {
        FileUtils.deleteDirectory(ROOT_DIR);
    }

    protected Pipeline buildPipeline(final int fileDataStartIndex, int fileDataCount) {
        final Pipeline pipeline = new Pipeline();
        pipeline.setParameters(new PipelineParameter());

        int dataMediaPairCount = fileDataCount / NUMBER_OF_FILE_DATA_COPIES;
        pipeline.setPairs(new ArrayList<DataMediaPair>(dataMediaPairCount));
        for (int i = fileDataStartIndex; i < dataMediaPairCount; i++) {
            DataMediaPair dataMediaPair = buildDataMediaPair(i, i);
            pipeline.getPairs().add(dataMediaPair);
        }
        return pipeline;
    }
View Full Code Here

        dbLoadAction = (DbLoadAction) TestedObject.getSpringBeanFactory().getBean("dbLoadAction");

        final Channel channel = new Channel();
        channel.setId(1L);

        final Pipeline pipeline = new Pipeline();
        pipeline.setId(100L);
        List<DataMediaPair> pairs = generatorDataMediaPairForOracle(20);
        pipeline.setPairs(pairs);
        pipeline.getParameters().merge(new SystemParameter());
        pipeline.getParameters().merge(new ChannelParameter());

        // final Pipeline oppositePipeline = new Pipeline();
        // oppositePipeline.setId(101L);
        channel.setPipelines(Arrays.asList(pipeline));
View Full Code Here

TOP

Related Classes of com.alibaba.otter.shared.common.model.config.pipeline.Pipeline

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.