Package com.alibaba.otter.shared.arbitrate.model

Examples of com.alibaba.otter.shared.arbitrate.model.EtlEventData

147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
        final List<Long> initProcessIds = new ArrayList<Long>();
        try {
            // 获取数据

            // select stage
            EtlEventData sdata1 = selectEvent.await(pipelineId);
            EtlEventData sdata2 = selectEvent.await(pipelineId);

            initProcessIds.add(sdata1.getProcessId());
            initProcessIds.add(sdata2.getProcessId());

            selectEvent.single(sdata1);
            selectEvent.single(sdata2);

            // extract stage
            EtlEventData edata1 = extractEvent.await(pipelineId);
            EtlEventData edata2 = extractEvent.await(pipelineId);

            extractEvent.single(edata1);
            extractEvent.single(edata2);

            // transform stage
            EtlEventData tdata1 = transformEvent.await(pipelineId);
            EtlEventData tdata2 = transformEvent.await(pipelineId);

            transformEvent.single(tdata1);
            transformEvent.single(tdata2);

            // SelectStageListener selectStageListener =
            // ArbitrateFactory.getInstance(pipelineId,
            // SelectStageListener.class);
            // selectStageListener.destory();
            // load stage
            EtlEventData ldata1 = loadEvent.await(pipelineId);
            loadEvent.single(ldata1);
            Long p1 = ldata1.getProcessId();

            TerminEventData terminData1 = new TerminEventData();
            terminData1.setPipelineId(pipelineId);
            terminData1.setProcessId(p1);
            terminEvent.ack(terminData1);// 发送ack信号，删除termin节点
View Full Code Here
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
        final List<Long> initProcessIds = new ArrayList<Long>();
        try {
            // 获取数据

            // select stage
            EtlEventData sdata1 = selectEvent.await(pipelineId);
            EtlEventData sdata2 = selectEvent.await(pipelineId);

            initProcessIds.add(sdata1.getProcessId());
            initProcessIds.add(sdata2.getProcessId());

            selectEvent.single(sdata1);
            selectEvent.single(sdata2);
            // extract stage
            EtlEventData edata1 = extractEvent.await(pipelineId);
            EtlEventData edata2 = extractEvent.await(pipelineId);

            extractEvent.single(edata1);
            extractEvent.single(edata2);

            // transform stage
            EtlEventData tdata1 = transformEvent.await(pipelineId);
            EtlEventData tdata2 = transformEvent.await(pipelineId);

            transformEvent.single(tdata1);
            transformEvent.single(tdata2);
            ArbitrateFactory.destory(pipelineId);
        } catch (InterruptedException e) {
View Full Code Here
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
        final List<Long> initProcessIds = new ArrayList<Long>();
        try {
            // 获取数据

            // select stage
            EtlEventData sdata1 = selectEvent.await(pipelineId);
            EtlEventData sdata2 = selectEvent.await(pipelineId);

            initProcessIds.add(sdata1.getProcessId());
            initProcessIds.add(sdata2.getProcessId());

            selectEvent.single(sdata1);
            selectEvent.single(sdata2);

            // extract stage
            EtlEventData edata1 = extractEvent.await(pipelineId);
            EtlEventData edata2 = extractEvent.await(pipelineId);

            extractEvent.single(edata1);
            extractEvent.single(edata2);

            // transform stage
            EtlEventData tdata1 = transformEvent.await(pipelineId);
            EtlEventData tdata2 = transformEvent.await(pipelineId);

            transformEvent.single(tdata1);
            transformEvent.single(tdata2);

            // SelectStageListener selectStageListener =
            // ArbitrateFactory.getInstance(pipelineId,
            // SelectStageListener.class);
            // selectStageListener.destory();
            // load stage
            EtlEventData ldata1 = loadEvent.await(pipelineId);
            loadEvent.single(ldata1);
            Long p1 = ldata1.getProcessId();

            TerminEventData terminData1 = new TerminEventData();
            terminData1.setPipelineId(pipelineId);
            terminData1.setProcessId(p1);
            terminEvent.ack(terminData1);// 发送ack信号，删除termin节点
View Full Code Here
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
        selectEvent = new SelectZooKeeperArbitrateEvent();
        final List<Long> initProcessIds = new ArrayList<Long>();
        try {
            //获取数据
            EtlEventData data1 = selectEvent.await(pipelineId);
            EtlEventData data2 = selectEvent.await(pipelineId);

            want.bool(data1 != null).is(true);
            want.bool(data2 != null).is(true);
            initProcessIds.add(data1.getProcessId());
            initProcessIds.add(data2.getProcessId());

            selectEvent.single(data1);
            selectEvent.single(data2);

            ArbitrateFactory.destory(pipelineId);
View Full Code Here
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
        public void run() {
            while (true) {
                try {
                    // 注意single时需传递同一个eventData对象，里面带着对应的nextNid信息
                    final EtlEventData eventData = arbitrateEventService.loadEvent().await(pipelineId);
                    // 提交对应的任务工作任务
                    executor.submit(new Callable() {

                        @Override
                        public Object call() throws Exception {
                            Long nextNid = eventData.getNextNid();
                            // 业务处理
                            Thread.sleep(500 + RandomUtils.nextInt(2000)); // sleep一下代表处理业务

                            if (isLocal(nextNid)) {// 判断是否本地
                                // 调用本地的pipe工具进行数据处理
                                eventData.setDesc(new Object());// 并设置相关信息
                            } else {
                                // 调用HTTP的pipe工具进行数据处理
                                eventData.setDesc(new Object());// 并设置相关信息
                            }

                            // 处理完成，通知一下
                            arbitrateEventService.loadEvent().single(eventData);
                            return true;
View Full Code Here
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
                    byte[] bytes = orginZk.getData(stagePath, false, zkStat);
                    if (bytes != null && bytes.length > 0) {
                        // 特殊处理zookeeper里的data信息，manager没有对应node中PipeKey的对象，所以导致反序列化会失败，需要特殊处理，删除'@'符号
                        String json = StringUtils.remove(new String(bytes, "UTF-8"), '@');
                        EtlEventData data = JsonUtils.unmarshalFromString(json, EtlEventData.class);
                        stageStat.setNumber(data.getNumber());
                        stageStat.setSize(data.getSize());

                        Map exts = new HashMap();
                        if (!CollectionUtils.isEmpty(data.getExts())) {
                            exts.putAll(data.getExts());
                        }
                        exts.put("currNid", data.getCurrNid());
                        exts.put("nextNid", data.getNextNid());
                        exts.put("desc", data.getDesc());
                        stageStat.setExts(exts);
                    }
                    if (prev != null) {// 对应的start时间为上一个节点的结束时间
                        stageStat.setStartTime(prev.getEndTime());
                    } else {
View Full Code Here
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
     */
    public synchronized void termin(TerminType type) {
        // 构建termin信号
        List<Long> processIds = new ArrayList<Long>(progress.keySet());
        for (Long processId : processIds) {
            EtlEventData eventData = progress.get(processId).getData();

            TerminEventData data = new TerminEventData();
            data.setPipelineId(getPipelineId());
            data.setType(type);
            data.setCode("channel");
            data.setDesc(type.toString());
            data.setProcessId(processId);
            if (eventData != null) {
                data.setBatchId(eventData.getBatchId());
                data.setCurrNid(eventData.getCurrNid());
                data.setStartTime(eventData.getStartTime());
                data.setEndTime(eventData.getEndTime());
                data.setFirstTime(eventData.getFirstTime());
                data.setNumber(eventData.getNumber());
                data.setSize(eventData.getSize());
                data.setExts(eventData.getExts());
            }
            offerTermin(data);
            progress.remove(processId);
        }

View Full Code Here
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
        MemoryStageController stageController = ArbitrateFactory.getInstance(pipelineId, MemoryStageController.class);
        Long processId = stageController.waitForProcess(StageType.SELECT); // 符合条件的processId

        ChannelStatus status = permitMonitor.getChannelPermit();
        if (status.isStart()) {// 即时查询一下当前的状态，状态随时可能会变
            EtlEventData eventData = new EtlEventData();
            eventData.setPipelineId(pipelineId);
            eventData.setProcessId(processId);
            eventData.setStartTime(new Date().getTime());// 返回当前时间
            Long nid = ArbitrateConfigUtils.getCurrentNid();
            eventData.setCurrNid(nid);
            eventData.setNextNid(nid);
            return eventData;// 只有这一条路返回
        } else {
            logger.warn("pipelineId[{}] select ignore processId[{}] by status[{}]", new Object[] { pipelineId,
                    processId, status });
            stageController.clearProgress(processId);//将progress中清理掉，避免阻塞后续调度，因为最小的Id一直处于extract未完成阶段
View Full Code Here
72
73
74
75
76
77
78
79
80
81
82
83
            // 2.2 判断是否存在了prev节点
            if (stageNodes.contains(prevNode)) {
                // 2.2.1 获取上一个节点的next node节点信息
                byte[] data = zookeeper.readData(path + "/" + prevNode);
                EtlEventData eventData = JsonUtils.unmarshalFromByte(data, EtlEventData.class);
                if (eventData.getNextNid().equals(ArbitrateConfigUtils.getCurrentNid())) {
                    List<Long> currentProcessIds = stageMonitor.getCurrentProcessIds(false);
                    if (currentProcessIds.contains(processId) && currentProcessIds.get(0).equals(processId)) {
                        // 判断是否是当前最小的processId节点，轮到自己处理了
                        addReply(processId);// 添加到返回队列,唤醒wait阻塞
                    }
View Full Code Here
73
74
75
76
77
78
79
80
81
82
83
84
            // 2.2 判断是否存在了prev节点
            if (stageNodes.contains(prevNode)) {
                // 2.2.1 获取上一个节点的next node节点信息
                byte[] data = zookeeper.readData(path + "/" + prevNode);
                EtlEventData eventData = JsonUtils.unmarshalFromByte(data, EtlEventData.class);
                if (eventData.getNextNid().equals(ArbitrateConfigUtils.getCurrentNid())) {
                    addReply(processId);// 添加到返回队列,唤醒wait阻塞
                }
            }
        } catch (ZkNoNodeException e) {
            // 出现节点不存在，说明出现了error情况
View Full Code Here
TOP

Related Classes of com.alibaba.otter.shared.arbitrate.model.EtlEventData

  • com.alibaba.otter.node.etl.extract.ExtractTask
  • com.alibaba.otter.node.etl.load.LoadTask
  • com.alibaba.otter.node.etl.select.SelectTask
  • com.alibaba.otter.node.etl.transform.TransformTask
  • com.alibaba.otter.shared.arbitrate.demo.servcie.ExtractServiceDemo$ExtractWorker
  • com.alibaba.otter.shared.arbitrate.demo.servcie.LoadServiceDemo$LoadWorker
  • com.alibaba.otter.shared.arbitrate.demo.servcie.SelectServiceDemo$SelectWorker
  • com.alibaba.otter.shared.arbitrate.demo.servcie.TransformServiceDemo$transformWorker
  • com.alibaba.otter.shared.arbitrate.impl.ArbitrateViewServiceImpl
  • com.alibaba.otter.shared.arbitrate.impl.setl.memory.MemoryStageController
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.