Package com.alibaba.otter.shared.etl.model

Examples of com.alibaba.otter.shared.etl.model.EventColumn

368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
                        }
                    } else {
                        // 构造反查的返回结果
                        List<EventColumn> newEventColumns = new ArrayList<EventColumn>();
                        for (int i = 0; i < newColumnValues.size(); i++) {
                            EventColumn column = new EventColumn();
                            column.setIndex(columnTableData.indexs[i]);
                            column.setColumnName(columnTableData.columnNames[i]);
                            column.setColumnType(columnTableData.columnTypes[i]);
                            column.setNull(newColumnValues.get(i) == null);
                            column.setColumnValue(newColumnValues.get(i));
                            column.setUpdate(true);
                            newEventColumns.add(column);
                        }

                        // 处理下columns中不在反查字段内的字段列表
                        for (EventColumn column : eventData.getColumns()) {
                            boolean override = false;
                            for (EventColumn newEventColumn : newEventColumns) {
                                if (StringUtils.equalsIgnoreCase(newEventColumn.getColumnName(), column.getColumnName())) {
                                    override = true;
                                    break;
                                }
                            }
View Full Code Here
639
640
641
642
643
644
645
646
647
648
649
650
        private String dumpEventColumn(List<EventColumn> columns) {
            StringBuilder builder = new StringBuilder(event_default_capacity);
            int size = columns.size();
            for (int i = 0; i < size; i++) {
                EventColumn column = columns.get(i);
                builder.append("\t").append(column.toString());
                if (i < columns.size() - 1) {
                    builder.append(SEP);
                }
            }
            return builder.toString();
View Full Code Here
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
                isRequiredMap.put(StringUtils.lowerCase(tableColumn.getName()), tableColumn.isRequired());
            }

            for (int i = 0; i < columns.size(); i++) {
                int paramIndex = i + 1;
                EventColumn column = columns.get(i);
                int sqlType = column.getColumnType();

                Boolean isRequired = isRequiredMap.get(StringUtils.lowerCase(column.getColumnName()));
                if (isRequired == null) {
                    throw new LoadException(String.format("column name %s is not found in Table[%s]",
                        column.getColumnName(),
                        table.toString()));
                }

                Object param = null;
                if (dbDialect instanceof MysqlDialect
                    && (sqlType == Types.TIME || sqlType == Types.TIMESTAMP || sqlType == Types.DATE)) {
                    // 解决mysql的0000-00-00 00:00:00问题，直接依赖mysql
                    // driver进行处理，如果转化为Timestamp会出错
                    param = column.getColumnValue();
                } else {
                    param = SqlUtils.stringToSqlValue(column.getColumnValue(),
                        sqlType,
                        isRequired,
                        dbDialect.isEmptyStringNulled());
                }
View Full Code Here
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
        col.put("columns", array);
        col.put("dml", eventData.getEventType());
        col.put("exectime", eventData.getExecuteTime());

        // 构造新的主键
        EventColumn id = new EventColumn();
        id.setColumnValue(eventData.getSchemaName());
        id.setColumnType(Types.BIGINT);
        id.setColumnName("id");
        // 构造新的字段
        EventColumn schema = new EventColumn();
        schema.setColumnValue(eventData.getSchemaName());
        schema.setColumnType(Types.VARCHAR);
        schema.setColumnName("oschema");

        EventColumn table = new EventColumn();
        table.setColumnValue(eventData.getTableName());
        table.setColumnType(Types.VARCHAR);
        table.setColumnName("otable");

        EventColumn ovalue = new EventColumn();
        ovalue.setColumnValue(col.toJSONString());
        ovalue.setColumnType(Types.VARCHAR);
        ovalue.setColumnName("ovalue");

        EventColumn gtime = new EventColumn();
        gtime.setColumnValue(eventData.getExecuteTime() + "");
        gtime.setColumnType(Types.VARCHAR);
        gtime.setColumnName("gtime");

        // 替换为新的字段和主键信息
        List<EventColumn> cols = new ArrayList<EventColumn>();
        cols.add(schema);
        cols.add(table);
View Full Code Here
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
import com.alibaba.otter.shared.etl.model.EventData;

public class TestEventProcessor extends AbstractEventProcessor {

    public boolean process(EventData eventData) {
        EventColumn belong = getColumn(eventData, "belong_to");
        if (belong == null) {
            return doProcess(eventData);
        }

        // 判断规则：
        // (belong_to字段是包含4: || visited_idcs是数字并且visited_idcs & 2 >0) &&
        // write_source不为空且write_source不包含"OT"字符串且不包含"CIW"字符串)

        boolean pass = false;
        pass |= StringUtils.contains(belong.getColumnValue(), "4:");

        if (!pass) {// 第一个条件不满足，看一下第二个条件，两者是或者关系
            EventColumn visited = getColumn(eventData, "visited_idcs");
            if (visited == null) {
                return doProcess(eventData);
            }

            pass |= NumberUtils.isDigits(visited.getColumnValue())
                    && (Integer.valueOf(visited.getColumnValue()) & 2) > 0;
        }

        if (pass) {// 第三个条件和前两个条件是与关系，delete类型不判断write_source
            // write_source为字符类型
            EventColumn writeSource = getColumn(eventData, "write_source");
            if (writeSource == null) {
                return doProcess(eventData);
            } else {
                // 继续与操作
                if (eventData.getEventType().isDelete() // delete类型直接放过
                    || (StringUtils.isNotEmpty(writeSource.getColumnValue()) && !StringUtils.contains(writeSource.getColumnValue(),
                                                                                                      "CIW"))) {
                    return doProcess(eventData);
                }
            }
        }
View Full Code Here
67
68
69
70
71
72
73
74
75
76
77
78
79
    }

    private boolean doProcess(EventData eventData) {
        if (!eventData.getEventType().isDelete()) {
            // 缺省值为“OT:时间戳:随机串”.
            EventColumn writeSource = getColumn(eventData, "write_source");
            if (writeSource != null) {
                // 填入otter产生的信息
                writeSource.setColumnValue("OT:" + System.currentTimeMillis() + ":" + RandomUtils.nextInt(10000));
            }
        }
        return true;
    }
View Full Code Here
TOP

Related Classes of com.alibaba.otter.shared.etl.model.EventColumn

  • com.alibaba.otter.node.etl.common.pipe.HttpPipeIntegration
  • com.alibaba.otter.node.etl.common.pipe.impl.http.RowDataHttpPipe
  • com.alibaba.otter.node.etl.extract.DatabaseExtractorTest
  • com.alibaba.otter.node.etl.extract.extractor.DatabaseExtractor$DatabaseExtractWorker
  • com.alibaba.otter.node.etl.extract.extractor.FreedomExtractor
  • com.alibaba.otter.node.etl.extract.extractor.GroupExtractor
  • com.alibaba.otter.node.etl.extract.FreedomExtractorTest
  • com.alibaba.otter.node.etl.load.DbLoadActionTest
  • com.alibaba.otter.node.etl.load.DbLoadMergerTest
  • com.alibaba.otter.node.etl.load.loader.db.DbLoadAction$DbLoadWorker
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.