Examples of ExtractException


Examples of com.alibaba.otter.node.etl.extract.exceptions.ExtractException

        List<EventData> eventDatas = rowBatch.getDatas();
        for (EventData eventData : eventDatas) {
            List<DataMediaPair> dataMediaPairs = ConfigHelper.findDataMediaPairByMediaId(pipeline,
                                                                                         eventData.getTableId());
            if (dataMediaPairs == null) {
                throw new ExtractException("ERROR ## the dataMediaId = " + eventData.getTableId()
                                           + " dataMediaPair is null,please check");
            }

            for (DataMediaPair dataMediaPair : dataMediaPairs) {
                if (dataMediaPair.getResolverData() == null
                    || dataMediaPair.getResolverData().getExtensionDataType() == null
                    || (dataMediaPair.getResolverData().getExtensionDataType().isClazz() && StringUtils.isBlank(dataMediaPair.getResolverData().getClazzPath()))
                    || (dataMediaPair.getResolverData().getExtensionDataType().isSource() && StringUtils.isBlank(dataMediaPair.getResolverData().getSourceText()))) {
                    continue;
                }

                FileResolver fileResolver = null;

                if (dataMediaPair.getResolverData() != null) {
                    fileResolver = extensionFactory.getExtension(FileResolver.class, dataMediaPair.getResolverData());
                } else {
                    continue;
                }

                if (fileResolver == null) {
                    throw new ExtractException("ERROR ## the dataMediaId = " + eventData.getTableId()
                                               + " the fileResolver className  = "
                                               + dataMediaPair.getResolverData().getClazzPath()
                                               + " is null ,please check the class");
                }
View Full Code Here

Examples of com.alibaba.otter.node.etl.extract.exceptions.ExtractException

                    index++;
                }
            }

            if (i != keys.size()) {
                throw new ExtractException("keys is not found in table " + table.toString() + " keys : "
                                           + dumpEventColumn(keys));
            }
            return data;
        }
View Full Code Here

Examples of com.alibaba.otter.node.etl.extract.exceptions.ExtractException

                        index++;
                    }
                }

                if (i != columns.size()) {
                    throw new ExtractException("columns is not found in table " + table.toString() + " columns : "
                                               + dumpEventColumn(columns));
                }
            }

            return data;
View Full Code Here

Examples of com.alibaba.otter.node.etl.extract.exceptions.ExtractException

            for (int i = 0; i < columnTypes.length; i++) {
                try {
                    String value = SqlUtils.sqlValueToString(rs, i + 1, columnTypes[i]);
                    result.add(value);
                } catch (Exception e) {
                    throw new ExtractException("ERROR ## get columnName has an error", e);
                }
            }
            return result;
        }
View Full Code Here

Examples of com.alibaba.otter.node.etl.extract.exceptions.ExtractException

                if (!CollectionUtils.isEmpty(eventData.getOldKeys())) {
                    eventData.setOldKeys(columnFilter(eventData.getOldKeys(), columns, mode));
                }

                if (CollectionUtils.isEmpty(eventData.getKeys())) { // 无主键,报错
                    throw new ExtractException(
                                               String.format("eventData after viewExtractor has no pks , pls check! identity:%s, new eventData:%s",
                                                             dbBatch.getRowBatch().getIdentity().toString(),
                                                             eventData.toString()));
                }
View Full Code Here

Examples of com.alibaba.otter.node.etl.extract.exceptions.ExtractException

                        Table table = dbDialect.findTable(eventData.getSchemaName(), eventData.getTableName());
                        List<EventColumn> newColumns = new ArrayList<EventColumn>();
                        Column[] primaryKeyColumns = table.getPrimaryKeyColumns();
                        if (primaryKeyColumns.length > pks.length) {
                            throw new ExtractException("data pk column size not match , data:" + eventData.toString());
                        }
                        // 构建字段
                        Column[] allColumns = table.getColumns();
                        for (int i = 0; i < allColumns.length; i++) {
                            Column column = allColumns[i];
View Full Code Here

Examples of com.alibaba.otter.node.etl.extract.exceptions.ExtractException

        // 读取一次配置
        adjustPoolSize(pipeline.getParameters().getExtractPoolSize()); // 调整下线程池,Extractor会被池化处理
        ExecutorCompletionService completionService = new ExecutorCompletionService(executor);

        // 进行并发提交
        ExtractException exception = null;
        // 每个表进行处理
        List<DataItem> items = new ArrayList<DataItem>();
        List<Future> futures = new ArrayList<Future>();
        List<EventData> eventDatas = dbBatch.getRowBatch().getDatas();
        for (EventData eventData : eventDatas) {
            DataItem item = new DataItem(eventData);
            // 针对row模式,需要去检查一下当前是否已经包含row记录的所有字段,如果发现字段不足,则执行一次数据库查询
            boolean flag = mustDb
                           || (eventData.getSyncConsistency() != null && eventData.getSyncConsistency().isMedia());

            // 增加一种case, 针对oracle erosa有时侯结果记录只有主键,没有变更字段,需要做一次反查
            if (!flag && CollectionUtils.isEmpty(eventData.getUpdatedColumns())) {
                DataMedia dataMedia = ConfigHelper.findDataMedia(pipeline, eventData.getTableId());
                if (dataMedia.getSource().getType().isOracle()) {
                    flag |= true;
                    eventData.setRemedy(true);// 针对这类数据,也统一视为补救的操作,可能erosa解析时反查数据库也不存在记录
                }
            }

            if (isRow && !flag) {
                // 提前判断一次,避免进入多线程进行竞争
                // 针对view视图的情况,会有后续再判断一次
                flag = checkNeedDbForRowMode(pipeline, eventData);
            }

            if (flag && (eventData.getEventType().isInsert() || eventData.getEventType().isUpdate())) {// 判断是否需要反查
                Future future = completionService.submit(new DatabaseExtractWorker(pipeline, item), null); // 提交进行并行查询
                if (future.isDone()) {
                    // 立即判断一次,因为使用了CallerRun可能当场跑出结果,针对有异常时快速响应,而不是等跑完所有的才抛异常
                    try {
                        future.get();
                    } catch (InterruptedException e) {
                        cancel(futures);// 取消完之后立马退出
                        throw new ExtractException(e);
                    } catch (ExecutionException e) {
                        cancel(futures); // 取消完之后立马退出
                        throw new ExtractException(e);
                    }
                }

                futures.add(future);// 记录一下添加的任务
            }

            items.add(item);// 按顺序添加
        }

        // 开始处理结果
        int index = 0;
        while (index < futures.size()) { // 循环处理发出去的所有任务
            try {
                Future future = completionService.take();// 它也可能被打断
                future.get();
            } catch (InterruptedException e) {
                exception = new ExtractException(e);
                break;// 如何一个future出现了异常,就退出
            } catch (ExecutionException e) {
                exception = new ExtractException(e);
                break;// 如何一个future出现了异常,就退出
            }

            index++;
        }
View Full Code Here

Examples of com.alibaba.otter.node.etl.extract.exceptions.ExtractException

            }

            List<DataMediaPair> dataMediaPairs = ConfigHelper.findDataMediaPairByMediaId(pipeline,
                                                                                         eventData.getTableId());
            if (dataMediaPairs == null) {
                throw new ExtractException("ERROR ## the dataMediaId = " + eventData.getTableId()
                                           + " dataMediaPair is null,please check");
            }

            for (DataMediaPair dataMediaPair : dataMediaPairs) {
                if (dataMediaPair.getResolverData() == null
                    || dataMediaPair.getResolverData().getExtensionDataType() == null
                    || (dataMediaPair.getResolverData().getExtensionDataType().isClazz() && StringUtils.isBlank(dataMediaPair.getResolverData().getClazzPath()))
                    || (dataMediaPair.getResolverData().getExtensionDataType().isSource() && StringUtils.isBlank(dataMediaPair.getResolverData().getSourceText()))) {
                    continue;
                }

                FileResolver fileResolver = null;

                if (dataMediaPair.getResolverData() != null) {
                    fileResolver = extensionFactory.getExtension(FileResolver.class, dataMediaPair.getResolverData());
                } else {
                    continue;
                }

                if (fileResolver == null) {
                    throw new ExtractException("ERROR ## the dataMediaId = " + eventData.getTableId()
                                               + " the fileResolver className  = "
                                               + dataMediaPair.getResolverData().getClazzPath()
                                               + " is null ,please check the class");
                }
View Full Code Here

Examples of com.alibaba.otter.node.etl.extract.exceptions.ExtractException

        Set<EventData> removeDatas = new HashSet<EventData>();// 使用set,提升remove时的查找速度
        for (EventData eventData : eventDatas) {
            List<DataMediaPair> dataMediaPairs = ConfigHelper.findDataMediaPairByMediaId(pipeline,
                                                                                         eventData.getTableId());
            if (dataMediaPairs == null) {
                throw new ExtractException("ERROR ## the dataMediaId = " + eventData.getTableId()
                                           + " dataMediaPair is null,please check");
            }

            for (DataMediaPair dataMediaPair : dataMediaPairs) {
                if (!dataMediaPair.isExistFilter()) {
View Full Code Here

Examples of com.alibaba.otter.node.etl.extract.exceptions.ExtractException

                        Table table = dbDialect.findTable(eventData.getSchemaName(), eventData.getTableName());
                        List<EventColumn> newColumns = new ArrayList<EventColumn>();
                        Column[] primaryKeyColumns = table.getPrimaryKeyColumns();
                        if (primaryKeyColumns.length > pks.length) {
                            throw new ExtractException("data pk column size not match , data:" + eventData.toString());
                        }
                        // 构建字段
                        Column[] allColumns = table.getColumns();
                        for (int i = 0; i < allColumns.length; i++) {
                            Column column = allColumns[i];
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.