Package com.alibaba.otter.node.etl.load.loader.db.context

Examples of com.alibaba.otter.node.etl.load.loader.db.context.FileLoadContext

110
111
112
113
114
115
116
117
118
119
120
        Identity id = buildIdentity(1L, 2L, 3L);
        FileBatch fileBatch = buildFileBatch(id);
        fileBatch.getFiles().addAll(buildFileDatas(null, EventType.INSERT, fileDataStartIndex, fileDataCount, true));

        WeightController controller = new WeightController(1);
        FileLoadContext context = fileLoadAction.load(fileBatch, ROOT_DIR, controller);
        want.object(context.getChannel()).isEqualTo(channel);
        want.object(context.getPipeline()).isEqualTo(pipeline);
        want.object(context.getPrepareDatas()).isEqualTo(fileBatch.getFiles());
        want.number(context.getProcessedDatas().size()).isEqualTo(fileBatch.getFiles().size());
    }
View Full Code Here
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
     */
    public FileLoadContext load(FileBatch fileBatch, File rootDir, WeightController controller) {
        if (false == rootDir.exists()) {
            throw new LoadException(rootDir.getPath() + " is not exist");
        }
        FileLoadContext context = buildContext(fileBatch.getIdentity());
        context.setPrepareDatas(fileBatch.getFiles());
        boolean isDryRun = context.getPipeline().getParameters().isDryRun();
        try {
            // 复制成功的文件信息
            WeightBuckets<FileData> buckets = buildWeightBuckets(fileBatch.getIdentity(), fileBatch.getFiles());
            List<Long> weights = buckets.weights();
            controller.start(weights);
            // 处理数据
            for (int i = 0; i < weights.size(); i++) {
                Long weight = weights.get(i);
                controller.await(weight.intValue());
                if (logger.isInfoEnabled()) {
                    logger.debug("##start load for weight:{}\n", weight);
                }

                // 处理同一个weight下的数据
                List<FileData> items = buckets.getItems(weight);
                if (context.getPipeline().getParameters().isDryRun()) {
                    dryRun(context, items, rootDir);
                } else {
                    moveFiles(context, items, rootDir);
                }

View Full Code Here
157
158
159
160
161
162
163
164
165
166
167
168
169
        }

    }

    private FileLoadContext buildContext(Identity identity) {
        FileLoadContext context = new FileLoadContext();
        context.setIdentity(identity);
        Channel channel = configClientService.findChannel(identity.getChannelId());
        Pipeline pipeline = configClientService.findPipeline(identity.getPipelineId());
        context.setChannel(channel);
        context.setPipeline(pipeline);
        return context;
    }
View Full Code Here
TOP

Related Classes of com.alibaba.otter.node.etl.load.loader.db.context.FileLoadContext

  • com.alibaba.otter.node.etl.load.FileLoadActionTest
  • com.alibaba.otter.node.etl.load.loader.db.FileLoadAction
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.