Package com.alibaba.otter.canal.meta

Source Code of com.alibaba.otter.canal.meta.FileMixedMetaManager$FileMetaClientIdentityData

package com.alibaba.otter.canal.meta;

import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;

import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.MDC;
import org.springframework.util.Assert;

import com.alibaba.otter.canal.common.utils.JsonUtils;
import com.alibaba.otter.canal.meta.exception.CanalMetaManagerException;
import com.alibaba.otter.canal.protocol.ClientIdentity;
import com.alibaba.otter.canal.protocol.position.LogPosition;
import com.alibaba.otter.canal.protocol.position.Position;
import com.google.common.base.Function;
import com.google.common.collect.Lists;
import com.google.common.collect.MapMaker;

/**
* 基于文件刷新的metaManager实现
*
* <pre>
* 策略:
* 1. 先写内存,然后定时刷新数据到File
* 2. 数据采取overwrite模式(只保留最后一次),通过logger实施append模式(记录历史版本)
* </pre>
*
* @author jianghang 2013-4-15 下午05:55:57
* @version 1.0.4
*/
public class FileMixedMetaManager extends MemoryMetaManager implements CanalMetaManager {

    private static final Logger      logger       = LoggerFactory.getLogger(FileMixedMetaManager.class);
    private static final Charset     charset      = Charset.forName("UTF-8");
    private File                     dataDir;
    private String                   dataFileName = "meta.dat";
    private Map<String, File>        dataFileCaches;
    private ScheduledExecutorService executor;
    @SuppressWarnings("serial")
    private final Position           nullCursor   = new Position() {
                                                  };
    private long                     period       = 1000;                                               // 单位ms
    private Set<ClientIdentity>      updateCursorTasks;

    public void start() {
        super.start();
        Assert.notNull(dataDir);
        if (!dataDir.exists()) {
            try {
                FileUtils.forceMkdir(dataDir);
            } catch (IOException e) {
                e.printStackTrace();
            }
        }

        if (!dataDir.canRead() || !dataDir.canWrite()) {
            throw new CanalMetaManagerException("dir[" + dataDir.getPath() + "] can not read/write");
        }

        dataFileCaches = new MapMaker().makeComputingMap(new Function<String, File>() {

            public File apply(String destination) {
                return getDataFile(destination);
            }
        });

        executor = Executors.newScheduledThreadPool(1);
        destinations = new MapMaker().makeComputingMap(new Function<String, List<ClientIdentity>>() {

            public List<ClientIdentity> apply(String destination) {
                return loadClientIdentiry(destination);
            }
        });

        cursors = new MapMaker().makeComputingMap(new Function<ClientIdentity, Position>() {

            public Position apply(ClientIdentity clientIdentity) {
                Position position = loadCursor(clientIdentity.getDestination(), clientIdentity);
                if (position == null) {
                    return nullCursor; // 返回一个空对象标识,避免出现异常
                } else {
                    return position;
                }
            }
        });

        batches = new MapMaker().makeComputingMap(new Function<ClientIdentity, MemoryClientIdentityBatch>() {

            public MemoryClientIdentityBatch apply(ClientIdentity clientIdentity) {
                // 读取一下zookeeper信息,初始化一次
                return loadBatch(clientIdentity.getDestination(), clientIdentity);
            }
        });

        updateCursorTasks = Collections.synchronizedSet(new HashSet<ClientIdentity>());

        // 启动定时工作任务
        executor.scheduleAtFixedRate(new Runnable() {

            public void run() {
                List<ClientIdentity> tasks = new ArrayList<ClientIdentity>(updateCursorTasks);
                for (ClientIdentity clientIdentity : tasks) {
                    MDC.put("destination", String.valueOf(clientIdentity.getDestination()));
                    try {
                        // 定时将内存中的最新值刷到zookeeper中,多次变更只刷一次
                        if (logger.isInfoEnabled()) {
                            LogPosition cursor = (LogPosition) getCursor(clientIdentity);
                            logger.info("clientId:{} cursor:[{},{},{}] address[{}]", new Object[] {
                                    clientIdentity.getClientId(), cursor.getPostion().getJournalName(),
                                    cursor.getPostion().getPosition(), cursor.getPostion().getTimestamp(),
                                    cursor.getIdentity().getSourceAddress().toString() });
                        }
                        flushDataToFile(clientIdentity.getDestination());
                        updateCursorTasks.remove(clientIdentity);
                    } catch (Throwable e) {
                        // ignore
                        logger.error("period update" + clientIdentity.toString() + " curosr failed!", e);
                    }
                }
            }
        }, period, period, TimeUnit.MILLISECONDS);
    }

    public void stop() {
        super.stop();

        flushDataToFile();// 刷新数据
        executor.shutdownNow();
        destinations.clear();
        batches.clear();
    }

    public void subscribe(final ClientIdentity clientIdentity) throws CanalMetaManagerException {
        super.subscribe(clientIdentity);

        // 订阅信息频率发生比较低,不需要做定时merge处理
        executor.submit(new Runnable() {

            public void run() {
                flushDataToFile(clientIdentity.getDestination());
            }
        });
    }

    public void unsubscribe(final ClientIdentity clientIdentity) throws CanalMetaManagerException {
        super.unsubscribe(clientIdentity);

        // 订阅信息频率发生比较低,不需要做定时merge处理
        executor.submit(new Runnable() {

            public void run() {
                flushDataToFile(clientIdentity.getDestination());
            }
        });
    }

    public void updateCursor(ClientIdentity clientIdentity, Position position) throws CanalMetaManagerException {
        updateCursorTasks.add(clientIdentity);// 添加到任务队列中进行触发
        super.updateCursor(clientIdentity, position);
    }

    public Position getCursor(ClientIdentity clientIdentity) throws CanalMetaManagerException {
        Position position = super.getCursor(clientIdentity);
        if (position == nullCursor) {
            return null;
        } else {
            return position;
        }
    }

    // ============================ helper method ======================

    private File getDataFile(String destination) {
        File destinationMetaDir = new File(dataDir, destination);
        if (!destinationMetaDir.exists()) {
            try {
                FileUtils.forceMkdir(destinationMetaDir);
            } catch (IOException e) {
                throw new CanalMetaManagerException(e);
            }
        }

        return new File(destinationMetaDir, dataFileName);
    }

    private FileMetaInstanceData loadDataFromFile(File dataFile) {
        try {
            if (!dataFile.exists()) {
                return null;
            }

            String json = FileUtils.readFileToString(dataFile, charset.name());
            return JsonUtils.unmarshalFromString(json, FileMetaInstanceData.class);
        } catch (IOException e) {
            throw new CanalMetaManagerException(e);
        }
    }

    private void flushDataToFile() {
        for (String destination : destinations.keySet()) {
            flushDataToFile(destination);
        }
    }

    private void flushDataToFile(String destination) {
        flushDataToFile(destination, dataFileCaches.get(destination));
    }

    private void flushDataToFile(String destination, File dataFile) {
        FileMetaInstanceData data = new FileMetaInstanceData();
        if (destinations.containsKey(destination)) {
            synchronized (destination.intern()) { // 基于destination控制一下并发更新
                data.setDestination(destination);

                List<FileMetaClientIdentityData> clientDatas = Lists.newArrayList();
                List<ClientIdentity> clientIdentitys = destinations.get(destination);
                for (ClientIdentity clientIdentity : clientIdentitys) {
                    FileMetaClientIdentityData clientData = new FileMetaClientIdentityData();
                    clientData.setClientIdentity(clientIdentity);
                    Position position = cursors.get(clientIdentity);
                    if (position != null && position != nullCursor) {
                        clientData.setCursor((LogPosition) position);
                    }
                    clientData.setBatch(batches.get(clientIdentity));

                    clientDatas.add(clientData);
                }

                data.setClientDatas(clientDatas);
            }

            String json = JsonUtils.marshalToString(data);
            try {
                FileUtils.writeStringToFile(dataFile, json);
            } catch (IOException e) {
                throw new CanalMetaManagerException(e);
            }
        }
    }

    private List<ClientIdentity> loadClientIdentiry(String destination) {
        List<ClientIdentity> result = Lists.newArrayList();

        FileMetaInstanceData data = loadDataFromFile(dataFileCaches.get(destination));
        if (data == null) {
            return result;
        }

        List<FileMetaClientIdentityData> clientDatas = data.getClientDatas();
        if (clientDatas == null) {
            return result;
        }

        for (FileMetaClientIdentityData clientData : clientDatas) {
            result.add(clientData.getClientIdentity());
        }

        return result;
    }

    private Position loadCursor(String destination, ClientIdentity clientIdentity) {
        FileMetaInstanceData data = loadDataFromFile(dataFileCaches.get(destination));
        if (data == null) {
            return null;
        }

        List<FileMetaClientIdentityData> clientDatas = data.getClientDatas();
        if (clientDatas == null) {
            return null;
        }

        for (FileMetaClientIdentityData clientData : clientDatas) {
            if (clientData.getClientIdentity() != null && clientData.getClientIdentity().equals(clientIdentity)) {
                return clientData.getCursor();
            }
        }

        return null;
    }

    private MemoryClientIdentityBatch loadBatch(String destination, ClientIdentity clientIdentity) {
        FileMetaInstanceData data = loadDataFromFile(dataFileCaches.get(destination));
        if (data == null) {
            return MemoryClientIdentityBatch.create(clientIdentity);
        }

        List<FileMetaClientIdentityData> clientDatas = data.getClientDatas();
        if (clientDatas == null) {
            return MemoryClientIdentityBatch.create(clientIdentity);
        }

        for (FileMetaClientIdentityData clientData : clientDatas) {
            if (clientData.getClientIdentity() != null && clientData.getClientIdentity().equals(clientIdentity)) {
                return clientData.getBatch();
            }
        }

        return MemoryClientIdentityBatch.create(clientIdentity);
    }

    /**
     * 描述一个clientIdentity对应的数据对象
     *
     * @author jianghang 2013-4-15 下午06:19:40
     * @version 1.0.4
     */
    public static class FileMetaClientIdentityData {

        private ClientIdentity            clientIdentity;
        private MemoryClientIdentityBatch batch;
        private LogPosition               cursor;

        public FileMetaClientIdentityData(){

        }

        public FileMetaClientIdentityData(ClientIdentity clientIdentity, MemoryClientIdentityBatch batch,
                                          LogPosition cursor){
            this.clientIdentity = clientIdentity;
            this.batch = batch;
            this.cursor = cursor;
        }

        public ClientIdentity getClientIdentity() {
            return clientIdentity;
        }

        public void setClientIdentity(ClientIdentity clientIdentity) {
            this.clientIdentity = clientIdentity;
        }

        public MemoryClientIdentityBatch getBatch() {
            return batch;
        }

        public void setBatch(MemoryClientIdentityBatch batch) {
            this.batch = batch;
        }

        public Position getCursor() {
            return cursor;
        }

        public void setCursor(LogPosition cursor) {
            this.cursor = cursor;
        }

    }

    /**
     * 描述整个canal instance对应数据对象
     *
     * @author jianghang 2013-4-15 下午06:20:22
     * @version 1.0.4
     */
    public static class FileMetaInstanceData {

        private String                           destination;
        private List<FileMetaClientIdentityData> clientDatas;

        public FileMetaInstanceData(){

        }

        public FileMetaInstanceData(String destination, List<FileMetaClientIdentityData> clientDatas){
            this.destination = destination;
            this.clientDatas = clientDatas;
        }

        public String getDestination() {
            return destination;
        }

        public void setDestination(String destination) {
            this.destination = destination;
        }

        public List<FileMetaClientIdentityData> getClientDatas() {
            return clientDatas;
        }

        public void setClientDatas(List<FileMetaClientIdentityData> clientDatas) {
            this.clientDatas = clientDatas;
        }

    }

    public void setDataDir(String dataDir) {
        this.dataDir = new File(dataDir);
    }

    public void setDataDir(File dataDir) {
        this.dataDir = dataDir;
    }

    public void setPeriod(long period) {
        this.period = period;
    }

}
TOP

Related Classes of com.alibaba.otter.canal.meta.FileMixedMetaManager$FileMetaClientIdentityData

TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.