├── README.md ├── pom.xml └── src ├── main ├── assembly │ └── release.xml ├── bin │ └── start-server.sh ├── java │ └── mysql │ │ └── redis │ │ └── replicate │ │ ├── Conf.java │ │ ├── Constant.java │ │ ├── CoordinatorController.java │ │ ├── HttpClientUtils.java │ │ ├── Lifecycle.java │ │ ├── LogbackConfigLoader.java │ │ ├── LoggerFactory.java │ │ ├── Monitor.java │ │ ├── MysqlRedisReplicateBootstrap.java │ │ ├── Threads.java │ │ ├── Tuple.java │ │ ├── ZkPathUtils.java │ │ ├── ZookeeperLeaderElector.java │ │ ├── ZookeeperUtils.java │ │ ├── canal │ │ ├── AbstractSink.java │ │ ├── ControllerService.java │ │ ├── MessagePuller.java │ │ └── MessageSink.java │ │ ├── config │ │ ├── DestinationConfig.java │ │ └── DestinationConfigManager.java │ │ ├── groovy │ │ ├── IRow2map.java │ │ ├── Row2map.java │ │ └── Row2mapManager.java │ │ ├── redis │ │ ├── RedisSink.java │ │ └── RedisUtils.java │ │ └── web │ │ ├── AliveServerServlet.java │ │ ├── AllDestinationServlet.java │ │ ├── DestinationOptServlet.java │ │ ├── EndpointServlet.java │ │ ├── GetConfigServlet.java │ │ ├── MonitorServlet.java │ │ ├── SaveConfigServlet.java │ │ ├── TestConfigServlet.java │ │ ├── Utils.java │ │ └── WebConsole.java └── resources │ ├── config.properties │ ├── logback.xml │ ├── row2map.groovy.tpl │ └── site │ ├── css │ └── bootstrap.css │ ├── destination.html │ ├── index.html │ └── js │ └── jquery-1.7.2.min.js └── test ├── doc └── test_script.sql └── java └── mysql └── redis └── replicate ├── HttpClientUtilsTest.java ├── ZookeeperLeaderElectorTest.java └── canal └── MysqlUtils.java /README.md: -------------------------------------------------------------------------------- 1 | # mysql-redis-replicate 2 | 3 | --- 4 | 目前还不稳定,还在开发完善中 5 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | 4.0.0 7 | com.gone 8 | mysql-redis-replicate 9 | 1.0 10 | 11 | 12 | 13 | 14 | org.slf4j 15 | slf4j-api 16 | 1.7.12 17 | 18 | 19 | 20 | com.alibaba.otter 21 | canal.server 22 | 1.0.20 23 | 24 | 25 | 26 | mysql 27 | mysql-connector-java 28 | 5.1.36 29 | 30 | 31 | 32 | org.codehaus.groovy 33 | groovy-all 34 | 2.4.5 35 | 36 | 37 | 38 | com.google.guava 39 | guava 40 | 18.0 41 | 42 | 43 | 44 | org.eclipse.jetty.aggregate 45 | jetty-all-server 46 | 8.0.3.v20111011 47 | 48 | 49 | 50 | redis.clients 51 | jedis 52 | 2.6.2 53 | 54 | 55 | 56 | junit 57 | junit 58 | 4.1 59 | test 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | org.apache.maven.plugins 72 | maven-jar-plugin 73 | 2.6 74 | 75 | 76 | true 77 | 78 | 79 | **/logback.xml 80 | **/main.properties 81 | **/destinations/** 82 | 83 | 84 | 85 | 86 | 87 | org.apache.maven.plugins 88 | maven-assembly-plugin 89 | 2.6 90 | 91 | 92 | assemble 93 | 94 | single 95 | 96 | package 97 | 98 | 99 | 100 | false 101 | false 102 | 103 | 104 | ${basedir}/src/main/assembly/release.xml 105 | 106 | 107 | ${project.artifactId}-${project.version} 108 | 109 | ${project.parent.build.directory} 110 | 111 | 112 | 113 | 114 | maven-compiler-plugin 115 | 116 | 1.8 117 | 1.8 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | -------------------------------------------------------------------------------- /src/main/assembly/release.xml: -------------------------------------------------------------------------------- 1 | 4 | dist 5 | 6 | tar.gz 7 | 8 | true 9 | 10 | 11 | . 12 | / 13 | 14 | README* 15 | 16 | 17 | 18 | ./src/main/bin 19 | bin 20 | 21 | **/* 22 | 23 | 0755 24 | 25 | 26 | ./src/main/conf 27 | /conf 28 | 29 | **/* 30 | 31 | 32 | 33 | ./src/main/resources 34 | /conf 35 | 36 | **/* 37 | 38 | 39 | 40 | target 41 | logs 42 | 43 | **/* 44 | 45 | 46 | 47 | 48 | 49 | lib 50 | 51 | junit:junit 52 | 53 | 54 | 55 | 56 | -------------------------------------------------------------------------------- /src/main/bin/start-server.sh: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | 3 | if [ -z "$JAVA_HOME" ] ; then 4 | export JAVA_HOME=/usr/local/java 5 | fi 6 | 7 | 8 | SCRIPT="$0" 9 | while [ -h "$SCRIPT" ] ; do 10 | ls=`ls -ld "$SCRIPT"` 11 | # Drop everything prior to -> 12 | link=`expr "$ls" : '.*-> \(.*\)$'` 13 | if expr "$link" : '/.*' > /dev/null; then 14 | SCRIPT="$link" 15 | else 16 | SCRIPT=`dirname "$SCRIPT"`/"$link" 17 | fi 18 | done 19 | 20 | SERVER_HOME=`dirname "$SCRIPT"` 21 | SERVER_HOME=`cd "$SERVER_HOME" ; cd .. ; pwd` 22 | export SERVER_HOME 23 | 24 | LIBDIR=$SERVER_HOME/lib 25 | 26 | CLASSPATH=${CLASSPATH}:${SERVER_HOME}/conf 27 | 28 | for lib in ${LIBDIR}/*.jar 29 | do 30 | CLASSPATH=$CLASSPATH:$lib 31 | done 32 | 33 | java=$JAVA_HOME/bin/java 34 | 35 | JAVA_OPTS=" 36 | -Xmx2G 37 | -Xms2G 38 | -XX:PermSize=128M 39 | -XX:MaxPermSize=256M 40 | -XX:+UseConcMarkSweepGC 41 | -XX:+UseParNewGC 42 | -XX:+CMSConcurrentMTEnabled 43 | -XX:+CMSParallelRemarkEnabled 44 | -XX:+UseCMSCompactAtFullCollection 45 | -XX:CMSFullGCsBeforeCompaction=0 46 | -XX:+CMSClassUnloadingEnabled 47 | -XX:LargePageSizeInBytes=128M 48 | -XX:+UseFastAccessorMethods 49 | -XX:+UseCMSInitiatingOccupancyOnly 50 | -XX:CMSInitiatingOccupancyFraction=80 51 | -XX:SoftRefLRUPolicyMSPerMB=0 52 | -XX:+PrintClassHistogram 53 | -XX:+PrintGCDetails 54 | -XX:+PrintGCTimeStamps 55 | -XX:+PrintHeapAtGC 56 | -Xloggc:/data/logs/gc.log 57 | -XX:+HeapDumpOnOutOfMemoryError 58 | -XX:HeapDumpPath=/data/logs/dump.hprof 59 | " 60 | 61 | echo "JAVA_HOME :$JAVA_HOME" 62 | echo "SERVER_HOME:$SERVER_HOME" 63 | echo "CLASSPATH :$CLASSPATH" 64 | echo "JAVA_OPTS :$JAVA_OPTS" 65 | 66 | cd $SERVER_HOME 67 | 68 | exec $java -classpath $CLASSPATH $JAVA_OPTS mysql.redis.replicate.MysqlRedisReplicateBootstrap & -------------------------------------------------------------------------------- /src/main/java/mysql/redis/replicate/Conf.java: -------------------------------------------------------------------------------- 1 | package mysql.redis.replicate; 2 | 3 | import com.google.common.collect.Maps; 4 | import org.apache.commons.beanutils.BeanUtils; 5 | import org.apache.commons.lang.StringUtils; 6 | import redis.clients.jedis.JedisPoolConfig; 7 | 8 | import java.io.FileInputStream; 9 | import java.io.IOException; 10 | import java.util.Map; 11 | import java.util.Properties; 12 | import java.util.Set; 13 | 14 | /** 15 | * Created by wens on 15-10-14. 16 | */ 17 | public class Conf { 18 | 19 | private String id; 20 | 21 | private String zookeeperServer; 22 | 23 | private String zookeeperRootPath; 24 | 25 | private int canalBatchSize; 26 | 27 | private int webConsolePort; 28 | 29 | private String httpEndpoin; 30 | 31 | private JedisPoolConfig jedisPoolConfig; 32 | 33 | public void setCanalBatchSize(int canalBatchSize) { 34 | this.canalBatchSize = canalBatchSize; 35 | } 36 | 37 | public int getWebConsolePort() { 38 | return webConsolePort; 39 | } 40 | 41 | public void setWebConsolePort(int webConsolePort) { 42 | this.webConsolePort = webConsolePort; 43 | } 44 | 45 | public JedisPoolConfig getJedisPoolConfig() { 46 | return jedisPoolConfig; 47 | } 48 | 49 | public void setJedisPoolConfig(JedisPoolConfig jedisPoolConfig) { 50 | this.jedisPoolConfig = jedisPoolConfig; 51 | } 52 | 53 | public String getId() { 54 | return id; 55 | } 56 | 57 | public void setId(String id) { 58 | this.id = id; 59 | } 60 | 61 | public String getZookeeperRootPath() { 62 | return zookeeperRootPath; 63 | } 64 | 65 | public void setZookeeperRootPath(String zookeeperRootPath) { 66 | this.zookeeperRootPath = zookeeperRootPath; 67 | } 68 | 69 | private static Conf INSTANCE = new Conf(); 70 | 71 | public static Conf getInstance() { 72 | return INSTANCE; 73 | } 74 | 75 | private Conf() { 76 | 77 | String conf = System.getProperty("canal.conf", "classpath:config.properties"); 78 | Properties properties = new Properties(); 79 | if (conf.startsWith(Constant.CLASSPATH_URL_PREFIX)) { 80 | conf = StringUtils.substringAfter(conf, Constant.CLASSPATH_URL_PREFIX); 81 | try { 82 | properties.load(Thread.currentThread().getContextClassLoader().getResourceAsStream(conf)); 83 | } catch (IOException e) { 84 | throw new RuntimeException(e); 85 | } 86 | } else { 87 | try { 88 | properties.load(new FileInputStream(conf)); 89 | } catch (IOException e) { 90 | throw new RuntimeException(e); 91 | } 92 | } 93 | init(properties); 94 | 95 | } 96 | 97 | 98 | private void init(Properties properties) { 99 | this.zookeeperServer = properties.getProperty("zk.server"); 100 | this.id = properties.getProperty("id"); 101 | this.zookeeperRootPath = properties.getProperty("zk.root.path", "/mysql-redis-replicate"); 102 | String cbs = properties.getProperty("canal.batch.size"); 103 | this.canalBatchSize = cbs == null || cbs.length() == 0 ? 100 : Integer.parseInt(cbs); 104 | this.webConsolePort = Integer.parseInt(properties.getProperty("web.console.port")); 105 | this.httpEndpoin = properties.getProperty("http.endpoint", "http://localhost:" + this.getWebConsolePort() + "/endpoint"); 106 | jedisPoolConfig = new JedisPoolConfig(); 107 | Set redisKeys = properties.keySet(); 108 | Map map = Maps.newHashMap(); 109 | for (Object key_ : redisKeys) { 110 | String key = (String) key_; 111 | if (!key.startsWith("redis.")) continue; 112 | map.put(key.substring(5), (String) properties.get(key)); 113 | } 114 | try { 115 | BeanUtils.populate(jedisPoolConfig, map); 116 | } catch (Exception e) { 117 | throw new RuntimeException(e); 118 | } 119 | } 120 | 121 | public String getZookeeperServer() { 122 | return zookeeperServer == null ? "localhost:2181" : zookeeperServer; 123 | } 124 | 125 | public void setZookeeperServer(String zookeeperServer) { 126 | this.zookeeperServer = zookeeperServer; 127 | } 128 | 129 | public int getCanalBatchSize() { 130 | return canalBatchSize; 131 | } 132 | 133 | public String getHttpEndpoin() { 134 | return httpEndpoin; 135 | } 136 | } 137 | -------------------------------------------------------------------------------- /src/main/java/mysql/redis/replicate/Constant.java: -------------------------------------------------------------------------------- 1 | package mysql.redis.replicate; 2 | 3 | /** 4 | * Created by wens on 15-10-30. 5 | */ 6 | public class Constant { 7 | 8 | public static final String CLASSPATH_URL_PREFIX = "classpath:"; 9 | 10 | } 11 | -------------------------------------------------------------------------------- /src/main/java/mysql/redis/replicate/CoordinatorController.java: -------------------------------------------------------------------------------- 1 | package mysql.redis.replicate; 2 | 3 | import com.alibaba.otter.canal.common.utils.AddressUtils; 4 | import mysql.redis.replicate.config.DestinationConfig; 5 | import mysql.redis.replicate.config.DestinationConfigManager; 6 | import org.I0Itec.zkclient.IZkChildListener; 7 | import org.I0Itec.zkclient.IZkStateListener; 8 | import org.apache.zookeeper.Watcher; 9 | import org.slf4j.Logger; 10 | 11 | import java.util.List; 12 | import java.util.Random; 13 | 14 | /** 15 | * Created by wens on 15/12/5. 16 | */ 17 | public class CoordinatorController implements ZookeeperLeaderElector.LeaderListener { 18 | 19 | private final Logger logger = LoggerFactory.getLogger(); 20 | 21 | private DestinationConfigManager destinationConfigManager; 22 | 23 | private String myId; 24 | private String httpEndpoint; 25 | private volatile List aliveServerIds; 26 | 27 | private ZookeeperLeaderElector zookeeperLeaderElector; 28 | 29 | public CoordinatorController(DestinationConfigManager destinationConfigManager, String myId, String httpEndPoint) { 30 | zookeeperLeaderElector = new ZookeeperLeaderElector(myId, this); 31 | this.destinationConfigManager = destinationConfigManager; 32 | this.myId = myId; 33 | this.httpEndpoint = httpEndPoint; 34 | } 35 | 36 | 37 | public void start() { 38 | 39 | ServerInfo serverInfo = new ServerInfo(myId, AddressUtils.getHostIp(), httpEndpoint, System.currentTimeMillis()); 40 | ZookeeperUtils.createEphemeral(ZkPathUtils.getIdsPath(myId), serverInfo); 41 | ZookeeperUtils.subscribeStateChanges(new IZkStateListener() { 42 | @Override 43 | public void handleStateChanged(Watcher.Event.KeeperState state) throws Exception { 44 | System.out.println("----"); 45 | } 46 | 47 | @Override 48 | public void handleNewSession() throws Exception { 49 | 50 | try { 51 | ZookeeperUtils.createEphemeral(ZkPathUtils.getIdsPath(myId), serverInfo); 52 | } catch (Exception e) { 53 | logger.error("Create ephemeral path fail : " + ZkPathUtils.getIdsPath(myId)); 54 | } 55 | 56 | } 57 | }); 58 | this.aliveServerIds = ZookeeperUtils.getChildren(ZkPathUtils.getIdsPath()); 59 | zookeeperLeaderElector.start(); 60 | } 61 | 62 | 63 | public void stop() { 64 | zookeeperLeaderElector.stop(); 65 | ZookeeperUtils.delete(ZkPathUtils.getIdsPath(myId)); 66 | } 67 | 68 | @Override 69 | public void onBecomingLeader() { 70 | ZookeeperUtils.subscribeChildChanges(ZkPathUtils.getIdsPath(), new ServerChangeListener()); 71 | this.aliveServerIds = ZookeeperUtils.getChildren(ZkPathUtils.getIdsPath()); 72 | checkDestination(); 73 | 74 | } 75 | 76 | public boolean stopDestination(String destination) { 77 | DestinationConfig destinationConfig = destinationConfigManager.getDestinationConfig(destination); 78 | String ret = doStopDestination(destination, destinationConfig); 79 | destinationConfig.setStopped(true); 80 | destinationConfigManager.saveOrUpdate(destinationConfig); 81 | return "ok".equals(ret) ? true : false; 82 | 83 | } 84 | 85 | private String doStopDestination(String destination, DestinationConfig destinationConfig) { 86 | String ret = "ok"; 87 | if (aliveServerIds.contains(destinationConfig.getRunOn())) { 88 | ServerInfo serverInfo = getServerInfo(destinationConfig.getRunOn()); 89 | ret = HttpClientUtils.post(serverInfo.getHttpEndpoint(), Tuple.of("cmd", "stop"), Tuple.of("destination", destination)); 90 | } else { 91 | logger.warn("stop destination on {},but {} is not alive.", destinationConfig.getRunOn(), destinationConfig.getRunOn()); 92 | } 93 | return ret; 94 | } 95 | 96 | public boolean startDestination(String destination, String serverId) { 97 | 98 | DestinationConfig destinationConfig = destinationConfigManager.getDestinationConfig(destination); 99 | if (!destinationConfig.isStopped()) { 100 | doStopDestination(destination, destinationConfig); 101 | } 102 | 103 | String ret = doStartDestination(destination, serverId, destinationConfig); 104 | if ("ok".equals(ret)) { 105 | destinationConfig.setStopped(false); 106 | destinationConfig.setRunFail(false); 107 | destinationConfig.setRunOn(serverId); 108 | } else { 109 | destinationConfig.setStopped(true); 110 | destinationConfig.setRunFail(true); 111 | destinationConfig.setRunOn(serverId); 112 | } 113 | 114 | destinationConfigManager.saveOrUpdate(destinationConfig); 115 | return "ok".equals(ret) ? true : false; 116 | 117 | } 118 | 119 | private String doStartDestination(String destination, String serverId, DestinationConfig destinationConfig) { 120 | String ret = "ok"; 121 | if (aliveServerIds.contains(serverId)) { 122 | ServerInfo serverInfo = getServerInfo(serverId); 123 | ret = HttpClientUtils.post(serverInfo.getHttpEndpoint(), Tuple.of("cmd", "start"), Tuple.of("destination", destination)); 124 | } else { 125 | logger.warn("start destination on {},but {} is not alive.", serverId, serverId); 126 | throw new RuntimeException("Server " + serverId + " is not alive"); 127 | } 128 | return ret; 129 | } 130 | 131 | public void deleteDestination(String destination) { 132 | DestinationConfig destinationConfig = destinationConfigManager.getDestinationConfig(destination); 133 | if (!destinationConfig.isRunFail()) { 134 | doStopDestination(destination, destinationConfig); 135 | } 136 | destinationConfigManager.delete(destination); 137 | } 138 | 139 | public List getAliveServerIds() { 140 | return aliveServerIds; 141 | } 142 | 143 | private class ServerChangeListener implements IZkChildListener { 144 | 145 | @Override 146 | public void handleChildChange(String parentPath, List currentChilds) throws Exception { 147 | aliveServerIds = currentChilds; 148 | checkDestination(); 149 | } 150 | 151 | 152 | } 153 | 154 | private void checkDestination() { 155 | 156 | List allDestinationConfig = destinationConfigManager.getAllDestinationConfig(); 157 | 158 | for (DestinationConfig destinationConfig : allDestinationConfig) { 159 | if (!destinationConfig.isStopped()) { 160 | String serverId = destinationConfig.getRunOn(); 161 | if (!aliveServerIds.contains(destinationConfig.getRunOn())) { 162 | Random random = new Random(); 163 | serverId = aliveServerIds.get(random.nextInt(aliveServerIds.size())); 164 | } 165 | ServerInfo serverInfo = getServerInfo(serverId); 166 | boolean fail = invokeEndpointForStart(destinationConfig, serverInfo); 167 | destinationConfig.setRunOn(serverInfo.id); 168 | destinationConfig.setRunFail(fail); 169 | destinationConfigManager.saveOrUpdate(destinationConfig); 170 | } 171 | } 172 | } 173 | 174 | private boolean invokeEndpointForStart(DestinationConfig destinationConfig, ServerInfo serverInfo) { 175 | boolean fail = true; 176 | int retry = 0; 177 | while (retry <= 2) { 178 | try { 179 | String ret = HttpClientUtils.post(serverInfo.getHttpEndpoint(), Tuple.of("cmd", "start"), Tuple.of("destination", destinationConfig.getDestination())); 180 | if ("ok".equals(ret)) { 181 | fail = false; 182 | break; 183 | } 184 | } catch (Exception e) { 185 | logger.error("Retry " + retry + " Run destination fail : dest=" + destinationConfig.getDestination() + ",serverId = " + serverInfo.getId(), e); 186 | retry++; 187 | } 188 | } 189 | return fail; 190 | } 191 | 192 | public ServerInfo getServerInfo(String id) { 193 | return ZookeeperUtils.readData(ZkPathUtils.getIdsPath(id), ServerInfo.class); 194 | } 195 | 196 | public static class ServerInfo { 197 | 198 | String id; 199 | String host; 200 | String httpEndpoint; 201 | long timestamp; 202 | 203 | public ServerInfo() { 204 | } 205 | 206 | public ServerInfo(String id, String host, String httpEndpoint, long timestamp) { 207 | this.id = id; 208 | this.host = host; 209 | this.httpEndpoint = httpEndpoint; 210 | this.timestamp = timestamp; 211 | } 212 | 213 | public String getId() { 214 | return id; 215 | } 216 | 217 | public void setId(String id) { 218 | this.id = id; 219 | } 220 | 221 | public String getHost() { 222 | return host; 223 | } 224 | 225 | public void setHost(String host) { 226 | this.host = host; 227 | } 228 | 229 | public String getHttpEndpoint() { 230 | return httpEndpoint; 231 | } 232 | 233 | public void setHttpEndpoint(String httpEndpoint) { 234 | this.httpEndpoint = httpEndpoint; 235 | } 236 | 237 | public long getTimestamp() { 238 | return timestamp; 239 | } 240 | 241 | public void setTimestamp(long timestamp) { 242 | this.timestamp = timestamp; 243 | } 244 | } 245 | } 246 | -------------------------------------------------------------------------------- /src/main/java/mysql/redis/replicate/HttpClientUtils.java: -------------------------------------------------------------------------------- 1 | package mysql.redis.replicate; 2 | 3 | import com.google.common.base.Charsets; 4 | 5 | import java.io.ByteArrayOutputStream; 6 | import java.io.IOException; 7 | import java.io.InputStream; 8 | import java.net.HttpURLConnection; 9 | import java.net.URL; 10 | import java.net.URLEncoder; 11 | 12 | /** 13 | * Created by wens on 15/12/6. 14 | */ 15 | public class HttpClientUtils { 16 | 17 | public static String post(String url, Tuple... params) { 18 | 19 | HttpURLConnection httpURLConnection = null; 20 | try { 21 | httpURLConnection = (HttpURLConnection) (new URL(url).openConnection()); 22 | 23 | httpURLConnection.setRequestMethod("GET"); 24 | httpURLConnection.setReadTimeout(60000); 25 | httpURLConnection.setDoInput(true); 26 | httpURLConnection.setDoOutput(true); 27 | httpURLConnection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded"); 28 | 29 | if (params != null && params.length != 0) { 30 | StringBuilder playload = new StringBuilder(1); 31 | 32 | for (Tuple p : params) { 33 | if (playload.length() != 0) { 34 | playload.append("&"); 35 | } 36 | playload.append(p.getOne()).append("=").append(URLEncoder.encode(p.getTwo(), "utf-8")); 37 | } 38 | 39 | httpURLConnection.getOutputStream().write(playload.toString().getBytes(Charsets.UTF_8)); 40 | } 41 | 42 | if (httpURLConnection.getResponseCode() == 200) { 43 | 44 | InputStream inputStream = httpURLConnection.getInputStream(); 45 | 46 | ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(1024); 47 | byte[] buf = new byte[1024]; 48 | int n; 49 | while ((n = inputStream.read(buf)) != -1) { 50 | byteArrayOutputStream.write(buf, 0, n); 51 | } 52 | return byteArrayOutputStream.toString(); 53 | } else { 54 | throw new RuntimeException("Response code is " + httpURLConnection.getResponseCode()); 55 | } 56 | 57 | } catch (IOException e) { 58 | throw new RuntimeException(e); 59 | } finally { 60 | if (httpURLConnection != null) { 61 | httpURLConnection.disconnect(); 62 | } 63 | } 64 | 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /src/main/java/mysql/redis/replicate/Lifecycle.java: -------------------------------------------------------------------------------- 1 | package mysql.redis.replicate; 2 | 3 | /** 4 | * Created by wens on 15-12-8. 5 | */ 6 | public interface Lifecycle { 7 | 8 | void start(); 9 | 10 | void stop(); 11 | } 12 | -------------------------------------------------------------------------------- /src/main/java/mysql/redis/replicate/LogbackConfigLoader.java: -------------------------------------------------------------------------------- 1 | package mysql.redis.replicate; 2 | 3 | import ch.qos.logback.classic.LoggerContext; 4 | import ch.qos.logback.classic.joran.JoranConfigurator; 5 | import ch.qos.logback.core.joran.spi.JoranException; 6 | import ch.qos.logback.core.util.StatusPrinter; 7 | import org.slf4j.LoggerFactory; 8 | 9 | import java.io.File; 10 | import java.io.IOException; 11 | import java.net.URL; 12 | 13 | /** 14 | * Created by wens on 15-10-21. 15 | */ 16 | public class LogbackConfigLoader { 17 | 18 | 19 | public static void load() throws IOException { 20 | URL resource = Thread.currentThread().getContextClassLoader().getResource("logback.xml"); 21 | load(resource.getFile()); 22 | } 23 | 24 | public static void main(String[] args) throws IOException { 25 | load(); 26 | } 27 | 28 | 29 | public static void load(String location) throws IOException { 30 | 31 | LoggerContext lc = (LoggerContext) LoggerFactory.getILoggerFactory(); 32 | File externalConfigFile = new File(location); 33 | if (!externalConfigFile.exists()) { 34 | throw new IOException("Logback External Conf File Parameter does not reference a file that exists"); 35 | } else { 36 | if (!externalConfigFile.isFile()) { 37 | throw new IOException("Logback External Conf File Parameter exists, but does not reference a file"); 38 | } else { 39 | if (!externalConfigFile.canRead()) { 40 | throw new IOException("Logback External Conf File exists and is a file, but cannot be read."); 41 | } else { 42 | JoranConfigurator configurator = new JoranConfigurator(); 43 | configurator.setContext(lc); 44 | lc.reset(); 45 | try { 46 | configurator.doConfigure(location); 47 | } catch (JoranException e) { 48 | throw new RuntimeException(e); 49 | } 50 | StatusPrinter.printInCaseOfErrorsOrWarnings(lc); 51 | } 52 | } 53 | } 54 | 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/main/java/mysql/redis/replicate/LoggerFactory.java: -------------------------------------------------------------------------------- 1 | package mysql.redis.replicate; 2 | 3 | import org.slf4j.Logger; 4 | 5 | /** 6 | * Created by wens on 15-10-29. 7 | */ 8 | public class LoggerFactory { 9 | 10 | public static final String LOG_NAME = "mysql-redis-replicate"; 11 | 12 | 13 | public static Logger getLogger() { 14 | return org.slf4j.LoggerFactory.getLogger(LoggerFactory.LOG_NAME); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/main/java/mysql/redis/replicate/Monitor.java: -------------------------------------------------------------------------------- 1 | package mysql.redis.replicate; 2 | 3 | import java.util.concurrent.ConcurrentHashMap; 4 | import java.util.concurrent.atomic.AtomicLong; 5 | 6 | /** 7 | * Created by wens on 15-10-20. 8 | */ 9 | public class Monitor { 10 | 11 | public static class MonitorData extends Thread { 12 | public MonitorData(){ 13 | start(); 14 | } 15 | public volatile String logfileName = ""; 16 | public volatile long logfileOffset; 17 | public AtomicLong insertCount = new AtomicLong(0); 18 | public AtomicLong updateCount = new AtomicLong(0); 19 | public AtomicLong deleteCount = new AtomicLong(0); 20 | public AtomicLong speed = new AtomicLong(0); 21 | 22 | @Override 23 | public void run() { 24 | long lastTotal = 0 ; 25 | while (true){ 26 | long total = insertCount.get() + updateCount.get() + deleteCount.get() ; 27 | speed.set( ( total - lastTotal) / 5 ); 28 | lastTotal = total ; 29 | try { 30 | Thread.sleep(5000); 31 | } catch (InterruptedException e) { 32 | Thread.currentThread().interrupt(); 33 | } 34 | 35 | 36 | } 37 | 38 | } 39 | } 40 | 41 | private static ConcurrentHashMap cache = new ConcurrentHashMap<>(); 42 | 43 | 44 | 45 | public static void updateLogPosition(String dest, String logfileName, long logfileOffset) { 46 | MonitorData monitorData = getMonitorData(dest); 47 | monitorData.logfileName = logfileName; 48 | monitorData.logfileOffset = logfileOffset; 49 | } 50 | 51 | public static void incrInsertCount(String dest, int step) { 52 | getMonitorData(dest).insertCount.addAndGet(step); 53 | } 54 | 55 | public static void incrUpdateCount(String dest, int step) { 56 | getMonitorData(dest).updateCount.addAndGet(step); 57 | } 58 | 59 | public static void incrDeleteCount(String dest, int step) { 60 | getMonitorData(dest).deleteCount.addAndGet(step); 61 | } 62 | 63 | public static MonitorData getMonitorData(String dest) { 64 | MonitorData monitorData = cache.get(dest); 65 | 66 | if (monitorData == null) { 67 | monitorData = new MonitorData(); 68 | MonitorData old = cache.putIfAbsent(dest, monitorData); 69 | if (old != null) { 70 | monitorData = old; 71 | } 72 | } 73 | return monitorData; 74 | } 75 | 76 | } 77 | -------------------------------------------------------------------------------- /src/main/java/mysql/redis/replicate/MysqlRedisReplicateBootstrap.java: -------------------------------------------------------------------------------- 1 | package mysql.redis.replicate; 2 | 3 | import mysql.redis.replicate.canal.ControllerService; 4 | import mysql.redis.replicate.config.DestinationConfigManager; 5 | import mysql.redis.replicate.web.WebConsole; 6 | import org.apache.commons.lang.exception.ExceptionUtils; 7 | import org.slf4j.Logger; 8 | 9 | /** 10 | * Created by wens on 15-10-14. 11 | */ 12 | public class MysqlRedisReplicateBootstrap { 13 | 14 | private static final Logger logger = LoggerFactory.getLogger(); 15 | 16 | public static void main(String[] args) throws Throwable { 17 | try { 18 | 19 | LogbackConfigLoader.load(); 20 | Conf conf = Conf.getInstance(); 21 | ZookeeperUtils.init(conf); 22 | final DestinationConfigManager destinationConfigManager = new DestinationConfigManager(); 23 | final CoordinatorController coordinatorController = new CoordinatorController(destinationConfigManager, conf.getId(), conf.getHttpEndpoin()); 24 | final ControllerService controllerService = new ControllerService(conf); 25 | controllerService.setDestinationConfigManager(destinationConfigManager); 26 | final WebConsole webConsole = new WebConsole(conf, controllerService, destinationConfigManager, coordinatorController); 27 | 28 | controllerService.start(); 29 | 30 | logger.info("## start the controller service success."); 31 | webConsole.start(); 32 | coordinatorController.start(); 33 | Runtime.getRuntime().addShutdownHook(new Thread() { 34 | 35 | public void run() { 36 | try { 37 | webConsole.stop(); 38 | controllerService.stop(); 39 | coordinatorController.stop(); 40 | logger.info("## stop the controller service success"); 41 | } catch (Throwable e) { 42 | logger.warn("##something goes wrong when stopping canal Server:\n{}", 43 | ExceptionUtils.getFullStackTrace(e)); 44 | } finally { 45 | logger.info("## canal server is down."); 46 | } 47 | 48 | } 49 | }); 50 | 51 | logger.info("## All Component is ready."); 52 | 53 | } catch (Throwable e) { 54 | logger.error("## Something goes wrong when starting up the canal Server:\n{}", 55 | ExceptionUtils.getFullStackTrace(e)); 56 | System.exit(0); 57 | } 58 | } 59 | 60 | } 61 | -------------------------------------------------------------------------------- /src/main/java/mysql/redis/replicate/Threads.java: -------------------------------------------------------------------------------- 1 | package mysql.redis.replicate; 2 | 3 | import java.util.concurrent.ThreadFactory; 4 | import java.util.concurrent.atomic.AtomicLong; 5 | 6 | /** 7 | * Created by wens on 15-11-5. 8 | */ 9 | public class Threads { 10 | 11 | public static ThreadFactory makeThreadFactory(final String baseName) { 12 | return new ThreadFactory() { 13 | AtomicLong atomicLong = new AtomicLong(0); 14 | 15 | @Override 16 | public Thread newThread(Runnable r) { 17 | return new Thread(r, String.format("%s-%s", baseName, atomicLong.incrementAndGet())); 18 | } 19 | }; 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/main/java/mysql/redis/replicate/Tuple.java: -------------------------------------------------------------------------------- 1 | package mysql.redis.replicate; 2 | 3 | /** 4 | * Created by wens on 15-10-22. 5 | */ 6 | public class Tuple { 7 | 8 | private T1 one; 9 | private T2 two; 10 | 11 | public Tuple(T1 one, T2 two) { 12 | this.one = one; 13 | this.two = two; 14 | } 15 | 16 | public T1 getOne() { 17 | return one; 18 | } 19 | 20 | public void setOne(T1 one) { 21 | this.one = one; 22 | } 23 | 24 | public T2 getTwo() { 25 | return two; 26 | } 27 | 28 | public void setTwo(T2 two) { 29 | this.two = two; 30 | } 31 | 32 | public static Tuple of(T1 v1, T2 v2) { 33 | return new Tuple<>(v1, v2); 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /src/main/java/mysql/redis/replicate/ZkPathUtils.java: -------------------------------------------------------------------------------- 1 | package mysql.redis.replicate; 2 | 3 | import org.I0Itec.zkclient.ZkClient; 4 | 5 | /** 6 | * Created by wens on 15-12-4. 7 | */ 8 | public class ZkPathUtils { 9 | 10 | private static String ROOT; 11 | 12 | public static void init(String root, ZkClient zkClient) { 13 | ROOT = root; 14 | 15 | if (!zkClient.exists(getDestinationConfigPath())) { 16 | zkClient.createPersistent(getDestinationConfigPath(), true); 17 | } 18 | 19 | if (!zkClient.exists(getIdsPath())) { 20 | zkClient.createPersistent(getIdsPath(), true); 21 | } 22 | 23 | if (!zkClient.exists(getDestinationSinkLogOffsetPath())) { 24 | zkClient.createPersistent(getDestinationSinkLogOffsetPath(), true); 25 | } 26 | } 27 | 28 | public static String getDestinationConfigPath() { 29 | return ROOT + "/destinations"; 30 | } 31 | 32 | public static String getDestinationConfigPath(String destination) { 33 | return getDestinationConfigPath() + "/" + destination; 34 | } 35 | 36 | public static String getControllerPath() { 37 | return ROOT + "/controller"; 38 | } 39 | 40 | public static String getIdsPath() { 41 | return ROOT + "/ids"; 42 | } 43 | 44 | public static String getIdsPath(String id) { 45 | return getIdsPath() + "/" + id; 46 | } 47 | 48 | public static String getDestinationSinkLogOffsetPath() { 49 | return ROOT + "/sink"; 50 | } 51 | 52 | public static String getDestinationSinkLogOffsetPath(String destination) { 53 | return getDestinationSinkLogOffsetPath() + "/" + destination; 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/main/java/mysql/redis/replicate/ZookeeperLeaderElector.java: -------------------------------------------------------------------------------- 1 | package mysql.redis.replicate; 2 | 3 | import org.I0Itec.zkclient.IZkDataListener; 4 | import org.I0Itec.zkclient.IZkStateListener; 5 | import org.apache.zookeeper.Watcher; 6 | import org.slf4j.Logger; 7 | 8 | /** 9 | * Created by wens on 15/12/5. 10 | */ 11 | public class ZookeeperLeaderElector { 12 | 13 | private final Logger logger = LoggerFactory.getLogger(); 14 | 15 | private String myId; 16 | 17 | private volatile String leaderId; 18 | 19 | private LeaderListener leaderListener; 20 | 21 | public ZookeeperLeaderElector(String myId, LeaderListener leaderListener) { 22 | this.myId = myId; 23 | this.leaderListener = leaderListener; 24 | } 25 | 26 | public void start() { 27 | ZookeeperUtils.subscribeStateChanges(new ZkStateListener()); 28 | ZookeeperUtils.subscribeDataChanges(ZkPathUtils.getControllerPath(), new ZkDataChangeListener()); 29 | elect(); 30 | 31 | } 32 | 33 | public boolean elect() { 34 | 35 | leaderId = getLeaderId(); 36 | 37 | ControllerInfo controllerInfo = new ControllerInfo(); 38 | controllerInfo.id = myId; 39 | controllerInfo.timestamp = System.currentTimeMillis(); 40 | try { 41 | ZookeeperUtils.createEphemeral(ZkPathUtils.getControllerPath(), controllerInfo); 42 | leaderId = myId; 43 | logger.info("I am leader : {}", myId); 44 | if (leaderListener != null) { 45 | try { 46 | leaderListener.onBecomingLeader(); 47 | } catch (Exception e) { 48 | logger.error("Call leaderListener fail.", e); 49 | } 50 | } 51 | return true; 52 | } catch (Exception e) { 53 | logger.info("Leader is {}", leaderId); 54 | } 55 | 56 | return false; 57 | 58 | } 59 | 60 | private String getLeaderId() { 61 | ControllerInfo controllerInfo = ZookeeperUtils.readData(ZkPathUtils.getControllerPath(), ControllerInfo.class); 62 | if (controllerInfo != null) { 63 | return controllerInfo.id; 64 | } 65 | return null; 66 | } 67 | 68 | public void stop() { 69 | if (myId.equals(leaderId)) { 70 | ZookeeperUtils.delete(ZkPathUtils.getControllerPath()); 71 | } 72 | 73 | } 74 | 75 | private class ZkStateListener implements IZkStateListener { 76 | 77 | public void handleStateChanged(Watcher.Event.KeeperState keeperState) throws Exception { 78 | 79 | } 80 | 81 | public void handleNewSession() throws Exception { 82 | elect(); 83 | 84 | } 85 | } 86 | 87 | private class ZkDataChangeListener implements IZkDataListener { 88 | 89 | public void handleDataChange(String s, Object o) throws Exception { 90 | 91 | } 92 | 93 | public void handleDataDeleted(String s) throws Exception { 94 | elect(); 95 | } 96 | } 97 | 98 | private static class ControllerInfo { 99 | String id; 100 | long timestamp; 101 | 102 | public String getId() { 103 | return id; 104 | } 105 | 106 | public void setId(String id) { 107 | this.id = id; 108 | } 109 | 110 | public long getTimestamp() { 111 | return timestamp; 112 | } 113 | 114 | public void setTimestamp(long timestamp) { 115 | this.timestamp = timestamp; 116 | } 117 | } 118 | 119 | interface LeaderListener { 120 | void onBecomingLeader(); 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /src/main/java/mysql/redis/replicate/ZookeeperUtils.java: -------------------------------------------------------------------------------- 1 | package mysql.redis.replicate; 2 | 3 | import com.alibaba.fastjson.serializer.SerializerFeature; 4 | import com.alibaba.otter.canal.common.utils.JsonUtils; 5 | import org.I0Itec.zkclient.IZkChildListener; 6 | import org.I0Itec.zkclient.IZkDataListener; 7 | import org.I0Itec.zkclient.IZkStateListener; 8 | import org.I0Itec.zkclient.ZkClient; 9 | import org.I0Itec.zkclient.exception.ZkException; 10 | import org.I0Itec.zkclient.serialize.BytesPushThroughSerializer; 11 | 12 | import java.util.List; 13 | 14 | /** 15 | * Created by wens on 15-12-7. 16 | */ 17 | public class ZookeeperUtils { 18 | 19 | private volatile static ZkClient zkClient; 20 | 21 | public static void init(Conf conf) { 22 | zkClient = openClient(conf.getZookeeperServer()); 23 | ZkPathUtils.init(conf.getZookeeperRootPath(), zkClient); 24 | } 25 | 26 | public static ZkClient openClient(String zkServer) { 27 | return new ZkClient(zkServer, 10000, 10000, new BytesPushThroughSerializer()); 28 | } 29 | 30 | public static List getChildren(String path) { 31 | return zkClient.getChildren(path); 32 | } 33 | 34 | 35 | public static T readData(String path, Class dataClass) { 36 | byte[] data = zkClient.readData(path, true); 37 | if (data != null && data.length != 0) { 38 | return JsonUtils.unmarshalFromByte(data, dataClass); 39 | } 40 | return null; 41 | } 42 | 43 | public static void writeData(String path, Object object, boolean createPathIfNotExist) { 44 | try { 45 | zkClient.writeData(path, JsonUtils.marshalToByte(object, SerializerFeature.WriteClassName)); 46 | } catch (ZkException e) { 47 | if (createPathIfNotExist) { 48 | createIfNotExist(path); 49 | zkClient.writeData(path, JsonUtils.marshalToByte(object, SerializerFeature.WriteClassName)); 50 | } else { 51 | throw e; 52 | } 53 | } 54 | } 55 | 56 | public static void delete(String path) { 57 | zkClient.delete(path); 58 | } 59 | 60 | public static void createIfNotExist(String path) { 61 | try { 62 | zkClient.createPersistent(path, true); 63 | } catch (Exception e) { 64 | // 65 | } 66 | } 67 | 68 | public static void createEphemeral(String path, Object object) { 69 | zkClient.createEphemeral(path, JsonUtils.marshalToByte(object)); 70 | } 71 | 72 | public static void subscribeChildChanges(String path, IZkChildListener listener) { 73 | zkClient.subscribeChildChanges(path, listener); 74 | } 75 | 76 | public static void subscribeStateChanges(IZkStateListener listener) { 77 | zkClient.subscribeStateChanges(listener); 78 | } 79 | 80 | public static void subscribeDataChanges(String path, IZkDataListener listener) { 81 | zkClient.subscribeDataChanges(path, listener); 82 | } 83 | 84 | public static void deleteRecursive(String path) { 85 | zkClient.deleteRecursive(path); 86 | } 87 | 88 | 89 | } 90 | -------------------------------------------------------------------------------- /src/main/java/mysql/redis/replicate/canal/AbstractSink.java: -------------------------------------------------------------------------------- 1 | package mysql.redis.replicate.canal; 2 | 3 | import com.alibaba.otter.canal.protocol.CanalEntry; 4 | import com.alibaba.otter.canal.protocol.Message; 5 | import com.google.common.collect.Maps; 6 | import mysql.redis.replicate.Lifecycle; 7 | import mysql.redis.replicate.LoggerFactory; 8 | import mysql.redis.replicate.Monitor; 9 | import mysql.redis.replicate.config.DestinationConfig; 10 | import org.slf4j.Logger; 11 | 12 | import java.util.List; 13 | import java.util.Map; 14 | import java.util.concurrent.BlockingQueue; 15 | import java.util.concurrent.LinkedBlockingQueue; 16 | import java.util.concurrent.TimeUnit; 17 | 18 | /** 19 | * Created by wens on 15-12-4. 20 | */ 21 | public abstract class AbstractSink implements MessageSink, Lifecycle { 22 | 23 | private final static Logger logger = LoggerFactory.getLogger(); 24 | 25 | protected DestinationConfig destinationConfig; 26 | private Map sinkWorkerMap; 27 | 28 | public AbstractSink(DestinationConfig destinationConfig) { 29 | this.destinationConfig = destinationConfig; 30 | this.sinkWorkerMap = Maps.newHashMap(); 31 | for (DestinationConfig.TableConfig tableConfig : destinationConfig.getTableConfigs()) { 32 | SinkWorker sinkWorker = createSinkWorker(tableConfig); 33 | sinkWorkerMap.put(tableConfig.getTableName(), sinkWorker); 34 | new Thread(sinkWorker, tableConfig.getTableName() + "-sink-worker-thread").start(); 35 | } 36 | } 37 | 38 | protected abstract SinkWorker createSinkWorker(DestinationConfig.TableConfig tableConfig); 39 | 40 | @Override 41 | public void sink(Message message) { 42 | 43 | for (CanalEntry.Entry entry : message.getEntries()) { 44 | 45 | if (entry.getEntryType() == CanalEntry.EntryType.ROWDATA) { 46 | String tableName = String.format("%s.%s", entry.getHeader().getSchemaName(), entry.getHeader().getTableName()); 47 | DestinationConfig.TableConfig tableConfig = null; 48 | for (DestinationConfig.TableConfig t : destinationConfig.getTableConfigs()) { 49 | if (t.getTableName().equals(tableName)) { 50 | tableConfig = t; 51 | break; 52 | } 53 | } 54 | if (tableConfig == null) { 55 | continue; 56 | } 57 | CanalEntry.RowChange rowChange = null; 58 | try { 59 | rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue()); 60 | } catch (Exception e) { 61 | throw new RuntimeException("parse event has an error , data:" + entry.toString(), e); 62 | } 63 | sinkWorkerMap.get(tableConfig.getTableName()).push(new RowChangeWrapper(rowChange ,message.getId() )); 64 | } 65 | Monitor.updateLogPosition(destinationConfig.getDestination() , entry.getHeader().getLogfileName() , entry.getHeader().getLogfileOffset() ); 66 | } 67 | 68 | } 69 | 70 | @Override 71 | public void stop() { 72 | for (SinkWorker sinkWorker : sinkWorkerMap.values()) { 73 | sinkWorker.stop(); 74 | } 75 | } 76 | 77 | protected abstract class SinkWorker implements Runnable, Lifecycle { 78 | 79 | protected volatile boolean stopped = false; 80 | protected volatile boolean running = true; 81 | 82 | protected volatile long commitBatchId = Long.MAX_VALUE ; 83 | 84 | protected DestinationConfig.TableConfig tableConfig; 85 | 86 | protected BlockingQueue rowChangeQueue; 87 | 88 | protected SinkWorker(DestinationConfig.TableConfig tableConfig) { 89 | this.tableConfig = tableConfig; 90 | this.rowChangeQueue = new LinkedBlockingQueue<>(10000) ; 91 | } 92 | 93 | public void push(RowChangeWrapper rowChange) { 94 | try { 95 | rowChangeQueue.put(rowChange); 96 | } catch (InterruptedException e) { 97 | Thread.currentThread().interrupt(); 98 | throw new RuntimeException(e); 99 | } 100 | } 101 | 102 | public void run() { 103 | 104 | while (!stopped) { 105 | try{ 106 | doRun(); 107 | }catch (Exception e){ 108 | logger.error("doRun fail",e); 109 | } 110 | } 111 | 112 | running = false ; 113 | } 114 | 115 | private void doRun() { 116 | RowChangeWrapper rowChange = null; 117 | try { 118 | rowChange = rowChangeQueue.poll(100, TimeUnit.MILLISECONDS); 119 | } catch (InterruptedException e) { 120 | Thread.currentThread().interrupt(); 121 | } 122 | if (rowChange != null) { 123 | CanalEntry.EventType eventType = rowChange.rowChange.getEventType(); 124 | int retry = 0 ; 125 | while (retry <= 5 ){ 126 | try{ 127 | doSink(rowChange.rowChange, eventType); 128 | break; 129 | }catch (Exception e){ 130 | logger.error("doSink fail : retry = "+retry+",tableName=" + tableConfig.getTableName() , e ); 131 | retry++; 132 | try { 133 | Thread.sleep(retry * 100 ); 134 | } catch (InterruptedException e1) { 135 | Thread.currentThread().interrupt(); 136 | } 137 | } 138 | } 139 | commitBatchId = rowChange.batchId ; 140 | }else{ 141 | commitBatchId = Long.MAX_VALUE ; 142 | } 143 | } 144 | 145 | private void doSink(CanalEntry.RowChange rowChange, CanalEntry.EventType eventType) { 146 | if (eventType == CanalEntry.EventType.INSERT) { 147 | handleInsert(rowChange.getRowDatasList()); 148 | Monitor.incrInsertCount(destinationConfig.getDestination(), rowChange.getRowDatasCount()); 149 | }else if (eventType == CanalEntry.EventType.UPDATE) { 150 | handleUpdate(rowChange.getRowDatasList()); 151 | Monitor.incrUpdateCount(destinationConfig.getDestination(), rowChange.getRowDatasCount()); 152 | }else if (eventType == CanalEntry.EventType.DELETE) { 153 | handleDelete(rowChange.getRowDatasList()); 154 | Monitor.incrDeleteCount(destinationConfig.getDestination(), rowChange.getRowDatasCount()); 155 | } 156 | } 157 | 158 | protected abstract void handleInsert(List rowDatasList); 159 | 160 | protected abstract void handleUpdate(List rowDatasList); 161 | 162 | protected abstract void handleDelete(List rowDatasList); 163 | 164 | 165 | protected String toString(List columns) { 166 | StringBuilder builder = new StringBuilder(); 167 | for (CanalEntry.Column column : columns) { 168 | 169 | builder.append(column.getName() + " : " + column.getValue()); 170 | builder.append(" type=" + column.getMysqlType()); 171 | if (column.getUpdated()) { 172 | builder.append(" update=" + column.getUpdated()); 173 | } 174 | builder.append("\n"); 175 | } 176 | return builder.toString(); 177 | } 178 | 179 | 180 | @Override 181 | public void start() { 182 | 183 | } 184 | 185 | @Override 186 | public void stop() { 187 | stopped = true; 188 | while (running ){ 189 | try { 190 | Thread.sleep(50); 191 | } catch (InterruptedException e) { 192 | Thread.currentThread().interrupt(); 193 | } 194 | } 195 | } 196 | 197 | public long getCommitBatchId() { 198 | return commitBatchId; 199 | } 200 | } 201 | 202 | protected long getCommitBatchId() { 203 | 204 | long min = Long.MAX_VALUE; 205 | 206 | for (SinkWorker sinkWorker : sinkWorkerMap.values()) { 207 | min = Math.min(min, sinkWorker.getCommitBatchId()); 208 | } 209 | return min; 210 | } 211 | 212 | static class RowChangeWrapper { 213 | CanalEntry.RowChange rowChange ; 214 | long batchId ; 215 | 216 | public RowChangeWrapper(CanalEntry.RowChange rowChange, long batchId) { 217 | this.rowChange = rowChange; 218 | this.batchId = batchId; 219 | } 220 | } 221 | } 222 | -------------------------------------------------------------------------------- /src/main/java/mysql/redis/replicate/canal/ControllerService.java: -------------------------------------------------------------------------------- 1 | package mysql.redis.replicate.canal; 2 | 3 | import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils; 4 | import com.alibaba.otter.canal.instance.core.CanalInstance; 5 | import com.alibaba.otter.canal.instance.core.CanalInstanceGenerator; 6 | import com.alibaba.otter.canal.instance.manager.CanalInstanceWithManager; 7 | import com.alibaba.otter.canal.instance.manager.model.Canal; 8 | import com.alibaba.otter.canal.instance.manager.model.CanalParameter; 9 | import com.alibaba.otter.canal.server.embedded.CanalServerWithEmbedded; 10 | import mysql.redis.replicate.Conf; 11 | import mysql.redis.replicate.LoggerFactory; 12 | import mysql.redis.replicate.ZookeeperUtils; 13 | import mysql.redis.replicate.config.DestinationConfig; 14 | import mysql.redis.replicate.config.DestinationConfigManager; 15 | import mysql.redis.replicate.redis.RedisSink; 16 | import org.slf4j.Logger; 17 | 18 | import java.io.IOException; 19 | import java.net.InetSocketAddress; 20 | import java.util.ArrayList; 21 | import java.util.Arrays; 22 | import java.util.List; 23 | import java.util.concurrent.ConcurrentHashMap; 24 | import java.util.concurrent.Executors; 25 | import java.util.concurrent.ScheduledExecutorService; 26 | 27 | public class ControllerService { 28 | 29 | private final Logger logger = LoggerFactory.getLogger(); 30 | 31 | private final ConcurrentHashMap runningTasks = new ConcurrentHashMap<>(); 32 | private DestinationConfigManager destinationConfigManager; 33 | 34 | private final Conf conf; 35 | 36 | private CanalServerWithEmbedded server; 37 | 38 | private ScheduledExecutorService scheduledExecutorService ; 39 | 40 | public ControllerService(Conf conf) throws IOException { 41 | this.conf = conf; 42 | this.scheduledExecutorService = Executors.newScheduledThreadPool(Runtime.getRuntime().availableProcessors()); 43 | } 44 | 45 | public void setDestinationConfigManager(DestinationConfigManager destinationConfigManager) { 46 | this.destinationConfigManager = destinationConfigManager; 47 | } 48 | 49 | public void start() { 50 | server = new CanalServerWithEmbedded(); 51 | server.setCanalInstanceGenerator(new CanalInstanceGenerator() { 52 | 53 | public CanalInstance generate(String destination) { 54 | return new CanalInstanceWithManager(buildCanal(destination), buildFilterTable(destination)); 55 | } 56 | 57 | 58 | }); 59 | server.start(); 60 | } 61 | 62 | public void stopTask(String destination) { 63 | MessagePuller task = runningTasks.get(destination); 64 | if (task != null) { 65 | task.safeStop(); 66 | runningTasks.remove(destination); 67 | } 68 | 69 | if (server.isStart(destination)) { 70 | server.stop(destination); 71 | } 72 | logger.info("## Stopped destination task:" + destination); 73 | } 74 | 75 | public void startTask(String destination) { 76 | DestinationConfig destinationConfig = getDestinationConfig(destination); 77 | MessagePuller task = runningTasks.get(destination); 78 | if (task != null) { 79 | task.safeStop(); 80 | } 81 | server.start(destination); 82 | MessagePuller puller = new MessagePuller(conf.getCanalBatchSize(), destination, server, new RedisSink(destinationConfig , conf.getJedisPoolConfig() ) , scheduledExecutorService ); 83 | puller.start(); 84 | runningTasks.put(destination, puller); 85 | logger.info("## Started destination task:" + destinationConfig.getDestination()); 86 | } 87 | 88 | 89 | public void stop() { 90 | for (String dest : runningTasks.keySet()) { 91 | stopTask(dest); 92 | } 93 | server.stop(); 94 | scheduledExecutorService.shutdownNow(); 95 | } 96 | 97 | 98 | private String buildFilterTable(String destination) { 99 | DestinationConfig destinationConfig = getDestinationConfig(destination); 100 | if (destinationConfig.getTableConfigs() == null || destinationConfig.getTableConfigs().size() == 0) { 101 | return "not-exist-table"; 102 | } 103 | 104 | StringBuilder sb = new StringBuilder(100 * destinationConfig.getTableConfigs().size()); 105 | 106 | for (DestinationConfig.TableConfig tableConfig : destinationConfig.getTableConfigs()) { 107 | sb.append(tableConfig.getTableName().replaceFirst("\\.", "\\\\.")); 108 | sb.append(","); 109 | } 110 | sb.append("not-exist-table"); 111 | return sb.toString(); 112 | } 113 | 114 | 115 | private Canal buildCanal(String destination) { 116 | DestinationConfig destinationConfig = getDestinationConfig(destination); 117 | 118 | recovery(destinationConfig); 119 | 120 | Canal canal = new Canal(); 121 | canal.setId(1L); 122 | canal.setName(destination); 123 | 124 | CanalParameter parameter = new CanalParameter(); 125 | 126 | parameter.setZkClusters(Arrays.asList(conf.getZookeeperServer().split(","))); 127 | 128 | parameter.setMetaMode(CanalParameter.MetaMode.ZOOKEEPER); 129 | parameter.setHaMode(CanalParameter.HAMode.HEARTBEAT); 130 | parameter.setIndexMode(CanalParameter.IndexMode.MEMORY_META_FAILBACK); 131 | 132 | parameter.setStorageMode(CanalParameter.StorageMode.MEMORY); 133 | parameter.setMemoryStorageBufferSize(32 * 1024); 134 | 135 | parameter.setSourcingType(CanalParameter.SourcingType.MYSQL); 136 | 137 | String[] dbAddresses = destinationConfig.getDbAddress().split(","); 138 | 139 | List dbAddressList = new ArrayList<>(dbAddresses.length); 140 | 141 | for (String address : dbAddresses) { 142 | String[] strings = address.split(":"); 143 | String ip = strings[0]; 144 | int port = 3306; 145 | if (strings.length == 2) { 146 | port = Integer.parseInt(strings[1]); 147 | } 148 | dbAddressList.add(new InetSocketAddress(ip, port)); 149 | } 150 | 151 | parameter.setDbAddresses(dbAddressList); 152 | parameter.setDbUsername(destinationConfig.getDbUser()); 153 | parameter.setDbPassword(destinationConfig.getDbPassword()); 154 | 155 | parameter.setSlaveId(1688L); 156 | 157 | parameter.setDefaultConnectionTimeoutInSeconds(30); 158 | parameter.setConnectionCharset("UTF-8"); 159 | parameter.setConnectionCharsetNumber((byte) 33); 160 | parameter.setReceiveBufferSize(8 * 1024); 161 | parameter.setSendBufferSize(8 * 1024); 162 | 163 | parameter.setDetectingEnable(false); 164 | 165 | 166 | canal.setCanalParameter(parameter); 167 | return canal; 168 | } 169 | 170 | private void recovery(DestinationConfig destinationConfig) { 171 | ZookeeperUtils.deleteRecursive(ZookeeperPathUtils.getBatchMarkPath(destinationConfig.getDestination(), (short) 1)); 172 | } 173 | 174 | public DestinationConfig getDestinationConfig(String destination) { 175 | return destinationConfigManager.getDestinationConfig(destination); 176 | } 177 | 178 | 179 | } 180 | -------------------------------------------------------------------------------- /src/main/java/mysql/redis/replicate/canal/MessagePuller.java: -------------------------------------------------------------------------------- 1 | package mysql.redis.replicate.canal; 2 | 3 | import com.alibaba.otter.canal.protocol.ClientIdentity; 4 | import com.alibaba.otter.canal.protocol.Message; 5 | import mysql.redis.replicate.Lifecycle; 6 | import mysql.redis.replicate.ZkPathUtils; 7 | import mysql.redis.replicate.ZookeeperUtils; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | import java.util.Iterator; 12 | import java.util.NavigableSet; 13 | import java.util.concurrent.*; 14 | 15 | /** 16 | * Created by wens on 15-10-15. 17 | */ 18 | public class MessagePuller extends Thread implements Lifecycle { 19 | 20 | private final static Logger logger = LoggerFactory.getLogger(ControllerService.class); 21 | 22 | private volatile boolean running = true; 23 | private volatile boolean stopped = false; 24 | 25 | private final String destination; 26 | 27 | private final com.alibaba.otter.canal.server.CanalService canalService; 28 | 29 | private final int batchSize; 30 | 31 | private final AbstractSink writer; 32 | 33 | private final ConcurrentSkipListSet commitBatchIds = new ConcurrentSkipListSet() ; 34 | 35 | private final ClientIdentity clientIdentity ; 36 | 37 | public MessagePuller(final int batchSize, final String destination, final com.alibaba.otter.canal.server.CanalService canalService, final AbstractSink writer , ScheduledExecutorService scheduledExecutorService ) { 38 | this.destination = destination; 39 | this.canalService = canalService; 40 | this.batchSize = batchSize; 41 | this.writer = writer; 42 | this.clientIdentity = new ClientIdentity(destination, (short) 1) ; 43 | 44 | setName("puller-" + this.destination + "-thread"); 45 | } 46 | 47 | 48 | public void safeStop() { 49 | 50 | if (stopped) { 51 | return; 52 | } 53 | 54 | running = false; 55 | while (!stopped) { 56 | try { 57 | Thread.sleep(10); 58 | } catch (InterruptedException e) { 59 | Thread.currentThread().interrupt(); 60 | } 61 | } 62 | writer.stop(); 63 | } 64 | 65 | 66 | @Override 67 | public void run() { 68 | 69 | try { 70 | canalService.subscribe(clientIdentity); 71 | while (running) { 72 | 73 | long commitBatchId = writer.getCommitBatchId(); 74 | NavigableSet batchIds = commitBatchIds.headSet(commitBatchId, false); 75 | Iterator iterator = batchIds.iterator(); 76 | while ( iterator.hasNext() ){ 77 | Long batchId = iterator.next(); 78 | canalService.ack(clientIdentity , batchId); 79 | commitBatchIds.remove(batchId); 80 | } 81 | 82 | Message message = canalService.getWithoutAck(clientIdentity, this.batchSize, 100L, TimeUnit.MICROSECONDS); // 获取指定数量的数据 83 | 84 | long batchId = message.getId(); 85 | int size = message.getEntries().size(); 86 | if (batchId == -1 || size == 0) { 87 | continue; 88 | } else { 89 | try { 90 | writer.sink(message); 91 | delayAck(batchId) ; 92 | } catch (Exception e) { 93 | logger.error("Got an Exception, when sink message.", e); 94 | canalService.rollback(clientIdentity, batchId); 95 | 96 | } 97 | } 98 | } 99 | } finally { 100 | stopped = true; 101 | } 102 | } 103 | 104 | private void delayAck(long batchId) { 105 | commitBatchIds.add(batchId); 106 | } 107 | 108 | 109 | } 110 | -------------------------------------------------------------------------------- /src/main/java/mysql/redis/replicate/canal/MessageSink.java: -------------------------------------------------------------------------------- 1 | package mysql.redis.replicate.canal; 2 | 3 | import com.alibaba.otter.canal.protocol.Message; 4 | 5 | /** 6 | * Created by wens on 15-10-20. 7 | */ 8 | public interface MessageSink { 9 | void sink(Message message); 10 | } 11 | -------------------------------------------------------------------------------- /src/main/java/mysql/redis/replicate/config/DestinationConfig.java: -------------------------------------------------------------------------------- 1 | package mysql.redis.replicate.config; 2 | 3 | import org.apache.commons.lang.StringUtils; 4 | 5 | import java.io.Serializable; 6 | import java.util.List; 7 | 8 | /** 9 | * Created by wens on 15-10-14. 10 | */ 11 | public class DestinationConfig implements Serializable { 12 | 13 | private String destination; 14 | 15 | private String dbAddress; 16 | 17 | private String dbUser; 18 | 19 | private String dbPassword; 20 | 21 | 22 | private String runOn; 23 | 24 | private boolean runFail; 25 | 26 | private List tableConfigs; 27 | 28 | private boolean stopped = true; 29 | 30 | 31 | public List getTableConfigs() { 32 | return tableConfigs; 33 | } 34 | 35 | public String getDestination() { 36 | return destination; 37 | } 38 | 39 | public void setDestination(String destination) { 40 | this.destination = destination; 41 | } 42 | 43 | public String getDbAddress() { 44 | return dbAddress; 45 | } 46 | 47 | public void setDbAddress(String dbAddress) { 48 | this.dbAddress = dbAddress; 49 | } 50 | 51 | public String getDbUser() { 52 | return dbUser; 53 | } 54 | 55 | public void setDbUser(String dbUser) { 56 | this.dbUser = dbUser; 57 | } 58 | 59 | public String getDbPassword() { 60 | return dbPassword; 61 | } 62 | 63 | public void setDbPassword(String dbPassword) { 64 | this.dbPassword = dbPassword; 65 | } 66 | 67 | public boolean isStopped() { 68 | return stopped; 69 | } 70 | 71 | public void setStopped(boolean stopped) { 72 | this.stopped = stopped; 73 | } 74 | 75 | public String getRunOn() { 76 | return runOn; 77 | } 78 | 79 | public void setRunOn(String runOn) { 80 | this.runOn = runOn; 81 | } 82 | 83 | public boolean isRunFail() { 84 | return runFail; 85 | } 86 | 87 | public void setRunFail(boolean runFail) { 88 | this.runFail = runFail; 89 | } 90 | 91 | public static class TableConfig { 92 | 93 | private String tableName; 94 | 95 | private String script; 96 | 97 | private String redisInfos; 98 | 99 | private String redisPassword; 100 | 101 | public String getTableName() { 102 | return tableName; 103 | } 104 | 105 | public void setTableName(String tableName) { 106 | this.tableName = tableName; 107 | } 108 | 109 | public String getScript() { 110 | return script; 111 | } 112 | 113 | public void setScript(String script) { 114 | this.script = script; 115 | } 116 | 117 | public String getRedisInfos() { 118 | return redisInfos; 119 | } 120 | 121 | public void setRedisInfos(String redisInfos) { 122 | this.redisInfos = redisInfos; 123 | } 124 | 125 | public String getRedisPassword() { 126 | return redisPassword; 127 | } 128 | 129 | public void setRedisPassword(String redisPassword) { 130 | this.redisPassword = redisPassword; 131 | } 132 | 133 | public String getPureTable() { 134 | return this.getTableName().substring(this.getTableName().indexOf(".") + 1); 135 | } 136 | } 137 | 138 | 139 | public void validate() { 140 | 141 | if (StringUtils.isEmpty(this.dbAddress)) { 142 | throw new IllegalArgumentException("Require `dbAddress`"); 143 | } 144 | 145 | if (tableConfigs == null || tableConfigs.size() == 0) { 146 | throw new IllegalArgumentException("Require `tableConfigs`"); 147 | } 148 | 149 | for (TableConfig tc : tableConfigs) { 150 | 151 | if (StringUtils.isEmpty(tc.getTableName())) { 152 | throw new IllegalArgumentException("Table config require `tableName`"); 153 | } 154 | 155 | if (StringUtils.isEmpty(tc.getScript())) { 156 | throw new IllegalArgumentException("Table config require `script`"); 157 | } 158 | 159 | if (StringUtils.isEmpty(tc.getRedisInfos())) { 160 | throw new IllegalArgumentException("Table config require `script`"); 161 | } 162 | 163 | 164 | } 165 | 166 | } 167 | 168 | public void setTableConfigs(List tableConfigs) { 169 | this.tableConfigs = tableConfigs; 170 | } 171 | } 172 | -------------------------------------------------------------------------------- /src/main/java/mysql/redis/replicate/config/DestinationConfigManager.java: -------------------------------------------------------------------------------- 1 | package mysql.redis.replicate.config; 2 | 3 | import com.google.common.collect.Sets; 4 | import mysql.redis.replicate.ZkPathUtils; 5 | import mysql.redis.replicate.ZookeeperUtils; 6 | 7 | import java.util.ArrayList; 8 | import java.util.List; 9 | import java.util.Set; 10 | 11 | /** 12 | * Created by wens on 15-10-14. 13 | */ 14 | public class DestinationConfigManager { 15 | 16 | 17 | public Set getAllDestination() { 18 | String path = ZkPathUtils.getDestinationConfigPath(); 19 | return Sets.newHashSet(ZookeeperUtils.getChildren(path)); 20 | } 21 | 22 | public List getAllDestinationConfig() { 23 | Set destinations = getAllDestination(); 24 | List destinationConfigs = new ArrayList<>(destinations.size()); 25 | for (String destination : destinations) { 26 | DestinationConfig destinationConfig = getDestinationConfig(destination); 27 | if (destinationConfig != null) { 28 | destinationConfigs.add(destinationConfig); 29 | } 30 | } 31 | return destinationConfigs; 32 | } 33 | 34 | public DestinationConfig getDestinationConfig(String destination) { 35 | return ZookeeperUtils.readData(ZkPathUtils.getDestinationConfigPath(destination), DestinationConfig.class); 36 | } 37 | 38 | public void saveOrUpdate(DestinationConfig destinationConfig) { 39 | String path = ZkPathUtils.getDestinationConfigPath(destinationConfig.getDestination()); 40 | ZookeeperUtils.writeData(path, destinationConfig, true); 41 | } 42 | 43 | 44 | public void delete(String destination) { 45 | String path = ZkPathUtils.getDestinationConfigPath(destination); 46 | ZookeeperUtils.delete(path); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/main/java/mysql/redis/replicate/groovy/IRow2map.java: -------------------------------------------------------------------------------- 1 | package mysql.redis.replicate.groovy; 2 | 3 | import java.util.Map; 4 | 5 | /** 6 | * Created by wens on 15-11-13. 7 | */ 8 | public interface IRow2map { 9 | void convert(Map type, Map row, Map map); 10 | 11 | } 12 | -------------------------------------------------------------------------------- /src/main/java/mysql/redis/replicate/groovy/Row2map.java: -------------------------------------------------------------------------------- 1 | package mysql.redis.replicate.groovy; 2 | 3 | import groovy.lang.GroovyClassLoader; 4 | 5 | import java.util.Map; 6 | 7 | /** 8 | * Created by wens on 15-11-13. 9 | */ 10 | public class Row2map implements IRow2map { 11 | 12 | private IRow2map delegate; 13 | 14 | public Row2map(String source) { 15 | 16 | GroovyClassLoader groovyCl = new GroovyClassLoader(); 17 | 18 | Class aClass = groovyCl.parseClass(source); 19 | 20 | try { 21 | delegate = (IRow2map) aClass.newInstance(); 22 | } catch (InstantiationException e) { 23 | throw new RuntimeException(e); 24 | } catch (IllegalAccessException e) { 25 | throw new RuntimeException(e); 26 | } 27 | 28 | } 29 | 30 | 31 | @Override 32 | public void convert(Map type, Map row, Map map) { 33 | delegate.convert(type, row, map); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/main/java/mysql/redis/replicate/groovy/Row2mapManager.java: -------------------------------------------------------------------------------- 1 | package mysql.redis.replicate.groovy; 2 | 3 | import com.google.common.collect.Maps; 4 | import com.google.common.io.Files; 5 | import mysql.redis.replicate.config.DestinationConfig; 6 | 7 | import java.io.File; 8 | import java.io.IOException; 9 | import java.nio.charset.StandardCharsets; 10 | import java.util.Map; 11 | 12 | /** 13 | * Created by wens on 15-11-13. 14 | */ 15 | public class Row2mapManager { 16 | 17 | public final static String SCRIPT_TEMPLATE_FILE = "row2map.groovy.tpl"; 18 | 19 | private DestinationConfig destinationConfig; 20 | 21 | private Map row2mapMap = Maps.newHashMap(); 22 | 23 | public Row2mapManager(DestinationConfig destinationConfig) { 24 | this.destinationConfig = destinationConfig; 25 | init(); 26 | } 27 | 28 | private void init() { 29 | 30 | String template; 31 | try { 32 | template = Files.toString(new File(Thread.currentThread().getContextClassLoader().getResource(SCRIPT_TEMPLATE_FILE).getPath()), StandardCharsets.UTF_8); 33 | } catch (IOException e) { 34 | throw new RuntimeException(e); 35 | } 36 | 37 | for (DestinationConfig.TableConfig tableConfig : destinationConfig.getTableConfigs()) { 38 | row2mapMap.put(tableConfig.getTableName(), new Row2map(template.replace("#script#", tableConfig.getScript()))); 39 | } 40 | } 41 | 42 | public IRow2map get(String tableName) { 43 | return row2mapMap.get(tableName); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/main/java/mysql/redis/replicate/redis/RedisSink.java: -------------------------------------------------------------------------------- 1 | package mysql.redis.replicate.redis; 2 | 3 | import com.alibaba.otter.canal.common.utils.JsonUtils; 4 | import com.alibaba.otter.canal.protocol.CanalEntry; 5 | import com.google.common.collect.Maps; 6 | import mysql.redis.replicate.LoggerFactory; 7 | import mysql.redis.replicate.Threads; 8 | import mysql.redis.replicate.canal.AbstractSink; 9 | import mysql.redis.replicate.config.DestinationConfig; 10 | import mysql.redis.replicate.groovy.IRow2map; 11 | import mysql.redis.replicate.groovy.Row2mapManager; 12 | import org.apache.commons.pool2.impl.GenericObjectPoolConfig; 13 | import org.slf4j.Logger; 14 | import redis.clients.jedis.Jedis; 15 | import redis.clients.jedis.Pipeline; 16 | import redis.clients.jedis.ShardedJedis; 17 | import redis.clients.jedis.ShardedJedisPool; 18 | import redis.clients.jedis.exceptions.JedisException; 19 | 20 | import java.util.List; 21 | import java.util.Map; 22 | import java.util.concurrent.Callable; 23 | import java.util.concurrent.ExecutorService; 24 | import java.util.concurrent.Executors; 25 | import java.util.concurrent.TimeUnit; 26 | 27 | /** 28 | * Created by wens on 15-12-3. 29 | */ 30 | public class RedisSink extends AbstractSink { 31 | 32 | private final static Logger logger = LoggerFactory.getLogger(); 33 | 34 | private final Row2mapManager row2mapManager; 35 | 36 | private final GenericObjectPoolConfig redisConfig; 37 | 38 | public RedisSink(DestinationConfig destinationConfig, GenericObjectPoolConfig redisConfig) { 39 | super(destinationConfig); 40 | this.row2mapManager = new Row2mapManager(destinationConfig); 41 | this.redisConfig = redisConfig; 42 | } 43 | 44 | @Override 45 | public void start() { 46 | 47 | } 48 | 49 | @Override 50 | protected AbstractSink.SinkWorker createSinkWorker(DestinationConfig.TableConfig tableConfig) { 51 | return new AbstractSink.SinkWorker(tableConfig) { 52 | private final ShardedJedisPool shardedJedisPool = RedisUtils.createSharedJedisPool(redisConfig, tableConfig.getRedisInfos(), tableConfig.getRedisPassword()); 53 | 54 | private final ExecutorService executorService = Executors.newCachedThreadPool(Threads.makeThreadFactory("redis-sync-thread")); 55 | 56 | @Override 57 | protected void handleInsert(List rowDatasList) { 58 | 59 | ShardedJedis shardedJedis = shardedJedisPool.getResource(); 60 | boolean broken = false; 61 | try { 62 | Map jedisPipelineMap = Maps.newHashMap(); 63 | for (CanalEntry.RowData rowData : rowDatasList) { 64 | List afterColumnsList = rowData.getAfterColumnsList(); 65 | 66 | if (logger.isDebugEnabled()) { 67 | logger.debug("{} receive insert data:\n{}", tableConfig.getTableName(), toString(afterColumnsList)); 68 | } 69 | StringBuilder id = new StringBuilder(); 70 | Map row = Maps.newHashMap(); 71 | Map rowType = Maps.newHashMap(); 72 | for (CanalEntry.Column c : afterColumnsList) { 73 | if (c.getIsKey()) { 74 | id.append(c.getValue()); 75 | } 76 | row.put(c.getName(), c.getValue()); 77 | rowType.put(c.getName(), c.getMysqlType().toUpperCase()); 78 | } 79 | IRow2map row2document = row2mapManager.get(tableConfig.getTableName()); 80 | Map map = Maps.newHashMap(); 81 | row2document.convert(rowType, row, map); 82 | 83 | if (map.isEmpty()) { 84 | return; 85 | } 86 | String key = String.format("%s_%s", tableConfig.getPureTable(), id.toString()); 87 | Jedis jedis = shardedJedis.getShard(key); 88 | PipelineSyncTask pilelineSyncTask = jedisPipelineMap.get(jedis); 89 | if (pilelineSyncTask == null) { 90 | pilelineSyncTask = new PipelineSyncTask(jedis.pipelined()); 91 | jedisPipelineMap.put(jedis, pilelineSyncTask); 92 | } 93 | for (String column : map.keySet()) { 94 | pilelineSyncTask.pipeline.hset(key, column, map.get(column)); 95 | } 96 | //pilelineSyncTask.pipeline.set(key, JsonUtils.marshalToString(map)); 97 | } 98 | executorService.invokeAll(jedisPipelineMap.values(), 1, TimeUnit.MINUTES); 99 | } catch (JedisException e) { 100 | broken = true; 101 | throw e; 102 | } catch (Exception e) { 103 | throw new RuntimeException(e); 104 | } finally { 105 | RedisUtils.returnResource(shardedJedisPool, shardedJedis, broken); 106 | } 107 | 108 | 109 | } 110 | 111 | 112 | @Override 113 | protected void handleUpdate(List rowDatasList) { 114 | 115 | ShardedJedis shardedJedis = shardedJedisPool.getResource(); 116 | boolean broken = false; 117 | try { 118 | Map jedisPipelineMap = Maps.newHashMap(); 119 | 120 | for (CanalEntry.RowData rowData : rowDatasList) { 121 | 122 | List afterColumnsList = rowData.getAfterColumnsList(); 123 | if (logger.isDebugEnabled()) { 124 | logger.debug("{} receive update data:\n{}", tableConfig.getTableName(), toString(afterColumnsList)); 125 | } 126 | boolean drop = true; 127 | StringBuilder id = new StringBuilder(); 128 | Map row = Maps.newHashMap(); 129 | Map rowType = Maps.newHashMap(); 130 | for (CanalEntry.Column c : afterColumnsList) { 131 | if (c.getIsKey()) { 132 | id.append(c.getValue()); 133 | } 134 | 135 | if (c.getUpdated()) { 136 | drop = false; 137 | } 138 | 139 | row.put(c.getName(), c.getValue()); 140 | rowType.put(c.getName(), c.getMysqlType().toUpperCase()); 141 | } 142 | if (!drop) { 143 | IRow2map row2document = row2mapManager.get(tableConfig.getTableName()); 144 | Map map = Maps.newHashMap(); 145 | row2document.convert(rowType, row, map); 146 | if (map.isEmpty()) { 147 | return; 148 | } 149 | String key = String.format("%s_%s", tableConfig.getPureTable(), id.toString()); 150 | 151 | Jedis jedis = shardedJedis.getShard(key); 152 | PipelineSyncTask pilelineSyncTask = jedisPipelineMap.get(jedis); 153 | if (pilelineSyncTask == null) { 154 | pilelineSyncTask = new PipelineSyncTask(jedis.pipelined()); 155 | jedisPipelineMap.put(jedis, pilelineSyncTask); 156 | } 157 | for (String column : map.keySet()) { 158 | pilelineSyncTask.pipeline.hset(key, column, map.get(column)); 159 | } 160 | //pilelineSyncTask.pipeline.set(key, JsonUtils.marshalToString(map)); 161 | } 162 | } 163 | executorService.invokeAll(jedisPipelineMap.values(), 1, TimeUnit.MINUTES); 164 | } catch (JedisException e) { 165 | broken = true; 166 | throw e; 167 | } catch (Exception e) { 168 | throw new RuntimeException(e); 169 | } finally { 170 | RedisUtils.returnResource(shardedJedisPool, shardedJedis, broken); 171 | } 172 | } 173 | 174 | @Override 175 | protected void handleDelete(List rowDatasList) { 176 | 177 | ShardedJedis shardedJedis = shardedJedisPool.getResource(); 178 | boolean broken = false; 179 | try { 180 | Map jedisPipelineMap = Maps.newHashMap(); 181 | for (CanalEntry.RowData rowData : rowDatasList) { 182 | List beforeColumnsList = rowData.getBeforeColumnsList(); 183 | if (logger.isDebugEnabled()) { 184 | logger.debug("{} receive delete data :\n {}", tableConfig.getTableName(), toString(beforeColumnsList)); 185 | } 186 | StringBuilder id = new StringBuilder(); 187 | for (CanalEntry.Column c : beforeColumnsList) { 188 | if (c.getIsKey()) { 189 | id.append(c.getValue()); 190 | } 191 | } 192 | String key = String.format("%s_%s", tableConfig.getPureTable(), id.toString()); 193 | 194 | Jedis jedis = shardedJedis.getShard(key); 195 | PipelineSyncTask pilelineSyncTask = jedisPipelineMap.get(jedis); 196 | if (pilelineSyncTask == null) { 197 | pilelineSyncTask = new PipelineSyncTask(jedis.pipelined()); 198 | jedisPipelineMap.put(jedis, pilelineSyncTask); 199 | } 200 | pilelineSyncTask.pipeline.del(key); 201 | } 202 | executorService.invokeAll(jedisPipelineMap.values(), 1, TimeUnit.MINUTES); 203 | } catch (JedisException e) { 204 | broken = true; 205 | throw e; 206 | } catch (Exception e) { 207 | throw new RuntimeException(e); 208 | } finally { 209 | RedisUtils.returnResource(shardedJedisPool, shardedJedis, broken); 210 | } 211 | } 212 | 213 | @Override 214 | public void stop() { 215 | super.stop(); 216 | executorService.shutdown(); 217 | shardedJedisPool.destroy(); 218 | } 219 | }; 220 | } 221 | 222 | 223 | private class PipelineSyncTask implements Callable { 224 | 225 | Pipeline pipeline; 226 | 227 | PipelineSyncTask(Pipeline pipeline) { 228 | this.pipeline = pipeline; 229 | } 230 | 231 | @Override 232 | public Void call() throws Exception { 233 | pipeline.sync(); 234 | return null; 235 | } 236 | } 237 | 238 | } 239 | -------------------------------------------------------------------------------- /src/main/java/mysql/redis/replicate/redis/RedisUtils.java: -------------------------------------------------------------------------------- 1 | package mysql.redis.replicate.redis; 2 | 3 | import org.apache.commons.pool2.impl.GenericObjectPoolConfig; 4 | import redis.clients.jedis.*; 5 | 6 | import java.util.*; 7 | import java.util.concurrent.atomic.AtomicLong; 8 | 9 | /** 10 | * Created by wens on 15-12-4. 11 | */ 12 | public class RedisUtils { 13 | 14 | public static ShardedJedisPool createSharedJedisPool(GenericObjectPoolConfig config, String redisInfos, String redisPassword) { 15 | 16 | if (config == null) { 17 | config = new JedisPoolConfig(); 18 | } 19 | String[] redisAddressInfos = redisInfos.split(",| |(\\r)?\\n"); 20 | List jdsInfoList = new ArrayList(redisAddressInfos.length); 21 | for (String address : redisAddressInfos) { 22 | if (address.trim().length() == 0) { 23 | break; 24 | } 25 | String[] hostAndPort = address.trim().split(":"); 26 | String host = hostAndPort[0]; 27 | int port = 6379; 28 | if (hostAndPort.length == 2) { 29 | port = Integer.parseInt(hostAndPort[1]); 30 | } 31 | JedisShardInfo jedisShardInfo = new JedisShardInfo(host, port, 6000); 32 | if (redisPassword != null && redisPassword.length() != 0) { 33 | jedisShardInfo.setPassword(redisPassword); 34 | } 35 | jdsInfoList.add(jedisShardInfo); 36 | } 37 | return new ShardedJedisPool(config, jdsInfoList); 38 | } 39 | 40 | 41 | public static ShardedJedis createSharedJedis(String redisInfos, String redisPassword) { 42 | String[] redisAddressInfos = redisInfos.split(",| |(\\r)?\\n"); 43 | List jdsInfoList = new ArrayList(redisAddressInfos.length); 44 | for (String address : redisAddressInfos) { 45 | String[] hostAndPort = address.trim().split(":"); 46 | String host = hostAndPort[0]; 47 | int port = 6379; 48 | if (hostAndPort.length == 2) { 49 | port = Integer.parseInt(hostAndPort[1]); 50 | } 51 | JedisShardInfo jedisShardInfo = new JedisShardInfo(host, port, 6000); 52 | if (redisPassword != null && redisPassword.length() != 0) { 53 | jedisShardInfo.setPassword(redisPassword); 54 | } 55 | jdsInfoList.add(jedisShardInfo); 56 | } 57 | return new ShardedJedis(jdsInfoList); 58 | } 59 | 60 | 61 | public static void putMap(Jedis client, String key, Map data) { 62 | Pipeline pipelined = client.pipelined(); 63 | for (String column : data.keySet()) { 64 | pipelined.hset(key, column, data.get(column)); 65 | } 66 | pipelined.sync(); 67 | } 68 | 69 | 70 | public static void returnResource( JedisPool jedisPool , Jedis jedis, boolean broken) { 71 | if (jedis != null) { 72 | if (broken) { 73 | jedisPool.returnBrokenResource(jedis); 74 | } else { 75 | jedisPool.returnResource(jedis); 76 | } 77 | } 78 | } 79 | 80 | public static void returnResource(ShardedJedisPool shardedJedisPool ,ShardedJedis shardedJedis, boolean broken) { 81 | if (shardedJedis != null) { 82 | if (broken) { 83 | shardedJedisPool.returnBrokenResource(shardedJedis); 84 | } else { 85 | shardedJedisPool.returnResource(shardedJedis); 86 | } 87 | } 88 | } 89 | 90 | 91 | } 92 | -------------------------------------------------------------------------------- /src/main/java/mysql/redis/replicate/web/AliveServerServlet.java: -------------------------------------------------------------------------------- 1 | package mysql.redis.replicate.web; 2 | 3 | import com.alibaba.otter.canal.common.utils.JsonUtils; 4 | import mysql.redis.replicate.CoordinatorController; 5 | 6 | import javax.servlet.ServletException; 7 | import javax.servlet.http.HttpServlet; 8 | import javax.servlet.http.HttpServletRequest; 9 | import javax.servlet.http.HttpServletResponse; 10 | import java.io.IOException; 11 | import java.util.List; 12 | 13 | /** 14 | * Created by wens on 15/12/6. 15 | */ 16 | public class AliveServerServlet extends HttpServlet { 17 | 18 | private CoordinatorController coordinatorController; 19 | 20 | public AliveServerServlet(CoordinatorController coordinatorController) { 21 | this.coordinatorController = coordinatorController; 22 | } 23 | 24 | @Override 25 | protected void service(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { 26 | List serverIds = coordinatorController.getAliveServerIds(); 27 | resp.getWriter().write(JsonUtils.marshalToString(serverIds)); 28 | } 29 | 30 | 31 | } 32 | -------------------------------------------------------------------------------- /src/main/java/mysql/redis/replicate/web/AllDestinationServlet.java: -------------------------------------------------------------------------------- 1 | package mysql.redis.replicate.web; 2 | 3 | import com.alibaba.otter.canal.common.utils.JsonUtils; 4 | import com.google.common.collect.Lists; 5 | import com.google.common.collect.Maps; 6 | import mysql.redis.replicate.CoordinatorController; 7 | import mysql.redis.replicate.Tuple; 8 | import mysql.redis.replicate.ZkPathUtils; 9 | import mysql.redis.replicate.ZookeeperUtils; 10 | import mysql.redis.replicate.config.DestinationConfig; 11 | import mysql.redis.replicate.config.DestinationConfigManager; 12 | 13 | import javax.servlet.ServletException; 14 | import javax.servlet.http.HttpServlet; 15 | import javax.servlet.http.HttpServletRequest; 16 | import javax.servlet.http.HttpServletResponse; 17 | import java.io.IOException; 18 | import java.sql.*; 19 | import java.util.HashMap; 20 | import java.util.List; 21 | import java.util.Map; 22 | import java.util.Set; 23 | 24 | /** 25 | * Created by wens on 15-11-16. 26 | */ 27 | public class AllDestinationServlet extends HttpServlet { 28 | 29 | 30 | 31 | 32 | private DestinationConfigManager destinationConfigManager; 33 | 34 | 35 | public AllDestinationServlet(DestinationConfigManager destinationConfigManager) { 36 | this.destinationConfigManager = destinationConfigManager; 37 | } 38 | 39 | protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { 40 | Set allDestination = destinationConfigManager.getAllDestination(); 41 | List> retList = Lists.newArrayList(); 42 | 43 | for (String dest : allDestination) { 44 | 45 | DestinationConfig config = destinationConfigManager.getDestinationConfig(dest); 46 | 47 | String monitorUrl = ""; 48 | 49 | CoordinatorController.ServerInfo serverInfo = ZookeeperUtils.readData(ZkPathUtils.getIdsPath(config.getRunOn()), CoordinatorController.ServerInfo.class); 50 | if(serverInfo != null ){ 51 | monitorUrl = String.format("%s?destination=%s&mysqlAddress=%s&mysqlUser=%s&mysqlPassword=%s", serverInfo.getHttpEndpoint().replace("endpoint", "monitor"), config.getDestination(), config.getDbAddress(), config.getDbUser(), config.getDbPassword()) ; 52 | } 53 | 54 | HashMap map = Maps.newHashMap(); 55 | map.put("destination", dest); 56 | map.put("stopped", config.isStopped()); 57 | map.put("runOn", config.getRunOn()); 58 | map.put("runFail", config.isRunFail()); 59 | map.put("monitorUrl", monitorUrl ); 60 | retList.add(map); 61 | } 62 | response.getWriter().write(JsonUtils.marshalToString(retList)); 63 | } 64 | 65 | 66 | 67 | } 68 | -------------------------------------------------------------------------------- /src/main/java/mysql/redis/replicate/web/DestinationOptServlet.java: -------------------------------------------------------------------------------- 1 | package mysql.redis.replicate.web; 2 | 3 | import mysql.redis.replicate.CoordinatorController; 4 | import mysql.redis.replicate.LoggerFactory; 5 | import org.slf4j.Logger; 6 | 7 | import javax.servlet.ServletException; 8 | import javax.servlet.http.HttpServlet; 9 | import javax.servlet.http.HttpServletRequest; 10 | import javax.servlet.http.HttpServletResponse; 11 | import java.io.IOException; 12 | 13 | /** 14 | * Created by wens on 15-11-17. 15 | */ 16 | public class DestinationOptServlet extends HttpServlet { 17 | 18 | private final Logger logger = LoggerFactory.getLogger(); 19 | 20 | private CoordinatorController coordinatorController; 21 | 22 | 23 | public DestinationOptServlet(CoordinatorController coordinatorController) { 24 | this.coordinatorController = coordinatorController; 25 | } 26 | 27 | @Override 28 | protected void service(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { 29 | 30 | try { 31 | String opt = req.getParameter("opt"); 32 | String destination = req.getParameter("destination"); 33 | String runOn = req.getParameter("runOn"); 34 | if (opt != null && destination != null) { 35 | if (opt.equalsIgnoreCase("stop")) { 36 | coordinatorController.stopDestination(destination); 37 | } else if (opt.equalsIgnoreCase("start")) { 38 | coordinatorController.startDestination(destination, runOn); 39 | } else if (opt.equalsIgnoreCase("delete")) { 40 | coordinatorController.deleteDestination(destination); 41 | } else { 42 | resp.getWriter().write("unknown opt :" + opt); 43 | return; 44 | } 45 | 46 | resp.getWriter().write("ok"); 47 | } 48 | } catch (Exception e) { 49 | logger.error("opt fail : opt = {} ", e); 50 | resp.getWriter().write("fail"); 51 | } 52 | 53 | 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/main/java/mysql/redis/replicate/web/EndpointServlet.java: -------------------------------------------------------------------------------- 1 | package mysql.redis.replicate.web; 2 | 3 | import mysql.redis.replicate.canal.ControllerService; 4 | import mysql.redis.replicate.config.DestinationConfigManager; 5 | 6 | import javax.servlet.ServletException; 7 | import javax.servlet.http.HttpServlet; 8 | import javax.servlet.http.HttpServletRequest; 9 | import javax.servlet.http.HttpServletResponse; 10 | import java.io.IOException; 11 | 12 | /** 13 | * Created by wens on 15/12/6. 14 | */ 15 | public class EndpointServlet extends HttpServlet { 16 | 17 | private ControllerService controllerService; 18 | 19 | private DestinationConfigManager destinationConfigManager; 20 | 21 | public EndpointServlet(DestinationConfigManager destinationConfigManager, ControllerService controllerService) { 22 | this.destinationConfigManager = destinationConfigManager; 23 | this.controllerService = controllerService; 24 | } 25 | 26 | @Override 27 | protected void service(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { 28 | 29 | try { 30 | String opt = req.getParameter("cmd"); 31 | String destination = req.getParameter("destination"); 32 | if (opt != null && destination != null) { 33 | if (opt.equalsIgnoreCase("stop")) { 34 | controllerService.stopTask(destination); 35 | } else if (opt.equalsIgnoreCase("start")) { 36 | controllerService.startTask(destination); 37 | } else if (opt.equalsIgnoreCase("delete")) { 38 | controllerService.stopTask(destination); 39 | destinationConfigManager.delete(destination); 40 | } 41 | resp.getWriter().write("ok"); 42 | } 43 | } catch (Exception e) { 44 | resp.getWriter().write("fail :" + e.getMessage()); 45 | } 46 | 47 | 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/main/java/mysql/redis/replicate/web/GetConfigServlet.java: -------------------------------------------------------------------------------- 1 | package mysql.redis.replicate.web; 2 | 3 | import com.alibaba.otter.canal.common.utils.JsonUtils; 4 | import mysql.redis.replicate.config.DestinationConfig; 5 | import mysql.redis.replicate.config.DestinationConfigManager; 6 | 7 | import javax.servlet.ServletException; 8 | import javax.servlet.http.HttpServlet; 9 | import javax.servlet.http.HttpServletRequest; 10 | import javax.servlet.http.HttpServletResponse; 11 | import java.io.IOException; 12 | 13 | /** 14 | * Created by wens on 15-11-16. 15 | */ 16 | public class GetConfigServlet extends HttpServlet { 17 | 18 | 19 | private DestinationConfigManager destinationConfigManager; 20 | 21 | public GetConfigServlet(DestinationConfigManager destinationConfigManager) { 22 | this.destinationConfigManager = destinationConfigManager; 23 | } 24 | 25 | protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { 26 | String destination = request.getParameter("destination"); 27 | DestinationConfig config = destinationConfigManager.getDestinationConfig(destination); 28 | response.getWriter().write(JsonUtils.marshalToString(config)); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/mysql/redis/replicate/web/MonitorServlet.java: -------------------------------------------------------------------------------- 1 | package mysql.redis.replicate.web; 2 | 3 | import mysql.redis.replicate.Monitor; 4 | import mysql.redis.replicate.Tuple; 5 | import mysql.redis.replicate.ZkPathUtils; 6 | import mysql.redis.replicate.ZookeeperUtils; 7 | 8 | import javax.servlet.ServletException; 9 | import javax.servlet.http.HttpServlet; 10 | import javax.servlet.http.HttpServletRequest; 11 | import javax.servlet.http.HttpServletResponse; 12 | import java.io.IOException; 13 | import java.io.PrintWriter; 14 | import java.sql.*; 15 | 16 | /** 17 | * Created by wens on 15-12-8. 18 | */ 19 | public class MonitorServlet extends HttpServlet { 20 | 21 | static { 22 | try { 23 | Class.forName("com.mysql.jdbc.Driver"); 24 | } catch (ClassNotFoundException e) { 25 | throw new RuntimeException("Load mysql driver fail."); 26 | } 27 | } 28 | 29 | @Override 30 | protected void service(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { 31 | 32 | String destination = req.getParameter("destination"); 33 | String mysqlAddress = req.getParameter("mysqlAddress"); 34 | String mysqlUser = req.getParameter("mysqlUser"); 35 | String mysqlPassword = req.getParameter("mysqlPassword"); 36 | 37 | Tuple masterBinlogPosition = null; 38 | try { 39 | masterBinlogPosition = getMasterBinlogPosition("jdbc:mysql://" +mysqlAddress , mysqlUser , mysqlPassword ); 40 | } catch (SQLException e) { 41 | throw new RuntimeException(e); 42 | } 43 | String mysqlLogOffset = masterBinlogPosition == null ? "" : masterBinlogPosition.getOne() + ":" + masterBinlogPosition.getTwo(); 44 | 45 | Monitor.MonitorData monitorData = Monitor.getMonitorData(destination); 46 | 47 | PrintWriter writer = resp.getWriter(); 48 | 49 | writer.write("mysql binlog offset:" + mysqlLogOffset + "\n"); 50 | 51 | if(monitorData != null ){ 52 | writer.write("replicate binlog offset:" +monitorData.logfileName + ":" + monitorData.logfileOffset + "\n" ); 53 | writer.write("insert count:" +monitorData.insertCount+ "\n" ); 54 | writer.write("update count:" +monitorData.updateCount+ "\n" ); 55 | writer.write("delete count:" +monitorData.deleteCount+ "\n" ); 56 | writer.write("speed:" +monitorData.speed+ "\n" ); 57 | writer.flush(); 58 | } 59 | } 60 | 61 | public Tuple getMasterBinlogPosition(String url, String user, String password) throws SQLException { 62 | Connection conn = null; 63 | Tuple position = null; 64 | try { 65 | conn = DriverManager.getConnection(url, user, password); 66 | 67 | Statement statement = conn.createStatement(); 68 | 69 | ResultSet resultSet = statement.executeQuery("show master status "); 70 | 71 | while (resultSet.next()) { 72 | position = new Tuple<>(resultSet.getString("File"), resultSet.getString("Position")); 73 | break; 74 | } 75 | resultSet.close(); 76 | statement.close(); 77 | } finally { 78 | if (conn != null) { 79 | conn.close(); 80 | } 81 | } 82 | return position; 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /src/main/java/mysql/redis/replicate/web/SaveConfigServlet.java: -------------------------------------------------------------------------------- 1 | package mysql.redis.replicate.web; 2 | 3 | import com.alibaba.otter.canal.common.utils.JsonUtils; 4 | import mysql.redis.replicate.config.DestinationConfig; 5 | import mysql.redis.replicate.config.DestinationConfigManager; 6 | 7 | import javax.servlet.ServletException; 8 | import javax.servlet.http.HttpServlet; 9 | import javax.servlet.http.HttpServletRequest; 10 | import javax.servlet.http.HttpServletResponse; 11 | import java.io.IOException; 12 | 13 | /** 14 | * Created by wens on 15-11-16. 15 | */ 16 | public class SaveConfigServlet extends HttpServlet { 17 | 18 | 19 | private DestinationConfigManager destinationConfigManager; 20 | 21 | public SaveConfigServlet(DestinationConfigManager destinationConfigManager) { 22 | this.destinationConfigManager = destinationConfigManager; 23 | } 24 | 25 | protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { 26 | String jsonStr = request.getParameter("json"); 27 | try { 28 | DestinationConfig destinationConfig = JsonUtils.unmarshalFromString(jsonStr, DestinationConfig.class); 29 | Utils.testConfig(destinationConfig); 30 | DestinationConfig old = destinationConfigManager.getDestinationConfig(destinationConfig.getDestination()); 31 | 32 | if (old != null) { 33 | destinationConfig.setStopped(old.isStopped()); 34 | destinationConfig.setRunFail(old.isRunFail()); 35 | destinationConfig.setRunOn(old.getRunOn()); 36 | } 37 | 38 | destinationConfigManager.saveOrUpdate(destinationConfig); 39 | response.getWriter().write("ok"); 40 | } catch (Exception e) { 41 | response.getWriter().write("fail:" + e.getMessage()); 42 | } 43 | 44 | 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/main/java/mysql/redis/replicate/web/TestConfigServlet.java: -------------------------------------------------------------------------------- 1 | package mysql.redis.replicate.web; 2 | 3 | import com.alibaba.otter.canal.common.utils.JsonUtils; 4 | import mysql.redis.replicate.config.DestinationConfig; 5 | import mysql.redis.replicate.config.DestinationConfigManager; 6 | 7 | import javax.servlet.ServletException; 8 | import javax.servlet.http.HttpServlet; 9 | import javax.servlet.http.HttpServletRequest; 10 | import javax.servlet.http.HttpServletResponse; 11 | import java.io.IOException; 12 | 13 | /** 14 | * Created by wens on 15-11-16. 15 | */ 16 | public class TestConfigServlet extends HttpServlet { 17 | 18 | 19 | private DestinationConfigManager destinationConfigManager; 20 | 21 | public TestConfigServlet(DestinationConfigManager destinationConfigManager) { 22 | this.destinationConfigManager = destinationConfigManager; 23 | } 24 | 25 | protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { 26 | String jsonStr = request.getParameter("json"); 27 | try { 28 | DestinationConfig destinationConfig = JsonUtils.unmarshalFromString(jsonStr, DestinationConfig.class); 29 | response.getWriter().write("ok:\n" + Utils.testConfig(destinationConfig)); 30 | } catch (Exception e) { 31 | response.getWriter().write("fail:" + e.getMessage()); 32 | } 33 | 34 | 35 | } 36 | 37 | 38 | } 39 | -------------------------------------------------------------------------------- /src/main/java/mysql/redis/replicate/web/Utils.java: -------------------------------------------------------------------------------- 1 | package mysql.redis.replicate.web; 2 | 3 | import com.google.common.collect.Maps; 4 | import com.google.common.io.Files; 5 | import mysql.redis.replicate.config.DestinationConfig; 6 | import mysql.redis.replicate.groovy.IRow2map; 7 | import mysql.redis.replicate.groovy.Row2map; 8 | import mysql.redis.replicate.groovy.Row2mapManager; 9 | import mysql.redis.replicate.redis.RedisUtils; 10 | import redis.clients.jedis.Jedis; 11 | import redis.clients.jedis.ShardedJedis; 12 | 13 | import java.io.File; 14 | import java.io.IOException; 15 | import java.nio.charset.StandardCharsets; 16 | import java.sql.*; 17 | import java.util.Map; 18 | import java.util.Random; 19 | 20 | /** 21 | * Created by wens on 15-11-16. 22 | */ 23 | public class Utils { 24 | 25 | static { 26 | try { 27 | Class.forName("com.mysql.jdbc.Driver"); 28 | } catch (ClassNotFoundException e) { 29 | throw new RuntimeException("Load mysql driver fail."); 30 | } 31 | } 32 | 33 | public static String testConfig(DestinationConfig destinationConfig) throws SQLException { 34 | Connection conn = null; 35 | try { 36 | StringBuilder sb = new StringBuilder(); 37 | conn = DriverManager.getConnection(String.format("jdbc:mysql://%s?connectTimeout=4000", destinationConfig.getDbAddress()), destinationConfig.getDbUser(), destinationConfig.getDbPassword()); 38 | for (DestinationConfig.TableConfig tableConfig : destinationConfig.getTableConfigs()) { 39 | String template; 40 | try { 41 | template = Files.toString(new File(Thread.currentThread().getContextClassLoader().getResource(Row2mapManager.SCRIPT_TEMPLATE_FILE).getPath()), StandardCharsets.UTF_8); 42 | } catch (IOException e) { 43 | throw new RuntimeException(e); 44 | } 45 | IRow2map row2map = new Row2map(template.replace("#script#", tableConfig.getScript())); 46 | 47 | PreparedStatement statement = conn.prepareStatement("select * from " + tableConfig.getTableName() + " limit 1 "); 48 | ResultSet resultSet = statement.executeQuery(); 49 | 50 | if (resultSet.next()) { 51 | Map data = Maps.newHashMap(); 52 | Map type = Maps.newHashMap(); 53 | int columnCount = resultSet.getMetaData().getColumnCount(); 54 | for (int i = 1; i <= columnCount; i++) { 55 | String columnName = resultSet.getMetaData().getColumnName(i); 56 | data.put(columnName, resultSet.getString(columnName)); 57 | type.put(columnName, resultSet.getMetaData().getColumnTypeName(i)); 58 | } 59 | Map map = Maps.newHashMap(); 60 | row2map.convert(type, data, map); 61 | 62 | ShardedJedis sharedJedis = RedisUtils.createSharedJedis(tableConfig.getRedisInfos(), tableConfig.getRedisPassword()); 63 | 64 | Random random = new Random(500); 65 | String id = String.valueOf(random.nextInt()); 66 | String key = "_test_key_" + id; 67 | Jedis jedis = sharedJedis.getShard(id); 68 | jedis.del(key); 69 | RedisUtils.putMap(jedis, key, map); 70 | sb.append(map).append("\n"); 71 | } 72 | resultSet.close(); 73 | statement.close(); 74 | } 75 | return sb.toString(); 76 | } finally { 77 | if (conn != null) { 78 | conn.close(); 79 | } 80 | } 81 | } 82 | 83 | } 84 | -------------------------------------------------------------------------------- /src/main/java/mysql/redis/replicate/web/WebConsole.java: -------------------------------------------------------------------------------- 1 | package mysql.redis.replicate.web; 2 | 3 | import mysql.redis.replicate.Conf; 4 | import mysql.redis.replicate.CoordinatorController; 5 | import mysql.redis.replicate.LoggerFactory; 6 | import mysql.redis.replicate.canal.ControllerService; 7 | import mysql.redis.replicate.config.DestinationConfigManager; 8 | import org.eclipse.jetty.server.Handler; 9 | import org.eclipse.jetty.server.Server; 10 | import org.eclipse.jetty.server.handler.HandlerList; 11 | import org.eclipse.jetty.server.handler.ResourceHandler; 12 | import org.eclipse.jetty.server.nio.SelectChannelConnector; 13 | import org.eclipse.jetty.servlet.ServletContextHandler; 14 | import org.eclipse.jetty.servlet.ServletHolder; 15 | import org.slf4j.Logger; 16 | 17 | /** 18 | * Created by wens on 15-11-16. 19 | */ 20 | public class WebConsole { 21 | 22 | private final Logger logger = LoggerFactory.getLogger(); 23 | 24 | private Server server; 25 | 26 | public WebConsole(Conf conf, ControllerService controllerService, DestinationConfigManager destinationConfigManager, CoordinatorController coordinatorController) { 27 | String siteResourcePath = Thread.currentThread().getContextClassLoader().getResource("site").getPath(); 28 | this.server = new Server(); 29 | SelectChannelConnector connector = new SelectChannelConnector(); 30 | logger.info("Web console bind on 0.0.0.0:{}", conf.getWebConsolePort()); 31 | connector.setPort(conf.getWebConsolePort()); 32 | server.addConnector(connector); 33 | 34 | ResourceHandler resource_handler = new ResourceHandler(); 35 | resource_handler.setDirectoriesListed(true); 36 | resource_handler.setWelcomeFiles(new String[]{"index.html"}); 37 | resource_handler.setResourceBase(siteResourcePath); 38 | ServletContextHandler context = new ServletContextHandler(ServletContextHandler.SESSIONS); 39 | context.setContextPath("/"); 40 | context.addServlet(new ServletHolder(new TestConfigServlet(destinationConfigManager)), "/config/test"); 41 | context.addServlet(new ServletHolder(new SaveConfigServlet(destinationConfigManager)), "/config/save"); 42 | context.addServlet(new ServletHolder(new GetConfigServlet(destinationConfigManager)), "/config/get"); 43 | context.addServlet(new ServletHolder(new AllDestinationServlet(destinationConfigManager)), "/destination/all"); 44 | context.addServlet(new ServletHolder(new DestinationOptServlet(coordinatorController)), "/destination/opt"); 45 | context.addServlet(new ServletHolder(new EndpointServlet(destinationConfigManager, controllerService)), "/endpoint"); 46 | context.addServlet(new ServletHolder(new AliveServerServlet(coordinatorController)), "/alive/server/ids"); 47 | context.addServlet(new ServletHolder(new MonitorServlet()), "/monitor"); 48 | 49 | HandlerList handlers = new HandlerList(); 50 | handlers.setHandlers(new Handler[]{resource_handler, context}); 51 | server.setHandler(handlers); 52 | 53 | } 54 | 55 | public void start() { 56 | try { 57 | logger.info("Web console start"); 58 | server.start(); 59 | } catch (Exception e) { 60 | throw new RuntimeException(); 61 | } 62 | } 63 | 64 | public void stop() { 65 | try { 66 | logger.info("Web console stop"); 67 | server.stop(); 68 | } catch (Exception e) { 69 | throw new RuntimeException(); 70 | } 71 | } 72 | 73 | } 74 | -------------------------------------------------------------------------------- /src/main/resources/config.properties: -------------------------------------------------------------------------------- 1 | id=0 2 | 3 | #zookeeper 4 | zk.root.path=/mysql-redis-replicate 5 | zk.server=localhost 6 | 7 | canal.batch.size=3000 8 | 9 | web.console.port=9090 10 | 11 | http.endpoint=http://localhost:9090/endpoint 12 | 13 | redis.maxActive = 16 14 | redis.minIdle = 4 15 | redis.maxIdle = 16 16 | redis.maxWait = 30000 17 | redis.minEvictableIdleTimeMillis = 60000 18 | redis.numTestsPerEvictionRun = -1 19 | redis.softMinEvictableIdleTimeMillis = -1 20 | redis.timeBetweenEvictionRunsMillis = 30000 21 | redis.whenExhaustedAction =1 -------------------------------------------------------------------------------- /src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{56} - %msg%n 7 | 8 | 9 | 10 | 11 | 12 | logs/mysql-redis-replicate.log 13 | 15 | 16 | logs/%d{yyyy-MM-dd}/mysql-redis-replicate-%d{yyyy-MM-dd}-%i.log.gz 17 | 18 | 19 | 512MB 20 | 21 | 60 22 | 23 | 24 | 25 | %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{56} - %msg%n 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /src/main/resources/row2map.groovy.tpl: -------------------------------------------------------------------------------- 1 | import mysql.redis.replicate.groovy.IRow2map 2 | 3 | import java.text.ParseException 4 | import java.text.SimpleDateFormat; 5 | 6 | import java.util.Date; 7 | /** 8 | * Created by wens on 15-11-13. 9 | */ 10 | public class MyRow2map implements IRow2map { 11 | 12 | @Override 13 | public void convert(Map type , Map row, Map map) { 14 | #script# 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/main/resources/site/destination.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | mysql redis replicate web console 6 | 7 | 8 | 14 | 15 | 16 | 17 | 18 | 19 | 20 |
21 |
22 | Basic Config 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 |
33 | 34 |
35 | Config Table 36 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 |
Table NameRedis InfoSCRIPT
53 |
54 | 55 | 56 |
57 | 238 | 239 | 240 | 241 | 242 | 243 | -------------------------------------------------------------------------------- /src/main/resources/site/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | mysql redis replicate web console 6 | 7 | 8 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 |
DestinationRun OnRun FailMonitor
38 | 39 |
40 | 注意:配置修改后需要重新加载才生效( STOP -> START) 41 | 42 |
43 | 130 | 131 | 132 | 133 | 134 | -------------------------------------------------------------------------------- /src/test/doc/test_script.sql: -------------------------------------------------------------------------------- 1 | 2 | CREATE USER canal IDENTIFIED BY 'canal'; 3 | GRANT SELECT, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'canal'@'%'; 4 | -- GRANT ALL PRIVILEGES ON *.* TO 'canal'@'%' ; 5 | FLUSH PRIVILEGES; 6 | 7 | create database user_db charset utf8 ; 8 | 9 | use user_db ; 10 | 11 | create table user ( id int auto_increment , name varchar(100) , age smallint , height float , sex char(1) , comment text , birthday date , create_at timestamp , primary key (id) ); 12 | 13 | insert into user (name , age , height , sex , comment , birthday , create_at ) values ('wens' , 29 , 165.66, 'M' , '音乐 读书 数学 编程' , '1986-01-06' , now() ) ; 14 | -------------------------------------------------------------------------------- /src/test/java/mysql/redis/replicate/HttpClientUtilsTest.java: -------------------------------------------------------------------------------- 1 | package mysql.redis.replicate; 2 | 3 | import java.io.IOException; 4 | import java.io.OutputStream; 5 | 6 | /** 7 | * Created by wens on 15/12/6. 8 | */ 9 | public class HttpClientUtilsTest { 10 | 11 | public static void main(String[] args) throws IOException, InterruptedException { 12 | 13 | Process nc = Runtime.getRuntime().exec("nc -l 9999"); 14 | new Thread() { 15 | @Override 16 | public void run() { 17 | String ret = HttpClientUtils.post("http://localhost:9999/post", Tuple.of("name", "wens")); 18 | System.out.println(ret); 19 | } 20 | }.start(); 21 | 22 | Thread.sleep(1000); 23 | 24 | StringBuffer resp = new StringBuffer(); 25 | 26 | resp.append("HTTP/1.1 200 OK\n"); 27 | resp.append("Content-Length: 2\n"); 28 | resp.append("Content-Type: text/html; charset=utf-8\n\n"); 29 | resp.append("ok"); 30 | OutputStream outputStream = nc.getOutputStream(); 31 | outputStream.write(resp.toString().getBytes()); 32 | outputStream.flush(); 33 | Thread.sleep(100); 34 | 35 | 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/test/java/mysql/redis/replicate/ZookeeperLeaderElectorTest.java: -------------------------------------------------------------------------------- 1 | package mysql.redis.replicate; 2 | 3 | import org.I0Itec.zkclient.ZkClient; 4 | 5 | import java.util.concurrent.BrokenBarrierException; 6 | import java.util.concurrent.CountDownLatch; 7 | import java.util.concurrent.CyclicBarrier; 8 | import java.util.concurrent.atomic.AtomicLong; 9 | 10 | /** 11 | * Created by wens on 15/12/5. 12 | */ 13 | public class ZookeeperLeaderElectorTest { 14 | 15 | public static void main(String[] args) throws InterruptedException { 16 | final ZkClient zkClient = new ZkClient("localhost"); 17 | ZkPathUtils.init("", zkClient); 18 | 19 | final AtomicLong atomicLong = new AtomicLong(0); 20 | 21 | final ZookeeperLeaderElector.LeaderListener leaderListener = new ZookeeperLeaderElector.LeaderListener() { 22 | public void onBecomingLeader() { 23 | atomicLong.incrementAndGet(); 24 | } 25 | }; 26 | 27 | int n = 3; 28 | final CountDownLatch countDownLatch = new CountDownLatch(n); 29 | final CyclicBarrier cyclicBarrier = new CyclicBarrier(n); 30 | 31 | for (int i = 0; i < n; i++) { 32 | final int ii = i; 33 | new Thread() { 34 | @Override 35 | public void run() { 36 | try { 37 | cyclicBarrier.await(); 38 | } catch (InterruptedException e) { 39 | e.printStackTrace(); 40 | } catch (BrokenBarrierException e) { 41 | e.printStackTrace(); 42 | } 43 | new ZookeeperLeaderElector(String.valueOf(ii), leaderListener).start(); 44 | 45 | countDownLatch.countDown(); 46 | } 47 | }.start(); 48 | } 49 | 50 | try { 51 | countDownLatch.await(); 52 | } catch (InterruptedException e) { 53 | e.printStackTrace(); 54 | } 55 | 56 | if (atomicLong.get() != 1) { 57 | throw new RuntimeException("Not pass"); 58 | } 59 | 60 | zkClient.delete(ZkPathUtils.getControllerPath()); 61 | 62 | Thread.sleep(1000); 63 | 64 | if (atomicLong.get() != 2) { 65 | throw new RuntimeException("Not pass"); 66 | } 67 | 68 | 69 | } 70 | 71 | 72 | } 73 | -------------------------------------------------------------------------------- /src/test/java/mysql/redis/replicate/canal/MysqlUtils.java: -------------------------------------------------------------------------------- 1 | package mysql.redis.replicate.canal; 2 | 3 | import java.sql.Connection; 4 | import java.sql.DriverManager; 5 | import java.sql.SQLException; 6 | import java.sql.Statement; 7 | 8 | /** 9 | * Created by wens on 15-10-15. 10 | */ 11 | public class MysqlUtils { 12 | 13 | 14 | public static Connection getConnection(String user, String pwd, String url) throws SQLException { 15 | return DriverManager.getConnection(url, user, pwd); 16 | } 17 | 18 | 19 | public static void executeSQL(Connection connection, String sql) throws SQLException { 20 | Statement statement = null; 21 | try { 22 | statement = connection.createStatement(); 23 | statement.execute(sql); 24 | } finally { 25 | if (statement != null) { 26 | statement.close(); 27 | } 28 | } 29 | } 30 | 31 | public static void executeUpdateSQL(Connection connection, String sql) throws SQLException { 32 | Statement statement = null; 33 | try { 34 | statement = connection.createStatement(); 35 | statement.executeUpdate(sql); 36 | } finally { 37 | if (statement != null) { 38 | statement.close(); 39 | } 40 | } 41 | } 42 | 43 | } 44 | --------------------------------------------------------------------------------