currentDatas, DbDialect dialect);
19 |
20 | public void after(L context, D currentData);
21 |
22 | public void commit(L context);
23 |
24 | public void error(L context);
25 | }
26 |
--------------------------------------------------------------------------------
/src/main/java/com/alibaba/otter/clave/progress/load/weight/WeightBarrier.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.otter.clave.progress.load.weight;
2 |
3 | import java.util.concurrent.TimeUnit;
4 | import java.util.concurrent.TimeoutException;
5 | import java.util.concurrent.locks.Condition;
6 | import java.util.concurrent.locks.ReentrantLock;
7 |
8 | /**
9 | * 构建基于weight的barrier控制
10 | *
11 | *
12 | * 场景：
13 | * 多个loader模块会进行并行加载，但每个loader的加载数据的进度统一受到weight的调度，只有当前的weight的所有数据都完成后，不同loader中的下一个weight才允许开始
14 | *
15 | * 实现：
16 | * 1. 使用AQS构建了一个基于weight的barrier处理，使用一个state进行控制(代表当前运行
19 | *
20 | * @author jianghang 2013-3-28 下午10:30:52
21 | * @version 1.0.0
22 | */
23 | public class WeightBarrier {
24 |
25 | private ReentrantLock lock = new ReentrantLock();
26 | private Condition condition = lock.newCondition();
27 | private volatile long threshold;
28 |
29 | public WeightBarrier(){
30 | this(Long.MAX_VALUE);
31 | }
32 |
33 | public WeightBarrier(long weight){
34 | this.threshold = weight;
35 | }
36 |
37 | /**
38 | * 阻塞等待weight允许执行
39 | *
40 | *
41 | * 阻塞返回条件：
42 | * 1. 中断事件
43 | * 2. 其他线程single()的weight > 当前阻塞等待的weight
44 | *
45 | *
46 | * @throws InterruptedException
47 | */
48 | public void await(long weight) throws InterruptedException {
49 | try {
50 | lock.lockInterruptibly();
51 | while (isPermit(weight) == false) {
52 | condition.await();
53 | }
54 | } finally {
55 | lock.unlock();
56 | }
57 | }
58 |
59 | /**
60 | * 阻塞等待当前的weight处理,允许设置超时时间
61 | *
62 | *
63 | * 阻塞返回条件：
64 | * 1. 中断事件
65 | * 2. 其他线程single()的weight > 当前阻塞等待的weight
66 | * 3. 超时
67 | *
68 | *
69 | * @param timeout
70 | * @param unit
71 | * @throws InterruptedException
72 | * @throws TimeoutException
73 | */
74 | public void await(long weight, long timeout, TimeUnit unit) throws InterruptedException, TimeoutException {
75 | try {
76 | lock.lockInterruptibly();
77 | while (isPermit(weight) == false) {
78 | condition.await(timeout, unit);
79 | }
80 | } finally {
81 | lock.unlock();
82 | }
83 | }
84 |
85 | /**
86 | * 重新设置weight信息
87 | *
88 | * @throws InterruptedException
89 | */
90 | public void single(long weight) throws InterruptedException {
91 | try {
92 | lock.lockInterruptibly();
93 | threshold = weight;
94 | condition.signalAll();
95 | } finally {
96 | lock.unlock();
97 | }
98 | }
99 |
100 | public long state() {
101 | return threshold;
102 | }
103 |
104 | private boolean isPermit(long state) {
105 | return state <= state();
106 | }
107 | }
108 |
--------------------------------------------------------------------------------
/src/main/java/com/alibaba/otter/clave/progress/load/weight/WeightBuckets.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.otter.clave.progress.load.weight;
2 |
3 | import java.util.ArrayList;
4 | import java.util.LinkedList;
5 | import java.util.List;
6 |
7 | import com.google.common.base.Function;
8 | import com.google.common.collect.Lists;
9 |
10 | /**
11 | * buckets的集合操作对象
12 | *
13 | * @author jianghang 2013-3-28 下午10:30:52
14 | * @version 1.0.0
15 | */
16 | public class WeightBuckets {
17 |
18 | private List> buckets = new ArrayList>(); // 对应的桶信息
19 |
20 | /**
21 | * 获取对应的weight的列表，从小到大的排序结果
22 | */
23 | public synchronized List weights() {
24 | return Lists.transform(buckets, new Function, Long>() {
25 |
26 | public Long apply(WeightBucket input) {
27 | return input.getWeight();
28 | }
29 | });
30 | }
31 |
32 | /**
33 | * 添加一个节点
34 | */
35 | public synchronized void addItem(long weight, T item) {
36 | WeightBucket bucket = new WeightBucket(weight);
37 | int index = indexedSearch(buckets, bucket);
38 | if (index > buckets.size() - 1) {// 先加一个bucket
39 | bucket.addLastItem(item);
40 | buckets.add(index, bucket);
41 | } else if (buckets.get(index).getWeight() != weight) {// 不匹配的
42 | bucket.addLastItem(item);
43 | buckets.add(index, bucket);
44 | } else {
45 | buckets.get(index).addLastItem(item);// 添加到已有的bucket上
46 | }
47 |
48 | }
49 |
50 | public synchronized List getItems(long weight) {
51 | WeightBucket bucket = new WeightBucket(weight);
52 | int index = indexedSearch(buckets, bucket);
53 | if (index < buckets.size() && index >= 0) {
54 | return buckets.get(index).getBucket();
55 | } else {
56 | return new LinkedList();
57 | }
58 | }
59 |
60 | // ========================= helper method =====================
61 |
62 | private int indexedSearch(List> list, WeightBucket item) {
63 | int i = 0;
64 | for (; i < list.size(); i++) {
65 | Comparable midVal = list.get(i);
66 | int cmp = midVal.compareTo(item);
67 | if (cmp == 0) {// item等于中间值
68 | return i;
69 | } else if (cmp > 0) {// item比中间值小
70 | return i;
71 | } else if (cmp < 0) {// item比中间值大
72 | // next
73 | }
74 | }
75 |
76 | return i;
77 | }
78 |
79 | }
80 |
81 | /**
82 | * 相同weight的item集合对象
83 | *
84 | * @author jianghang 2011-11-1 上午11:09:58
85 | * @version 4.0.0
86 | * @param
87 | */
88 | class WeightBucket implements Comparable {
89 |
90 | private long weight = -1;
91 | private LinkedList bucket = new LinkedList();
92 |
93 | public WeightBucket(){
94 | }
95 |
96 | public WeightBucket(long weight){
97 | this.weight = weight;
98 | }
99 |
100 | public long getWeight() {
101 | return weight;
102 | }
103 |
104 | public void setWeight(long weight) {
105 | this.weight = weight;
106 | }
107 |
108 | public List getBucket() {
109 | return bucket;
110 | }
111 |
112 | public void setBucket(LinkedList bucket) {
113 | this.bucket = bucket;
114 | }
115 |
116 | public void addFirstItem(T item) {
117 | this.bucket.addFirst(item);
118 | }
119 |
120 | public T getFirstItem() {
121 | return this.bucket.getFirst();
122 | }
123 |
124 | public void addLastItem(T item) {
125 | this.bucket.addLast(item);
126 | }
127 |
128 | public T getLastItem() {
129 | return this.bucket.getLast();
130 | }
131 |
132 | public int compareTo(WeightBucket o) {
133 | if (this.getWeight() > o.getWeight()) {
134 | return 1;
135 | } else if (this.getWeight() == o.getWeight()) {
136 | return 0;
137 | } else {
138 | return -1;
139 | }
140 | }
141 |
142 | }
143 |
--------------------------------------------------------------------------------
/src/main/java/com/alibaba/otter/clave/progress/load/weight/WeightController.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.otter.clave.progress.load.weight;
2 |
3 | import java.util.List;
4 | import java.util.concurrent.BlockingQueue;
5 | import java.util.concurrent.PriorityBlockingQueue;
6 | import java.util.concurrent.TimeUnit;
7 | import java.util.concurrent.TimeoutException;
8 | import java.util.concurrent.atomic.AtomicInteger;
9 |
10 | /**
11 | * 权重控制器
12 | *
13 | * @author jianghang 2013-3-28 下午10:30:52
14 | * @version 1.0.0
15 | */
16 | public class WeightController {
17 |
18 | private AtomicInteger latch;
19 | private WeightBarrier barrier;
20 | private BlockingQueue weights = new PriorityBlockingQueue();
21 |
22 | public WeightController(int load){
23 | latch = new AtomicInteger(load);
24 | barrier = new WeightBarrier(Integer.MIN_VALUE);
25 | }
26 |
27 | /**
28 | * 每个loader任务报告启动的第一个任务的weight
29 | *
30 | * @throws InterruptedException
31 | */
32 | public synchronized void start(List weights) throws InterruptedException {
33 | for (int i = 0; i < weights.size(); i++) {
34 | this.weights.add(weights.get(i));
35 | }
36 |
37 | int number = latch.decrementAndGet();
38 | if (number == 0) {
39 | Long initWeight = this.weights.peek();
40 | if (initWeight != null) {
41 | barrier.single(initWeight);
42 | }
43 | }
44 | }
45 |
46 | /**
47 | * 等待自己当前的weight任务可以被执行
48 | *
49 | * @throws InterruptedException
50 | */
51 | public void await(long weight) throws InterruptedException {
52 | barrier.await(weight);
53 | }
54 |
55 | /**
56 | * 等待自己当前的weight任务可以被执行,带超时控制
57 | *
58 | * @throws InterruptedException
59 | * @throws TimeoutException
60 | */
61 | public void await(long weight, long timeout, TimeUnit unit) throws InterruptedException, TimeoutException {
62 | barrier.await(weight, timeout, unit);
63 | }
64 |
65 | /**
66 | * 通知下一个weight任务可以被执行
67 | *
68 | * @throws InterruptedException
69 | */
70 | public synchronized void single(long weight) throws InterruptedException {
71 | this.weights.remove(weight);
72 | // 触发下一个可运行的weight
73 | Long nextWeight = this.weights.peek();
74 | if (nextWeight != null) {
75 | barrier.single(nextWeight);
76 | }
77 | }
78 | }
79 |
--------------------------------------------------------------------------------
/src/main/java/com/alibaba/otter/clave/progress/select/ClaveSelector.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.otter.clave.progress.select;
2 |
3 | import com.alibaba.otter.clave.common.lifecycle.ClaveLifeCycle;
4 |
5 | /**
6 | * 增量数据获取
7 | *
8 | * @author jianghang 2013-3-28 下午09:20:16
9 | * @version 1.0.0
10 | */
11 | public interface ClaveSelector extends ClaveLifeCycle {
12 |
13 | /**
14 | * 获取一批待处理的数据
15 | */
16 | public Message selector() throws InterruptedException;
17 |
18 | /**
19 | * 反馈一批数据处理失败，需要下次重新被处理
20 | */
21 | public void rollback(Long batchId);
22 |
23 | /**
24 | * 反馈所有的batch数据需要被重新处理
25 | */
26 | public void rollback();
27 |
28 | /**
29 | * 反馈一批数据处理完成
30 | */
31 | public void ack(Long batchId);
32 |
33 | }
34 |
--------------------------------------------------------------------------------
/src/main/java/com/alibaba/otter/clave/progress/select/Message.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.otter.clave.progress.select;
2 |
3 | import java.io.Serializable;
4 | import java.util.List;
5 |
6 | /**
7 | * 数据对象
8 | *
9 | * @author jianghang 2013-3-28 下午09:21:32
10 | * @version 1.0.0
11 | */
12 | public class Message implements Serializable {
13 |
14 | private static final long serialVersionUID = 4999493579483771204L;
15 | private Long id;
16 | private List datas;
17 |
18 | public Message(Long id, List datas){
19 | this.id = id;
20 | this.datas = datas;
21 | }
22 |
23 | public Long getId() {
24 | return id;
25 | }
26 |
27 | public void setId(Long id) {
28 | this.id = id;
29 | }
30 |
31 | public List getDatas() {
32 | return datas;
33 | }
34 |
35 | public void setDatas(List datas) {
36 | this.datas = datas;
37 | }
38 |
39 | }
40 |
--------------------------------------------------------------------------------
/src/main/java/com/alibaba/otter/clave/progress/select/canal/AbstractCanalSelector.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.otter.clave.progress.select.canal;
2 |
3 | import java.text.SimpleDateFormat;
4 | import java.util.Date;
5 | import java.util.List;
6 |
7 | import org.apache.commons.lang.SystemUtils;
8 | import org.slf4j.Logger;
9 | import org.slf4j.LoggerFactory;
10 | import org.slf4j.MDC;
11 | import org.springframework.util.CollectionUtils;
12 |
13 | import com.alibaba.otter.canal.protocol.CanalEntry.Entry;
14 | import com.alibaba.otter.clave.ClaveConfig;
15 | import com.alibaba.otter.clave.model.EventData;
16 | import com.alibaba.otter.clave.progress.select.ClaveSelector;
17 | import com.alibaba.otter.clave.progress.select.Message;
18 |
19 | public abstract class AbstractCanalSelector implements ClaveSelector {
20 |
21 | protected static final Logger logger = LoggerFactory.getLogger(AbstractCanalSelector.class);
22 | protected static final String SEP = SystemUtils.LINE_SEPARATOR;
23 | protected static final String DATE_FORMAT = "yyyy-MM-dd HH:mm:ss";
24 | protected String destination;
25 | protected int logSplitSize = 50;
26 | protected boolean dump = true;
27 | protected boolean dumpDetail = true;
28 | protected MessageParser messageParser;
29 |
30 | protected Message selector(com.alibaba.otter.canal.protocol.Message message) throws InterruptedException {
31 | List eventDatas = messageParser.parse(message.getEntries()); // 过滤事务头/尾和回环数据
32 | Message result = new Message(message.getId(), eventDatas);
33 |
34 | if (dump && logger.isInfoEnabled()) {
35 | String startPosition = null;
36 | String endPosition = null;
37 | if (!CollectionUtils.isEmpty(message.getEntries())) {
38 | startPosition = buildPositionForDump(message.getEntries().get(0));
39 | endPosition = buildPositionForDump(message.getEntries().get(message.getEntries().size() - 1));
40 | }
41 |
42 | dumpMessages(result, startPosition, endPosition, message.getEntries().size());// 记录一下，方便追查问题
43 | }
44 | return result;
45 | }
46 |
47 | /**
48 | * 记录一下message对象
49 | */
50 | protected void dumpMessages(Message message, String startPosition, String endPosition, int total) {
51 | try {
52 | MDC.put(ClaveConfig.splitLogFileKey, destination);
53 | logger.info(SEP + "****************************************************" + SEP);
54 | logger.info(MessageDumper.dumpMessageInfo(message, startPosition, endPosition, total));
55 | logger.info("****************************************************" + SEP);
56 | if (dumpDetail) {// 判断一下是否需要打印详细信息
57 | dumpEventDatas(message.getDatas());
58 | logger.info("****************************************************" + SEP);
59 | }
60 | } finally {
61 | MDC.remove(ClaveConfig.splitLogFileKey);
62 | }
63 | }
64 |
65 | /**
66 | * 分批输出多个数据
67 | */
68 | protected void dumpEventDatas(List eventDatas) {
69 | int size = eventDatas.size();
70 | // 开始输出每条记录
71 | int index = 0;
72 | do {
73 | if (index + logSplitSize >= size) {
74 | logger.info(MessageDumper.dumpEventDatas(eventDatas.subList(index, size)));
75 | } else {
76 | logger.info(MessageDumper.dumpEventDatas(eventDatas.subList(index, index + logSplitSize)));
77 | }
78 | index += logSplitSize;
79 | } while (index < size);
80 | }
81 |
82 | protected String buildPositionForDump(Entry entry) {
83 | long time = entry.getHeader().getExecuteTime();
84 | Date date = new Date(time);
85 | SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT);
86 | return entry.getHeader().getLogfileName() + ":" + entry.getHeader().getLogfileOffset() + ":"
87 | + entry.getHeader().getExecuteTime() + "(" + format.format(date) + ")";
88 | }
89 |
90 | public void setLogSplitSize(int logSplitSize) {
91 | this.logSplitSize = logSplitSize;
92 | }
93 |
94 | public void setDump(boolean dump) {
95 | this.dump = dump;
96 | }
97 |
98 | public void setDumpDetail(boolean dumpDetail) {
99 | this.dumpDetail = dumpDetail;
100 | }
101 |
102 | public void setMessageParser(MessageParser messageParser) {
103 | this.messageParser = messageParser;
104 | }
105 |
106 | }
107 |
--------------------------------------------------------------------------------
/src/main/java/com/alibaba/otter/clave/progress/select/canal/CanalClientSelector.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.otter.clave.progress.select.canal;
2 |
3 | import java.net.InetSocketAddress;
4 | import java.util.List;
5 |
6 | import org.apache.commons.lang.StringUtils;
7 | import org.springframework.util.CollectionUtils;
8 |
9 | import com.alibaba.otter.canal.client.CanalConnector;
10 | import com.alibaba.otter.canal.client.CanalConnectors;
11 | import com.alibaba.otter.clave.exceptions.ClaveException;
12 | import com.alibaba.otter.clave.model.EventData;
13 | import com.alibaba.otter.clave.progress.select.Message;
14 |
15 | /**
16 | * 基于canal client模式实现数据获取方式
17 | *
18 | * @author jianghang 2013-3-28 下午10:01:19
19 | * @version 1.0.0
20 | */
21 | public class CanalClientSelector extends AbstractCanalSelector {
22 |
23 | private String destination;
24 | private String zkServers;
25 | private List address;
26 | private String username = "";
27 | private String password = "";
28 | private volatile boolean running = false; // 是否处于运行中
29 |
30 | private CanalConnector connector;
31 | private String filter;
32 | private int batchSize = 5000;
33 |
34 | public CanalClientSelector(){
35 |
36 | }
37 |
38 | public CanalClientSelector(String destination, String zkServers){
39 | this.destination = destination;
40 | this.zkServers = zkServers;
41 | }
42 |
43 | public CanalClientSelector(String destination, List address){
44 | this.destination = destination;
45 | this.address = address;
46 | }
47 |
48 | public void start() {
49 | if (StringUtils.isNotEmpty(zkServers)) {
50 | connector = CanalConnectors.newClusterConnector(zkServers, destination, username, password);
51 | } else if (!CollectionUtils.isEmpty(address)) {
52 | connector = CanalConnectors.newClusterConnector(address, destination, username, password);
53 | } else {
54 | throw new ClaveException("no server zkservers or canal server address!");
55 | }
56 |
57 | connector.connect();
58 | connector.subscribe(filter);
59 | running = true;
60 | }
61 |
62 | public void stop() {
63 | if (!running) {
64 | return;
65 | }
66 |
67 | connector.disconnect();
68 | running = false;
69 | }
70 |
71 | public boolean isStart() {
72 | return running;
73 | }
74 |
75 | public Message selector() throws InterruptedException {
76 | com.alibaba.otter.canal.protocol.Message message = null;
77 | while (running) {
78 | message = connector.getWithoutAck(batchSize);
79 | if (message == null || message.getId() == -1L) { // 代表没数据
80 | continue;
81 | } else {
82 | break;
83 | }
84 | }
85 |
86 | if (!running) {
87 | throw new InterruptedException();
88 | }
89 |
90 | return selector(message);
91 | }
92 |
93 | public void ack(Long batchId) {
94 | connector.ack(batchId);
95 | }
96 |
97 | public void rollback(Long batchId) {
98 | connector.rollback(batchId);
99 | }
100 |
101 | public void rollback() {
102 | connector.rollback();
103 | }
104 |
105 | // ================= setter / getter =====================
106 |
107 | public void setDestination(String destination) {
108 | this.destination = destination;
109 | }
110 |
111 | public void setZkServers(String zkServers) {
112 | this.zkServers = zkServers;
113 | }
114 |
115 | public void setAddress(List address) {
116 | this.address = address;
117 | }
118 |
119 | public void setUsername(String username) {
120 | this.username = username;
121 | }
122 |
123 | public void setPassword(String password) {
124 | this.password = password;
125 | }
126 |
127 | public void setFilter(String filter) {
128 | this.filter = filter;
129 | }
130 |
131 | public void setBatchSize(int batchSize) {
132 | this.batchSize = batchSize;
133 | }
134 | }
135 |
--------------------------------------------------------------------------------
/src/main/java/com/alibaba/otter/clave/progress/select/canal/MessageDumper.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.otter.clave.progress.select.canal;
2 |
3 | import java.text.MessageFormat;
4 | import java.text.SimpleDateFormat;
5 | import java.util.Date;
6 | import java.util.List;
7 |
8 | import org.apache.commons.lang.StringUtils;
9 | import org.apache.commons.lang.SystemUtils;
10 | import org.springframework.util.CollectionUtils;
11 |
12 | import com.alibaba.otter.clave.model.EventColumn;
13 | import com.alibaba.otter.clave.model.EventData;
14 | import com.alibaba.otter.clave.progress.select.Message;
15 |
16 | /**
17 | * dump记录
18 | *
19 | * @author jianghang 2013-3-28 下午09:57:42
20 | * @version 1.0.0
21 | */
22 | public class MessageDumper {
23 |
24 | private static final String SEP = SystemUtils.LINE_SEPARATOR;
25 | private static final String TIMESTAMP_FORMAT = "yyyy-MM-dd HH:mm:ss:SSS";
26 | private static String context_format = null;
27 | private static String eventData_format = null;
28 | private static int event_default_capacity = 1024; // 预设值StringBuilder，减少扩容影响
29 |
30 | static {
31 | context_format = "* Batch Id: [{0}] ,total : [{1}] , normal : [{2}] , filter :[{3}] , Time : {4}" + SEP;
32 | context_format += "* Start : [{5}] " + SEP;
33 | context_format += "* End : [{6}] " + SEP;
34 |
35 | eventData_format = "-----------------" + SEP;
36 | eventData_format += "- Schema: {0} , Table: {1} " + SEP;
37 | eventData_format += "- Type: {2} , ExecuteTime: {3} " + SEP;
38 | eventData_format += "-----------------" + SEP;
39 | eventData_format += "---START" + SEP;
40 | eventData_format += "---Pks" + SEP;
41 | eventData_format += "{4}" + SEP;
42 | eventData_format += "---oldPks" + SEP;
43 | eventData_format += "{5}" + SEP;
44 | eventData_format += "---Columns" + SEP;
45 | eventData_format += "{6}" + SEP;
46 | eventData_format += "---END" + SEP;
47 |
48 | }
49 |
50 | public static String dumpMessageInfo(Message message, String startPosition, String endPosition, int total) {
51 | Date now = new Date();
52 | SimpleDateFormat format = new SimpleDateFormat(TIMESTAMP_FORMAT);
53 | int normal = message.getDatas().size();
54 | return MessageFormat.format(context_format, String.valueOf(message.getId()), total, normal, total - normal,
55 | format.format(now), startPosition, endPosition);
56 | }
57 |
58 | public static String dumpEventDatas(List eventDatas) {
59 | if (CollectionUtils.isEmpty(eventDatas)) {
60 | return StringUtils.EMPTY;
61 | }
62 |
63 | // 预先设定容量大小
64 | StringBuilder builder = new StringBuilder(event_default_capacity * eventDatas.size());
65 | for (EventData data : eventDatas) {
66 | builder.append(dumpEventData(data));
67 | }
68 | return builder.toString();
69 | }
70 |
71 | public static String dumpEventData(EventData eventData) {
72 | return MessageFormat.format(eventData_format, eventData.getSchemaName(), eventData.getTableName(),
73 | eventData.getEventType().getValue(), String.valueOf(eventData.getExecuteTime()),
74 | dumpEventColumn(eventData.getKeys()), dumpEventColumn(eventData.getOldKeys()),
75 | dumpEventColumn(eventData.getColumns()), "\t" + eventData.getSql());
76 | }
77 |
78 | private static String dumpEventColumn(List columns) {
79 | StringBuilder builder = new StringBuilder(event_default_capacity);
80 | int size = columns.size();
81 | for (int i = 0; i < size; i++) {
82 | EventColumn column = columns.get(i);
83 | builder.append("\t").append(column.toString());
84 | if (i < columns.size() - 1) {
85 | builder.append(SEP);
86 | }
87 | }
88 | return builder.toString();
89 | }
90 | }
91 |
--------------------------------------------------------------------------------
/src/main/java/com/alibaba/otter/clave/progress/transform/ClaveTransform.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.otter.clave.progress.transform;
2 |
3 | import com.alibaba.otter.clave.common.lifecycle.ClaveLifeCycle;
4 |
5 | public interface ClaveTransform extends ClaveLifeCycle {
6 |
7 | public T transform(T data);
8 | }
9 |
--------------------------------------------------------------------------------
/src/main/java/com/alibaba/otter/clave/utils/ClaveToStringStyle.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.otter.clave.utils;
2 |
3 | import java.text.SimpleDateFormat;
4 | import java.util.Date;
5 |
6 | import org.apache.commons.lang.builder.ToStringStyle;
7 |
8 | /**
9 | * clave项目内部使用的ToStringStyle
10 | *
11 | *
12 | * 默认Style输出格式：
13 | * Person[name=John Doe,age=33,smoker=false ,time=2010-04-01 00:00:00]
14 | *
15 | *
16 | * @author jianghang 2010-6-18 上午11:35:27
17 | */
18 | public class ClaveToStringStyle extends ToStringStyle {
19 |
20 | private static final long serialVersionUID = -6568177374288222145L;
21 |
22 | private static final String DEFAULT_TIME = "yyyy-MM-dd HH:mm:ss";
23 | private static final String DEFAULT_DAY = "yyyy-MM-dd";
24 |
25 | /**
26 | *
27 | * 输出格式：
28 | * Person[name=John Doe,age=33,smoker=false ,time=2010-04-01 00:00:00]
29 | *
30 | */
31 | public static final ToStringStyle TIME_STYLE = new ClaveDateStyle(DEFAULT_TIME);
32 |
33 | /**
34 | *
35 | * 输出格式：
36 | * Person[name=John Doe,age=33,smoker=false ,day=2010-04-01]
37 | *
38 | */
39 | public static final ToStringStyle DAY_STYLE = new ClaveDateStyle(DEFAULT_DAY);
40 |
41 | /**
42 | *
43 | * 输出格式：
44 | * Person[name=John Doe,age=33,smoker=false ,time=2010-04-01 00:00:00]
45 | *
46 | */
47 | public static final ToStringStyle DEFAULT_STYLE = ClaveToStringStyle.TIME_STYLE;
48 |
49 | // =========================== 自定义style =============================
50 |
51 | /**
52 | * 支持日期格式化的ToStringStyle
53 | *
54 | * @author li.jinl
55 | */
56 | private static class ClaveDateStyle extends ToStringStyle {
57 |
58 | private static final long serialVersionUID = 5208917932254652886L;
59 |
60 | // 日期format格式
61 | private String pattern;
62 |
63 | public ClaveDateStyle(String pattern){
64 | super();
65 | this.setUseShortClassName(true);
66 | this.setUseIdentityHashCode(false);
67 | // 设置日期format格式
68 | this.pattern = pattern;
69 | }
70 |
71 | protected void appendDetail(StringBuffer buffer, String fieldName, Object value) {
72 | // 增加自定义的date对象处理
73 | if (value instanceof Date) {
74 | value = new SimpleDateFormat(pattern).format(value);
75 | }
76 | // 后续可以增加其他自定义对象处理
77 | buffer.append(value);
78 | }
79 | }
80 | }
81 |
--------------------------------------------------------------------------------
/src/main/java/com/alibaba/otter/clave/utils/spring/PropertyPlaceholderConfigurer.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.otter.clave.utils.spring;
2 |
3 | import java.util.ArrayList;
4 | import java.util.List;
5 | import java.util.Properties;
6 |
7 | import org.springframework.beans.factory.InitializingBean;
8 | import org.springframework.context.ResourceLoaderAware;
9 | import org.springframework.core.io.Resource;
10 | import org.springframework.core.io.ResourceLoader;
11 | import org.springframework.util.Assert;
12 |
13 | /**
14 | * 扩展Spring的{@linkplain org.springframework.beans.factory.config.PropertyPlaceholderConfigurer} ，增加默认值的功能。
15 | * 例如：${placeholder:defaultValue}，假如placeholder的值不存在，则默认取得 defaultValue。
16 | *
17 | * @author jianghang 2013-1-24 下午03:37:56
18 | * @version 1.0.0
19 | */
20 | public class PropertyPlaceholderConfigurer extends org.springframework.beans.factory.config.PropertyPlaceholderConfigurer implements ResourceLoaderAware, InitializingBean {
21 |
22 | private static final String PLACEHOLDER_PREFIX = "${";
23 | private static final String PLACEHOLDER_SUFFIX = "}";
24 | private ResourceLoader loader;
25 | private String[] locationNames;
26 |
27 | public PropertyPlaceholderConfigurer(){
28 | setIgnoreUnresolvablePlaceholders(true);
29 | }
30 |
31 | public void setResourceLoader(ResourceLoader loader) {
32 | this.loader = loader;
33 | }
34 |
35 | public void setLocationNames(String[] locations) {
36 | this.locationNames = locations;
37 | }
38 |
39 | public void afterPropertiesSet() throws Exception {
40 | Assert.notNull(loader, "no resourceLoader");
41 |
42 | if (locationNames != null) {
43 | for (int i = 0; i < locationNames.length; i++) {
44 | locationNames[i] = resolveSystemPropertyPlaceholders(locationNames[i]);
45 | }
46 | }
47 |
48 | if (locationNames != null) {
49 | List resources = new ArrayList(locationNames.length);
50 |
51 | for (String location : locationNames) {
52 | location = trimToNull(location);
53 |
54 | if (location != null) {
55 | resources.add(loader.getResource(location));
56 | }
57 | }
58 |
59 | super.setLocations(resources.toArray(new Resource[resources.size()]));
60 | }
61 | }
62 |
63 | private String resolveSystemPropertyPlaceholders(String text) {
64 | StringBuilder buf = new StringBuilder(text);
65 |
66 | for (int startIndex = buf.indexOf(PLACEHOLDER_PREFIX); startIndex >= 0;) {
67 | int endIndex = buf.indexOf(PLACEHOLDER_SUFFIX, startIndex + PLACEHOLDER_PREFIX.length());
68 |
69 | if (endIndex != -1) {
70 | String placeholder = buf.substring(startIndex + PLACEHOLDER_PREFIX.length(), endIndex);
71 | int nextIndex = endIndex + PLACEHOLDER_SUFFIX.length();
72 |
73 | try {
74 | String value = resolveSystemPropertyPlaceholder(placeholder);
75 |
76 | if (value != null) {
77 | buf.replace(startIndex, endIndex + PLACEHOLDER_SUFFIX.length(), value);
78 | nextIndex = startIndex + value.length();
79 | } else {
80 | System.err.println("Could not resolve placeholder '"
81 | + placeholder
82 | + "' in ["
83 | + text
84 | + "] as system property: neither system property nor environment variable found");
85 | }
86 | } catch (Throwable ex) {
87 | System.err.println("Could not resolve placeholder '" + placeholder + "' in [" + text
88 | + "] as system property: " + ex);
89 | }
90 |
91 | startIndex = buf.indexOf(PLACEHOLDER_PREFIX, nextIndex);
92 | } else {
93 | startIndex = -1;
94 | }
95 | }
96 |
97 | return buf.toString();
98 | }
99 |
100 | private String resolveSystemPropertyPlaceholder(String placeholder) {
101 | DefaultablePlaceholder dp = new DefaultablePlaceholder(placeholder);
102 | String value = System.getProperty(dp.placeholder);
103 |
104 | if (value == null) {
105 | value = System.getenv(dp.placeholder);
106 | }
107 |
108 | if (value == null) {
109 | value = dp.defaultValue;
110 | }
111 |
112 | return value;
113 | }
114 |
115 | @Override
116 | protected String resolvePlaceholder(String placeholder, Properties props, int systemPropertiesMode) {
117 | DefaultablePlaceholder dp = new DefaultablePlaceholder(placeholder);
118 | String value = super.resolvePlaceholder(dp.placeholder, props, systemPropertiesMode);
119 |
120 | if (value == null) {
121 | value = dp.defaultValue;
122 | }
123 |
124 | return trimToEmpty(value);
125 | }
126 |
127 | private static class DefaultablePlaceholder {
128 |
129 | private final String defaultValue;
130 | private final String placeholder;
131 |
132 | public DefaultablePlaceholder(String placeholder){
133 | int commaIndex = placeholder.indexOf(":");
134 | String defaultValue = null;
135 |
136 | if (commaIndex >= 0) {
137 | defaultValue = trimToEmpty(placeholder.substring(commaIndex + 1));
138 | placeholder = trimToEmpty(placeholder.substring(0, commaIndex));
139 | }
140 |
141 | this.placeholder = placeholder;
142 | this.defaultValue = defaultValue;
143 | }
144 | }
145 |
146 | private String trimToNull(String str) {
147 | if (str == null) {
148 | return null;
149 | }
150 |
151 | String result = str.trim();
152 |
153 | if (result == null || result.length() == 0) {
154 | return null;
155 | }
156 |
157 | return result;
158 | }
159 |
160 | public static String trimToEmpty(String str) {
161 | if (str == null) {
162 | return "";
163 | }
164 |
165 | return str.trim();
166 | }
167 | }
168 |
--------------------------------------------------------------------------------
/src/main/java/com/alibaba/otter/clave/utils/spring/SocketAddressEditor.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.otter.clave.utils.spring;
2 |
3 | import java.beans.PropertyEditorSupport;
4 | import java.net.InetSocketAddress;
5 |
6 | import org.apache.commons.lang.StringUtils;
7 | import org.springframework.beans.PropertyEditorRegistrar;
8 | import org.springframework.beans.PropertyEditorRegistry;
9 |
10 | public class SocketAddressEditor extends PropertyEditorSupport implements PropertyEditorRegistrar {
11 |
12 | public void registerCustomEditors(PropertyEditorRegistry registry) {
13 | registry.registerCustomEditor(InetSocketAddress.class, this);
14 | }
15 |
16 | public void setAsText(String text) throws IllegalArgumentException {
17 | String[] addresses = StringUtils.split(text, ":");
18 | if (addresses.length > 0) {
19 | setValue(new InetSocketAddress(addresses[0], Integer.valueOf(addresses[1])));
20 | } else {
21 | setValue(null);
22 | }
23 | }
24 |
25 | }
26 |
--------------------------------------------------------------------------------
/src/main/resources/clave.properties:
--------------------------------------------------------------------------------
1 | clave.destination=example
2 | clave.zkServers=10.20.144.51:2181
3 | clave.filter=test.ljh_demo
4 | clave.batchSize=100
5 | clave.rowMode=false
6 | clave.poolSize=5
7 | clave.serverId=1
8 | clave.debug=true
9 | clave.skipLoadException=false
10 | clave.db.type=MYSQL
11 | clave.db.url=jdbc:mysql://10.20.144.29:3306
12 | clave.db.driver=com.mysql.jdbc.Driver
13 | #clave.db.driver=oracle.jdbc.driver.OracleDriver
14 | clave.db.username=ottermysql
15 | clave.db.password=ottermysql
16 | clave.db.encode=UTF-8
--------------------------------------------------------------------------------
/src/main/resources/logback.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{56} - %msg%n
6 |
7 |
8 |
9 |
10 |
11 | ../logs/clave.log
12 |
13 |
14 | ../logs/%d{yyyy-MM-dd}/clave-%d{yyyy-MM-dd}-%i.log.gz
15 |
16 |
17 | 512MB
18 |
19 | 60
20 |
21 |
22 |
23 | %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{56} - %msg%n
24 |
25 |
26 |
27 |
28 |
29 | ../logs/select.log
30 |
31 |
32 | ../logs/%d{yyyy-MM-dd}/select-%d{yyyy-MM-dd}-%i.log.gz
33 |
34 |
35 | 512MB
36 |
37 | 60
38 |
39 |
40 | %msg
41 |
42 |
43 |
44 |
45 | ../logs/load.log
46 |
47 |
48 | ../logs/%d{yyyy-MM-dd}/load-%d{yyyy-MM-dd}-%i.log.gz
49 |
50 |
51 | 512MB
52 |
53 | 60
54 |
55 |
56 | %msg
57 |
58 |
59 |
60 |
61 |
62 |
63 |
64 |
65 |
66 |
67 |
68 |
69 |
70 |
71 |
74 |
75 |
76 |
--------------------------------------------------------------------------------
/src/main/resources/spring/clave.xml:
--------------------------------------------------------------------------------
1 |
2 |
12 |
13 |
14 |
15 |
16 |
17 |
18 |
19 | classpath:clave.properties
20 |
21 |
22 |
23 |
24 |
25 |
26 |
27 |
28 |
30 |
31 |
32 |
33 |
34 |
35 |
36 |
37 |
38 |
39 |
40 |
41 |
42 |
43 |
44 |
45 |
46 |
47 |
48 |
49 |
50 |
51 |
52 |
53 |
54 |
55 |
56 |
57 |
58 |
59 |
60 |
61 |
62 |
63 |
64 |
65 |
66 |
67 |
68 |
69 |
70 |
71 |
72 |
73 |
74 |
75 |
76 |
77 |
78 |
79 |
85 |
86 |
87 |
88 |
89 |
90 |
91 |
92 |
93 |
94 |
95 |
96 |
97 |
98 |
99 |
100 |
101 |
102 |
103 |
104 |
105 |
107 |
108 |
109 |
110 |
112 |
113 |
114 |
115 |
116 |
117 |
118 |
119 |
120 |
121 |
122 |
123 |
124 |
125 |
126 |
127 |
128 |
129 |
130 |
131 |
132 |
133 |
145 |
146 |
147 |
148 |
--------------------------------------------------------------------------------
/src/test/java/com/alibaba/otter/clave/progress/select/CanalClientSelectorTest.java:
--------------------------------------------------------------------------------
1 | package com.alibaba.otter.clave.progress.select;
2 |
3 | import org.junit.Test;
4 |
5 | import com.alibaba.otter.clave.model.EventData;
6 | import com.alibaba.otter.clave.progress.select.Message;
7 | import com.alibaba.otter.clave.progress.select.canal.CanalClientSelector;
8 | import com.alibaba.otter.clave.progress.select.canal.MessageParser;
9 |
10 | public class CanalClientSelectorTest {
11 |
12 | @Test
13 | public void testSimple() throws Exception {
14 | CanalClientSelector selector = new CanalClientSelector("example", "10.20.144.51:2181");
15 | MessageParser messageParser = new MessageParser();
16 | selector.setMessageParser(messageParser);
17 | selector.setBatchSize(100);
18 | selector.setFilter("");
19 | selector.start();
20 |
21 | selector.rollback();
22 | int totalEmtryCount = 120;
23 | int emptyCount = 0;
24 | while (emptyCount < totalEmtryCount) {
25 | Message message = selector.selector();
26 | long batchId = message.getId();
27 | int size = message.getDatas().size();
28 | if (batchId == -1 || size == 0) {
29 | emptyCount++;
30 | System.out.println("empty count : " + emptyCount);
31 | try {
32 | Thread.sleep(1000);
33 | } catch (InterruptedException e) {
34 | }
35 | } else {
36 | emptyCount = 0;
37 | System.out.printf("message[batchId=%s,size=%s] \n", batchId, size);
38 | }
39 |
40 | selector.ack(batchId); // 提交确认
41 | // connector.rollback(batchId); // 处理失败, 回滚数据
42 | }
43 |
44 | System.out.println("empty too many times, exit");
45 | selector.stop();
46 | }
47 | }
48 |
--------------------------------------------------------------------------------