filter_columns) {
56 | this.filter_columns = filter_columns;
57 | }
58 | /**
59 | * @decription 数据配置
60 | * @author yi.zhang
61 | * @time 2017年7月13日 下午5:19:28
62 | * @since 1.0
63 | * @jdk 1.8
64 | */
65 | public class Config implements Serializable{
66 | private String servers;
67 | private int port=0;
68 | /**
69 | * 资源类型->
70 | * 0:Canal服务,
71 | * 1:Elasticsearch服务,
72 | * 2:NoSQL服务[2.1->Cassandra,2.2->MongoDB,2.3->Redis,2.4->Memecached]
73 | * 3:SQL服务[3.1->MySQL,3.2->SQL Server,3.3->Oracle]
74 | * 4:数据仓库(Greenplum)
75 | * 5:消息队列(Kafka)
76 | */
77 | private double type = -1;
78 | private String version = "";
79 | private String username = "";
80 | private String password = "";
81 | private String database = "";
82 | private String schema = "";
83 | private String keyspace = "";
84 | private String other = "";
85 | private int batch_size = 100;
86 |
87 | public String getServers() {
88 | return servers;
89 | }
90 | public void setServers(String servers) {
91 | this.servers = servers;
92 | }
93 | public int getPort() {
94 | return port;
95 | }
96 | public void setPort(int port) {
97 | this.port = port;
98 | }
99 | /**
100 | * 资源类型->
101 | * 0:Canal服务,
102 | * 1:Elasticsearch服务,
103 | * 2:NoSQL服务[2.1->Cassandra,2.2->MongoDB,2.3->Redis,2.4->Memecached]
104 | * 3:SQL服务[3.1->MySQL,3.2->SQL Server,3.3->Oracle]
105 | * 4:数据仓库(Greenplum)
106 | * 5:消息队列(Kafka)
107 | */
108 | public double getType() {
109 | return type;
110 | }
111 | /**
112 | * 资源类型->
113 | * 0:Canal服务,
114 | * 1:Elasticsearch服务,
115 | * 2:NoSQL服务[2.1->Cassandra,2.2->MongoDB,2.3->Redis,2.4->Memecached]
116 | * 3:SQL服务[3.1->MySQL,3.2->SQL Server,3.3->Oracle]
117 | * 4:数据仓库(Greenplum)
118 | * 5:消息队列(Kafka)
119 | */
120 | public void setType(double type) {
121 | this.type = type;
122 | }
123 | public String getVersion() {
124 | return version;
125 | }
126 | public void setVersion(String version) {
127 | this.version = version;
128 | }
129 | public String getUsername() {
130 | return username;
131 | }
132 | public void setUsername(String username) {
133 | this.username = username;
134 | }
135 | public String getPassword() {
136 | return password;
137 | }
138 | public void setPassword(String password) {
139 | this.password = password;
140 | }
141 | public String getDatabase() {
142 | return database;
143 | }
144 | public void setDatabase(String database) {
145 | this.database = database;
146 | }
147 | public String getSchema() {
148 | return schema;
149 | }
150 | public void setSchema(String schema) {
151 | this.schema = schema;
152 | }
153 | public String getKeyspace() {
154 | return keyspace;
155 | }
156 | public void setKeyspace(String keyspace) {
157 | this.keyspace = keyspace;
158 | }
159 | public String getOther() {
160 | return other;
161 | }
162 | public void setOther(String other) {
163 | this.other = other;
164 | }
165 | public int getBatch_size() {
166 | return batch_size;
167 | }
168 | public void setBatch_size(int batch_size) {
169 | this.batch_size = batch_size;
170 | }
171 | }
172 | }
173 |
--------------------------------------------------------------------------------
/src/main/java/com/share/swing/Main.java:
--------------------------------------------------------------------------------
1 | package com.share.swing;
2 |
3 | import java.awt.Color;
4 | import java.awt.Font;
5 | import java.awt.event.ActionEvent;
6 | import java.awt.event.ActionListener;
7 | import java.util.Locale;
8 |
9 | import javax.swing.ButtonGroup;
10 | import javax.swing.DropMode;
11 | import javax.swing.JButton;
12 | import javax.swing.JFrame;
13 | import javax.swing.JLabel;
14 | import javax.swing.JRadioButton;
15 | import javax.swing.JTextPane;
16 | import javax.swing.SwingConstants;
17 |
18 | public class Main extends CommonFrame {
19 | private static final long serialVersionUID = 8927006591622687297L;
20 |
21 | public Main() {
22 | super._init();
23 | language(Locale.CHINESE);
24 | }
25 |
26 | protected void init(){
27 | this.setTitle("®™"+title+" "+ version +" ©ZhangYi");
28 | container = this.getContentPane();
29 | container.setForeground(Color.BLACK);
30 | container.setBackground(new Color(204, 255, 255));
31 | container.setFont(new Font("仿宋", Font.PLAIN, 12));
32 | container.setLayout(null);
33 | int cwidth = container.getSize().width>0?container.getSize().width:width;
34 | int cheight = container.getSize().height>0?container.getSize().height:height;
35 |
36 | JLabel jtitle = new JLabel(title);
37 | jtitle.setLabelFor(getContentPane());
38 | jtitle.setToolTipText(decription);
39 | jtitle.setHorizontalAlignment(SwingConstants.CENTER);
40 | jtitle.setForeground(Color.RED);
41 | jtitle.setBackground(Color.WHITE);
42 | //为标签设置及添加到框架
43 | jtitle.setBounds(cwidth*3/8,98,cwidth/4,81);
44 | jtitle.setFont(new Font("华文新魏", Font.BOLD, 30));
45 | container.add(jtitle);
46 |
47 | JTextPane textPane = new JTextPane();
48 | textPane.setDropMode(DropMode.USE_SELECTION);
49 | textPane.setForeground(Color.BLACK);
50 | textPane.setBackground(new Color(245, 255, 250));
51 | textPane.setEditable(false);
52 | textPane.setFont(new Font("华文宋体", Font.BOLD, 16));
53 | textPane.setText(decription.replace("\n", "\n\n"));
54 | textPane.setBounds(cwidth*5/16, 200, cwidth*3/8, cheight*3/8);
55 | container.add(textPane);
56 |
57 | JButton button = new JButton(ResourceHolder.getProperty("dstt.btn.next"));
58 | button.setFont(new Font("宋体", Font.BOLD, 14));
59 | button.addActionListener(new ActionListener() {
60 | public void actionPerformed(ActionEvent e) {
61 | setVisible(false);
62 | new ManageTable();
63 | }
64 | });
65 | button.setBounds(cwidth*7/8, cheight*15/16, 93, 30);
66 | container.add(button);
67 |
68 | final ButtonGroup language = new ButtonGroup();
69 |
70 | final JRadioButton chinese = new JRadioButton(ResourceHolder.getProperty("system.language.zh_CN"));
71 | chinese.setBackground(new Color(224, 255, 255));
72 | chinese.setHorizontalAlignment(SwingConstants.CENTER);
73 | chinese.setFont(new Font("仿宋", Font.BOLD, 12));
74 | chinese.setBounds(cwidth*14/16, 16, 75, 23);
75 | language.add(chinese);
76 | final JRadioButton english = new JRadioButton(ResourceHolder.getProperty("system.language.en_US"));
77 | english.setBackground(new Color(224, 255, 255));
78 | english.setHorizontalAlignment(SwingConstants.CENTER);
79 | english.setFont(new Font("仿宋", Font.BOLD, 12));
80 | english.setBounds(cwidth*15/16, 16, 75, 23);
81 | language.add(english);
82 | chinese.addActionListener(new ActionListener() {
83 | public void actionPerformed(ActionEvent e) {
84 | language.clearSelection();
85 | language(Locale.CHINESE);
86 | }
87 | });
88 | english.addActionListener(new ActionListener() {
89 | public void actionPerformed(ActionEvent e) {
90 | language.clearSelection();
91 | language(Locale.ENGLISH);
92 | }
93 | });
94 | if(this.getLocale().equals(Locale.ENGLISH)){
95 | english.setSelected(true);
96 | }else{
97 | chinese.setSelected(true);
98 | }
99 | container.add(chinese);
100 | container.add(english);
101 | }
102 |
103 | public void language(Locale locale){
104 | if(locale==null)locale=Locale.getDefault();
105 | ResourceHolder.locale=locale;
106 | this.setLocale(locale);
107 | title = ResourceHolder.getProperty("system.name");
108 | version = ResourceHolder.getProperty("system.version");
109 | decription = ResourceHolder.getProperty("system.decription");
110 | this.getContentPane().removeAll();
111 | init();
112 |
113 | this.setVisible(true);
114 | this.setResizable(false);
115 | this.validate();
116 | this.repaint();
117 | }
118 | public static void main(String[] args) {
119 | JFrame frame = new Main();
120 | frame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);
121 | frame.setVisible(true);
122 | }
123 | }
124 |
--------------------------------------------------------------------------------
/src/main/java/com/share/swing/ResourceHolder.java:
--------------------------------------------------------------------------------
1 | package com.share.swing;
2 |
3 | import java.io.IOException;
4 | import java.io.InputStream;
5 | import java.util.Locale;
6 | import java.util.Properties;
7 |
8 | import org.apache.log4j.Logger;
9 |
10 | import com.share.util.StringUtil;
11 |
12 |
13 | /**
14 | *
15 | * 项目:数据转换
16 | * 描述:资源配置文件取值
17 | * 作者:ZhangYi
18 | * 时间:2016年1月26日 上午9:54:00
19 | * 版本:dtt_v1.0
20 | * JDK:1.7.80
21 | *
22 | */
23 | public class ResourceHolder {
24 |
25 | private static final Logger logger = Logger.getLogger(ResourceHolder.class);
26 | public static Locale locale = Locale.getDefault();
27 | /**
28 | *
29 | * 描述:获取资源文件值(messages_en.properties/messages_zh.properties)
30 | * 作者:ZhangYi
31 | * 时间:2016年1月26日 上午9:52:03
32 | * 参数：(参数列表)
33 | * @param key 国际化key
34 | * @param language (ZH:中文,EN:英文)
35 | * @return
36 | *
37 | */
38 | public static String getProperty(String key) {
39 | String value = "";
40 | String file = "/messages_zh_CN.properties";
41 | if (locale.getLanguage().contains("en")) {
42 | file = "/messages_en.properties";
43 | }
44 | try {
45 | InputStream stream = ResourceHolder.class.getResourceAsStream(file);
46 | Properties properties = new Properties();
47 | properties.load(stream);
48 | value = properties.getProperty(key);
49 | } catch (IOException e) {
50 | logger.error("--资源文件取值失败--", e);
51 | }
52 | return value;
53 | }
54 |
55 | /**
56 | *
57 | * 描述:获取本地语言
58 | * 作者:ZhangYi
59 | * 时间:2015年1月30日 下午1:24:06
60 | * 参数：(参数列表)
61 | * @param language
62 | * @return
63 | *
64 | */
65 | public static Locale getLocale(String language) {
66 | Locale locale = Locale.CHINESE;
67 | if (!StringUtil.isEmpty(language)) {
68 | language = language.toLowerCase();
69 | if (language.indexOf("en") != -1) {
70 | locale = Locale.ENGLISH;
71 | }
72 | if (language.indexOf("zh_hk") != -1) {
73 | locale = Locale.TRADITIONAL_CHINESE;
74 | }
75 | if (language.indexOf("zh_cn") != -1) {
76 | locale = Locale.SIMPLIFIED_CHINESE;
77 | }
78 | }
79 | return locale;
80 | }
81 | }
82 |
--------------------------------------------------------------------------------
/src/main/java/com/share/util/ClazzUtil.java:
--------------------------------------------------------------------------------
1 | package com.share.util;
2 |
3 | import java.lang.reflect.Field;
4 | import java.lang.reflect.Method;
5 | import java.util.Date;
6 | import java.util.LinkedHashMap;
7 | import java.util.Map;
8 |
9 | public class ClazzUtil {
10 |
11 | public Map reflect(Object obj){
12 | Map map = new LinkedHashMap();
13 | Class clazz = obj.getClass();
14 | Field[] fields = clazz.getDeclaredFields();
15 | for (Field field : fields) {
16 | String name = field.getName();
17 | if(name.equalsIgnoreCase("serialVersionUID")||name.contains("$this")){
18 | continue;
19 | }
20 | String type = field.getType().getSimpleName();
21 | try {
22 | Method method = clazz.getMethod((type.equalsIgnoreCase("boolean")?"is":"get")+name.substring(0, 1).toUpperCase()+ name.substring(1));
23 | Object value = method.invoke(obj);
24 | if(type.equalsIgnoreCase("String")||type.equalsIgnoreCase("Date")){
25 | if(type.equalsIgnoreCase("Date")){
26 | value = DateUtil.formatDateTimeStr((Date)value);
27 | }
28 | value = value.toString();
29 | }
30 | map.put(name, value);
31 | }catch (Exception e) {
32 | e.printStackTrace();
33 | }
34 | }
35 | return map;
36 | }
37 | }
38 |
--------------------------------------------------------------------------------
/src/main/java/com/share/util/HttpUtil.java:
--------------------------------------------------------------------------------
1 | package com.share.util;
2 |
3 | import java.io.ByteArrayOutputStream;
4 | import java.io.DataOutputStream;
5 | import java.io.InputStream;
6 | import java.io.UnsupportedEncodingException;
7 | import java.net.Authenticator;
8 | import java.net.HttpURLConnection;
9 | import java.net.PasswordAuthentication;
10 | import java.net.URL;
11 | import java.net.URLDecoder;
12 | import java.net.URLEncoder;
13 |
14 | import org.apache.commons.codec.binary.Base64;
15 | import org.apache.http.Consts;
16 | import org.apache.http.HttpEntity;
17 | import org.apache.http.HttpResponse;
18 | import org.apache.http.client.methods.HttpDelete;
19 | import org.apache.http.client.methods.HttpGet;
20 | import org.apache.http.client.methods.HttpHead;
21 | import org.apache.http.client.methods.HttpOptions;
22 | import org.apache.http.client.methods.HttpPost;
23 | import org.apache.http.client.methods.HttpPut;
24 | import org.apache.http.client.methods.HttpRequestBase;
25 | import org.apache.http.client.methods.HttpTrace;
26 | import org.apache.http.client.utils.HttpClientUtils;
27 | import org.apache.http.entity.ContentType;
28 | import org.apache.http.entity.StringEntity;
29 | import org.apache.http.impl.client.CloseableHttpClient;
30 | import org.apache.http.impl.client.HttpClients;
31 | import org.apache.http.util.EntityUtils;
32 | import org.apache.logging.log4j.LogManager;
33 | import org.apache.logging.log4j.Logger;
34 |
35 | public class HttpUtil {
36 | private static Logger logger = LogManager.getLogger(HttpUtil.class);
37 | /**
38 | * http客户端
39 | */
40 | private static CloseableHttpClient httpClient = HttpClients.createDefault();
41 | /**
42 | * Get请求
43 | */
44 | public final static String METHOD_GET = "GET";
45 | /**
46 | * Post请求
47 | */
48 | public final static String METHOD_POST = "POST";
49 | /**
50 | * Head请求
51 | */
52 | public final static String METHOD_HEAD = "HEAD";
53 | /**
54 | * Options请求
55 | */
56 | public final static String METHOD_OPTIONS = "OPTIONS";
57 | /**
58 | * Put请求
59 | */
60 | public final static String METHOD_PUT = "PUT";
61 | /**
62 | * Delete请求
63 | */
64 | public final static String METHOD_DELETE = "DELETE";
65 | /**
66 | * Trace请求
67 | */
68 | public final static String METHOD_TRACE = "TRACE";
69 | /**
70 | * @param proxyHost 代理地址
71 | * @param port 代理端口
72 | * @param account 认证账号
73 | * @param password 认证密码
74 | */
75 | public static void auth(String proxyHost,int port,final String account,final String password){
76 | System.setProperty("https.proxyHost", proxyHost);
77 | System.setProperty("https.proxyPort", port+"");
78 | Authenticator.setDefault(new Authenticator() {
79 | protected PasswordAuthentication getPasswordAuthentication(){
80 | return new PasswordAuthentication(account, new String(password).toCharArray());
81 | }
82 | });
83 | }
84 | /**
85 | * @description 判断服务连通性
86 | * @author yi.zhang
87 | * @time 2017年4月19日 下午6:00:40
88 | * @param url
89 | * @param auth 认证信息(username+":"+password)
90 | * @return (true:连接成功,false:连接失败)
91 | */
92 | public static boolean checkConnection(String url,String auth){
93 | boolean flag = false;
94 | try {
95 | HttpURLConnection connection = (HttpURLConnection)new URL(url).openConnection();
96 | connection.setConnectTimeout(5*1000);
97 | if(auth!=null&&!"".equals(auth)){
98 | String authorization = "Basic "+new String(Base64.encodeBase64(auth.getBytes()));
99 | connection.setRequestProperty("Authorization", authorization);
100 | }
101 | connection.connect();
102 | if(connection.getResponseCode()==HttpURLConnection.HTTP_OK){
103 | flag = true;
104 | }
105 | connection.disconnect();
106 | }catch (Exception e) {
107 | logger.error("--Server Connect Error !",e);
108 | }
109 | return flag;
110 | }
111 | /**
112 | * @param url 请求URL
113 | * @param method 请求URL
114 | * @param param json参数(post|put)
115 | * @param auth 认证信息(username+":"+password)
116 | * @return
117 | */
118 | public static String httpRequest(String url,String method,String param,String auth){
119 | String result = null;
120 | HttpResponse httpResponse = null;
121 | try {
122 | HttpRequestBase http = new HttpGet(url);
123 | if(method.equalsIgnoreCase(METHOD_POST)){
124 | http = new HttpPost(url);
125 | StringEntity body = new StringEntity(param,ContentType.APPLICATION_JSON);
126 | body.setContentType("application/json");
127 | ((HttpPost)http).setEntity(body);
128 | }else if(method.equalsIgnoreCase(METHOD_PUT)){
129 | http = new HttpPut(url);
130 | StringEntity body = new StringEntity(param,ContentType.APPLICATION_JSON);
131 | body.setContentType("application/json");
132 | ((HttpPut)http).setEntity(body);
133 | }else if(method.equalsIgnoreCase(METHOD_DELETE)){
134 | http = new HttpDelete(url);
135 | }else if(method.equalsIgnoreCase(METHOD_HEAD)){
136 | http = new HttpHead(url);
137 | }else if(method.equalsIgnoreCase(METHOD_OPTIONS)){
138 | http = new HttpOptions(url);
139 | }else if(method.equalsIgnoreCase(METHOD_TRACE)){
140 | http = new HttpTrace(url);
141 | }
142 | if(auth!=null&&!"".equals(auth)){
143 | String authorization = "Basic "+new String(Base64.encodeBase64(auth.getBytes()));
144 | http.setHeader("Authorization", authorization);
145 | }
146 | httpResponse = httpClient.execute(http);
147 | HttpEntity entity = httpResponse.getEntity();
148 | result = EntityUtils.toString(entity,Consts.UTF_8);
149 | }catch (Exception e) {
150 | logger.error("--http request error !",e);
151 | result = e.getMessage();
152 | }finally {
153 | HttpClientUtils.closeQuietly(httpResponse);
154 | }
155 | return result;
156 | }
157 | /**
158 | * @param url 请求URL
159 | * @param method 请求URL
160 | * @param param json参数(post|put)
161 | * @return
162 | */
163 | public static String urlRequest(String url,String method,String param,String auth){
164 | String result = null;
165 | try {
166 | HttpURLConnection connection = (HttpURLConnection)new URL(url).openConnection();
167 | connection.setConnectTimeout(60*1000);
168 | connection.setRequestMethod(method.toUpperCase());
169 | if(auth!=null&&!"".equals(auth)){
170 | String authorization = "Basic "+new String(Base64.encodeBase64(auth.getBytes()));
171 | connection.setRequestProperty("Authorization", authorization);
172 | }
173 | if(param!=null&&!"".equals(param)){
174 | connection.setDoInput(true);
175 | connection.setDoOutput(true);
176 | connection.connect();
177 | DataOutputStream dos = new DataOutputStream(connection.getOutputStream());
178 | dos.write(param.getBytes(Consts.UTF_8));
179 | dos.flush();
180 | dos.close();
181 | }else{
182 | connection.connect();
183 | }
184 | if(connection.getResponseCode()==HttpURLConnection.HTTP_OK){
185 | InputStream in = connection.getInputStream();
186 | ByteArrayOutputStream out = new ByteArrayOutputStream();
187 | byte[] buff = new byte[1024];
188 | int len = 0;
189 | while((len=in.read(buff, 0, buff.length))>0){
190 | out.write(buff, 0, len);
191 | }
192 | byte[] data = out.toByteArray();
193 | in.close();
194 | result = data!=null&&data.length>0?new String(data, Consts.UTF_8):null;
195 | }else{
196 | result = "{\"status\":"+connection.getResponseCode()+",\"msg\":\""+connection.getResponseMessage()+"\"}";
197 | }
198 | connection.disconnect();
199 | }catch (Exception e) {
200 | logger.error("--http request error !",e);
201 | }
202 | return result;
203 | }
204 | /**
205 | * @decription URL编码
206 | * @author yi.zhang
207 | * @time 2017年9月15日 下午3:33:38
208 | * @param target
209 | * @return
210 | */
211 | public static String encode(String target){
212 | String result = target;
213 | try {
214 | result = URLEncoder.encode(target, Consts.UTF_8.name());
215 | } catch (UnsupportedEncodingException e) {
216 | logger.error("--http encode error !",e);
217 | }
218 | return result;
219 | }
220 | /**
221 | * @decription URL解码
222 | * @author yi.zhang
223 | * @time 2017年9月15日 下午3:33:38
224 | * @param target
225 | * @return
226 | */
227 | public static String decode(String target){
228 | String result = target;
229 | try {
230 | result = URLDecoder.decode(target, Consts.UTF_8.name());
231 | } catch (UnsupportedEncodingException e) {
232 | logger.error("--http decode error !",e);
233 | }
234 | return result;
235 | }
236 |
237 | public static void main(String[] args) {
238 | String index = "testlog";
239 | String type = "servicelog";
240 | String id = "";
241 | String url = "http://127.0.0.1:9200/"+index+"/"+type;
242 | if(!"".equals(id)){
243 | url=url+"/"+id;
244 | }else{
245 | // url=url+"/_search";
246 | }
247 | String method = "post";
248 | // String body = "{\"query\":{\"match\":{\"operator\":\"test\"}}}";
249 | String body = "{\"name\":\"mobile music\",\"operator\":\"10000\",\"content\":\"I like music!\",\"createTime\":\"2017-04-20\"}";
250 | String result = null;
251 | String auth="elastic:elastic";
252 | result = checkConnection("http://127.0.0.1:9200",auth)+"";
253 | result = httpRequest(url, method, body,null);
254 | System.out.println(result);
255 | System.out.println("---------------------------------------------------------");
256 | // result = urlRequest(url, method, param);
257 | System.out.println(result);
258 | }
259 | }
260 |
261 |
--------------------------------------------------------------------------------
/src/main/java/com/share/util/NumberUtil.java:
--------------------------------------------------------------------------------
1 | package com.share.util;
2 |
3 | public class NumberUtil {
4 | public static boolean isNumber(String target){
5 | if(target==null||"".equals(target)||"null".equals(target)){
6 | return false;
7 | }
8 | for(int i=0;i='0'&&c<='9')){
11 | return false;
12 | }
13 | }
14 | return true;
15 | }
16 | }
17 |
--------------------------------------------------------------------------------
/src/main/java/com/share/util/StringUtil.java:
--------------------------------------------------------------------------------
1 | package com.share.util;
2 |
3 | public class StringUtil {
4 | public static boolean isEmpty(String target){
5 | if(target==null||"".equals(target)||"null".equals(target)){
6 | return true;
7 | }
8 | return false;
9 | }
10 | }
11 |
--------------------------------------------------------------------------------
/src/main/resources/bin/start.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | path="${BASH_SOURCE-$0}"
3 | path="$(dirname "${path}")"
4 | path="$(cd "${path}";pwd)"
5 | base=${path}/..
6 | base_path="$(cd "${base}";pwd)"
7 |
8 | app_name=database-transform-tool
9 | conf=${base_path}/config/config.properties
10 | log=${base_path}/logs/${app_name}.log
11 | pid=${base_path}/data/${app_name}.pid
12 | if [ -f $pid ] ; then
13 | echo "please run stop.sh first,then start.sh" 2>$2
14 | exit 1;
15 | fi
16 |
17 | if [ -n "${app_name}" ] ; then
18 | kid = `ps -ef |grep ${app_name}|grep -v grep|awk '{print $2}'`
19 | echo pid[$kid] from `uname` system process!
20 | fi
21 |
22 | if [ -n $kid ] ; then
23 | echo [`uname`] ${app_name} process [$kid] is Running!
24 | exit 1;
25 | fi
26 |
27 | if [ -f $log ] ; then
28 | rm -rf ${base_path}/logs/*
29 | rm -rf $pid
30 | fi
31 |
32 | if [ ! -d ${base_path}/logs ] ; then
33 | mkdir -p ${base_path}/logs
34 | fi
35 |
36 | if [ ! -d ${base_path}/data ] ; then
37 | mkdir -p ${base_path}/data
38 | fi
39 |
40 | if [ "$JAVA_HOME" != "" ]; then
41 | JAVA="$JAVA_HOME/bin/java"
42 | else
43 | JAVA=java
44 | fi
45 | JAVA_ENV="-server -Xms2g -Xmx2g -Xss1m "
46 | JAVA_OPTS="$JAVA_ENV -XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=75 -XX:+UseCMSInitiatingOccupancyOnly -XX:+AlwaysPreTouch -Djava.awt.headless=true -Dfile.encoding=UTF-8 -Djna.nosys=true -Djdk.io.permissionsUseCanonicalPath=true -Dio.netty.noUnsafe=true -Dio.netty.noKeySetOptimization=true -Dio.netty.recycler.maxCapacityPerThread=0 -Dlog4j.shutdownHookEnabled=false -Dlog4j2.disable.jmx=true -Dlog4j.skipJansi=true -XX:+HeapDumpOnOutOfMemoryError "
47 |
48 | for i in "${base_path}"/lib/*.jar
49 | do
50 | CLASSPATH="$i:$CLASSPATH"
51 | done
52 |
53 | if [ -e $conf -a -d ${base_path}/logs ]
54 | then
55 | echo -------------------------------------------------------------------------------------------
56 | cd ${base_path}
57 |
58 | for file in "${base_path}"/*.jar
59 | do
60 | file=${file##*/}
61 | filename=${file%.*}
62 | echo -----------------file=${file},filename=${filename}------------------
63 | if [[ $filename =~ $app_name ]]; then
64 | app=$file
65 | echo app jar:$app
66 | break;
67 | fi
68 | done
69 |
70 | echo ${app_name} Starting ...
71 | $JAVA $JAVA_OPTS -classpath=.:$CLASSPATH -cp $app:"${base_path}"/lib/*.jar com.ucloudlink.css.Application -spring.config.location=$conf -base.path=${base_path} >$log 2>&1 &
72 | echo $! > $pid
73 |
74 | kid = `ps -ef |grep ${app_name}|grep -v grep|awk '{print $2}'`
75 | if [ -n $kid ] ; then
76 | echo ----------------------------${app_name} STARTED SUCCESS------------------------------------
77 | else
78 | echo ----------------------------${app_name} STARTED ERROR------------------------------------
79 | fi
80 | echo -------------------------------------------------------------------------------------------
81 | else
82 | echo "${app_name} config($conf) Or logs direction is not exist,please create first!"
83 | rm -rf $pid
84 | fi
--------------------------------------------------------------------------------
/src/main/resources/bin/stop.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | path="${BASH_SOURCE-$0}"
3 | path="$(dirname "${path}")"
4 | path="$(cd "${path}";pwd)"
5 | base=${path}/..
6 | base_path="$(cd "${base}";pwd)"
7 |
8 | app_name=database-transform-tool
9 | conf=${base_path}/config/application.properties
10 | log=${base_path}/logs/${app_name}.log
11 | pid=${base_path}/data/${app_name}.pid
12 |
13 | if [ -n "${app_name}" ] ; then
14 | kid=`ps -ef |grep ${app_name}|grep -v grep|awk '{print $2}'`
15 | echo pid[$kid] from `uname` system process!
16 | fi
17 |
18 | if [ -z "$kid" -a -e "$pid" ] ; then
19 | chmod +x $pid
20 | kid=`cat $pid`
21 | echo pid[$kid] from pid file!
22 | fi
23 |
24 | if [ -n "${kid}" ];
25 | then
26 | echo ${app_name} pid:${kid}
27 | kill -9 ${kid}
28 | echo ----------------------------${app_name} STOPED SUCCESS------------------------------------
29 | else
30 | echo ${app_name} pid isn't exist or has stoped !
31 | fi
32 |
33 | if [ -f $pid ]; then
34 | rm -rf $pid
35 | echo "If there is a problem, Please check the log!"
36 | fi
--------------------------------------------------------------------------------
/src/main/resources/canal.properties:
--------------------------------------------------------------------------------
1 | #########################################################################################
2 | #################################### Canal Config #################################
3 | #########################################################################################
4 | #\u670d\u52a1\u5668\u5b9e\u4f8b\u5217\u8868(\u591a\u5b9e\u4f8b\u4ee5','\u5206\u5272)
5 | canal.destinations=example
6 | canal.servers=127.0.0.1:11111
7 | canal.username=canal
8 | canal.password=canal
9 | canal.batch_size=1000
10 | #\u76d1\u63a7\u8fc7\u6ee4\u89c4\u5219(\u9ed8\u8ba4\u6240\u6709)
11 | canal.filter_regex=.*\\..*
12 | #\u662f\u5426\u4f7f\u7528zookeeper\u5730\u5740\u8bbf\u95ee\u65b9\u5f0f
13 | canal.zookeeper.enabled=false
14 | #\u662f\u5426\u542f\u7528\u6570\u636e\u8f6c\u5316\u65b9\u5f0f
15 | canal.transform.enabled=false
16 | #\u6570\u636e\u8f6c\u5316\u5b58\u50a8\u7c7b\u578b:elasticsearch|cassandra|jdbc(MySQL|SQL Server|Oracle)|greenplum
17 | canal.transform.type=elasticsearch
--------------------------------------------------------------------------------
/src/main/resources/config.properties:
--------------------------------------------------------------------------------
1 | #########################################################################################
2 | ########## Transform Config(\u6570\u636e\u8f6c\u5b58)-->transform_enabled=true #########################
3 | #########################################################################################
4 | #1.Elasticsearch\u914d\u7f6e[transform_type=elasticsearch](\u652f\u6301\u96c6\u7fa4\u914d\u7f6e,\u591a\u670d\u52a1\u4ee5','\u5206\u5272)
5 | elasticsearch.cluster.name=elasticsearch
6 | elasticsearch.cluster.servers=localhost:9300
7 | elasticsearch.cluster.username=elastic
8 | elasticsearch.cluster.password=elastic
9 | #2.Cassandra\u914d\u7f6e[transform_type=cassandra](\u652f\u6301\u96c6\u7fa4\u914d\u7f6e,\u591a\u670d\u52a1\u4ee5','\u5206\u5272)
10 | cassandra.servers=localhost:9402
11 | cassandra.keyspace=css_analyse
12 | cassandra.username=cassandra
13 | cassandra.password=cassandra
14 | #3.MongoDB\u914d\u7f6e[transform_type=mongodb](\u652f\u6301\u96c6\u7fa4\u914d\u7f6e,\u591a\u670d\u52a1\u4ee5','\u5206\u5272)
15 | mongodb.servers=localhost:27017
16 | mongodb.database=admin
17 | mongodb.schema=cdr
18 | mongodb.username=root
19 | mongodb.password=root
20 | #4.Greenplum\u914d\u7f6e[transform_type=greenplum](\u6570\u636e\u4ed3\u5e93)
21 | greenplum.address=localhost:5432
22 | greenplum.database=bitest
23 | greenplum.schema=cdr
24 | greenplum.username=bitest
25 | greenplum.password=123456
26 | #5.\u6570\u636e\u5e93\u914d\u7f6e[transform_type=jdbc](\u652f\u6301[MySQL|SQL Server|Oracle]\u7b49\u6570\u636e\u5e93)
27 | jdbc.driver=com.mysql.jdbc.Driver
28 | jdbc.url=jdbc:mysql://localhost:3306/cdr?useUnicode=true&characterEncoding=UTF8
29 | jdbc.username=root
30 | jdbc.password=root
31 | jdbc.druid.enabled=true
32 | jdbc.druid.max_pool_size=100
33 | jdbc.druid.init_pool_size=10
34 | #########################################################################################
35 | #(\u652f\u6301\u96c6\u7fa4\u914d\u7f6e,\u591a\u670d\u52a1\u4ee5','\u5206\u5272)
36 | kafka.servers=127.0.0.1:9092
37 | #\u662f\u5426\u4f7f\u7528zookeeper\u5730\u5740\u8bbf\u95ee\u65b9\u5f0f
38 | kafka.zookeeper.enabled=false
39 | kafka.zookeeper.servers=127.0.0.1:2181
40 | #\u751f\u4ea7\u7aef\u6301\u4e45\u8bbe\u7f6e
41 | kafka.productor.acks=all
42 | #\u6d88\u8d39\u7aef\u6279\u91cf\u83b7\u53d6\u6570\u91cf
43 | kafka.consumer.batch_size=1000
--------------------------------------------------------------------------------
/src/main/resources/log4j2.properties:
--------------------------------------------------------------------------------
1 | appenders = console,rolling
2 |
3 | appender.console.type = Console
4 | appender.console.name = System
5 | appender.console.target = SYSTEM_OUT
6 | appender.console.layout.type = PatternLayout
7 | appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss}|%-5p|[%c{1}(%L)-->%M] -- %m%n
8 |
9 | rootLogger.level = info
10 | rootLogger.appenderRefs= console
11 | rootLogger.appenderRef.console.ref = System
12 |
13 | appender.rolling.type = RollingFile
14 | appender.rolling.name = LogFile
15 | appender.rolling.append = false
16 | appender.rolling.layout.type = PatternLayout
17 | appender.rolling.layout.pattern = %d{yyyy-MM-dd HH:mm:ss}|%-5p|[%c{1}(%L)-->%M] -- %m%n
18 | appender.rolling.fileName = target/logs/sys.log
19 | appender.rolling.filePattern = target/logs/log_[%d{yyyy-MM-dd-HH-mm}]-%i.log.gz
20 | appender.rolling.policies.type = Policies
21 | appender.rolling.policies.time.type = TimeBasedTriggeringPolicy
22 | appender.rolling.policies.time.interval = 24
23 | appender.rolling.policies.time.modulate = true
24 | appender.rolling.policies.size.type = SizeBasedTriggeringPolicy
25 | appender.rolling.policies.size.size = 10MB
26 | appender.rolling.strategy.type = DefaultRolloverStrategy
27 | appender.rolling.strategy.max = 5
28 |
29 | loggers=rolling,other
30 | logger.rolling.name=com.ucloudlink
31 | logger.rolling.level=info
32 | logger.rolling.additivity=true
33 | logger.rolling.appenderRefs= rolling
34 | logger.rolling.appenderRef.rolling.ref = LogFile
35 | logger.other.name=org
36 | logger.other.level=warn
37 | logger.other.additivity=true
38 | logger.other.appenderRefs= other
39 | logger.other.appenderRef.other.ref = LogFile
--------------------------------------------------------------------------------
/src/main/resources/messages_en.properties:
--------------------------------------------------------------------------------
1 | system.language.zh_CN=Chinese
2 | system.language.en_US=English
3 | system.name=DataSource Transform Tool
4 | system.version=v1.0
5 | system.decription=Software supports DataSource:\n1.Canal Monitor(MySQL) Syn\n2. Data Syn Transform: \n1)SQL DataSource (MySQL|SQL Server|Oracle) \n2)NoSQL DataSource (Cassandra|MongoDB) \n3)Search Engine (Elasticsearch) \n4)Message Queue (Kafka) etc.
6 | ###################################################################################################################
7 | dstt.btn.continue=Continue
8 | dstt.btn.prev=Prev
9 | dstt.btn.next=Next
10 | dstt.btn.back=Back
11 | dstt.btn.open=Open
12 | dstt.btn.close=Close
13 | dstt.btn.enabled=Enabled
14 | dstt.btn.disabled=Disabled
15 | dstt.btn.submit=Submit
16 | dstt.btn.add=Add
17 | dstt.btn.update=Update
18 | dstt.btn.cancel=Cancel
19 | dstt.btn.remove=Remove
20 | dstt.btn.refresh=Refresh
21 | dstt.btn.syn=Syn
22 | dstt.btn.synall=Syn All
23 |
24 | dstt.service.canal=Canal Monitor
25 | dstt.service.datasyn=Data Syn
26 | dstt.service.datalog=Data Log
27 |
28 | dstt.ds.source=Source
29 | dstt.ds.target=Target
30 |
31 | dstt.ds.seq=No.
32 | dstt.ds.saddress=Source Adress
33 | dstt.ds.stype=Source Type
34 | dstt.ds.taddress=Target Address
35 | dstt.ds.ttype=Target Type
36 | dstt.ds.action=Action
37 |
38 | dstt.ds.type=Service Type
39 | dstt.ds.type.0=Canal Service
40 | dstt.ds.type.0.1=Zookeeper Service
41 | dstt.ds.type.1=Elasticsearch Service
42 | dstt.ds.type.2=NoSQL Service
43 | dstt.ds.type.2.1=Cassandra
44 | dstt.ds.type.2.2=MongoDB
45 | dstt.ds.type.2.3=Redis
46 | dstt.ds.type.2.4=Memecached
47 | dstt.ds.type.3=SQL Service
48 | dstt.ds.type.3.1=MySQL
49 | dstt.ds.type.3.2=SQL\u3000Server
50 | dstt.ds.type.3.3=Oracle
51 | dstt.ds.type.4=Data Store
52 | dstt.ds.type.4.1=Greenplum
53 | dstt.ds.type.5=Message Queue
54 | dstt.ds.type.5.1=Kafka
55 |
56 | dstt.ds.servers=Servers
57 | dstt.ds.servers.tip=(Support cluster and multi service use ',' split)
58 | dstt.ds.port=Port
59 | dstt.ds.username=Username
60 | dstt.ds.password=Password
61 | dstt.ds.database=Database
62 | dstt.ds.schema=Schema
63 | dstt.ds.keyspace=Keyspace
64 | dstt.ds.mapping=Table Mapping
65 | dstt.ds.filter=Filter Columns
66 | dstt.ds.filter.tip=(Multi columns use ',' split)
--------------------------------------------------------------------------------
/src/main/resources/messages_zh_CN.properties:
--------------------------------------------------------------------------------
1 | system.language.zh_CN=\u4e2d\u6587
2 | system.language.en_US=\u82f1\u6587
3 | system.name=\u6570\u636e\u6e90\u8f6c\u6362\u5de5\u5177
4 | system.version=v1.0
5 | system.decription=\u8f6f\u4ef6\u652f\u6301\u6570\u636e\u8d44\u6e90:\n1.Canal\u670d\u52a1\u76d1\u63a7(MySQL)\n2.\u6570\u636e\u540c\u6b65\u8f6c\u6362:\n1)SQL\u6570\u636e\u6e90(MySQL|SQL Server|Oracle\u7b49)\n2)NoSQL\u6570\u636e\u6e90(Cassandra|MongoDB\u7b49)\n3)\u641c\u7d22\u5f15\u64ce(Elasticsearch)\n4)\u6d88\u606f\u961f\u5217(Kafka)\u7b49
6 | ###################################################################################################################
7 | dstt.btn.continue=\u7ee7\u7eed
8 | dstt.btn.prev=\u4e0a\u4e00\u6b65
9 | dstt.btn.next=\u4e0b\u4e00\u6b65
10 | dstt.btn.back=\u8fd4\u56de
11 | dstt.btn.open=\u5f00\u542f
12 | dstt.btn.close=\u5173\u95ed
13 | dstt.btn.enabled=\u542f\u7528
14 | dstt.btn.disabled=\u7981\u7528
15 | dstt.btn.submit=\u63d0\u4ea4
16 | dstt.btn.add=\u589e\u52a0
17 | dstt.btn.update=\u4fee\u6539
18 | dstt.btn.cancel=\u53d6\u6d88
19 | dstt.btn.remove=\u79fb\u9664
20 | dstt.btn.refresh=\u5237\u65b0
21 | dstt.btn.syn=\u540c\u6b65
22 | dstt.btn.synall=\u540c\u6b65\u5168\u90e8
23 |
24 | dstt.service.canal=Canal\u76d1\u63a7
25 | dstt.service.datasyn=\u6570\u636e\u540c\u6b65
26 | dstt.service.datalog=\u65e5\u5fd7
27 |
28 | dstt.ds.source=\u6570\u636e\u6e90
29 | dstt.ds.target=\u76ee\u6807\u6e90
30 |
31 | dstt.ds.seq=\u5e8f\u53f7
32 | dstt.ds.saddress=\u6570\u636e\u5730\u5740
33 | dstt.ds.stype=\u6570\u636e\u7c7b\u578b
34 | dstt.ds.taddress=\u76ee\u6807\u5730\u5740
35 | dstt.ds.ttype=\u76ee\u6807\u7c7b\u578b
36 | dstt.ds.action=\u64cd\u4f5c
37 |
38 | dstt.ds.type=\u670d\u52a1\u7c7b\u578b
39 | dstt.ds.type.0=Canal\u670d\u52a1
40 | dstt.ds.type.0.1=Zookeeper\u670d\u52a1
41 | dstt.ds.type.1=Elasticsearch\u670d\u52a1
42 | dstt.ds.type.2=NoSQL\u670d\u52a1
43 | dstt.ds.type.2.1=Cassandra
44 | dstt.ds.type.2.2=MongoDB
45 | dstt.ds.type.2.3=Redis
46 | dstt.ds.type.2.4=Memecached
47 | dstt.ds.type.3=SQL\u670d\u52a1
48 | dstt.ds.type.3.1=MySQL
49 | dstt.ds.type.3.2=SQL\u3000Server
50 | dstt.ds.type.3.3=Oracle
51 | dstt.ds.type.4=\u6570\u636e\u4ed3\u5e93
52 | dstt.ds.type.4.1=Greenplum
53 | dstt.ds.type.5=\u6d88\u606f\u961f\u5217
54 | dstt.ds.type.5.1=Kafka
55 |
56 | dstt.ds.servers=\u670d\u52a1\u5730\u5740
57 | dstt.ds.servers.tip=(\u652f\u6301\u96c6\u7fa4\u914d\u7f6e\u5219\u591a\u670d\u52a1\u4ee5','\u5206\u5272)
58 | dstt.ds.port=\u670d\u52a1\u7aef\u53e3
59 | dstt.ds.username=\u7528 \u6237 \u540d
60 | dstt.ds.password=\u5bc6 \u7801
61 | dstt.ds.database=\u6570 \u636e \u5e93
62 | dstt.ds.schema=Schema
63 | dstt.ds.keyspace=Keyspace
64 | dstt.ds.mapping=\u5e93\u8868\u6620\u5c04
65 | dstt.ds.filter=\u8868\u5b57\u6bb5\u8fc7\u6ee4
66 | dstt.ds.filter.tip=(\u591a\u5b57\u6bb5\u4ee5','\u5206\u5272)
--------------------------------------------------------------------------------