├── .gitattributes ├── .gitignore ├── LICENSE ├── README.md ├── docs └── images │ ├── binlog-add.png │ ├── binlog-log.png │ ├── binlog-start.png │ ├── es-edit1.png │ ├── es-edit2.png │ ├── es-edit3.png │ ├── es-log.png │ ├── main.png │ ├── system.png │ └── system2.png ├── easy-kafka-common ├── pom.xml └── src │ └── main │ └── java │ └── com │ └── cehome │ └── easykafka │ ├── Consumer.java │ └── Producer.java ├── easy-kafka-proxy ├── pom.xml └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── cehome │ │ │ └── easykafka │ │ │ ├── App.java │ │ │ ├── JarClassLoader.java │ │ │ ├── StandardKafkaClassLoader.java │ │ │ ├── consumer │ │ │ ├── ConsumerRecord.java │ │ │ └── SimpleKafkaConsumer.java │ │ │ ├── enums │ │ │ └── VersionEnum.java │ │ │ └── producer │ │ │ └── SimpleKafkaProducer.java │ └── resources │ │ ├── del-application.yml │ │ └── kafka │ │ ├── 0.10.1.0 │ │ └── kafka-clients-0.10.1.0.jar │ │ ├── 0.11.0.1 │ │ └── kafka-clients-0.11.0.1.jar │ │ └── 0.9.0.1 │ │ └── kafka-clients-0.9.0.1.jar │ └── test │ └── java │ └── com │ └── cehome │ └── easykafka │ ├── KafkaConsumerTest.java │ └── KafkaProducerTest.java ├── easy-sync ├── pom.xml └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── cehome │ │ │ └── easysync │ │ │ ├── Application.java │ │ │ ├── BeanConfig.java │ │ │ ├── DefaultTimeTaskController.java │ │ │ ├── DemoPlugin.java │ │ │ ├── HTTPBasicAuthorizeFilter.java │ │ │ ├── RemoteController.java │ │ │ ├── ServletInitializer.java │ │ │ ├── TableCreator.java │ │ │ ├── dao │ │ │ ├── PositionDao.java │ │ │ └── SyncConfigDao.java │ │ │ ├── domain │ │ │ ├── Position.java │ │ │ └── SyncConfig.java │ │ │ ├── jest │ │ │ ├── AliasIndices.java │ │ │ └── Jest.java │ │ │ ├── objects │ │ │ ├── Column.java │ │ │ ├── MysqlUrl.java │ │ │ ├── Row.java │ │ │ ├── Table.java │ │ │ ├── TableTask.java │ │ │ └── config │ │ │ │ ├── Es.java │ │ │ │ ├── EsConfig.java │ │ │ │ ├── EsField.java │ │ │ │ ├── EsMapping.java │ │ │ │ ├── Kafka.java │ │ │ │ ├── Mysql.java │ │ │ │ ├── MysqlConfig.java │ │ │ │ ├── MysqlSource.java │ │ │ │ ├── Rebuild.java │ │ │ │ └── Repair.java │ │ │ ├── plugins │ │ │ ├── EsSyncFull.java │ │ │ ├── EsSyncIncrement.java │ │ │ ├── EsSyncPlugin.java │ │ │ ├── EsSyncPluginRunner.java │ │ │ ├── MysqlSyncListener.java │ │ │ └── MysqlSyncPlugin.java │ │ │ ├── service │ │ │ ├── DatabaseService.java │ │ │ ├── PositionSaveService.java │ │ │ ├── PositionService.java │ │ │ └── TableTasksMapService.java │ │ │ └── utils │ │ │ ├── CharsetConversion.java │ │ │ ├── Const.java │ │ │ ├── EsUtils.java │ │ │ ├── Global.java │ │ │ ├── MemTable.java │ │ │ └── MysqlTypeUtils.java │ └── resources │ │ ├── application-dev.properties │ │ ├── application.properties │ │ ├── logback.xml │ │ ├── sql │ │ ├── posi.txt │ │ ├── sync_conf_h2.txt │ │ ├── sync_conf_mysql.txt │ │ ├── task_cache.txt │ │ ├── task_h2.txt │ │ └── task_mysql.txt │ │ ├── static │ │ ├── favicon.ico │ │ ├── res │ │ │ ├── img │ │ │ │ ├── logo.png │ │ │ │ └── question.png │ │ │ └── js │ │ │ │ ├── lang-cn2.js │ │ │ │ └── lang-en2.js │ │ └── vue.html │ │ ├── templates │ │ ├── include │ │ │ ├── head-detail.ftl │ │ │ ├── head-master.ftl │ │ │ └── head-meta.ftl │ │ └── timeTask │ │ │ ├── esEdit.ftl │ │ │ ├── esEditBody.ftl │ │ │ ├── esEditMapping.ftl │ │ │ ├── esList.ftl │ │ │ ├── getLog.ftl │ │ │ ├── sourceEdit.ftl │ │ │ ├── sourceEditBody.ftl │ │ │ └── sourceList.ftl │ │ └── time-task-type-custom.json │ └── test │ └── java │ ├── BinlogTest.java │ ├── ConvertTest.java │ ├── EsTest.java │ └── KafkaTest.java ├── pack.bat └── pom.xml /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | *.js linguist-language=Java 4 | *.css linguist-language=Java 5 | *.html linguist-language=Java 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | !.mvn/wrapper/maven-wrapper.jar 3 | 4 | ### STS ### 5 | .apt_generated 6 | .classpath 7 | .factorypath 8 | .project 9 | .settings 10 | .springBeans 11 | 12 | ### IntelliJ IDEA ### 13 | .idea 14 | *.iws 15 | *.iml 16 | *.ipr 17 | 18 | ### NetBeans ### 19 | nbproject/private/ 20 | build/ 21 | nbbuild/ 22 | dist/ 23 | nbdist/ 24 | .nb-gradle/ 25 | 26 | *.log 27 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # easy-sync 2 | 3 | Sync mysql to elasticsearch in real-time via binlog and kafka. 4 | - Easy to use. 5 | - Support multi tasks and add new task without restart. 6 | - Change index structure smoothly without affecting online business. 7 | 8 | ![demoͼ](docs/images/main.png) 9 | 10 | # How does it work? 11 | There are two kinds of tasks. 12 | Binlog Task - Read binlog and send different tables' data to different kafka topics. 13 | Elasticsearch Task - Each task read own kafka topic and write data to elasticsearch. 14 | 15 | ![system](docs/images/system.png) 16 | 17 | # Why use kafka? 18 | mysql binlog records all tables changes one by one. When you sync two tables to es, 19 | one does not work will block another one. So we send two tables' data to kafka with different topics, 20 | two tables will work independently. 21 | # Quick start 22 | - Install jdk(>=1.8), install kafka(>=0.9.0.1) 23 | - download eays-sync from https://github.com/cehome-com/easy-sync/releases, 24 | https://github.com/cehome-com/easy-sync/releases/download/2.0.5/easy-sync.zip 25 | , unzip and run: 26 | **java -jar easy-sync.jar** 27 | - Visit http://localhost:8080 28 | - Add and start binlog task, add and start elasticsearch task 29 | 30 | # Add binlog task 31 | - Mysql binlog config 32 | Before you can read mysql binlog, you have to enable binlog with Row mode, 33 | and grant REPLICATION privilege to your mysql user. 34 | 1. enable mysql binlog. 35 | 36 | ``` 37 | [mysqld] 38 | log-bin=mysql-bin 39 | binlog-format=ROW 40 | server_id=1 41 | ``` 42 | 43 | 2. create mysql user and grant REPLICATION privilege. 44 | 45 | ``` 46 | CREATE USER sync IDENTIFIED BY 'sync'; 47 | GRANT SELECT, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'sync'@'%'; 48 | -- GRANT ALL PRIVILEGES ON *.* TO 'sync'@'%' ; 49 | FLUSH PRIVILEGES; 50 | ``` 51 | 52 | - At home page, click "Add" button to open a dialog. Finish fields "Task Name, mysql url/user/password, 53 | kafka servers", click "Save" and "Close". 54 | "Server"(which machine task runs on) field needn't edit, default value is your local ip such as 55 | "192.168.137.1" 56 | 57 | ![add](docs/images/binlog-add.png) 58 | 59 | 60 | - A binlog task is in list, click "Start". 61 | 62 | ![start](docs/images/binlog-start.png) 63 | 64 | - Wait a minute, click "Log" button, a new window is opened and you can see some logs. 65 | Because none of elasticsearch task has been added, binlog task will not send any data to kafka. 66 | You must continue to add an elasticsearch task. 67 | 68 | ![log](docs/images/binlog-log.png) 69 | 70 | # Add elasticsearch task 71 | - Click top menu "Elasticsearch" , click "Add" button to open a new window, configure "Base Info" and "Datasource Config". 72 | Task. 73 | Select mysql instance: select the binlog task "task1" you have just configured. 74 | Database Names: existing databases such as "ershouji". 75 | Table Names: an existing table such as "equipment". If you have more than on tables such as 76 | equipment_1,equipment_2 with the same structure, then use "equipment_%" or "equipment_1,equipment_2". 77 | Primary key: currently only support increase, number type (for full sync). 78 | 79 | ![Datasource Config](docs/images/es-edit1.png) 80 | 81 | - Elasticsearch Config 82 | Be careful to select elasticsearch version. 83 | 84 | ![Elasticsearch Config](docs/images/es-edit2.png) 85 | 86 | - Field Mapping. Mapping for mysql and elasticsearch fields. Just click "Auto Mapping" button, all fields 87 | will be listed. You can delete or modify some fields. 88 | 89 | ![Auto Mapping](docs/images/es-edit3.png) 90 | 91 | - Click "Rebuild Index". This step will save config and create new elasticsearch index. 92 | - Switch to home page, refresh the page. You can see the task created, click "Start" button. 93 | - Wait a minute, click "Log" to see logs. By default, full sync and increase sync runs 94 | together. 95 | 96 | ![es log](docs/images/es-log.png) 97 | 98 | 99 | # FAQ 100 | - mysql versions 101 | binlog tools use https://github.com/shyiko/mysql-binlog-connector-java, so mysql 5.x or 8.x is supported. 102 | - kafka versions 103 | kafka client can use 0.9.0.1,0.10.1.0,0.11.0.1 three versions. Hight version kafka servers normally 104 | support low version clients. So you kafka server must >= 0.9.0.1 105 | - where does task config store in? 106 | By default, task config stores in H2 database in user home such as C:\Users\xxx\easy_sync. 107 | You can create application.properties in the same directory easy-sync.jar is in to change it. 108 | Mysql or h2 database is supported. 109 | ``` 110 | task.datasource.driverClassName=org.h2.Driver 111 | task.datasource.url=jdbc:h2:tcp://localhost:9000/~/easy_sync;MODE=MYSQL 112 | task.datasource.username=sa 113 | task.datasource.password= 114 | ``` 115 | - easy-sync cluster supports. 116 | You must have more than one computer nodes such as 192.168.0.10 and 192.168.0.11. 117 | 1. At each node, create application.properties in the same directory easy-sync.jar is in , 118 | add "cluster name" and "config storage datasource" to file. 119 | ``` 120 | #cluster name , also is config table_name, default is easy_sync 121 | task.factory.name=easy_sync 122 | 123 | #config storage datasource, support mysql or h2 database 124 | task.datasource.driverClassName=com.mysql.jdbc.Driver 125 | task.datasource.url=jdbc:mysql://192.168.0.13:3306/sync?useUnicode=true&characterEncoding=utf8&allowMultiQueries=true 126 | task.datasource.username=xxxx 127 | task.datasource.password=xxxx 128 | ``` 129 | 2. start easy_sync at each node. 130 | 3. configure you tasks. You can select which ip task runs on(set "server" to 192.168.0.10 131 | or 192.168.0.11 ) 132 | 4. Once one node is down, it will auto switch to another node after a while ( You can see 133 | server ip changing from 192.168.0.10 to 192.168.0.11, but after the node is up again, the ip will 134 | switch back) 135 | 136 | - Languages 137 | English(en) and chinese(cn) is supported. Default is english, 138 | Add "**language=cn**" to application.properties to enable chinese. 139 | 140 | - change web port 141 | modify application.properties 142 | **server.port=8080** 143 | 144 | - about index sync 145 | 146 | 147 | 148 | 149 | -------------------------------------------------------------------------------- /docs/images/binlog-add.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/easy30/easy-sync/60ebd9bf998ac7fde38ea68ff2b4e8a6c65875c8/docs/images/binlog-add.png -------------------------------------------------------------------------------- /docs/images/binlog-log.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/easy30/easy-sync/60ebd9bf998ac7fde38ea68ff2b4e8a6c65875c8/docs/images/binlog-log.png -------------------------------------------------------------------------------- /docs/images/binlog-start.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/easy30/easy-sync/60ebd9bf998ac7fde38ea68ff2b4e8a6c65875c8/docs/images/binlog-start.png -------------------------------------------------------------------------------- /docs/images/es-edit1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/easy30/easy-sync/60ebd9bf998ac7fde38ea68ff2b4e8a6c65875c8/docs/images/es-edit1.png -------------------------------------------------------------------------------- /docs/images/es-edit2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/easy30/easy-sync/60ebd9bf998ac7fde38ea68ff2b4e8a6c65875c8/docs/images/es-edit2.png -------------------------------------------------------------------------------- /docs/images/es-edit3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/easy30/easy-sync/60ebd9bf998ac7fde38ea68ff2b4e8a6c65875c8/docs/images/es-edit3.png -------------------------------------------------------------------------------- /docs/images/es-log.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/easy30/easy-sync/60ebd9bf998ac7fde38ea68ff2b4e8a6c65875c8/docs/images/es-log.png -------------------------------------------------------------------------------- /docs/images/main.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/easy30/easy-sync/60ebd9bf998ac7fde38ea68ff2b4e8a6c65875c8/docs/images/main.png -------------------------------------------------------------------------------- /docs/images/system.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/easy30/easy-sync/60ebd9bf998ac7fde38ea68ff2b4e8a6c65875c8/docs/images/system.png -------------------------------------------------------------------------------- /docs/images/system2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/easy30/easy-sync/60ebd9bf998ac7fde38ea68ff2b4e8a6c65875c8/docs/images/system2.png -------------------------------------------------------------------------------- /easy-kafka-common/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | com.cehome 7 | easy-sync-parent 8 | 2.0.5 9 | 10 | 11 | 4.0.0 12 | com.cehome 13 | easy-kafka-common 14 | 1.1-SNAPSHOT 15 | jar 16 | 17 | 18 | -------------------------------------------------------------------------------- /easy-kafka-common/src/main/java/com/cehome/easykafka/Consumer.java: -------------------------------------------------------------------------------- 1 | package com.cehome.easykafka; 2 | 3 | /** 4 | * Created by houyanlin on 2018/06/25 5 | **/ 6 | public interface Consumer { 7 | 8 | public void createKafkaConsumer() throws Exception; 9 | 10 | public void subscribe(String... topic) throws Exception; 11 | 12 | 13 | public Object poll(long timeout); 14 | 15 | public void commitSync(); 16 | 17 | public void close(); 18 | 19 | } 20 | -------------------------------------------------------------------------------- /easy-kafka-common/src/main/java/com/cehome/easykafka/Producer.java: -------------------------------------------------------------------------------- 1 | package com.cehome.easykafka; 2 | 3 | /** 4 | * kafka生产者 5 | * 6 | * @author houyanlin 7 | * @create 2018-06-22 11:03 8 | **/ 9 | public interface Producer { 10 | 11 | public Object send(String topic,String key, String value) throws Exception; 12 | 13 | public void close() throws Exception; 14 | 15 | } 16 | -------------------------------------------------------------------------------- /easy-kafka-proxy/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | 11 | 12 | org.springframework.boot 13 | spring-boot-starter-parent 14 | 1.5.9.RELEASE 15 | 16 | com.cehome 17 | 4.0.0 18 | jar 19 | 20 | easy-kafka-proxy 21 | 1.1-SNAPSHOT 22 | 23 | 24 | 25 | org.springframework.boot 26 | spring-boot-starter-test 27 | test 28 | 29 | 30 | org.springframework.boot 31 | spring-boot-starter 32 | provided 33 | 34 | 35 | 36 | 37 | org.springframework.boot 38 | spring-boot-starter-web 39 | provided 40 | 41 | 42 | commons-io 43 | commons-io 44 | 2.5 45 | 46 | 47 | commons-beanutils 48 | commons-beanutils 49 | 1.9.3 50 | 51 | 66 | 67 | com.cehome 68 | easy-kafka-common 69 | 1.1-SNAPSHOT 70 | 71 | 72 | 73 | 74 | -------------------------------------------------------------------------------- /easy-kafka-proxy/src/main/java/com/cehome/easykafka/App.java: -------------------------------------------------------------------------------- 1 | package com.cehome.easykafka; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | import org.springframework.beans.factory.annotation.Value; 6 | import org.springframework.boot.SpringApplication; 7 | import org.springframework.boot.autoconfigure.SpringBootApplication; 8 | import org.springframework.context.ApplicationContext; 9 | import org.springframework.web.bind.annotation.RestController; 10 | 11 | /** 12 | * Created by houyanlin on 2018/06/21 13 | **/ 14 | @SpringBootApplication 15 | @RestController 16 | public class App { 17 | 18 | Logger logger = LoggerFactory.getLogger(App.class); 19 | 20 | @Value("${kafka.producer.bootstrapServers}") 21 | private String producerBootstrapServers; 22 | 23 | @Value("${kafka.producer.retries}") 24 | private String producerRetries; 25 | 26 | @Value("${kafka.producer.batchSize}") 27 | private String producerBatchSize; 28 | 29 | @Value("${kafka.producer.lingerMs}") 30 | private String producerLingerMs; 31 | 32 | @Value("${kafka.producer.bufferMemory}") 33 | private String producerBufferMemory; 34 | 35 | @Value("${kafka.consumer.bootstrapServers}") 36 | private String consumerBootstrapServers; 37 | @Value("${kafka.consumer.groupId}") 38 | private String consumerGroupId; 39 | 40 | public static void main(String[] args) { 41 | ApplicationContext ctx = SpringApplication.run(App.class, args); 42 | } 43 | 44 | } 45 | -------------------------------------------------------------------------------- /easy-kafka-proxy/src/main/java/com/cehome/easykafka/JarClassLoader.java: -------------------------------------------------------------------------------- 1 | package com.cehome.easykafka; 2 | 3 | import org.apache.commons.io.IOUtils; 4 | 5 | import java.io.IOException; 6 | import java.io.InputStream; 7 | import java.net.URL; 8 | import java.net.URLClassLoader; 9 | import java.security.ProtectionDomain; 10 | import java.util.jar.JarEntry; 11 | import java.util.jar.JarInputStream; 12 | 13 | public class JarClassLoader extends ClassLoader { 14 | 15 | private String version; 16 | public JarClassLoader(String version,ClassLoader classLoader){ 17 | super(classLoader); 18 | this.version=version; 19 | } 20 | 21 | 22 | 23 | @Override 24 | protected Class findClass(String name) throws ClassNotFoundException{ 25 | //Class c=findLoadedClass(name); 26 | //if(c!=null) return c; 27 | //System.out.println("find class: "+name); 28 | String jar="/kafka/" + version + "/kafka-clients-" + version + ".jar"; 29 | InputStream is = this.getClass().getResourceAsStream(jar); 30 | if(is==null) throw new ClassNotFoundException("jar "+jar+" not exists"); 31 | JarInputStream jis= null; 32 | try { 33 | String path=name.replace('.','/')+".class"; 34 | jis = new JarInputStream(is); 35 | JarEntry entry=null; 36 | while( ( entry = jis.getNextJarEntry())!=null){ 37 | if (entry.getName().equals(path)) { 38 | byte[] bs = IOUtils.toByteArray(jis); 39 | 40 | //ProtectionDomain protectionDomain=new ProtectionDomain(null,null); 41 | return defineClass(name,bs, 0, bs.length); 42 | 43 | } 44 | } 45 | return getSystemClassLoader().loadClass(name); 46 | //throw new ClassNotFoundException(); 47 | 48 | } catch (IOException e) { 49 | throw new ClassNotFoundException(e.getMessage()); 50 | } 51 | 52 | } 53 | 54 | public static void main(String[] args) throws ClassNotFoundException { 55 | JarClassLoader jarClassLoader=new JarClassLoader("0.10.1.0",null); 56 | Class c= jarClassLoader.loadClass( "org.apache.kafka.clients.consumer.KafkaConsumer"); 57 | c=jarClassLoader.loadClass("java.lang.String"); 58 | System.out.println(c.getName()); 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /easy-kafka-proxy/src/main/java/com/cehome/easykafka/StandardKafkaClassLoader.java: -------------------------------------------------------------------------------- 1 | package com.cehome.easykafka; 2 | 3 | import org.apache.commons.io.FileUtils; 4 | import org.apache.commons.io.IOUtils; 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | import org.springframework.core.io.ClassPathResource; 8 | 9 | import java.io.File; 10 | import java.io.IOException; 11 | import java.io.InputStream; 12 | import java.net.MalformedURLException; 13 | import java.net.URL; 14 | import java.net.URLClassLoader; 15 | 16 | /** 17 | * Created by houyanlin on 2018/06/22 18 | **/ 19 | public class StandardKafkaClassLoader extends URLClassLoader { 20 | private Logger logger = LoggerFactory.getLogger(StandardKafkaClassLoader.class); 21 | public StandardKafkaClassLoader(String version) { 22 | super(new URL[] {}, null); // 将 Parent 设置为 null 23 | loadResource(version); 24 | try { 25 | Thread.sleep(5000); 26 | } catch (InterruptedException e) { 27 | e.printStackTrace(); 28 | } 29 | } 30 | 31 | @Override 32 | public Class loadClass(String name) throws ClassNotFoundException { 33 | try { 34 | return super.loadClass(name); 35 | }catch (ClassNotFoundException e){ 36 | return StandardKafkaClassLoader.class.getClassLoader().loadClass(name); 37 | } 38 | } 39 | 40 | @Override 41 | public Class findClass(String name) throws ClassNotFoundException { 42 | logger.debug("findClass: {}", name); 43 | try { 44 | return super.findClass(name); 45 | } catch(ClassNotFoundException e) { 46 | return StandardKafkaClassLoader.class.getClassLoader().loadClass(name); 47 | } 48 | } 49 | 50 | private void loadResource(String version) { 51 | InputStream inputStream = null; 52 | try { 53 | ClassPathResource classPathResource = new ClassPathResource("kafka/" + version + "/kafka-clients-" + version + ".jar"); 54 | inputStream = classPathResource.getInputStream(); 55 | File dir = File.createTempFile("kafka-clients-" + version, ".jar"); 56 | FileUtils.copyInputStreamToFile(inputStream, dir); 57 | logger.info("loadResource:{}",dir.getAbsoluteFile()); 58 | this.addURL(dir); 59 | }catch (Exception e){ 60 | e.printStackTrace(); 61 | }finally { 62 | IOUtils.closeQuietly(inputStream); 63 | } 64 | 65 | } 66 | 67 | private void tryLoadJarInDir(File dir) { 68 | // File dir = new File(dirPath); 69 | logger.info("tryLoadJarInDir-dirPath:{}",dir.getAbsoluteFile()); 70 | // 自动加载目录下的jar包 71 | if (dir.exists() && dir.isDirectory()) { 72 | for (File file : dir.listFiles()) { 73 | if (file.isFile() && file.getName().endsWith(".jar")) { 74 | logger.info("tryLoadJarInDir-jar:{}",file.getAbsoluteFile()); 75 | this.addURL(file); 76 | continue; 77 | } 78 | } 79 | } 80 | } 81 | private void addURL(File file) { 82 | try { 83 | super.addURL(new URL("file", null, file.getCanonicalPath())); 84 | } catch (MalformedURLException e) { 85 | e.printStackTrace(); 86 | } catch (IOException e) { 87 | e.printStackTrace(); 88 | } 89 | } 90 | 91 | } 92 | -------------------------------------------------------------------------------- /easy-kafka-proxy/src/main/java/com/cehome/easykafka/consumer/ConsumerRecord.java: -------------------------------------------------------------------------------- 1 | package com.cehome.easykafka.consumer; 2 | 3 | /** 4 | * Created by houyanlin on 2018/06/26 5 | **/ 6 | public class ConsumerRecord { 7 | 8 | private String topic; 9 | private int partition; 10 | private long offset; 11 | private long timestamp; 12 | private int serializedKeySize; 13 | private int serializedValueSize; 14 | private String key; 15 | private String value; 16 | 17 | public ConsumerRecord(String key,String value){ 18 | this.key = key; 19 | this.value = value; 20 | } 21 | 22 | public ConsumerRecord(){ 23 | 24 | } 25 | 26 | public ConsumerRecord(String topic,int partition,long offset,long timestamp,int serializedKeySize,int serializedValueSize,String key,String value){ 27 | this.topic = topic; 28 | this.partition = partition; 29 | this.offset = offset; 30 | this.timestamp = timestamp; 31 | this.serializedKeySize = serializedKeySize; 32 | this.serializedValueSize = serializedValueSize; 33 | this.key = key; 34 | this.value = value; 35 | } 36 | 37 | 38 | public String topic() { 39 | return this.topic; 40 | } 41 | 42 | public int partition() { 43 | return this.partition; 44 | } 45 | 46 | 47 | public String key() { 48 | return this.key; 49 | } 50 | 51 | public String value() { 52 | return this.value; 53 | } 54 | 55 | public long offset() { 56 | return this.offset; 57 | } 58 | 59 | public long timestamp() { 60 | return this.timestamp; 61 | } 62 | 63 | 64 | public int serializedKeySize() { 65 | return this.serializedKeySize; 66 | } 67 | 68 | public int serializedValueSize() { 69 | return this.serializedValueSize; 70 | } 71 | 72 | @Override 73 | public String toString() { 74 | final StringBuffer sb = new StringBuffer("ConsumerRecord{"); 75 | sb.append("topic='").append(topic).append('\''); 76 | sb.append(", key='").append(key).append('\''); 77 | sb.append(", value='").append(value).append('\''); 78 | sb.append('}'); 79 | return sb.toString(); 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /easy-kafka-proxy/src/main/java/com/cehome/easykafka/consumer/SimpleKafkaConsumer.java: -------------------------------------------------------------------------------- 1 | package com.cehome.easykafka.consumer; 2 | 3 | import com.cehome.easykafka.Consumer; 4 | import com.cehome.easykafka.JarClassLoader; 5 | import com.cehome.easykafka.enums.VersionEnum; 6 | 7 | import java.lang.reflect.Constructor; 8 | import java.lang.reflect.Method; 9 | import java.util.*; 10 | 11 | /** 12 | * Created by houyanlin on 2018/06/22 13 | **/ 14 | public class SimpleKafkaConsumer implements Consumer{ 15 | 16 | private String version; 17 | private Properties props; 18 | //private StandardKafkaClassLoader standardKafkaClassLoader; 19 | private JarClassLoader jarClassLoader; 20 | private Class consumerClazz; 21 | private Constructor consumerConstructor; 22 | private Object consumerInstance; 23 | private Class consumerRecordClazz; 24 | 25 | 26 | public SimpleKafkaConsumer(String version, Properties props) throws Exception { 27 | this.version = version; 28 | this.props = props; 29 | //standardKafkaClassLoader = new StandardKafkaClassLoader(version); 30 | jarClassLoader=new JarClassLoader(version,SimpleKafkaConsumer.class.getClassLoader()); 31 | this.consumerClazz = jarClassLoader.loadClass("org.apache.kafka.clients.consumer.KafkaConsumer"); 32 | this.consumerRecordClazz =jarClassLoader.loadClass("org.apache.kafka.clients.consumer.ConsumerRecord"); 33 | } 34 | @Override 35 | public void createKafkaConsumer() throws Exception{ 36 | ClassLoader oldClassLoader = Thread.currentThread().getContextClassLoader(); 37 | try { 38 | // 临时更改 ClassLoader 39 | //Thread.currentThread().setContextClassLoader(consumerClazz.getClassLoader()); 40 | 41 | this.consumerConstructor = consumerClazz.getConstructor(Properties.class); 42 | if (version.startsWith(VersionEnum.KAFKA_VERSION_8.getValue())) { 43 | props.put("partition.assignment.strategy", "roundrobin"); 44 | } 45 | this.consumerInstance = consumerConstructor.newInstance(props); 46 | }finally { 47 | // Thread.currentThread().setContextClassLoader(oldClassLoader); 48 | } 49 | 50 | 51 | 52 | } 53 | 54 | 55 | @Override 56 | public void subscribe(String[] topic) throws Exception{ 57 | if (version.startsWith(VersionEnum.KAFKA_VERSION_8.getValue())){ 58 | Method method = consumerClazz.getMethod("subscribe", String[].class); 59 | method.invoke(consumerInstance, (Object)topic); 60 | }else if (version.startsWith(VersionEnum.KAFKA_VERSION_9.getValue())){ 61 | Method method = consumerClazz.getMethod("subscribe", List.class); 62 | method.invoke(consumerInstance, Arrays.asList(topic)); 63 | }else if (version.startsWith(VersionEnum.KAFKA_VERSION_10.getValue())){ 64 | Method method = consumerClazz.getMethod("subscribe", Collection.class); 65 | method.invoke(consumerInstance, Arrays.asList(topic)); 66 | }else { 67 | Method method = consumerClazz.getMethod("subscribe", Collection.class); 68 | method.invoke(consumerInstance, Arrays.asList(topic)); 69 | } 70 | 71 | } 72 | 73 | 74 | @Override 75 | public List poll(long timeout) { 76 | List list = new ArrayList(); 77 | try { 78 | Method method = consumerClazz.getMethod("poll", long.class); 79 | Iterable iterable = (Iterable) method.invoke(consumerInstance, timeout); 80 | if (iterable != null){ 81 | Iterator iterator = iterable.iterator(); 82 | while (iterator.hasNext()){ 83 | Method valueMethod = consumerRecordClazz.getMethod("value"); 84 | Method keyMethod = consumerRecordClazz.getMethod("key"); 85 | Object record = iterator.next(); 86 | list.add( new ConsumerRecord((String) keyMethod.invoke(record),(String)valueMethod.invoke(record))); 87 | } 88 | } 89 | }catch (Exception e){ 90 | throw new RuntimeException(e); 91 | } 92 | return list; 93 | } 94 | 95 | @Override 96 | public void commitSync() { 97 | try { 98 | if (version.startsWith(VersionEnum.KAFKA_VERSION_8.getValue())){ 99 | // Method method = consumerClazz.getMethod("commit",boolean.class); 100 | // method.invoke(consumerInstance,true); 101 | }else{ 102 | Method method = consumerClazz.getMethod("commitSync"); 103 | method.invoke(consumerInstance); 104 | } 105 | 106 | }catch (Exception e){ 107 | e.printStackTrace(); 108 | } 109 | } 110 | @Override 111 | public void close(){ 112 | try { 113 | Method method = consumerClazz.getMethod("close"); 114 | method.invoke(consumerInstance); 115 | }catch (Exception e){ 116 | e.printStackTrace(); 117 | } 118 | } 119 | 120 | 121 | } 122 | -------------------------------------------------------------------------------- /easy-kafka-proxy/src/main/java/com/cehome/easykafka/enums/VersionEnum.java: -------------------------------------------------------------------------------- 1 | package com.cehome.easykafka.enums; 2 | 3 | /** 4 | * kafka版本 5 | * 6 | * @author houyanlin 7 | * @create 2018-07-02 11:29 8 | **/ 9 | public enum VersionEnum { 10 | 11 | KAFKA_VERSION_8(8,"0.8","0.8.2.2"), 12 | KAFKA_VERSION_9(8,"0.9","0.9.0.1"), 13 | KAFKA_VERSION_10(8,"0.10","0.10.1.0"), 14 | KAFKA_VERSION_11(8,"0.11","0.11.0.2"); 15 | 16 | private Integer code; 17 | private String value; 18 | private String desc; 19 | 20 | VersionEnum(Integer code,String value,String desc){ 21 | this.code = code; 22 | this.value = value; 23 | this.desc = desc; 24 | } 25 | 26 | public Integer getCode() { 27 | return code; 28 | } 29 | 30 | public void setCode(Integer code) { 31 | this.code = code; 32 | } 33 | 34 | public String getValue() { 35 | return value; 36 | } 37 | 38 | public void setValue(String value) { 39 | this.value = value; 40 | } 41 | 42 | public String getDesc() { 43 | return desc; 44 | } 45 | 46 | public void setDesc(String desc) { 47 | this.desc = desc; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /easy-kafka-proxy/src/main/java/com/cehome/easykafka/producer/SimpleKafkaProducer.java: -------------------------------------------------------------------------------- 1 | package com.cehome.easykafka.producer; 2 | 3 | import com.cehome.easykafka.JarClassLoader; 4 | import com.cehome.easykafka.Producer; 5 | 6 | import java.lang.reflect.Constructor; 7 | import java.lang.reflect.Method; 8 | import java.util.Properties; 9 | import java.util.concurrent.Future; 10 | 11 | /** 12 | * Created by houyanlin on 2018/06/22 13 | **/ 14 | public class SimpleKafkaProducer implements Producer{ 15 | 16 | private String version; 17 | private Properties props; 18 | //private StandardKafkaClassLoader standardKafkaClassLoader; 19 | private JarClassLoader jarClassLoader; 20 | private Class recordClazz; 21 | private Class producerClazz; 22 | private Object producerInstance; 23 | private Constructor recordConstructor; 24 | 25 | 26 | 27 | public SimpleKafkaProducer(String version, Properties props){ 28 | this.version = version; 29 | this.props = props; 30 | //standardKafkaClassLoader = new StandardKafkaClassLoader(version); 31 | jarClassLoader=new JarClassLoader(version,SimpleKafkaProducer.class.getClassLoader()); 32 | //ClassLoader oldClassLoader = Thread.currentThread().getContextClassLoader(); 33 | try { 34 | //Thread.currentThread().setContextClassLoader(jarClassLoader); 35 | this.producerClazz = jarClassLoader.loadClass("org.apache.kafka.clients.producer.KafkaProducer"); 36 | this.recordClazz = jarClassLoader.loadClass("org.apache.kafka.clients.producer.ProducerRecord"); 37 | Constructor pruducerConstructor = producerClazz.getConstructor(Properties.class); 38 | this.producerInstance = pruducerConstructor.newInstance(props); 39 | this.recordConstructor = recordClazz.getConstructor(String.class,Object.class,Object.class); 40 | }catch (Exception e){ 41 | e.printStackTrace(); 42 | throw new RuntimeException(e); 43 | } 44 | finally { 45 | //Thread.currentThread().setContextClassLoader(oldClassLoader); 46 | } 47 | 48 | } 49 | @Override 50 | public Object send(String topic, String key, String value) throws Exception{ 51 | Object recordInstance = recordConstructor.newInstance(topic,key,value); 52 | Method method = producerClazz.getMethod("send", recordClazz); 53 | Future future= (Future)method.invoke(producerInstance,recordInstance); 54 | return future.get(); 55 | } 56 | 57 | @Override 58 | public void close() throws Exception{ 59 | Method method = producerClazz.getMethod("close"); 60 | method.invoke(producerInstance); 61 | } 62 | 63 | } 64 | -------------------------------------------------------------------------------- /easy-kafka-proxy/src/main/resources/del-application.yml: -------------------------------------------------------------------------------- 1 | server: 2 | port: 8808 3 | spring: 4 | application: 5 | name: easy-kafka-proxy 6 | 7 | kafka: 8 | producer: 9 | bootstrapServers: houstan.com:9094 10 | retries: 3 11 | batchSize: 16384 12 | lingerMs: 1 13 | #32M 14 | bufferMemory: 33554432 15 | 16 | consumer: 17 | bootstrapServers: houstan.com:9094 18 | groupId: 0 19 | enableAutoCommit: false 20 | autoCommitIntervalMs: 1000 21 | sessionTimeoutMs: 30000 22 | maxPollRecords: 100 23 | #earliest,latest 24 | autoOffsetReset: earliest 25 | 26 | logging: 27 | level: 28 | root: debug 29 | -------------------------------------------------------------------------------- /easy-kafka-proxy/src/main/resources/kafka/0.10.1.0/kafka-clients-0.10.1.0.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/easy30/easy-sync/60ebd9bf998ac7fde38ea68ff2b4e8a6c65875c8/easy-kafka-proxy/src/main/resources/kafka/0.10.1.0/kafka-clients-0.10.1.0.jar -------------------------------------------------------------------------------- /easy-kafka-proxy/src/main/resources/kafka/0.11.0.1/kafka-clients-0.11.0.1.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/easy30/easy-sync/60ebd9bf998ac7fde38ea68ff2b4e8a6c65875c8/easy-kafka-proxy/src/main/resources/kafka/0.11.0.1/kafka-clients-0.11.0.1.jar -------------------------------------------------------------------------------- /easy-kafka-proxy/src/main/resources/kafka/0.9.0.1/kafka-clients-0.9.0.1.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/easy30/easy-sync/60ebd9bf998ac7fde38ea68ff2b4e8a6c65875c8/easy-kafka-proxy/src/main/resources/kafka/0.9.0.1/kafka-clients-0.9.0.1.jar -------------------------------------------------------------------------------- /easy-kafka-proxy/src/test/java/com/cehome/easykafka/KafkaConsumerTest.java: -------------------------------------------------------------------------------- 1 | package com.cehome.easykafka; 2 | 3 | import ch.qos.logback.classic.Level; 4 | import ch.qos.logback.classic.LoggerContext; 5 | import com.cehome.easykafka.consumer.SimpleKafkaConsumer; 6 | import org.junit.Test; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | import java.util.List; 11 | import java.util.Properties; 12 | 13 | /** 14 | * Created by houyanlin on 2018/06/26 15 | **/ 16 | //@RunWith(SpringRunner.class) 17 | //@SpringBootTest(classes = App.class) 18 | public class KafkaConsumerTest { 19 | static Logger logger = LoggerFactory.getLogger(KafkaConsumerTest.class); 20 | static { 21 | ( (LoggerContext) LoggerFactory.getILoggerFactory()).getLogger("ROOT").setLevel(Level.INFO); 22 | } 23 | 24 | @Test 25 | public void pollTest() throws Exception{ 26 | String version="0.11.0.1"; 27 | //version="0.9.0.1"; 28 | Properties props = new Properties(); 29 | props.put("bootstrap.servers", "localhost:9092"); 30 | props.put("group.id","test1"); 31 | props.put("acks", "all"); 32 | props.put("retries", "3"); 33 | props.put("batch.size", "16384"); 34 | props.put("linger.ms", 1); 35 | props.put("buffer.memory", "33554432"); 36 | props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); 37 | props.put("value.deserializer","org.apache.kafka.common.serialization.StringDeserializer"); 38 | // props.put("partition.assignment.strategy","range"); 39 | props.put("message.max.bytes", "10MB"); 40 | props.put("replica.fetch.max.bytes", "10MBB"); 41 | Thread.currentThread().setContextClassLoader(null); 42 | SimpleKafkaConsumer kafkaConsumer = new SimpleKafkaConsumer(version,props); 43 | kafkaConsumer.createKafkaConsumer(); 44 | String[] st = {"test3"}; 45 | kafkaConsumer.subscribe(st); 46 | while (true) { 47 | List result = kafkaConsumer.poll(5000); 48 | if (result.size() > 0) { 49 | logger.info("record:{}", result.get(0)); 50 | } 51 | logger.info("result:{}", result); 52 | } 53 | } 54 | 55 | } 56 | -------------------------------------------------------------------------------- /easy-kafka-proxy/src/test/java/com/cehome/easykafka/KafkaProducerTest.java: -------------------------------------------------------------------------------- 1 | package com.cehome.easykafka; 2 | 3 | import ch.qos.logback.classic.Level; 4 | import ch.qos.logback.classic.LoggerContext; 5 | import com.cehome.easykafka.producer.SimpleKafkaProducer; 6 | import org.junit.Test; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | import java.util.Properties; 11 | import java.util.Random; 12 | 13 | /** 14 | * Created by houyanlin on 2018/06/26 15 | **/ 16 | public class KafkaProducerTest { 17 | Logger logger = LoggerFactory.getLogger(KafkaProducerTest.class); 18 | static { 19 | ( (LoggerContext) LoggerFactory.getILoggerFactory()).getLogger("ROOT").setLevel(Level.INFO); 20 | } 21 | 22 | @Test 23 | public void sendTest() throws Exception{ 24 | String version="0.11.0.1"; 25 | //version="0.9.0.1"; 26 | Properties props = new Properties(); 27 | props.put("bootstrap.servers", "localhost:9092"); 28 | props.put("acks", "all"); 29 | props.put("retries", "3"); 30 | props.put("batch.size", "16384"); 31 | props.put("linger.ms", 1); 32 | props.put("buffer.memory", "33554432"); 33 | props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer"); 34 | props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer"); 35 | props.put("message.max.bytes", "10MB"); 36 | props.put("replica.fetch.max.bytes", "10MBB"); 37 | props.put("max.block.ms", "15000"); 38 | Thread.currentThread().setContextClassLoader(null); 39 | SimpleKafkaProducer kafkaProducer = new SimpleKafkaProducer(version,props); 40 | while(true) { 41 | System.out.println("result=" + kafkaProducer.send("test3", "7", "data"+new Random().nextInt(100)).getClass()); 42 | Thread.sleep(new Random().nextInt(5000)); 43 | } 44 | //logger.info("send success"); 45 | 46 | 47 | } 48 | 49 | 50 | } 51 | -------------------------------------------------------------------------------- /easy-sync/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4 | org.springframework.boot 5 | spring-boot-starter-parent 6 | 1.5.9.RELEASE 7 | 8 | 9 | 10 | 4.0.0 11 | com.cehome 12 | easy-sync 13 | jar 14 | easy-sync 15 | 2.0.5 16 | 17 | http://maven.apache.org 18 | 19 | 3.2.2.RELEASE 20 | UTF-8 21 | 1.7 22 | 23 | 24 | 25 | 83 | 84 | 85 | 86 | 87 | org.springframework.boot 88 | spring-boot-starter-web 89 | 90 | 91 | org.springframework.boot 92 | spring-boot-starter-freemarker 93 | 94 | 95 | 96 | org.springframework.boot 97 | spring-boot-starter-test 98 | test 99 | 100 | 101 | 102 | org.springframework.boot 103 | spring-boot-starter-cache 104 | 105 | 106 | 107 | junit 108 | junit 109 | 3.8.1 110 | test 111 | 112 | 113 | 114 | com.cehome 115 | task-console-resource 116 | 2.0.5 117 | 118 | 119 | 120 | com.cehome 121 | easy-kafka-proxy 122 | 1.1-SNAPSHOT 123 | 124 | 125 | 126 | ch.qos.logback 127 | logback-classic 128 | 1.1.11 129 | 130 | 131 | 132 | 133 | 134 | com.github.shyiko 135 | mysql-binlog-connector-java 136 | 0.16.1 137 | 138 | 139 | junit 140 | junit 141 | 4.12 142 | test 143 | 144 | 145 | 146 | io.searchbox 147 | jest 148 | 5.3.3 149 | 150 | 151 | 152 | org.apache.commons 153 | commons-csv 154 | 1.5 155 | 156 | 157 | 158 | io.swagger.core.v3 159 | swagger-annotations 160 | 2.0.2 161 | 162 | 163 | 164 | org.apache.kafka 165 | kafka-clients 166 | 0.11.0.1 167 | provided 168 | 169 | 170 | 171 | 172 | 173 | ${project.artifactId} 174 | 175 | 176 | org.springframework.boot 177 | spring-boot-maven-plugin 178 | 179 | 180 | org.apache.maven.plugins 181 | maven-deploy-plugin 182 | 2.7 183 | 184 | true 185 | 186 | 187 | 188 | 189 | 190 | 191 | -------------------------------------------------------------------------------- /easy-sync/src/main/java/com/cehome/easysync/Application.java: -------------------------------------------------------------------------------- 1 | package com.cehome.easysync; 2 | 3 | import com.cehome.task.annotation.EnableTimeTaskClient; 4 | import com.cehome.task.annotation.EnableTimeTaskConsole; 5 | import org.apache.commons.lang.StringUtils; 6 | import org.springframework.beans.factory.annotation.Value; 7 | import org.springframework.boot.SpringApplication; 8 | import org.springframework.boot.autoconfigure.SpringBootApplication; 9 | import org.springframework.boot.web.servlet.FilterRegistrationBean; 10 | import org.springframework.cache.annotation.EnableCaching; 11 | import org.springframework.context.ApplicationContext; 12 | import org.springframework.context.annotation.Bean; 13 | import org.springframework.context.annotation.ComponentScan; 14 | import org.springframework.web.bind.annotation.RequestMapping; 15 | import org.springframework.web.bind.annotation.RestController; 16 | 17 | import javax.servlet.*; 18 | import javax.servlet.http.HttpServletResponse; 19 | import java.io.IOException; 20 | import java.util.ArrayList; 21 | import java.util.List; 22 | 23 | @SpringBootApplication 24 | @RestController 25 | @EnableTimeTaskConsole 26 | @EnableTimeTaskClient 27 | @ComponentScan("com.cehome.easysync") 28 | @EnableCaching 29 | public class Application { 30 | @Value("${language:en}") 31 | String language; 32 | 33 | @Value("${login.user:}") 34 | String user; 35 | @Value("${login.password:}") 36 | String password; 37 | 38 | private static ApplicationContext applicationContext; 39 | public static void main(String[] args) { 40 | applicationContext= SpringApplication.run(Application.class,args); 41 | } 42 | 43 | public static T getBean(Class c){ 44 | return applicationContext.getBean(c); 45 | } 46 | 47 | @RequestMapping("/") 48 | public void index(HttpServletResponse response) throws IOException { 49 | response.sendRedirect("timeTask/sourceList.htm"); 50 | } 51 | 52 | 53 | @Bean 54 | public FilterRegistrationBean createFilterRegistrationBean() { 55 | FilterRegistrationBean registration = new FilterRegistrationBean(); 56 | registration.setFilter(new Filter() { 57 | 58 | 59 | @Override 60 | public void init(FilterConfig filterConfig) throws ServletException { 61 | 62 | } 63 | 64 | @Override 65 | public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain) throws IOException, ServletException { 66 | servletRequest.setAttribute("language", language); 67 | filterChain.doFilter(servletRequest, servletResponse); 68 | } 69 | 70 | @Override 71 | public void destroy() { 72 | 73 | } 74 | }); 75 | registration.addUrlPatterns("/*"); 76 | //registration.setName("MyFilter"); 77 | // registration.setOrder(1); 78 | return registration; 79 | } 80 | 81 | 82 | 83 | @Bean 84 | public FilterRegistrationBean createAuthFilter() { 85 | FilterRegistrationBean registrationBean = new FilterRegistrationBean(); 86 | HTTPBasicAuthorizeFilter httpBasicFilter = new HTTPBasicAuthorizeFilter(); 87 | registrationBean.setFilter(httpBasicFilter); 88 | List urlPatterns = new ArrayList(); 89 | System.out.println("login.user="+user); 90 | if(StringUtils.isBlank(user)){ 91 | urlPatterns.add("/not_need_filter"); 92 | }else{ 93 | httpBasicFilter.setUser(user); 94 | httpBasicFilter.setPassword(password); 95 | urlPatterns.add("/*"); 96 | } 97 | 98 | 99 | registrationBean.setUrlPatterns(urlPatterns); 100 | registrationBean.setOrder(200); 101 | return registrationBean; 102 | } 103 | 104 | 105 | 106 | 107 | 108 | } 109 | -------------------------------------------------------------------------------- /easy-sync/src/main/java/com/cehome/easysync/BeanConfig.java: -------------------------------------------------------------------------------- 1 | package com.cehome.easysync; 2 | 3 | import com.cehome.easysync.dao.PositionDao; 4 | import com.cehome.easysync.dao.SyncConfigDao; 5 | import com.cehome.easysync.domain.Position; 6 | import com.cehome.easysync.domain.SyncConfig; 7 | import com.cehome.task.TimeTaskFactory; 8 | import jsharp.sql.SessionFactory; 9 | import jsharp.support.BeanAnn; 10 | import org.springframework.beans.factory.annotation.Autowired; 11 | import org.springframework.cache.Cache; 12 | import org.springframework.cache.CacheManager; 13 | import org.springframework.cache.concurrent.ConcurrentMapCache; 14 | import org.springframework.cache.support.SimpleCacheManager; 15 | import org.springframework.context.annotation.Bean; 16 | import org.springframework.context.annotation.Configuration; 17 | 18 | import java.util.ArrayList; 19 | import java.util.List; 20 | 21 | @Configuration 22 | public class BeanConfig { 23 | @Autowired 24 | private TimeTaskFactory timeTaskFactory; 25 | @Bean 26 | public SessionFactory createSessionFactory(){ 27 | return timeTaskFactory.getSessionFactory(); 28 | } 29 | 30 | 31 | private T createDao(Class dao,Class e,String name) throws Exception { 32 | BeanAnn beanAnn= BeanAnn.getBeanAnn(e); 33 | String tableName=timeTaskFactory.getName()+"_"+name; 34 | beanAnn.setTable(tableName); 35 | T t= dao.newInstance(); 36 | //positionDao.setTableName(tableName); 37 | //positionDao.setSessionFactory(timeTaskFactory.getSessionFactory()); 38 | return t; 39 | } 40 | 41 | @Bean 42 | public PositionDao createPositionDao() throws Exception { 43 | 44 | return createDao(PositionDao.class, Position.class,"posi"); 45 | 46 | } 47 | 48 | 49 | @Bean 50 | public SyncConfigDao createSyncConfigDao() throws Exception { 51 | 52 | return createDao(SyncConfigDao.class,SyncConfig.class,"sync_conf"); 53 | 54 | } 55 | 56 | 57 | /** 58 | * https://www.cnblogs.com/imyijie/p/6518547.html 59 | * @return 60 | */ 61 | @Bean 62 | public CacheManager cacheManager() { 63 | SimpleCacheManager cacheManager = new SimpleCacheManager(); 64 | List caches=new ArrayList(){{ 65 | add(new ConcurrentMapCache("tableTasksMap")); 66 | }}; 67 | cacheManager.setCaches(caches); 68 | return cacheManager; 69 | } 70 | 71 | 72 | } 73 | 74 | 75 | -------------------------------------------------------------------------------- /easy-sync/src/main/java/com/cehome/easysync/DemoPlugin.java: -------------------------------------------------------------------------------- 1 | package com.cehome.easysync; 2 | 3 | import com.alibaba.fastjson.JSONObject; 4 | import com.cehome.task.client.TimeTaskContext; 5 | import com.cehome.task.client.TimeTaskPlugin; 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | import org.springframework.stereotype.Component; 9 | 10 | import java.net.Inet4Address; 11 | 12 | @Component 13 | public class DemoPlugin extends TimeTaskPlugin { 14 | private static final Logger logger = LoggerFactory.getLogger(DemoPlugin.class); 15 | @Override 16 | public void run(TimeTaskContext context, JSONObject args) throws Exception { 17 | logger.info("task id="+context.getId()); 18 | logger.info("task name="+context.getName()); 19 | logger.info("task run on ip="+ Inet4Address.getLocalHost().getHostAddress()); 20 | logger.info("task sleep 2000"); 21 | Thread.sleep(2000); 22 | logger.info("task run count="+context.getRunTimes()); 23 | } 24 | 25 | @Override 26 | public void stop(TimeTaskContext context) throws Exception { 27 | logger.info("task "+context.getName()+" is stopped "); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /easy-sync/src/main/java/com/cehome/easysync/HTTPBasicAuthorizeFilter.java: -------------------------------------------------------------------------------- 1 | package com.cehome.easysync; 2 | 3 | import sun.misc.BASE64Decoder; 4 | 5 | import javax.servlet.*; 6 | import javax.servlet.http.HttpServletRequest; 7 | import javax.servlet.http.HttpServletResponse; 8 | import java.io.IOException; 9 | 10 | @SuppressWarnings("restriction") 11 | public class HTTPBasicAuthorizeFilter implements Filter{ 12 | 13 | private String user; 14 | private String password; 15 | 16 | public String getUser() { 17 | return user; 18 | } 19 | 20 | public void setUser(String user) { 21 | this.user = user; 22 | } 23 | 24 | public String getPassword() { 25 | return password; 26 | } 27 | 28 | public void setPassword(String password) { 29 | this.password = password; 30 | } 31 | 32 | @Override 33 | public void destroy() { 34 | 35 | } 36 | 37 | @Override 38 | public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) 39 | throws IOException, ServletException { 40 | boolean ok = checkHTTPBasicAuthorize(request); 41 | if (!ok) 42 | { 43 | HttpServletResponse httpResponse = (HttpServletResponse) response; 44 | httpResponse.setCharacterEncoding("UTF-8"); 45 | httpResponse.setContentType("application/json; charset=utf-8"); 46 | httpResponse.setStatus(HttpServletResponse.SC_UNAUTHORIZED); 47 | httpResponse.setHeader("Cache-Control", "no-store"); 48 | httpResponse.setDateHeader("Expires", 0); 49 | httpResponse.setHeader("WWW-authenticate", "Basic Realm=cehome"); 50 | httpResponse.getWriter().write("no auth"); 51 | return; 52 | } 53 | else 54 | { 55 | chain.doFilter(request, response); 56 | } 57 | } 58 | 59 | @Override 60 | public void init(FilterConfig arg0) throws ServletException { 61 | 62 | } 63 | 64 | private boolean checkHTTPBasicAuthorize(ServletRequest request) 65 | { 66 | 67 | 68 | try 69 | { 70 | HttpServletRequest httpRequest = (HttpServletRequest)request; 71 | String path=httpRequest.getServletPath(); 72 | //ignore 73 | if(path.startsWith("/remote") || path.startsWith("/timeTaskClientService")){ 74 | return true; 75 | } 76 | if(httpRequest.getSession().getAttribute("user")!=null){ 77 | return true; 78 | } 79 | /* //-- xiao ke 80 | if(httpRequest.getParameter("codeSig")!=null && httpRequest.getParameter("code")!=null){ 81 | JSONObject j= XiaoKeService.getOpenUserIdByCode(XiaoKeService.getAppAccessToken(), httpRequest.getParameter("code")); 82 | if ("0".equals(j.getString("errorCode"))) { 83 | httpRequest.getSession().setAttribute("userId",1L); 84 | return true; 85 | } 86 | }*/ 87 | 88 | String auth = httpRequest.getHeader("Authorization"); 89 | if ((auth != null) && (auth.length() > 6)) 90 | { 91 | String HeadStr = auth.substring(0, 5).toLowerCase(); 92 | if (HeadStr.compareTo("basic") == 0) 93 | { 94 | auth = auth.substring(6, auth.length()); 95 | String decodedAuth = getFromBASE64(auth); 96 | if (decodedAuth != null) 97 | { 98 | String[] userArray = decodedAuth.split(":"); 99 | String u=userArray[0]; 100 | String p=userArray.length==1?"":userArray[1]; 101 | if(u.equals(user)&& p.equals(password)){ 102 | httpRequest.getSession().setAttribute("user",user); 103 | return true; 104 | } 105 | 106 | } 107 | } 108 | } 109 | return false; 110 | } 111 | catch(Exception ex) 112 | { 113 | return false; 114 | } 115 | 116 | } 117 | 118 | private String getFromBASE64(String s) { 119 | if (s == null) 120 | return null; 121 | BASE64Decoder decoder = new BASE64Decoder(); 122 | try { 123 | byte[] b = decoder.decodeBuffer(s); 124 | return new String(b); 125 | } catch (Exception e) { 126 | return null; 127 | } 128 | } 129 | 130 | } -------------------------------------------------------------------------------- /easy-sync/src/main/java/com/cehome/easysync/RemoteController.java: -------------------------------------------------------------------------------- 1 | package com.cehome.easysync; 2 | 3 | import com.cehome.easysync.service.TableTasksMapService; 4 | import org.springframework.beans.factory.annotation.Autowired; 5 | import org.springframework.web.bind.annotation.RequestMapping; 6 | import org.springframework.web.bind.annotation.RestController; 7 | 8 | @RestController 9 | @RequestMapping("/remote") 10 | public class RemoteController { 11 | @Autowired 12 | TableTasksMapService tableTasksMapService; 13 | 14 | @RequestMapping("cleanTableTasksMap.htm") 15 | public void cleanTableTasksMap(long timeTaskId){ 16 | tableTasksMapService.clean(timeTaskId); 17 | 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /easy-sync/src/main/java/com/cehome/easysync/ServletInitializer.java: -------------------------------------------------------------------------------- 1 | package com.cehome.easysync; 2 | 3 | import org.springframework.boot.builder.SpringApplicationBuilder; 4 | import org.springframework.boot.web.support.SpringBootServletInitializer; 5 | 6 | public class ServletInitializer extends SpringBootServletInitializer { 7 | 8 | @Override 9 | protected SpringApplicationBuilder configure(SpringApplicationBuilder application) { 10 | 11 | 12 | SpringApplicationBuilder builder= application.sources(Application.class); 13 | return builder; 14 | 15 | } 16 | 17 | } 18 | -------------------------------------------------------------------------------- /easy-sync/src/main/java/com/cehome/easysync/TableCreator.java: -------------------------------------------------------------------------------- 1 | package com.cehome.easysync; 2 | 3 | import com.cehome.task.Constants; 4 | import com.cehome.task.util.IpAddressUtil; 5 | import jsharp.util.Common; 6 | import org.apache.commons.io.IOUtils; 7 | import org.h2.tools.Server; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | import org.springframework.beans.BeansException; 11 | import org.springframework.beans.factory.InitializingBean; 12 | import org.springframework.beans.factory.annotation.Value; 13 | import org.springframework.beans.factory.config.BeanPostProcessor; 14 | import org.springframework.stereotype.Component; 15 | 16 | import java.io.InputStream; 17 | import java.sql.*; 18 | 19 | 20 | @Component 21 | public class TableCreator implements InitializingBean,BeanPostProcessor { 22 | 23 | 24 | private static final Logger logger = LoggerFactory.getLogger(TableCreator.class); 25 | 26 | @Value("${task.h2.start:true}") 27 | private boolean h2Start; 28 | 29 | @Value("${task.h2.port:9092}") 30 | private String h2Port; 31 | 32 | 33 | @Value("${task.autoCreateTable:false}") 34 | private boolean autoCreateTable; 35 | 36 | @Value("${task.factory.appName}") 37 | private String appName; 38 | 39 | @Value("${task.factory.name}") 40 | private String table; 41 | 42 | @Value("${task.fuseHostName:false}") 43 | private boolean useHostName; 44 | 45 | @Value(Constants.CONFIG_DRIVER) 46 | private String driverClassName; 47 | 48 | @Value("${task.datasource.url}") 49 | private String url; 50 | 51 | @Value("${task.datasource.username}") 52 | private String username; 53 | 54 | @Value("${task.datasource.password}") 55 | private String password; 56 | 57 | @Override 58 | public void afterPropertiesSet() throws Exception { 59 | boolean mysql=driverClassName.indexOf("mysql") >= 0; 60 | logger.info(mysql?"Use mysql database":"Use h2 database"); 61 | logger.info("url={}, table={}",url,table); 62 | if(autoCreateTable) { 63 | execute(); 64 | } 65 | } 66 | 67 | private boolean tableExists(Statement st ,String table ,boolean mysql) throws SQLException { 68 | ResultSet rs = null; 69 | try { 70 | if (mysql) { 71 | String sql = "show tables like '" + table + "'"; 72 | rs = st.executeQuery(sql); 73 | return rs.next(); 74 | }else{ 75 | String sql = "show tables"; 76 | rs = st.executeQuery(sql); 77 | while(rs.next()){ 78 | if(rs.getString(1).equalsIgnoreCase(table)){ 79 | return true; 80 | } 81 | } 82 | return false; 83 | } 84 | }finally { 85 | Common.closeObjects(rs); 86 | } 87 | } 88 | 89 | public void execute() { 90 | 91 | Connection conn = null; 92 | Statement st = null; 93 | //ResultSet rs = null; 94 | 95 | 96 | try 97 | { 98 | boolean mysql=driverClassName.indexOf("mysql") >= 0; 99 | //String table2=table1+"_cache"; 100 | 101 | Class.forName(driverClassName); 102 | 103 | if(!mysql && h2Start){ 104 | try { 105 | Server server = Server.createTcpServer("-tcpPort", h2Port, "-tcpAllowOthers").start(); 106 | 107 | System.out.println("h2 server listen at " + server.getURL()); 108 | }catch (Exception e){ 109 | e.printStackTrace(); 110 | } 111 | 112 | } 113 | 114 | conn = DriverManager.getConnection(url, username, password); 115 | 116 | 117 | try { 118 | logger.info("create begin"); 119 | st = conn.createStatement(); 120 | execute(st,mysql,mysql?"task_mysql.txt":"task_h2.txt",table); 121 | execute(st,mysql,"task_cache.txt",table+"_cache"); 122 | //st.execute("drop table if exists easy_search_posi "); 123 | execute(st,mysql,"posi.txt",table+"_posi"); 124 | //st.execute("drop table if exists easy_search_sync_conf "); 125 | execute(st,mysql,mysql?"sync_conf_mysql.txt":"sync_conf_h2.txt",table+"_sync_conf"); 126 | 127 | /*{ 128 | 129 | if (tableExists(st,table1,mysql)) { 130 | logger.info("table " + table1 + " exists."); 131 | 132 | }else{ 133 | //File file = ResourceUtils.getFile("sql/"+(mysql?"task_mysql.txt":"task_h2.txt")); 134 | InputStream is= this.getClass().getClassLoader().getResourceAsStream("sql/"+(mysql?"task_mysql.txt":"task_h2.txt")); 135 | String sql1=IOUtils.toString(is,"UTF-8"); 136 | is.close(); 137 | //String sql1 = FileUtils.readFileToString(file,"UTF-8"); 138 | sql1 = sql1.replace("${tableName}", table1); 139 | 140 | sql1 = sql1.replace("${ip}", useHostName?IpAddressUtil.getLocalHostName(): IpAddressUtil.getLocalHostAddress()); 141 | sql1 = sql1.replace("${appName}", appName); 142 | st.execute(sql1); 143 | logger.info("created table " + table1); 144 | } 145 | 146 | 147 | }*/ 148 | 149 | /* { 150 | if (tableExists(st, table2,mysql)) { 151 | logger.info("table " + table2 + " exists."); 152 | 153 | } else { 154 | InputStream is= this.getClass().getClassLoader().getResourceAsStream("sql/task_cache.txt"); 155 | String sql2=IOUtils.toString(is,"UTF-8"); 156 | is.close(); 157 | //File file = ResourceUtils.getFile("classpath:sql/task_cache.txt"); 158 | // String sql2 = FileUtils.readFileToString(file, "UTF-8"); 159 | sql2 = sql2.replace("${tableName}", table2); 160 | st.execute(sql2); 161 | logger.info("created table " + table2); 162 | } 163 | }*/ 164 | 165 | logger.info("create end"); 166 | }finally { 167 | Common.closeObjects(st); 168 | } 169 | 170 | 171 | 172 | } 173 | catch (Exception e){ 174 | e.printStackTrace(); 175 | } 176 | 177 | finally 178 | { 179 | Common.closeObjects(st,conn); 180 | 181 | } 182 | 183 | 184 | } 185 | 186 | private void execute( Statement st,boolean mysql,String sqlFile,String table) throws Exception { 187 | if (tableExists(st,table,mysql)) { 188 | logger.info("table " + table + " exists."); 189 | 190 | }else{ 191 | //File file = ResourceUtils.getFile("sql/"+(mysql?"task_mysql.txt":"task_h2.txt")); 192 | InputStream is= this.getClass().getClassLoader().getResourceAsStream("sql/"+sqlFile); 193 | String sql1=IOUtils.toString(is,"UTF-8"); 194 | is.close(); 195 | //String sql1 = FileUtils.readFileToString(file,"UTF-8"); 196 | sql1 = sql1.replace("${tableName}", table); 197 | 198 | sql1 = sql1.replace("${ip}", useHostName?IpAddressUtil.getLocalHostName(): IpAddressUtil.getLocalHostAddress()); 199 | sql1 = sql1.replace("${appName}", appName); 200 | st.execute(sql1); 201 | logger.info("created table " + table); 202 | } 203 | 204 | } 205 | 206 | 207 | 208 | @Override 209 | public Object postProcessBeforeInitialization(Object o, String s) throws BeansException { 210 | return o; 211 | } 212 | 213 | @Override 214 | public Object postProcessAfterInitialization(Object o, String s) throws BeansException { 215 | return o; 216 | } 217 | } 218 | -------------------------------------------------------------------------------- /easy-sync/src/main/java/com/cehome/easysync/dao/PositionDao.java: -------------------------------------------------------------------------------- 1 | package com.cehome.easysync.dao; 2 | 3 | import com.cehome.easysync.domain.Position; 4 | import jsharp.sql.SimpleDao; 5 | import jsharp.support.BeanAnn; 6 | 7 | //@Component 8 | public class PositionDao extends SimpleDao { 9 | 10 | 11 | @Override 12 | public String getTableName() { 13 | BeanAnn beanAnn= BeanAnn.getBeanAnn(PositionDao.class); 14 | return beanAnn.getTable(); 15 | } 16 | 17 | 18 | } 19 | -------------------------------------------------------------------------------- /easy-sync/src/main/java/com/cehome/easysync/dao/SyncConfigDao.java: -------------------------------------------------------------------------------- 1 | package com.cehome.easysync.dao; 2 | 3 | import com.cehome.easysync.domain.SyncConfig; 4 | import jsharp.sql.SimpleDao; 5 | import jsharp.support.BeanAnn; 6 | 7 | //@Component 8 | public class SyncConfigDao extends SimpleDao { 9 | 10 | 11 | @Override 12 | public String getTableName() { 13 | BeanAnn beanAnn= BeanAnn.getBeanAnn(SyncConfig.class); 14 | return beanAnn.getTable(); 15 | } 16 | 17 | 18 | public SyncConfig getByTimeTaskId(long timeTaskId){ 19 | return this.queryOneByProps(null,"timeTaskId",timeTaskId); 20 | } 21 | 22 | 23 | } 24 | -------------------------------------------------------------------------------- /easy-sync/src/main/java/com/cehome/easysync/domain/Position.java: -------------------------------------------------------------------------------- 1 | package com.cehome.easysync.domain; 2 | 3 | import jsharp.sql.BaseDO; 4 | import jsharp.sql.anno.Table; 5 | 6 | import javax.persistence.Id; 7 | 8 | @Table 9 | public class Position extends BaseDO { 10 | @Id 11 | private long id; //主键 12 | private long serverId; 13 | private long timeTaskId; 14 | 15 | private String filename; 16 | private long position; 17 | private String gtidSet; 18 | //private String gtid; 19 | 20 | 21 | public long getId() { 22 | return id; 23 | } 24 | 25 | public void setId(long id) { 26 | this.id = id; 27 | } 28 | 29 | public long getServerId() { 30 | return serverId; 31 | } 32 | 33 | public void setServerId(long serverId) { 34 | this.serverId = serverId; 35 | } 36 | 37 | public String getFilename() { 38 | return filename; 39 | } 40 | 41 | public void setFilename(String filename) { 42 | this.filename = filename; 43 | } 44 | 45 | public long getPosition() { 46 | return position; 47 | } 48 | 49 | public void setPosition(long position) { 50 | this.position = position; 51 | } 52 | 53 | public long getTimeTaskId() { 54 | return timeTaskId; 55 | } 56 | 57 | public void setTimeTaskId(long timeTaskId) { 58 | this.timeTaskId = timeTaskId; 59 | } 60 | 61 | /* public String getGtid() { 62 | return gtid; 63 | } 64 | 65 | public void setGtid(String gtid) { 66 | this.gtid = gtid; 67 | }*/ 68 | 69 | public String getGtidSet() { 70 | return gtidSet; 71 | } 72 | 73 | public void setGtidSet(String gtidSet) { 74 | this.gtidSet = gtidSet; 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /easy-sync/src/main/java/com/cehome/easysync/domain/SyncConfig.java: -------------------------------------------------------------------------------- 1 | package com.cehome.easysync.domain; 2 | 3 | import jsharp.sql.BaseDO; 4 | import jsharp.sql.anno.Table; 5 | 6 | import javax.persistence.Id; 7 | 8 | @Table 9 | public class SyncConfig extends BaseDO { 10 | 11 | @Id 12 | private long id; 13 | 14 | private long timeTaskId; 15 | 16 | //can not modify 17 | private long mysqlTimeTaskId; 18 | 19 | // 20 | private String mysqlDatabases; 21 | private String mysqlTables; 22 | private String mysqlKeys; 23 | private String mysqlKeySep; 24 | 25 | 26 | private String indexName1; 27 | private String waitIndexName1; 28 | private String taskConfig1; 29 | private String fullTable1; 30 | private long fullPosition1; 31 | private int fullStatus1; 32 | 33 | private String indexName2; 34 | private String waitIndexName2; 35 | private String taskConfig2; 36 | private String fullTable2; 37 | private long fullPosition2; 38 | private int fullStatus2; 39 | 40 | private String repairTaskConfig; 41 | private String repairTable; 42 | private long repairPosition; 43 | private int repairStatus; 44 | 45 | 46 | //private String rebuildTables; 47 | /* private String rebuildTable; 48 | 49 | private long rebuildPosition; 50 | 51 | private int rebuildStatus;*/ 52 | 53 | public long getId() { 54 | return id; 55 | } 56 | 57 | public void setId(long id) { 58 | this.id = id; 59 | } 60 | 61 | public long getTimeTaskId() { 62 | return timeTaskId; 63 | } 64 | 65 | public void setTimeTaskId(long timeTaskId) { 66 | this.timeTaskId = timeTaskId; 67 | } 68 | 69 | public long getMysqlTimeTaskId() { 70 | return mysqlTimeTaskId; 71 | } 72 | 73 | public void setMysqlTimeTaskId(long mysqlTimeTaskId) { 74 | this.mysqlTimeTaskId = mysqlTimeTaskId; 75 | } 76 | 77 | public String getMysqlDatabases() { 78 | return mysqlDatabases; 79 | } 80 | 81 | public void setMysqlDatabases(String mysqlDatabases) { 82 | this.mysqlDatabases = mysqlDatabases; 83 | } 84 | 85 | public String getMysqlTables() { 86 | return mysqlTables; 87 | } 88 | 89 | public void setMysqlTables(String mysqlTables) { 90 | this.mysqlTables = mysqlTables; 91 | } 92 | 93 | public String getMysqlKeys() { 94 | return mysqlKeys; 95 | } 96 | 97 | public void setMysqlKeys(String mysqlKeys) { 98 | this.mysqlKeys = mysqlKeys; 99 | } 100 | 101 | public String getMysqlKeySep() { 102 | return mysqlKeySep; 103 | } 104 | 105 | public void setMysqlKeySep(String mysqlKeySep) { 106 | this.mysqlKeySep = mysqlKeySep; 107 | } 108 | 109 | public String getIndexName1() { 110 | return indexName1; 111 | } 112 | 113 | public void setIndexName1(String indexName1) { 114 | this.indexName1 = indexName1; 115 | } 116 | 117 | public String getTaskConfig1() { 118 | return taskConfig1; 119 | } 120 | 121 | public void setTaskConfig1(String taskConfig1) { 122 | this.taskConfig1 = taskConfig1; 123 | } 124 | 125 | public String getFullTable1() { 126 | return fullTable1; 127 | } 128 | 129 | public void setFullTable1(String fullTable1) { 130 | this.fullTable1 = fullTable1; 131 | } 132 | 133 | public long getFullPosition1() { 134 | return fullPosition1; 135 | } 136 | 137 | public void setFullPosition1(long fullPosition1) { 138 | this.fullPosition1 = fullPosition1; 139 | } 140 | 141 | public int getFullStatus1() { 142 | return fullStatus1; 143 | } 144 | 145 | public void setFullStatus1(int fullStatus1) { 146 | this.fullStatus1 = fullStatus1; 147 | } 148 | 149 | public String getIndexName2() { 150 | return indexName2; 151 | } 152 | 153 | public void setIndexName2(String indexName2) { 154 | this.indexName2 = indexName2; 155 | } 156 | 157 | public String getTaskConfig2() { 158 | return taskConfig2; 159 | } 160 | 161 | public void setTaskConfig2(String taskConfig2) { 162 | this.taskConfig2 = taskConfig2; 163 | } 164 | 165 | public String getFullTable2() { 166 | return fullTable2; 167 | } 168 | 169 | public void setFullTable2(String fullTable2) { 170 | this.fullTable2 = fullTable2; 171 | } 172 | 173 | public long getFullPosition2() { 174 | return fullPosition2; 175 | } 176 | 177 | public void setFullPosition2(long fullPosition2) { 178 | this.fullPosition2 = fullPosition2; 179 | } 180 | 181 | public int getFullStatus2() { 182 | return fullStatus2; 183 | } 184 | 185 | public void setFullStatus2(int fullStatus2) { 186 | this.fullStatus2 = fullStatus2; 187 | } 188 | 189 | public String getWaitIndexName1() { 190 | return waitIndexName1; 191 | } 192 | 193 | public void setWaitIndexName1(String waitIndexName1) { 194 | this.waitIndexName1 = waitIndexName1; 195 | } 196 | 197 | public String getWaitIndexName2() { 198 | return waitIndexName2; 199 | } 200 | 201 | public void setWaitIndexName2(String waitIndexName2) { 202 | this.waitIndexName2 = waitIndexName2; 203 | } 204 | 205 | public String getRepairTaskConfig() { 206 | return repairTaskConfig; 207 | } 208 | 209 | public void setRepairTaskConfig(String repairTaskConfig) { 210 | this.repairTaskConfig = repairTaskConfig; 211 | } 212 | 213 | public String getRepairTable() { 214 | return repairTable; 215 | } 216 | 217 | public void setRepairTable(String repairTable) { 218 | this.repairTable = repairTable; 219 | } 220 | 221 | public long getRepairPosition() { 222 | return repairPosition; 223 | } 224 | 225 | public void setRepairPosition(long repairPosition) { 226 | this.repairPosition = repairPosition; 227 | } 228 | 229 | public int getRepairStatus() { 230 | return repairStatus; 231 | } 232 | 233 | public void setRepairStatus(int repairStatus) { 234 | this.repairStatus = repairStatus; 235 | } 236 | } 237 | -------------------------------------------------------------------------------- /easy-sync/src/main/java/com/cehome/easysync/jest/AliasIndices.java: -------------------------------------------------------------------------------- 1 | package com.cehome.easysync.jest; 2 | 3 | import com.google.gson.JsonObject; 4 | import io.searchbox.action.GenericResultAbstractAction; 5 | import io.searchbox.client.JestResult; 6 | 7 | public class AliasIndices extends GenericResultAbstractAction { 8 | private String alias; 9 | public AliasIndices(String alias){ 10 | //super(builder); 11 | this.alias = alias; 12 | setURI(buildURI()); 13 | } 14 | 15 | @Override 16 | public String getRestMethodName() { 17 | return "GET"; 18 | } 19 | 20 | @Override 21 | protected String buildURI() { 22 | return super.buildURI() + "/*/_alias/" + alias; 23 | } 24 | 25 | 26 | public String[] parse(JestResult jestResult){ 27 | if(!jestResult.isSucceeded()) return new String[0]; 28 | JsonObject jsonObject= jestResult.getJsonObject(); 29 | return jsonObject.keySet().toArray(new String[0]); 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /easy-sync/src/main/java/com/cehome/easysync/jest/Jest.java: -------------------------------------------------------------------------------- 1 | package com.cehome.easysync.jest; 2 | 3 | import com.cehome.easysync.objects.config.Es; 4 | import io.searchbox.client.JestClient; 5 | import io.searchbox.client.JestClientFactory; 6 | import io.searchbox.client.JestResult; 7 | import io.searchbox.client.config.HttpClientConfig; 8 | import io.searchbox.indices.DeleteIndex; 9 | import io.searchbox.indices.IndicesExists; 10 | import io.searchbox.indices.aliases.AddAliasMapping; 11 | import io.searchbox.indices.aliases.AliasMapping; 12 | import io.searchbox.indices.aliases.ModifyAliases; 13 | import io.searchbox.indices.aliases.RemoveAliasMapping; 14 | import org.apache.commons.lang.StringUtils; 15 | import org.slf4j.Logger; 16 | import org.slf4j.LoggerFactory; 17 | 18 | import java.io.IOException; 19 | import java.util.ArrayList; 20 | import java.util.Arrays; 21 | import java.util.List; 22 | 23 | public class Jest { 24 | private static final Logger logger = LoggerFactory.getLogger(Jest.class); 25 | JestClient jestClient; 26 | 27 | public Jest(Es es) { 28 | this.jestClient = createJestClient(es); 29 | } 30 | 31 | public boolean bindAliasAndIndex(String alias, String indexName) throws IOException { 32 | logger.info("begin to bind index "+indexName+" to alias "+alias); 33 | List list = new ArrayList<>(); 34 | AliasIndices aliasIndices = new AliasIndices(alias); 35 | JestResult jestResult = jestClient.execute(aliasIndices); 36 | 37 | if (jestResult.isSucceeded()) { 38 | String[] indexNames = aliasIndices.parse(jestResult); 39 | if(indexNames.length==1 && indexNames[0].equalsIgnoreCase(indexName)){ 40 | logger.warn("old index is same as new index : {}, ignore binding.",indexName); 41 | return true; 42 | } 43 | for (String s : indexNames) { 44 | logger.info("old index "+s); 45 | RemoveAliasMapping removeAliasMapping = new RemoveAliasMapping.Builder(s, alias).build(); 46 | list.add(removeAliasMapping); 47 | } 48 | } else if (jestResult.getResponseCode() == 404) { 49 | logger.info("no alias found, ignore"); 50 | } else { 51 | 52 | logger.error(jestResult.getErrorMessage()); 53 | return false; 54 | } 55 | 56 | 57 | AddAliasMapping addAliasMapping = new AddAliasMapping.Builder(indexName, alias).build(); 58 | list.add(addAliasMapping); 59 | ModifyAliases modifyAliases = new ModifyAliases.Builder(list).build(); 60 | JestResult jr = jestClient.execute(modifyAliases); 61 | if (jr.isSucceeded()) { 62 | return true; 63 | 64 | } else { 65 | logger.error("modifyAliases error :" + jr.getErrorMessage()); 66 | return false; 67 | } 68 | 69 | 70 | } 71 | 72 | public boolean deleteIndex(String indexName) { 73 | if(StringUtils.isBlank(indexName)){ 74 | logger.warn("index name is blank"); 75 | return false; 76 | } 77 | JestResult jr = null; 78 | try { 79 | //logger.info("delete index "+indexName); 80 | jr = jestClient.execute(new DeleteIndex.Builder(indexName).build()); 81 | if (jr.isSucceeded() || jr.getResponseCode()==404) { 82 | return true; 83 | } 84 | logger.error("delete index error :" + jr.getErrorMessage()); 85 | } catch (IOException e) { 86 | logger.error("", e); 87 | } 88 | 89 | return false; 90 | 91 | 92 | } 93 | 94 | public boolean indexExists(String indexName) throws IOException { 95 | 96 | AliasIndices aliasIndices = new AliasIndices(indexName); 97 | JestResult jestResult = jestClient.execute(aliasIndices); 98 | 99 | //because index and alias can not be the same, alias found , means index not exists 100 | if (jestResult.isSucceeded()) { 101 | return false; 102 | } else if (jestResult.getResponseCode() == 404) { 103 | 104 | } else { 105 | 106 | throw new IOException(jestResult.getResponseCode()+","+jestResult.getErrorMessage()); 107 | 108 | } 109 | 110 | 111 | 112 | jestResult = jestClient.execute(new IndicesExists.Builder(indexName).build()); 113 | if (jestResult.isSucceeded()) { 114 | return true; 115 | }else if(jestResult.getResponseCode()==404){ 116 | return false; 117 | 118 | }else{ 119 | throw new IOException(jestResult.getResponseCode()+","+jestResult.getErrorMessage()); 120 | } 121 | 122 | 123 | 124 | 125 | } 126 | 127 | 128 | 129 | public JestClient getJestClient() { 130 | return jestClient; 131 | } 132 | 133 | 134 | private JestClient createJestClient(Es es) { 135 | JestClientFactory factory = new JestClientFactory(); 136 | List addresses = Arrays.asList(es.getAddresses().split(";")); 137 | factory.setHttpClientConfig(new HttpClientConfig 138 | .Builder(addresses)// "http://localhost:9200") 139 | .multiThreaded(true) 140 | .connTimeout(10 * 000) 141 | .readTimeout(10 * 000) 142 | //Per default this implementation will create no more than 2 concurrent connections per given route 143 | // .defaultMaxTotalConnectionPerRoute() 144 | // and no more 20 connections in total 145 | // .maxTotalConnection() 146 | .build()); 147 | JestClient jestClient = factory.getObject(); 148 | return jestClient; 149 | } 150 | } 151 | -------------------------------------------------------------------------------- /easy-sync/src/main/java/com/cehome/easysync/objects/Column.java: -------------------------------------------------------------------------------- 1 | package com.cehome.easysync.objects; 2 | 3 | public class Column { 4 | private String name; 5 | 6 | //mysql type :bigint 7 | private String dataType; 8 | //mysql type detail : bigint(20) 9 | private String columnType; 10 | 11 | private String charset;//collation; 12 | 13 | 14 | public String getName() { 15 | return name; 16 | } 17 | 18 | public void setName(String name) { 19 | this.name = name; 20 | } 21 | 22 | public String getCharset() { 23 | return charset; 24 | } 25 | 26 | public void setCharset(String charset) { 27 | this.charset = charset; 28 | } 29 | 30 | public String getDataType() { 31 | return dataType; 32 | } 33 | 34 | public void setDataType(String dataType) { 35 | this.dataType = dataType; 36 | } 37 | 38 | public String getColumnType() { 39 | return columnType; 40 | } 41 | 42 | public void setColumnType(String columnType) { 43 | this.columnType = columnType; 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /easy-sync/src/main/java/com/cehome/easysync/objects/MysqlUrl.java: -------------------------------------------------------------------------------- 1 | package com.cehome.easysync.objects; 2 | 3 | import java.util.HashMap; 4 | import java.util.Map; 5 | 6 | public class MysqlUrl { 7 | private String host; 8 | private String port; 9 | 10 | private String url; 11 | private long connectTimeout=5000; 12 | private boolean useSSL; 13 | 14 | // jdbc:mysql://127.0.0.1:3306?zeroDateTimeBehavior=convertToNull&connectTimeout=5000 15 | public MysqlUrl(String url){ 16 | try { 17 | this.url = url; 18 | int n1 = url.indexOf("://"); 19 | int n2 = url.indexOf("?"); 20 | 21 | if (n2 == -1) { 22 | n2 = url.length(); 23 | } else { 24 | String p = url.substring(n2 + 1); 25 | String[] params = p.split("&"); 26 | Map map = new HashMap<>(); 27 | for (String param : params) { 28 | String[] ss = param.split("="); 29 | map.put(ss[0], ss[1]); 30 | } 31 | 32 | String value = map.get("connectTimeout"); 33 | if (value != null) { 34 | setConnectTimeout(Long.parseLong(value)); 35 | } 36 | 37 | value = map.get("useSSL"); 38 | if (value != null) { 39 | setUseSSL(Boolean.parseBoolean(value)); 40 | } 41 | 42 | 43 | } 44 | String h = url.substring(n1 + 3, n2); 45 | n1 = h.indexOf(':'); 46 | setHost(h.substring(0, n1)); 47 | setPort(h.substring(n1 + 1)); 48 | }catch (Exception e){ 49 | throw new RuntimeException(url+" is not valid mysql url.",e); 50 | } 51 | 52 | } 53 | 54 | 55 | public String getHost() { 56 | return host; 57 | } 58 | 59 | public void setHost(String host) { 60 | this.host = host; 61 | } 62 | 63 | public String getPort() { 64 | return port; 65 | } 66 | 67 | public void setPort(String port) { 68 | this.port = port; 69 | } 70 | 71 | public String getUrl() { 72 | return url; 73 | } 74 | 75 | public long getConnectTimeout() { 76 | return connectTimeout; 77 | } 78 | 79 | public void setConnectTimeout(long connectTimeout) { 80 | this.connectTimeout = connectTimeout; 81 | } 82 | 83 | public boolean isUseSSL() { 84 | return useSSL; 85 | } 86 | 87 | public void setUseSSL(boolean useSSL) { 88 | this.useSSL = useSSL; 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /easy-sync/src/main/java/com/cehome/easysync/objects/Row.java: -------------------------------------------------------------------------------- 1 | package com.cehome.easysync.objects; 2 | 3 | 4 | import jsharp.util.DataMap; 5 | 6 | public class Row { 7 | private String type; 8 | private String database; 9 | private String table; 10 | private Long timestamp; 11 | private Long serverId; 12 | 13 | private DataMap data=new DataMap(); 14 | private DataMap oldData; 15 | 16 | public String getType() { 17 | return type; 18 | } 19 | 20 | public void setType(String type) { 21 | this.type = type; 22 | } 23 | 24 | public String getDatabase() { 25 | return database; 26 | } 27 | 28 | public void setDatabase(String database) { 29 | this.database = database; 30 | } 31 | 32 | public String getTable() { 33 | return table; 34 | } 35 | 36 | public void setTable(String table) { 37 | this.table = table; 38 | } 39 | 40 | public Long getTimestamp() { 41 | return timestamp; 42 | } 43 | 44 | public void setTimestamp(Long timestamp) { 45 | this.timestamp = timestamp; 46 | } 47 | 48 | public Long getServerId() { 49 | return serverId; 50 | } 51 | 52 | public void setServerId(Long serverId) { 53 | this.serverId = serverId; 54 | } 55 | 56 | public DataMap getData() { 57 | return data; 58 | } 59 | 60 | public void setData(DataMap data) { 61 | this.data = data; 62 | } 63 | 64 | public DataMap getOldData() { 65 | return oldData; 66 | } 67 | 68 | public void setOldData(DataMap oldData) { 69 | this.oldData = oldData; 70 | } 71 | 72 | } 73 | -------------------------------------------------------------------------------- /easy-sync/src/main/java/com/cehome/easysync/objects/Table.java: -------------------------------------------------------------------------------- 1 | package com.cehome.easysync.objects; 2 | 3 | public class Table { 4 | private String database; 5 | private String table; 6 | private Column[] columns; 7 | //private String[] 8 | } 9 | -------------------------------------------------------------------------------- /easy-sync/src/main/java/com/cehome/easysync/objects/TableTask.java: -------------------------------------------------------------------------------- 1 | package com.cehome.easysync.objects; 2 | 3 | public class TableTask { 4 | private long timeTaskId; 5 | private String keys; 6 | 7 | public long getTimeTaskId() { 8 | return timeTaskId; 9 | } 10 | 11 | public void setTimeTaskId(long timeTaskId) { 12 | this.timeTaskId = timeTaskId; 13 | } 14 | 15 | public String getKeys() { 16 | return keys; 17 | } 18 | 19 | public void setKeys(String keys) { 20 | this.keys = keys; 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /easy-sync/src/main/java/com/cehome/easysync/objects/config/Es.java: -------------------------------------------------------------------------------- 1 | package com.cehome.easysync.objects.config; 2 | 3 | public class Es { 4 | private String addresses; 5 | private String analyzer; 6 | private String indexAlias; 7 | private String indexSettings; 8 | // 0 <5 1 >=5 9 | private int version; 10 | 11 | public String getAddresses() { 12 | return addresses; 13 | } 14 | 15 | public void setAddresses(String addresses) { 16 | this.addresses = addresses; 17 | } 18 | 19 | public String getAnalyzer() { 20 | return analyzer; 21 | } 22 | 23 | public void setAnalyzer(String analyzer) { 24 | this.analyzer = analyzer; 25 | } 26 | 27 | public String getIndexAlias() { 28 | return indexAlias; 29 | } 30 | 31 | public void setIndexAlias(String indexAlias) { 32 | this.indexAlias = indexAlias; 33 | } 34 | 35 | public String getIndexSettings() { 36 | return indexSettings; 37 | } 38 | 39 | public void setIndexSettings(String indexSettings) { 40 | this.indexSettings = indexSettings; 41 | } 42 | 43 | public int getVersion() { 44 | return version; 45 | } 46 | 47 | public void setVersion(int version) { 48 | this.version = version; 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /easy-sync/src/main/java/com/cehome/easysync/objects/config/EsConfig.java: -------------------------------------------------------------------------------- 1 | package com.cehome.easysync.objects.config; 2 | 3 | public class EsConfig { 4 | private MysqlSource mysqlSource; 5 | private Es es; 6 | private EsMapping esMapping; 7 | private Rebuild rebuild; 8 | private Repair repair; 9 | 10 | public MysqlSource getMysqlSource() { 11 | return mysqlSource; 12 | } 13 | 14 | public void setMysqlSource(MysqlSource mysqlSource) { 15 | this.mysqlSource = mysqlSource; 16 | } 17 | 18 | public Es getEs() { 19 | return es; 20 | } 21 | 22 | public void setEs(Es es) { 23 | this.es = es; 24 | } 25 | 26 | public EsMapping getEsMapping() { 27 | return esMapping; 28 | } 29 | 30 | public void setEsMapping(EsMapping esMapping) { 31 | this.esMapping = esMapping; 32 | } 33 | 34 | public Rebuild getRebuild() { 35 | return rebuild; 36 | } 37 | 38 | public void setRebuild(Rebuild rebuild) { 39 | this.rebuild = rebuild; 40 | } 41 | 42 | public Repair getRepair() { 43 | return repair; 44 | } 45 | 46 | public void setRepair(Repair repair) { 47 | this.repair = repair; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /easy-sync/src/main/java/com/cehome/easysync/objects/config/EsField.java: -------------------------------------------------------------------------------- 1 | package com.cehome.easysync.objects.config; 2 | 3 | public class EsField { 4 | private String source; 5 | private String sourceType; 6 | private String target; 7 | private String type; 8 | private String custom; 9 | 10 | public String getSource() { 11 | return source; 12 | } 13 | 14 | public void setSource(String source) { 15 | this.source = source; 16 | } 17 | 18 | public String getSourceType() { 19 | return sourceType; 20 | } 21 | 22 | public void setSourceType(String sourceType) { 23 | this.sourceType = sourceType; 24 | } 25 | 26 | public String getTarget() { 27 | return target; 28 | } 29 | 30 | public void setTarget(String target) { 31 | this.target = target; 32 | } 33 | 34 | public String getType() { 35 | return type; 36 | } 37 | 38 | public void setType(String type) { 39 | this.type = type; 40 | } 41 | 42 | public String getCustom() { 43 | return custom; 44 | } 45 | 46 | public void setCustom(String custom) { 47 | this.custom = custom; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /easy-sync/src/main/java/com/cehome/easysync/objects/config/EsMapping.java: -------------------------------------------------------------------------------- 1 | package com.cehome.easysync.objects.config; 2 | 3 | public class EsMapping { 4 | 5 | private EsField[] esFields; 6 | 7 | 8 | 9 | public EsField[] getEsFields() { 10 | return esFields; 11 | } 12 | 13 | public void setEsFields(EsField[] esFields) { 14 | this.esFields = esFields; 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /easy-sync/src/main/java/com/cehome/easysync/objects/config/Kafka.java: -------------------------------------------------------------------------------- 1 | package com.cehome.easysync.objects.config; 2 | 3 | public class Kafka { 4 | private String version="0.10.1.0"; 5 | private String servers; 6 | private String producerConfigs; 7 | private String consumerConfigs; 8 | /* private String acks; 9 | private String retries; 10 | private String batchSize; 11 | private String lingerMs; 12 | private String bufferMemory; 13 | private String keySerializer; 14 | private String valueSerializer; 15 | private String messageMaxBytes; 16 | private String replicaFetchMaxBytes; 17 | 18 | private String enableAutoCommit; 19 | private String keyDeserializer; 20 | private String valueDeserializer; 21 | private String fetchMessageMaxBytes;*/ 22 | 23 | public String getServers() { 24 | return servers; 25 | } 26 | 27 | public void setServers(String servers) { 28 | this.servers = servers; 29 | } 30 | 31 | public String getVersion() { 32 | return version; 33 | } 34 | 35 | public void setVersion(String version) { 36 | this.version = version; 37 | } 38 | 39 | public String getProducerConfigs() { 40 | return producerConfigs; 41 | } 42 | 43 | public void setProducerConfigs(String producerConfigs) { 44 | this.producerConfigs = producerConfigs; 45 | } 46 | 47 | public String getConsumerConfigs() { 48 | return consumerConfigs; 49 | } 50 | 51 | public void setConsumerConfigs(String consumerConfigs) { 52 | this.consumerConfigs = consumerConfigs; 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /easy-sync/src/main/java/com/cehome/easysync/objects/config/Mysql.java: -------------------------------------------------------------------------------- 1 | package com.cehome.easysync.objects.config; 2 | 3 | import java.io.Serializable; 4 | 5 | public class Mysql implements Serializable { 6 | 7 | //defalut = jdbc:mysql://127.0.0.1:3306?zeroDateTimeBehavior=convertToNull&connectTimeout=5000 8 | private String url; 9 | private String user; 10 | private String password; 11 | //default 12 | //private String bingLogCharset; 13 | 14 | 15 | public String getUser() { 16 | return user; 17 | } 18 | 19 | public void setUser(String user) { 20 | this.user = user; 21 | } 22 | 23 | public String getPassword() { 24 | return password; 25 | } 26 | 27 | public void setPassword(String password) { 28 | this.password = password; 29 | } 30 | 31 | public String getUrl() { 32 | return url; 33 | } 34 | 35 | public void setUrl(String url) { 36 | this.url = url; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /easy-sync/src/main/java/com/cehome/easysync/objects/config/MysqlConfig.java: -------------------------------------------------------------------------------- 1 | package com.cehome.easysync.objects.config; 2 | 3 | public class MysqlConfig { 4 | 5 | private Mysql mysql; 6 | private Kafka kafka; 7 | 8 | public Mysql getMysql() { 9 | return mysql; 10 | } 11 | 12 | public void setMysql(Mysql mysql) { 13 | this.mysql = mysql; 14 | } 15 | 16 | public Kafka getKafka() { 17 | return kafka; 18 | } 19 | 20 | public void setKafka(Kafka kafka) { 21 | this.kafka = kafka; 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /easy-sync/src/main/java/com/cehome/easysync/objects/config/MysqlSource.java: -------------------------------------------------------------------------------- 1 | package com.cehome.easysync.objects.config; 2 | 3 | public class MysqlSource { 4 | private long timeTaskId; 5 | private String databases; 6 | private String tables; 7 | private String keys; 8 | private String keySep; 9 | 10 | public long getTimeTaskId() { 11 | return timeTaskId; 12 | } 13 | 14 | public void setTimeTaskId(long timeTaskId) { 15 | this.timeTaskId = timeTaskId; 16 | } 17 | 18 | public String getDatabases() { 19 | return databases; 20 | } 21 | 22 | public void setDatabases(String databases) { 23 | this.databases = databases; 24 | } 25 | 26 | public String getTables() { 27 | return tables; 28 | } 29 | 30 | public void setTables(String tables) { 31 | this.tables = tables; 32 | } 33 | 34 | public String getKeys() { 35 | return keys; 36 | } 37 | 38 | public void setKeys(String keys) { 39 | this.keys = keys; 40 | } 41 | 42 | public String getKeySep() { 43 | return keySep; 44 | } 45 | 46 | public void setKeySep(String keySep) { 47 | this.keySep = keySep; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /easy-sync/src/main/java/com/cehome/easysync/objects/config/Rebuild.java: -------------------------------------------------------------------------------- 1 | package com.cehome.easysync.objects.config; 2 | 3 | public class Rebuild { 4 | private boolean enableFullSync; 5 | private String fullWhere; 6 | private int fullBatchSize; 7 | private int fullBatchInterval; 8 | private boolean switchAfterFullSync; 9 | private boolean deleteOldIndex; 10 | 11 | public boolean isEnableFullSync() { 12 | return enableFullSync; 13 | } 14 | 15 | public void setEnableFullSync(boolean enableFullSync) { 16 | this.enableFullSync = enableFullSync; 17 | } 18 | 19 | public String getFullWhere() { 20 | return fullWhere; 21 | } 22 | 23 | public void setFullWhere(String fullWhere) { 24 | this.fullWhere = fullWhere; 25 | } 26 | 27 | public int getFullBatchSize() { 28 | return fullBatchSize; 29 | } 30 | 31 | public void setFullBatchSize(int fullBatchSize) { 32 | this.fullBatchSize = fullBatchSize; 33 | } 34 | 35 | public int getFullBatchInterval() { 36 | return fullBatchInterval; 37 | } 38 | 39 | public void setFullBatchInterval(int fullBatchInterval) { 40 | this.fullBatchInterval = fullBatchInterval; 41 | } 42 | 43 | public boolean isSwitchAfterFullSync() { 44 | return switchAfterFullSync; 45 | } 46 | 47 | public void setSwitchAfterFullSync(boolean switchAfterFullSync) { 48 | this.switchAfterFullSync = switchAfterFullSync; 49 | } 50 | 51 | public boolean isDeleteOldIndex() { 52 | return deleteOldIndex; 53 | } 54 | 55 | public void setDeleteOldIndex(boolean deleteOldIndex) { 56 | this.deleteOldIndex = deleteOldIndex; 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /easy-sync/src/main/java/com/cehome/easysync/objects/config/Repair.java: -------------------------------------------------------------------------------- 1 | package com.cehome.easysync.objects.config; 2 | 3 | public class Repair { 4 | 5 | private String where; 6 | private int batchSize; 7 | private int batchInterval; 8 | 9 | public String getWhere() { 10 | return where; 11 | } 12 | 13 | public void setWhere(String where) { 14 | this.where = where; 15 | } 16 | 17 | public int getBatchSize() { 18 | return batchSize; 19 | } 20 | 21 | public void setBatchSize(int batchSize) { 22 | this.batchSize = batchSize; 23 | } 24 | 25 | public int getBatchInterval() { 26 | return batchInterval; 27 | } 28 | 29 | public void setBatchInterval(int batchInterval) { 30 | this.batchInterval = batchInterval; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /easy-sync/src/main/java/com/cehome/easysync/plugins/EsSyncIncrement.java: -------------------------------------------------------------------------------- 1 | package com.cehome.easysync.plugins; 2 | 3 | import com.alibaba.fastjson.JSON; 4 | import com.cehome.easykafka.Consumer; 5 | import com.cehome.easykafka.consumer.ConsumerRecord; 6 | import com.cehome.easykafka.consumer.SimpleKafkaConsumer; 7 | import com.cehome.easysync.jest.Jest; 8 | import com.cehome.easysync.objects.Column; 9 | import com.cehome.easysync.objects.Row; 10 | import com.cehome.easysync.objects.config.EsConfig; 11 | import com.cehome.easysync.objects.config.EsField; 12 | import com.cehome.easysync.objects.config.Kafka; 13 | import com.cehome.easysync.service.DatabaseService; 14 | import com.cehome.easysync.utils.Const; 15 | import com.cehome.easysync.utils.Global; 16 | import com.cehome.task.client.TimeTaskContext; 17 | import io.searchbox.core.Bulk; 18 | import io.searchbox.core.BulkResult; 19 | import io.searchbox.core.Delete; 20 | import io.searchbox.core.Index; 21 | import jsharp.util.Common; 22 | import jsharp.util.DataMap; 23 | import org.apache.commons.beanutils.ConvertUtils; 24 | import org.apache.commons.lang.StringUtils; 25 | import org.slf4j.Logger; 26 | import org.slf4j.LoggerFactory; 27 | import org.slf4j.MDC; 28 | 29 | import java.io.UnsupportedEncodingException; 30 | import java.util.*; 31 | import java.util.concurrent.atomic.AtomicLong; 32 | 33 | public class EsSyncIncrement extends Thread { 34 | private static final Logger logger = LoggerFactory.getLogger(EsSyncIncrement.class); 35 | 36 | private TimeTaskContext context; 37 | private Kafka kafka; 38 | private volatile String indexName; 39 | 40 | Jest jest; 41 | private EsConfig esConfig; 42 | private EsField[] esFields; 43 | Map columnMap; 44 | private Consumer consumer = null; 45 | DatabaseService databaseService; 46 | Properties props = new Properties(); 47 | private volatile boolean running = true; 48 | private AtomicLong atomicLong = new AtomicLong(); 49 | private static AtomicLong clientId=new AtomicLong(0); 50 | 51 | public EsSyncIncrement(TimeTaskContext context, DatabaseService databaseService, Kafka kafka, String indexName, Jest jest, String taskConfig) throws Exception { 52 | this.context = context; 53 | this.kafka = kafka; 54 | this.indexName = indexName; 55 | this.databaseService=databaseService; 56 | 57 | this.jest = jest; 58 | esConfig = Global.toObject(taskConfig, EsConfig.class); 59 | esFields= esConfig.getEsMapping().getEsFields(); 60 | 61 | } 62 | 63 | /* public String getIndexName() { 64 | return indexName; 65 | }*/ 66 | 67 | private void createKafkaConsumer() throws Exception{ 68 | if(consumer!=null)Common.closeObject(consumer); 69 | consumer = new SimpleKafkaConsumer(kafka.getVersion(),props); 70 | consumer.createKafkaConsumer(); 71 | consumer.subscribe(Const.TOPIC_PREFIX + context.getId()); 72 | } 73 | 74 | @Override 75 | public void run() { 76 | props.put("client.id",String.valueOf(context.getId() + atomicLong.incrementAndGet())); 77 | props.put("bootstrap.servers", kafka.getServers()); 78 | props.put("group.id", "es_" + indexName); 79 | 80 | props.put("enable.auto.commit", "false"); 81 | props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); 82 | props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); 83 | props.put("fetch.message.max.bytes", "10MB"); 84 | /*solution for Caused by: org.apache.kafka.clients.consumer.CommitFailedException: Commit cannot be completed since the group has already rebalanced and assigned the partitions to another member. This means that the time between subsequent calls to poll() was longer than the configured max.poll.interval.ms, which typically implies that the poll loop is spending too much time message processing. You can address this either by increasing the session timeout or by reducing the maximum size of batches returned in poll() with max.poll.records.*/ 85 | props.put("session.timeout.ms", "30000"); 86 | 87 | //kafka在0.9版本无max.poll.records参数，默认拉取记录是500，直到0.10版本才引入该参数 88 | props.put("max.poll.records", "10"); 89 | 90 | //消息发送的最长等待时间.需大于session.timeout.ms这个时间 91 | props.put("request.timeout.ms", "40000"); 92 | 93 | 94 | 95 | props.put("client.id",""+clientId.incrementAndGet()); 96 | if(StringUtils.isNotBlank(kafka.getConsumerConfigs())){ 97 | String[] lines=kafka.getConsumerConfigs().split("[\\r\\n]+"); 98 | for (String line:lines){ 99 | line=line.trim(); 100 | if(line.length()==0) continue; 101 | String[] e= line.split("\\s+=\\s+"); 102 | props.put(e[0],e[1]); 103 | } 104 | } 105 | /* props.put("group.id", "es_" + indexName); 106 | props.put("enable.auto.commit", kafka.getEnableAutoCommit()); 107 | props.put("key.deserializer", kafka.getKeyDeserializer()); 108 | props.put("value.deserializer", kafka.getValueDeserializer()); 109 | props.put("fetch.message.max.bytes", kafka.getFetchMessageMaxBytes());*/ 110 | final int minBatchSize = 10; 111 | MDC.put("shard", "task/"+context.getId()); 112 | while (canRun()) { 113 | try { 114 | 115 | createKafkaConsumer(); 116 | break; 117 | } catch (Exception e) { 118 | logger.error("", e); 119 | context.sleep(3000); 120 | } 121 | } 122 | 123 | //List> buffer = new ArrayList<>(); 124 | 125 | while (canRun()) { 126 | 127 | try { 128 | List records = (List) consumer.poll(500); 129 | if(!records.isEmpty()) { 130 | doSync(records); 131 | } 132 | consumer.commitSync(); 133 | } catch (Exception e) { 134 | logger.error("", e); 135 | context.sleep(3000); 136 | 137 | 138 | } 139 | 140 | } 141 | Common.closeObject(consumer); 142 | 143 | MDC.remove("shard"); 144 | } 145 | 146 | 147 | private boolean canRun() { 148 | return context.isRunning() && running; 149 | } 150 | 151 | 152 | public void terminate(long waitMS) { 153 | running = false; 154 | if (waitMS > 0) { 155 | try { 156 | this.join(waitMS); 157 | } catch (InterruptedException e) { 158 | logger.error("", e); 159 | } 160 | } 161 | } 162 | 163 | 164 | private Map convertKafka2Es(Map dataMap) throws UnsupportedEncodingException { 165 | 166 | DataMap result=new DataMap(); 167 | for(EsField esField :esFields){ 168 | String name=esField.getSource(); 169 | String targetType=esField.getType(); 170 | Column column=columnMap.get(name); 171 | Object value=dataMap.get(name); 172 | if(value!=null) { 173 | if ("boolean".equalsIgnoreCase(targetType)) { 174 | value = ConvertUtils.convert(value, Boolean.class); 175 | } else if ("date".equalsIgnoreCase(targetType)) { 176 | if (value instanceof Date) { 177 | value = ((Date) value).getTime(); 178 | } 179 | } else if ("binary".equalsIgnoreCase(targetType)) { 180 | if (value instanceof byte[]) { 181 | value = org.apache.commons.codec.binary.Base64.encodeBase64String((byte[]) value); 182 | } else { 183 | value = org.apache.commons.codec.binary.Base64.encodeBase64String(value.toString().getBytes("UTF-8")); 184 | } 185 | } 186 | } 187 | 188 | result.put(esField.getTarget(),value); 189 | 190 | 191 | } 192 | 193 | 194 | return result; 195 | 196 | 197 | } 198 | 199 | private void doSync(List records) throws Exception { 200 | 201 | Bulk.Builder bulk = new Bulk.Builder().defaultIndex(indexName).defaultType(Const.ES_TYPE); 202 | for (ConsumerRecord record : records) { 203 | logger.info("[incr] {}={}",indexName,record.value()); 204 | Row row = JSON.parseObject(record.value(), Row.class); 205 | if(columnMap==null){ 206 | columnMap=databaseService.getColumnMap(row.getDatabase(),row.getTable()); 207 | } 208 | String id = record.key(); 209 | if (row.getType().equalsIgnoreCase("insert") || (row.getType().equalsIgnoreCase("update"))) { 210 | LinkedHashMap data = row.getData(); 211 | Map map = (convertKafka2Es(data)); 212 | Index index = new Index.Builder(map).id(id).build(); 213 | bulk.addAction(index); 214 | 215 | } else if (row.getType().equalsIgnoreCase("delete")) { 216 | Delete delete = new Delete.Builder(id).build(); 217 | bulk.addAction(delete); 218 | } else { 219 | // 220 | } 221 | } 222 | 223 | 224 | BulkResult br = jest.getJestClient().execute(bulk.build()); 225 | if (!br.isSucceeded()) { 226 | logger.error("error={}, failItems={}", br.getErrorMessage(), JSON.toJSONString(br.getFailedItems())); 227 | // br.getFailedItems().get(0). 228 | throw new RuntimeException("bulk error"); 229 | } 230 | 231 | 232 | // buffer.add(record); 233 | 234 | } 235 | } 236 | -------------------------------------------------------------------------------- /easy-sync/src/main/java/com/cehome/easysync/plugins/EsSyncPlugin.java: -------------------------------------------------------------------------------- 1 | package com.cehome.easysync.plugins; 2 | 3 | import com.alibaba.fastjson.JSONObject; 4 | import com.cehome.easysync.Application; 5 | import com.cehome.easysync.dao.SyncConfigDao; 6 | import com.cehome.task.client.TimeTaskContext; 7 | import com.cehome.task.client.TimeTaskPlugin; 8 | import com.cehome.task.dao.TimeTaskDao; 9 | import org.slf4j.Logger; 10 | import org.slf4j.LoggerFactory; 11 | import org.springframework.beans.factory.annotation.Autowired; 12 | import org.springframework.stereotype.Component; 13 | 14 | import java.net.Inet4Address; 15 | 16 | @Component 17 | public class EsSyncPlugin extends TimeTaskPlugin { 18 | private static final Logger logger = LoggerFactory.getLogger(EsSyncPlugin.class); 19 | @Autowired 20 | private SyncConfigDao syncConfigDao; 21 | @Autowired 22 | private TimeTaskDao timeTaskDao; 23 | @Override 24 | public void run(TimeTaskContext context, JSONObject args) throws Exception { 25 | logger.info("task id="+context.getId()); 26 | logger.info("task name="+context.getName()); 27 | logger.info("task run on ip="+ Inet4Address.getLocalHost().getHostAddress()); 28 | logger.info("get es task config"); 29 | EsSyncPluginRunner esSync= Application.getBean(EsSyncPluginRunner.class); 30 | esSync.run(context); 31 | 32 | logger.info("task run count="+context.getRunTimes()); 33 | } 34 | 35 | @Override 36 | public void stop(TimeTaskContext context) throws Exception { 37 | logger.info("task "+context.getName()+" is stopped "); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /easy-sync/src/main/java/com/cehome/easysync/plugins/MysqlSyncListener.java: -------------------------------------------------------------------------------- 1 | package com.cehome.easysync.plugins; 2 | 3 | import com.alibaba.fastjson.JSON; 4 | import com.cehome.easykafka.Producer; 5 | import com.cehome.easysync.domain.Position; 6 | import com.cehome.easysync.objects.Column; 7 | import com.cehome.easysync.objects.Row; 8 | import com.cehome.easysync.objects.TableTask; 9 | import com.cehome.easysync.service.DatabaseService; 10 | import com.cehome.easysync.service.PositionSaveService; 11 | import com.cehome.easysync.service.TableTasksMapService; 12 | import com.cehome.easysync.utils.Const; 13 | import com.cehome.easysync.utils.MysqlTypeUtils; 14 | import com.cehome.task.client.TimeTaskContext; 15 | import com.github.shyiko.mysql.binlog.BinaryLogClient; 16 | import com.github.shyiko.mysql.binlog.event.*; 17 | import jsharp.util.TimeCal; 18 | import org.apache.commons.collections.CollectionUtils; 19 | import org.slf4j.Logger; 20 | import org.slf4j.LoggerFactory; 21 | import org.springframework.beans.factory.annotation.Autowired; 22 | import org.springframework.context.annotation.Scope; 23 | import org.springframework.stereotype.Component; 24 | 25 | import java.io.IOException; 26 | import java.io.Serializable; 27 | import java.io.UnsupportedEncodingException; 28 | import java.util.*; 29 | 30 | @Component 31 | @Scope("prototype") 32 | public class MysqlSyncListener implements BinaryLogClient.EventListener { 33 | private static final Logger logger = LoggerFactory.getLogger(MysqlSyncListener.class); 34 | private Position position = null; 35 | private String database; 36 | private String table; 37 | private long tableId; 38 | private List tableTasks; 39 | BinaryLogClient client; 40 | DatabaseService databaseService; 41 | private long timezoneInMS; 42 | Producer producer; 43 | TimeTaskContext context; 44 | TimeCal timeCal1 = new TimeCal(5); 45 | TimeCal timeCal2 = new TimeCal(30); 46 | private volatile boolean running = true; 47 | 48 | 49 | @Autowired 50 | TableTasksMapService tableTasksMapService; 51 | 52 | 53 | private PositionSaveService positionSaveService; 54 | 55 | public MysqlSyncListener() { 56 | 57 | } 58 | 59 | public void init(TimeTaskContext context, Producer producer, BinaryLogClient client, DatabaseService databaseService, PositionSaveService positionSaveService) { 60 | this.context = context; 61 | this.producer = producer; 62 | this.client = client; 63 | this.databaseService = databaseService; 64 | this.positionSaveService = positionSaveService; 65 | this.timezoneInMS = databaseService.getTimezoneHours() * 3600 * 1000; 66 | 67 | /* position=positionService.getPosition(databaseService.getTimeTaskId(),databaseService.getServerId()); 68 | if(position==null) { 69 | position = EntityUtils.create(Position.class); 70 | position.setTimeTaskId(databaseService.getTimeTaskId()); 71 | position.setServerId(databaseService.getServerId()); 72 | }*/ 73 | 74 | 75 | } 76 | 77 | public boolean isRunning() { 78 | return running; 79 | } 80 | 81 | public void setRunning(boolean running) { 82 | this.running = running; 83 | } 84 | 85 | @Override 86 | public void onEvent(Event event) { 87 | try { 88 | doOnEvent(event); 89 | } catch (Throwable e) { 90 | //logger.error("event error", e); 91 | setRunning(false); 92 | // in order to jump out BinlogClient loop and end 93 | throw new Error(e); 94 | /* new Thread(new Runnable() { 95 | @Override 96 | public void run() { 97 | try { 98 | //disconnect in order to restart with last postion, then will not lose any event. 99 | client.disconnect(); 100 | logger.info("disconnect ok"); 101 | } catch (IOException e1) { 102 | e1.printStackTrace(); 103 | } 104 | } 105 | }).start(); 106 | */ 107 | 108 | 109 | } 110 | } 111 | 112 | 113 | public void doOnEvent(Event event) throws Exception { 114 | logger.debug("" + event); 115 | 116 | 117 | TimeCal timeCal = timeCal2; 118 | EventType eventType = event.getHeader().getEventType(); 119 | if (eventType == EventType.GTID) { 120 | String gtid = ((GtidEventData) event.getData()).getGtid(); 121 | } 122 | if (eventType == EventType.TABLE_MAP) { 123 | TableMapEventData data = (TableMapEventData) event.getData(); 124 | database = data.getDatabase(); 125 | table = data.getTable(); 126 | tableId = data.getTableId(); 127 | Map> tableTasksMap = tableTasksMapService.get(databaseService); 128 | tableTasks = tableTasksMap.get(database + "." + table); 129 | 130 | } else if (CollectionUtils.isNotEmpty(tableTasks)) { 131 | ArrayList list = new ArrayList<>(); 132 | if (eventType == EventType.WRITE_ROWS || eventType == EventType.EXT_WRITE_ROWS) { 133 | WriteRowsEventData data = (WriteRowsEventData) event.getData(); 134 | for (Serializable[] rowData : data.getRows()) { 135 | list.add(buildRow("insert", rowData, data.getIncludedColumns())); 136 | } 137 | 138 | } else if (eventType == EventType.UPDATE_ROWS || eventType == EventType.EXT_UPDATE_ROWS) { 139 | 140 | UpdateRowsEventData data = (UpdateRowsEventData) event.getData(); 141 | 142 | for (Map.Entry e : data.getRows()) { 143 | Serializable[] rowData = e.getValue(); 144 | list.add(buildRow("update", rowData, data.getIncludedColumns())); 145 | } 146 | 147 | 148 | } else if (eventType == EventType.DELETE_ROWS || eventType == EventType.EXT_DELETE_ROWS) { 149 | DeleteRowsEventData data = (DeleteRowsEventData) event.getData(); 150 | for (Serializable[] rowData : data.getRows()) { 151 | list.add(buildRow("delete", rowData, data.getIncludedColumns())); 152 | } 153 | } 154 | 155 | 156 | //todo: 事务性 157 | for (TableTask tableTask : tableTasks) { 158 | 159 | for (Row row : list) { 160 | 161 | String id = row.getData().getStr(tableTask.getKeys()); 162 | try { 163 | send(Const.TOPIC_PREFIX + tableTask.getTimeTaskId(), id, JSON.toJSONString(row)); 164 | logger.info("[binlog] success task={}, {}", tableTask.getTimeTaskId(), JSON.toJSONString(row)); 165 | } catch (Exception e) { 166 | logger.error("[binlog] fail task={}, {}", tableTask.getTimeTaskId(), JSON.toJSONString(row)); 167 | throw e; 168 | } 169 | //logger 170 | 171 | } 172 | 173 | } 174 | 175 | timeCal = timeCal1; 176 | 177 | 178 | } 179 | 180 | 181 | positionSaveService.set(client.getBinlogFilename(), client.getBinlogPosition(), client.getGtidSet()); 182 | if (timeCal.isTimeUp() > 0) { 183 | positionSaveService.save(); 184 | } 185 | 186 | 187 | } 188 | 189 | 190 | private void send(String topic, String key, String value) throws Exception { 191 | final int count = 20; 192 | for (int i = 0; i < count; i++) { 193 | if (!context.isRunning()) { 194 | throw new RuntimeException("Task is stopped, send end."); 195 | } 196 | try { 197 | producer.send(topic, key, value); 198 | return; 199 | } catch (Exception e) { 200 | if (i == count - 1) { 201 | logger.error("try {} times and exit", count); 202 | throw e; 203 | } else { 204 | if (i == 0) { 205 | logger.error("send error", e); 206 | } else { 207 | logger.error("send error: {}", e.getMessage()); 208 | } 209 | context.sleep(5000); 210 | } 211 | 212 | } 213 | } 214 | 215 | } 216 | 217 | 218 | private Row buildRow(String type, Serializable[] rowData, BitSet includedColumns) throws Exception { 219 | Row row = new Row(); 220 | row.setType(type); 221 | row.setDatabase(database); 222 | row.setTable(table); 223 | int dataIdx = 0; 224 | List columns = databaseService.getColumns(database, table); 225 | for (int i = 0; i < columns.size(); i++) { 226 | Column column = columns.get(i); 227 | if (includedColumns.get(i)) { 228 | Object value = null; 229 | if (rowData[dataIdx] != null) { 230 | value = convertColumn(column, rowData[dataIdx]); 231 | } 232 | row.getData().put(column.getName(), value); 233 | dataIdx++; 234 | } 235 | 236 | } 237 | 238 | return row; 239 | 240 | } 241 | 242 | /** 243 | * 数字型、日期型转long 不变 244 | * char/varchar/text 型转byte[]，需要根据字段编码还原 245 | * blob转byte[] 246 | * 247 | * @param object 248 | * @return 249 | */ 250 | private Object convertColumn(Column column, Object object) throws UnsupportedEncodingException { 251 | //https://www.cnblogs.com/waterystone/p/6226356.html 252 | 253 | if (object instanceof byte[]) { 254 | String dataType = column.getDataType(); 255 | if (MysqlTypeUtils.isString(dataType)) { 256 | byte[] bs = (byte[]) object; 257 | return column.getCharset() == null ? new String(bs) : new String(bs, column.getCharset()); 258 | } 259 | } else if (object instanceof Long) { 260 | String dataType = column.getDataType(); 261 | if (MysqlTypeUtils.isDateNormal(dataType)) { 262 | return ((Long) object) - timezoneInMS; 263 | } 264 | } 265 | 266 | return object; 267 | 268 | } 269 | 270 | 271 | } 272 | -------------------------------------------------------------------------------- /easy-sync/src/main/java/com/cehome/easysync/plugins/MysqlSyncPlugin.java: -------------------------------------------------------------------------------- 1 | package com.cehome.easysync.plugins; 2 | 3 | import com.alibaba.fastjson.JSON; 4 | import com.alibaba.fastjson.JSONObject; 5 | import com.cehome.easykafka.Producer; 6 | import com.cehome.easykafka.producer.SimpleKafkaProducer; 7 | import com.cehome.easysync.Application; 8 | import com.cehome.easysync.domain.Position; 9 | import com.cehome.easysync.objects.MysqlUrl; 10 | import com.cehome.easysync.objects.config.Kafka; 11 | import com.cehome.easysync.objects.config.Mysql; 12 | import com.cehome.easysync.objects.config.MysqlConfig; 13 | import com.cehome.easysync.service.DatabaseService; 14 | import com.cehome.easysync.service.PositionSaveService; 15 | import com.cehome.easysync.service.PositionService; 16 | import com.cehome.easysync.service.TableTasksMapService; 17 | import com.cehome.easysync.utils.Global; 18 | import com.cehome.task.client.TimeTaskContext; 19 | import com.cehome.task.client.TimeTaskPlugin; 20 | import com.github.shyiko.mysql.binlog.BinaryLogClient; 21 | import com.github.shyiko.mysql.binlog.event.deserialization.EventDeserializer; 22 | import jsharp.util.EntityUtils; 23 | import org.apache.commons.lang.StringUtils; 24 | import org.slf4j.Logger; 25 | import org.slf4j.LoggerFactory; 26 | import org.springframework.beans.factory.annotation.Autowired; 27 | import org.springframework.stereotype.Component; 28 | 29 | import java.net.Inet4Address; 30 | import java.util.Properties; 31 | 32 | /** 33 | * --读取基本配置 34 | * ----初始化Database服务，获取serverid 35 | * -- 获取 last position. 通过serverid 和 timetaskId去查找位置，如果找不到 36 | * 37 | */ 38 | @Component 39 | public class MysqlSyncPlugin extends TimeTaskPlugin { 40 | private static final Logger logger = LoggerFactory.getLogger(MysqlSyncPlugin.class); 41 | @Autowired 42 | private PositionService positionService; 43 | @Autowired 44 | TableTasksMapService tableTasksMapService; 45 | @Override 46 | public void run(TimeTaskContext context, JSONObject args) throws Exception { 47 | 48 | final String taskConfig=context.getTaskConfig(); 49 | long timeTaskId=context.getId(); 50 | MysqlConfig mysqlConfig= Global.toObject(taskConfig, MysqlConfig.class); 51 | Mysql mysql=mysqlConfig.getMysql(); 52 | MysqlUrl mysqlUrl=new MysqlUrl(mysql.getUrl()); 53 | logger.info("create database meta service"); 54 | final DatabaseService databaseService=DatabaseService.newInstance(mysql); 55 | databaseService.setTimeTaskId(timeTaskId); 56 | long serverId=databaseService.getServerId(); 57 | 58 | tableTasksMapService.clean(timeTaskId); 59 | 60 | logger.info("create kafka producer"); 61 | Kafka kafka=mysqlConfig.getKafka(); 62 | Properties props = new Properties(); 63 | props.put("bootstrap.servers", kafka.getServers()); 64 | props.put("acks", "all"); 65 | props.put("batch.size", 16384); 66 | props.put("linger.ms", 1); 67 | props.put("buffer.memory", 33554432); 68 | props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer"); 69 | props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer"); 70 | props.put("message.max.bytes", "10MB"); 71 | props.put("replica.fetch.max.bytes", "10MB"); 72 | props.put("retries", 0); 73 | //default 60s 74 | props.put("max.block.ms", "30000"); 75 | if(StringUtils.isNotBlank(kafka.getProducerConfigs())){ 76 | String[] lines=kafka.getProducerConfigs().split("[\\r\\n]+"); 77 | for (String line:lines){ 78 | line=line.trim(); 79 | if(line.length()==0) continue; 80 | String[] e= line.split("\\s*=\\s*"); 81 | props.put(e[0],e[1]); 82 | } 83 | } 84 | /*props.put("acks", kafka.getAcks()); 85 | props.put("retries", kafka.getRetries()); 86 | props.put("batch.size", kafka.getBatchSize()); 87 | props.put("linger.ms", kafka.getLingerMs()); 88 | props.put("buffer.memory", kafka.getBufferMemory()); 89 | props.put("key.serializer", kafka.getKeySerializer()); 90 | props.put("value.serializer", kafka.getValueSerializer()); 91 | props.put("message.max.bytes", kafka.getMessageMaxBytes()); 92 | props.put("replica.fetch.max.bytes", kafka.getReplicaFetchMaxBytes());*/ 93 | 94 | Producer producer = new SimpleKafkaProducer(kafka.getVersion(),props); 95 | context.put("producer",producer); 96 | 97 | BinaryLogClient client =null; 98 | PositionSaveService positionSaveService=null; 99 | while(context.isRunning()) { 100 | try { 101 | logger.info("create BinaryLogClient"); 102 | 103 | //client = new BinaryLogClient("192.168.0.13", 3306, "root", "asdf1234!"); 104 | //client.setConnectTimeout(30 * 1000); 105 | client = new BinaryLogClient(mysqlUrl.getHost(), Integer.parseInt(mysqlUrl.getPort()), mysql.getUser() , mysql.getPassword()); 106 | client.setConnectTimeout(mysqlUrl.getConnectTimeout()); 107 | client.setKeepAliveInterval(5 * 1000); 108 | //client.setKeepAlive(); 109 | EventDeserializer eventDeserializer = new EventDeserializer(); 110 | eventDeserializer.setCompatibilityMode( 111 | EventDeserializer.CompatibilityMode.DATE_AND_TIME_AS_LONG, 112 | EventDeserializer.CompatibilityMode.CHAR_AND_BINARY_AS_BYTE_ARRAY 113 | // EventDeserializer.CompatibilityMode.In INVALID_DATE_AND_TIME_AS_MIN_VALUE 114 | ); 115 | client.setEventDeserializer(eventDeserializer); 116 | logger.info("get last position"); 117 | Position position=positionService.getPosition(timeTaskId,serverId); 118 | if (position == null) { 119 | logger.info("last position is null. maybe first run"); 120 | position = EntityUtils.create(Position.class); 121 | position.setTimeTaskId(databaseService.getTimeTaskId()); 122 | position.setServerId(databaseService.getServerId()); 123 | } else { 124 | logger.info("last position :" + JSON.toJSONString(position)); 125 | client.setBinlogFilename(position.getFilename()); 126 | client.setBinlogPosition(position.getPosition()); 127 | client.setGtidSet(position.getGtidSet()); 128 | } 129 | positionSaveService=new PositionSaveService(positionService,position); 130 | 131 | 132 | MysqlSyncListener mysqlSyncListener= Application.getBean(MysqlSyncListener.class); 133 | mysqlSyncListener.init(context,producer, client, databaseService,positionSaveService); 134 | client.registerEventListener(mysqlSyncListener); 135 | synchronized (context) { 136 | if(context.isRunning()) { 137 | context.put("client", client); 138 | 139 | } 140 | } 141 | if(context.isRunning()) { 142 | logger.info("begin to connect"); 143 | client.connect(); 144 | //logger.info("after connect"); 145 | } 146 | while (context.isRunning() && mysqlSyncListener.isRunning()){ 147 | context.sleep(5000); 148 | //logger.info("sleep 50000"); 149 | } 150 | logger.info("this turn end"); 151 | 152 | } catch (Throwable e) { 153 | logger.error("MysqlSyncPlugin error",e); 154 | } 155 | try { 156 | client.disconnect(); 157 | } catch (Exception ex) { 158 | 159 | } 160 | if(positionSaveService!=null){ 161 | positionSaveService.save(); 162 | } 163 | context.sleep(5000); 164 | } 165 | 166 | System.out.println("task end"); 167 | 168 | 169 | 170 | logger.info("task id="+context.getId()); 171 | logger.info("task name="+context.getName()); 172 | logger.info("task run on ip="+ Inet4Address.getLocalHost().getHostAddress()); 173 | logger.info("task run count="+context.getRunTimes()); 174 | } 175 | 176 | 177 | 178 | 179 | 180 | 181 | 182 | @Override 183 | public void stop(TimeTaskContext context) throws Exception { 184 | synchronized (context) { 185 | try { 186 | BinaryLogClient client = (BinaryLogClient) context.get("client"); 187 | client.disconnect(); 188 | } catch (Exception e) { 189 | logger.error("BinaryLogClient disconnect error", e); 190 | } 191 | } 192 | try{ 193 | Producer producer=(Producer)context.get("producer"); 194 | producer.close(); 195 | } catch (Exception e) { 196 | logger.error("producer disconnect error", e); 197 | } 198 | 199 | logger.info("task "+context.getName()+" is stopped "); 200 | } 201 | } 202 | -------------------------------------------------------------------------------- /easy-sync/src/main/java/com/cehome/easysync/service/DatabaseService.java: -------------------------------------------------------------------------------- 1 | package com.cehome.easysync.service; 2 | 3 | import com.alibaba.fastjson.JSONArray; 4 | import com.cehome.easysync.Application; 5 | import com.cehome.easysync.dao.SyncConfigDao; 6 | import com.cehome.easysync.objects.Column; 7 | import com.cehome.easysync.objects.config.Mysql; 8 | import com.cehome.easysync.utils.CharsetConversion; 9 | import jsharp.sql.ObjectSessionFactory; 10 | import jsharp.sql.SessionFactory; 11 | import jsharp.util.Common; 12 | import jsharp.util.DataMap; 13 | import jsharp.util.DataValue; 14 | import org.apache.commons.lang.StringUtils; 15 | import org.apache.tomcat.jdbc.pool.DataSource; 16 | import org.springframework.beans.factory.annotation.Autowired; 17 | import org.springframework.cache.concurrent.ConcurrentMapCache; 18 | import org.springframework.context.annotation.Scope; 19 | import org.springframework.stereotype.Service; 20 | import org.springframework.util.LinkedCaseInsensitiveMap; 21 | 22 | import java.sql.*; 23 | import java.util.*; 24 | 25 | @Service 26 | @Scope("prototype") 27 | public class DatabaseService { 28 | 29 | private static String driverClassName = "com.mysql.jdbc.Driver"; 30 | DataSource dataSource; 31 | SessionFactory sessionFactory; 32 | private long timeTaskId; 33 | private long serverId; 34 | 35 | //private volatile Map> tableTasksMap = null; 36 | @Autowired 37 | private SyncConfigDao syncConfigDao; 38 | 39 | private ConcurrentMapCache columnsCache=new ConcurrentMapCache("columns"); 40 | private ConcurrentMapCache columnMapCache=new ConcurrentMapCache("columnMap"); 41 | 42 | 43 | protected DataSource createDataSource(Mysql mysql) { 44 | 45 | DataSource dataSource = new org.apache.tomcat.jdbc.pool.DataSource(); 46 | dataSource.setDriverClassName(driverClassName); 47 | dataSource.setUrl(mysql.getUrl()); 48 | dataSource.setUsername(mysql.getUser()); 49 | dataSource.setPassword(mysql.getPassword()); 50 | dataSource.setValidationQuery("select 1"); 51 | dataSource.setTestWhileIdle(true); 52 | dataSource.setTimeBetweenEvictionRunsMillis(30*1000); 53 | return dataSource; 54 | } 55 | 56 | public void setMysql(Mysql mysql) { 57 | dataSource = createDataSource(mysql); 58 | sessionFactory = new ObjectSessionFactory(); 59 | sessionFactory.setDataSource(dataSource); 60 | 61 | DataValue dataValue = sessionFactory.queryValue("SELECT @@server_id as server_id"); 62 | //if(dataValue==null) throw new JSException("server id is null"); 63 | serverId = dataValue.getLong(-1); 64 | 65 | } 66 | 67 | public static DatabaseService newInstance(Mysql mysql) { 68 | 69 | DatabaseService databaseService = Application.getBean(DatabaseService.class); 70 | databaseService.setMysql(mysql); 71 | return databaseService; 72 | } 73 | 74 | 75 | public long getTimeTaskId() { 76 | return timeTaskId; 77 | } 78 | 79 | public void setTimeTaskId(long timeTaskId) { 80 | this.timeTaskId = timeTaskId; 81 | } 82 | 83 | public long getServerId() { 84 | return this.serverId; 85 | 86 | } 87 | 88 | /*public synchronized List getTableTasks(String database, String table) { 89 | 90 | if (tableTasksMap == null) { 91 | 92 | long mysqlTimeTaskId = getTimeTaskId(); 93 | List syncConfigs = syncConfigDao.queryListByProps(null, "mysqlTimeTaskId", mysqlTimeTaskId); 94 | Map> map = new HashMap<>(); 95 | for (SyncConfig syncConfig : syncConfigs) { 96 | long timeTaskId = syncConfig.getTimeTaskId(); 97 | String[] tables = getMatchTables(syncConfig.getMysqlDatabases(), syncConfig.getMysqlTables(), syncConfig.getMysqlKeys()); 98 | if (ArrayUtils.isNotEmpty(tables)) { 99 | for (String t : tables) { 100 | List tableTasks = map.get(t); 101 | if (tableTasks == null) { 102 | tableTasks = new ArrayList<>(); 103 | map.put(t, tableTasks); 104 | } 105 | TableTask tableTask=new TableTask(); 106 | tableTask.setTimeTaskId(timeTaskId); 107 | tableTask.setKeys(syncConfig.getMysqlKeys()); 108 | tableTasks.add(tableTask); 109 | 110 | } 111 | } 112 | 113 | } 114 | tableTasksMap = map; 115 | } 116 | 117 | 118 | return tableTasksMap.get(database+"."+table); 119 | 120 | } 121 | 122 | public synchronized void clearTableTasks() { 123 | tableTasksMap = null; 124 | }*/ 125 | 126 | 127 | public String[] getMatchTables(String databases, String tables) { 128 | 129 | return getMatchTables(databases,tables,0); 130 | } 131 | public String[] getFirstMatchTable(String databases, String tables) { 132 | String[] ss=getMatchTables(databases,tables,1); 133 | if(ss.length>0) return ss[0].split("\\."); 134 | return new String[0]; 135 | } 136 | private String[] getMatchTables(String databases, String tables,int limit) { 137 | String oper1=databases.indexOf('%')>=0?"like":"="; 138 | String oper2=tables.indexOf('%')>=0?"like":"="; 139 | String sql = "select table_schema,table_name from information_schema.tables where "; 140 | sql+=parseValues("table_name",tables); 141 | if(StringUtils.isNotBlank(databases)){ 142 | sql+=" and "+parseValues("table_schema",databases); 143 | } 144 | 145 | sql+=" order by table_schema,table_name"; 146 | 147 | if(limit>0){ 148 | sql+=" limit "+limit+" "; 149 | } 150 | 151 | List list = sessionFactory.queryList(sql); 152 | String[] result = new String[list.size()]; 153 | 154 | int i = 0; 155 | for (DataMap dataMap : list) { 156 | String tableName = dataMap.getString("table_schema") + "." + dataMap.getString("table_name"); 157 | result[i++] = tableName; 158 | } 159 | return result; 160 | 161 | 162 | } 163 | 164 | private String parseValues(String field,String values){ 165 | 166 | if(values.indexOf(',')>=-1){ 167 | String oper=values.indexOf('%')>=0?" like ":" = "; 168 | return " ( "+field+oper +" '"+values+"' ) "; 169 | }else{ 170 | // a='a' or b='b' or c like 'c%' 171 | StringBuilder sb=new StringBuilder(); 172 | String[] ss=values.split(","); 173 | for(String s :ss){ 174 | if(sb.length()>0) sb.append( " or "); 175 | String oper=s.indexOf('%')>=0?" like ":" = "; 176 | sb.append( field +oper +" '"+s+"' "); 177 | } 178 | return " (" +sb.toString()+") "; 179 | } 180 | } 181 | 182 | public JSONArray getMatchTables2(String databases, String tables, String pk) { 183 | String oper1=databases.indexOf('%')>=0?"like":"="; 184 | String oper2=tables.indexOf('%')>=0?"like":"="; 185 | String sql = "select table_schema,table_name from information_schema.tables where table_schema "+oper1 186 | +" ? and table_name "+oper2+" ? order by table_schema,table_name"; 187 | List list = sessionFactory.queryList(sql, databases, tables); 188 | JSONArray result = new JSONArray(list.size()); 189 | 190 | for (DataMap dataMap : list) { 191 | String tableName = dataMap.getString("table_schema") + "." + dataMap.getString("table_name"); 192 | DataValue dataValue = sessionFactory.queryValue("select max(" + pk + ") from " + tableName); 193 | long max = dataValue.getLong(0); 194 | JSONArray row = new JSONArray(); 195 | row.add(tableName); 196 | row.add(max); 197 | result.add(row); 198 | 199 | 200 | } 201 | return result; 202 | // 203 | 204 | } 205 | 206 | 207 | 208 | public List getColumns(String database, String table) throws SQLException { 209 | 210 | String key=database+"."+table; 211 | List list= columnsCache.get(key, List.class); 212 | if(list==null) { 213 | 214 | 215 | String sql = "select column_name name,data_type dataType , column_type columnType, character_set_name charset from information_schema.columns " + 216 | 217 | "where table_name=? and table_schema=? order by ordinal_position"; 218 | list = sessionFactory.queryList(sql, Column.class, table, database); 219 | for (Column column : list) { 220 | if (StringUtils.isNotBlank(column.getCharset())) { 221 | column.setCharset(CharsetConversion.getJavaCharset(column.getCharset())); 222 | } 223 | } 224 | columnsCache.putIfAbsent(key,list); 225 | } 226 | 227 | return list; 228 | 229 | } 230 | 231 | public Map getColumnMap(String database, String table) throws SQLException { 232 | String key=database+"."+table; 233 | Map map=columnMapCache.get(key,Map.class); 234 | if(map==null) { 235 | List list = getColumns(database, table); 236 | map=new LinkedCaseInsensitiveMap(); 237 | for(Column column :list){ 238 | map.put(column.getName(),column); 239 | } 240 | columnMapCache.putIfAbsent(key,map); 241 | } 242 | return map; 243 | } 244 | 245 | @Deprecated 246 | public Column[] getColumns2(String database, String table) throws SQLException { 247 | Connection conn = null; 248 | PreparedStatement stmt = null; 249 | ResultSet rs = null; 250 | try { 251 | conn = dataSource.getConnection(); 252 | String sql = "select * from " + database + "." + table + " limit 1 "; 253 | 254 | stmt = conn.prepareStatement(sql); 255 | rs = stmt.executeQuery(sql); 256 | ResultSetMetaData data = rs.getMetaData(); 257 | Column[] columns = new Column[data.getColumnCount()]; 258 | for (int i = 1; i <= data.getColumnCount(); i++) { 259 | Column column = new Column(); 260 | column.setName(data.getColumnName(i)); 261 | //column.setType(data.getColumnType(i)); 262 | columns[i - 1] = column; 263 | } 264 | return columns; 265 | 266 | } finally { 267 | Common.closeObjects(rs, stmt, conn); 268 | 269 | } 270 | } 271 | 272 | public List queryList(String table, String where, String pk, long min ,int limit) { 273 | if(StringUtils.isNotBlank(where)) { 274 | where=" ( "+where+" ) and "; 275 | } 276 | if(where==null) where=""; 277 | String sql = String.format("select * from %s where %s %s > ? order by %s limit ? ", table, where, pk, pk); 278 | List list = sessionFactory.queryList(sql, min, limit); 279 | return list; 280 | } 281 | 282 | 283 | public int getTimezoneHours(){ 284 | String sql="SELECT EXTRACT(HOUR FROM (TIMEDIFF(NOW(), UTC_TIMESTAMP))) AS `timezone`"; 285 | return sessionFactory.queryValue(sql).getInt(0); 286 | 287 | } 288 | 289 | } 290 | -------------------------------------------------------------------------------- /easy-sync/src/main/java/com/cehome/easysync/service/PositionSaveService.java: -------------------------------------------------------------------------------- 1 | package com.cehome.easysync.service; 2 | 3 | import com.cehome.easysync.domain.Position; 4 | 5 | import java.util.Objects; 6 | 7 | public class PositionSaveService { 8 | 9 | PositionService positionService; 10 | Position position; 11 | Position oldPosition=new Position(); 12 | public PositionSaveService(PositionService positionService, Position position){ 13 | this.positionService=positionService; 14 | this.position=position; 15 | updateOldPosition(); 16 | 17 | 18 | } 19 | public void set(String filename,long posi,String gtidSet){ 20 | 21 | position.setFilename(filename); 22 | position.setPosition(posi); 23 | position.setGtidSet(gtidSet); 24 | } 25 | public void save(){ 26 | //if( (timeCal.isTimeUp()>0 || now) && !same() ) { 27 | if(!same()){ 28 | positionService.save(position); 29 | updateOldPosition(); 30 | } 31 | } 32 | 33 | private void updateOldPosition(){ 34 | oldPosition.setFilename(position.getFilename()); 35 | oldPosition.setPosition(position.getPosition()); 36 | oldPosition.setGtidSet(position.getGtidSet()); 37 | } 38 | 39 | private boolean same(){ 40 | return Objects.equals(position.getFilename(),oldPosition.getFilename()) && 41 | position.getPosition()==oldPosition.getPosition() && 42 | Objects.equals( position.getGtidSet(),oldPosition.getGtidSet()); 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /easy-sync/src/main/java/com/cehome/easysync/service/PositionService.java: -------------------------------------------------------------------------------- 1 | package com.cehome.easysync.service; 2 | 3 | import com.cehome.easysync.dao.PositionDao; 4 | import com.cehome.easysync.domain.Position; 5 | import jsharp.sql.JSException; 6 | import org.springframework.beans.factory.annotation.Autowired; 7 | import org.springframework.stereotype.Service; 8 | 9 | import java.util.List; 10 | 11 | @Service 12 | public class PositionService { 13 | 14 | @Autowired 15 | private PositionDao positionDao; 16 | 17 | public Position getPosition(long timeTaskId,long serverId){ 18 | List list= positionDao.queryListByProps(null, "timeTaskId",timeTaskId); 19 | if(list.size()==0) return null; 20 | for(Position position :list){ 21 | if(position.getServerId()==serverId) return position; 22 | } 23 | throw new JSException("can not find serverId "+serverId); 24 | } 25 | 26 | public int save(Position position){ 27 | return positionDao.save(position); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /easy-sync/src/main/java/com/cehome/easysync/service/TableTasksMapService.java: -------------------------------------------------------------------------------- 1 | package com.cehome.easysync.service; 2 | 3 | import com.cehome.easysync.dao.SyncConfigDao; 4 | import com.cehome.easysync.domain.SyncConfig; 5 | import com.cehome.easysync.objects.TableTask; 6 | import com.cehome.task.dao.TimeTaskDao; 7 | import com.cehome.task.domain.TimeTask; 8 | import org.apache.commons.lang.ArrayUtils; 9 | import org.slf4j.Logger; 10 | import org.slf4j.LoggerFactory; 11 | import org.springframework.beans.factory.annotation.Autowired; 12 | import org.springframework.cache.annotation.CacheEvict; 13 | import org.springframework.cache.annotation.Cacheable; 14 | import org.springframework.stereotype.Service; 15 | 16 | import java.util.ArrayList; 17 | import java.util.HashMap; 18 | import java.util.List; 19 | import java.util.Map; 20 | @Service 21 | public class TableTasksMapService { 22 | private static final Logger logger = LoggerFactory.getLogger(TableTasksMapService.class); 23 | 24 | @Autowired 25 | private SyncConfigDao syncConfigDao; 26 | @Autowired 27 | private TimeTaskDao timeTaskDao; 28 | @Cacheable(value = "tableTasksMap",key="#databaseService.timeTaskId") 29 | public Map> get(DatabaseService databaseService) { 30 | 31 | long mysqlTimeTaskId = databaseService.getTimeTaskId(); 32 | List syncConfigs = syncConfigDao.queryListByProps(null, "mysqlTimeTaskId", mysqlTimeTaskId); 33 | Map> map = new HashMap<>(); 34 | for (SyncConfig syncConfig : syncConfigs) { 35 | long timeTaskId = syncConfig.getTimeTaskId(); 36 | TimeTask timeTask=timeTaskDao.get(timeTaskId); 37 | if(timeTask==null || timeTask.getStatus()==2){ 38 | continue; 39 | } 40 | String[] tables = databaseService.getMatchTables(syncConfig.getMysqlDatabases(), syncConfig.getMysqlTables()); 41 | if (ArrayUtils.isNotEmpty(tables)) { 42 | for (String t : tables) { 43 | List tableTasks = map.get(t); 44 | if (tableTasks == null) { 45 | tableTasks = new ArrayList<>(); 46 | map.put(t, tableTasks); 47 | } 48 | TableTask tableTask=new TableTask(); 49 | tableTask.setTimeTaskId(timeTaskId); 50 | tableTask.setKeys(syncConfig.getMysqlKeys()); 51 | tableTasks.add(tableTask); 52 | 53 | } 54 | } 55 | 56 | } 57 | return map; 58 | 59 | 60 | } 61 | 62 | // 清空 缓存 63 | @CacheEvict(value = "tableTasksMap",key="#mysqlTimeTaskId") 64 | public void clean(long mysqlTimeTaskId) { 65 | logger.info("clear cache for timetaskid=" + mysqlTimeTaskId); 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /easy-sync/src/main/java/com/cehome/easysync/utils/Const.java: -------------------------------------------------------------------------------- 1 | package com.cehome.easysync.utils; 2 | 3 | public class Const { 4 | public static String TOPIC_PREFIX="es_t_"; 5 | public static String ES_TYPE="doc"; 6 | public static int REBUILD_NONE =0; 7 | public static int REBUILD_NEED=1; 8 | public static int REBUILD_ALLOW=2; 9 | public static int REBUILD_START=3; 10 | public static int REBUILD_END=4; 11 | public static int REBUILD_STOP=5; 12 | 13 | 14 | public static int Task_TYPE_MYSQL=0; 15 | public static int Task_TYPE_ES=1; 16 | } 17 | -------------------------------------------------------------------------------- /easy-sync/src/main/java/com/cehome/easysync/utils/EsUtils.java: -------------------------------------------------------------------------------- 1 | package com.cehome.easysync.utils; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | 6 | import java.util.HashMap; 7 | import java.util.Map; 8 | 9 | public class EsUtils { 10 | private static final Logger logger = LoggerFactory.getLogger(EsUtils.class); 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | private static Map mapping1=new HashMap(){{ 19 | put("bit","boolean");//todo:bit(1) , bit(2)?? 20 | put("boolean","boolean"); 21 | 22 | put("date","date"); 23 | put("datetime","date"); 24 | put("time","date"); 25 | put("timestamp","date"); 26 | 27 | 28 | put("decimal","double"); 29 | put("double","double"); 30 | put("real","double"); 31 | put("float","float"); 32 | put("bigint","long"); 33 | put("int","integer"); 34 | put("mediumint","integer"); 35 | put("smallint","short"); 36 | put("tinyint","byte"); 37 | 38 | 39 | put("char","string"); 40 | put("nchar","string"); 41 | put("nvarchar","string"); 42 | put("varchar","string"); 43 | 44 | put("tinytext","string"); 45 | put("mediumtext","string"); 46 | put("longtext","string"); 47 | put("text","string"); 48 | 49 | put("binary","binary"); 50 | put("varbinary","binary"); 51 | put("blob","binary"); 52 | put("tinyblob","binary"); 53 | put("mediumblob","binary"); 54 | put("longblob","binary"); 55 | 56 | 57 | }}; 58 | 59 | 60 | private static Map mapping2=new HashMap(mapping1){{ 61 | put("char","keyword"); 62 | put("nchar","keyword"); 63 | put("nvarchar","keyword"); 64 | put("varchar","keyword"); 65 | 66 | put("tinytext","text"); 67 | put("mediumtext","text"); 68 | put("longtext","text"); 69 | put("text","text"); 70 | }}; 71 | 72 | 73 | /** 74 | * https://www.cnblogs.com/waterystone/p/6226356.html 75 | * @param mysqlType 76 | * @return 77 | */ 78 | public static String mysqlType2EsType(String mysqlType,boolean newVersion){ 79 | 80 | return newVersion?mapping2.get(mysqlType):mapping1.get(mysqlType); 81 | } 82 | 83 | public static void main(String[] args) { 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /easy-sync/src/main/java/com/cehome/easysync/utils/Global.java: -------------------------------------------------------------------------------- 1 | package com.cehome.easysync.utils; 2 | 3 | import com.alibaba.fastjson.JSON; 4 | import com.alibaba.fastjson.JSONObject; 5 | import org.apache.commons.lang.StringUtils; 6 | 7 | public class Global { 8 | public static JSONObject toJSON(String s){ 9 | if(StringUtils.isBlank(s)) return new JSONObject(); 10 | return JSONObject.parseObject(s); 11 | } 12 | 13 | public static T toObject(String s, Class c) { 14 | try { 15 | if (StringUtils.isBlank(s)) return c.newInstance(); 16 | return JSON.parseObject(s, c); 17 | }catch (Exception e){ 18 | throw new RuntimeException(e); 19 | } 20 | 21 | } 22 | 23 | public static JSONObject toObject(String s) { 24 | try { 25 | if (StringUtils.isBlank(s)) return new JSONObject(); 26 | return JSON.parseObject(s); 27 | }catch (Exception e){ 28 | throw new RuntimeException(e); 29 | } 30 | 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /easy-sync/src/main/java/com/cehome/easysync/utils/MemTable.java: -------------------------------------------------------------------------------- 1 | package com.cehome.easysync.utils; 2 | 3 | import org.apache.commons.csv.CSVFormat; 4 | import org.apache.commons.csv.CSVParser; 5 | import org.apache.commons.csv.CSVRecord; 6 | 7 | import java.io.InputStreamReader; 8 | import java.util.HashMap; 9 | import java.util.List; 10 | import java.util.Map; 11 | 12 | public class MemTable { 13 | List records; 14 | Map> map=new HashMap<>(); 15 | public MemTable(){ 16 | 17 | } 18 | public MemTable(String resource,String charset) throws Exception { 19 | InputStreamReader r=new InputStreamReader( MemTable.class.getResourceAsStream(resource),charset); 20 | CSVParser parser= CSVFormat.DEFAULT.parse(r); 21 | records=parser.getRecords(); 22 | 23 | 24 | 25 | } 26 | 27 | public void setIndex(String[] names){ 28 | for(String name :names){ 29 | Map m=new HashMap<>(); 30 | for(CSVRecord record :records){ 31 | m.put(record.get(name),record); 32 | } 33 | map.put(name,m); 34 | } 35 | 36 | } 37 | 38 | public CSVRecord find(String name ,String value){ 39 | Map m=map.get(name); 40 | if(m!=null){ 41 | return m.get(value); 42 | } 43 | for(CSVRecord record :records){ 44 | if(record.get(name).equals(value)){ 45 | return record; 46 | } 47 | 48 | } 49 | return null; 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /easy-sync/src/main/java/com/cehome/easysync/utils/MysqlTypeUtils.java: -------------------------------------------------------------------------------- 1 | package com.cehome.easysync.utils; 2 | 3 | import java.util.HashSet; 4 | import java.util.Set; 5 | 6 | public class MysqlTypeUtils { 7 | private static Set stringTypeSet = new HashSet() { 8 | { 9 | add("char"); 10 | add("varchar"); 11 | add("nchar"); 12 | add("nvarchar"); 13 | add("char"); 14 | add("tinytext"); 15 | add("mediumtext"); 16 | add("longtext"); 17 | add("text"); 18 | } }; 19 | 20 | private static Set dateTypeSet = new HashSet() { 21 | { 22 | add("date"); 23 | add("datetime"); 24 | add("time"); 25 | add("timestamp"); 26 | //year?? 27 | } }; 28 | 29 | private static Set dateTypeSet2 = new HashSet() { 30 | { 31 | add("date"); 32 | add("datetime"); 33 | add("time"); 34 | } }; 35 | 36 | public static boolean isString(String dataType) { 37 | return stringTypeSet.contains(dataType); 38 | } 39 | public static boolean isDate(String dataType) { 40 | return dateTypeSet.contains(dataType); 41 | } 42 | public static boolean isDateNormal(String dataType) { 43 | return dateTypeSet2.contains(dataType); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /easy-sync/src/main/resources/application-dev.properties: -------------------------------------------------------------------------------- 1 | #en or cn 2 | language=en 3 | 4 | spring.freemarker.request-context-attribute= request 5 | spring.freemarker.prefer-file-system-access=false 6 | 7 | #------ main options -------- 8 | spring.application.name=easy-sync 9 | task.factory.appName=easy-sync 10 | #cluster name (also is table_name) 11 | task.factory.name=easy_sync 12 | 13 | #mysql 14 | task.datasource.driverClassName=com.mysql.jdbc.Driver 15 | task.datasource.url=jdbc:mysql://192.168.0.13:3306/sync?useUnicode=true&characterEncoding=utf8&allowMultiQueries=true 16 | task.datasource.username=root 17 | task.datasource.password=asdf1234! 18 | ##h2 19 | #task.datasource.driverClassName=org.h2.Driver 20 | #task.datasource.url=jdbc:h2:tcp://localhost:9002/~/easy_sync;MODE=MYSQL 21 | #task.datasource.username=sa 22 | #task.datasource.password= 23 | 24 | 25 | ## redis config 26 | #task.factory.redis.host=192.168.0.11 27 | #task.factory.redis.port=6379 28 | 29 | #------ client options -------- 30 | task.log.packages=ROOT 31 | task.log.path=/logs/easy_sync 32 | task.taskCheckInterval=5000 33 | task.heartBeatSendInterval=10000 34 | 35 | #------ console options -------- 36 | task.h2.start=true 37 | task.h2.port=9002 38 | task.autoCreateTable=true 39 | task.heartBeatSwitchEnable=true 40 | task.heartBeatCheckInterval=30000 41 | task.heartBeatFailSwitchTime=60000 42 | 43 | 44 | logging.level.com.cehome.task.console=WARN 45 | logging.level.org.apache.kafka.clients.NetworkClient=ERROR 46 | 47 | login.user= 48 | login.password= 49 | 50 | 51 | -------------------------------------------------------------------------------- /easy-sync/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | #en or cn 2 | #spring.profiles.active= --spring.profiles.active=dev 3 | language=en 4 | spring.freemarker.request-context-attribute= request 5 | spring.freemarker.prefer-file-system-access=false 6 | 7 | #------ main options -------- 8 | task.factory.appName=easy-sync 9 | spring.application.name=easy-sync 10 | #cluster name (also is table_name) 11 | task.factory.name=easy_sync 12 | 13 | ##mysql 14 | #task.datasource.driverClassName=com.mysql.jdbc.Driver 15 | #task.datasource.url=jdbc:mysql://192.168.0.13:3306/sync?useUnicode=true&characterEncoding=utf8&allowMultiQueries=true 16 | #task.datasource.username=root 17 | #task.datasource.password=asdf1234! 18 | #h2 19 | task.datasource.driverClassName=org.h2.Driver 20 | task.datasource.url=jdbc:h2:tcp://localhost:9002/~/easy_sync;MODE=MYSQL 21 | task.datasource.username=sa 22 | task.datasource.password= 23 | 24 | 25 | ## redis config 26 | #task.factory.redis.host=192.168.0.11 27 | #task.factory.redis.port=6379 28 | 29 | #------ client options -------- 30 | task.log.packages=ROOT 31 | task.log.path=/logs/easy_sync 32 | task.taskCheckInterval=5000 33 | task.heartBeatSendInterval=10000 34 | 35 | #------ console options -------- 36 | task.h2.start=true 37 | task.h2.port=9002 38 | task.autoCreateTable=true 39 | task.heartBeatSwitchEnable=true 40 | task.heartBeatCheckInterval=30000 41 | task.heartBeatFailSwitchTime=60000 42 | 43 | 44 | logging.level.com.cehome.task.console=WARN 45 | logging.level.org.apache.kafka.clients.NetworkClient=ERROR 46 | 47 | login.user= 48 | login.password= 49 | 50 | 51 | 52 | 53 | -------------------------------------------------------------------------------- /easy-sync/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | %date %level %file:%line -- %msg%n 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /easy-sync/src/main/resources/sql/posi.txt: -------------------------------------------------------------------------------- 1 | CREATE TABLE `${tableName}` ( 2 | `id` BIGINT NOT NULL AUTO_INCREMENT, 3 | `server_id` BIGINT UNSIGNED NOT NULL, 4 | `time_task_id` BIGINT NOT NULL, 5 | `filename` VARCHAR(255) NULL DEFAULT NULL, 6 | `position` BIGINT NULL, 7 | `gtid_set` VARCHAR(45) NULL, 8 | `create_time` DATETIME NULL, 9 | `update_time` DATETIME NULL, 10 | PRIMARY KEY (`id`), 11 | UNIQUE KEY `${tableName}_keys` (`time_task_id`,`server_id`)); -------------------------------------------------------------------------------- /easy-sync/src/main/resources/sql/sync_conf_h2.txt: -------------------------------------------------------------------------------- 1 | CREATE TABLE `${tableName}` ( 2 | `id` BIGINT NOT NULL AUTO_INCREMENT, 3 | `time_task_id` BIGINT NOT NULL, 4 | `mysql_time_task_id` BIGINT NULL, 5 | `mysql_databases` VARCHAR(500) NULL, 6 | `mysql_tables` VARCHAR(1000) NULL, 7 | `mysql_keys` VARCHAR(500) NULL, 8 | `mysql_key_sep` VARCHAR(50) NULL, 9 | 10 | 11 | `index_name1` VARCHAR(255) NULL, 12 | `wait_index_name1` VARCHAR(255) NULL, 13 | `task_config1` VARCHAR(100000) NULL, 14 | `full_table1` VARCHAR(255) NULL, 15 | `full_position1` BIGINT NOT NULL DEFAULT 0, 16 | `full_status1` INT NOT NULL DEFAULT 0, 17 | 18 | `index_name2` VARCHAR(255) NULL, 19 | `wait_index_name2` VARCHAR(255) NULL, 20 | `task_config2` VARCHAR(100000) NULL, 21 | `full_table2` VARCHAR(255) NULL, 22 | `full_position2` BIGINT NOT NULL DEFAULT 0, 23 | `full_status2` INT NOT NULL DEFAULT 0, 24 | 25 | `repair_task_config` VARCHAR(100000) NULL, 26 | `repair_table` VARCHAR(255) NULL, 27 | `repair_position` BIGINT NOT NULL DEFAULT 0, 28 | `repair_status` INT NOT NULL DEFAULT 0, 29 | 30 | `create_time` DATETIME NULL, 31 | `update_time` DATETIME NULL, 32 | 33 | 34 | PRIMARY KEY (`id`), 35 | UNIQUE INDEX `${tableName}_ttid` (`time_task_id`), 36 | INDEX `${tableName}_mttid` (`mysql_time_task_id`)); -------------------------------------------------------------------------------- /easy-sync/src/main/resources/sql/sync_conf_mysql.txt: -------------------------------------------------------------------------------- 1 | CREATE TABLE `${tableName}` ( 2 | `id` BIGINT NOT NULL AUTO_INCREMENT, 3 | `time_task_id` BIGINT NOT NULL, 4 | `mysql_time_task_id` BIGINT NULL, 5 | `mysql_databases` VARCHAR(500) NULL, 6 | `mysql_tables` VARCHAR(1000) NULL, 7 | `mysql_keys` VARCHAR(500) NULL, 8 | `mysql_key_sep` VARCHAR(50) NULL, 9 | 10 | 11 | `index_name1` VARCHAR(255) NULL, 12 | `wait_index_name1` VARCHAR(255) NULL, 13 | `task_config1` mediumtext NULL, 14 | `full_table1` VARCHAR(255) NULL, 15 | `full_position1` BIGINT NOT NULL DEFAULT 0, 16 | `full_status1` INT NOT NULL DEFAULT 0, 17 | 18 | `index_name2` VARCHAR(255) NULL, 19 | `wait_index_name2` VARCHAR(255) NULL, 20 | `task_config2` mediumtext NULL, 21 | `full_table2` VARCHAR(255) NULL, 22 | `full_position2` BIGINT NOT NULL DEFAULT 0, 23 | `full_status2` INT NOT NULL DEFAULT 0, 24 | 25 | `repair_task_config` mediumtext NULL, 26 | `repair_table` VARCHAR(255) NULL, 27 | `repair_position` BIGINT NOT NULL DEFAULT 0, 28 | `repair_status` INT NOT NULL DEFAULT 0, 29 | 30 | `create_time` DATETIME NULL, 31 | `update_time` DATETIME NULL, 32 | 33 | 34 | PRIMARY KEY (`id`), 35 | UNIQUE INDEX `${tableName}_ttid` (`time_task_id`), 36 | INDEX `${tableName}_mttid` (`mysql_time_task_id`)); -------------------------------------------------------------------------------- /easy-sync/src/main/resources/sql/task_cache.txt: -------------------------------------------------------------------------------- 1 | CREATE TABLE `${tableName}` ( 2 | `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT, 3 | `main_key` varchar(200) NOT NULL, 4 | `sub_key` varchar(200) NOT NULL, 5 | `value` varchar(500) DEFAULT NULL, 6 | `dataType` int(11) DEFAULT '0', 7 | `expire` datetime DEFAULT NULL, 8 | `version` bigint(20) DEFAULT '0', 9 | `create_time` datetime NOT NULL COMMENT '', 10 | `update_time` datetime NOT NULL COMMENT '', 11 | PRIMARY KEY (`id`), 12 | UNIQUE KEY `${tableName}_keys` (`main_key`,`sub_key`) 13 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8; 14 | -------------------------------------------------------------------------------- /easy-sync/src/main/resources/sql/task_h2.txt: -------------------------------------------------------------------------------- 1 | -- mediumtext to varchar 2 | CREATE TABLE `${tableName}` ( 3 | `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'id', 4 | `target_ip` varchar(100) NOT NULL DEFAULT '' COMMENT '', 5 | `name` varchar(255) NOT NULL DEFAULT '' COMMENT '', 6 | `status` int(11) NOT NULL DEFAULT '0' COMMENT '', 7 | `config` varchar(255) DEFAULT NULL COMMENT '', 8 | `last_target_ip` varchar(100) DEFAULT '' COMMENT '', 9 | `last_start_time` datetime DEFAULT NULL COMMENT '', 10 | `last_end_time` datetime DEFAULT NULL COMMENT '', 11 | `create_time` datetime NOT NULL COMMENT '', 12 | `update_time` datetime NOT NULL COMMENT '', 13 | `category1` varchar(255) DEFAULT NULL COMMENT '', 14 | `priority` int(11) DEFAULT '0' COMMENT '', 15 | `app_name` varchar(100) DEFAULT NULL COMMENT '', 16 | `cron_type` int(11) NOT NULL DEFAULT '0' COMMENT 'cron_type', 17 | `category2` varchar(255) DEFAULT NULL COMMENT '', 18 | `category3` varchar(255) DEFAULT NULL COMMENT '', 19 | `category4` varchar(255) DEFAULT NULL COMMENT '', 20 | `user_id` bigint(20) unsigned DEFAULT NULL COMMENT 'user_id', 21 | `cron` varchar(100) DEFAULT NULL COMMENT '', 22 | `task_type` int(11) NOT NULL DEFAULT '0' COMMENT '', 23 | `task_config` varchar(100000) COMMENT '', 24 | `log_level` varchar(10) DEFAULT NULL COMMENT '', 25 | `mon_flag` smallint(6) NOT NULL DEFAULT '0' COMMENT '', 26 | `mon_config` varchar(100) DEFAULT NULL COMMENT '', 27 | `create_user` varchar(100) DEFAULT NULL COMMENT '', 28 | `oper_time` datetime DEFAULT NULL COMMENT '', 29 | `oper_user` varchar(100) DEFAULT NULL, 30 | `task_result` varchar(100000) COMMENT 'run result', 31 | PRIMARY KEY (`id`), 32 | KEY `idx_${tableName}_au` (`app_name`,`update_time`), 33 | KEY `ids_${tableName}_ai` (`app_name`,`id`) 34 | ) COMMENT='time task'; 35 | 36 | 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /easy-sync/src/main/resources/sql/task_mysql.txt: -------------------------------------------------------------------------------- 1 | CREATE TABLE `${tableName}` ( 2 | `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'id', 3 | `target_ip` varchar(100) NOT NULL DEFAULT '' COMMENT 'run ip', 4 | `name` varchar(255) NOT NULL DEFAULT '' COMMENT 'task name', 5 | `status` int(11) NOT NULL DEFAULT '0' COMMENT 'status', 6 | `config` varchar(255) DEFAULT NULL COMMENT 'bean config', 7 | `last_target_ip` varchar(100) DEFAULT '' COMMENT 'last run ip', 8 | `last_start_time` datetime DEFAULT NULL COMMENT 'last start time', 9 | `last_end_time` datetime DEFAULT NULL COMMENT 'last end time', 10 | `create_time` datetime NOT NULL COMMENT 'create time', 11 | `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'update time', 12 | `category1` varchar(255) DEFAULT NULL COMMENT 'category', 13 | `priority` int(11) DEFAULT '0' COMMENT 'pool', 14 | `app_name` varchar(100) DEFAULT NULL COMMENT 'app name', 15 | `cron_type` int(11) NOT NULL DEFAULT '0' COMMENT 'cron_type', 16 | `category2` varchar(255) DEFAULT NULL COMMENT 'category', 17 | `category3` varchar(255) DEFAULT NULL COMMENT 'category', 18 | `category4` varchar(255) DEFAULT NULL COMMENT 'category', 19 | `user_id` bigint(20) unsigned DEFAULT NULL COMMENT 'user_id', 20 | `cron` varchar(100) DEFAULT NULL COMMENT 'cron express', 21 | `task_type` int(11) NOT NULL DEFAULT '0' COMMENT 'custom task type', 22 | `task_config` mediumtext COMMENT 'task extras config', 23 | `log_level` varchar(10) DEFAULT NULL COMMENT 'log level', 24 | `mon_flag` smallint(6) NOT NULL DEFAULT '0' COMMENT '0:no 1:yes', 25 | `mon_config` varchar(100) DEFAULT NULL COMMENT 'monitor config', 26 | `create_user` varchar(100) DEFAULT NULL COMMENT 'creator', 27 | `oper_time` datetime DEFAULT NULL COMMENT 'operation time', 28 | `oper_user` varchar(100) DEFAULT NULL, 29 | `task_result` mediumtext COMMENT 'run result', 30 | PRIMARY KEY (`id`), 31 | KEY `idx_app_update` (`app_name`,`update_time`), 32 | KEY `ids_app_id` (`app_name`,`id`) 33 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='time task'; 34 | 35 | 36 | -- INSERT INTO `${tableName}` (target_ip,name,status,config,create_time,update_time,category1,app_name,oper_user,oper_time,user_id,cron) 37 | --values('${ip}','demo',1,'{\"args\":{},\"method\":\"run\",\"stopMethod\":\"\",\"bean\":\"demoPlugin\"}', 38 | --now(),now(),'default','${appName}','admin',now(),1,'5s'); 39 | 40 | 41 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /easy-sync/src/main/resources/static/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/easy30/easy-sync/60ebd9bf998ac7fde38ea68ff2b4e8a6c65875c8/easy-sync/src/main/resources/static/favicon.ico -------------------------------------------------------------------------------- /easy-sync/src/main/resources/static/res/img/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/easy30/easy-sync/60ebd9bf998ac7fde38ea68ff2b4e8a6c65875c8/easy-sync/src/main/resources/static/res/img/logo.png -------------------------------------------------------------------------------- /easy-sync/src/main/resources/static/res/img/question.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/easy30/easy-sync/60ebd9bf998ac7fde38ea68ff2b4e8a6c65875c8/easy-sync/src/main/resources/static/res/img/question.png -------------------------------------------------------------------------------- /easy-sync/src/main/resources/static/res/js/lang-cn2.js: -------------------------------------------------------------------------------- 1 | function lang_text2() {/* 2 | mysqlUrl=mysql 链接 3 | mysqlHost=mysql主机 4 | mysqlPort=mysql端口 5 | mysqlUser=mysql用户 6 | mysqlPassword=mysql密码 7 | kafkaServers=Kafka服务器 8 | kafkaVersion=Kafka版本 9 | config=配置 10 | mysqlTasks=Binlog 11 | elasticsearchTasks=Elasticsearch 12 | 13 | baseInfo=基本信息 14 | datasourceConfig=数据源配置 15 | selectMysqlInstance=选择mysql实例 16 | databaseName=数据库名称 17 | tableName=表名 18 | primaryKey=主键 19 | elasticsearchConfig=Elasticsearch配置 20 | elasticsearchAddresses=Elasticserch地址 21 | defaultAnalyzer=缺省分析器 22 | indexAlias=索引别名 23 | indexSettings=索引设置 24 | esVersion=Elasticserch版本 25 | fieldMapping=字段映射 26 | addField=添加字段 27 | autoMapping=自动映射 28 | rebuildSettings=创建/重建索引设置 29 | enableFullSync=启用全量数据同步 30 | millisecond=毫秒 31 | switchAfterFullSync=安全切换索引(重建结束后才启用新索引) 32 | deleteOldIndex=重建索引后删除旧索引 33 | saveConfig=保存配置 34 | rebuildIndex=创建/重建索引 35 | fullBatchSize=每批大小 36 | fullBatchInterval=批次间隔 37 | fullWhere=同步 where 条件 38 | deleteAll=删除全部 39 | saveConfigDesc=保存后，数据库和表变化将立即生效，字段修改需要重建索引才会生效 40 | rebuildDesc= 保存配置、创建新索引、同步数据，结束后将替换旧索引（修改别名映射）。 41 | 42 | sourceFieldName=源字段 43 | sourceType=源字段类型 44 | targetFieldName=目标字段 45 | targetType=目标字段类型 46 | convert=转换函数 47 | custom=自定义 48 | repairSettings=数据补偿 49 | repairData=数据补偿 50 | repairDataDesc=根据条件重新同步数据 51 | stopRebuildIndex=取消重建 52 | stopRepairData=取消数据补偿 53 | producerConfigs=生产者配置 54 | consumerConfigs=消费者配置 55 | */} 56 | -------------------------------------------------------------------------------- /easy-sync/src/main/resources/static/res/js/lang-en2.js: -------------------------------------------------------------------------------- 1 | function lang_text2() {/* 2 | mysqlUrl=mysql url 3 | mysqlHost=mysql host 4 | mysqlPort=mysql port 5 | mysqlUser=mysql user 6 | mysqlPassword=mysql password 7 | kafkaServers=kafka servers 8 | kafkaVersion=kafka version 9 | config=Config 10 | mysqlTasks=Binlog 11 | elasticsearchTasks=Elasticsearch 12 | 13 | baseInfo=Base Info 14 | datasourceConfig=Datasource Config 15 | selectMysqlInstance=Select mysql instance 16 | databaseName=Database Names 17 | tableName=Table Names 18 | primaryKey=Primary Key 19 | elasticsearchConfig=Elasticsearch Config 20 | elasticsearchAddresses=Elasticserch Addresses 21 | defaultAnalyzer=Default Analyzer 22 | indexAlias=Index Alias 23 | indexSettings=Index Settings 24 | esVersion=Elasticserch Version 25 | fieldMapping=Field Mapping 26 | addField=Add Field 27 | autoMapping=Auto Mapping 28 | rebuildSettings=Build/Rebuild Index Settings 29 | enableFullSync=Enable Full Sync 30 | millisecond=Millisecond 31 | switchAfterFullSync=Safe switch index(Use new index After rebuild finish) 32 | deleteOldIndex=Delete old index after rebuild finish 33 | saveConfig=Save Config 34 | rebuildIndex=Rebuild Index 35 | fullBatchSize=Batch size 36 | fullBatchInterval=Interval between batch 37 | fullWhere=Where condition(SQL) 38 | deleteAll=Delete All 39 | saveConfigDesc=After save config , databases or tables changes will effect immediately. But fields changes will not apply until finishing rebuild opration. 40 | rebuildDesc= Save config , create a new index and sync data , after done then replace the old index（change alias mapping）. 41 | 42 | sourceFieldName=Source Field 43 | sourceType=Source Type 44 | targetFieldName=Target Field 45 | targetType=Target Type 46 | convert=Convert 47 | custom=Custom 48 | repairSettings=Repair Settings 49 | repairData=Repair Data 50 | repairDataDesc=resync some records 51 | 52 | stopRebuildIndex=Cancal Rebuild 53 | stopRepairData=Cancal Repair 54 | 55 | producerConfigs=Producer Configs 56 | consumerConfigs=Consumer Configs 57 | 58 | */} 59 | -------------------------------------------------------------------------------- /easy-sync/src/main/resources/static/vue.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Title 6 | 7 | 8 | 9 |
    10 |
  • 11 | 12 | {{ item.message }} 13 |
    • 14 |
    • 15 |
16 | 17 | 18 | 47 | 48 | -------------------------------------------------------------------------------- /easy-sync/src/main/resources/templates/include/head-detail.ftl: -------------------------------------------------------------------------------- 1 | <#ftl encoding="utf-8"> 2 | 24 |

-------------------------------------------------------------------------------- /easy-sync/src/main/resources/templates/include/head-master.ftl: -------------------------------------------------------------------------------- 1 | <#ftl encoding="utf-8"> 2 | 29 |

30 | -------------------------------------------------------------------------------- /easy-sync/src/main/resources/templates/include/head-meta.ftl: -------------------------------------------------------------------------------- 1 | <#ftl encoding="utf-8"> 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | Easy Sync 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 20 | 21 | 22 | 23 | 24 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | <#----> 40 | <#----> 41 | <#----> 42 | 43 | 58 | -------------------------------------------------------------------------------- /easy-sync/src/main/resources/templates/timeTask/esEdit.ftl: -------------------------------------------------------------------------------- 1 | <#ftl encoding="utf-8"> 2 | <#include "../include/constants.ftl"> 3 | <#import "../include/detail.ftl" as frame> 4 | <@frame.page title="" customerCss=[] customerJs=[] specificLib=[]> 5 | 11 | <#include "esEditBody.ftl"> 12 | 143 | 144 | <#include "res.ftl"> 145 | 146 | 147 | -------------------------------------------------------------------------------- /easy-sync/src/main/resources/templates/timeTask/esEditMapping.ftl: -------------------------------------------------------------------------------- 1 | <#ftl encoding="utf-8"> 2 | 3 | 4 |
5 |
6 |

Field mapping

7 | 8 | 9 |
10 | 11 |
12 | 13 |
14 |
15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 46 | 47 | 48 | 52 | 53 | 54 | 59 | 60 | 61 |
source fieldsource typefieldtypeconvertcustomoperation
49 | 51 |
55 | 56 | 57 | 58 |
62 | 63 |
64 |
65 | 66 | 67 |
68 | 69 |
70 | 71 | 72 | 73 | 74 | -------------------------------------------------------------------------------- /easy-sync/src/main/resources/templates/timeTask/esList.ftl: -------------------------------------------------------------------------------- 1 | <#ftl encoding="utf-8"> 2 | <#include "../include/constants.ftl"> 3 | <#import "../include/master.ftl" as frame> 4 | <@frame.page title="${areaCustom.desc}" menu="timeTask_${taskType}" customerCss=[] customerJs=[] specificLib=[]> 5 | 6 | 7 | 23 | 29 | 45 | 46 | 47 | 48 |
49 | 51 |
52 | <#include "searchForm.ftl"> 53 | 54 |
55 | <#include "page.ftl"> 56 | 57 | 58 | 59 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | <#-- --> 70 | 71 | 72 | 73 | 74 | <#list list as item> 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | <#-- --> 85 | 103 | 104 | 105 | 106 |
taskIdtaskNameserverstatusmodifyTimeoperatoroperation
${item.id} <#if item.priority==2 > high${item.name}<#if (appName!)=="" >${item.appName}|${item.targetIp}${(item.status==0)?string("停止",(item.status==1)?string("运行","删除"))} ${item.operTime?string("yyyy-MM-dd HH:mm:ss")}${item.operUser}modify 86 | 87 | 88 | <#if areaCustom.taskConfig??> 89 | browseData 91 | 92 |   93 | 94 | copy 95 |   96 | 99 | 100 | viewLog 101 | 102 |
107 | <#include "page.ftl"> 108 | 109 |
110 | 111 | <#include "dialog.ftl"> 112 | <#include "res.ftl"> 113 | -------------------------------------------------------------------------------- /easy-sync/src/main/resources/templates/timeTask/getLog.ftl: -------------------------------------------------------------------------------- 1 | <#ftl encoding="utf-8"> 2 | 3 | 4 | <#include "../include/constants.ftl"> 5 | <#include "../include/head-meta.ftl"> 6 | <#-- 7 | log 8 | 9 | --> 10 | 11 | 12 | 13 | <#include "../include/head-detail.ftl"> 14 | 15 | 16 | 17 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 77 |
${taskName }
18 | pageSize: ${pageSize} K   19 | totalSize: ${totalSize}K   20 | ${pn}/${pageCount?c}页   21 | 22 | 24 | 26 | 28 | 30 |    31 | 32 |      33 | 34 | pageSize（K）： 35 | 36 | 37 | No Wrap 38 | 39 | 40 |
78 | 79 | 80 | -------------------------------------------------------------------------------- /easy-sync/src/main/resources/templates/timeTask/sourceEdit.ftl: -------------------------------------------------------------------------------- 1 | <#ftl encoding="utf-8"> 2 |
3 | 4 | <#include "sourceEditBody.ftl"> 5 | 6 |
7 | 8 | 66 | -------------------------------------------------------------------------------- /easy-sync/src/main/resources/templates/timeTask/sourceList.ftl: -------------------------------------------------------------------------------- 1 | <#ftl encoding="utf-8"> 2 | <#include "../include/constants.ftl"> 3 | <#import "../include/master.ftl" as frame> 4 | <@frame.page title="${areaCustom.desc}" menu="timeTask_${taskType}" customerCss=[] customerJs=[] specificLib=[]> 5 | 6 | 7 | 22 | 28 | 44 | 45 | 46 | 47 |
48 | 50 |
51 | <#include "searchForm.ftl"> 52 | 53 |
54 | <#include "page.ftl"> 55 | 56 | 57 | 58 | 60 | 61 | 62 | 63 | 64 | <#-- --> 65 | 66 | 67 | 68 | <#-- --> 69 | 70 | 71 | 72 | 73 | <#list list as item> 74 | 75 | 76 | 77 | 78 | 79 | 80 | <#-- --> 81 | 82 | 83 | <#-- --> 84 | 102 | 103 | 104 | 105 |
taskIdtaskNameserverplanstatusmodifyTimeoperatoroperation
${item.id} <#if item.priority==2 > 高${item.name}<#if (appName!)=="" >${item.appName}|${item.targetIp}${item.cron}${(item.status==0)?string("停止",(item.status==1)?string("运行","删除"))} ${item.operTime?string("yyyy-MM-dd HH:mm:ss")}${item.operUser}modify 85 | 86 | 87 | <#if areaCustom.taskConfig??> 88 | 90 | 91 |   92 | 93 | 94 |   95 | 98 | 99 | viewLog 100 | 101 |
106 | <#include "page.ftl"> 107 | 108 |
109 | 110 | <#include "dialog.ftl"> 111 | <#include "res.ftl"> 112 | -------------------------------------------------------------------------------- /easy-sync/src/main/resources/time-task-type-custom.json: -------------------------------------------------------------------------------- 1 | { 2 | 3 | "0":{ 4 | "desc":"Binlog", 5 | "categoryNames":["group"], 6 | "bean":{ 7 | "fixed":"1", 8 | "bean":"mysqlSyncPlugin", 9 | "method":"run", 10 | "args":{}, 11 | "stopMethod":"stop" 12 | } 13 | }, 14 | "1":{ 15 | "desc":"Elasticsearch", 16 | "categoryNames":["group"], 17 | "bean":{ 18 | "fixed":"1", 19 | "bean":"esSyncPlugin", 20 | "method":"run", 21 | "args":{}, 22 | "stopMethod":"stop" 23 | } 24 | }, 25 | "2":{ 26 | "desc":"task", 27 | "categoryNames":["group"] 28 | } 29 | } -------------------------------------------------------------------------------- /easy-sync/src/test/java/BinlogTest.java: -------------------------------------------------------------------------------- 1 | 2 | import com.alibaba.fastjson.JSON; 3 | import com.github.shyiko.mysql.binlog.BinaryLogClient; 4 | import com.github.shyiko.mysql.binlog.event.*; 5 | import com.github.shyiko.mysql.binlog.event.deserialization.EventDeserializer; 6 | import org.junit.Test; 7 | 8 | import java.io.IOException; 9 | import java.io.Serializable; 10 | import java.io.UnsupportedEncodingException; 11 | import java.util.Calendar; 12 | import java.util.Date; 13 | import java.util.TimeZone; 14 | 15 | /** 16 | * https://github.com/shyiko/mysql-binlog-connector-java 17 | */ 18 | public class BinlogTest { 19 | @Test 20 | public void test() throws IOException, InterruptedException { 21 | BinaryLogClient client = new BinaryLogClient("192.168.0.13", 3306, "root", "asdf1234!"); 22 | client.setConnectTimeout(30 * 1000); 23 | client.setKeepAliveInterval(20 * 1000); 24 | client.setKeepAlive(true); 25 | EventDeserializer eventDeserializer = new EventDeserializer(); 26 | eventDeserializer.setCompatibilityMode( 27 | EventDeserializer.CompatibilityMode.DATE_AND_TIME_AS_LONG, 28 | EventDeserializer.CompatibilityMode.CHAR_AND_BINARY_AS_BYTE_ARRAY 29 | // EventDeserializer.CompatibilityMode.In INVALID_DATE_AND_TIME_AS_MIN_VALUE 30 | ); 31 | // client.setEventDeserializer(eventDeserializer); 32 | client.registerEventListener(new BinaryLogClient.EventListener() { 33 | private String table; 34 | 35 | @Override 36 | public void onEvent(Event event) { 37 | System.out.println(event); 38 | if(true) return; 39 | if (event.getHeader().getEventType() == EventType.TABLE_MAP) { 40 | TableMapEventData data = (TableMapEventData) event.getData(); 41 | 42 | table = data.getTable(); 43 | 44 | } 45 | if (table == null || !table.startsWith("coolma")) { 46 | return; 47 | } 48 | System.out.println("------------------------"); 49 | if (event.getHeader().getEventType() == EventType.WRITE_ROWS) { 50 | WriteRowsEventData data = (WriteRowsEventData) event.getData(); 51 | Serializable[] row = data.getRows().get(0); 52 | for (Serializable s : row) 53 | if (s != null) { 54 | if (s instanceof byte[]) { 55 | System.out.println("byte[]=" + to(s)); 56 | } else 57 | System.out.println(s.getClass() + "=" + s); 58 | } else System.out.println("null"); 59 | } 60 | if (event.getHeader().getEventType() == EventType.UPDATE_ROWS) { 61 | UpdateRowsEventData data = (UpdateRowsEventData) event.getData(); 62 | Serializable[] row = data.getRows().iterator().next().getValue(); 63 | for (Serializable s : row) 64 | if (s != null) { 65 | if (s instanceof byte[]) { 66 | System.out.println("byte[]=" + to(s)); 67 | } else if(s instanceof Number){ 68 | Long l=((Number)s).longValue(); 69 | l=l-8*3600*1000; 70 | System.out.println(s+"="+new Date(l).toLocaleString()); 71 | }else 72 | System.out.println(s.getClass() + "="+s);// + new Date(Long.parseLong(s.toString())).toLocaleString()); 73 | } else System.out.println("null"); 74 | } 75 | 76 | } 77 | }); 78 | //client.setBlocking(); 79 | client.connect(); 80 | while (true){ 81 | Thread.sleep(1000); 82 | System.out.println(client.isConnected()); 83 | } 84 | 85 | //System.out.println("ok"); 86 | //java.lang.Thread.sleep(999999); 87 | } 88 | 89 | private static String to(Object s) { 90 | try { 91 | return new String((byte[]) s, "GBK"); 92 | } catch (UnsupportedEncodingException e) { 93 | 94 | e.printStackTrace(); 95 | return ""; 96 | } 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /easy-sync/src/test/java/ConvertTest.java: -------------------------------------------------------------------------------- 1 | import org.apache.commons.beanutils.ConvertUtils; 2 | import org.junit.Assert; 3 | import org.junit.Test; 4 | 5 | 6 | public class ConvertTest { 7 | 8 | @Test 9 | public void booleanTest(){ 10 | Object[][] objects={ 11 | {"",false}, {"0",false} ,{0,false}, {"false",false}, {10,false},{"sss",false}, 12 | {"-100",false}, 13 | {new Object(),false}, 14 | {null,false}, 15 | {"1",true},{"true",true},{"TRUE",true}, 16 | }; 17 | convert(objects); 18 | 19 | } 20 | 21 | 22 | public void convert(Object[][] objects){ 23 | for(Object[] o :objects){ 24 | System.out.println(o[0]+","+o[1]); 25 | Object v= ConvertUtils.convert(o[0], o[1].getClass()); 26 | Assert.assertEquals(v,o[1]); 27 | } 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /easy-sync/src/test/java/EsTest.java: -------------------------------------------------------------------------------- 1 | import com.alibaba.fastjson.JSON; 2 | import com.cehome.easysync.jest.AliasIndices; 3 | import com.cehome.easysync.jest.Jest; 4 | import com.cehome.easysync.objects.config.Es; 5 | import io.searchbox.client.JestClient; 6 | import io.searchbox.client.JestResult; 7 | import io.searchbox.indices.aliases.*; 8 | import org.junit.Test; 9 | 10 | import java.io.IOException; 11 | import java.text.ParseException; 12 | import java.text.SimpleDateFormat; 13 | import java.util.ArrayList; 14 | import java.util.Date; 15 | import java.util.List; 16 | 17 | public class EsTest { 18 | 19 | static JestClient client; 20 | static Jest jest; 21 | static { 22 | Es es=new Es(); 23 | es.setAddresses("http://192.168.0.38:9200"); 24 | jest=new Jest(es); 25 | client=jest.getJestClient(); 26 | } 27 | String alias="crawler_test1"; 28 | 29 | @Test 30 | public void test1(){ 31 | 32 | } 33 | @Test 34 | public void addIndexToAlias() throws IOException { 35 | String index="crawler_test1_180604_143527"; 36 | index="crawler_test1_180604_143156"; 37 | 38 | List list=new ArrayList<>(); 39 | 40 | AddAliasMapping addAliasMapping = new AddAliasMapping.Builder(index,alias).build(); 41 | list.add(addAliasMapping); 42 | ModifyAliases modifyAliases = new ModifyAliases.Builder(list).build(); 43 | print(client.execute(modifyAliases)); 44 | 45 | } 46 | 47 | @Test 48 | public void getAliasesIndexList() throws IOException { 49 | AliasIndices aliasIndices=new AliasIndices(alias); 50 | JestResult jestResult=client.execute(aliasIndices); 51 | print(jestResult); 52 | System.out.println(JSON.toJSONString(aliasIndices.parse(jestResult))); 53 | } 54 | 55 | @Test 56 | public void testAliasesExists() throws IOException { 57 | 58 | alias="crawler_test111"; 59 | AliasExists aliasExists = new AliasExists.Builder().alias(alias).build(); 60 | JestResult jestResult=client.execute(aliasExists); 61 | print(jestResult); 62 | 63 | } 64 | 65 | @Test 66 | public void testIndexExists() throws IOException { 67 | System.out.println( jest.indexExists("equipment_test")); 68 | } 69 | 70 | @Test 71 | public void testDeleteIndex() throws IOException { 72 | System.out.println( jest.deleteIndex("crawler_content11")); 73 | } 74 | 75 | @Test 76 | public void testTime1() throws IOException, ParseException { 77 | System.out.println(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").parse("2016-08-04 14:33:35").getTime()); 78 | } 79 | 80 | @Test 81 | public void testTime2() throws IOException, ParseException { 82 | System.out.println(new Date(1529952232000L)); 83 | } 84 | 85 | private void print(JestResult jestResult){ 86 | if(jestResult.isSucceeded()){ 87 | 88 | System.out.println("ok:"+jestResult.getJsonObject()); 89 | }else{ 90 | System.out.println("error:"+jestResult.getErrorMessage()); 91 | } 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /easy-sync/src/test/java/KafkaTest.java: -------------------------------------------------------------------------------- 1 | import com.cehome.easykafka.Consumer; 2 | import com.cehome.easykafka.Producer; 3 | import com.cehome.easykafka.consumer.ConsumerRecord; 4 | import com.cehome.easykafka.consumer.SimpleKafkaConsumer; 5 | import com.cehome.easykafka.producer.SimpleKafkaProducer; 6 | import org.junit.Test; 7 | 8 | import java.util.List; 9 | import java.util.Properties; 10 | 11 | public class KafkaTest { 12 | @Test 13 | public void test() throws Exception { 14 | 15 | Properties props = new Properties(); 16 | // props.put("bootstrap.servers", "localhost:9092");//"localhost:9092"); 17 | // props.put("acks", "all"); 18 | // props.put("retries", 0); 19 | // props.put("batch.size", 16384); 20 | // props.put("linger.ms", 1); 21 | // props.put("buffer.memory", 33554432); 22 | // props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer"); 23 | // props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer"); 24 | props.put("bootstrap.servers", "localhost:9094"); 25 | props.put("acks", "all"); 26 | props.put("retries", "3"); 27 | props.put("batch.size", "16384"); 28 | props.put("linger.ms", 1); 29 | props.put("buffer.memory", "33554432"); 30 | props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer"); 31 | props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer"); 32 | props.put("message.max.bytes", "10MB"); 33 | props.put("replica.fetch.max.bytes", "10MBB"); 34 | 35 | Producer producer = new SimpleKafkaProducer("0.10.1.0",props); 36 | for (int i = 100; i < 110; i++) 37 | System.out.println( 38 | producer.send("my-topic", Integer.toString(i), Integer.toString(i)) 39 | 40 | ); 41 | 42 | producer.close(); 43 | } 44 | 45 | @Test 46 | public void consumer()throws Exception{ 47 | Properties props = new Properties(); 48 | props.put("bootstrap.servers", "localhost:9094"); 49 | props.put("group.id", "test"); 50 | props.put("enable.auto.commit", "true"); 51 | props.put("auto.commit.interval.ms", "1000"); 52 | props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); 53 | props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); 54 | Consumer consumer = new SimpleKafkaConsumer("0.10.1.0",props); 55 | consumer.createKafkaConsumer(); 56 | consumer.subscribe("my-topic"); 57 | while (true) { 58 | List records = (List) consumer.poll(100); 59 | for (ConsumerRecord record : records) 60 | System.out.printf("offset = %d, key = %s, value = %s%n", record.offset(), record.key(), record.value()); 61 | } 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /pack.bat: -------------------------------------------------------------------------------- 1 | mvn clean package -Dmaven.test.skip=true -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4 | 4.0.0 5 | com.cehome 6 | easy-sync-parent 7 | pom 8 | easy-sync-parent 9 | 2.0.5 10 | https://github.com/cehome-com/easy-sync 11 | easy sync 12 | 13 | 3.2.2.RELEASE 14 | UTF-8 15 | 1.7 16 | 17 | 18 | 19 | 20 | 21 | junit 22 | junit 23 | 3.8.1 24 | test 25 | 26 | 27 | 28 | ch.qos.logback 29 | logback-classic 30 | 1.1.11 31 | 32 | 33 | 34 | javax.servlet 35 | servlet-api 36 | 2.5 37 | provided 38 | 39 | 40 | org.springframework 41 | spring-web 42 | ${spring-version} 43 | provided 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | easy-sync 55 | 58 | easy-kafka-common 59 | easy-kafka-proxy 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | The Apache Software License, Version 2.0 70 | http://www.apache.org/licenses/LICENSE-2.0.txt 71 | 72 | 73 | 74 | 75 | 76 | maruixiang 77 | maruixiang@sina.com 78 | 79 | 80 | 81 | 82 | git@github.com:cehome-com/easy-sync.git 83 | git@github.com:cehome-com/easy-sync.git 84 | https://github.com/cehome-com/easy-sync 85 | 86 | 87 | 88 | 89 | oss 90 | 91 | 92 | oss-snapshots 93 | https://oss.sonatype.org/content/repositories/snapshots/ 94 | 95 | 96 | oss 97 | https://oss.sonatype.org/service/local/staging/deploy/maven2/ 98 | 99 | 100 | 101 | 102 | 103 | org.apache.maven.plugins 104 | maven-compiler-plugin 105 | 3.2 106 | 107 | UTF-8 108 | 1.7 109 | 1.7 110 | 111 | 112 | 113 | 114 | org.apache.maven.plugins 115 | maven-source-plugin 116 | 3.0.1 117 | 118 | 119 | compile 120 | 121 | jar 122 | 123 | 124 | 125 | 126 | 127 | 128 | org.apache.maven.plugins 129 | maven-javadoc-plugin 130 | 2.10.4 131 | 132 | 133 | package 134 | 135 | jar 136 | 137 | 138 | -Xdoclint:none 139 | 140 | 141 | 142 | 143 | 144 | 145 | org.apache.maven.plugins 146 | maven-gpg-plugin 147 | 1.6 148 | 149 | 150 | sign-artifacts 151 | verify 152 | 153 | sign 154 | 155 | 156 | 157 | 158 | 159 | 160 | 161 | 162 | 163 | default 164 | 165 | true 166 | 167 | 168 | 169 | 170 | org.apache.maven.plugins 171 | maven-compiler-plugin 172 | 3.2 173 | 174 | UTF-8 175 | 1.7 176 | 1.7 177 | 178 | 179 | 180 | 181 | org.apache.maven.plugins 182 | maven-source-plugin 183 | 3.0.1 184 | 185 | 186 | compile 187 | 188 | jar 189 | 190 | 191 | 192 | 193 | 194 | 195 | 196 | 197 | internal-releases 198 | Local Release Repository 199 | http://192.168.0.12:8080/nexus/content/repositories/releases/ 200 | 201 | 202 | internal-snapshots 203 | Local Snapshot Repository 204 | http://192.168.0.12:8080/nexus/content/repositories/snapshots/ 205 | 206 | 207 | 208 | 209 | 210 | 211 | 212 | 213 | 214 | 215 | 216 | --------------------------------------------------------------------------------