├── .gitignore ├── LICENSE ├── README.md ├── app ├── build.gradle ├── settings.gradle └── src │ └── main │ ├── java │ ├── ElasticRabbitApp.java │ └── TestDisruptor │ │ ├── LongEvent.java │ │ ├── LongEventFactory.java │ │ ├── LongEventHandler.java │ │ ├── LongEventMain.java │ │ ├── LongEventProducer.java │ │ └── LongEventProducerWithTranslator.java │ └── resources │ ├── application.properties │ └── log4j.properties ├── build.gradle ├── commonutil ├── build.gradle ├── settings.gradle └── src │ └── main │ └── java │ └── springcommon │ ├── ApplicationContextHolder.java │ └── UserInfoContextHolder.java ├── elasticservice ├── .gitignore ├── build.gradle ├── settings.gradle └── src │ └── main │ ├── config.properties │ ├── java │ └── elasticsearch │ │ ├── ClusterFailureListener.java │ │ ├── ESQueryBodyBuilder.java │ │ ├── ElasticRestClient.java │ │ ├── ElasticTransportClient.java │ │ ├── constant │ │ └── ESConstants.java │ │ ├── esapi │ │ ├── ClusterService.java │ │ ├── DocumentService.java │ │ ├── HotDocumentService.java │ │ ├── IndexService.java │ │ └── resp │ │ │ ├── ESBaseResponse.java │ │ │ ├── ESBulkResponse.java │ │ │ ├── ESCountResponse.java │ │ │ ├── ESDeleteResponse.java │ │ │ ├── ESGetByIdResponse.java │ │ │ ├── ESMultiGetResponse.java │ │ │ ├── ESQueryResponse.java │ │ │ ├── ESSaveResponse.java │ │ │ ├── Hit.java │ │ │ └── Hits.java │ │ ├── exception │ │ ├── ElasticAPIException.java │ │ ├── ElasticQueryException.java │ │ └── ElasticVersionConflictException.java │ │ ├── exportimport │ │ ├── LoadData.java │ │ └── mainCrtl.java │ │ └── searchservice │ │ ├── ESService.java │ │ ├── SearchService.java │ │ ├── SearchServiceImpl.java │ │ ├── config │ │ ├── ESConnectionConfig.java │ │ └── ElasticServiceConfig.java │ │ ├── dsl │ │ ├── DLSGenerateServiceImpl.java │ │ ├── DSLGenerateService.java │ │ └── DSLMeta.java │ │ └── models │ │ ├── CacheSearchService.java │ │ ├── CacheSearchServiceImpl.java │ │ ├── Collection.java │ │ ├── ESResp │ │ ├── ESCountResp.java │ │ ├── ESSearchResp.java │ │ ├── Hit.java │ │ ├── HitResult.java │ │ └── HitSource.java │ │ ├── Meta.java │ │ ├── QueryParam.java │ │ └── QueryResp.java │ └── resources │ ├── config.properties │ ├── elasticsearch │ ├── stores-product-mapping.json │ └── storessearch-mapping.json │ ├── log4j.properties │ └── templates │ ├── base.dsl.twig │ ├── count.dsl.twig │ ├── query.dsl.twig │ └── search.dsl.twig ├── flow.png ├── flow.xml ├── generatedata ├── .gitignore ├── build.gradle ├── settings.gradle └── src │ └── main │ ├── java │ └── initdata │ │ ├── RabbitMQApp.java │ │ ├── publish │ │ ├── MessagePublishService.java │ │ └── model │ │ │ ├── Messages.java │ │ │ └── MessagesDef.java │ │ └── sqlexecute │ │ ├── SQLExecute │ │ └── MariaDBSQLExecute.java │ │ └── SQLScriptExecuteService.java │ └── resources │ ├── application.properties │ ├── log4j.properties │ ├── messages │ └── source.json │ ├── rabbit │ └── rabbitmq.properties │ └── sql │ ├── data.sql │ └── db.properties ├── gradle.properties ├── netty-http ├── build.gradle ├── settings.gradle └── src │ ├── main │ ├── java │ │ ├── HttpServerBoot.java │ │ ├── http │ │ │ ├── Config.java │ │ │ ├── DefaultExceptionHandler.java │ │ │ ├── DefaultServerInitializer.java │ │ │ ├── HttpSearchHandler.java │ │ │ ├── HttpSearchResponseHandler.java │ │ │ ├── SearchQueryDecoder.java │ │ │ ├── elasticaction │ │ │ │ ├── RequestMeta.java │ │ │ │ ├── RestRequest.java │ │ │ │ ├── RestResponse.java │ │ │ │ ├── SearchDSL.java │ │ │ │ └── SearchDSLImpl.java │ │ │ ├── exception │ │ │ │ └── BadRequestException.java │ │ │ ├── message │ │ │ │ ├── DecodedSearchRequest.java │ │ │ │ └── QueryMeta.java │ │ │ ├── searchcommand │ │ │ │ ├── RestCommand.java │ │ │ │ └── SearchRequest.java │ │ │ └── worker │ │ │ │ ├── SearchWorker.java │ │ │ │ ├── WorkerThread.java │ │ │ │ └── notifyexecutor │ │ │ │ ├── IFutureListener.java │ │ │ │ ├── INotifyingExecutorService.java │ │ │ │ ├── INotifyingFuture.java │ │ │ │ └── SearchingFuture.java │ │ └── querydsl │ │ │ └── TemplateGenerator.java │ └── resources │ │ ├── log4j.properties │ │ └── templates │ │ └── filters.twig │ └── test │ └── java │ └── TestEhcache.java ├── notes ├── Java_mem_track.md ├── Search.xml ├── TT.xml ├── all_about_search&filter.md ├── basic&sourcestart.md ├── cap.png ├── cluster.xml ├── cluster_deploy.png ├── cluster_relateddesign.md ├── conflict1.png ├── conflict2.png ├── country_sel.png ├── elasticsearch-kibana-grafana.md ├── full-mesh-double.png ├── invent_show.PNG ├── invertindex.png ├── metrics.png ├── msa-arch-analytic.md ├── optimisticlock&versionconflicthandle.md ├── overwrite.png ├── profiling.gif ├── shardsplit.png ├── threenodes.png ├── tokenizer.png ├── tokenizer.xml ├── twonodes.png └── type_sel.png ├── rabbitmqservice ├── .gitignore ├── build.gradle ├── settings.gradle └── src │ └── main │ └── java │ ├── rabbitmq │ ├── ListenerJobBuilder.java │ ├── MQListenerAdmin.java │ ├── MQParsedConfig.java │ └── MessageListenerProxy.java │ └── sync │ ├── common │ ├── ESHandleMessage.java │ ├── MQMessageConst.java │ └── OperateAction.java │ ├── handler │ ├── ESHandler.java │ ├── ESHandlerManager.java │ └── impl │ │ ├── ESAbstractHandler.java │ │ ├── ESProductListSyncHandler.java │ │ ├── ESProductSyncHandler.java │ │ └── ESSKUSyncHandler.java │ └── listener │ └── ESMessageListener.java └── settings.gradle /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | .gradle 3 | build/ 4 | 5 | # Ignore Gradle GUI config 6 | gradle-app.setting 7 | 8 | # Avoid ignoring Gradle wrapper jar file (.jar files are usually ignored) 9 | !gradle-wrapper.jar 10 | 11 | # Cache of project 12 | .gradletasknamecache 13 | 14 | # # Work around https://youtrack.jetbrains.com/issue/IDEA-116898 15 | # gradle/wrapper/gradle-wrapper.properties 16 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2016 Jet He 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # elastic-rabbitmq 2 | load data into elasticsearch from consume rabbitmq, use version control make data consistent. 3 | 4 | # data consume flow 5 | ![flow](./flow.png) 6 | 7 | 从RabbitMQ中消费message，并persist到ES中。一个明显的问题就是ES没有 8 | 事务一致性的保证，通过乐观锁控制每个Document的版本号。在并发写入的情况下这个是一大挑战。 9 | 10 | # Some notes 11 | 1. [ElasticSearch 深入理解 一：基础概念&源码启动](./notes/basic&sourcestart.md) 12 | 2. [ElasticSearch 深入理解 二：乐观锁&版本冲突管理](./notes/optimisticlock&versionconflicthandle.md) 13 | 3. [ElasticSearch 深入理解 三：集群部署设计](./notes/cluster_relateddesign.md) 14 | 4. [ElasticSearch 深入理解 四：Search & Query DSL的生成](./notes/all_about_search&filter.md) 15 | 16 | -------------------------------------------------------------------------------- /app/build.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * This build file was auto generated by running the Gradle 'init' task 3 | * by 'I311352' at '12/7/16 5:12 PM' with Gradle 3.2.1 4 | * 5 | * This generated file contains a sample Java project to get you started. 6 | * For more details take a look at the Java Quickstart chapter in the Gradle 7 | * user guide available at https://docs.gradle.org/3.2.1/userguide/tutorial_java_projects.html 8 | */ 9 | 10 | // Apply the java plugin to add support for Java 11 | apply plugin: 'java' 12 | 13 | buildscript { 14 | repositories { 15 | mavenCentral() 16 | } 17 | dependencies { 18 | classpath("org.springframework.boot:spring-boot-gradle-plugin:1.4.2.RELEASE") 19 | } 20 | } 21 | 22 | //configurations { 23 | // compile.exclude module: 'spring-boot-starter-logging' 24 | //} 25 | 26 | dependencies { 27 | // compile ('org.springframework.boot:spring-boot-starter-web:1.4.3.RELEASE') 28 | // compile("org.springframework.boot:spring-boot-starter-actuator") 29 | 30 | compile 'com.lmax:disruptor:3.3.6' 31 | } 32 | -------------------------------------------------------------------------------- /app/settings.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * This settings file was auto generated by the Gradle buildInit task 3 | * by 'I311352' at '12/7/16 5:18 PM' with Gradle 3.2.1 4 | * 5 | * The settings file is used to specify which projects to include in your build. 6 | * In a single project build this file can be empty or even removed. 7 | * 8 | * Detailed information about configuring a multi-project build in Gradle can be found 9 | * in the user guide at https://docs.gradle.org/3.2.1/userguide/multi_project_builds.html 10 | */ 11 | 12 | /* 13 | // To declare projects as part of a multi-project build use the 'include' method 14 | include 'shared' 15 | include 'api' 16 | include 'services:webservice' 17 | */ 18 | 19 | rootProject.name = 'app' -------------------------------------------------------------------------------- /app/src/main/java/ElasticRabbitApp.java: -------------------------------------------------------------------------------- 1 | import http.Config; 2 | import org.springframework.beans.factory.annotation.Autowired; 3 | import org.springframework.boot.Banner; 4 | import org.springframework.boot.CommandLineRunner; 5 | import org.springframework.boot.SpringApplication; 6 | import org.springframework.boot.SpringBootConfiguration; 7 | import org.springframework.boot.autoconfigure.EnableAutoConfiguration; 8 | import org.springframework.boot.builder.SpringApplicationBuilder; 9 | import org.springframework.cloud.client.circuitbreaker.EnableCircuitBreaker; 10 | import org.springframework.cloud.netflix.hystrix.EnableHystrix; 11 | import org.springframework.cloud.netflix.hystrix.dashboard.EnableHystrixDashboard; 12 | import org.springframework.context.ApplicationContext; 13 | import org.springframework.context.ConfigurableApplicationContext; 14 | import org.springframework.context.annotation.ComponentScan; 15 | import rabbitmq.ListenerJobBuilder; 16 | import rabbitmq.MQListenerAdmin; 17 | 18 | import java.util.concurrent.ConcurrentHashMap; 19 | import java.util.concurrent.locks.ReadWriteLock; 20 | 21 | /** 22 | * Created by I311352 on 12/28/2016. 23 | */ 24 | @SpringBootConfiguration 25 | @EnableAutoConfiguration 26 | @EnableHystrix 27 | @EnableCircuitBreaker 28 | @EnableHystrixDashboard 29 | @ComponentScan({ "elasticsearch","rabbitmq", "sync"}) 30 | public class ElasticRabbitApp implements CommandLineRunner { 31 | @Autowired 32 | private ConfigurableApplicationContext context; 33 | 34 | public static void main(String[] args) { 35 | SpringApplication.run(ElasticRabbitApp.class, args); 36 | } 37 | 38 | @Override 39 | public void run(String... args) throws Exception { 40 | MQListenerAdmin listenerAdmin = ListenerJobBuilder.buildMQListenerAdmin(context); 41 | listenerAdmin.start(); 42 | new HttpServerBoot(context).run(); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /app/src/main/java/TestDisruptor/LongEvent.java: -------------------------------------------------------------------------------- 1 | package TestDisruptor; 2 | 3 | import javafx.event.Event; 4 | 5 | /** 6 | * Created by i311352 on 2/6/2017. 7 | */ 8 | public class LongEvent{ 9 | private long value; 10 | 11 | public void set(long value) { 12 | this.value = value; 13 | } 14 | 15 | @Override 16 | public String toString() { 17 | return "LongEvent{" + 18 | "value=" + value + 19 | '}'; 20 | } 21 | } 22 | 23 | -------------------------------------------------------------------------------- /app/src/main/java/TestDisruptor/LongEventFactory.java: -------------------------------------------------------------------------------- 1 | package TestDisruptor; 2 | 3 | import com.lmax.disruptor.EventFactory; 4 | 5 | /** 6 | * Created by i311352 on 2/6/2017. 7 | */ 8 | public class LongEventFactory implements EventFactory { 9 | public LongEvent newInstance() { 10 | return new LongEvent(); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /app/src/main/java/TestDisruptor/LongEventHandler.java: -------------------------------------------------------------------------------- 1 | package TestDisruptor; 2 | 3 | 4 | import com.lmax.disruptor.EventHandler; 5 | 6 | /** 7 | * Created by i311352 on 2/6/2017. 8 | */ 9 | public class LongEventHandler implements EventHandler { 10 | public void handle(LongEvent event) { 11 | System.out.println("Event: " + event); 12 | } 13 | 14 | public void onEvent(LongEvent event, long sequence, boolean endOfBatch) { 15 | System.out.println("Event: " + event); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /app/src/main/java/TestDisruptor/LongEventMain.java: -------------------------------------------------------------------------------- 1 | package TestDisruptor; 2 | 3 | import com.lmax.disruptor.RingBuffer; 4 | import com.lmax.disruptor.dsl.Disruptor; 5 | 6 | import java.nio.ByteBuffer; 7 | import java.util.concurrent.CountDownLatch; 8 | import java.util.concurrent.Executor; 9 | import java.util.concurrent.Executors; 10 | import java.util.concurrent.ThreadPoolExecutor; 11 | 12 | /** 13 | * Created by i311352 on 2/6/2017. 14 | */ 15 | public class LongEventMain { 16 | public static void main(String[] args) throws Exception { 17 | Executor executor = Executors.newCachedThreadPool(); 18 | 19 | LongEventFactory eventFactory = new LongEventFactory(); 20 | int bufferSize = 1024; 21 | 22 | // Disruptor disruptor = new Disruptor(eventFactory, bufferSize, executor); 23 | // disruptor.handleEventsWith(new LongEventHandler()); 24 | // disruptor.start(); 25 | 26 | Disruptor disruptor = new Disruptor<>(LongEvent::new, bufferSize, executor); 27 | disruptor.handleEventsWith((event, sequence, endOfBatch) -> {System.out.println("Event: " + event); 28 | System.out.println("CurrentThreadName:" + Thread.currentThread().getName()); 29 | }); 30 | disruptor.start(); 31 | 32 | RingBuffer ringBuffer = disruptor.getRingBuffer(); 33 | LongEventProducer producer = new LongEventProducer(ringBuffer); 34 | 35 | ByteBuffer bb = ByteBuffer.allocate(8); 36 | for (long l = 0; true; l++) { 37 | bb.putLong(0, l); 38 | ringBuffer.publishEvent((event, sequence, buffer) -> event.set(buffer.getLong(0)), bb); 39 | 40 | // producer.onData(bb); 41 | //Thread.sleep(1000); 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /app/src/main/java/TestDisruptor/LongEventProducer.java: -------------------------------------------------------------------------------- 1 | package TestDisruptor; 2 | 3 | import com.lmax.disruptor.RingBuffer; 4 | 5 | import java.nio.ByteBuffer; 6 | 7 | /** 8 | * Created by i311352 on 2/6/2017. 9 | */ 10 | public class LongEventProducer { 11 | private final RingBuffer ringBuffer; 12 | public LongEventProducer(RingBuffer ringBuffer) { 13 | this.ringBuffer = ringBuffer; 14 | } 15 | 16 | public void onData(ByteBuffer bb) { 17 | long sequence = ringBuffer.next(); 18 | 19 | try { 20 | LongEvent event = ringBuffer.get(sequence); 21 | event.set(bb.getLong(0)); 22 | } finally { 23 | ringBuffer.publish(sequence); 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /app/src/main/java/TestDisruptor/LongEventProducerWithTranslator.java: -------------------------------------------------------------------------------- 1 | package TestDisruptor; 2 | 3 | import com.lmax.disruptor.EventTranslatorOneArg; 4 | import com.lmax.disruptor.RingBuffer; 5 | 6 | import java.nio.ByteBuffer; 7 | 8 | /** 9 | * Created by i311352 on 2/6/2017. 10 | */ 11 | public class LongEventProducerWithTranslator { 12 | private final RingBuffer ringBuffer; 13 | public LongEventProducerWithTranslator(RingBuffer ringBuffer) { 14 | this.ringBuffer = ringBuffer; 15 | } 16 | 17 | private static final EventTranslatorOneArg TRANSLATOR_ONE_ARG = 18 | new EventTranslatorOneArg() { 19 | @Override 20 | public void translateTo(LongEvent event, long sequence, ByteBuffer arg0) { 21 | event.set(arg0.getLong(0)); 22 | } 23 | }; 24 | public void onData(ByteBuffer bb) { 25 | ringBuffer.publishEvent(TRANSLATOR_ONE_ARG, bb); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /app/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | server.port=20161 2 | 3 | logging.level.org.springframework.web = DEBUG 4 | logging.level.rabbitmq = DEBUG 5 | logging.level.elasticservice = DEBUG -------------------------------------------------------------------------------- /app/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=debug, console 2 | 3 | log4j.appender.console=org.apache.log4j.ConsoleAppender 4 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 5 | log4j.appender.console.layout.conversionPattern=[%d{ISO8601}][%-35p][%-35c{1}] %marker%.10000m%n 6 | -------------------------------------------------------------------------------- /build.gradle: -------------------------------------------------------------------------------- 1 | allprojects { 2 | task hello << { task -> println "I'm $task.project.name" } 3 | } 4 | 5 | subprojects { 6 | apply plugin: 'java' 7 | apply plugin: 'idea' 8 | 9 | repositories { 10 | // Use 'jcenter' for resolving your dependencies. 11 | // You can declare any Maven/Ivy/file repository here. 12 | mavenCentral() 13 | mavenLocal() 14 | jcenter() 15 | maven { 16 | url 'http://nexus.smec/nexus/content/groups/allrepository' 17 | } 18 | } 19 | dependencies { 20 | // The production code uses the SLF4J logging API at compile time 21 | //compile group: 'org.springframework', name: 'spring-webmvc', version: '4.3.5.RELEASE' 22 | //compile group: 'org.springframework.boot', name: 'spring-boot', version: '1.4.3.RELEASE' 23 | //compile group: 'org.springframework', name: 'spring-context', version: '4.3.5.RELEASE' 24 | //compile group: 'org.springframework', name: 'spring-orm', version: '4.3.5.RELEASE' 25 | 26 | compile group: 'org.apache.commons', name: 'commons-lang3', version: '3.5' 27 | compile group: 'commons-collections', name: 'commons-collections', version: '3.2.2' 28 | compile group: 'org.springframework.boot', name: 'spring-boot-actuator', version: '1.4.3.RELEASE' 29 | 30 | // compile group: 'org.hibernate', name: 'hibernate-core', version: '5.2.5.Final' 31 | // compile group: 'org.hibernate', name: 'hibernate-annotations', version: '3.5.6-Final' 32 | // compile group: 'org.springframework', name: 'spring-tx', version: '4.3.5.RELEASE' 33 | compile group: 'org.apache.logging.log4j', name: 'log4j-api', version: '2.7' 34 | compile group: 'org.apache.logging.log4j', name: 'log4j-core', version: '2.7' 35 | 36 | 37 | compile group: 'log4j', name: 'log4j', version: '1.2.17' 38 | compile 'org.slf4j:slf4j-api:1.7.21' 39 | compile 'joda-time:joda-time:2.2' 40 | compile 'com.google.code.gson:gson:2.7' 41 | compile 'commons-io:commons-io:2.4' 42 | compile group: 'org.elasticsearch', name: 'elasticsearch', version: '5.1.1' 43 | compile group: 'org.elasticsearch.client', name: 'rest', version: '5.1.1' 44 | compile group: 'org.elasticsearch.client', name: 'sniffer', version: '5.1.1' 45 | 46 | compile group: 'org.apache.httpcomponents', name: 'httpclient', version: '4.5.2' 47 | 48 | 49 | // Declare the dependency for your favourite test framework you want to use in your tests. 50 | // TestNG is also supported by the Gradle Test task. Just change the 51 | // testCompile dependency to testCompile 'org.testng:testng:6.8.1' and add 52 | // 'test.useTestNG()' to your build script. 53 | testCompile 'junit:junit:4.12' 54 | 55 | compile("org.springframework.cloud:spring-cloud-netflix-core:1.2.5.RELEASE") 56 | compile 'org.springframework.cloud:spring-cloud-starter-hystrix:1.2.5.RELEASE' 57 | compile 'org.springframework.cloud:spring-cloud-starter-hystrix-dashboard:1.2.5.RELEASE' 58 | compile "org.jtwig:jtwig-core:5.85.3.RELEASE" 59 | } 60 | } 61 | 62 | project(':app') { 63 | dependencies { 64 | compile project(':commonutil') 65 | compile project(':elasticservice') 66 | compile project(':rabbitmqservice') 67 | compile project(':netty-http') 68 | } 69 | } 70 | 71 | project(':rabbitmqservice') { 72 | dependencies { 73 | compile project(':commonutil') 74 | compile project(':elasticservice') 75 | } 76 | } 77 | 78 | project(':netty-http') { 79 | dependencies { 80 | compile project(':commonutil') 81 | compile project(':elasticservice') 82 | } 83 | } -------------------------------------------------------------------------------- /commonutil/build.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * This build file was auto generated by running the Gradle 'init' task 3 | * by 'I311352' at '12/7/16 5:12 PM' with Gradle 3.2.1 4 | * 5 | * This generated file contains a sample Java project to get you started. 6 | * For more details take a look at the Java Quickstart chapter in the Gradle 7 | * user guide available at https://docs.gradle.org/3.2.1/userguide/tutorial_java_projects.html 8 | */ 9 | 10 | // Apply the java plugin to add support for Java 11 | apply plugin: 'java' 12 | 13 | buildscript { 14 | repositories { 15 | mavenCentral() 16 | } 17 | dependencies { 18 | classpath("org.springframework.boot:spring-boot-gradle-plugin:1.4.2.RELEASE") 19 | } 20 | } 21 | 22 | //configurations { 23 | // compile.exclude module: 'spring-boot-starter-logging' 24 | //} 25 | 26 | dependencies { 27 | compile group: 'javax.servlet', name: 'javax.servlet-api', version: '3.1.0' 28 | compile ('org.springframework.boot:spring-boot-starter-web:1.4.3.RELEASE') 29 | 30 | } 31 | -------------------------------------------------------------------------------- /commonutil/settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'commonutil' 2 | -------------------------------------------------------------------------------- /commonutil/src/main/java/springcommon/UserInfoContextHolder.java: -------------------------------------------------------------------------------- 1 | package springcommon; 2 | 3 | /** 4 | * Created by I311352 on 12/29/2016. 5 | */ 6 | public class UserInfoContextHolder { 7 | 8 | private static final ThreadLocal userInfoLocal = new InheritableThreadLocal() { 9 | @Override 10 | protected UserInfo initialValue() { 11 | return new UserInfo(); 12 | } 13 | }; 14 | 15 | public static void setUserInfo(UserInfo userInfo) { 16 | if (userInfo != null) { 17 | userInfoLocal.set(userInfo); 18 | } 19 | } 20 | 21 | public static void clear() { 22 | userInfoLocal.remove(); 23 | } 24 | 25 | 26 | public static void setUserInfo(Long tenantId, Long userId, Long employeeId, String locale, String messageId, 27 | String traceId, String sessionId, String jwt) { 28 | UserInfo info = userInfoLocal.get(); 29 | info.employeeId = employeeId; 30 | info.userId = userId; 31 | info.tenantId = tenantId; 32 | info.locale = locale; 33 | info.messageId = messageId; 34 | info.traceId = traceId; 35 | info.sessionId = sessionId; 36 | info.userType = "key"; 37 | info.jwt = jwt; 38 | } 39 | 40 | public static Long getTenantId() { 41 | return userInfoLocal.get().tenantId; 42 | } 43 | 44 | public static Long getUserId() { 45 | return userInfoLocal.get().userId; 46 | } 47 | 48 | public static Long getEmployeeId() { 49 | return userInfoLocal.get().employeeId; 50 | } 51 | 52 | public static String getLocale() { 53 | return userInfoLocal.get().locale; 54 | } 55 | 56 | public static String getMessageId() { 57 | return userInfoLocal.get().messageId; 58 | } 59 | 60 | public static String getTraceId() { 61 | return userInfoLocal.get().traceId; 62 | } 63 | 64 | public static String getSessionId() { 65 | return userInfoLocal.get().sessionId; 66 | } 67 | 68 | public static String getUserType() { 69 | return userInfoLocal.get().userType; 70 | } 71 | 72 | public static String getSource() { 73 | return userInfoLocal.get().source; 74 | } 75 | 76 | public static String getSystemLocale(){ 77 | return userInfoLocal.get().systemlocale; 78 | } 79 | 80 | public static String getRoId(){ 81 | return userInfoLocal.get().roId; 82 | } 83 | 84 | public static String getJwt(){ 85 | return userInfoLocal.get().jwt; 86 | } 87 | 88 | public static class UserInfo { 89 | private Long tenantId = null; 90 | private Long userId = null; 91 | private Long employeeId = null; 92 | private String locale = null; 93 | private String messageId = null; 94 | private String traceId = null; 95 | private String sessionId = null; 96 | private String userType = null; 97 | private String source = null; 98 | private String systemlocale=null; 99 | private String roId=null; 100 | private String jwt = null; 101 | 102 | public Long getTenantId() { 103 | return tenantId; 104 | } 105 | 106 | public void setTenantId(Long tenantId) { 107 | this.tenantId = tenantId; 108 | } 109 | 110 | public Long getUserId() { 111 | return userId; 112 | } 113 | 114 | public void setUserId(Long userId) { 115 | this.userId = userId; 116 | } 117 | 118 | public Long getEmployeeId() { 119 | return employeeId; 120 | } 121 | 122 | public void setEmployeeId(Long employeeId) { 123 | this.employeeId = employeeId; 124 | } 125 | 126 | public String getLocale() { 127 | return locale; 128 | } 129 | 130 | public void setLocale(String locale) { 131 | this.locale = locale; 132 | } 133 | 134 | public String getMessageId() { 135 | return messageId; 136 | } 137 | 138 | public void setMessageId(String messageId) { 139 | this.messageId = messageId; 140 | } 141 | 142 | public String getTraceId() { 143 | return traceId; 144 | } 145 | 146 | public void setTraceId(String traceId) { 147 | this.traceId = traceId; 148 | } 149 | 150 | public String getSessionId() { 151 | return sessionId; 152 | } 153 | 154 | public void setSessionId(String sessionId) { 155 | this.sessionId = sessionId; 156 | } 157 | 158 | public String getUserType() { 159 | return userType; 160 | } 161 | 162 | public void setUserType(String userType) { 163 | this.userType = userType; 164 | } 165 | 166 | public String getSource() { 167 | return source; 168 | } 169 | 170 | public void setSource(String source) { 171 | this.source = source; 172 | } 173 | 174 | public String getSystemlocale() { 175 | return systemlocale; 176 | } 177 | 178 | public void setSystemlocale(String systemlocale) { 179 | this.systemlocale = systemlocale; 180 | } 181 | 182 | public String getRoId() { 183 | return roId; 184 | } 185 | 186 | public void setRoId(String roId) { 187 | this.roId = roId; 188 | } 189 | 190 | public String getJwt() { 191 | return jwt; 192 | } 193 | 194 | public void setJwt(String jwt) { 195 | this.jwt = jwt; 196 | } 197 | 198 | 199 | } 200 | } 201 | 202 | -------------------------------------------------------------------------------- /elasticservice/.gitignore: -------------------------------------------------------------------------------- 1 | .gradle 2 | gradle 3 | gradlew 4 | build/ 5 | 6 | # Ignore Gradle GUI config 7 | gradle-app.setting 8 | 9 | # Avoid ignoring Gradle wrapper jar file (.jar files are usually ignored) 10 | !gradle-wrapper.jar 11 | 12 | # Cache of project 13 | .gradletasknamecache 14 | 15 | # # Work around https://youtrack.jetbrains.com/issue/IDEA-116898 16 | # gradle/wrapper/gradle-wrapper.properties 17 | -------------------------------------------------------------------------------- /elasticservice/build.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * This build file was auto generated by running the Gradle 'init' task 3 | * by 'I311352' at '12/7/16 5:12 PM' with Gradle 3.2.1 4 | * 5 | * This generated file contains a sample Java project to get you started. 6 | * For more details take a look at the Java Quickstart chapter in the Gradle 7 | * user guide available at https://docs.gradle.org/3.2.1/userguide/tutorial_java_projects.html 8 | */ 9 | 10 | // Apply the java plugin to add support for Java 11 | apply plugin: 'java' 12 | apply plugin: 'application' 13 | mainClassName = 'elasticsearch.exportimport.mainCtrl' 14 | 15 | jar { 16 | manifest { 17 | attributes 'Main-Class': 'elasticsearch.exportimport.mainCtrl' 18 | } 19 | } 20 | dependencies { 21 | compile 'com.mitchellbosecke:pebble:2.3.0' 22 | } 23 | buildscript { 24 | repositories { 25 | mavenCentral() 26 | } 27 | dependencies { 28 | classpath("org.springframework.boot:spring-boot-gradle-plugin:1.4.2.RELEASE") 29 | } 30 | } 31 | 32 | -------------------------------------------------------------------------------- /elasticservice/settings.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * This settings file was auto generated by the Gradle buildInit task 3 | * by 'I311352' at '12/7/16 5:12 PM' with Gradle 3.2.1 4 | * 5 | * The settings file is used to specify which projects to include in your build. 6 | * In a single project build this file can be empty or even removed. 7 | * 8 | * Detailed information about configuring a multi-project build in Gradle can be found 9 | * in the user guide at https://docs.gradle.org/3.2.1/userguide/multi_project_builds.html 10 | */ 11 | 12 | /* 13 | // To declare projects as part of a multi-project build use the 'include' method 14 | include 'shared' 15 | include 'api' 16 | include 'services:webservice' 17 | */ 18 | 19 | rootProject.name = 'elasticservice' 20 | -------------------------------------------------------------------------------- /elasticservice/src/main/config.properties: -------------------------------------------------------------------------------- 1 | ES_HOST=127.0.0.1:9200 2 | SAVE_FILE=export.json 3 | TENANTID=35401674516640 4 | S3_BUKET=testBucket -------------------------------------------------------------------------------- /elasticservice/src/main/java/elasticsearch/ClusterFailureListener.java: -------------------------------------------------------------------------------- 1 | package elasticsearch; 2 | 3 | import org.apache.http.HttpHost; 4 | import org.apache.log4j.Logger; 5 | import org.elasticsearch.client.RestClient; 6 | 7 | /** 8 | * Created by I311352 on 9/30/2016. 9 | */ 10 | public class ClusterFailureListener extends RestClient.FailureListener { 11 | private static final Logger logger = Logger.getLogger(ClusterFailureListener.class); 12 | 13 | @Override 14 | public void onFailure(HttpHost host) { 15 | logger.error("Node down! {}:{}" + host.getHostName() + host.getPort()); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /elasticservice/src/main/java/elasticsearch/ElasticRestClient.java: -------------------------------------------------------------------------------- 1 | package elasticsearch; 2 | 3 | import elasticsearch.constant.ESConstants; 4 | import org.apache.http.HttpHost; 5 | import org.apache.log4j.Logger; 6 | import org.elasticsearch.client.RestClient; 7 | import org.elasticsearch.client.sniff.Sniffer; 8 | import org.springframework.beans.factory.config.AbstractFactoryBean; 9 | import org.springframework.stereotype.Component; 10 | 11 | import java.io.IOException; 12 | import java.util.ArrayList; 13 | import java.util.List; 14 | 15 | /** 16 | * Created by I311352 on 9/30/2016. 17 | */ 18 | 19 | @Component 20 | public class ElasticRestClient extends AbstractFactoryBean { 21 | private static Logger logger = Logger.getLogger(ElasticRestClient.class); 22 | private ClusterFailureListener clusterFailureListener; 23 | private volatile RestClient restClient; 24 | private static List hosts = new ArrayList<>(); 25 | private Sniffer sniffer; 26 | 27 | static { 28 | hosts.add("10.128.161.107:9200"); 29 | } 30 | 31 | @Override 32 | public Class getObjectType() { 33 | return RestClient.class; 34 | } 35 | 36 | 37 | @Override 38 | public RestClient createInstance() throws Exception { 39 | if (this.restClient != null) { 40 | return this.restClient; 41 | } 42 | 43 | HttpHost[] addresses = new HttpHost[hosts.size()]; 44 | for (int i = 0; i < hosts.size(); i++) { 45 | addresses[i] = HttpHost.create(hosts.get(i)); 46 | } 47 | 48 | this.restClient = RestClient 49 | .builder(addresses) 50 | .setMaxRetryTimeoutMillis(ESConstants.RESTCLIENT_TIMEOUT) 51 | .build(); 52 | 53 | // this.sniffer = Sniffer.builder(this.restClient) 54 | // .setSniffIntervalMillis(60000).build(); 55 | 56 | return this.restClient; 57 | } 58 | 59 | @Override 60 | protected void destroyInstance(RestClient instance) throws Exception { 61 | try { 62 | logger.info("Closing sniffer"); 63 | instance.close(); 64 | this.sniffer.close(); 65 | } catch (IOException e) { 66 | logger.warn("Error during close sniffer", e); 67 | } 68 | } 69 | 70 | public void setHosts(List hosts) { 71 | this.hosts = hosts; 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /elasticservice/src/main/java/elasticsearch/ElasticTransportClient.java: -------------------------------------------------------------------------------- 1 | package elasticsearch; 2 | 3 | 4 | import org.apache.log4j.Logger; 5 | import org.elasticsearch.client.transport.TransportClient; 6 | 7 | /** 8 | * Created by i325622 on 10/20/16. 9 | */ 10 | public class ElasticTransportClient { 11 | 12 | private static Logger logger = Logger.getLogger(ElasticTransportClient.class); 13 | 14 | private TransportClient client; 15 | 16 | public ElasticTransportClient() { 17 | 18 | // try { 19 | // 20 | // client = TransportClient.builder() 21 | // .settings(Settings.EMPTY) 22 | // .build() 23 | // .addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(ESConstants.clusterIPAdress), 9200)); 24 | // 25 | // } catch (UnknownHostException e) { 26 | // logger.error("init TransportClient fail.", e); 27 | // } 28 | 29 | } 30 | 31 | public TransportClient getClient() { 32 | return client; 33 | } 34 | 35 | public void setClient(TransportClient client) { 36 | this.client = client; 37 | } 38 | } 39 | 40 | -------------------------------------------------------------------------------- /elasticservice/src/main/java/elasticsearch/constant/ESConstants.java: -------------------------------------------------------------------------------- 1 | package elasticsearch.constant; 2 | 3 | import java.util.regex.Pattern; 4 | 5 | /** 6 | * Created by I311352 on 10/17/2016. 7 | */ 8 | public class ESConstants { 9 | public static final String STORE_INDEX = "stores"; 10 | public static final String PRODUCT_TYPE = "product"; 11 | public static final String SKU_TYPE = "sku"; 12 | public static final String PRODUCT_PROPERTY = "productproperty"; 13 | public static final String CHANNEL_TYPE = "channels"; 14 | public static final int SNIFFER_INTERVAL = 15000; 15 | public static final int RESTCLIENT_TIMEOUT = 20000; // 20s 16 | 17 | 18 | public static final Pattern ROUTINGKEY_PATTERN = Pattern.compile("([\\w\\.]+)\\.(\\w+)\\.(\\d+)"); 19 | 20 | } 21 | -------------------------------------------------------------------------------- /elasticservice/src/main/java/elasticsearch/esapi/ClusterService.java: -------------------------------------------------------------------------------- 1 | package elasticsearch.esapi; 2 | 3 | import com.google.gson.JsonObject; 4 | import elasticsearch.constant.ESConstants; 5 | import org.apache.http.HttpHost; 6 | import org.apache.log4j.Logger; 7 | import org.elasticsearch.client.RestClient; 8 | import org.elasticsearch.client.sniff.ElasticsearchHostsSniffer; 9 | import org.elasticsearch.client.sniff.HostsSniffer; 10 | import org.springframework.beans.factory.annotation.Autowired; 11 | import org.springframework.stereotype.Component; 12 | 13 | import java.io.IOException; 14 | import java.util.List; 15 | import java.util.concurrent.Executors; 16 | import java.util.concurrent.ScheduledExecutorService; 17 | import java.util.concurrent.ScheduledFuture; 18 | import java.util.concurrent.TimeUnit; 19 | import java.util.concurrent.atomic.AtomicBoolean; 20 | 21 | /** 22 | * Created by I311352 on 11/17/2016. 23 | */ 24 | 25 | @Component 26 | public class ClusterService { 27 | private static final Logger logger = Logger.getLogger(ClusterService.class); 28 | private final RestClient client; 29 | private final ElasticsearchHostsSniffer elasticsearchHostsSniffer; 30 | private static Task task = null; 31 | 32 | @Autowired 33 | public ClusterService(RestClient client) { 34 | this.client = client; 35 | this.elasticsearchHostsSniffer = new ElasticsearchHostsSniffer(client); 36 | } 37 | 38 | public List getNodesInfo() { 39 | try { 40 | return elasticsearchHostsSniffer.sniffHosts(); 41 | } catch (IOException e) { 42 | logger.error("Cannot get elastic search nodes.." + e); 43 | } 44 | return null; 45 | } 46 | 47 | public JsonObject getNodesJvmInfo() { 48 | return null; 49 | } 50 | 51 | public void startSniffer() { 52 | this.task = new Task(elasticsearchHostsSniffer, client, ESConstants.SNIFFER_INTERVAL); 53 | } 54 | 55 | // sniffer implementation 56 | private static class Task implements Runnable { 57 | private final HostsSniffer hostsSniffer; 58 | private final RestClient restClient; 59 | //private NodeStatusService nodeStatusService = null; 60 | private int lastCheckHttpHosts = 0; 61 | 62 | private final long sniffIntervalMillis; 63 | private final ScheduledExecutorService scheduledExecutorService; 64 | private final AtomicBoolean running = new AtomicBoolean(false); 65 | private ScheduledFuture scheduledFuture; 66 | 67 | private Task(HostsSniffer hostsSniffer, RestClient restClient, int sniffIntervalMillis) { 68 | this.hostsSniffer = hostsSniffer; 69 | this.restClient = restClient; 70 | this.sniffIntervalMillis = sniffIntervalMillis; 71 | this.scheduledExecutorService = Executors.newScheduledThreadPool(1); 72 | //this.nodeStatusService = null;// ApplicationContextHolder.getBean(NodeStatusService.class); 73 | scheduleNextRun(0); 74 | } 75 | 76 | synchronized void scheduleNextRun(long delayMillis) { 77 | if (scheduledExecutorService.isShutdown() == false) { 78 | try { 79 | if (scheduledFuture != null) { 80 | //regardless of when the next sniff is scheduled, cancel it and schedule a new one with updated delay 81 | this.scheduledFuture.cancel(false); 82 | } 83 | logger.debug("scheduling next sniff in " + delayMillis + " ms"); 84 | this.scheduledFuture = this.scheduledExecutorService.schedule(this, delayMillis, TimeUnit.MILLISECONDS); 85 | } catch(Exception e) { 86 | logger.error("error while scheduling next sniffer task", e); 87 | } 88 | } 89 | } 90 | 91 | @Override 92 | public void run() { 93 | sniff(null, sniffIntervalMillis); 94 | } 95 | 96 | void sniff(HttpHost excludeHost, long nextSniffDelayMillis) { 97 | List sniffedHosts = null; 98 | if (running.compareAndSet(false, true)) { 99 | try { 100 | sniffedHosts = hostsSniffer.sniffHosts(); 101 | logger.debug("sniffed hosts: " + sniffedHosts); 102 | if (excludeHost != null) { 103 | sniffedHosts.remove(excludeHost); 104 | } 105 | if (sniffedHosts.isEmpty()) { 106 | logger.warn("no hosts to set, hosts will be updated at the next sniffing round"); 107 | } else { 108 | this.restClient.setHosts(sniffedHosts.toArray(new HttpHost[sniffedHosts.size()])); 109 | } 110 | } catch (Exception e) { 111 | logger.error("error while sniffing nodes", e); 112 | } finally { 113 | scheduleNextRun(nextSniffDelayMillis); 114 | running.set(false); 115 | // exception happen 116 | if (sniffedHosts == null) { 117 | // rest client is round robin check next time 118 | lastCheckHttpHosts --; 119 | if (lastCheckHttpHosts >= 1) { 120 | return; 121 | } 122 | } 123 | // nodeStatusService.checkStatus(sniffedHosts); 124 | lastCheckHttpHosts = sniffedHosts.size(); 125 | } 126 | } 127 | } 128 | 129 | synchronized void shutdown() { 130 | scheduledExecutorService.shutdown(); 131 | try { 132 | if (scheduledExecutorService.awaitTermination(1000, TimeUnit.MILLISECONDS)) { 133 | return; 134 | } 135 | scheduledExecutorService.shutdownNow(); 136 | } catch (InterruptedException e) { 137 | Thread.currentThread().interrupt(); 138 | } 139 | } 140 | } 141 | } 142 | -------------------------------------------------------------------------------- /elasticservice/src/main/java/elasticsearch/esapi/IndexService.java: -------------------------------------------------------------------------------- 1 | package elasticsearch.esapi; 2 | 3 | import elasticsearch.constant.ESConstants; 4 | import org.apache.commons.io.IOUtils; 5 | import org.apache.http.HttpEntity; 6 | import org.apache.http.entity.StringEntity; 7 | import org.apache.log4j.Logger; 8 | import org.elasticsearch.client.Response; 9 | import org.elasticsearch.client.RestClient; 10 | import org.springframework.beans.factory.annotation.Autowired; 11 | import org.springframework.stereotype.Component; 12 | 13 | import java.io.IOException; 14 | import java.io.InputStream; 15 | import java.io.UnsupportedEncodingException; 16 | import java.util.Hashtable; 17 | 18 | /** 19 | * Created by I311352 on 9/30/2016. 20 | */ 21 | 22 | @Component 23 | public class IndexService { 24 | private static final String CURRENT_INDEX_VERSION = ESConstants.STORE_INDEX + "_v1"; 25 | private static final Logger logger = Logger.getLogger(IndexService.class); 26 | private static volatile boolean initialized = false; 27 | private final RestClient client; 28 | 29 | @Autowired 30 | public IndexService(RestClient client) { 31 | this.client = client; 32 | 33 | if (!initialized) { 34 | CreateIndexIfNotExist(); 35 | initialized = true; 36 | } 37 | } 38 | 39 | public void CreateIndexIfNotExist() { 40 | if (indexExist(CURRENT_INDEX_VERSION)) { 41 | logger.info("Index Exist, index name:" + CURRENT_INDEX_VERSION ); 42 | addAlias(CURRENT_INDEX_VERSION, ESConstants.STORE_INDEX); 43 | return; 44 | } 45 | 46 | InputStream inputStream = null; 47 | try { 48 | String INDEX_MAPING_FILE = "/elasticsearch/stores-product-mapping.json"; 49 | inputStream = this.getClass().getResourceAsStream(INDEX_MAPING_FILE); 50 | String mapInfo = IOUtils.toString(inputStream); 51 | createIndex(CURRENT_INDEX_VERSION, mapInfo); 52 | } catch (IOException e) { 53 | logger.error("Could not read mapping file for creating index", e); 54 | } finally { 55 | IOUtils.closeQuietly(inputStream); 56 | } 57 | } 58 | 59 | public Boolean indexExist(String indexName) { 60 | try { 61 | Response response = client.performRequest("HEAD", indexName); 62 | 63 | int statusCode = response.getStatusLine().getStatusCode(); 64 | if (statusCode == 200) { 65 | return true; 66 | } else if (statusCode == 404) { 67 | return false; 68 | } 69 | logger.error("Error checking index existence: {}" + response.getStatusLine().getReasonPhrase()); 70 | 71 | } catch (IOException e) { 72 | logger.error("Failed to verify the index existence ", e); 73 | } 74 | 75 | return true; 76 | } 77 | 78 | public void createIndex(String indexName, String requestBody) { 79 | try { 80 | HttpEntity entity = new StringEntity(requestBody); 81 | Response response = client.performRequest("PUT", 82 | indexName, 83 | new Hashtable<>(), 84 | entity); 85 | 86 | int statusCode = response.getStatusLine().getStatusCode(); 87 | 88 | if (statusCode > 299) { 89 | logger.warn("Error while creating an index: {}" + response.getStatusLine().getReasonPhrase()); 90 | } 91 | 92 | addAlias(indexName, ESConstants.STORE_INDEX); 93 | } catch (UnsupportedEncodingException e) { 94 | logger.warn("Failed to converting the request body into an http entity"); 95 | } catch (IOException e) { 96 | logger.warn("Failed to creating new index."); 97 | } 98 | 99 | logger.info("Create index successful, indexName:" + indexName); 100 | } 101 | 102 | public void refreshIndex(String indexName){ 103 | try { 104 | client.performRequest("POST", 105 | indexName + "/_refresh"); 106 | } catch (UnsupportedEncodingException e) { 107 | logger.warn("Failed to refresh the store index"); 108 | } catch (IOException e) { 109 | logger.warn("Failed to refresh the store index."); 110 | } 111 | logger.info("Refresh index successful, indexName:" + indexName); 112 | } 113 | 114 | private void addAlias(String indexName, String alias) { 115 | try { 116 | Response response = client.performRequest("PUT", 117 | indexName+"/_alias/"+alias); 118 | int statusCode = response.getStatusLine().getStatusCode(); 119 | if (statusCode > 299) { 120 | logger.warn("Error while addAlias an index: {}" + response.getStatusLine().getReasonPhrase()); 121 | } 122 | } catch (IOException e) { 123 | logger.warn("Failed to addAlias." + indexName + alias); 124 | } 125 | } 126 | 127 | public void reIndex(String newIndex, String oldIndex) { 128 | 129 | } 130 | } 131 | -------------------------------------------------------------------------------- /elasticservice/src/main/java/elasticsearch/esapi/resp/ESBaseResponse.java: -------------------------------------------------------------------------------- 1 | package elasticsearch.esapi.resp; 2 | 3 | import com.google.gson.annotations.SerializedName; 4 | 5 | /** 6 | * Created by I311352 on 10/18/2016. 7 | */ 8 | public class ESBaseResponse { 9 | @SerializedName("_index") 10 | protected String index; 11 | @SerializedName("_type") 12 | protected String type; 13 | @SerializedName("_id") 14 | protected String id; 15 | @SerializedName("_version") 16 | protected Long version; 17 | @SerializedName("_routing") 18 | protected String routing; 19 | 20 | public String getIndex() { 21 | return index; 22 | } 23 | 24 | public void setIndex(String index) { 25 | this.index = index; 26 | } 27 | 28 | public String getType() { 29 | return type; 30 | } 31 | 32 | public void setType(String type) { 33 | this.type = type; 34 | } 35 | 36 | public String getId() { 37 | return id; 38 | } 39 | 40 | public void setId(String id) { 41 | this.id = id; 42 | } 43 | 44 | public Long getVersion() { 45 | return version; 46 | } 47 | 48 | public void setVersion(Long version) { 49 | this.version = version; 50 | } 51 | 52 | public String getRouting() { 53 | return routing; 54 | } 55 | 56 | public void setRouting(String routing) { 57 | this.routing = routing; 58 | } 59 | 60 | @Override 61 | public String toString() { 62 | return "ESBaseResponse{" + 63 | "index='" + index + '\'' + 64 | ", type='" + type + '\'' + 65 | ", id='" + id + '\'' + 66 | ", version=" + version + 67 | ", routing='" + routing + '\'' + 68 | '}'; 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /elasticservice/src/main/java/elasticsearch/esapi/resp/ESBulkResponse.java: -------------------------------------------------------------------------------- 1 | package elasticsearch.esapi.resp; 2 | 3 | import com.google.gson.JsonArray; 4 | 5 | /** 6 | * Created by I311352 on 11/8/2016. 7 | */ 8 | public class ESBulkResponse { 9 | protected Long took; 10 | protected Boolean errors; 11 | protected JsonArray items; 12 | 13 | public Long getTook() { 14 | return took; 15 | } 16 | 17 | public void setTook(Long took) { 18 | this.took = took; 19 | } 20 | 21 | public Boolean getErrors() { 22 | return errors; 23 | } 24 | 25 | public void setErrors(Boolean errors) { 26 | this.errors = errors; 27 | } 28 | 29 | public JsonArray getItems() { 30 | return items; 31 | } 32 | 33 | public void setItems(JsonArray items) { 34 | this.items = items; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /elasticservice/src/main/java/elasticsearch/esapi/resp/ESCountResponse.java: -------------------------------------------------------------------------------- 1 | package elasticsearch.esapi.resp; 2 | 3 | import com.google.gson.JsonObject; 4 | import com.google.gson.annotations.SerializedName; 5 | 6 | /** 7 | * Created by i325622 on 10/19/16. 8 | */ 9 | public class ESCountResponse { 10 | @SerializedName("count") 11 | protected Long count; 12 | @SerializedName("_shards") 13 | protected JsonObject _shards; 14 | 15 | public Long getCount() { 16 | return count; 17 | } 18 | 19 | public void setCount(Long count) { 20 | this.count = count; 21 | } 22 | 23 | public JsonObject getShards() { 24 | return _shards; 25 | } 26 | 27 | public void setShards(JsonObject shards) { 28 | this._shards = shards; 29 | } 30 | 31 | } 32 | -------------------------------------------------------------------------------- /elasticservice/src/main/java/elasticsearch/esapi/resp/ESDeleteResponse.java: -------------------------------------------------------------------------------- 1 | package elasticsearch.esapi.resp; 2 | 3 | import com.google.gson.JsonObject; 4 | import com.google.gson.annotations.SerializedName; 5 | 6 | /** 7 | * Created by I311352 on 10/18/2016. 8 | */ 9 | public class ESDeleteResponse extends ESBaseResponse { 10 | protected Boolean found; 11 | @SerializedName("_shards") 12 | protected JsonObject shards; 13 | 14 | public Boolean getFound() { 15 | return found; 16 | } 17 | 18 | public void setFound(Boolean found) { 19 | this.found = found; 20 | } 21 | 22 | public JsonObject getShards() { 23 | return shards; 24 | } 25 | 26 | public void setShards(JsonObject shards) { 27 | this.shards = shards; 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /elasticservice/src/main/java/elasticsearch/esapi/resp/ESGetByIdResponse.java: -------------------------------------------------------------------------------- 1 | package elasticsearch.esapi.resp; 2 | 3 | import com.google.gson.JsonObject; 4 | import com.google.gson.annotations.SerializedName; 5 | 6 | /** 7 | * Created by I311352 on 10/26/2016. 8 | */ 9 | public class ESGetByIdResponse extends ESBaseResponse { 10 | private Boolean found; 11 | @SerializedName("_source") 12 | private JsonObject object; 13 | 14 | public Boolean getFound() { 15 | return found; 16 | } 17 | 18 | public void setFound(Boolean found) { 19 | this.found = found; 20 | } 21 | 22 | public JsonObject getObject() { 23 | return object; 24 | } 25 | 26 | public void setObject(JsonObject object) { 27 | this.object = object; 28 | } 29 | 30 | @Override 31 | public String toString() { 32 | return super.toString() + "ESGetByIdResponse{" + 33 | "found=" + found + 34 | ", object=" + object + 35 | '}'; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /elasticservice/src/main/java/elasticsearch/esapi/resp/ESMultiGetResponse.java: -------------------------------------------------------------------------------- 1 | package elasticsearch.esapi.resp; 2 | 3 | import java.util.ArrayList; 4 | 5 | /** 6 | * Created by I311352 on 11/14/2016. 7 | */ 8 | public class ESMultiGetResponse { 9 | private ArrayList docs; 10 | 11 | public ArrayList getDocs() { 12 | return docs; 13 | } 14 | 15 | public void setDocs(ArrayList docs) { 16 | this.docs = docs; 17 | } 18 | 19 | @Override 20 | public String toString() { 21 | return "ESMultiGetResponse{" + 22 | "docs=" + docs + 23 | '}'; 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /elasticservice/src/main/java/elasticsearch/esapi/resp/ESQueryResponse.java: -------------------------------------------------------------------------------- 1 | package elasticsearch.esapi.resp; 2 | 3 | import com.google.gson.JsonObject; 4 | import com.google.gson.annotations.SerializedName; 5 | 6 | /** 7 | * Created by I311352 on 10/17/2016. 8 | */ 9 | public class ESQueryResponse { 10 | @SerializedName("_scroll_id") 11 | protected String scrollId; 12 | 13 | protected Long took; 14 | 15 | @SerializedName("timed_out") 16 | protected Boolean timeOut; 17 | 18 | @SerializedName("_shards") 19 | protected JsonObject shards; 20 | 21 | @SerializedName("hits") 22 | protected Hit hit; 23 | 24 | public Long getTook() { 25 | return took; 26 | } 27 | 28 | public void setTook(Long took) { 29 | this.took = took; 30 | } 31 | 32 | public Boolean getTimeOut() { 33 | return timeOut; 34 | } 35 | 36 | public void setTimeOut(Boolean timeOut) { 37 | this.timeOut = timeOut; 38 | } 39 | 40 | public JsonObject getShards() { 41 | return shards; 42 | } 43 | 44 | public void setShards(JsonObject shards) { 45 | this.shards = shards; 46 | } 47 | 48 | public Hit getHit() { 49 | return hit; 50 | } 51 | 52 | public void setHit(Hit hit) { 53 | this.hit = hit; 54 | } 55 | 56 | public String getScrollId() { 57 | return scrollId; 58 | } 59 | 60 | public void setScrollId(String scrollId) { 61 | this.scrollId = scrollId; 62 | } 63 | 64 | @Override 65 | public String toString() { 66 | return "ESQueryResponse{" + 67 | "scrollId='" + scrollId + '\'' + 68 | ", took=" + took + 69 | ", timeOut=" + timeOut + 70 | ", shards=" + shards + 71 | ", hit=" + hit + 72 | '}'; 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /elasticservice/src/main/java/elasticsearch/esapi/resp/ESSaveResponse.java: -------------------------------------------------------------------------------- 1 | package elasticsearch.esapi.resp; 2 | 3 | import com.google.gson.JsonObject; 4 | import com.google.gson.annotations.SerializedName; 5 | 6 | /** 7 | * Created by I311352 on 10/18/2016. 8 | */ 9 | public class ESSaveResponse extends ESBaseResponse { 10 | protected Boolean created; 11 | @SerializedName("_shards") 12 | protected JsonObject shards; 13 | 14 | public Boolean getCreated() { 15 | return created; 16 | } 17 | 18 | public void setCreated(Boolean created) { 19 | this.created = created; 20 | } 21 | 22 | public JsonObject getShards() { 23 | return shards; 24 | } 25 | 26 | public void setShards(JsonObject shards) { 27 | this.shards = shards; 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /elasticservice/src/main/java/elasticsearch/esapi/resp/Hit.java: -------------------------------------------------------------------------------- 1 | package elasticsearch.esapi.resp; 2 | 3 | import com.google.gson.annotations.SerializedName; 4 | 5 | import java.util.List; 6 | 7 | /** 8 | * Created by I311352 on 10/17/2016. 9 | */ 10 | public class Hit { 11 | private Long total; 12 | 13 | @SerializedName("max_score") 14 | private String maxScore; 15 | private List hits; 16 | 17 | public Long getTotal() { 18 | return total; 19 | } 20 | 21 | public void setTotal(Long total) { 22 | this.total = total; 23 | } 24 | 25 | public String getMaxScore() { 26 | return maxScore; 27 | } 28 | 29 | public void setMaxScore(String maxScore) { 30 | this.maxScore = maxScore; 31 | } 32 | 33 | public List getHits() { 34 | return hits; 35 | } 36 | 37 | public void setHits(List hits) { 38 | this.hits = hits; 39 | } 40 | 41 | @Override 42 | public String toString() { 43 | return "Hit{" + 44 | "total=" + total + 45 | ", maxScore='" + maxScore + '\'' + 46 | ", hits=" + hits + 47 | '}'; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /elasticservice/src/main/java/elasticsearch/esapi/resp/Hits.java: -------------------------------------------------------------------------------- 1 | package elasticsearch.esapi.resp; 2 | 3 | import com.google.gson.JsonObject; 4 | import com.google.gson.annotations.SerializedName; 5 | 6 | /** 7 | * Created by I311352 on 10/17/2016. 8 | */ 9 | 10 | /** 11 | * no version in query response 12 | */ 13 | public class Hits extends ESBaseResponse { 14 | @SerializedName("_score") 15 | private String score; 16 | @SerializedName("_source") 17 | private JsonObject source; 18 | 19 | public String getScore() { 20 | return score; 21 | } 22 | 23 | public void setScore(String score) { 24 | this.score = score; 25 | } 26 | 27 | public JsonObject getSource() { 28 | return source; 29 | } 30 | 31 | public void setSource(JsonObject source) { 32 | this.source = source; 33 | } 34 | 35 | @Override 36 | public String toString() { 37 | return "Hits{" + 38 | "index='" + index + '\'' + 39 | ", type='" + type + '\'' + 40 | ", id='" + id + '\'' + 41 | ", score='" + score + '\'' + 42 | ", routing='" + routing + '\'' + 43 | ", source=" + source + 44 | '}'; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /elasticservice/src/main/java/elasticsearch/exception/ElasticAPIException.java: -------------------------------------------------------------------------------- 1 | package elasticsearch.exception; 2 | 3 | /** 4 | * Created by I311352 on 9/30/2016. 5 | */ 6 | public class ElasticAPIException extends RuntimeException { 7 | public ElasticAPIException(String message) { 8 | super(message); 9 | } 10 | 11 | public ElasticAPIException(String message, Throwable cause) { 12 | super(message, cause); 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /elasticservice/src/main/java/elasticsearch/exception/ElasticQueryException.java: -------------------------------------------------------------------------------- 1 | package elasticsearch.exception; 2 | 3 | /** 4 | * Created by I311352 on 9/30/2016. 5 | */ 6 | public class ElasticQueryException extends RuntimeException { 7 | public ElasticQueryException(String message) { 8 | super(message); 9 | } 10 | 11 | public ElasticQueryException(String message, Throwable cause) { 12 | super(message, cause); 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /elasticservice/src/main/java/elasticsearch/exception/ElasticVersionConflictException.java: -------------------------------------------------------------------------------- 1 | package elasticsearch.exception; 2 | 3 | /** 4 | * Created by I311352 on 10/25/2016. 5 | */ 6 | public class ElasticVersionConflictException extends RuntimeException { 7 | public ElasticVersionConflictException(String message) { 8 | super(message); 9 | } 10 | 11 | public ElasticVersionConflictException(String message, Throwable cause) { 12 | super(message, cause); 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /elasticservice/src/main/java/elasticsearch/exportimport/mainCrtl.java: -------------------------------------------------------------------------------- 1 | package elasticsearch.exportimport; 2 | 3 | import org.apache.log4j.Logger; 4 | import org.springframework.boot.CommandLineRunner; 5 | import org.springframework.boot.SpringApplication; 6 | import org.springframework.boot.autoconfigure.SpringBootApplication; 7 | import org.springframework.context.ApplicationContext; 8 | import org.springframework.context.annotation.Bean; 9 | 10 | import java.io.File; 11 | import java.io.FileInputStream; 12 | import java.io.IOException; 13 | import java.util.Properties; 14 | 15 | /** 16 | * Created by I311352 on 4/24/2017. 17 | */ 18 | @SpringBootApplication 19 | public class mainCrtl { 20 | private static final Logger logger = Logger.getLogger(mainCrtl.class); 21 | 22 | public static void main(String args[]) { 23 | SpringApplication.run(mainCrtl.class, args); 24 | 25 | 26 | } 27 | 28 | @Bean 29 | public CommandLineRunner commandLineRunner(ApplicationContext ctx) { 30 | return args -> { 31 | String fileName = "export.json"; 32 | String eshost = "127.0.0.1:9200"; 33 | String buket = "s3_buket"; 34 | Long tenantId = 35401674516640L; 35 | 36 | LoadData loadData = new LoadData(); 37 | loadData.saveData(fileName, tenantId, buket, eshost); 38 | 39 | if (System.getProperty("config.properties") != null) { 40 | logger.info("Use user config " + System.getProperty("config.properties")); 41 | try { 42 | File file = new File(System.getProperty("config.properties")); 43 | if (file.exists()) { 44 | FileInputStream inputStream = new FileInputStream(file); 45 | Properties props = new Properties(); 46 | props.load(inputStream); 47 | 48 | // String fileName = props.getProperty("SAVE_FILE"); 49 | // String eshost = props.getProperty("ES_HOST"); 50 | // String buket = props.getProperty("S3_BUKET"); 51 | // Long tenantId = Long.parseLong(props.getProperty("TENANTID")); 52 | // 53 | // logger.info("going to export data fileName=" + fileName +" ESHOST="+eshost + " buket="+buket 54 | // +" tenantId=" +tenantId); 55 | // 56 | // LoadData loadData = new LoadData(); 57 | // loadData.saveData(fileName, tenantId, buket, eshost); 58 | } 59 | } catch (IOException e) { 60 | logger.error("file load fail "+e); 61 | } 62 | } else { 63 | logger.error("cannot get properties"); 64 | } 65 | 66 | System.exit(0); 67 | }; 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /elasticservice/src/main/java/elasticsearch/searchservice/ESService.java: -------------------------------------------------------------------------------- 1 | package elasticsearch.searchservice; 2 | 3 | import elasticsearch.ElasticRestClient; 4 | import org.elasticsearch.client.RestClient; 5 | import org.elasticsearch.rest.RestResponse; 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | import org.springframework.beans.factory.annotation.Autowired; 9 | import org.springframework.stereotype.Service; 10 | import org.springframework.web.client.RestClientException; 11 | 12 | /** 13 | * Created by i311352 on 5/2/2017. 14 | */ 15 | @Service 16 | public class ESService { 17 | private static final Logger logger = LoggerFactory.getLogger(ESService.class); 18 | 19 | private static String baseURL; 20 | private static String searchProductURL; 21 | private static String countProductURL; 22 | 23 | @Autowired 24 | ElasticRestClient client; 25 | 26 | public ESService(String url) { 27 | this.setBaseURL(url); 28 | this.setSearchProductURL(url+"/storessearch/product/_search"); 29 | this.setCountProductURL(url+"/storessearch/product/_count"); 30 | logger.info("Initial ESService done, url: " + url); 31 | } 32 | 33 | public void setBaseURL(String url) { 34 | this.baseURL = url; 35 | } 36 | 37 | public void setSearchProductURL(String searchURL) { 38 | ESService.searchProductURL = searchURL; 39 | } 40 | 41 | public static void setCountProductURL(String countProductURL) { 42 | ESService.countProductURL = countProductURL; 43 | } 44 | 45 | // public RestResponse postSearch(byte[] body) { 46 | // // only return ids 47 | // map.put("_source", "id"); 48 | // try { 49 | // return client.post(searchProductURL, body, map); 50 | // } catch (RestClientException e) { 51 | // //logger.warn("DO search failed " + e); 52 | // return null; 53 | // } 54 | // } 55 | // 56 | // public RestResponse postSearchCount(byte[] body) { 57 | // try { 58 | // return client.post(countProductURL, body, map); 59 | // } catch (RestClientException e) { 60 | // //logger.warn("DO search failed " + e); 61 | // return null; 62 | // } 63 | // } 64 | } 65 | -------------------------------------------------------------------------------- /elasticservice/src/main/java/elasticsearch/searchservice/SearchService.java: -------------------------------------------------------------------------------- 1 | package elasticsearch.searchservice; 2 | 3 | 4 | import elasticsearch.searchservice.models.QueryParam; 5 | import elasticsearch.searchservice.models.QueryResp; 6 | import org.springframework.stereotype.Service; 7 | 8 | /** 9 | * Created by i311352 on 5/2/2017. 10 | */ 11 | @Service 12 | public interface SearchService { 13 | QueryResp doSearch(QueryParam queryFromPHP); 14 | QueryResp doCount(QueryParam queryFromPHP); 15 | } 16 | -------------------------------------------------------------------------------- /elasticservice/src/main/java/elasticsearch/searchservice/SearchServiceImpl.java: -------------------------------------------------------------------------------- 1 | package elasticsearch.searchservice; 2 | 3 | import elasticsearch.searchservice.dsl.DSLGenerateService; 4 | import elasticsearch.searchservice.models.CacheSearchService; 5 | import elasticsearch.searchservice.models.QueryParam; 6 | import elasticsearch.searchservice.models.QueryResp; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | import org.springframework.beans.factory.annotation.Autowired; 10 | import org.springframework.stereotype.Service; 11 | 12 | /** 13 | * Created by i311352 on 5/2/2017. 14 | */ 15 | @Service 16 | public class SearchServiceImpl implements SearchService { 17 | private static final Logger logger = LoggerFactory.getLogger(SearchServiceImpl.class); 18 | 19 | @Autowired 20 | private CacheSearchService cacheSearchService; 21 | 22 | @Autowired 23 | private DSLGenerateService dslGenerateService; 24 | 25 | @Override 26 | public QueryResp doCount(QueryParam queryFromPHP) { 27 | if (queryFromPHP.getChannelId() == null) { 28 | logger.warn("Must have channelId for elasticsearch "); 29 | return null; 30 | } 31 | 32 | logger.info("start do count"); 33 | queryFromPHP.setCount(true); 34 | String body = dslGenerateService.fromQueryParam(queryFromPHP); 35 | logger.info("count DSL \r\n " + body); 36 | Integer count = cacheSearchService.count(body); 37 | logger.info("count done " + count); 38 | return new QueryResp(count.longValue()); 39 | } 40 | 41 | @Override 42 | public QueryResp doSearch(QueryParam queryFromPHP) { 43 | if (queryFromPHP.getChannelId() == null) { 44 | logger.warn("Must have channelId for elasticsearch "); 45 | return null; 46 | } 47 | 48 | logger.info("start do search"); 49 | String body = dslGenerateService.fromQueryParam(queryFromPHP); 50 | logger.info("search DSL \r\n" + body); 51 | return cacheSearchService.query(body); 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /elasticservice/src/main/java/elasticsearch/searchservice/config/ESConnectionConfig.java: -------------------------------------------------------------------------------- 1 | package elasticsearch.searchservice.config; 2 | 3 | /** 4 | * Created by I311352 on 6/9/2017. 5 | */ 6 | public class ESConnectionConfig { 7 | private String esDatabase; 8 | private String esHost; 9 | private Integer esPort; 10 | 11 | public ESConnectionConfig() { 12 | } 13 | 14 | public String getEsDatabase() { 15 | return this.esDatabase; 16 | } 17 | 18 | public void setEsDatabase(String esDatabase) { 19 | this.esDatabase = esDatabase; 20 | } 21 | 22 | public String getEsHost() { 23 | return this.esHost; 24 | } 25 | 26 | public void setEsHost(String esHost) { 27 | this.esHost = esHost; 28 | } 29 | 30 | public Integer getEsPort() { 31 | return this.esPort; 32 | } 33 | 34 | public void setEsPort(Integer esPort) { 35 | this.esPort = esPort; 36 | } 37 | } -------------------------------------------------------------------------------- /elasticservice/src/main/java/elasticsearch/searchservice/config/ElasticServiceConfig.java: -------------------------------------------------------------------------------- 1 | package elasticsearch.searchservice.config; 2 | 3 | import elasticsearch.searchservice.ESService; 4 | import org.apache.commons.lang3.StringUtils; 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | import org.springframework.context.annotation.Bean; 8 | import org.springframework.context.annotation.Configuration; 9 | import org.springframework.context.annotation.Primary; 10 | import org.springframework.web.servlet.config.annotation.WebMvcConfigurerAdapter; 11 | 12 | import java.io.File; 13 | import java.io.FileInputStream; 14 | import java.io.IOException; 15 | import java.io.InputStream; 16 | import java.util.Properties; 17 | 18 | 19 | /** 20 | * Created by I311352 on 10/18/2016. 21 | */ 22 | @Configuration 23 | public class ElasticServiceConfig extends WebMvcConfigurerAdapter { 24 | public static final String LOCAL_CONFIG_PATH = "META-INF/resources/es.properties"; 25 | public static final String PRODUCTION_CONFIG_PATH = "/etc/secrets/es/es.properties"; 26 | private static final Logger logger = LoggerFactory.getLogger(ElasticServiceConfig.class); 27 | 28 | @Bean 29 | public ESConnectionConfig esConnectionConfig(){ 30 | Properties properties = loadProperties(LOCAL_CONFIG_PATH, PRODUCTION_CONFIG_PATH); 31 | 32 | ESConnectionConfig config = new ESConnectionConfig(); 33 | config.setEsDatabase(StringUtils.trimToNull(properties.getProperty("ES_DATABASE"))); 34 | config.setEsHost(StringUtils.trimToNull(properties.getProperty("ES_HOST"))); 35 | config.setEsPort(Integer.parseInt(StringUtils.trimToNull(properties.getProperty("ES_PORT")))); 36 | return config; 37 | } 38 | 39 | @Bean(name = {"ESService"}) 40 | @Primary 41 | public ESService elasticRestClient(ESConnectionConfig config) { 42 | 43 | String host = "http://" + config.getEsHost() + ":" + config.getEsPort(); 44 | ESService esService = new ESService(host); 45 | return esService; 46 | } 47 | 48 | public Properties loadProperties(String localPath, String productivePath) { 49 | Properties props = new Properties(); 50 | File file = new File(productivePath); 51 | Object inputStream = null; 52 | 53 | try { 54 | if(!file.exists()) { 55 | logger.error("Can't find the properties file {}, this will leads to using local properties file {}.", productivePath, localPath); 56 | } else { 57 | inputStream = new FileInputStream(file); 58 | } 59 | 60 | props.load((InputStream)inputStream); 61 | } catch (Exception e) { 62 | logger.warn("Load resource properties error.", e); 63 | } finally { 64 | if(null != inputStream) { 65 | try { 66 | ((InputStream)inputStream).close(); 67 | } catch (IOException e) { 68 | logger.warn("Resource input stream close error.", e); 69 | } 70 | } 71 | 72 | } 73 | 74 | return props; 75 | } 76 | } -------------------------------------------------------------------------------- /elasticservice/src/main/java/elasticsearch/searchservice/dsl/DSLGenerateService.java: -------------------------------------------------------------------------------- 1 | package elasticsearch.searchservice.dsl; 2 | 3 | import elasticsearch.searchservice.models.QueryParam; 4 | import org.springframework.stereotype.Service; 5 | 6 | /** 7 | * Created by i311352 on 5/3/2017. 8 | */ 9 | @Service 10 | public interface DSLGenerateService { 11 | String fromQueryParam(QueryParam param); 12 | } 13 | -------------------------------------------------------------------------------- /elasticservice/src/main/java/elasticsearch/searchservice/dsl/DSLMeta.java: -------------------------------------------------------------------------------- 1 | package elasticsearch.searchservice.dsl; 2 | 3 | import java.util.HashMap; 4 | import java.util.List; 5 | 6 | /** 7 | * Created by i311352 on 5/8/2017. 8 | */ 9 | public class DSLMeta { 10 | private String key; 11 | private List value; 12 | private KeyType keyType; 13 | private ValType valueType; 14 | private String operator; 15 | private String nestPath; 16 | private HashMap> sysAttributes; 17 | 18 | 19 | public DSLMeta() { 20 | } 21 | 22 | public DSLMeta(String key, List value, KeyType keyType, ValType valueType) { 23 | this.key = key; 24 | this.value = value; 25 | this.keyType = keyType; 26 | this.valueType = valueType; 27 | } 28 | 29 | public DSLMeta(HashMap> sysAttributes, KeyType keyType, ValType valueType) { 30 | this.keyType = keyType; 31 | this.valueType = valueType; 32 | this.sysAttributes = sysAttributes; 33 | } 34 | 35 | public enum KeyType { 36 | NORMAL, NESTED 37 | } 38 | 39 | public enum ValType { 40 | STRING, DATE, NUMERIC, IDLIST, WILDCARD, PREFIX 41 | } 42 | 43 | public enum OPERATOR { 44 | EQ, LE, GE 45 | } 46 | 47 | public String getKey() { 48 | return key; 49 | } 50 | 51 | public void setKey(String key) { 52 | this.key = key; 53 | } 54 | 55 | public KeyType getKeyType() { 56 | return keyType; 57 | } 58 | 59 | public void setKeyType(KeyType keyType) { 60 | this.keyType = keyType; 61 | } 62 | 63 | public List getValue() { 64 | return value; 65 | } 66 | 67 | public void setValue(List value) { 68 | this.value = value; 69 | } 70 | 71 | public ValType getValueType() { 72 | return valueType; 73 | } 74 | 75 | public void setValueType(ValType valueType) { 76 | this.valueType = valueType; 77 | } 78 | 79 | public String getNestPath() { 80 | return nestPath; 81 | } 82 | 83 | public void setNestPath(String nestPath) { 84 | this.nestPath = nestPath; 85 | } 86 | 87 | public String getOperator() { 88 | return operator; 89 | } 90 | 91 | public void setOperator(String operator) { 92 | this.operator = operator; 93 | } 94 | 95 | public HashMap> getSysAttributes() { 96 | return sysAttributes; 97 | } 98 | 99 | public void setSysAttributes(HashMap> sysAttributes) { 100 | this.sysAttributes = sysAttributes; 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /elasticservice/src/main/java/elasticsearch/searchservice/models/CacheSearchService.java: -------------------------------------------------------------------------------- 1 | package elasticsearch.searchservice.models; 2 | 3 | import org.springframework.stereotype.Service; 4 | 5 | /** 6 | * Created by i311352 on 5/13/2017. 7 | */ 8 | @Service 9 | public interface CacheSearchService { 10 | QueryResp query(String body); 11 | Integer count(String body); 12 | } 13 | -------------------------------------------------------------------------------- /elasticservice/src/main/java/elasticsearch/searchservice/models/Collection.java: -------------------------------------------------------------------------------- 1 | package elasticsearch.searchservice.models; 2 | 3 | import java.util.List; 4 | 5 | /** 6 | * Created by i311352 on 5/2/2017. 7 | */ 8 | 9 | public class Collection { 10 | private String conditionType; 11 | private List conditions; 12 | 13 | public Collection() {} 14 | public String getConditionType() { 15 | return conditionType; 16 | } 17 | 18 | public void setConditionType(String conditionType) { 19 | this.conditionType = conditionType; 20 | } 21 | 22 | public List getConditions() { 23 | return conditions; 24 | } 25 | 26 | public void setConditions(List conditions) { 27 | this.conditions = conditions; 28 | } 29 | } -------------------------------------------------------------------------------- /elasticservice/src/main/java/elasticsearch/searchservice/models/ESResp/ESCountResp.java: -------------------------------------------------------------------------------- 1 | package elasticsearch.searchservice.models.ESResp; 2 | 3 | /** 4 | * Created by i311352 on 5/10/2017. 5 | */ 6 | public class ESCountResp { 7 | private Integer count; 8 | 9 | public Integer getCount() { 10 | return count; 11 | } 12 | 13 | public void setCount(Integer count) { 14 | this.count = count; 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /elasticservice/src/main/java/elasticsearch/searchservice/models/ESResp/ESSearchResp.java: -------------------------------------------------------------------------------- 1 | package elasticsearch.searchservice.models.ESResp; 2 | 3 | /** 4 | * Created by i311352 on 5/4/2017. 5 | */ 6 | public class ESSearchResp { 7 | Hit hits; 8 | 9 | 10 | public Hit getHits() { 11 | return hits; 12 | } 13 | 14 | public void setHits(Hit hits) { 15 | this.hits = hits; 16 | } 17 | } -------------------------------------------------------------------------------- /elasticservice/src/main/java/elasticsearch/searchservice/models/ESResp/Hit.java: -------------------------------------------------------------------------------- 1 | package elasticsearch.searchservice.models.ESResp; 2 | 3 | import java.util.List; 4 | 5 | /** 6 | * Created by i311352 on 5/4/2017. 7 | */ 8 | public class Hit { 9 | Long total; 10 | List hits; 11 | 12 | public Long getTotal() { 13 | return total; 14 | } 15 | 16 | public void setTotal(Long total) { 17 | this.total = total; 18 | } 19 | 20 | public List getHits() { 21 | return hits; 22 | } 23 | 24 | public void setHits(List hits) { 25 | this.hits = hits; 26 | } 27 | } -------------------------------------------------------------------------------- /elasticservice/src/main/java/elasticsearch/searchservice/models/ESResp/HitResult.java: -------------------------------------------------------------------------------- 1 | package elasticsearch.searchservice.models.ESResp; 2 | 3 | /** 4 | * Created by i311352 on 5/4/2017. 5 | */ 6 | public class HitResult { 7 | HitSource _source; 8 | 9 | public HitSource get_source() { 10 | return _source; 11 | } 12 | 13 | public void set_source(HitSource _source) { 14 | this._source = _source; 15 | } 16 | } -------------------------------------------------------------------------------- /elasticservice/src/main/java/elasticsearch/searchservice/models/ESResp/HitSource.java: -------------------------------------------------------------------------------- 1 | package elasticsearch.searchservice.models.ESResp; 2 | 3 | /** 4 | * Created by i311352 on 5/4/2017. 5 | */ 6 | public class HitSource { 7 | Long id; 8 | 9 | public Long getId() { 10 | return id; 11 | } 12 | 13 | public void setId(Long id) { 14 | this.id = id; 15 | } 16 | } -------------------------------------------------------------------------------- /elasticservice/src/main/java/elasticsearch/searchservice/models/Meta.java: -------------------------------------------------------------------------------- 1 | package elasticsearch.searchservice.models; 2 | 3 | import java.util.List; 4 | 5 | /** 6 | * Created by i311352 on 5/2/2017. 7 | */ 8 | public class Meta { 9 | private String key; 10 | private List value; 11 | private String operator; 12 | 13 | public Meta() {} 14 | public Meta(String key, List value, String operator) { 15 | this.key = key; 16 | this.value = value; 17 | this.operator = operator; 18 | } 19 | 20 | public String getKey() { 21 | return key; 22 | } 23 | 24 | public void setKey(String key) { 25 | this.key = key; 26 | } 27 | 28 | public List getValue() { 29 | return value; 30 | } 31 | 32 | public void setValue(List value) { 33 | this.value = value; 34 | } 35 | 36 | public String getOperator() { 37 | return operator; 38 | } 39 | 40 | public void setOperator(String operator) { 41 | this.operator = operator; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /elasticservice/src/main/java/elasticsearch/searchservice/models/QueryParam.java: -------------------------------------------------------------------------------- 1 | package elasticsearch.searchservice.models; 2 | 3 | import java.util.List; 4 | 5 | /** 6 | * Created by i311352 on 5/2/2017. 7 | */ 8 | public class QueryParam { 9 | private List filter; 10 | private Collection collection; 11 | private String orderByField; 12 | private String orderByOrder; 13 | private Long channelId; 14 | private Integer pageSize; 15 | private Integer offSet; 16 | private Boolean isCount = Boolean.FALSE; 17 | 18 | public List getFilter() { 19 | return filter; 20 | } 21 | 22 | public void setFilter(List filter) { 23 | this.filter = filter; 24 | } 25 | 26 | public Collection getCollection() { 27 | return collection; 28 | } 29 | 30 | public void setCollection(Collection collection) { 31 | this.collection = collection; 32 | } 33 | 34 | public String getOrderByField() { 35 | return orderByField; 36 | } 37 | 38 | public void setOrderByField(String orderByField) { 39 | this.orderByField = orderByField; 40 | } 41 | 42 | public String getOrderByOrder() { 43 | return orderByOrder; 44 | } 45 | 46 | public void setOrderByOrder(String orderByOrder) { 47 | this.orderByOrder = orderByOrder; 48 | } 49 | 50 | public Integer getPageSize() { 51 | return pageSize; 52 | } 53 | 54 | public void setPageSize(Integer pageSize) { 55 | this.pageSize = pageSize; 56 | } 57 | 58 | public Integer getOffSet() { 59 | return offSet; 60 | } 61 | 62 | public void setOffSet(Integer offSet) { 63 | this.offSet = offSet; 64 | } 65 | 66 | public Long getChannelId() { 67 | return channelId; 68 | } 69 | 70 | public void setChannelId(Long channelId) { 71 | this.channelId = channelId; 72 | } 73 | 74 | public Boolean getCount() { 75 | return isCount; 76 | } 77 | 78 | public void setCount(Boolean count) { 79 | isCount = count; 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /elasticservice/src/main/java/elasticsearch/searchservice/models/QueryResp.java: -------------------------------------------------------------------------------- 1 | package elasticsearch.searchservice.models; 2 | 3 | 4 | import com.google.gson.JsonObject; 5 | import elasticsearch.esapi.resp.ESQueryResponse; 6 | import elasticsearch.esapi.resp.Hits; 7 | import elasticsearch.searchservice.models.ESResp.ESSearchResp; 8 | import elasticsearch.searchservice.models.ESResp.HitResult; 9 | 10 | import java.util.ArrayList; 11 | import java.util.List; 12 | 13 | /** 14 | * Created by i311352 on 5/4/2017. 15 | */ 16 | public class QueryResp { 17 | Long total; 18 | List productIds; 19 | 20 | public QueryResp() { 21 | } 22 | 23 | public QueryResp(Long total) { 24 | this.total = total; 25 | } 26 | 27 | public QueryResp(ESQueryResponse response) { 28 | total = response.getHit().getTotal(); 29 | productIds = new ArrayList<>(); 30 | for (Hits hitResult : response.getHit().getHits()) { 31 | JsonObject object = hitResult.getSource(); 32 | Long id = object.get("id").getAsLong(); 33 | productIds.add(id); 34 | } 35 | } 36 | 37 | public QueryResp(ESSearchResp resp) { 38 | total = resp.getHits().getTotal(); 39 | productIds = new ArrayList<>(); 40 | for (HitResult hitResult : resp.getHits().getHits()) { 41 | productIds.add(hitResult.get_source().getId()); 42 | } 43 | } 44 | 45 | public Long getTotal() { 46 | return total; 47 | } 48 | 49 | public void setTotal(Long total) { 50 | this.total = total; 51 | } 52 | 53 | public List getProductIds() { 54 | return productIds; 55 | } 56 | 57 | public void setProductIds(List productIds) { 58 | this.productIds = productIds; 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /elasticservice/src/main/resources/config.properties: -------------------------------------------------------------------------------- 1 | ES_HOST=127.0.0.1:9200 2 | SAVE_FILE=export.json 3 | TENANTID=35401674516640 4 | S3_BUKET=testBucket -------------------------------------------------------------------------------- /elasticservice/src/main/resources/elasticsearch/stores-product-mapping.json: -------------------------------------------------------------------------------- 1 | { 2 | "settings": { 3 | "number_of_shards": 15, 4 | "number_of_replicas": 1, 5 | "max_result_window" : 500000 6 | }, 7 | "aliases" : { 8 | "stores" : {} 9 | }, 10 | "mappings": { 11 | "product": { 12 | "dynamic": "true", 13 | "_routing": {"required": true}, 14 | "properties": { 15 | "id": { 16 | "type": "long" 17 | }, 18 | "name": { 19 | "type": "string" 20 | } 21 | } 22 | }, 23 | "sku": { 24 | "_parent": { 25 | "type": "product" 26 | }, 27 | "properties": { 28 | "id": { 29 | "type": "long" 30 | }, 31 | "parentId": { 32 | "type": "long" 33 | }} 34 | }, 35 | "productproperty": { 36 | "dynamic": "true", 37 | "_routing": {"required": true}, 38 | "properties": { 39 | "id": { 40 | "type": "long" 41 | }, 42 | "propertyType": { 43 | "type": "string", 44 | "index": "not_analyzed" 45 | } 46 | } 47 | } 48 | } 49 | } -------------------------------------------------------------------------------- /elasticservice/src/main/resources/elasticsearch/storessearch-mapping.json: -------------------------------------------------------------------------------- 1 | { 2 | "settings": { 3 | "number_of_shards": 15, 4 | "number_of_replicas": 0, 5 | "analysis": { 6 | "analyzer": { 7 | "suggest_name_synonyms": { 8 | "type": "custom", 9 | "tokenizer": "standard", 10 | "filter": [ "name_synonyms","lowercase","myNGramFilter" ] 11 | }, 12 | "suggest_category_synonyms": { 13 | "type": "custom", 14 | "tokenizer": "standard", 15 | "filter": [ "category_synonyms","lowercase", "myNGramFilter" ] 16 | } 17 | }, 18 | "filter": { 19 | "name_synonyms": { 20 | "type": "synonym", 21 | "synonyms": [""] 22 | }, 23 | "category_synonyms": { 24 | "type": "synonym", 25 | "synonyms": [""] 26 | }, 27 | "myNGramFilter": { 28 | "type": "edgeNGram", 29 | "min_gram": 1, 30 | "max_gram": 40 31 | } 32 | } 33 | } 34 | }, 35 | "aliases" : { 36 | "storessearch" : {} 37 | }, 38 | "mappings": { 39 | "product": { 40 | "dynamic": "true", 41 | "numeric_detection": "false", 42 | "_routing": {"required": true}, 43 | "properties": { 44 | "name_suggest" : { 45 | "type" : "completion", 46 | "analyzer" : "suggest_name_synonyms", 47 | "search_analyzer": "standard" 48 | }, 49 | "category_suggest" : { 50 | "type" : "completion", 51 | "analyzer" : "suggest_category_synonyms", 52 | "search_analyzer": "standard" 53 | }, 54 | "systemField": { 55 | "properties": { 56 | "name":{ 57 | "type":"keyword", 58 | "copy_to": ["name_suggest"] 59 | } 60 | } 61 | }, 62 | "customField": { 63 | "properties": { 64 | } 65 | }, 66 | "sku": { 67 | "type": "nested", 68 | "properties": { 69 | "id": { 70 | "type": "long" 71 | }, 72 | "updateTime": { 73 | "type": "date", 74 | "format": "strict_date_optional_time||epoch_millis" 75 | }, 76 | "creationTime":{ 77 | "type": "date", 78 | "format": "strict_date_optional_time||epoch_millis" 79 | } 80 | } 81 | }, 82 | "attributes": { 83 | "type": "nested", 84 | "properties": { 85 | "attribute1": { 86 | "type": "keyword", 87 | "index": "not_analyzed" 88 | }, 89 | "attribute2": { 90 | "type": "keyword", 91 | "index": "not_analyzed" 92 | }, 93 | "creationTime": { 94 | "type": "date", 95 | "format": "strict_date_optional_time||epoch_millis" 96 | }, 97 | "id": { 98 | "type": "long" 99 | }, 100 | "updateTime": { 101 | "type": "date", 102 | "format": "strict_date_optional_time||epoch_millis" 103 | } 104 | } 105 | }, 106 | "channels": { 107 | "type": "nested", 108 | "properties": { 109 | "id":{ 110 | "type": "long" 111 | } 112 | } 113 | }, 114 | "collection":{ 115 | "type": "long" 116 | }, 117 | "description":{ 118 | "type": "text" 119 | }, 120 | "category": { 121 | "properties": { 122 | "chain": { 123 | "type": "keyword", 124 | "copy_to":["name_suggest","category_suggest"] 125 | }, 126 | "order": { 127 | "type": "keyword" 128 | } 129 | } 130 | }, 131 | "updateTime": { 132 | "type": "date", 133 | "format": "strict_date_optional_time||epoch_millis" 134 | }, 135 | "creationTime":{ 136 | "type": "date", 137 | "format": "strict_date_optional_time||epoch_millis" 138 | } 139 | } 140 | } 141 | } 142 | } -------------------------------------------------------------------------------- /elasticservice/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=debug, console 2 | 3 | log4j.appender.console=org.apache.log4j.ConsoleAppender 4 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 5 | log4j.appender.console.layout.conversionPattern=[%d{ISO8601}][%-35p][%-35c{1}] %marker%.10000m%n 6 | -------------------------------------------------------------------------------- /elasticservice/src/main/resources/templates/base.dsl.twig: -------------------------------------------------------------------------------- 1 | {% macro term(meta) %} 2 | {"term":{ 3 | "{{meta.key}}":{% if meta.valueType == "STRING" or meta.valueType == "DATE" %} "{{meta.value}}" {% else %} {{meta.value}} {% endif %} 4 | }} 5 | {% endmacro %} 6 | 7 | {% macro range_query(meta) %} 8 | {"range":{ 9 | "{{ meta.key }}": { 10 | "{{ meta.operator }}" : {% if meta.valueType == "STRING" or meta.valueType == "DATE" %} "{{meta.value|first}}" {% else %} {{meta.value|first}} {% endif %} 11 | } 12 | }} 13 | {% endmacro %} 14 | 15 | {% macro terms_query(meta, isStr="false") %} 16 | {"terms":{ 17 | "{{meta.key}}": [ 18 | {% for value in meta.value %} 19 | {% if meta.valueType == "STRING" or meta.valueType == "DATE" or isStr == "true" %} "{{value}}" {% else %} {{value}} {% endif %} {% if loop.index != loop.length-1 %}, {% endif %} 20 | {% endfor %} 21 | ] 22 | }} 23 | {% endmacro %} 24 | 25 | {% macro wildcard_query(meta) %} 26 | {"wildcard":{ 27 | "{{meta.key}}": { 28 | "value": "{{meta.value|first}}" 29 | } 30 | }} 31 | {% endmacro %} 32 | 33 | {% macro prefix_query(meta) %} 34 | {"prefix":{ 35 | "{{meta.key}}": { 36 | "value": "{{meta.value|first}}" 37 | } 38 | }} 39 | {% endmacro %} 40 | 41 | {% macro channel(channelId) %} 42 | "must": [{"nested": { 43 | "path": "channels", 44 | "query": { 45 | "bool": { 46 | "must": [ 47 | { 48 | "match": { 49 | "channels.id": {{ channelId }} 50 | } 51 | } 52 | ] 53 | } 54 | } 55 | } 56 | }] 57 | {% endmacro %} 58 | 59 | {% macro nest_range(meta)%} 60 | { 61 | "nested": { 62 | "path": "{{ meta.nestPath }}", 63 | "query": { 64 | "range": { 65 | "{{ meta.key }}": { 66 | "{{ meta.operator }}": {% if meta.valueType == "STRING" or meta.valueType == "DATE" %} "{{meta.value|first}}" {% else %} {{meta.value|first}} {% endif %} 67 | } 68 | } 69 | } 70 | } 71 | } 72 | {% endmacro %} 73 | 74 | {% macro nest_query(meta) %} 75 | { 76 | "nested": { 77 | "path": "{{ meta.nestPath }}", 78 | "query": { 79 | "bool": { 80 | "must": [ 81 | {{ terms_query(meta) }} 82 | ] 83 | } 84 | } 85 | } 86 | } 87 | {% endmacro %} 88 | 89 | {% macro nest_sys_attribute_query(meta) %} 90 | { 91 | "nested": { 92 | "path": "{{ meta.nestPath }}", 93 | "query": { 94 | "bool": { 95 | "must": [ 96 | {% for entry in meta.sysAttributes %} 97 | {{ terms_query(entry, "true") }} {% if loop.index != loop.length-1 %}, {% endif %} 98 | {% endfor %} 99 | ] 100 | } 101 | } 102 | } 103 | } 104 | {% endmacro %} 105 | 106 | {% macro meta_query(meta)%} 107 | {% if meta.keyType == "NORMAL" %} 108 | {% if meta.valueType == "STRING" %} 109 | {{ terms_query(meta) }} 110 | {% elseif meta.valueType == "WILDCARD" %} 111 | {{ wildcard_query(meta) }} 112 | {% elseif meta.valueType == "PREFIX" %} 113 | {{ prefix_query(meta) }} 114 | {% else %} 115 | {{ range_query(meta) }} 116 | {% endif %} 117 | {% else %} 118 | {% if meta.valueType == "STRING" or meta.valueType == "IDLIST" %} 119 | {% if meta.nestPath == "attributes" %} 120 | {{ nest_sys_attribute_query(meta) }} 121 | {% else %} 122 | {{ nest_query(meta) }} 123 | {% endif %} 124 | {% else %} 125 | {{ nest_range(meta) }} 126 | {% endif %} 127 | {% endif %} 128 | {% endmacro %} 129 | 130 | {% macro query_base(filter, collectionForOr, channelId) %} 131 | {% if filter | length > 0 %} 132 | "bool" : { 133 | "must" : [ 134 | {% for term in filter %} 135 | {{ meta_query(term) }} {% if loop.index != loop.length-1 %}, {% endif %} 136 | {% endfor %} 137 | ] 138 | {% if collectionForOr %} 139 | ,"should": [ 140 | {% for term in collectionForOr %} 141 | {{ meta_query(term) }} {% if loop.index != loop.length-1 %}, {% endif %} 142 | {% endfor %} 143 | ] 144 | {% endif %} 145 | ,{{ channel(channelId) }} 146 | } 147 | {% elseif collectionForOr %} 148 | "bool" : { 149 | "should": [ 150 | {% for term in collectionForOr %} 151 | {{ meta_query(term) }} {% if loop.index != loop.length-1 %}, {% endif %} 152 | {% endfor %} 153 | ] 154 | ,{{ channel(channelId) }} 155 | } 156 | {% else %} 157 | "bool":{ 158 | {{ channel(channelId) }} 159 | } 160 | {% endif %} 161 | {% endmacro %} 162 | -------------------------------------------------------------------------------- /elasticservice/src/main/resources/templates/count.dsl.twig: -------------------------------------------------------------------------------- 1 | {% import "templates/base.dsl.twig" %} 2 | 3 | { 4 | "query":{ 5 | {{ query_base(filter, collectionForOr, channelId) }} 6 | } 7 | } -------------------------------------------------------------------------------- /elasticservice/src/main/resources/templates/query.dsl.twig: -------------------------------------------------------------------------------- 1 | {% import "templates/base.dsl.twig" %} 2 | 3 | { 4 | "query":{ 5 | {{ query_base(filter, collectionForOr, channelId) }} 6 | }, 7 | "sort":{ 8 | "{{ orderByField }}":{ 9 | "order" : "{{orderByOrder}}" 10 | {% if orderByField contains "price" %} 11 | ,"nested_path": "channels" 12 | {% endif %} 13 | }}, 14 | "from": {{ from }}, 15 | "size": {{ size }} 16 | } -------------------------------------------------------------------------------- /elasticservice/src/main/resources/templates/search.dsl.twig: -------------------------------------------------------------------------------- 1 | {% import "templates/base.dsl.twig" %} 2 | {"query": { 3 | "bool": { 4 | "must": [ 5 | {"query_string": { 6 | "query": "*{{ search_string }}*", 7 | "default_operator": "or" 8 | } 9 | } 10 | ], 11 | {{ channel(channelId) }} 12 | }}, 13 | "from": {{ from }}, 14 | "size": {{ size }} 15 | } -------------------------------------------------------------------------------- /flow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/compasses/elastic-rabbitmq/3c70d3a7ea310e0411b828452c79787737bed99f/flow.png -------------------------------------------------------------------------------- /flow.xml: -------------------------------------------------------------------------------- 1 | 7Vxbc5s4FP41fkxGoAvwmDhJ29npTrbZmd0+YpBtpthkAefSX78SSBhdSLCBOmnrPASEEELfdz6dcyR7Buebpw95eL/+nMU0nbkgfprBq5nrOsgl7B8vea5LAt+vC1Z5EtdFYF9wl3yn4k5ZuktiWoiyuqjMsrRM7tXCKNtuaVQqZWGeZ49qtWWWxkrBfbiiRsFdFKZm6T9JXK7rUl++Fi//SJPVWj7ZIUF9ZRFG31Z5ttuK581cuKw+9eVNKNsSL1qswzh7bBXB6xmc51lW1kebpzlN+diqw3bTcbXpd063ZZ8b3PqGhzDdUdljkrJbL+PkgfevfBZjQv7b8U5dlvSpPAvTZLWdwQtWI6XLcn+VHa34/+s0LMokuqNhHq1lk6wbVat1FTEAzQPcx3VS0rv7MOLnj4xYrNK63KTszGGHRZln3+g8S7O8qg9J5NPFsrkigeJ1l0matmrGIfWXES/PtqVgm0PEeaseYJ/5nJWL97uqXg5ePtCcvUyYXojiMuN9i8NiTWPxRHPgBRb8XvrUKhJAfKDZhpb5M6sirzpefYswGggFOo97CmJpSOsW/VxPUl/QftW0vYeeHQj07UyABhOSbcw6rqPEGHvPD6PnlFdgI3f5A4Bb5WGcUAUsj4aEgiNAjVg7Vb9HwAwiV8EMA2hg5gDPxMwPhkOGuiADcRbtNuytCgO+Spsa1h6KnDMHF+DqVeS22ZZaQBPFp8SLGZVqY15g4BU4Jlx4BLiwAVehyGNLGwWOvWTzF0cU+a6GqG9aIAQmpMQdDikxIP0SLhZJ+fmvTt2MwzK8K7O88kAOl87q00c6l5j/DQHHormL6m8kU3SIApz0/Vq4Qc+CmywbgptncXs0vOg2vuCeJB8R5swUSfSaJQUAQHjAgLOhy5//ZYXg3MHy/Gt1DprzW5on7PU4IFfgpYEvsl0eUWViKMN8RUtleqex4vh2z2vAPYcgIAgRDwAMfFU2ETSNDFuwwi9gJZ59myXsTToFGntYbaJ+T3FX27PVG9K6jF2toXp0jIYq6jQD0otNvsGmeU7DkrKyT1YfivvQKptyWiTfw0VVgcN8z7tVdRRfzvCVzTx1p3STxDG//zINFzS9bAKRtibUoUh/ippsk5ajm3kTi4mXmLXjGRvDGMmRjJoERMIIc5qGZfKghmL9GaTdkC2XBR2KcDCBXix8jPAh/it9SspaLlwsTr92aL8f0ajDbWZe86KaQXopCR6sJAgG547ngoBJCnQxViFHyFT9cYQEIfZc32N3BgEB2lyDIDlOVbCcpaSqyBhtfFWRT2qRTobTP5+sSAP7tWTFMYNukTg56/Qj32CWZEx33pGSI73CwJYEQaZCkBHcQseMqIfr/OF+4V7nPVcTen58lEsoqdZWcnSYkrsyspJ2hS3QjKLd6nPOjtVqrRmstTOiVJuhPQOpSAremysW8FlC/C+0unpx+6lnnP+O1b2xq8Hyfsb03QMaD4fp+7iKbmYEXlIQkT+ZSD4w8tvywUauS0JYl24S/lLVWcEsozRFrioW1V5Mebd1h5iy4xzoQTaZGoG3P5XL+O5kx0xjzLNtsdsw7fmp5EMa1RjygQI8TC+kYzKFQ2hJKA53QA4PNNuJKU/LTJHO1NReeAjWA1Sb6EwWtNp8HXyY5iBfXYubztdBGClPgvDIhBcGakOuP1nCy5I//TOLqYSixdU0Te4LLhBN+jvbMVu9bspfCWd66IrBd87jasZUIx7XpNxN9em3ruHaDaZ6UEcgZNErg7mdDCSuGsV6BBsM9IjJQBn8DomDLBlNjq9Fi37jOxa+PnF+HL5mPpPjC3/jOx2+ATZ3BkyFL3QMfP/YLeg83RX8/bpRngDNjvyTBU1b/slFPxg2fROOD4gBmyOXnNu4OWOsS0IzkTDOfizZyiLXS06zKevIjQP992PlWcm8+4xfDcA41MDS5ZczsiU1CR1LarLhyyBqmKHB9R077zLqo+K9XZkVAo6+o14wbiTb1d/85OoMHWC+Y4gsULPFnhTdtrVK31/ZMjeGyppe0kdugbd59vR8qCFNOFEO3rtjF9sJLAyi130iB9smzRG0Fx2b/O9Il72kknxUK8fFopI98WgF6a6ye2SfmtOCXWvcrKzsYjs6PWPegQtsCKtrtD7UbLR3IIy0hpDWl/ECYWSbr98FZ/giqZLYCTzUldh5n2xSZwY/8I9jkz7F+LrWjMimY7N/J2eTj12NTb77c7FJza0Fctv3UDYFMtyYgE2me/JO2GRMZzKzLI+/NtX6k6mdIIY2gvmnJJhjbIYNNGL0ZZhD9JbQZBTD5hYlg2L77enc+KNsw8nUYlHba23RcZFm0be/18lWZR5nz024SVL+dvOqMRfchduC/ft8J+kmnu6otAIYqiIFSadINWsZIzFOZszbjHNPyjiM1GjW1xNPvd2tQGvI0RoakW9mXuvN8s1RuYbbMnYOXKIQC8l9PsdPlgrbfAvbvFOyzYPqvkk/0JroyzbP1xqazrmXS35vmW3NkiuAgcIo4sIOSu0V0WToiHQLTipumvfvALkmPNhhI1pDI/LN3A36Vvl2gILtBVFydE85Av1xo4TTTqkeUNcOmm+9HyxyekNgOtL1yHq9EdKpvpjXfAmiW+AkS1sun+cNZ5yifcFbm2qJtg/EAeDIuZa4akte0C/1wQgQPreqiYWGbrUOtDW3QPnRAnZQt3g0x3tk6d4Ix8eKcE/s7Tk6cY7c0+QBdUGAcRlNQkFPfo+nUVwyhILsdP8bHnX1/Q+lwOv/AQ== -------------------------------------------------------------------------------- /generatedata/.gitignore: -------------------------------------------------------------------------------- 1 | .gradle 2 | gradle 3 | gradlew 4 | build/ 5 | 6 | # Ignore Gradle GUI config 7 | gradle-app.setting 8 | 9 | # Avoid ignoring Gradle wrapper jar file (.jar files are usually ignored) 10 | !gradle-wrapper.jar 11 | 12 | # Cache of project 13 | .gradletasknamecache 14 | 15 | # # Work around https://youtrack.jetbrains.com/issue/IDEA-116898 16 | # gradle/wrapper/gradle-wrapper.properties 17 | -------------------------------------------------------------------------------- /generatedata/build.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * This build file was auto generated by running the Gradle 'init' task 3 | * by 'I311352' at '11/23/16 1:11 PM' with Gradle 2.14.1 4 | * 5 | * This generated file contains a sample Java project to get you started. 6 | * For more details take a look at the Java Quickstart chapter in the Gradle 7 | * user guide available at https://docs.gradle.org/2.14.1/userguide/tutorial_java_projects.html 8 | */ 9 | buildscript { 10 | repositories { 11 | mavenCentral() 12 | } 13 | dependencies { 14 | classpath("org.springframework.boot:spring-boot-gradle-plugin:1.4.2.RELEASE") 15 | } 16 | } 17 | 18 | // Apply the java plugin to add support for Java 19 | apply plugin: 'application' 20 | apply plugin: 'maven' 21 | apply plugin: 'org.springframework.boot' 22 | 23 | mainClassName = 'initdata.RabbitMQApp' 24 | 25 | sourceCompatibility = 1.8 26 | targetCompatibility = 1.8 27 | //jar { 28 | // baseName = 'elastic-rabbit-job' 29 | // version = '1.0.0' 30 | // manifest { 31 | // attributes 'Main-Class': 'initdata.RabbitMQApp' 32 | // } 33 | //} 34 | 35 | 36 | // In this section you declare where to find the dependencies of your project 37 | repositories { 38 | // Use 'jcenter' for resolving your dependencies. 39 | // You can declare any Maven/Ivy/file repository here. 40 | mavenCentral() 41 | mavenLocal() 42 | maven { 43 | url 'http://nexus.smec/nexus/content/groups/allrepository' 44 | } 45 | } 46 | 47 | // In this section you declare the dependencies for your production and test code 48 | dependencies { 49 | // The production code uses the SLF4J logging API at compile time 50 | compile("org.springframework.boot:spring-boot-starter-web") 51 | compile 'org.slf4j:slf4j-api:1.7.21' 52 | compile 'joda-time:joda-time:2.2' 53 | compile 'com.google.code.gson:gson:2.7' 54 | compile 'com.rabbitmq:amqp-client:3.6.5' 55 | compile 'commons-io:commons-io:2.4' 56 | compile group: 'org.mariadb.jdbc', name: 'mariadb-java-client', version: '1.5.5' 57 | 58 | // Declare the dependency for your favourite test framework you want to use in your tests. 59 | // TestNG is also supported by the Gradle Test task. Just change the 60 | // testCompile dependency to testCompile 'org.testng:testng:6.8.1' and add 61 | // 'test.useTestNG()' to your build script. 62 | testCompile 'junit:junit:4.12' 63 | testCompile('org.springframework.boot:spring-boot-starter-test') 64 | } -------------------------------------------------------------------------------- /generatedata/settings.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * This settings file was auto generated by the Gradle buildInit task 3 | * by 'I311352' at '12/7/16 5:18 PM' with Gradle 3.2.1 4 | * 5 | * The settings file is used to specify which projects to include in your build. 6 | * In a single project build this file can be empty or even removed. 7 | * 8 | * Detailed information about configuring a multi-project build in Gradle can be found 9 | * in the user guide at https://docs.gradle.org/3.2.1/userguide/multi_project_builds.html 10 | */ 11 | 12 | /* 13 | // To declare projects as part of a multi-project build use the 'include' method 14 | include 'shared' 15 | include 'api' 16 | include 'services:webservice' 17 | */ 18 | 19 | rootProject.name = 'generatedata' 20 | -------------------------------------------------------------------------------- /generatedata/src/main/java/initdata/RabbitMQApp.java: -------------------------------------------------------------------------------- 1 | package initdata; 2 | 3 | /** 4 | * Created by I311352 on 11/23/2016. 5 | */ 6 | import initdata.publish.MessagePublishService; 7 | import initdata.sqlexecute.SQLScriptExecuteService; 8 | import org.apache.log4j.Logger; 9 | import org.springframework.boot.CommandLineRunner; 10 | import org.springframework.boot.SpringApplication; 11 | import org.springframework.boot.autoconfigure.SpringBootApplication; 12 | import org.springframework.context.ApplicationContext; 13 | import org.springframework.context.annotation.Bean; 14 | 15 | @SpringBootApplication 16 | public class RabbitMQApp { 17 | private static final Logger logger = Logger.getLogger(RabbitMQApp.class); 18 | 19 | public static void main(String[] args) { 20 | SpringApplication.run(RabbitMQApp.class, args); 21 | } 22 | 23 | private void publishMessageTask(ApplicationContext ctx) { 24 | logger.info("Start publish message..."); 25 | MessagePublishService publishService = ctx.getBean(MessagePublishService.class); 26 | if (System.getProperty("rabbit.cfg") != null) { 27 | logger.info("Use user config rabbit: " + System.getProperty("rabbit.cfg")); 28 | publishService.setRabbitProperties(System.getProperty("rabbit.cfg")); 29 | } 30 | if (System.getProperty("message.src") != null) { 31 | logger.info("Use user message source: " + System.getProperty("message.src")); 32 | publishService.setMessageSource(System.getProperty("message.src")); 33 | } 34 | publishService.publish(); 35 | } 36 | 37 | @Bean 38 | public CommandLineRunner commandLineRunner(ApplicationContext ctx) { 39 | return args -> { 40 | publishMessageTask(ctx); 41 | // SQLScriptExecuteService sqlScriptExecuteService = ctx.getBean(SQLScriptExecuteService.class); 42 | // if (System.getProperty("sql.source") != null) { 43 | // sqlScriptExecuteService.setDefaultSqlSource(System.getProperty("sql.source")); 44 | // } 45 | // 46 | // //sqlScriptExecuteService.runSQL(); 47 | // sqlScriptExecuteService.sendMsgToRabbit(); 48 | System.exit(0); 49 | }; 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /generatedata/src/main/java/initdata/publish/model/Messages.java: -------------------------------------------------------------------------------- 1 | package initdata.publish.model; 2 | 3 | import com.google.gson.annotations.SerializedName; 4 | 5 | import java.util.List; 6 | import java.util.Map; 7 | 8 | /** 9 | * Created by I311352 on 11/24/2016. 10 | */ 11 | public class Messages { 12 | private Map messageHeader; 13 | private List messages; 14 | public Map getMessageHeader() { 15 | return messageHeader; 16 | } 17 | 18 | public void setMessageHeader(Map messageHeader) { 19 | this.messageHeader = messageHeader; 20 | } 21 | 22 | public List getMessages() { 23 | return messages; 24 | } 25 | 26 | public void setMessages(List messages) { 27 | this.messages = messages; 28 | } 29 | 30 | class MessageHeader { 31 | @SerializedName("X-User-ID") 32 | private Long userId; 33 | @SerializedName("X-Employee-ID") 34 | private Long employeeId; 35 | @SerializedName("X-Tenant-ID") 36 | private Long tenantId; 37 | 38 | public Long getUserId() { 39 | return userId; 40 | } 41 | 42 | public void setUserId(Long userId) { 43 | this.userId = userId; 44 | } 45 | 46 | public Long getEmployeeId() { 47 | return employeeId; 48 | } 49 | 50 | public void setEmployeeId(Long employeeId) { 51 | this.employeeId = employeeId; 52 | } 53 | 54 | public Long getTenantId() { 55 | return tenantId; 56 | } 57 | 58 | public void setTenantId(Long tenantId) { 59 | this.tenantId = tenantId; 60 | } 61 | 62 | @Override 63 | public String toString() { 64 | return "MessageHeader{" + 65 | "userId=" + userId + 66 | ", employeeId=" + employeeId + 67 | ", tenantId=" + tenantId + 68 | '}'; 69 | } 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /generatedata/src/main/java/initdata/publish/model/MessagesDef.java: -------------------------------------------------------------------------------- 1 | package initdata.publish.model; 2 | 3 | /** 4 | * Created by I311352 on 11/24/2016. 5 | */ 6 | public class MessagesDef { 7 | public String body; 8 | public String routingKey; 9 | 10 | public String getBody() { 11 | return body; 12 | 13 | } 14 | public void setBody(String body) { 15 | this.body = body; 16 | } 17 | 18 | public String getRoutingKey() { 19 | return routingKey; 20 | } 21 | 22 | public void setRoutingKey(String routingKey) { 23 | this.routingKey = routingKey; 24 | } 25 | } -------------------------------------------------------------------------------- /generatedata/src/main/java/initdata/sqlexecute/SQLScriptExecuteService.java: -------------------------------------------------------------------------------- 1 | package initdata.sqlexecute; 2 | 3 | import initdata.sqlexecute.SQLExecute.MariaDBSQLExecute; 4 | import org.apache.log4j.Logger; 5 | import org.springframework.beans.factory.annotation.Autowired; 6 | import org.springframework.stereotype.Service; 7 | 8 | import java.io.BufferedReader; 9 | import java.io.FileReader; 10 | import java.io.IOException; 11 | import java.util.ArrayList; 12 | import java.util.List; 13 | import java.util.concurrent.TimeoutException; 14 | 15 | /** 16 | * Created by I311352 on 12/1/2016. 17 | */ 18 | @Service 19 | public class SQLScriptExecuteService { 20 | private static final Logger logger = Logger.getLogger(SQLScriptExecuteService.class); 21 | private static String DEFAULT_SQL_SOURCE = "/sql/data.sql"; 22 | private static boolean USER_CONFIG = false; 23 | 24 | private static final String DEFAULT_DELIMITER = ";"; 25 | private static final String COMMENTS = "--"; 26 | 27 | 28 | private List sqlLine = new ArrayList<>(); 29 | 30 | @Autowired 31 | private MariaDBSQLExecute mariaDBSQLExecute; 32 | 33 | public void runSQL() { 34 | if (!loadSQLScriptLine()) { 35 | logger.error("Load file fail...do nothing"); 36 | return; 37 | } 38 | 39 | mariaDBSQLExecute.executeSQL(sqlLine); 40 | } 41 | 42 | public void sendMsgToRabbit() throws TimeoutException, IOException { 43 | mariaDBSQLExecute.publish(); 44 | } 45 | 46 | private boolean loadSQLScriptLine() { 47 | BufferedReader bufferedReader = null; 48 | try { 49 | if (USER_CONFIG == true) { 50 | bufferedReader = new BufferedReader(new FileReader(DEFAULT_SQL_SOURCE)); 51 | } else { 52 | bufferedReader = new BufferedReader(new FileReader(this.getClass().getResource(DEFAULT_SQL_SOURCE).getFile())); 53 | } 54 | 55 | String line; 56 | StringBuffer stringBuffer = null; 57 | while ((line = bufferedReader.readLine()) != null) { 58 | logger.debug("Got sql ..." + line); 59 | String trimLine = line.trim(); 60 | if (trimLine.isEmpty() || trimLine.startsWith(COMMENTS)) { 61 | continue; 62 | } 63 | 64 | if (stringBuffer == null) { 65 | stringBuffer = new StringBuffer(64); 66 | } 67 | 68 | if (trimLine.endsWith(DEFAULT_DELIMITER)) { 69 | stringBuffer.append(" ").append(trimLine); 70 | sqlLine.add(stringBuffer.toString()); 71 | stringBuffer = null; 72 | } else { 73 | stringBuffer.append(" ").append(trimLine); 74 | } 75 | } 76 | return true; 77 | } catch (IOException e) { 78 | logger.error("Error reading sql file..."); 79 | } finally { 80 | try { 81 | if (bufferedReader != null) { 82 | bufferedReader.close(); 83 | } 84 | } catch (IOException e) { 85 | logger.error("close error " + e); 86 | } 87 | } 88 | 89 | return false; 90 | } 91 | 92 | public static void setDefaultSqlSource(String defaultSqlSource) { 93 | DEFAULT_SQL_SOURCE = defaultSqlSource; 94 | USER_CONFIG = true; 95 | } 96 | 97 | } 98 | -------------------------------------------------------------------------------- /generatedata/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | server.port=20361 2 | 3 | -------------------------------------------------------------------------------- /generatedata/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=debug, console 2 | 3 | log4j.appender.console=org.apache.log4j.ConsoleAppender 4 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 5 | log4j.appender.console.layout.conversionPattern=%5p [%t] (%F:%L) - %m%n 6 | -------------------------------------------------------------------------------- /generatedata/src/main/resources/messages/source.json: -------------------------------------------------------------------------------- 1 | { 2 | "messages":[ 3 | { 4 | "body":"[ { \"propertyList\" : [ ], \"channelId\" : 230429851385888, \"type\" : \"LightProduct\", \"action\" : \"SYNC\", \"sourceId\" : 230440885796896, \"time\" : 1476068186936, \"tenantId\" : 57606670888640 } ]", 5 | "routingKey":"LightProduct.SYNC" 6 | } 7 | ], 8 | "messageHeader":{ 9 | "X-Tenant-ID": 57606670888640, 10 | "X-User-ID": -1, 11 | "X-Employee-ID": -1 12 | } 13 | } -------------------------------------------------------------------------------- /generatedata/src/main/resources/rabbit/rabbitmq.properties: -------------------------------------------------------------------------------- 1 | # only for local env 2 | MQ_HOST=10.128.165.206 3 | MQ_PORT=5672 4 | MQ_DB_USR=test 5 | MQ_DB_PWD=test -------------------------------------------------------------------------------- /generatedata/src/main/resources/sql/data.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/compasses/elastic-rabbitmq/3c70d3a7ea310e0411b828452c79787737bed99f/generatedata/src/main/resources/sql/data.sql -------------------------------------------------------------------------------- /generatedata/src/main/resources/sql/db.properties: -------------------------------------------------------------------------------- 1 | MYSQL_HOST=10.128.163.99 2 | MYSQL_PORT=3306 3 | MYSQL_SCHEMA=channeladapter 4 | MYSQL_DB_USR=root 5 | MYSQL_DB_PWD=Initial0 -------------------------------------------------------------------------------- /gradle.properties: -------------------------------------------------------------------------------- 1 | #spring boot 2 | theSpringBootVersion=1.3.3.RELEASE 3 | theSpringCloudVersion=Brixton.RC1 4 | theSpringPlatformBomVersion=2.0.3.RELEASE 5 | theDependencyManagementPlugin=0.6.1.RELEASE 6 | -------------------------------------------------------------------------------- /netty-http/build.gradle: -------------------------------------------------------------------------------- 1 | 2 | apply plugin: 'java' 3 | 4 | sourceCompatibility = 1.8 5 | 6 | repositories { 7 | mavenCentral() 8 | } 9 | 10 | dependencies { 11 | testCompile group: 'junit', name: 'junit', version: '4.11' 12 | compile 'io.netty:netty-all:4.1.5.Final' 13 | compile 'com.netflix.hystrix:hystrix-core:1.5.9' 14 | compile 'com.netflix.hystrix:hystrix-metrics-event-stream:1.5.9' 15 | compile 'org.ehcache:ehcache:3.3.1' 16 | } 17 | -------------------------------------------------------------------------------- /netty-http/settings.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * This settings file was auto generated by the Gradle buildInit task 3 | * by 'I311352' at '12/7/16 5:18 PM' with Gradle 3.2.1 4 | * 5 | * The settings file is used to specify which projects to include in your build. 6 | * In a single project build this file can be empty or even removed. 7 | * 8 | * Detailed information about configuring a multi-project build in Gradle can be found 9 | * in the user guide at https://docs.gradle.org/3.2.1/userguide/multi_project_builds.html 10 | */ 11 | 12 | /* 13 | // To declare projects as part of a multi-project build use the 'include' method 14 | include 'shared' 15 | include 'api' 16 | include 'services:webservice' 17 | */ 18 | 19 | rootProject.name = 'netty-http' 20 | -------------------------------------------------------------------------------- /netty-http/src/main/java/HttpServerBoot.java: -------------------------------------------------------------------------------- 1 | 2 | import http.Config; 3 | import http.DefaultServerInitializer; 4 | import io.netty.bootstrap.ServerBootstrap; 5 | import io.netty.channel.Channel; 6 | import io.netty.channel.ChannelOption; 7 | import io.netty.channel.EventLoopGroup; 8 | import io.netty.channel.epoll.Epoll; 9 | import io.netty.channel.epoll.EpollEventLoopGroup; 10 | import io.netty.channel.epoll.EpollServerSocketChannel; 11 | import io.netty.channel.nio.NioEventLoopGroup; 12 | import io.netty.channel.socket.nio.NioServerSocketChannel; 13 | import org.springframework.context.ConfigurableApplicationContext; 14 | 15 | /** 16 | * Created by i311352 on 2/16/2017. 17 | */ 18 | public class HttpServerBoot { 19 | ConfigurableApplicationContext context; 20 | private Config conf = new Config(); 21 | 22 | 23 | public HttpServerBoot(ConfigurableApplicationContext configurableApplicationContext) { 24 | this.context = configurableApplicationContext; 25 | } 26 | 27 | protected static boolean isEpollAvailable = false; 28 | 29 | static { 30 | isEpollAvailable = Epoll.isAvailable(); 31 | } 32 | 33 | public void run() throws Exception { 34 | ServerBootstrap b = new ServerBootstrap(); 35 | try { 36 | if (isEpollAvailable) { 37 | b.group(new EpollEventLoopGroup(this.conf.getEventLoopThreadCount())) 38 | .channel(EpollServerSocketChannel.class); 39 | } else { 40 | b.group(new NioEventLoopGroup(this.conf.getEventLoopThreadCount())) 41 | .channel(NioServerSocketChannel.class); 42 | } 43 | b.childHandler(new DefaultServerInitializer(conf, context)) 44 | .option(ChannelOption.SO_BACKLOG, conf.getBacklog()) 45 | .option(ChannelOption.SO_REUSEADDR, true); 46 | 47 | Channel ch = b.bind(conf.getPort()).sync().channel(); 48 | ch.closeFuture().sync(); 49 | } finally { 50 | 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /netty-http/src/main/java/http/Config.java: -------------------------------------------------------------------------------- 1 | package http; 2 | 3 | /** 4 | * Created by i311352 on 2/13/2017. 5 | */ 6 | public class Config { 7 | public Integer getBacklog() { 8 | return 1280; 9 | } 10 | 11 | public Integer getPort() { 12 | return 9999; 13 | } 14 | 15 | public Integer getClientMaxBodySize() { 16 | return 1024*1024*10; 17 | } 18 | 19 | public Integer getTaskThreadPoolSize() { 20 | return 0; 21 | } 22 | 23 | public Integer getEventLoopThreadCount() { 24 | return 10; 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /netty-http/src/main/java/http/DefaultExceptionHandler.java: -------------------------------------------------------------------------------- 1 | package http; 2 | 3 | import http.exception.BadRequestException; 4 | import io.netty.buffer.Unpooled; 5 | import io.netty.channel.ChannelHandlerContext; 6 | import io.netty.channel.ChannelInboundHandlerAdapter; 7 | import io.netty.handler.codec.http.DefaultFullHttpResponse; 8 | import io.netty.handler.codec.http.FullHttpResponse; 9 | import io.netty.handler.codec.http.HttpResponseStatus; 10 | import io.netty.util.CharsetUtil; 11 | import org.apache.log4j.Logger; 12 | 13 | import java.io.PrintWriter; 14 | import java.io.StringWriter; 15 | 16 | import static io.netty.handler.codec.http.HttpHeaders.Names.CONTENT_LENGTH; 17 | import static io.netty.handler.codec.http.HttpHeaders.Names.CONTENT_TYPE; 18 | import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1; 19 | 20 | /** 21 | * Created by i311352 on 2/13/2017. 22 | */ 23 | public class DefaultExceptionHandler extends ChannelInboundHandlerAdapter { 24 | private final Logger logger = Logger.getLogger(DefaultExceptionHandler.class); 25 | 26 | @Override 27 | public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { 28 | logger.error("Exception caught: " + cause); 29 | 30 | HttpResponseStatus status = (cause instanceof BadRequestException) ? HttpResponseStatus.BAD_REQUEST : 31 | HttpResponseStatus.INTERNAL_SERVER_ERROR; 32 | 33 | StringWriter stringWriter = new StringWriter(); 34 | PrintWriter printWriter = new PrintWriter(stringWriter); 35 | cause.printStackTrace(printWriter); 36 | String content = stringWriter.toString(); 37 | 38 | FullHttpResponse response = new DefaultFullHttpResponse(HTTP_1_1, 39 | status, Unpooled.copiedBuffer(content, CharsetUtil.UTF_8)); 40 | 41 | response.headers().set(CONTENT_TYPE, "application/json; charset=UTF-8"); 42 | response.headers().set(CONTENT_LENGTH, 43 | response.content().readableBytes()); 44 | 45 | ctx.writeAndFlush(response); 46 | ctx.close(); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /netty-http/src/main/java/http/DefaultServerInitializer.java: -------------------------------------------------------------------------------- 1 | package http; 2 | 3 | import elasticsearch.ElasticRestClient; 4 | import http.worker.SearchWorker; 5 | import io.netty.channel.ChannelInitializer; 6 | import io.netty.channel.ChannelPipeline; 7 | import io.netty.channel.socket.SocketChannel; 8 | import io.netty.handler.codec.http.HttpObjectAggregator; 9 | import io.netty.handler.codec.http.HttpRequestDecoder; 10 | import io.netty.handler.codec.http.HttpResponseEncoder; 11 | import io.netty.util.concurrent.DefaultEventExecutorGroup; 12 | import io.netty.util.concurrent.EventExecutorGroup; 13 | import org.elasticsearch.client.RestClient; 14 | import org.springframework.context.ConfigurableApplicationContext; 15 | 16 | import java.util.concurrent.ArrayBlockingQueue; 17 | import java.util.concurrent.ExecutorService; 18 | 19 | 20 | /** 21 | * Created by i311352 on 2/13/2017. 22 | */ 23 | public class DefaultServerInitializer extends ChannelInitializer { 24 | private final Config conf; 25 | 26 | //private final EventExecutorGroup executor; 27 | private final ExecutorService executor; 28 | private final ConfigurableApplicationContext applicationContext; 29 | 30 | public DefaultServerInitializer(Config conf, ConfigurableApplicationContext applicationContext) { 31 | this.conf = conf; 32 | this.applicationContext = applicationContext; 33 | 34 | this.executor = new SearchWorker(conf.getTaskThreadPoolSize(), "SearchWorkder", new ArrayBlockingQueue(5)); 35 | 36 | // new DefaultEventExecutorGroup( 37 | // conf.getTaskThreadPoolSize()); 38 | } 39 | 40 | @Override 41 | protected void initChannel(SocketChannel ch) throws Exception { 42 | final ChannelPipeline pipeline = ch.pipeline(); 43 | pipeline.addLast("httpDecoder", new HttpRequestDecoder()); 44 | 45 | pipeline.addLast("httpAggregator", new HttpObjectAggregator(conf.getClientMaxBodySize())); 46 | pipeline.addLast("httpResponseEncoder", new HttpResponseEncoder()); 47 | pipeline.addLast("httpMyResponseHandler", new HttpSearchResponseHandler()); 48 | 49 | pipeline.addLast("httpSearchDecoder", new SearchQueryDecoder()); 50 | 51 | RestClient restClient = applicationContext.getBean("elasticRestClient", RestClient.class); 52 | 53 | pipeline.addLast("httpSearchHandler", new HttpSearchHandler(this.executor, restClient)); 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /netty-http/src/main/java/http/HttpSearchHandler.java: -------------------------------------------------------------------------------- 1 | package http; 2 | 3 | import com.google.gson.Gson; 4 | import http.elasticaction.RestRequest; 5 | import http.elasticaction.RestResponse; 6 | import http.elasticaction.SearchDSLImpl; 7 | import http.message.DecodedSearchRequest; 8 | import http.searchcommand.RestCommand; 9 | import http.searchcommand.SearchRequest; 10 | import http.worker.SearchWorker; 11 | import http.worker.notifyexecutor.IFutureListener; 12 | import http.worker.notifyexecutor.SearchingFuture; 13 | import io.netty.channel.ChannelHandlerContext; 14 | import io.netty.channel.SimpleChannelInboundHandler; 15 | import io.netty.util.concurrent.EventExecutorGroup; 16 | import io.netty.util.concurrent.Future; 17 | import io.netty.util.concurrent.GenericFutureListener; 18 | import org.elasticsearch.client.RestClient; 19 | import org.slf4j.Logger; 20 | import org.slf4j.LoggerFactory; 21 | import querydsl.TemplateGenerator; 22 | 23 | import java.util.concurrent.CompletableFuture; 24 | import java.util.concurrent.ExecutorService; 25 | import java.util.concurrent.RunnableFuture; 26 | 27 | 28 | /** 29 | * Created by i311352 on 2/13/2017. 30 | */ 31 | public class HttpSearchHandler extends SimpleChannelInboundHandler { 32 | private final static Gson gson = new Gson(); 33 | private final static Logger logger = LoggerFactory.getLogger(HttpSearchHandler.class); 34 | 35 | private final SearchWorker executor; 36 | private RestClient client; 37 | 38 | public HttpSearchHandler(ExecutorService group, RestClient client) { 39 | this.executor = (SearchWorker) group; 40 | this.client = client; 41 | } 42 | 43 | @Override 44 | protected void channelRead0(ChannelHandlerContext ctx, DecodedSearchRequest searchRequest) throws Exception { 45 | logger.info("Got search request " + searchRequest.getQueryMeta().toString()); 46 | //throw new BadRequestException(new Exception("adads")); 47 | 48 | RestCommand command = new RestCommand(client, new TemplateGenerator(searchRequest.getQueryMeta())); 49 | 50 | //SearchRequest request = new SearchRequest(); 51 | // Callable callable = new Callable() { 52 | // @Override 53 | // public Object call() throws Exception { 54 | // return null; 55 | // } 56 | // }; 57 | 58 | if (ctx.executor().inEventLoop()) { 59 | logger.info("In event loop"); 60 | } 61 | logger.info("Current info" + this.executor.monitorInfo()); 62 | 63 | SearchingFuture searchingFuture = (SearchingFuture) executor.doSearch(new SearchRequest(command));//(SearchingFuture) new SearchingFuture<>(new SearchRequest(command)); 64 | searchingFuture.setListener(new IFutureListener() { 65 | @Override 66 | public void onSuccess(Object result) { 67 | logger.info("Got Response for request: "); 68 | RestResponse restResponse = new RestResponse(); 69 | 70 | restResponse.setBody(result); 71 | restResponse.setHttpRequest(searchRequest.getHttpRequest()); 72 | ctx.writeAndFlush(restResponse); 73 | } 74 | 75 | @Override 76 | public void onCancel(RunnableFuture cancelledFuture) { 77 | 78 | } 79 | 80 | @Override 81 | public void onError(Throwable e, java.util.concurrent.Future future) { 82 | ctx.fireExceptionCaught(e); 83 | } 84 | }); 85 | 86 | // final Future future = executor.submit(new SearchRequest(command)); 87 | // 88 | // future.addListener(new GenericFutureListener>() { 89 | // @Override 90 | // public void operationComplete(Future future) throws Exception { 91 | // logger.info("Got Response for request: "); 92 | // if (future.isSuccess()) { 93 | // RestResponse restResponse = new RestResponse(); 94 | // 95 | // restResponse.setBody(future.get()); 96 | // restResponse.setHttpRequest(searchRequest.getHttpRequest()); 97 | // ctx.writeAndFlush(restResponse); 98 | // } else { 99 | // ctx.fireExceptionCaught(future.cause()); 100 | // } 101 | // if (ctx.executor().inEventLoop()) { 102 | // logger.info("In event loop"); 103 | // } else { 104 | // logger.info("Not in event loop"); 105 | // } 106 | // } 107 | // }); 108 | // 109 | // RestResponse restResponse = new RestResponse(); 110 | // restResponse.setBody(gson.toJson(searchRequest.getQueryMeta())); 111 | // restResponse.setHttpRequest(searchRequest.getHttpRequest()); 112 | // ctx.writeAndFlush(restResponse); 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /netty-http/src/main/java/http/HttpSearchResponseHandler.java: -------------------------------------------------------------------------------- 1 | package http; 2 | 3 | import com.google.gson.Gson; 4 | import http.elasticaction.RestResponse; 5 | import io.netty.buffer.Unpooled; 6 | import io.netty.channel.ChannelHandlerContext; 7 | import io.netty.channel.ChannelOutboundHandlerAdapter; 8 | import io.netty.channel.ChannelPromise; 9 | import io.netty.handler.codec.http.DefaultFullHttpResponse; 10 | import io.netty.handler.codec.http.FullHttpResponse; 11 | import io.netty.handler.codec.http.HttpRequest; 12 | import io.netty.handler.codec.http.HttpResponse; 13 | import io.netty.util.CharsetUtil; 14 | import io.netty.util.ReferenceCountUtil; 15 | 16 | import static io.netty.handler.codec.http.HttpHeaders.Names.CONTENT_LENGTH; 17 | import static io.netty.handler.codec.http.HttpHeaders.Names.CONTENT_TYPE; 18 | import static io.netty.handler.codec.http.HttpResponseStatus.OK; 19 | import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1; 20 | 21 | /** 22 | * Created by i311352 on 2/14/2017. 23 | */ 24 | public class HttpSearchResponseHandler extends ChannelOutboundHandlerAdapter { 25 | Gson gson = new Gson(); 26 | 27 | @Override 28 | public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) throws Exception { 29 | if (msg instanceof HttpResponse) { 30 | super.write(ctx, msg, promise); 31 | return; 32 | } 33 | 34 | RestResponse response = (RestResponse) msg; 35 | 36 | // Build the response object 37 | FullHttpResponse httpResponse = new DefaultFullHttpResponse(HTTP_1_1, 38 | OK, Unpooled.copiedBuffer((char[]) response.getBody(), CharsetUtil.UTF_8)); 39 | 40 | httpResponse.headers().set(CONTENT_TYPE, "application/json"); 41 | httpResponse.headers().set(CONTENT_LENGTH, 42 | httpResponse.content().readableBytes()); 43 | 44 | HttpRequest httpRequest = response.getHttpRequest(); 45 | 46 | // keep alive check 47 | // Boolean keepAlive = isKeepAlive(httpRequest); 48 | // if (keepAlive) { 49 | // // Add keep alive header as per 50 | // // http://www.w3.org/Protocols/HTTP/1.1/draft-ietf-http-v11-spec-01.html#Connection 51 | // httpResponse.headers().set(CONNECTION, 52 | // HttpHeaders.Values.KEEP_ALIVE); 53 | // } else { 54 | // httpResponse.headers().set(CONNECTION, HttpHeaders.Values.CLOSE); 55 | // } 56 | 57 | ReferenceCountUtil.release(httpRequest); 58 | ctx.writeAndFlush(httpResponse, promise); 59 | ctx.close(); 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /netty-http/src/main/java/http/SearchQueryDecoder.java: -------------------------------------------------------------------------------- 1 | package http; 2 | 3 | import http.exception.BadRequestException; 4 | import http.message.DecodedSearchRequest; 5 | import http.message.QueryMeta; 6 | import io.netty.channel.ChannelHandlerContext; 7 | import io.netty.channel.SimpleChannelInboundHandler; 8 | import io.netty.handler.codec.DecoderResult; 9 | import io.netty.handler.codec.http.FullHttpRequest; 10 | import io.netty.handler.codec.http.QueryStringDecoder; 11 | import org.slf4j.Logger; 12 | import org.slf4j.LoggerFactory; 13 | 14 | import java.util.List; 15 | import java.util.Map; 16 | 17 | /** 18 | * Created by i311352 on 2/13/2017. 19 | */ 20 | public class SearchQueryDecoder extends SimpleChannelInboundHandler { 21 | private long orderNumber; 22 | private static Logger logger = LoggerFactory.getLogger(SearchQueryDecoder.class); 23 | 24 | 25 | public SearchQueryDecoder() { 26 | super(false); 27 | } 28 | 29 | @Override 30 | protected void channelRead0(ChannelHandlerContext ctx, FullHttpRequest httpRequest) throws Exception { 31 | DecoderResult result = httpRequest.decoderResult(); 32 | if (!result.isSuccess()) { 33 | throw new BadRequestException(result.cause()); 34 | } 35 | logger.info("Get search request: " + httpRequest.uri()); 36 | 37 | // only decode get path is enough 38 | Map> requestParameters; 39 | QueryStringDecoder stringDecoder = new QueryStringDecoder(httpRequest.uri()); 40 | requestParameters = stringDecoder.parameters(); 41 | 42 | QueryMeta meta = new QueryMeta(); 43 | 44 | for(Map.Entry> entry : requestParameters.entrySet()) { 45 | if (entry.getKey().equals("options[]")) { 46 | // add filters 47 | List filters = entry.getValue(); 48 | filters.forEach(filter -> { 49 | String[] typeVal = filter.split(":"); 50 | meta.addMeta(typeVal[0], typeVal[1]); 51 | }); 52 | } else if (entry.getKey().equals("orderby")) { 53 | meta.setOrderBy(entry.getValue().get(0)); 54 | } else { 55 | logger.warn("Unknown query parameter, ignore it:" + entry.toString()); 56 | } 57 | } 58 | 59 | DecodedSearchRequest searchRequest = new DecodedSearchRequest(httpRequest, meta, orderNumber++); 60 | ctx.fireChannelRead(searchRequest); 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /netty-http/src/main/java/http/elasticaction/RequestMeta.java: -------------------------------------------------------------------------------- 1 | package http.elasticaction; 2 | 3 | /** 4 | * Created by i311352 on 2/13/2017. 5 | */ 6 | public interface RequestMeta { 7 | 8 | } 9 | -------------------------------------------------------------------------------- /netty-http/src/main/java/http/elasticaction/RestRequest.java: -------------------------------------------------------------------------------- 1 | package http.elasticaction; 2 | 3 | /** 4 | * Created by i311352 on 2/13/2017. 5 | */ 6 | public class RestRequest { 7 | } 8 | -------------------------------------------------------------------------------- /netty-http/src/main/java/http/elasticaction/RestResponse.java: -------------------------------------------------------------------------------- 1 | package http.elasticaction; 2 | 3 | import io.netty.handler.codec.http.HttpRequest; 4 | 5 | /** 6 | * Created by i311352 on 2/15/2017. 7 | */ 8 | public class RestResponse { 9 | 10 | private HttpRequest httpRequest; 11 | private Object body; 12 | 13 | public Object getBody() { 14 | return body; 15 | } 16 | 17 | public void setBody(Object body) { 18 | this.body = body; 19 | } 20 | 21 | public HttpRequest getHttpRequest() { 22 | return httpRequest; 23 | } 24 | 25 | public void setHttpRequest(HttpRequest httpRequest) { 26 | this.httpRequest = httpRequest; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /netty-http/src/main/java/http/elasticaction/SearchDSL.java: -------------------------------------------------------------------------------- 1 | package http.elasticaction; 2 | 3 | /** 4 | * Created by i311352 on 2/13/2017. 5 | */ 6 | @FunctionalInterface 7 | public interface SearchDSL { 8 | R getDSL(); 9 | } 10 | -------------------------------------------------------------------------------- /netty-http/src/main/java/http/elasticaction/SearchDSLImpl.java: -------------------------------------------------------------------------------- 1 | package http.elasticaction; 2 | 3 | import http.message.QueryMeta; 4 | import org.elasticsearch.index.query.MatchAllQueryBuilder; 5 | import org.elasticsearch.index.query.QueryBuilder; 6 | import org.elasticsearch.index.query.QueryBuilders; 7 | 8 | import static org.elasticsearch.index.query.QueryBuilders.*; 9 | 10 | /** 11 | * Created by i311352 on 2/21/2017. 12 | */ 13 | public class SearchDSLImpl implements SearchDSL { 14 | 15 | private final QueryMeta meta; 16 | 17 | public SearchDSLImpl(QueryMeta meta) { 18 | this.meta = meta; 19 | } 20 | 21 | @Override 22 | public QueryBuilder getDSL() { 23 | MatchAllQueryBuilder builders = matchAllQuery(); 24 | return builders; 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /netty-http/src/main/java/http/exception/BadRequestException.java: -------------------------------------------------------------------------------- 1 | package http.exception; 2 | 3 | /** 4 | * Created by i311352 on 2/13/2017. 5 | */ 6 | public class BadRequestException extends Exception{ 7 | 8 | public BadRequestException(Throwable throwable) { 9 | super(throwable); 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /netty-http/src/main/java/http/message/DecodedSearchRequest.java: -------------------------------------------------------------------------------- 1 | package http.message; 2 | 3 | import io.netty.handler.codec.http.HttpRequest; 4 | 5 | /** 6 | * Created by i311352 on 2/13/2017. 7 | */ 8 | public class DecodedSearchRequest { 9 | private final HttpRequest httpRequest; 10 | private final long orderNumber; 11 | 12 | private QueryMeta queryMeta; 13 | 14 | public DecodedSearchRequest(HttpRequest request, QueryMeta meta, long orderNumber) { 15 | this.httpRequest = request; 16 | this.queryMeta = meta; 17 | this.orderNumber = orderNumber; 18 | } 19 | public QueryMeta getQueryMeta() { 20 | return queryMeta; 21 | } 22 | 23 | public void setQueryMeta(QueryMeta queryMeta) { 24 | this.queryMeta = queryMeta; 25 | } 26 | 27 | public HttpRequest getHttpRequest() { 28 | return httpRequest; 29 | } 30 | 31 | public long getOrderNumber() { 32 | return orderNumber; 33 | } 34 | 35 | @Override 36 | public String toString() { 37 | return "DecodedSearchRequest{" + 38 | "httpRequest=" + httpRequest + 39 | ", orderNumber=" + orderNumber + 40 | ", queryMeta=" + queryMeta + 41 | '}'; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /netty-http/src/main/java/http/message/QueryMeta.java: -------------------------------------------------------------------------------- 1 | package http.message; 2 | 3 | import java.util.HashMap; 4 | 5 | /** 6 | * Created by i311352 on 2/13/2017. 7 | */ 8 | public class QueryMeta { 9 | 10 | private String orderBy; 11 | private HashMap filters = new HashMap(); 12 | 13 | public HashMap addMeta(String type, String value) { 14 | filters.put(type, value); 15 | return filters; 16 | } 17 | 18 | public String getOrderBy() { 19 | return orderBy; 20 | } 21 | 22 | public void setOrderBy(String orderBy) { 23 | this.orderBy = orderBy; 24 | } 25 | 26 | public HashMap getFilters() { 27 | return filters; 28 | } 29 | 30 | public void setFilters(HashMap filters) { 31 | this.filters = filters; 32 | } 33 | 34 | @Override 35 | public String toString() { 36 | return "QueryMeta{" + 37 | "orderBy='" + orderBy + '\'' + 38 | ", filters=" + filters + 39 | '}'; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /netty-http/src/main/java/http/searchcommand/RestCommand.java: -------------------------------------------------------------------------------- 1 | package http.searchcommand; 2 | 3 | import com.netflix.hystrix.*; 4 | import http.elasticaction.RestRequest; 5 | import http.elasticaction.SearchDSL; 6 | import http.elasticaction.SearchDSLImpl; 7 | import org.apache.commons.io.IOUtils; 8 | import org.apache.http.HttpEntity; 9 | import org.apache.http.entity.StringEntity; 10 | import org.elasticsearch.client.Response; 11 | import org.elasticsearch.client.RestClient; 12 | import org.elasticsearch.index.query.QueryBuilder; 13 | import org.slf4j.Logger; 14 | import org.slf4j.LoggerFactory; 15 | 16 | import java.nio.charset.Charset; 17 | import java.util.HashMap; 18 | import java.util.UUID; 19 | 20 | /** 21 | * Created by i311352 on 2/16/2017. 22 | */ 23 | public class RestCommand extends HystrixCommand { 24 | private static final Logger LOGGER = LoggerFactory.getLogger(RestCommand.class); 25 | 26 | private RestClient client; 27 | private SearchDSL restRequest; 28 | 29 | public RestCommand(RestClient client, SearchDSL restRequest) { 30 | super(getSetter(client.toString())); 31 | this.client = client; 32 | this.restRequest = restRequest; 33 | } 34 | 35 | private static HystrixCommand.Setter getSetter(String key) { 36 | LOGGER.info("Command key:" + key); 37 | // String uuid = UUID.randomUUID().toString(); 38 | // key += uuid; 39 | 40 | HystrixCommandGroupKey groupKey = HystrixCommandGroupKey.Factory.asKey(key); 41 | HystrixCommandKey commandKey = HystrixCommandKey.Factory.asKey(key); 42 | HystrixCommandProperties.Setter commandProperties = HystrixCommandProperties.Setter(); 43 | commandProperties.withExecutionTimeoutInMilliseconds(10000) 44 | .withCircuitBreakerErrorThresholdPercentage(60); 45 | 46 | HystrixThreadPoolProperties.Setter threadPoolPropertiesDefaults = HystrixThreadPoolProperties.Setter(); 47 | threadPoolPropertiesDefaults.withCoreSize(15) 48 | .withMaxQueueSize(300) 49 | .withQueueSizeRejectionThreshold(90); 50 | 51 | return Setter.withGroupKey(groupKey).andCommandKey(commandKey).andCommandPropertiesDefaults(commandProperties) 52 | .andThreadPoolPropertiesDefaults(threadPoolPropertiesDefaults); 53 | } 54 | 55 | @Override 56 | protected char[] run() throws Exception { 57 | LOGGER.info("Going to get request: " + "stores/product/_search?size=1000"); 58 | String query = this.restRequest.getDSL().toString(); 59 | LOGGER.info("Going to query..: " + query); 60 | HttpEntity requestBody = new StringEntity("{\n" + 61 | " \"match_all\" : {\n" + 62 | " \"boost\" : 1.0\n" + 63 | " }\n" + 64 | "}", Charset.defaultCharset()); 65 | 66 | Response response = this.client.performRequest("GET", "stores/product/_search");//, new HashMap(), requestBody); 67 | return IOUtils.toCharArray(response.getEntity().getContent()); 68 | } 69 | 70 | @Override 71 | protected char[] getFallback() { 72 | String string = new String("Just FallBack"); 73 | return string.toCharArray(); 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /netty-http/src/main/java/http/searchcommand/SearchRequest.java: -------------------------------------------------------------------------------- 1 | package http.searchcommand; 2 | 3 | import org.elasticsearch.client.Response; 4 | import org.slf4j.Logger; 5 | import org.slf4j.LoggerFactory; 6 | 7 | import java.util.concurrent.Callable; 8 | 9 | /** 10 | * Created by i311352 on 2/16/2017. 11 | */ 12 | public class SearchRequest implements Callable { 13 | private static final Logger LOGGER = LoggerFactory.getLogger(RestCommand.class); 14 | 15 | RestCommand command; 16 | public SearchRequest(RestCommand command) { 17 | this.command = command; 18 | } 19 | 20 | @Override 21 | public char[] call() throws Exception { 22 | LOGGER.info("Start request: " + command.toString()); 23 | return command.execute(); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /netty-http/src/main/java/http/worker/SearchWorker.java: -------------------------------------------------------------------------------- 1 | package http.worker; 2 | 3 | 4 | 5 | import http.searchcommand.SearchRequest; 6 | import http.worker.notifyexecutor.INotifyingFuture; 7 | import http.worker.notifyexecutor.SearchingFuture; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | import java.util.concurrent.BlockingQueue; 12 | import java.util.concurrent.ThreadPoolExecutor; 13 | import java.util.concurrent.TimeUnit; 14 | 15 | /** 16 | * Created by i311352 on 2/20/2017. 17 | */ 18 | public class SearchWorker extends ThreadPoolExecutor { 19 | private final static Logger logger = LoggerFactory.getLogger(SearchWorker.class); 20 | private static SearchWorker instance; 21 | 22 | public SearchWorker(int size, String name, BlockingQueue blockingQueue) { 23 | super(size, size+5, 0L, TimeUnit.MILLISECONDS, blockingQueue, new WorkerThread(name)); 24 | } 25 | 26 | @Override 27 | protected void beforeExecute(Thread t, Runnable r) { 28 | logger.info("start to exectue: " + t); 29 | super.beforeExecute(t, r); 30 | } 31 | 32 | @Override 33 | protected void afterExecute(Runnable r, Throwable t) { 34 | super.afterExecute(r, t); 35 | logger.info("after to exectue: " + t); 36 | } 37 | 38 | public INotifyingFuture doSearch(SearchRequest request) { 39 | SearchingFuture future = new SearchingFuture(request); 40 | future.setExecutor(this); 41 | future.run(); 42 | return future; 43 | } 44 | 45 | public String monitorInfo() { 46 | StringBuilder result = new StringBuilder(100); 47 | result.append("\r\n QueueSize :" + this.getQueue().size()); 48 | result.append("ThreadCount :" + this.getPoolSize() + "\r\n"); 49 | return result.toString(); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /netty-http/src/main/java/http/worker/WorkerThread.java: -------------------------------------------------------------------------------- 1 | package http.worker; 2 | 3 | import java.util.ArrayList; 4 | import java.util.Date; 5 | import java.util.Iterator; 6 | import java.util.List; 7 | import java.util.concurrent.ThreadFactory; 8 | 9 | /** 10 | * Created by i311352 on 2/20/2017. 11 | */ 12 | public class WorkerThread implements ThreadFactory { 13 | 14 | private int threadCount; 15 | private String name; 16 | private List states; 17 | 18 | 19 | public WorkerThread(String name) { 20 | this.name = name; 21 | this.threadCount = 1; 22 | this.states = new ArrayList<>(); 23 | } 24 | 25 | @Override 26 | public Thread newThread(Runnable r) { 27 | ThreadGroup group = new ThreadGroup("SearchWorder"); 28 | 29 | Thread t = new Thread(r, name + "_" + threadCount); 30 | threadCount ++; 31 | 32 | states.add(String.format("Created thread %d with name %s on %s \n", t.getId(), t.getName(), new Date())); 33 | 34 | return t; 35 | } 36 | 37 | public String getStats() 38 | { 39 | StringBuffer buffer = new StringBuffer(); 40 | Iterator it = this.states.iterator(); 41 | while (it.hasNext()) 42 | { 43 | buffer.append(it.next()); 44 | } 45 | return buffer.toString(); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /netty-http/src/main/java/http/worker/notifyexecutor/IFutureListener.java: -------------------------------------------------------------------------------- 1 | package http.worker.notifyexecutor; 2 | 3 | import java.util.concurrent.Future; 4 | import java.util.concurrent.RunnableFuture; 5 | 6 | /** 7 | * Created by i311352 on 2/20/2017. 8 | */ 9 | public interface IFutureListener { 10 | 11 | /** 12 | * The task was computed successfully. 13 | * @param result 14 | */ 15 | public void onSuccess(V result); 16 | 17 | /** 18 | * called when future state is canceled. 19 | */ 20 | public void onCancel(RunnableFuture cancelledFuture); 21 | 22 | /** 23 | * Called when there was an error while executing 24 | * this future. 25 | * @param e 26 | * @param future the future that fails 27 | */ 28 | public void onError(Throwable e, Future future); 29 | } 30 | -------------------------------------------------------------------------------- /netty-http/src/main/java/http/worker/notifyexecutor/INotifyingExecutorService.java: -------------------------------------------------------------------------------- 1 | package http.worker.notifyexecutor; 2 | 3 | import java.util.concurrent.Callable; 4 | import java.util.concurrent.ExecutorService; 5 | 6 | /** 7 | * Created by i311352 on 2/20/2017. 8 | */ 9 | public interface INotifyingExecutorService extends ExecutorService { 10 | 11 | @Override 12 | public INotifyingFuture submit(Callable task); 13 | 14 | @Override 15 | public INotifyingFuture submit(Runnable task); 16 | 17 | @Override 18 | public INotifyingFuture submit(Runnable task, T result); 19 | } -------------------------------------------------------------------------------- /netty-http/src/main/java/http/worker/notifyexecutor/INotifyingFuture.java: -------------------------------------------------------------------------------- 1 | package http.worker.notifyexecutor; 2 | 3 | import java.util.concurrent.ExecutorService; 4 | import java.util.concurrent.RunnableFuture; 5 | 6 | /** 7 | * Created by i311352 on 2/20/2017. 8 | */ 9 | public interface INotifyingFuture extends RunnableFuture { 10 | 11 | /** 12 | * Sets this listener to a {@link INotifyingFuture}. When the future is done 13 | * or canceled the listener gets notified.
14 | * @param listener 15 | * @param the executor that executes the shiet. 16 | */ 17 | public void setListener(IFutureListener listener, ExecutorService executor); 18 | 19 | /** 20 | * Sets this listener to a {@link INotifyingFuture}. When the future is done 21 | * or canceled the listener gets notified.
22 | * Attention: Be aware of the fact that everything that is done in that 23 | * listener is executed in same thread as the original task that this listener listens 24 | * to. Only use this method when you are sure that no long running task is performed 25 | * by the listener. When you want the listener's tasks to be performed asynchronous 26 | * use {@link #setListener(IFutureListener, ExecutorService)} instead. 27 | * @param listener 28 | */ 29 | public void setListener(IFutureListener listener); 30 | } 31 | -------------------------------------------------------------------------------- /netty-http/src/main/java/http/worker/notifyexecutor/SearchingFuture.java: -------------------------------------------------------------------------------- 1 | package http.worker.notifyexecutor; 2 | 3 | import http.worker.SearchWorker; 4 | 5 | import java.util.concurrent.*; 6 | import java.util.concurrent.atomic.AtomicBoolean; 7 | 8 | /** 9 | * Created by i311352 on 2/20/2017. 10 | */ 11 | 12 | public class SearchingFuture extends FutureTask implements INotifyingFuture { 13 | 14 | private static final ExecutorService DEFAULT_EXECUTOR = new SearchWorker(30, "SearchWorker", new LinkedBlockingDeque(100));; 15 | 16 | private IFutureListener listener = null; 17 | private ExecutorService executor = null; 18 | private final AtomicBoolean executed = new AtomicBoolean(); 19 | 20 | public SearchingFuture(Callable callable) { 21 | super(callable); 22 | setExecutor(DEFAULT_EXECUTOR); 23 | } 24 | 25 | public SearchingFuture(Runnable runnable, V result) { 26 | super(runnable,result); 27 | setExecutor(DEFAULT_EXECUTOR); 28 | } 29 | 30 | 31 | @Override 32 | protected void done() { 33 | if(listener == null){ 34 | return; 35 | } 36 | notifyListenerOnce(); 37 | } 38 | 39 | /** 40 | * Atomically executes the task only one time. 41 | */ 42 | protected void notifyListenerOnce(){ 43 | if(!this.executed.getAndSet(true)){ 44 | notifyListener(); 45 | } 46 | } 47 | 48 | protected void notifyListener() { 49 | this.executor.submit(new TaskCompletionRunner(delegateFuture(),this.listener)); 50 | } 51 | 52 | /** 53 | * @return the future that was processed. 54 | */ 55 | protected RunnableFuture delegateFuture(){ 56 | return this; 57 | } 58 | 59 | @Override 60 | public void setListener(IFutureListener listener, ExecutorService executor) { 61 | setExecutor(executor); 62 | setListener(listener); 63 | } 64 | 65 | public void setExecutor(ExecutorService executor) { 66 | this.executor = executor; 67 | } 68 | 69 | @Override 70 | public void setListener(IFutureListener listener) { 71 | this.listener = listener; 72 | /* 73 | * Probably the task was already executed. If so, call done() manually. 74 | */ 75 | runWhenDone(); 76 | } 77 | 78 | protected void runWhenDone(){ 79 | if(isDone()){ 80 | notifyListenerOnce(); 81 | } 82 | } 83 | 84 | private static class TaskCompletionRunner implements Runnable{ 85 | 86 | private final IFutureListener listener; 87 | private final RunnableFuture future; 88 | 89 | public TaskCompletionRunner(RunnableFuture future, IFutureListener listener) { 90 | this.future = future; 91 | this.listener = listener; 92 | } 93 | 94 | @Override 95 | public void run() { 96 | if (this.future.isCancelled()) { 97 | this.listener.onCancel(this.future); 98 | } else { 99 | try { 100 | this.listener.onSuccess(this.future.get()); 101 | } catch (InterruptedException e) { 102 | this.listener.onError(e, this.future); 103 | } catch (ExecutionException e) { 104 | this.listener.onError(e, this.future); 105 | } 106 | } 107 | } 108 | } 109 | } -------------------------------------------------------------------------------- /netty-http/src/main/java/querydsl/TemplateGenerator.java: -------------------------------------------------------------------------------- 1 | package querydsl; 2 | 3 | import http.elasticaction.SearchDSL; 4 | import http.message.QueryMeta; 5 | import org.jtwig.JtwigModel; 6 | import org.jtwig.JtwigTemplate; 7 | import org.jtwig.environment.DefaultEnvironmentConfiguration; 8 | import org.jtwig.environment.Environment; 9 | import org.jtwig.environment.EnvironmentConfiguration; 10 | import org.jtwig.environment.EnvironmentFactory; 11 | import org.jtwig.resource.reference.ResourceReference; 12 | import org.slf4j.Logger; 13 | import org.slf4j.LoggerFactory; 14 | 15 | /** 16 | * Created by I311352 on 3/8/2017. 17 | */ 18 | public class TemplateGenerator implements SearchDSL { 19 | private final static Logger logger = LoggerFactory.getLogger(TemplateGenerator.class); 20 | private QueryMeta meta; 21 | 22 | 23 | public TemplateGenerator(QueryMeta meta) { 24 | this.meta = meta; 25 | } 26 | 27 | @Override 28 | public String getDSL() { 29 | JtwigTemplate template = JtwigTemplate.classpathTemplate("templates/filters.twig"); 30 | JtwigModel model = JtwigModel.newModel().with("var", "World"); 31 | String result = template.render(model); 32 | logger.info("get search dsl " + result); 33 | return result; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /netty-http/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=debug, console 2 | 3 | log4j.appender.console=org.apache.log4j.ConsoleAppender 4 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 5 | log4j.appender.console.layout.conversionPattern=[%d{ISO8601}][%-35p][%-35c{1}] %marker%.10000m%n 6 | -------------------------------------------------------------------------------- /netty-http/src/main/resources/templates/filters.twig: -------------------------------------------------------------------------------- 1 | {
 2 | "query":{ 3 | {% if (length(name)==0) %} 4 | 
 "match_all": {} 5 | {% else %} 6 | 
 "match":{ 7 | 
 "name": { 8 | 
 "query": "{{ name }}", 9 | 
 "operator": "{{ default(operator, 'or') }}" 10 | 
 } 11 | 
 } 12 | {% endif %} 13 | 
 } 14 | 
} -------------------------------------------------------------------------------- /netty-http/src/test/java/TestEhcache.java: -------------------------------------------------------------------------------- 1 | import org.ehcache.Cache; 2 | import org.ehcache.CacheManager; 3 | import org.ehcache.config.builders.CacheConfigurationBuilder; 4 | import org.ehcache.config.builders.CacheManagerBuilder; 5 | import org.ehcache.config.builders.ResourcePoolsBuilder; 6 | import org.junit.Test; 7 | 8 | /** 9 | * Created by I311352 on 4/24/2017. 10 | */ 11 | public class TestEhcache { 12 | 13 | @Test 14 | public void testTem() { 15 | CacheManager cacheManager = CacheManagerBuilder.newCacheManagerBuilder() 16 | .withCache("preConfigured", 17 | CacheConfigurationBuilder.newCacheConfigurationBuilder(Long.class, String.class, ResourcePoolsBuilder.heap(10))) 18 | .build(); 19 | 20 | cacheManager.init(); 21 | 22 | Cache preConfigured = 23 | cacheManager.getCache("preConfigured", Long.class, String.class); 24 | 25 | Cache myCache = cacheManager.createCache("myCache", 26 | CacheConfigurationBuilder.newCacheConfigurationBuilder(Long.class, String.class, ResourcePoolsBuilder.heap(10))); 27 | 28 | myCache.put(1L, "da one!"); 29 | myCache.putIfAbsent(0L, "ee"); 30 | String value = myCache.get(1L); 31 | 32 | System.out.println("Value is " + value); 33 | cacheManager.removeCache("preConfigured"); 34 | cacheManager.close(); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /notes/Java_mem_track.md: -------------------------------------------------------------------------------- 1 | Native memory track use jemalloc for ES 2 | 从[这篇文章](https://www.elastic.co/blog/tracking-down-native-memory-leaks-in-elasticsearch)中得知ES在5.1版本中有原生内存泄露的问题， 3 | 而碰巧我们生产环境中也在使用5.1系列的版本，但是场景不一样。比如我们并没有使用x-pack，但是还是放心不下，决定在我们的场景下利用文章中的方法进行下测试。 4 | 5 | 先弄清楚要测试的问题，Java的内存问题，大多数集中在Java Heap上面，所以平时遇到native memory的问题几率较低。Heap是Java新对象创建的地方也是GC工作的 6 | 主要区域，平时如果遇到Heap内存问题，通过收集GC日志，基本上就能看出端倪。例如GC时间过长、GC过于频繁的话需要进行GC算法参数的调优，老年代持续增长的话就要看 7 | 对象是否分配的合理了，这也是一种泄露-导致GC无法回收。不同的GC算法调优的参数不一样，但目前还是用的CMS，有时间可以总结下。 8 | 9 | 另外就是native memory的问题，例如出现了Outofmemory的错误： 10 | 11 | ``` 12 | Allocated 1953546736 bytes of native memory before running out 13 | Exception in thread "main" java.lang.OutOfMemoryError: unable to create new native thread 14 | at java.lang.Thread.start0(Native Method) 15 | at java.lang.Thread.start(Thread.java:574) 16 | at com.ibm.jtc.demos.StartingAThreadUnderNativeStarvation.main( 17 | StartingAThreadUnderNativeStarvation.java:22) 18 | ``` 19 | 20 | 遇到这样的问题，首先要确定的是是不是由堆内存申请导致的，如果不是就有可能是下面两种： 21 | 1. MetaSpace 22 | 2. Native memory 23 | 24 | 只要是JVM相关的内存问题，通过相关的监控工具都能很好的判定，例如如果集成了[Metrics](http://metrics.dropwizard.io/3.2.2/index.html)，可以得到 25 | 26 | ![metrics](./metrics.png) 27 | 28 | 通过这样的监控可以很容易的可以看出内存问题是否是出自于JVM。 29 | 30 | 扯得有点远了，回归主题：native memory tracking。 31 | 需要清楚的是Native memory tracking是tracking Java进程申请的OS内存，即glibc申请内存的情况。而 32 | [jemalloc](https://github.com/jemalloc/jemalloc)这个工具可以帮助我们抓取到Java进程的内存申请行为。过程也颇为简单： 33 | 34 | ``` 35 | git clone https://github.com/jemalloc/jemalloc 36 | git checkout stable-4 37 | ./autogen.sh 38 | ./configure --enable-perf 39 | make 40 | sudo make install 41 | ``` 42 | 43 | 启动ES之前设置下环境变量： 44 | ``` 45 | export LD_PRELOAD=/usr/local/lib/libjemalloc.so 46 | export MALLOC_CONF=prof:true,lg_prof_interval:30,lg_prof_sample:17,prof_final:true 47 | ``` 48 | 49 | 环境变量LD_PRELOAD用来替换原生的glibc malloc。 50 | 然后启动ES，运行一段时间，停掉后会生成多个jeprof.{processId}的文件，通过下面命令生成内存tracking信息： 51 | 52 | ``` 53 | jeprof --show_bytes --gif ~/jre/bin/java jeprof*.heap > ./app-profiling.gif 54 | ``` 55 | 56 | ![profiling](./profiling.gif) 57 | 有将近95%的内存来自于jvm本身，也由于运行时间不是很长，所以可以认为是安全的，如果再持续一段时间这个百分比降低了，说明有内存泄露的可能。 58 | 59 | 参考： 60 | 1. http://www.evanjones.ca/java-native-leak-bug.html 61 | 2. https://gdstechnology.blog.gov.uk/2015/12/11/using-jemalloc-to-get-to-the-bottom-of-a-memory-leak/ -------------------------------------------------------------------------------- /notes/Search.xml: -------------------------------------------------------------------------------- 1 | 7V1bd6M4Ev41PCYHkMTl0fjSPef07vZues7sPCpGttlg8GCcy/z6lYRkIwkn2MGxnZCHBEpCoPqqPklFiVhguHz+VuDV4h95TFLLteNnC4ws13Wg69E/TPJSSULPrQTzIokrkb0T3CV/E3GllG6SmKyFrBKVeZ6WyUoVTvMsI9NSkeGiyJ/UarM8jRXBCs+JIbib4tSU/pHE5aKSBrJbTP6dJPOFvLPjhVXJPZ4+zIt8k4n7WS6Y8Z+qeIllW6Kj6wWO86eaCIwtMCzyvKyOls9DkjLdqmqb7CndPndBsrLNBQKWR5xuRNfHd4t8JR6ufJEKoVdR3dOT6GmRlORuhaes5InCT2WLcpnSM4cepviepNFWC8M8zQvehNQDiNZlkT+QWgkYI3/kbUukxllzsyRNZc0sz9gDzAscJ7R7mniWZ6UwJMcV57V72PQHACrHaTLPqGxKmyC0MDJVJrT4SIqSPNdEQoXfSL4kZfFCq8hSIOAU5u64qDp/2hkPFFUWNbuRMizMdb5teQcZPRCoNSMIegS7QDD0zoYg7BHsAEEXhWdDEPUIdoAgkJ5wBgS9HsEuEAzB2RAMegQ7QBCi8/lg2CPYAYJIInYGBOXyrQbhHcHFdHFHCtoDA8nPgl9KZlSLEQMpoYvIgRCX+aqj2akdKJi6oW1gimSzdVBdx+8AVcdAdZziNe3qMN2sy4vCldYcDieTsXe94KIwaAAXNYHrdACuGQEwCTeLByyowpiKAr9OpiqeBlhMh5NJM1gttU/1Wbz8lwrtW9p3cf4nO2cnz0lZlSFx9qdofi8e63xTTEV/RA9LXMyJrCUcisRKYMgErQ5KA4tKWUFSXCaPajipCSdxh595Qh94b0DBRVoTVXfEVfX4jt5QoDVkaw1VSjAa4maz7XY7SzIjEd/GvyzXS5n7xMkjPZyXHKNKdF/oEnoLpZ5mh9ShStXyCrJO/sb3vAIzjBXrB+8Ziiw0ahpidT9eJnHMrm/DSm2Nt8EKpZvptLCNYYpeWPU4YBNd3FCHcAJ1jAfvMzpZJZ/N1uS9ZgDNcIZz7Ui2Rq0J9L1I2rfQg64CJLwgIB0TyMsYGfaSf23MANBXBo1b296OIj9JkVA9MBviptZqzACXP2aEqi05njajbztmuL6jjhmSbU4wZpiRs37M2FqhdMBPP2Y4Zvjti1HNhXCI62tBdE9z/bYcApBKRm54unmn33PIfg6RnvX5OcQMH/Yccg4OAVALIR/LIVB69uk5xDUDlz2HbM1Letan5xDXjHR+MQ6pL3m8y6KVUGUDx4NH0oqvhcRCbZ3UIa24hj31tCItbm+k/PPRihkZ/WK0ciEcAj0ty+vYqQly1USHU3KIGYbrOWTLIdKzPj+HmJGynkPOwSEI6POQYzkkVNdJp+QQM77Wc8iWQ6RnfX4OMSNl/yRl+fK9LFcfl6HjTQNyP2smHTWTI8YkmE0PAPckmeHH5t747n6U22ZnNGQ1vppQdZQbbsp8LZRrtUx7WVOLSLL5L3YyuoHvCy+8GxGp6RoiQUO+jAM7AMSMMv7EBV5Sc/tV4Gw9y4vld5xR2irWH+FMgXsPvFZpUTEiQXwIUh/hTMAzoWt0Jn2xf9ROGTPIxzFb5UU5TFmm2Ecgdh8giOw2iM2CKZleGv1tt/a9hVgAOkDs6uJxezWu5BIIQ1SSCS5rlut66iwXQS2RtP2LQHU+5OnDYnezXGBG2xqmuNX4+fmnteDLhNegubqB147kkRln4HUyt28h0IB0LwhI0CJOmiY8iVvlbnjwsKwPtVLe9YaOGK8XJBbPxU5+4pKWZFzCknG72fOhp35Bc8/HNh1MmRJ3sQHZjEz9kRcPbHVikO+vRUFw3JJ8TzD5iv3w3m41+QrDySQML2zy1bT29H0T1i03vAvXhmhBHhMzmfek+7G2+n0TsQn/OYEP72VkA9u9QAKkvnP05Fq0PptrABJ1sH0HmDEE6YB0GZ9J92PYgppn1st6yA+HXJ9B+7KJj4C8KReqGXK3h7wzyCE4H+TQnDtdxlrZalrWttLqB610oRaSglrg4ti0eeAHakPdrXShOeUCPdhHhDWg3Jd1MNihOqBDfft0h2A3jd891m8ne2j+iMIjQ1h6Q+B0L2qhOXAjA+yvEcqozP6VUIaDZMBP4HJzSbvnkPmyoffaNl4L1MwqgLwjvdZW0yuAHsHe47UUEfxSqyZ8af8Da0MKEJ8C3JlO1eLRhmS+A/G+KCWgN+PU0AZqhp97SZTgvk0J6wVescM0yR50MsBFadIFF0+SVFarcYpYEVGJKGd28D+W8CGQYm/+qSgvykU+zzOc/sj5C/2Tf9PBubWB+lGHWw+G1p6MsOa3bfYBZKa9fhODbP31W7VyOxvpybQCfeF4+FRFbcjztYa6m6qgFsH6r2HNtmrJ6BQmG1yayVI9qCYrh9vDXxBroVGkNdShybb4osGXNdkODTM8q2FqXBrY4ZGGqXGpj7SGujNMr2mxUHt5VjNQ768N+zZ0xGZ8N2IKN2Dzp2oWty3X83eZSTQ2xApuqvw/1o4DV88NrYw9K4TWYMIOgqEVONY4tAaRFQbWGFlRaEUeOwgmVkiLJlY0tMIBL6KVXXbVYGQNELsqHFnhmB8EVhSxOgOHS3x2SQB/+1ctJF099uupyIcpiuc1vqkm+UVyWQ+alzi3tKL4QqFd6zztFe22/9eGUPjpGVVa5DMNRA5XBVcO1eEYWtGEaWOnUtoG1xjTyZhpVVb+/T8/eP3ACkb8QKpX3NNn6IQTARO7sCoKardDEsGqHZs/myfuO3A5BPSqgTWwS5LhrPxtJJqpAAzpo9kS2+g6MHIZRvX0Zt7rYMwRqVTkccQmvPseK2JWWlOasO2ahoPACm2hRoECVZqzqNI+axfTdqnRD/k1I6bi6ppoID6fJ1KRpOFXFuPao7sfV6ZmcCtcgBpI4P273g+PK7lSF9USLLWMy7qqVQca39E2pIr2uNG1KAhWCtKVIRHnnafdpnRyZR1DnAQrc+YUAzhDIdY/6jDNjhRZtKmds4m+S/8RlqSRJWJEFQ7oCJnEuCRXpibvVlAPIxrIFUXHznE1eowBY+PAvY6uvLpF6IKf+45OyGjVLC/JejdDuYpH17PU5dM/ELJiN3wkuwBakmfr6+iVupHFXuIkY7P/DfvND7l33Ayou692yWSX3Sc5HVzLJGO8ZNkR1e9ZkvIpui1pv5Q7Rthxznsshk2qAYqiHSezWTLdpOz28YbIakucscvnmN0KT6d5ESfZXJZONwVfsonmqPrw8crrfFuguqyVSRvml5lbbUNqykT5iGB1+5QQPYMvAGYGnxe45moYHLFbiZ7u/nNQtQzd/XsmMP4/ -------------------------------------------------------------------------------- /notes/all_about_search&filter.md: -------------------------------------------------------------------------------- 1 | ## ElasticSearch 深入理解 四：Search & Query DSL的生成 2 | ES最核心的功能就是提供了全文检索能力，並还有很强大的分析功能。项目中已经使用ES作为产品主数据，当然还有个MySQL的ID服务器。ES将负责完成所有的产品检索、精确匹配、同义词检索、搜索推荐等功能。ES天生支持那些功能，但是需要数据存储结构的支持，那么数据结构如何定义呢？答案就是index的mapping定义。所以说做好ES的search功能必须要很好的理解你的数据存储结构。 3 | 4 | ### index流程 5 | Mapping结构规定了字段存储方式，但是ES的存储流程是伴随着文本流处理的，大致有三个阶段： 6 | 7 | ![tokenizer](./tokenizer.png) 8 | 9 | 其中每个阶段都有很多选择，都可以通过mapping的setting设置。具体可以参照[官网介绍](https://www.elastic.co/guide/en/elasticsearch/reference/current/analysis-analyzers.html)。最终把预处理、分词好的值存储到倒排索引中，供后面的Search使用。理解这个流程对Search的理解也有好处，例如term query的时候为啥会是区分大小写查询的，而match query不区分。 10 | 11 | ### mapping结构 12 | ES支持多种应用场景，autosuggest，fuzzy Search，同义词等。这些功能的实现都是通过index的mapping结构定义支持的。所以这块需要多看看ES的文档，并结合index的mapping进行一定的验证实验。 13 | 14 | 下面具体说下项目中需要解决的问题。 15 | 16 | 搜索推荐的功能理解起来比较简单，例如用户在Search输入框输入的过程中，给一定的feedback，推荐出一个list供用户选择。关于这个功能，ES是可以支持的很好的。例如定义一个字段，type指定为“completion”。 17 | 18 | ```json 19 | {"properties": { 20 | "name_suggest" : { 21 | "type" : "completion", 22 | "analyzer" : "suggest_name_synonyms", 23 | "search_analyzer": "standard" 24 | }}} 25 | ``` 26 | 27 | 例如上面properties中定义了一个name_suggest字段，这个字段作为一个“虚拟字段”存在，即不存在doc value中。需要注意的是analyzer，这个对推荐效果有关键的影响作用。例如： 28 | 29 | ```json 30 | {"analysis": { 31 | "analyzer": { 32 | "suggest_name_synonyms": { 33 | "type": "custom", 34 | "tokenizer": "standard", 35 | "filter": [ "name_synonyms","lowercase","myNGramFilter" ] 36 | }}}} 37 | ``` 38 | 39 | tokenizer为分词器定义，filter中指定了所用的filter，有同义词的filter，转换大小写，边界词filter。可以参照ES的官方文档进行进一步理解。同义词filter支持定义对应的同义词，但是同义词的更新需要做reindex操作。 40 | 这里使用的是standard的分词器，如果需要同时支持中英文，需要安装特定的插件，例如官方的[smartcn](https://www.elastic.co/guide/en/elasticsearch/plugins/5.1/analysis-smartcn.html)就是个不错的选择。 41 | 42 | 有时我们需要做跨字段的推荐或者搜索，可以利用field的copy_to属性： 43 | 44 | ```json 45 | {"systemField": { 46 | "properties": { 47 | "name":{ 48 | "type":"keyword", 49 | "copy_to": ["name_suggest"] 50 | } 51 | } 52 | }} 53 | ``` 54 | 55 | 这样类似实现了字段的聚合。将多个字段聚合到一个可以支持complete、同义词搜索的字段了。 56 | 至于fuzzy Search可以在获取suggest的时候指定： 57 | 58 | ``` 59 | GET /storessearch/_suggest 60 | { 61 | "product": { 62 | "text": "myname", 63 | "completion": { 64 | "field": "name_suggest", 65 | "fuzzy": { 66 | "fuzziness": 2 67 | } 68 | } 69 | } 70 | } 71 | ``` 72 | 73 | ### Query DSL生成 74 | ES的[Java Client](https://www.elastic.co/guide/en/elasticsearch/client/java-api/current/java-compound-queries.html)支持创建Query DSL: 75 | 76 | ``` 77 | QueryBuilder qb = boolQuery() 78 | .must(termQuery("content", "test1")) 79 | .must(termQuery("content", "test4")) 80 | .mustNot(termQuery("content", "test2")) 81 | .should(termQuery("content", "test3")) 82 | .filter(termQuery("content", "test5")); 83 | ``` 84 | 85 | 但是这种创建复杂些的DSL就显得力不从心了，比如文档结构复杂点的，有nest path，也有child、parent关系的，如果用Java Client来写代码会非常复杂难懂，也不容易维护。 86 | 考虑到DSL也是json结构，可以写些helper class用来创建这种json结构，代码大概类似这种： 87 | 88 | ```java 89 | public static class SimpleQueryDSLBuilder { 90 | JsonObject match = new JsonObject(); 91 | JsonObject query = new JsonObject(); 92 | JsonObject matchMeta = new JsonObject(); 93 | JsonArray source = new JsonArray(); 94 | 95 | public SimpleQueryDSLBuilder addMatch(String fieldName, JsonPrimitive primitive) { 96 | this.matchMeta.add(fieldName, primitive); 97 | return this; 98 | } 99 | 100 | public SimpleQueryDSLBuilder addSource(JsonPrimitive primitive) { 101 | this.source.add(primitive); 102 | return this; 103 | } 104 | 105 | public JsonObject build() { 106 | query.add("_source", source); 107 | match.add("match", matchMeta); 108 | query.add("query", match); 109 | log.debug("query json: " + query.toString()); 110 | return query; 111 | } 112 | } 113 | ``` 114 | 115 | 这种生成DSL的方式跟Java Client的方式，大同小异，当查询非常复杂的时候，代码量也就上去了。 116 | 由此看来生成通过代码生成DSL的方式不太可行。有什么好的解决办法呢？答案就是使用模板引擎的方式。twig模板的语法非常适合做这件事，[这篇文章](https://amsterdam.luminis.eu/2016/07/25/the-new-elasticsearch-java-rest-client-part-2/)也给出了些例子。 117 | 118 | 实际使用的是[pebble](http://www.mitchellbosecke.com/pebble/home)模板引擎，语法是受twig启发。这样修改query语句时需要修改模板即可，代码只需要用这些模板生成对应的Query DSL即可。写任复杂的Query DSL也毫无压力。 119 | 例如定义一些base query： 120 | 121 | ``` 122 | {% macro terms_query(meta, isStr="false") %} 123 | {"terms":{ 124 | "{{meta.key}}": [ 125 | {% for value in meta.value %} 126 | {% if meta.valueType == "STRING" or meta.valueType == "DATE" or isStr == "true" %} "{{value}}" {% else %} {{value}} {% endif %} {% if loop.index != loop.length-1 %}, {% endif %} 127 | {% endfor %} 128 | ] 129 | }} 130 | {% endmacro %} 131 | 132 | {% macro wildcard_query(meta) %} 133 | {"wildcard":{ 134 | "{{meta.key}}": { 135 | "value": "{{meta.value|first}}" 136 | } 137 | }} 138 | {% endmacro %} 139 | 140 | {% macro prefix_query(meta) %} 141 | {"prefix":{ 142 | "{{meta.key}}": { 143 | "value": "{{meta.value|first}}" 144 | } 145 | }} 146 | {% endmacro %} 147 | 148 | {% macro channel(channelId) %} 149 | "must": [{"nested": { 150 | "path": "channels", 151 | "query": { 152 | "bool": { 153 | "must": [ 154 | { 155 | "match": { 156 | "channels.id": {{ channelId }} 157 | } 158 | } 159 | ] 160 | } 161 | } 162 | } 163 | }] 164 | {% endmacro %} 165 | ``` 166 | 167 | 这些base的query都被定义成了macro，可以被任意组合复用，从而为组成更复杂的query提供了基础，另外也变得非常灵活。只需要Java代码组织好所用的query data，当然这些都是基于业务需要。 168 | 169 | 参考： 170 | 1. https://www.oreilly.com/ideas/10-elasticsearch-metrics-to-watch 171 | 2. http://dogdogfish.com/guide/building-a-search-engine-for-e-commerce-with-elasticsearch/ 172 | 3. http://rea.tech/implementing-autosuggest-in-elasticsearch/ 173 | 174 | -------------------------------------------------------------------------------- /notes/basic&sourcestart.md: -------------------------------------------------------------------------------- 1 | ## ElasticSearch 深入理解 一：基础概念&源码启动 2 | 3 | Elasticsearch 是一个开源的全文检索、分析引擎，具有高伸缩性。使用广泛：Wikipedia、Stack Overflow、GitHub 等都基于 Elasticsearch 来构建他们的搜索引擎。 4 | 项目组用了将近两个月的时间将其做上线，包括大半个月的原型验证，中间经历了不少的挑战，这里打算一并总结归档下。开篇就从基础的几个概念开始，并准备好环境， 5 | 看看如何源码启动，从Idea IDE里面直接启动ES，方便调试分析。没有特别说明ElasticSearch的版本都是基于5.0.0。 6 | 7 | ### 几个基础概念 8 | 9 | #### Inverted index 10 | 如下图所示 11 | ![invertindex](./invertindex.png) 12 | 13 | 三个文档经过一系列的处理生成的倒排索引。 14 | Inverted index 就是建立单词到文档的映射关系，通过单词检索能很快的找到对应的文档，类似于词典的功能。 15 | 单词即terms是检索的基本单位，可以任意的组装，进行文档的全文检索或者精确匹配，ElasticSearch有对应的配置支持。 16 | 例如需要全文检索的字段可以将type定义为text，精确匹配的字段type定义为keyword即可。 17 | 18 | #### Index 19 | Index 在ElasticSearch里面既是动词也是名词。指具有相关性Document的集合；ES将数据存储于一个或多个索引中， 20 | 类比传统的关系型数据库领域来说，索引相当于SQL中的一个数据库，或者一个schema。 21 | 索引由其名称(必须为全小写字符)进行标识，并通过引用此名称完成文档的创建、搜索、更新及删除操作。 22 | 23 | #### Type 24 | 类型是索引内部的逻辑分区(category/partition)，然而其意义完全取决于用户需求。因此，一个索引内部可定义一个或多个类型(type)。一般来说，类型就是为那些拥有相同的域的文档做的预定义。 25 | 类比传统的关系型数据库领域来说，类型相当于“表”。 26 | 27 | #### Document 28 | Document是索引和检索的基本单位，JSON 格式存储。Document存储时必须赋予一个特定的type。 29 | 也就是API路径是index/type/document 30 | 31 | #### Mapping 32 | 规定了字段类型，需要在创建index 的时候指定。Document在索引时，首先会进行字段分析，不同的字段对后续的Search&Filter 影响较大； 33 | 比如是否参与全文检索，或者只是精确匹配，对应分别是text和keyword类型。 34 | 35 | ### 集群相关 36 | 37 | #### Cluster 38 | 集群里的每个服务器则被称为一个节点（node）。可以通过索引分片将海量数据进行分割并分布到不同节点。 39 | 通过副本可以实现更强的可用性和更高的性能。这些服务器被统称为一个集群（cluster）。 40 | 41 | #### Node 42 | 运行了单个实例的ES主机称为节点，它是集群的一个成员，可以存储数据、参与集群索引及搜索操作。节点通过为其配置的ES集群名称确定其所要加入的集群。 43 | Node本身也有自己的名字和类型，默认集群中Node之间是对等的。 44 | 45 | ### Shard & Replica 46 | 当需要存储大规模文档时，由于RAM空间、硬盘容量等的限制，仅使用一个节点是不够的。另一个问题是一个节点的计算能力达不到所期望的复杂功能的要求。 47 | 在这些情况下，可以将数据切分，每部分是一个单独的Apache Lucene索引，称为分片（shard）。每个分片可以存储在集群的不同节点上。 48 | 当需要查询一个由多个分片构成的索引时，ElasticSearch将该查询发送到每个相关的分片，并将结果合并。这些过程对具体应用而言是透明的，无须知道分片的存在。 49 | 50 | index 是一个逻辑命名空间，shard 是具体的物理概念，建索引、查询等都是具体的shard在工作。shard是ElasticSearch基本的 51 | 调度单位，shard包括primary shard和replica shard，写数据时，先写到primary shard，然后， 52 | 同步到replica shard，查询时，primary和replica充当相同的作用。 53 | replica shard可以有多份，也可以没有，replica shard的存在有两个作用，一是容灾，如果primary shard挂了， 54 | 数据也不会丢失，集群仍然能正常工作，ElasticSearch会把存活的replica shard提升为primary shard； 55 | 二是提高性能，因为replica和 primary shard都能处理查询，**需要注意的是index创建在多个节点之间是传播的**。 56 | shard数只能在建立index时设置，后期不能更改，但是，replica数可以随时通过ElasticSearch提供的API更改。 57 | 58 | ### 源码启动 59 | 这里使用intellij idea IDE，ElasticSearch源码使用[gradle](https://gradle.org/)构建，所以需要本地开发环境安装gradle。 60 | 61 | 1. checkout ElasticSearch的源码：https://github.com/elastic/elasticsearch 62 | 2. 切换到5.0.0版本：切换到源码目录里面执行： git checkout v5.0.0 63 | 3. 执行gradle任务导入idea IDE：gradle idea 64 | 4. 打开idea新建project from exist source，导入即可。然后通过命令行进行构建也行idea里面进行也可以：gradle build 65 | 5. main函数入口在Elasticsearch.java里面；idea运行前需要设置个JVM参数，例如 -Des.path.home=C:\elasticsearch-5.0.0\elasticsearch-5.0.0。 66 | 67 | 这个目录下需要有ElasticSearch运行的一些配置文件：包括elasticsearch.yml，jvm.options，log4j2.properties。初始没有可以从安装包里面取一份； 68 | 也可以从build结果里面取一份。 69 | 70 | 接下来直接启动运行即可。需要注意的可能会报权限问题导致启动失败，索性直接在java.policy文件中添加一行即可。 71 | ``` 72 | grant { 73 | permission java.security.AllPermission "", ""; 74 | ... 75 | ``` 76 | 77 | -------------------------------------------------------------------------------- /notes/cap.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/compasses/elastic-rabbitmq/3c70d3a7ea310e0411b828452c79787737bed99f/notes/cap.png -------------------------------------------------------------------------------- /notes/cluster.xml: -------------------------------------------------------------------------------- 1 | dZHBEoIgEIafhrvCjKNns7p08tCZAJEJXQdxtJ4+DcwYiwOzfP/u/rAgkjfTydCuvgAXGuGIT4gcEMZZSuZ9AQ8HksQDaRR3KN5AqZ7Cw8jTQXHRB4kWQFvVhZBB2wpmA0aNgTFMq0CHrh2VYgdKRvWeXhW3taMpTjZ+FkrWq3OcZE65UXaXBobW+yFMqvdyckPXXv6hfU05jF+IFIjkBsC6qJlyoZfRrmNzdcc/6ufeRrT2R8EcbL3nQ/B/pHgB -------------------------------------------------------------------------------- /notes/cluster_deploy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/compasses/elastic-rabbitmq/3c70d3a7ea310e0411b828452c79787737bed99f/notes/cluster_deploy.png -------------------------------------------------------------------------------- /notes/cluster_relateddesign.md: -------------------------------------------------------------------------------- 1 | ## ElasticSearch 深入理解 三：集群部署设计 2 | 3 | ElasticSearch从名字中也可以知道，它的Elastic跟Search是同等重要的，甚至以Elastic为主要导向。 4 | Elastic即可伸缩性，作为一个分布式的搜索引擎，可伸缩性是它的核心竞争力。 5 | 6 | ### Elastic&Shard 7 | 8 | 这两个词能放在在一起，因为ES的Elastic是建立在Shard的基础上。Shard作为ES伸缩性的基本单元，可以在不同的node之间迁移。例如我一个实际测试的例子： 9 | 两个节点的时候，一个index配置15个shard，1个replica： 10 | 11 | ![twonodes.png](./twonodes.png) 12 | 13 | 每个node都有15个shard，互为备份。 14 | 这时再增加一个节点： 15 | 16 | ![threenodes.png](./threenodes.png) 17 | 18 | ES会做一次rebalancing，这次每个node都平均分了10个shard。但是ES的shard rebalancing有限制，就是shard本身无法split，例如： 19 | 20 | ![shardsplit.png](./shardsplit.png) 21 | 22 | 一个index有5个shard，scale out到5个node的时候就是每个node一个shard，再增加一个node不会起作用了。当然shard split可以通过重新index，增加shard数量来解决。 23 | 24 | 每个shard都会有个主分片，副分片的个数由replica确定。对Document产生修改的操作，例如create update delete，都会路由到主分片上，路由的动作由集群中的一个node完成，这个node也称为coordinator。如果使用了ES的java Client用户的request会以round-robin的方式发给集群中的节点，自动实现了负载均衡。 25 | 26 | 这里有个重要的东西，就是Cluster的一致性，ES通过一个ClusterState结构来确定，ClusterState有一个版本号，在node之间进行同步。支持Diff，用于增量的更新。 27 | 28 | ### Network Model 29 | 在启用一个Cluster之前，需要了解Cluster内部各个node之间的网络连接情况。ElasticSearch集群中的各个node之间的连接是全连接的网络结构： 30 | 31 | ![full-mesh-double](./full-mesh-double.png) 32 | 33 | 通过TCP通信，而实际上node之间默认情况下会建立13个TCP连接，分别是： 34 | 35 | connectionsPerNodeRecovery = 2 36 | connectionsPerNodeBulk = 3 37 | connectionsPerNodeReg = 6 38 | connectionsPerNodeState = 1 39 | connectionsPerNodePing = 1 40 | 41 | 通过定义能够知道大概每个连接的目的是什么了。Ping的那个链接在不同类型node上的行为是不同的，例如如果节点是个master节点ping消息发送的是node fault detection，如果节点是node节点则发送的ping消息是master fault detection。这种类似于心跳消息，通过wireshark抓包可以看到node的ping消息发送还是非常频繁的。当然这个心跳消息发送频率可以通过ping_interval修改，默认是1s。ElasticSearch的集群最好不跨局域网，因为集群的稳定性大大依赖网络的稳定性和效率。 42 | 43 | ### Split-brain problem 44 | ElasticSearch作为一个分布式的搜索引擎，一大挑战就是集群中的一致性问题。在上篇文章里面说了在高并发下如何通过ES的乐观锁机制保证数据的一致性的，但是主要是从业务场景出发，并没有太多考虑集群中的一致性问题，因为那是ES本身要解决的。 45 | 46 | 有很多的一致性算法，例如ETCD使用的Raft，从协议层面上保证一致性。ES使用的是自己的一套一致性算法，称为zen discovery。在做一致性同步的同时也会做一些ES本身的数据同步，提高效率，减少带宽。它主要完成以下几个功能： 47 | 48 | 1. Master节点选举：首先会根据节点的StateVersion，较高的称为master，如果相等，再check node的type，是否能做master，然后进行比较节点的ID，较小的选为master。 49 | 2. FaultDetection：master会给所有的node发送node fault detection检测包；普通节点会给master节点发送master fault detection检测包。 50 | 3. Membership管理: 节点加入退出等相关处理。 51 | 4. Cluster State publish：集群状态的发布，自带版本号用于一致性的状态更新。 52 | 5. Unicast Ping：单播Ping包，ES目前已不推荐使用多播做集群发现了，因为不好控制也不安全。单播可以精确控制节点数，使伸缩性变得简单。一个重要的配置项是 ``` discovery.zen.ping.unicast.hosts: ["host1", "host2:port"] ```，指定了ES实例所在的机器。 53 | 54 | #### 几个重要配置 55 | 对于ElasticSearch如何发生Split-brain，[这篇文章](http://blog.trifork.com/2013/10/24/how-to-avoid-the-split-brain-problem-in-elasticsearch/)清晰地做了解释。 56 | 为了防止出现Split-brain问题，ES有几个配置在集群部署时需要格外注意： 57 | 58 | cluster.name: elasticsearch_production //集群的名字，不同名字不能组成一个Cluster 59 | node.name: elasticsearch_005_data //该节点的名字 60 | discovery.zen.minimum_master_nodes: 2 //对防脑裂起到关键作用。需要设置为quorum值，根据node节点的数量：(number of master-eligible nodes / 2) + 1 61 | 62 | 针对recovery有几个配置需要注意： 63 | 64 | gateway.recover_after_nodes: ax //hard limitation，集群正常工作的最少节点数 65 | gateway.expected_nodes: ex //集群中总共的节点数，也叫expected 66 | gateway.recover_after_time: 5m //5分钟后或者ex个节点加入后，开始recovery，看哪个先到 67 | 68 | 这些配置可以有效防止集群中的数据抖动现象，如果集群一少半存活节点就开始recovery，等后续节点online，还有重新做recovery和shard rebalancing，会浪费大量的CPU、带宽资源，严重可能导致集群短时间无法使用。 69 | 70 | ElasticSearch的CAP特性，如下图所示： 71 | 72 | ![cap](./cap.png) 73 | 74 | 任何分布式系统只能满足其中两个，ES满足其中的哪两个呢？好像没有定论，要看自己的实际应用场景了。根据不同的场景选择需要支持哪两种。例如配置minimum_master_nodes为最大数可以保证C，但A就保证不了了。实际生产环境应该都比较倾向于CA，因为P可以人为干预并做好预防。 75 | 76 | ### Cluster Design 77 | 理解了ElasticSearch的Cluster的运作机制，才能更好为业务做出合适的部署方式。例如把ES当成一个底层的主数据存储组件，那就需要高一致性，然后可用性其次；如果ES只是用来做辅助功能例如加速Search，那么高可用性就是首要解决的了。 78 | 79 | 例如下面的设计： 80 | 81 | ![cluster_deploy](./cluster_deploy.png) 82 | 83 | 注意点： 84 | 1. 三节点应该是最少的集群节点配置了，能保证较好的稳定性。 85 | 2. quorum值必须是一半以上节点的数目，所以三个节点的话，quorum值必须是。按照官方的说法这个设置能防止脑裂问题。 86 | 3. 这个集群的配置对Search是友好的，就是说down了两个节点的话就不允许写入了，而Search依然可以进行。 87 | 4. 针对写入的场景每个节点都是一个coordinator node，它会把index请求发给primary shard上。而Search不区分primary shard和replica shard。 88 | 5. 分片的load balance是ElasticSearch自动完成的，有的时候并不一定保证均衡。每个分片的大小也是不能保证均衡的。 89 | 90 | 参考： 91 | 1. https://www.elastic.co/blog/every-shard-deserves-a-home 92 | 2. https://www.elastic.co/blog/writing-your-own-ingest-processor-for-elasticsearch 93 | 3. https://www.elastic.co/blog/found-elasticsearch-networking 94 | 4. https://www.elastic.co/blog/found-keeping-elasticsearch-in-sync 95 | 5. https://www.elastic.co/blog/found-sizing-elasticsearch 96 | 6. https://www.elastic.co/blog/found-elasticsearch-top-down 97 | 7. https://www.elastic.co/blog/found-elasticsearch-from-the-bottom-up#index-segments 98 | 8. https://www.elastic.co/blog/found-crash-elasticsearch#mapping-explosion 99 | 9. https://blog.insightdatascience.com/anatomy-of-an-elasticsearch-cluster-part-i-7ac9a13b05db#.w2skctg2l 100 | 101 | -------------------------------------------------------------------------------- /notes/conflict1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/compasses/elastic-rabbitmq/3c70d3a7ea310e0411b828452c79787737bed99f/notes/conflict1.png -------------------------------------------------------------------------------- /notes/conflict2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/compasses/elastic-rabbitmq/3c70d3a7ea310e0411b828452c79787737bed99f/notes/conflict2.png -------------------------------------------------------------------------------- /notes/country_sel.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/compasses/elastic-rabbitmq/3c70d3a7ea310e0411b828452c79787737bed99f/notes/country_sel.png -------------------------------------------------------------------------------- /notes/elasticsearch-kibana-grafana.md: -------------------------------------------------------------------------------- 1 | ## ElasticSearch data visualization工具：Kibana and Grafana 2 | 3 | 如果能把数据的实时的可视化，应该是个很好的应用方向。比如目前用的比较多的[ELK](https://www.elastic.co/products)，做日志分析，做系统监控。中间使用Logstash或者Beats把数据收集到ElasticSearch里面，通过Kibana做查询，然后通过配置DashBoard把展示出来。 4 | 5 | 最近在用Kibana做一些数据的查询优化，发现他的可视化功能还是不错的，但是跟Grafana相比，功能上还是欠缺了些。可视化数据的需求，是方方面面的。大公司的报表，企业做BI、信息化，都离不开数据分析。如果数据能够通过各种维度展现出来，让人更好的理解数据，才能更好的利用数据。所以可视化将会是大数据分析的必然需求。 6 | 7 | 在研究怎么样更好的去检索ES时，利用了Kibana的Dev Tools并进一步了解到了他的visualize功能。因为之前也使用过Grafana，发现它也支持ES，发现其在可视化ES的数据时更加方便和自由。 8 | 9 | ### ES数据类型 10 | 数据分析跟全文检索不一样，如果字段类型是text的，ES会做分词处理，这种主要用于全文检索。数据分析时，检索的时候可能会看到不想要的结果，所以如果单纯的做可视化，最好将Mapping修改下，将字段类型改为keyword。这个目前只在用Grafana的时候有些问题，Kibana倒是问题不大。 11 | 12 | ### [Kibana](https://www.elastic.co/products/kibana) 13 | Kibana安装非常方便，官网下载，并配置下ES的URL就可以连上了。Kibana使用的时候会有副作用，它会在你的ES里面创建一个它所需要的Index，并会有较为频繁的query。如果是生产环境，ES存的是核心数据要慎用了。安装好启动指定需要分析的Index，就能看到以下几个Tab。 14 | 15 | 1. Discover，查询时用的比较多，查询日志信息的时候。可以进行任意的查询，能快速的定位问题。 16 | 2. Visualize，提供很多种类型的图展示方式。 17 | 3. Dev Tools，提供了很好的跟ES交互的工具，比Postman更方便，感觉可以不用Postman了。 18 | 4. Dashboard，Visuallize里面定义的视图可以保存到Dashboard上面，这样可以看到实时分析的结果。 19 | 20 | Kibana总体上来讲使用比较傻瓜式的，基本上用鼠标选择下就能用。Dev Tools是个很好的工具HTTP请求写起来很方便，也有历史记录。ES的Search都是Get带body的，在Dev Tools里面很方便使用。 21 | 22 | 例如： 23 | ``` 24 | GET _search 25 | { 26 | "query": { 27 | "match_all": {} 28 | } 29 | } 30 | ``` 31 | 鼠标点上GET那一行就能发出去了。手写JSON也毫不费力。 32 | 33 | ### [Grafana](https://grafana.com/) 34 | 印象中Grafana是[InfluxDB](https://www.influxdata.com/)可视化工具，但基于Grafana良好的架构，DataSource可以变得多样性了，现在是支持ElasticSearch作为DataSource的。 35 | Grafana的安装也非常简单，解压启动就可以了。DataSource都是动态添加上去的。Grafana作为一个纯粹的可视化工具，灵活性做到了极致。使用ElasticSearch作为DataSource，指定对应的Index，就可以建立DashBoard就行数据可视化了。相比Kibana，Grafana的template功能能把数据按照任意的维度进行切分展示，这就是它的强大之处。 36 | 37 | Grafana需要index有个时间序列的字段，这个其实不难，正常的数据都会有个时间字段的。下面举个例子，有一批专利申请的数据，通过某种手段导入到ES中。例如以专利申请的国家和类型进行切分的话，那就要建立两个模板： 38 | 国家的模板，query中填写： 39 | ``` 40 | {"find": "terms", "field": "国家"} 41 | ``` 42 | 专利类型的模板，query中填写： 43 | ``` 44 | {"find": "terms", "field": "专利类型"} 45 | ``` 46 | 然后添加panel类型为Graph，以“发明人”字段作为聚合数据。edit panel中的query填写： 47 | ``` 48 | 国家:$country AND 专利类型:$type 49 | ``` 50 | graph左上角就会有两个选择列表： 51 | 52 | ![country_sel.png](./country_sel.png) 53 | 54 | ![type_sel.png](./type_sel.png) 55 | 56 | 例如发明人的panel展示： 57 | 58 | ![invent_show.png](./invent_show.PNG) 59 | 60 | 可以随意选择维度就行分析。 61 | 62 | Grafana还有较为完善的用户管理，这也是强于Kibana的地方。 63 | 64 | ### Summary 65 | 不管是Kibana还是Grafana，都会容易联想到系统监控，日志分析。没错这是它们的强项，不管事序列化的数据还是Metrics Data都能很好的展示出来，让对应的DevOps做监控预警等。从另一方面讲，其他的任何数据，能把数据保存到ES，配合Grafana或者Kibana都能很好的做分析展示。当然更看好Grafana，随着Grafana的功能进一步增强，会变得功能更加强大，图表也会更加多样性。下一步结合Spark或者Hadoop，做个实时的BI也是个不错的方向哦。 66 | -------------------------------------------------------------------------------- /notes/full-mesh-double.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/compasses/elastic-rabbitmq/3c70d3a7ea310e0411b828452c79787737bed99f/notes/full-mesh-double.png -------------------------------------------------------------------------------- /notes/invent_show.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/compasses/elastic-rabbitmq/3c70d3a7ea310e0411b828452c79787737bed99f/notes/invent_show.PNG -------------------------------------------------------------------------------- /notes/invertindex.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/compasses/elastic-rabbitmq/3c70d3a7ea310e0411b828452c79787737bed99f/notes/invertindex.png -------------------------------------------------------------------------------- /notes/metrics.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/compasses/elastic-rabbitmq/3c70d3a7ea310e0411b828452c79787737bed99f/notes/metrics.png -------------------------------------------------------------------------------- /notes/msa-arch-analytic.md: -------------------------------------------------------------------------------- 1 | # 微服务化技术点滴 2 | 3 | -------------------------------------------------------------------------------- /notes/optimisticlock&versionconflicthandle.md: -------------------------------------------------------------------------------- 1 | ## ElasticSearch 深入理解 二：乐观锁&版本冲突管理 2 | 3 | ES用于持久化存储数据时，没有传统数据库的事务一致性保证。因为数据的来源不是单一的，更何况我们的使用场景是通过消费RabbitMQ消息来保存数据。 4 | RabbitMQ作为一个异步消息分发组件，消息的顺序性也是没有保证的，虽然RabbitMQ能保证从生产者接收到的消息和发给消费者的消息顺序保证一致，但是在多个生产者和多个消费者的模式下，消息的顺序是没有保证的。举个一般的例子，一个Business Object的更新操作，很有很多个消息，有更新价格的，有跟新品类名称的等等，这些消息都是没有顺序保证的，有的时候也可能只来了update消息，而create消息还没到来。作为消费者，肯定是并发消费队列消息的，这样并发更新过程中就难以避免版本冲突了。 5 | 6 | ### ES版本号 7 | ES中的每个Document都会有一个内部版本号，这个版本号是Document的元数据，由ES管理的，可以通过Get拿到Document的版本号。Search默认不返回Document的版本号，可以通过添加个参数到Search Body让ES返回版本号： 8 | ``` 9 | { 10 | "version":true 11 | } 12 | ``` 13 | 版本号是从1开始递增的，但是partial update接口不一定会递增版本号，如果数据没有发送更新的话。 14 | 版本号更新是atomic的。 15 | 16 | ### ES乐观锁 17 | ES的乐观锁建立在ES版本号的管理机制之上。乐观锁对应的就是悲观锁。SQL数据库例如MariaDB在满足事务一致性时，需要对所处理的数据提前加锁。对几种事务稍加总结下： 18 | 19 | 1. TRANSACTION_SERIALIZABLE: 它要求事务所涉及的所有数据都被锁定。不管是通过主键还是通过where字句表都会被锁定。这样就避免了事务进行期间数据被改动。 20 | 2. TRANSACTION_REPEATABLE_READ: 可重复读的事务一致性，要求事务进行期间所有访问的数据都被锁定，不过其他的事务可以加入新的数据，所以这种模式会出现幻读，即事务再次执行时可能会得到不同的执行结果。 21 | 3. TRANSACTION_READ_COMMITTED: 提交读的事务一致性，事务运行期间只有正在写入的行会被锁定。这种模式会发送不可重复读的问题，即在事务进行中，一个时间点读到的数据到另一个时间点再次读取时，就变得完全不同了。 22 | 4. TRANSACTION_READ_UNCOMMITTED: 未提交读的事务一致性，代价最低的事务模式，事务运行期间不会施加任何锁，会出现脏读，dirty read。一个事务可以读取另一个事务写入的数据，尽管这些数据未提交。 23 | 24 | ES的乐观锁机制不一样，suppose对某个数据的处理不会冲突，也不会加锁，通过API指定更新的版本号，如果版本号不符合就返回exception。 25 | 26 | #### Overwrite问题 27 | 如果不指定对应的版本号，就会存在overwrite的问题，导致数据丢失。因为ES通过API index数据，遵循Last write win： 28 | ![overwrite](overwrite.png) 29 | 30 | 如上图所示，两个process同时对一个Document进行name更新，process1的更新就丢失了。这是ES的机制问题，需要应用层小心对待。 31 | 32 | ### Conflict Handle 33 | Overwrite问题需要解决，特别把ES当作一个存储层来用，数据的一致性非常重要，是不能被Overwrite的。根据消费RabbitMQ的应用场景会存在聚合的场景，即多个不同类型的message，即routing key不同，需要aggregate到一个Document的情况，以Product作为主的Business Object为例。 34 | 35 | Conflict是ES的版本号管理机制，但其不管心哪个字段产生了Conflict，有的时候是相同的字段，有的时候是不同的字段，如果能统一去handle Conflict就好了。解决办法是有的，在应用层代码提供handle Conflict的能力。 36 | 当然Conflict产生需要有个前提条件，就是要指定更新Document的版本号，如果不指定就会直接被Overwrite了，也不会报409 Conflict的错误。指定版本号很简单，只需要在API URL拼接一个version参数：**version=XX**，这个version就是这次API操作的Document的版本号，如果不是ES就会返回409 Conflict。 37 | 不同字段： 38 | 39 | ![conflict1.png](./conflict1.png) 40 | 41 | 两个线程同时对一个product更新，一个更新的是category信息，另一个更新channel信息，由此产生了Conflict。 42 | 43 | 相同字段： 44 | 45 | ![conflict2.png](./conflict2.png) 46 | 47 | 两个线程同时对product上的channel进行更新，经过Conflict handle后正确写入。 48 | 49 | ### Consistent Base On Async Event Message 50 | 所以说最关键的就是这个问题，保证数据的consistent，而且是在异步高并发消息处理过程中。根据业务流程，不同类型的消息由不同的handler处理，每个handler要实现自己的Conflict Handler方法。 51 | 52 | 例如一般化的Conflict Handler处理： 53 | ``` 54 | protected BiFunction conflictFunction() { 55 | return (JsonObject currentObj, JsonObject newObj) -> { 56 | if (currentObj.get("version").getAsLong() > newObj.get("version").getAsLong()) { 57 | return currentObj; 58 | } else { 59 | return newObj; 60 | } 61 | }; 62 | } 63 | ``` 64 | 65 | 代码里面适用的是这些BO对象本身也有版本号。 66 | 67 | 带后Conflict Handle的ES update 流程： 68 | ``` 69 | try { 70 | //In case the arguments pollute 71 | HashMap localParam = new HashMap<>(); 72 | if (param != null) { 73 | localParam.putAll(param); 74 | } 75 | if (localParam.get("version") != null && localParam.get("version").equals("-1")) { 76 | localParam.remove("version"); 77 | return createSource(index, type, sourceId, localParam, newObj); 78 | } else if (localParam.get("version") != null) { 79 | logger.debug("do update with version" + localParam + " source=" + newObj.toString()); 80 | return updateOnInsert(index, type, sourceId, localParam, newObj); 81 | } else { 82 | ESGetByIdResponse esResponse = documentService.loadSourceById(index, type, sourceId, localParam); 83 | if (esResponse == null || esResponse.getFound() == false) { 84 | return createSource(index, type, sourceId, localParam, newObj); 85 | } else { 86 | logger.debug("Retrieval source " + esResponse.toString()); 87 | localParam.put("version", esResponse.getVersion().toString()); 88 | return updateOnInsert(index, type, sourceId, localParam, newObj); 89 | } 90 | } 91 | } catch (ElasticVersionConflictException e) { 92 | try { 93 | successful = Status.CONFILICT; 94 | //conflict 95 | //In case the arguments pollute 96 | HashMap localParam = new HashMap<>(); 97 | if (param != null) { 98 | localParam.putAll(param); 99 | } 100 | logger.warn("Conflict happen! type = " + type + " sourceId = " + sourceId); 101 | TimeUnit.MICROSECONDS.sleep(300); 102 | 103 | //need retrieve new version 104 | localParam.remove("version"); 105 | ESGetByIdResponse esResponse = documentService.loadSourceById(index, type, sourceId, localParam); 106 | if (esResponse == null) { 107 | logger.error("impossible is happen! need check data " + " type:" + type + " id" + sourceId); 108 | throw new IllegalStateException(); 109 | } 110 | 111 | JsonObject changedObj = f.apply(esResponse.getObject(), newObj); 112 | localParam.put("version", esResponse.getVersion().toString()); 113 | return updateOnInsert(index, type, sourceId, localParam, changedObj); 114 | ``` 115 | 116 | 如果BO是首次创建，就是Create的流程：parameter里面要添加参数**params.put("op_type", "create")**，如果不是就走updateOnInsert流程，updateOnInsert就是调用ES的partial update API并如果不存在的情况下就插入新的。 117 | 118 | 上述的代码也有一个缺陷，例如级联Conflict的情况。就是这次Conflict了，Conflict处理后，又产生了Conflict，这种情况可以多次catch，也可以做记录到另一个数据表中，后面统一进行恢复处理。 119 | 120 | 参考文档： 121 | 122 | 1. https://www.elastic.co/blog/elasticsearch-versioning-support -------------------------------------------------------------------------------- /notes/overwrite.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/compasses/elastic-rabbitmq/3c70d3a7ea310e0411b828452c79787737bed99f/notes/overwrite.png -------------------------------------------------------------------------------- /notes/profiling.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/compasses/elastic-rabbitmq/3c70d3a7ea310e0411b828452c79787737bed99f/notes/profiling.gif -------------------------------------------------------------------------------- /notes/shardsplit.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/compasses/elastic-rabbitmq/3c70d3a7ea310e0411b828452c79787737bed99f/notes/shardsplit.png -------------------------------------------------------------------------------- /notes/threenodes.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/compasses/elastic-rabbitmq/3c70d3a7ea310e0411b828452c79787737bed99f/notes/threenodes.png -------------------------------------------------------------------------------- /notes/tokenizer.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/compasses/elastic-rabbitmq/3c70d3a7ea310e0411b828452c79787737bed99f/notes/tokenizer.png -------------------------------------------------------------------------------- /notes/tokenizer.xml: -------------------------------------------------------------------------------- 1 | 7VlNc9owEP01PpbBNjbkGGhoL53pDJ1pehSybGsiW1QWGPrru7IkG3/QCZN0SBtziKWn1Uq7762QieOvsuMngXbpFx4R5njT6Oj4Hx3Pc2deCA+FnDRyF3oaSASNNDRtgA39RcxMi+5pRAqDaUhyziTdtUHM85xg2cKQELxsm8WcRS1ghxLSAzYYsT76nUYy1ejChqXwz4QmqV3ZDe/0yBbhp0TwfW7Wczw/rj56OEPWlwm0SFHEyzPIf3D8leBc6lZ2XBGmcttO2/rCaL1vQXL5nAmGlgNiexN6yUXkeCGD6cuIHqCZqCY8UbaDRr4t1KOQgqDMGsICLdsqNHmy6YQ1gTnoLMuUSrLZIaxGShAPYKnMGPRcaKJip+mM6ZHAFpcMbQlb1kldccZF5dOmtdoJfyJnI1P4+H49YgmE3C5jypi1zHmudpQIFFHIVgeOeS6NLkGUuj+0BmI0yQHD4IKIOvgDEZIcL5Li1lRDCRGeESlOYGIneEZNpnrcwPTLRovBQkPpmQwthoz6k9pzowBoGBEMCyLoCcJyjNRqgsQVGv7cK4kCd1KV5L3y763LspwQhgpJ8QRzAKo6hifJ1R89UhAkcAp98EUgJVgZ4L2osuOtUY7YqaDFB5wiAXxBVotJJZF6UXUaUGnEZcEVmCOsOPCmaz3tfIZWJm6MauGiM4kPqP6/iT2uBy5GvhVdZCzs1yxscH+zwg7fTGFL4C4HBp6n7W+1dV/StadhUY9ifYlYfX9+M7HO35ZYrziKK7n+6RiW2uBNf/28etDd2+P4XXTz8p7NbnfJXAyUd5fNPLpXL3IqahBvQXGbxEFC1useIe4VaYbMidMjgFPb+aE6k6DOOIl6742dfEMQfC8wab1eSSQSYvM3zMp51qcDWTeYgEqW9NDexBAVZoWvnKrSbt4sFi3SvUWHTb15M+v8vbHraN5x5Hcc6ZB7jipl1GE/Syx3/55YyJHKR+NPtV8qo6Avo3CU0XUysj8tvWcdhX0dzUcdXakjt6ejAkEG+rc3qq5ED5vx+vI3ri9BGNzs+uL2fzV9d0fJvH+UuBcYG88SVnebX/a1efPvE//hNw== -------------------------------------------------------------------------------- /notes/twonodes.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/compasses/elastic-rabbitmq/3c70d3a7ea310e0411b828452c79787737bed99f/notes/twonodes.png -------------------------------------------------------------------------------- /notes/type_sel.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/compasses/elastic-rabbitmq/3c70d3a7ea310e0411b828452c79787737bed99f/notes/type_sel.png -------------------------------------------------------------------------------- /rabbitmqservice/.gitignore: -------------------------------------------------------------------------------- 1 | .gradle 2 | gradle 3 | gradlew 4 | 5 | build/ 6 | 7 | # Ignore Gradle GUI config 8 | gradle-app.setting 9 | 10 | # Avoid ignoring Gradle wrapper jar file (.jar files are usually ignored) 11 | !gradle-wrapper.jar 12 | 13 | # Cache of project 14 | .gradletasknamecache 15 | 16 | # # Work around https://youtrack.jetbrains.com/issue/IDEA-116898 17 | # gradle/wrapper/gradle-wrapper.properties 18 | -------------------------------------------------------------------------------- /rabbitmqservice/build.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * This build file was auto generated by running the Gradle 'init' task 3 | * by 'I311352' at '12/7/16 5:12 PM' with Gradle 3.2.1 4 | * 5 | * This generated file contains a sample Java project to get you started. 6 | * For more details take a look at the Java Quickstart chapter in the Gradle 7 | * user guide available at https://docs.gradle.org/3.2.1/userguide/tutorial_java_projects.html 8 | */ 9 | 10 | // Apply the java plugin to add support for Java 11 | apply plugin: 'java' 12 | 13 | buildscript { 14 | repositories { 15 | mavenCentral() 16 | } 17 | dependencies { 18 | classpath("org.springframework.boot:spring-boot-gradle-plugin:1.4.2.RELEASE") 19 | } 20 | } 21 | 22 | // In this section you declare where to find the dependencies of your project 23 | repositories { 24 | // Use 'jcenter' for resolving your dependencies. 25 | // You can declare any Maven/Ivy/file repository here. 26 | mavenCentral() 27 | mavenLocal() 28 | maven { 29 | url 'http://nexus.smec/nexus/content/groups/allrepository' 30 | } 31 | } 32 | 33 | dependencies { 34 | compile group: 'org.springframework.amqp', name: 'spring-amqp', version: '1.6.6.RELEASE' 35 | compile 'org.springframework.amqp:spring-rabbit:1.6.6.RELEASE' 36 | compile 'com.rabbitmq:amqp-client:4.0.0' 37 | } 38 | 39 | // In this section you declare the dependencies for your production and test code 40 | -------------------------------------------------------------------------------- /rabbitmqservice/settings.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * This settings file was auto generated by the Gradle buildInit task 3 | * by 'I311352' at '12/7/16 5:18 PM' with Gradle 3.2.1 4 | * 5 | * The settings file is used to specify which projects to include in your build. 6 | * In a single project build this file can be empty or even removed. 7 | * 8 | * Detailed information about configuring a multi-project build in Gradle can be found 9 | * in the user guide at https://docs.gradle.org/3.2.1/userguide/multi_project_builds.html 10 | */ 11 | 12 | /* 13 | // To declare projects as part of a multi-project build use the 'include' method 14 | include 'shared' 15 | include 'api' 16 | include 'services:webservice' 17 | */ 18 | 19 | rootProject.name = 'rabbitmqservice' 20 | -------------------------------------------------------------------------------- /rabbitmqservice/src/main/java/rabbitmq/ListenerJobBuilder.java: -------------------------------------------------------------------------------- 1 | package rabbitmq; 2 | 3 | import org.springframework.amqp.rabbit.connection.CachingConnectionFactory; 4 | import org.springframework.amqp.rabbit.connection.ConnectionFactory; 5 | import org.springframework.context.ConfigurableApplicationContext; 6 | import org.springframework.jca.cci.connection.SingleConnectionFactory; 7 | import sync.listener.ESMessageListener; 8 | 9 | /** 10 | * Created by I311352 on 12/28/2016. 11 | */ 12 | 13 | public class ListenerJobBuilder { 14 | // just everything under default 15 | public static MQListenerAdmin buildMQListenerAdmin(ConfigurableApplicationContext context) { 16 | MessageListenerProxy listenerProxy = new MessageListenerProxy(new ESMessageListener(context), 17 | "Elastic_Queue"); 18 | return new MQListenerAdmin(getConnectionFactory(), listenerProxy); 19 | } 20 | 21 | public static ConnectionFactory getConnectionFactory() { 22 | CachingConnectionFactory connectionFactory = new CachingConnectionFactory("10.128.165.206"); 23 | connectionFactory.setUsername("guest"); 24 | connectionFactory.setPassword("guest"); 25 | return connectionFactory; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /rabbitmqservice/src/main/java/rabbitmq/MQListenerAdmin.java: -------------------------------------------------------------------------------- 1 | package rabbitmq; 2 | 3 | import org.aopalliance.aop.Advice; 4 | import org.apache.log4j.Logger; 5 | import org.springframework.amqp.core.*; 6 | import org.springframework.amqp.rabbit.connection.ConnectionFactory; 7 | import org.springframework.amqp.rabbit.core.RabbitAdmin; 8 | import org.springframework.amqp.rabbit.listener.SimpleMessageListenerContainer; 9 | 10 | /** 11 | * Created by I311352 on 12/28/2016. 12 | */ 13 | 14 | public class MQListenerAdmin { 15 | private static final Logger logger = Logger.getLogger(MQListenerAdmin.class); 16 | private final ConnectionFactory mqFactory;; 17 | private final RabbitAdmin amqpAdmin; 18 | private final SimpleMessageListenerContainer container; 19 | private Boolean startSucceeded = false; 20 | private Queue queue; 21 | private final MessageListener listener; 22 | 23 | public MQListenerAdmin(ConnectionFactory mqFactory, MessageListener listener) { 24 | container = new SimpleMessageListenerContainer(); 25 | this.mqFactory = mqFactory; 26 | amqpAdmin = new RabbitAdmin(mqFactory); 27 | this.listener = listener; 28 | } 29 | 30 | public void start() { 31 | declareQueue(); 32 | 33 | container.setAcknowledgeMode(AcknowledgeMode.AUTO); 34 | container.setAutoStartup(false); 35 | container.setChannelTransacted(false); 36 | container.setConcurrentConsumers(MQParsedConfig.conncurrentConsumers); 37 | container.setConnectionFactory(mqFactory); 38 | container.setPrefetchCount(1); 39 | container.setQueueNames(queue.getName()); 40 | // container.setTransactionManager(new 41 | // RabbitTransactionManager(mqFactory)); 42 | container.setMessageListener(listener); 43 | container.setDefaultRequeueRejected(true); 44 | container.start(); 45 | startSucceeded = true; 46 | } 47 | 48 | private void declareQueue() { 49 | amqpAdmin.declareExchange(new TopicExchange(MQParsedConfig.exchangeName)); 50 | queue = new Queue("ElasticRabbit"); 51 | amqpAdmin.declareQueue(queue); 52 | 53 | logger.info("Using queue name:" + queue.getName()); 54 | 55 | String[] routingKeys = MQParsedConfig.routingKeys; 56 | for (String routingKey : routingKeys) { 57 | amqpAdmin.declareBinding(new Binding(queue.getName(), 58 | Binding.DestinationType.QUEUE, MQParsedConfig.exchangeName, routingKey, null)); 59 | } 60 | } 61 | 62 | } 63 | -------------------------------------------------------------------------------- /rabbitmqservice/src/main/java/rabbitmq/MQParsedConfig.java: -------------------------------------------------------------------------------- 1 | package rabbitmq; 2 | 3 | import org.springframework.amqp.core.Queue; 4 | 5 | /** 6 | * Created by I311352 on 12/28/2016. 7 | */ 8 | public class MQParsedConfig { 9 | public final static String exchangeName = "SharedExchange"; 10 | public final static int inital_maxInterval = 10; 11 | public final static int multiplier = 2; 12 | public final static int max_retry = 4; 13 | public final static int max_interval = 50; 14 | public final static int conncurrentConsumers = 15; 15 | public final static String[] routingKeys = {"Product.CREATE.#", "SKU.CREATE.#", "CatalogSKU.LIST.#", "" + 16 | "Category.ASSOCIATE.#"}; 17 | } 18 | -------------------------------------------------------------------------------- /rabbitmqservice/src/main/java/rabbitmq/MessageListenerProxy.java: -------------------------------------------------------------------------------- 1 | package rabbitmq; 2 | 3 | import org.apache.log4j.Logger; 4 | import org.springframework.amqp.core.Message; 5 | import org.springframework.amqp.core.MessageListener; 6 | import sync.common.MQMessageConst; 7 | 8 | /** 9 | * Created by I311352 on 12/28/2016. 10 | */ 11 | public class MessageListenerProxy implements MessageListener { 12 | private static final Logger logger = Logger.getLogger(MessageListenerProxy.class); 13 | private final MessageListener delegate; 14 | private final String jobName; 15 | 16 | public MessageListenerProxy(MessageListener listener, String jobName) { 17 | this.delegate = listener; 18 | this.jobName = jobName; 19 | } 20 | 21 | private void beforeOnMessage(Message message) { 22 | logger.info("Get message"); 23 | this.markJobThread(message); 24 | } 25 | 26 | @Override 27 | public void onMessage(Message message) { 28 | try { 29 | beforeOnMessage(message); 30 | delegate.onMessage(message); 31 | } finally { 32 | afterOnMessage(message); 33 | } 34 | } 35 | 36 | private void afterOnMessage(Message message) { 37 | logger.info("Finish message"); 38 | this.unmarkJobThread(); 39 | } 40 | 41 | private void markJobThread(Message message) { 42 | StringBuilder sb = new StringBuilder(); 43 | sb.append("Thread_EventJob_").append(this.jobName).append("_tenant_"); 44 | Object tenantId = message.getMessageProperties().getHeaders() 45 | .get(MQMessageConst.H_TENANTID); 46 | sb.append(tenantId); 47 | Thread.currentThread().setName(sb.toString()); 48 | } 49 | 50 | private void unmarkJobThread() { 51 | Thread.currentThread().setName("Thread_EventJob_Idle_Parking"); 52 | } 53 | } -------------------------------------------------------------------------------- /rabbitmqservice/src/main/java/sync/common/ESHandleMessage.java: -------------------------------------------------------------------------------- 1 | package sync.common; 2 | 3 | import com.google.gson.JsonElement; 4 | import org.apache.commons.collections.CollectionUtils; 5 | 6 | import java.util.Arrays; 7 | import java.util.List; 8 | 9 | /** 10 | * Created by I311352 on 10/17/2016. 11 | */ 12 | 13 | public class ESHandleMessage { 14 | private Long tenantId; 15 | private String type; 16 | private OperateAction action; 17 | private byte[] msgBody; 18 | private JsonElement jsonElement; 19 | 20 | private boolean needSendEvent = true; 21 | private boolean isEventGenerated = false; 22 | 23 | private List productIds = null; 24 | private Long channelId = null; 25 | 26 | public ESHandleMessage() { 27 | } 28 | 29 | public ESHandleMessage(Long tenantId, String indexType, String action, byte[] msgBody) { 30 | this.tenantId = tenantId; 31 | this.type = indexType; 32 | this.action = OperateAction.valueOf(action); 33 | this.msgBody = msgBody; 34 | } 35 | 36 | public Long getTenantId() { 37 | return tenantId; 38 | } 39 | 40 | public void setTenantId(Long tenantId) { 41 | this.tenantId = tenantId; 42 | } 43 | 44 | public String getType() { 45 | return type; 46 | } 47 | 48 | public void setType(String type) { 49 | this.type = type; 50 | } 51 | 52 | public OperateAction getAction() { 53 | return action; 54 | } 55 | 56 | public void setAction(OperateAction action) { 57 | this.action = action; 58 | } 59 | 60 | public byte[] getMsgBody() { 61 | return msgBody; 62 | } 63 | 64 | public void setMsgBody(byte[] msgBody) { 65 | this.msgBody = msgBody; 66 | } 67 | 68 | public List getProductIds() { 69 | return productIds; 70 | } 71 | 72 | public void setProductIds(List productIds) { 73 | this.productIds = productIds; 74 | } 75 | 76 | public boolean isNeedSendEvent() { 77 | return needSendEvent; 78 | } 79 | 80 | public boolean isProductIdsEmpty() { 81 | return CollectionUtils.isEmpty(this.productIds); 82 | } 83 | 84 | public void setNeedSendEvent(boolean needSendEvent) { 85 | this.needSendEvent = needSendEvent; 86 | } 87 | 88 | public JsonElement getJsonElement() { 89 | return jsonElement; 90 | } 91 | 92 | public void setJsonElement(JsonElement jsonElement) { 93 | this.jsonElement = jsonElement; 94 | } 95 | 96 | public Long getChannelId() { 97 | return channelId; 98 | } 99 | 100 | public void setChannelId(Long channelId) { 101 | this.channelId = channelId; 102 | } 103 | 104 | public boolean isEventGenerated() { 105 | return isEventGenerated; 106 | } 107 | 108 | public void setEventGenerated(boolean eventGenerated) { 109 | isEventGenerated = eventGenerated; 110 | } 111 | 112 | @Override 113 | public String toString() { 114 | return "ESHandleMessage{" + 115 | "tenantId=" + tenantId + 116 | ", type='" + type + '\'' + 117 | ", searchcommand=" + action + 118 | ", needSendEvent=" + needSendEvent + 119 | ", isEventGenerated=" + isEventGenerated + 120 | ", productIds=" + productIds + 121 | ", channelId=" + channelId + 122 | '}'; 123 | } 124 | } 125 | -------------------------------------------------------------------------------- /rabbitmqservice/src/main/java/sync/common/MQMessageConst.java: -------------------------------------------------------------------------------- 1 | package sync.common; 2 | 3 | /** 4 | * Created by I311352 on 12/28/2016. 5 | */ 6 | public class MQMessageConst { 7 | public static final String H_MESSAGEID = "X-Message-ID"; 8 | public static final String H_TRACEID = "X-Trace-ID"; 9 | public static final String H_TENANTID = "X-Tenant-ID"; 10 | public static final String H_EMPLOYEEID = "X-Employee-ID"; 11 | public static final String H_USERID = "X-User-ID"; 12 | public static final String H_USERTYPE = "X-User-Type"; 13 | public static final String H_USER_LOCALE = "X-Locale"; 14 | public static final String H_SYSTEM_LOCALE = "X-System-Locale"; 15 | public static final String H_SESSIONID = "X-Session-ID"; 16 | public static final String H_SOURCE = "X-Source"; 17 | public static final String H_ORIGIN_USER_ID = "X-Origin-User-ID"; 18 | public static final String H_ORIGIN_EMPLOYEE_ID = "X-Origin-Employee-ID"; 19 | public static final String H_ACCESS_TOKEN = "X-Access-Token"; 20 | public static final String H_ORIGIN_USER_TYPE = "X-Origin-User-Type"; 21 | public static final String H_RO_ID = "X-Ro-ID"; 22 | public static String H_RETRIGGER_FLAG = "false"; 23 | public static final String H_EVENT_TYPE = "X-Event-Type"; 24 | public static final String H_JOBNAME = "header_jobname"; 25 | public static final String H_SIMPLEJOBNAME = "header_simplejobname"; 26 | public static final String H_IMPERSONATE = "header_impersonate"; 27 | public static final String H_STARTTIME = "header_starttime"; 28 | public static final String H_REPEATCOUNT = "header_repeatcount"; 29 | public static final String QUARTZMSGQUEUENAME = "quartz.msg.queue"; 30 | public static final String QUARTZMSGUSERDATAMAP = "header_quartzmsg_userdatamap"; 31 | public static final String H_INTERVALINSECONDS = "header_intervalInSeconds"; 32 | public static final String H_JOBDATA = "header_jobData"; 33 | public static final String H_JOBPREFIX = "header_jobPrefix"; 34 | 35 | public MQMessageConst() { 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /rabbitmqservice/src/main/java/sync/common/OperateAction.java: -------------------------------------------------------------------------------- 1 | package sync.common; 2 | 3 | /** 4 | * Created by I311352 on 10/18/2016. 5 | */ 6 | public enum OperateAction { 7 | CREATE, 8 | UPDATE, 9 | DELETE, 10 | ASSOCIATE, 11 | DISASSOCIATE, 12 | LIST, 13 | UNLIST 14 | } 15 | -------------------------------------------------------------------------------- /rabbitmqservice/src/main/java/sync/handler/ESHandler.java: -------------------------------------------------------------------------------- 1 | package sync.handler; 2 | 3 | import org.springframework.web.context.support.SpringBeanAutowiringSupport; 4 | import sync.common.ESHandleMessage; 5 | 6 | /** 7 | * Created by I311352 on 10/17/2016. 8 | */ 9 | public interface ESHandler { 10 | public void onMessage(ESHandleMessage message); 11 | } 12 | -------------------------------------------------------------------------------- /rabbitmqservice/src/main/java/sync/handler/ESHandlerManager.java: -------------------------------------------------------------------------------- 1 | package sync.handler; 2 | 3 | import elasticsearch.esapi.DocumentService; 4 | import elasticsearch.esapi.HotDocumentService; 5 | import org.springframework.context.ConfigurableApplicationContext; 6 | import springcommon.ApplicationContextHolder; 7 | import sync.common.ESHandleMessage; 8 | import sync.handler.impl.ESProductListSyncHandler; 9 | import sync.handler.impl.ESProductSyncHandler; 10 | import sync.handler.impl.ESSKUSyncHandler; 11 | 12 | import java.util.HashMap; 13 | import java.util.List; 14 | import java.util.Map; 15 | import java.util.stream.Collectors; 16 | 17 | /** 18 | * Created by I311352 on 10/17/2016. 19 | */ 20 | 21 | public class ESHandlerManager { 22 | private Map handlerMap = new HashMap(); 23 | 24 | public void initHandler(ConfigurableApplicationContext context) { 25 | //DocumentService service = ApplicationContextHolder.getBean("documentService", DocumentService.class); 26 | 27 | ESProductSyncHandler productSyncHandler = new ESProductSyncHandler(); 28 | productSyncHandler.setDocumentService(context.getBean("documentService", DocumentService.class)); 29 | productSyncHandler.setHotDocumentService(context.getBean("hotDocumentService", HotDocumentService.class)); 30 | handlerMap.put("Product.CREATE", productSyncHandler); 31 | handlerMap.put("Product.UPDATE", productSyncHandler); 32 | 33 | ESSKUSyncHandler esskuSyncHandler = new ESSKUSyncHandler(); 34 | esskuSyncHandler.setDocumentService(context.getBean("documentService", DocumentService.class)); 35 | esskuSyncHandler.setHotDocumentService(context.getBean("hotDocumentService", HotDocumentService.class)); 36 | handlerMap.put("SKU.CREATE", esskuSyncHandler); 37 | handlerMap.put("SKU.UPDATE", esskuSyncHandler); 38 | 39 | ESProductListSyncHandler listSyncHandler = new ESProductListSyncHandler(); 40 | listSyncHandler.setDocumentService(context.getBean("documentService", DocumentService.class)); 41 | listSyncHandler.setHotDocumentService(context.getBean("hotDocumentService", HotDocumentService.class)); 42 | handlerMap.put("CatalogSKU.LIST", listSyncHandler); 43 | handlerMap.put("CatalogSKU.UNLIST", listSyncHandler); 44 | } 45 | 46 | public List getRoutingKeyList() { 47 | List routingKeyList = handlerMap.entrySet().stream().map(val -> val.getKey()) 48 | .collect(Collectors.toList()); 49 | return routingKeyList; 50 | } 51 | 52 | public ESHandler getHandler(ESHandleMessage message) { 53 | String key = message.getType() + "." + message.getAction(); 54 | return handlerMap.get(key); 55 | } 56 | 57 | public Map getHandlerMap() { 58 | return handlerMap; 59 | } 60 | 61 | public void setHandlerMap(Map handlerMap) { 62 | this.handlerMap = handlerMap; 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /rabbitmqservice/src/main/java/sync/handler/impl/ESSKUSyncHandler.java: -------------------------------------------------------------------------------- 1 | package sync.handler.impl; 2 | 3 | import com.google.gson.JsonObject; 4 | import elasticsearch.constant.ESConstants; 5 | import elasticsearch.esapi.resp.ESGetByIdResponse; 6 | import sync.common.ESHandleMessage; 7 | import sync.common.OperateAction; 8 | 9 | import java.util.ArrayList; 10 | import java.util.HashMap; 11 | 12 | /** 13 | * Created by I311352 on 10/17/2016. 14 | */ 15 | public class ESSKUSyncHandler extends ESAbstractHandler{ 16 | 17 | @Override 18 | public boolean handleCreate(ESHandleMessage message) { 19 | JsonObject object = message.getJsonElement().getAsJsonObject(); 20 | Long skuId = object.get("id").getAsLong(); 21 | 22 | //sku body still contains product info, need delete 23 | if (object.get("product") == null) { 24 | // not valid message just pass 25 | logger.error("Invalid message for SKU Sync"); 26 | message.setNeedSendEvent(false); 27 | return true; 28 | } 29 | 30 | JsonObject product = object.get("product").getAsJsonObject(); 31 | Long productId = product.get("id").getAsLong(); 32 | object.addProperty("parentId", productId); 33 | object.remove("product"); 34 | 35 | HashMap params = new HashMap<>(); 36 | params.put("parent", productId.toString()); 37 | 38 | // check the version 39 | Long newVersion = object.get("version").getAsLong(); 40 | ESGetByIdResponse response = loadDocumentWithVersion(skuId, ESConstants.SKU_TYPE); 41 | if (response == null || response.getFound() != true) { 42 | logger.info("The first come message body"); 43 | params.put("version", "-1"); 44 | } else { 45 | // check the version 46 | Long currentVersion = response.getObject().get("version") == null ? -1 : 47 | response.getObject().get("version").getAsLong(); 48 | if (currentVersion >= newVersion) { 49 | // need ignore it 50 | logger.info("Older Version come in last."); 51 | message.setNeedSendEvent(false); 52 | return true; 53 | } else { 54 | params.put("version", response.getVersion().toString()); 55 | } 56 | } 57 | 58 | hotDocumentService.update(ESConstants.STORE_INDEX, ESConstants.SKU_TYPE, 59 | skuId, params, object, this.conflictFunction()); 60 | 61 | message.setNeedSendEvent(true); 62 | return true; 63 | } 64 | 65 | @Override 66 | public boolean handleUpdate(ESHandleMessage message) { 67 | return handleCreate(message); 68 | } 69 | 70 | @Override 71 | public boolean handleDelete(ESHandleMessage message) { 72 | Long skuId = jsonParser.parse(new String(message.getMsgBody())).getAsLong(); 73 | logger.info("Handle Delete SKU msg id is ." + skuId); 74 | 75 | documentService.Delete(ESConstants.STORE_INDEX, ESConstants.SKU_TYPE, 76 | skuId, null); 77 | 78 | return true; 79 | } 80 | 81 | @Override 82 | public boolean postHandleMessage(ESHandleMessage message) { 83 | if (!message.isNeedSendEvent()) { 84 | return true; 85 | } 86 | 87 | Long productId; 88 | if (message.getAction() == OperateAction.DELETE) { 89 | productId = jsonParser.parse(new String(message.getMsgBody())).getAsLong(); 90 | logger.info("Handle Delete id is ." + productId); 91 | } else { 92 | productId = message.getJsonElement() 93 | .getAsJsonObject() 94 | .get("parentId").getAsLong(); 95 | } 96 | 97 | ArrayList productIds = new ArrayList<>(); 98 | productIds.add(productId); 99 | message.setProductIds(productIds); 100 | return super.postHandleMessage(message); 101 | } 102 | 103 | } 104 | -------------------------------------------------------------------------------- /rabbitmqservice/src/main/java/sync/listener/ESMessageListener.java: -------------------------------------------------------------------------------- 1 | package sync.listener; 2 | 3 | 4 | import elasticsearch.constant.ESConstants; 5 | import org.apache.commons.lang3.ObjectUtils; 6 | import org.apache.commons.lang3.time.StopWatch; 7 | import org.apache.log4j.Logger; 8 | import org.springframework.amqp.core.Message; 9 | import org.springframework.amqp.core.MessageListener; 10 | import org.springframework.beans.factory.annotation.Autowired; 11 | import org.springframework.boot.actuate.metrics.CounterService; 12 | import org.springframework.boot.actuate.metrics.GaugeService; 13 | import org.springframework.context.ConfigurableApplicationContext; 14 | import org.springframework.stereotype.Component; 15 | import sun.misc.MessageUtils; 16 | import sync.common.ESHandleMessage; 17 | import sync.common.MQMessageConst; 18 | import sync.handler.ESHandler; 19 | import sync.handler.ESHandlerManager; 20 | 21 | import java.util.List; 22 | import java.util.Locale; 23 | import java.util.regex.Matcher; 24 | 25 | /** 26 | * Created by I311352 on 10/17/2016. 27 | */ 28 | @Component 29 | public class ESMessageListener implements MessageListener { 30 | private static final Logger lightLogger = Logger.getLogger(ESMessageListener.class); 31 | 32 | private ESHandlerManager handlerManager = null; 33 | private ConfigurableApplicationContext context; 34 | 35 | private final CounterService counterService; 36 | 37 | private final GaugeService gaugeService; 38 | 39 | public ESMessageListener(ConfigurableApplicationContext context) { 40 | this.counterService = context.getBean("counterService", CounterService.class); 41 | this.gaugeService = context.getBean("gaugeService", GaugeService.class); 42 | this.context = context; 43 | //handlerManager.initHandler(context); 44 | } 45 | 46 | @Override 47 | public void onMessage(Message message) { 48 | if (handlerManager == null) { 49 | handlerManager = new ESHandlerManager(); 50 | handlerManager.initHandler(context); 51 | } 52 | 53 | doWork(message); 54 | } 55 | 56 | protected void postHandle(Message message) { 57 | String messageId = String.valueOf(message.getMessageProperties() 58 | .getHeaders().get(MQMessageConst.H_MESSAGEID)); 59 | String jobName = String.valueOf(message.getMessageProperties() 60 | .getHeaders().get(MQMessageConst.H_JOBNAME)); 61 | } 62 | 63 | public void doWork(Message message) { 64 | String routingKey = message.getMessageProperties().getReceivedRoutingKey(); 65 | Matcher m = ESConstants.ROUTINGKEY_PATTERN.matcher(routingKey); 66 | try { 67 | if (m.find() && m.groupCount() == 3) { 68 | ESHandleMessage esHandleMessage = new ESHandleMessage(Long.parseLong(m.group(3)), 69 | m.group(1), m.group(2), message.getBody()); 70 | esHandleMessage.setEventGenerated(isGeneratedMessage(message)); 71 | handleMsg(esHandleMessage); 72 | } else { 73 | lightLogger.warn("Message is invalid: " + routingKey); 74 | } 75 | } finally { 76 | lightLogger.info("Done"); 77 | } 78 | 79 | } 80 | 81 | private boolean isGeneratedMessage(Message message) { 82 | return ObjectUtils.equals("EventGenerator", message.getMessageProperties().getHeaders().get("eventSource")); 83 | } 84 | 85 | private void handleMsg(ESHandleMessage message) { 86 | StopWatch stopWatch = new StopWatch(); 87 | stopWatch.start(); 88 | boolean successful = false; 89 | 90 | lightLogger.info("ES sync start on ESHandleMessage " + message.toString()); 91 | ESHandler handler = null; 92 | try { 93 | handler = handlerManager.getHandler(message); 94 | if (handler == null) { 95 | lightLogger.info("Handler is null for message" + message.toString()); 96 | return; 97 | } 98 | 99 | handler.onMessage(message); 100 | 101 | successful = true; 102 | } catch (Throwable t) { 103 | successful = false; 104 | throw t; 105 | } finally { 106 | stopWatch.stop(); 107 | String metricName = message.getClass().getSimpleName() + "." + handler.getClass().getSimpleName(); 108 | //counterService.increment(metricName + "." + (successful ? ".success" : "failed")); 109 | //gaugeService.submit(metricName, stopWatch.getTime()); 110 | lightLogger.info("ES sync end on execTime[" + stopWatch.getTime() + "]"); 111 | } 112 | } 113 | 114 | // @Override 115 | public String[] getRoutingKeys() { 116 | List routingList = handlerManager.getRoutingKeyList(); 117 | return routingList.toArray(new String[routingList.size()]); 118 | } 119 | 120 | public void setHandlerManager(ESHandlerManager handlerManager) { 121 | this.handlerManager = handlerManager; 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | include 'commonutil' 2 | include 'elasticservice' 3 | include 'rabbitmqservice' 4 | include 'generatedata' 5 | include 'netty-http' 6 | include 'app' 7 | 8 | --------------------------------------------------------------------------------