├── .github └── workflows │ └── ci.yml ├── .gitignore ├── .mvn └── wrapper │ ├── MavenWrapperDownloader.java │ ├── maven-wrapper.jar │ └── maven-wrapper.properties ├── README.md ├── core ├── pom.xml └── src │ └── main │ ├── kotlin │ └── elasticsearchpaginator │ │ └── core │ │ ├── configuration │ │ ├── JacksonConfiguration.kt │ │ └── ReactorRabbitmqAutoConfiguration.kt │ │ ├── model │ │ ├── DeletePages.kt │ │ ├── DeleteQuery.kt │ │ └── Query.kt │ │ ├── transport │ │ └── AbstractRabbitmqReceiver.kt │ │ └── util │ │ ├── ElasticsearchUtils.kt │ │ └── RabbitmqUtils.kt │ └── resources │ └── META-INF │ └── spring.factories ├── docker-compose.yml ├── mvnw ├── mvnw.cmd ├── pom.xml ├── worker-paginator-calc ├── Dockerfile ├── pom.xml └── src │ ├── main │ ├── kotlin │ │ └── elasticsearchpaginator │ │ │ └── workerpaginatorcalc │ │ │ ├── WorkerPaginatorCalcApplication.kt │ │ │ ├── configuration │ │ │ ├── ElasticsearchConfiguration.kt │ │ │ ├── ElasticsearchProperties.kt │ │ │ ├── RabbitmqConfiguration.kt │ │ │ └── RabbitmqProperties.kt │ │ │ ├── exception │ │ │ └── UnexpectedQueryException.kt │ │ │ ├── model │ │ │ └── Page.kt │ │ │ ├── repository │ │ │ ├── EntityElasticsearchRepository.kt │ │ │ └── PageRepository.kt │ │ │ ├── service │ │ │ ├── ComputePagesService.kt │ │ │ └── DeletePagesService.kt │ │ │ └── transport │ │ │ ├── ComputePagesReceiver.kt │ │ │ ├── DeletePagesReceiver.kt │ │ │ └── DeleteQuerySender.kt │ └── resources │ │ ├── application.yml │ │ └── pages-mappings.json │ └── test │ ├── kotlin │ └── elasticsearchpaginator │ │ └── workerpaginatorcalc │ │ ├── AbstractIntegrationTest.kt │ │ ├── ComputePagesIntegrationTest.kt │ │ └── DeletePagesIntegrationTest.kt │ └── resources │ └── junit-platform.properties └── worker-paginator ├── Dockerfile ├── pom.xml └── src ├── main ├── kotlin │ └── elasticsearchpaginator │ │ └── workerpaginator │ │ ├── WorkerPaginatorApplication.kt │ │ ├── configuration │ │ ├── ElasticsearchConfiguration.kt │ │ ├── ElasticsearchProperties.kt │ │ ├── RabbitmqConfiguration.kt │ │ └── RabbitmqProperties.kt │ │ ├── model │ │ └── QueryEntry.kt │ │ ├── repository │ │ └── QueryEntryRepository.kt │ │ ├── service │ │ ├── CleaningQueriesService.kt │ │ ├── QueriesService.kt │ │ └── RefreshQueriesService.kt │ │ ├── transport │ │ ├── ComputePagesSender.kt │ │ ├── DeletePagesSender.kt │ │ ├── DeleteQueriesReceiver.kt │ │ └── QueriesReceiver.kt │ │ └── web │ │ └── QueriesController.kt └── resources │ ├── application.yml │ └── query-entries-mappings.json └── test ├── kotlin └── elasticsearchpaginator │ └── workerpaginator │ ├── AbstractIntegrationTest.kt │ ├── ClearOutdatedQueriesIntegrationTest.kt │ ├── DeleteQueryIntegrationTest.kt │ └── HandleQueryIntegrationTest.kt └── resources └── junit-platform.properties /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: ci 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | 8 | jobs: 9 | build: 10 | 11 | runs-on: ubuntu-latest 12 | 13 | steps: 14 | - uses: actions/checkout@v1 15 | 16 | - name: Set up JDK 11 17 | uses: actions/setup-java@v1 18 | with: 19 | java-version: 11 20 | 21 | - name: Cache dependencies 22 | uses: actions/cache@v1 23 | with: 24 | path: ~/.m2/repository 25 | key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} 26 | restore-keys: | 27 | ${{ runner.os }}-maven- 28 | 29 | - name: Build with Maven 30 | run: mvn -B package --file pom.xml 31 | 32 | - name: Build Docker images 33 | run: docker-compose build 34 | 35 | - name: Login to Docker Hub 36 | run: docker login --username $DOCKER_USERNAME --password $DOCKER_PASSWORD 37 | env: 38 | DOCKER_USERNAME: ${{ secrets.GITHUB_DOCKER_USERNAME }} 39 | DOCKER_PASSWORD: ${{ secrets.GITHUB_DOCKER_PASSWORD }} 40 | 41 | - name: Publish Docker images 42 | run: docker-compose push 43 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | HELP.md 2 | target/ 3 | !.mvn/wrapper/maven-wrapper.jar 4 | !**/src/main/** 5 | !**/src/test/** 6 | 7 | ### STS ### 8 | .apt_generated 9 | .classpath 10 | .factorypath 11 | .project 12 | .settings 13 | .springBeans 14 | .sts4-cache 15 | 16 | ### IntelliJ IDEA ### 17 | .idea 18 | *.iws 19 | *.iml 20 | *.ipr 21 | 22 | ### NetBeans ### 23 | /nbproject/private/ 24 | /nbbuild/ 25 | /dist/ 26 | /nbdist/ 27 | /.nb-gradle/ 28 | build/ 29 | 30 | ### VS Code ### 31 | .vscode/ 32 | -------------------------------------------------------------------------------- /.mvn/wrapper/MavenWrapperDownloader.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2012-2019 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import java.net.*; 18 | import java.io.*; 19 | import java.nio.channels.*; 20 | import java.util.Properties; 21 | 22 | public class MavenWrapperDownloader { 23 | 24 | private static final String WRAPPER_VERSION = "0.5.5"; 25 | /** 26 | * Default URL to download the maven-wrapper.jar from, if no 'downloadUrl' is provided. 27 | */ 28 | private static final String DEFAULT_DOWNLOAD_URL = "https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/" 29 | + WRAPPER_VERSION + "/maven-wrapper-" + WRAPPER_VERSION + ".jar"; 30 | 31 | /** 32 | * Path to the maven-wrapper.properties file, which might contain a downloadUrl property to 33 | * use instead of the default one. 34 | */ 35 | private static final String MAVEN_WRAPPER_PROPERTIES_PATH = 36 | ".mvn/wrapper/maven-wrapper.properties"; 37 | 38 | /** 39 | * Path where the maven-wrapper.jar will be saved to. 40 | */ 41 | private static final String MAVEN_WRAPPER_JAR_PATH = 42 | ".mvn/wrapper/maven-wrapper.jar"; 43 | 44 | /** 45 | * Name of the property which should be used to override the default download url for the wrapper. 46 | */ 47 | private static final String PROPERTY_NAME_WRAPPER_URL = "wrapperUrl"; 48 | 49 | public static void main(String args[]) { 50 | System.out.println("- Downloader started"); 51 | File baseDirectory = new File(args[0]); 52 | System.out.println("- Using base directory: " + baseDirectory.getAbsolutePath()); 53 | 54 | // If the maven-wrapper.properties exists, read it and check if it contains a custom 55 | // wrapperUrl parameter. 56 | File mavenWrapperPropertyFile = new File(baseDirectory, MAVEN_WRAPPER_PROPERTIES_PATH); 57 | String url = DEFAULT_DOWNLOAD_URL; 58 | if (mavenWrapperPropertyFile.exists()) { 59 | FileInputStream mavenWrapperPropertyFileInputStream = null; 60 | try { 61 | mavenWrapperPropertyFileInputStream = new FileInputStream(mavenWrapperPropertyFile); 62 | Properties mavenWrapperProperties = new Properties(); 63 | mavenWrapperProperties.load(mavenWrapperPropertyFileInputStream); 64 | url = mavenWrapperProperties.getProperty(PROPERTY_NAME_WRAPPER_URL, url); 65 | } catch (IOException e) { 66 | System.out.println("- ERROR loading '" + MAVEN_WRAPPER_PROPERTIES_PATH + "'"); 67 | } finally { 68 | try { 69 | if (mavenWrapperPropertyFileInputStream != null) { 70 | mavenWrapperPropertyFileInputStream.close(); 71 | } 72 | } catch (IOException e) { 73 | // Ignore ... 74 | } 75 | } 76 | } 77 | System.out.println("- Downloading from: " + url); 78 | 79 | File outputFile = new File(baseDirectory.getAbsolutePath(), MAVEN_WRAPPER_JAR_PATH); 80 | if (!outputFile.getParentFile().exists()) { 81 | if (!outputFile.getParentFile().mkdirs()) { 82 | System.out.println( 83 | "- ERROR creating output directory '" + outputFile.getParentFile().getAbsolutePath() + "'"); 84 | } 85 | } 86 | System.out.println("- Downloading to: " + outputFile.getAbsolutePath()); 87 | try { 88 | downloadFileFromURL(url, outputFile); 89 | System.out.println("Done"); 90 | System.exit(0); 91 | } catch (Throwable e) { 92 | System.out.println("- Error downloading"); 93 | e.printStackTrace(); 94 | System.exit(1); 95 | } 96 | } 97 | 98 | private static void downloadFileFromURL(String urlString, File destination) throws Exception { 99 | if (System.getenv("MVNW_USERNAME") != null && System.getenv("MVNW_PASSWORD") != null) { 100 | String username = System.getenv("MVNW_USERNAME"); 101 | char[] password = System.getenv("MVNW_PASSWORD").toCharArray(); 102 | Authenticator.setDefault(new Authenticator() { 103 | @Override 104 | protected PasswordAuthentication getPasswordAuthentication() { 105 | return new PasswordAuthentication(username, password); 106 | } 107 | }); 108 | } 109 | URL website = new URL(urlString); 110 | ReadableByteChannel rbc; 111 | rbc = Channels.newChannel(website.openStream()); 112 | FileOutputStream fos = new FileOutputStream(destination); 113 | fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE); 114 | fos.close(); 115 | rbc.close(); 116 | } 117 | 118 | } 119 | -------------------------------------------------------------------------------- /.mvn/wrapper/maven-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/btravers/elasticsearch-paginator/97a09b3e54ad29b179492457869f5df8b6da2584/.mvn/wrapper/maven-wrapper.jar -------------------------------------------------------------------------------- /.mvn/wrapper/maven-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.6.2/apache-maven-3.6.2-bin.zip 2 | wrapperUrl=https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.5/maven-wrapper-0.5.5.jar 3 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Elasticsearch Paginator 2 | 3 | [![Build Status](https://github.com/btravers/elasticsearch-paginator/workflows/ci/badge.svg?branch=master)](https://github.com/btravers/elasticsearch-paginator/actions) 4 | 5 | WIP 6 | -------------------------------------------------------------------------------- /core/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | elasticsearch-paginator 7 | elasticsearch-paginator 8 | 0.0.1-SNAPSHOT 9 | 10 | 4.0.0 11 | 12 | core 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /core/src/main/kotlin/elasticsearchpaginator/core/configuration/JacksonConfiguration.kt: -------------------------------------------------------------------------------- 1 | package elasticsearchpaginator.core.configuration 2 | 3 | import com.fasterxml.jackson.databind.ObjectMapper 4 | import com.fasterxml.jackson.databind.SerializationFeature 5 | import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule 6 | import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper 7 | import org.springframework.boot.autoconfigure.AutoConfigureBefore 8 | import org.springframework.boot.autoconfigure.jackson.JacksonAutoConfiguration 9 | import org.springframework.context.annotation.Bean 10 | import org.springframework.context.annotation.Configuration 11 | 12 | @Configuration 13 | @AutoConfigureBefore(JacksonAutoConfiguration::class) 14 | class JacksonConfiguration { 15 | 16 | @Bean 17 | fun mapper(): ObjectMapper { 18 | return OBJECT_MAPPER 19 | } 20 | 21 | companion object { 22 | val OBJECT_MAPPER = jacksonObjectMapper() 23 | .registerModule(JavaTimeModule()) 24 | .configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false) 25 | } 26 | 27 | } 28 | -------------------------------------------------------------------------------- /core/src/main/kotlin/elasticsearchpaginator/core/configuration/ReactorRabbitmqAutoConfiguration.kt: -------------------------------------------------------------------------------- 1 | package elasticsearchpaginator.core.configuration 2 | 3 | import com.rabbitmq.client.Connection 4 | import com.rabbitmq.client.ConnectionFactory 5 | import org.springframework.beans.factory.DisposableBean 6 | import org.springframework.boot.autoconfigure.amqp.RabbitProperties 7 | import org.springframework.context.annotation.Bean 8 | import org.springframework.context.annotation.Configuration 9 | import reactor.core.publisher.Mono 10 | import reactor.rabbitmq.* 11 | 12 | @Configuration 13 | class ReactorRabbitmqAutoConfiguration(private val rabbitProperties: RabbitProperties) { 14 | 15 | @Bean 16 | fun connectionMono(): Mono { 17 | return Mono.fromCallable { 18 | ConnectionFactory() 19 | .apply { 20 | this.host = rabbitProperties.host 21 | this.port = rabbitProperties.port 22 | this.username = rabbitProperties.username 23 | this.password = rabbitProperties.password 24 | this.useNio() 25 | } 26 | .newConnection() 27 | } 28 | .cache() 29 | } 30 | 31 | @Bean 32 | fun receiver(connectionMono: Mono): Receiver { 33 | return RabbitFlux.createReceiver( 34 | ReceiverOptions() 35 | .connectionMono(connectionMono) 36 | ) 37 | } 38 | 39 | @Bean 40 | fun sender(connectionMono: Mono): Sender { 41 | return RabbitFlux.createSender( 42 | SenderOptions() 43 | .connectionMono(connectionMono) 44 | ) 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /core/src/main/kotlin/elasticsearchpaginator/core/model/DeletePages.kt: -------------------------------------------------------------------------------- 1 | package elasticsearchpaginator.core.model 2 | 3 | data class DeletePages( 4 | val queryId: String 5 | ) 6 | -------------------------------------------------------------------------------- /core/src/main/kotlin/elasticsearchpaginator/core/model/DeleteQuery.kt: -------------------------------------------------------------------------------- 1 | package elasticsearchpaginator.core.model 2 | 3 | data class DeleteQuery( 4 | val queryId: String 5 | ) 6 | -------------------------------------------------------------------------------- /core/src/main/kotlin/elasticsearchpaginator/core/model/Query.kt: -------------------------------------------------------------------------------- 1 | package elasticsearchpaginator.core.model 2 | 3 | import java.math.BigInteger 4 | import java.security.MessageDigest 5 | 6 | data class Query( 7 | val index: String, 8 | val query: String, 9 | val sort: String, 10 | val firstPageSize: Int?, 11 | val size: Int 12 | ) { 13 | 14 | fun hash(): String { 15 | val messageDigest = MessageDigest.getInstance("MD5") 16 | return BigInteger(1, messageDigest.digest(this.toString().toByteArray())) 17 | .toString(16) 18 | .padStart(32, '0') 19 | } 20 | 21 | } 22 | -------------------------------------------------------------------------------- /core/src/main/kotlin/elasticsearchpaginator/core/transport/AbstractRabbitmqReceiver.kt: -------------------------------------------------------------------------------- 1 | package elasticsearchpaginator.core.transport 2 | 3 | import com.fasterxml.jackson.databind.ObjectMapper 4 | import org.slf4j.LoggerFactory 5 | import org.springframework.beans.factory.InitializingBean 6 | import org.springframework.boot.ExitCodeGenerator 7 | import org.springframework.boot.SpringApplication 8 | import org.springframework.context.ApplicationContext 9 | import reactor.core.publisher.Mono 10 | import reactor.rabbitmq.Receiver 11 | 12 | abstract class AbstractRabbitmqReceiver : InitializingBean { 13 | 14 | private val logger = LoggerFactory.getLogger(AbstractRabbitmqReceiver::class.java) 15 | 16 | abstract val queueName: String 17 | abstract val receiver: Receiver 18 | abstract val mapper: ObjectMapper 19 | abstract val applicationContext: ApplicationContext 20 | abstract val eventClass: Class 21 | 22 | abstract fun eventHandler(event: T): Mono 23 | 24 | override fun afterPropertiesSet() { 25 | this.receiver 26 | .consumeManualAck(queueName) 27 | .flatMap { acknowledgableDelivery -> 28 | Mono.just(acknowledgableDelivery.body) 29 | .map { body -> this.mapper.readValue(body, eventClass) } 30 | .flatMap(this::eventHandler) 31 | .doOnSuccess { acknowledgableDelivery.ack() } 32 | .onErrorResume { err -> 33 | logger.error("Unexpected error during pagination computation", err) 34 | acknowledgableDelivery.nack(false) 35 | Mono.empty() 36 | } 37 | } 38 | .doOnError { err -> logger.error("Stop listening queue {} due to error", queueName, err) } 39 | .doOnComplete { logger.error("Unexpected end of listening queue {}", queueName) } 40 | .subscribe( 41 | {}, 42 | { SpringApplication.exit(applicationContext, ExitCodeGenerator { 1 }) }, 43 | { SpringApplication.exit(applicationContext, ExitCodeGenerator { 0 }) } 44 | ) 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /core/src/main/kotlin/elasticsearchpaginator/core/util/ElasticsearchUtils.kt: -------------------------------------------------------------------------------- 1 | package elasticsearchpaginator.core.util 2 | 3 | import org.elasticsearch.action.ActionListener 4 | import reactor.core.publisher.Mono 5 | 6 | object ElasticsearchUtils { 7 | 8 | const val NUMBER_SHARDS_SETTING = "number_of_shards" 9 | const val NUMBER_REPLICAS_SETTING = "number_of_replicas" 10 | 11 | fun async(f: (ActionListener) -> Unit): Mono { 12 | return Mono.create { sink -> 13 | f(object : ActionListener { 14 | override fun onResponse(response: T) { 15 | sink.success(response) 16 | } 17 | 18 | override fun onFailure(e: Exception) { 19 | sink.error(e) 20 | } 21 | }) 22 | } 23 | } 24 | 25 | } 26 | -------------------------------------------------------------------------------- /core/src/main/kotlin/elasticsearchpaginator/core/util/RabbitmqUtils.kt: -------------------------------------------------------------------------------- 1 | package elasticsearchpaginator.core.util 2 | 3 | import org.springframework.amqp.core.* 4 | 5 | object RabbitmqUtils { 6 | 7 | fun AmqpAdmin.createExchange(exchangeName: String): TopicExchange { 8 | return ExchangeBuilder.topicExchange(exchangeName) 9 | .durable(true) 10 | .build() 11 | .apply { 12 | this@createExchange.declareExchange(this) 13 | } 14 | } 15 | 16 | fun AmqpAdmin.createQueues(queueName: String, deadLetterQueueName: String, key: String, exchange: Exchange) { 17 | val deadLetterQueue = QueueBuilder 18 | .durable(deadLetterQueueName) 19 | .build() 20 | this.declareQueue(deadLetterQueue) 21 | 22 | val queue = QueueBuilder 23 | .durable(queueName) 24 | .withArgument("x-dead-letter-exchange", "") 25 | .withArgument("x-dead-letter-routing-key", deadLetterQueueName) 26 | .build() 27 | this.declareQueue(queue) 28 | 29 | val binding = BindingBuilder 30 | .bind(queue) 31 | .to(exchange) 32 | .with(key) 33 | .noargs() 34 | this.declareBinding(binding) 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /core/src/main/resources/META-INF/spring.factories: -------------------------------------------------------------------------------- 1 | org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ 2 | elasticsearchpaginator.core.configuration.ReactorRabbitmqAutoConfiguration,\ 3 | elasticsearchpaginator.core.configuration.JacksonConfiguration 4 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2.1' 2 | 3 | services: 4 | 5 | worker-paginator: 6 | image: elasticsearchpaginator/workerpaginator:${TAG:-latest} 7 | build: 8 | context: ./worker-paginator 9 | dockerfile: Dockerfile 10 | 11 | worker-paginator-calc: 12 | image: elasticsearchpaginator/workerpaginatorcalc:${TAG:-latest} 13 | build: 14 | context: ./worker-paginator-calc 15 | dockerfile: Dockerfile 16 | -------------------------------------------------------------------------------- /mvnw: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # ---------------------------------------------------------------------------- 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # https://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | # ---------------------------------------------------------------------------- 20 | 21 | # ---------------------------------------------------------------------------- 22 | # Maven2 Start Up Batch script 23 | # 24 | # Required ENV vars: 25 | # ------------------ 26 | # JAVA_HOME - location of a JDK home dir 27 | # 28 | # Optional ENV vars 29 | # ----------------- 30 | # M2_HOME - location of maven2's installed home dir 31 | # MAVEN_OPTS - parameters passed to the Java VM when running Maven 32 | # e.g. to debug Maven itself, use 33 | # set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 34 | # MAVEN_SKIP_RC - flag to disable loading of mavenrc files 35 | # ---------------------------------------------------------------------------- 36 | 37 | if [ -z "$MAVEN_SKIP_RC" ] ; then 38 | 39 | if [ -f /etc/mavenrc ] ; then 40 | . /etc/mavenrc 41 | fi 42 | 43 | if [ -f "$HOME/.mavenrc" ] ; then 44 | . "$HOME/.mavenrc" 45 | fi 46 | 47 | fi 48 | 49 | # OS specific support. $var _must_ be set to either true or false. 50 | cygwin=false; 51 | darwin=false; 52 | mingw=false 53 | case "`uname`" in 54 | CYGWIN*) cygwin=true ;; 55 | MINGW*) mingw=true;; 56 | Darwin*) darwin=true 57 | # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home 58 | # See https://developer.apple.com/library/mac/qa/qa1170/_index.html 59 | if [ -z "$JAVA_HOME" ]; then 60 | if [ -x "/usr/libexec/java_home" ]; then 61 | export JAVA_HOME="`/usr/libexec/java_home`" 62 | else 63 | export JAVA_HOME="/Library/Java/Home" 64 | fi 65 | fi 66 | ;; 67 | esac 68 | 69 | if [ -z "$JAVA_HOME" ] ; then 70 | if [ -r /etc/gentoo-release ] ; then 71 | JAVA_HOME=`java-config --jre-home` 72 | fi 73 | fi 74 | 75 | if [ -z "$M2_HOME" ] ; then 76 | ## resolve links - $0 may be a link to maven's home 77 | PRG="$0" 78 | 79 | # need this for relative symlinks 80 | while [ -h "$PRG" ] ; do 81 | ls=`ls -ld "$PRG"` 82 | link=`expr "$ls" : '.*-> \(.*\)$'` 83 | if expr "$link" : '/.*' > /dev/null; then 84 | PRG="$link" 85 | else 86 | PRG="`dirname "$PRG"`/$link" 87 | fi 88 | done 89 | 90 | saveddir=`pwd` 91 | 92 | M2_HOME=`dirname "$PRG"`/.. 93 | 94 | # make it fully qualified 95 | M2_HOME=`cd "$M2_HOME" && pwd` 96 | 97 | cd "$saveddir" 98 | # echo Using m2 at $M2_HOME 99 | fi 100 | 101 | # For Cygwin, ensure paths are in UNIX format before anything is touched 102 | if $cygwin ; then 103 | [ -n "$M2_HOME" ] && 104 | M2_HOME=`cygpath --unix "$M2_HOME"` 105 | [ -n "$JAVA_HOME" ] && 106 | JAVA_HOME=`cygpath --unix "$JAVA_HOME"` 107 | [ -n "$CLASSPATH" ] && 108 | CLASSPATH=`cygpath --path --unix "$CLASSPATH"` 109 | fi 110 | 111 | # For Mingw, ensure paths are in UNIX format before anything is touched 112 | if $mingw ; then 113 | [ -n "$M2_HOME" ] && 114 | M2_HOME="`(cd "$M2_HOME"; pwd)`" 115 | [ -n "$JAVA_HOME" ] && 116 | JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`" 117 | fi 118 | 119 | if [ -z "$JAVA_HOME" ]; then 120 | javaExecutable="`which javac`" 121 | if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then 122 | # readlink(1) is not available as standard on Solaris 10. 123 | readLink=`which readlink` 124 | if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then 125 | if $darwin ; then 126 | javaHome="`dirname \"$javaExecutable\"`" 127 | javaExecutable="`cd \"$javaHome\" && pwd -P`/javac" 128 | else 129 | javaExecutable="`readlink -f \"$javaExecutable\"`" 130 | fi 131 | javaHome="`dirname \"$javaExecutable\"`" 132 | javaHome=`expr "$javaHome" : '\(.*\)/bin'` 133 | JAVA_HOME="$javaHome" 134 | export JAVA_HOME 135 | fi 136 | fi 137 | fi 138 | 139 | if [ -z "$JAVACMD" ] ; then 140 | if [ -n "$JAVA_HOME" ] ; then 141 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 142 | # IBM's JDK on AIX uses strange locations for the executables 143 | JAVACMD="$JAVA_HOME/jre/sh/java" 144 | else 145 | JAVACMD="$JAVA_HOME/bin/java" 146 | fi 147 | else 148 | JAVACMD="`which java`" 149 | fi 150 | fi 151 | 152 | if [ ! -x "$JAVACMD" ] ; then 153 | echo "Error: JAVA_HOME is not defined correctly." >&2 154 | echo " We cannot execute $JAVACMD" >&2 155 | exit 1 156 | fi 157 | 158 | if [ -z "$JAVA_HOME" ] ; then 159 | echo "Warning: JAVA_HOME environment variable is not set." 160 | fi 161 | 162 | CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher 163 | 164 | # traverses directory structure from process work directory to filesystem root 165 | # first directory with .mvn subdirectory is considered project base directory 166 | find_maven_basedir() { 167 | 168 | if [ -z "$1" ] 169 | then 170 | echo "Path not specified to find_maven_basedir" 171 | return 1 172 | fi 173 | 174 | basedir="$1" 175 | wdir="$1" 176 | while [ "$wdir" != '/' ] ; do 177 | if [ -d "$wdir"/.mvn ] ; then 178 | basedir=$wdir 179 | break 180 | fi 181 | # workaround for JBEAP-8937 (on Solaris 10/Sparc) 182 | if [ -d "${wdir}" ]; then 183 | wdir=`cd "$wdir/.."; pwd` 184 | fi 185 | # end of workaround 186 | done 187 | echo "${basedir}" 188 | } 189 | 190 | # concatenates all lines of a file 191 | concat_lines() { 192 | if [ -f "$1" ]; then 193 | echo "$(tr -s '\n' ' ' < "$1")" 194 | fi 195 | } 196 | 197 | BASE_DIR=`find_maven_basedir "$(pwd)"` 198 | if [ -z "$BASE_DIR" ]; then 199 | exit 1; 200 | fi 201 | 202 | ########################################################################################## 203 | # Extension to allow automatically downloading the maven-wrapper.jar from Maven-central 204 | # This allows using the maven wrapper in projects that prohibit checking in binary data. 205 | ########################################################################################## 206 | if [ -r "$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" ]; then 207 | if [ "$MVNW_VERBOSE" = true ]; then 208 | echo "Found .mvn/wrapper/maven-wrapper.jar" 209 | fi 210 | else 211 | if [ "$MVNW_VERBOSE" = true ]; then 212 | echo "Couldn't find .mvn/wrapper/maven-wrapper.jar, downloading it ..." 213 | fi 214 | if [ -n "$MVNW_REPOURL" ]; then 215 | jarUrl="$MVNW_REPOURL/io/takari/maven-wrapper/0.5.5/maven-wrapper-0.5.5.jar" 216 | else 217 | jarUrl="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.5/maven-wrapper-0.5.5.jar" 218 | fi 219 | while IFS="=" read key value; do 220 | case "$key" in (wrapperUrl) jarUrl="$value"; break ;; 221 | esac 222 | done < "$BASE_DIR/.mvn/wrapper/maven-wrapper.properties" 223 | if [ "$MVNW_VERBOSE" = true ]; then 224 | echo "Downloading from: $jarUrl" 225 | fi 226 | wrapperJarPath="$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" 227 | if $cygwin; then 228 | wrapperJarPath=`cygpath --path --windows "$wrapperJarPath"` 229 | fi 230 | 231 | if command -v wget > /dev/null; then 232 | if [ "$MVNW_VERBOSE" = true ]; then 233 | echo "Found wget ... using wget" 234 | fi 235 | if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then 236 | wget "$jarUrl" -O "$wrapperJarPath" 237 | else 238 | wget --http-user=$MVNW_USERNAME --http-password=$MVNW_PASSWORD "$jarUrl" -O "$wrapperJarPath" 239 | fi 240 | elif command -v curl > /dev/null; then 241 | if [ "$MVNW_VERBOSE" = true ]; then 242 | echo "Found curl ... using curl" 243 | fi 244 | if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then 245 | curl -o "$wrapperJarPath" "$jarUrl" -f 246 | else 247 | curl --user $MVNW_USERNAME:$MVNW_PASSWORD -o "$wrapperJarPath" "$jarUrl" -f 248 | fi 249 | 250 | else 251 | if [ "$MVNW_VERBOSE" = true ]; then 252 | echo "Falling back to using Java to download" 253 | fi 254 | javaClass="$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.java" 255 | # For Cygwin, switch paths to Windows format before running javac 256 | if $cygwin; then 257 | javaClass=`cygpath --path --windows "$javaClass"` 258 | fi 259 | if [ -e "$javaClass" ]; then 260 | if [ ! -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then 261 | if [ "$MVNW_VERBOSE" = true ]; then 262 | echo " - Compiling MavenWrapperDownloader.java ..." 263 | fi 264 | # Compiling the Java class 265 | ("$JAVA_HOME/bin/javac" "$javaClass") 266 | fi 267 | if [ -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then 268 | # Running the downloader 269 | if [ "$MVNW_VERBOSE" = true ]; then 270 | echo " - Running MavenWrapperDownloader.java ..." 271 | fi 272 | ("$JAVA_HOME/bin/java" -cp .mvn/wrapper MavenWrapperDownloader "$MAVEN_PROJECTBASEDIR") 273 | fi 274 | fi 275 | fi 276 | fi 277 | ########################################################################################## 278 | # End of extension 279 | ########################################################################################## 280 | 281 | export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"} 282 | if [ "$MVNW_VERBOSE" = true ]; then 283 | echo $MAVEN_PROJECTBASEDIR 284 | fi 285 | MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" 286 | 287 | # For Cygwin, switch paths to Windows format before running java 288 | if $cygwin; then 289 | [ -n "$M2_HOME" ] && 290 | M2_HOME=`cygpath --path --windows "$M2_HOME"` 291 | [ -n "$JAVA_HOME" ] && 292 | JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"` 293 | [ -n "$CLASSPATH" ] && 294 | CLASSPATH=`cygpath --path --windows "$CLASSPATH"` 295 | [ -n "$MAVEN_PROJECTBASEDIR" ] && 296 | MAVEN_PROJECTBASEDIR=`cygpath --path --windows "$MAVEN_PROJECTBASEDIR"` 297 | fi 298 | 299 | # Provide a "standardized" way to retrieve the CLI args that will 300 | # work with both Windows and non-Windows executions. 301 | MAVEN_CMD_LINE_ARGS="$MAVEN_CONFIG $@" 302 | export MAVEN_CMD_LINE_ARGS 303 | 304 | WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain 305 | 306 | exec "$JAVACMD" \ 307 | $MAVEN_OPTS \ 308 | -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ 309 | "-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ 310 | ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@" 311 | -------------------------------------------------------------------------------- /mvnw.cmd: -------------------------------------------------------------------------------- 1 | @REM ---------------------------------------------------------------------------- 2 | @REM Licensed to the Apache Software Foundation (ASF) under one 3 | @REM or more contributor license agreements. See the NOTICE file 4 | @REM distributed with this work for additional information 5 | @REM regarding copyright ownership. The ASF licenses this file 6 | @REM to you under the Apache License, Version 2.0 (the 7 | @REM "License"); you may not use this file except in compliance 8 | @REM with the License. You may obtain a copy of the License at 9 | @REM 10 | @REM https://www.apache.org/licenses/LICENSE-2.0 11 | @REM 12 | @REM Unless required by applicable law or agreed to in writing, 13 | @REM software distributed under the License is distributed on an 14 | @REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | @REM KIND, either express or implied. See the License for the 16 | @REM specific language governing permissions and limitations 17 | @REM under the License. 18 | @REM ---------------------------------------------------------------------------- 19 | 20 | @REM ---------------------------------------------------------------------------- 21 | @REM Maven2 Start Up Batch script 22 | @REM 23 | @REM Required ENV vars: 24 | @REM JAVA_HOME - location of a JDK home dir 25 | @REM 26 | @REM Optional ENV vars 27 | @REM M2_HOME - location of maven2's installed home dir 28 | @REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands 29 | @REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a key stroke before ending 30 | @REM MAVEN_OPTS - parameters passed to the Java VM when running Maven 31 | @REM e.g. to debug Maven itself, use 32 | @REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 33 | @REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files 34 | @REM ---------------------------------------------------------------------------- 35 | 36 | @REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on' 37 | @echo off 38 | @REM set title of command window 39 | title %0 40 | @REM enable echoing by setting MAVEN_BATCH_ECHO to 'on' 41 | @if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO% 42 | 43 | @REM set %HOME% to equivalent of $HOME 44 | if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%") 45 | 46 | @REM Execute a user defined script before this one 47 | if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre 48 | @REM check for pre script, once with legacy .bat ending and once with .cmd ending 49 | if exist "%HOME%\mavenrc_pre.bat" call "%HOME%\mavenrc_pre.bat" 50 | if exist "%HOME%\mavenrc_pre.cmd" call "%HOME%\mavenrc_pre.cmd" 51 | :skipRcPre 52 | 53 | @setlocal 54 | 55 | set ERROR_CODE=0 56 | 57 | @REM To isolate internal variables from possible post scripts, we use another setlocal 58 | @setlocal 59 | 60 | @REM ==== START VALIDATION ==== 61 | if not "%JAVA_HOME%" == "" goto OkJHome 62 | 63 | echo. 64 | echo Error: JAVA_HOME not found in your environment. >&2 65 | echo Please set the JAVA_HOME variable in your environment to match the >&2 66 | echo location of your Java installation. >&2 67 | echo. 68 | goto error 69 | 70 | :OkJHome 71 | if exist "%JAVA_HOME%\bin\java.exe" goto init 72 | 73 | echo. 74 | echo Error: JAVA_HOME is set to an invalid directory. >&2 75 | echo JAVA_HOME = "%JAVA_HOME%" >&2 76 | echo Please set the JAVA_HOME variable in your environment to match the >&2 77 | echo location of your Java installation. >&2 78 | echo. 79 | goto error 80 | 81 | @REM ==== END VALIDATION ==== 82 | 83 | :init 84 | 85 | @REM Find the project base dir, i.e. the directory that contains the folder ".mvn". 86 | @REM Fallback to current working directory if not found. 87 | 88 | set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR% 89 | IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir 90 | 91 | set EXEC_DIR=%CD% 92 | set WDIR=%EXEC_DIR% 93 | :findBaseDir 94 | IF EXIST "%WDIR%"\.mvn goto baseDirFound 95 | cd .. 96 | IF "%WDIR%"=="%CD%" goto baseDirNotFound 97 | set WDIR=%CD% 98 | goto findBaseDir 99 | 100 | :baseDirFound 101 | set MAVEN_PROJECTBASEDIR=%WDIR% 102 | cd "%EXEC_DIR%" 103 | goto endDetectBaseDir 104 | 105 | :baseDirNotFound 106 | set MAVEN_PROJECTBASEDIR=%EXEC_DIR% 107 | cd "%EXEC_DIR%" 108 | 109 | :endDetectBaseDir 110 | 111 | IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig 112 | 113 | @setlocal EnableExtensions EnableDelayedExpansion 114 | for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a 115 | @endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS% 116 | 117 | :endReadAdditionalConfig 118 | 119 | SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" 120 | set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar" 121 | set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain 122 | 123 | set DOWNLOAD_URL="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.5/maven-wrapper-0.5.5.jar" 124 | 125 | FOR /F "tokens=1,2 delims==" %%A IN ("%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties") DO ( 126 | IF "%%A"=="wrapperUrl" SET DOWNLOAD_URL=%%B 127 | ) 128 | 129 | @REM Extension to allow automatically downloading the maven-wrapper.jar from Maven-central 130 | @REM This allows using the maven wrapper in projects that prohibit checking in binary data. 131 | if exist %WRAPPER_JAR% ( 132 | if "%MVNW_VERBOSE%" == "true" ( 133 | echo Found %WRAPPER_JAR% 134 | ) 135 | ) else ( 136 | if not "%MVNW_REPOURL%" == "" ( 137 | SET DOWNLOAD_URL="%MVNW_REPOURL%/io/takari/maven-wrapper/0.5.5/maven-wrapper-0.5.5.jar" 138 | ) 139 | if "%MVNW_VERBOSE%" == "true" ( 140 | echo Couldn't find %WRAPPER_JAR%, downloading it ... 141 | echo Downloading from: %DOWNLOAD_URL% 142 | ) 143 | 144 | powershell -Command "&{"^ 145 | "$webclient = new-object System.Net.WebClient;"^ 146 | "if (-not ([string]::IsNullOrEmpty('%MVNW_USERNAME%') -and [string]::IsNullOrEmpty('%MVNW_PASSWORD%'))) {"^ 147 | "$webclient.Credentials = new-object System.Net.NetworkCredential('%MVNW_USERNAME%', '%MVNW_PASSWORD%');"^ 148 | "}"^ 149 | "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; $webclient.DownloadFile('%DOWNLOAD_URL%', '%WRAPPER_JAR%')"^ 150 | "}" 151 | if "%MVNW_VERBOSE%" == "true" ( 152 | echo Finished downloading %WRAPPER_JAR% 153 | ) 154 | ) 155 | @REM End of extension 156 | 157 | @REM Provide a "standardized" way to retrieve the CLI args that will 158 | @REM work with both Windows and non-Windows executions. 159 | set MAVEN_CMD_LINE_ARGS=%* 160 | 161 | %MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %* 162 | if ERRORLEVEL 1 goto error 163 | goto end 164 | 165 | :error 166 | set ERROR_CODE=1 167 | 168 | :end 169 | @endlocal & set ERROR_CODE=%ERROR_CODE% 170 | 171 | if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost 172 | @REM check for post script, once with legacy .bat ending and once with .cmd ending 173 | if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat" 174 | if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd" 175 | :skipRcPost 176 | 177 | @REM pause the script if MAVEN_BATCH_PAUSE is set to 'on' 178 | if "%MAVEN_BATCH_PAUSE%" == "on" pause 179 | 180 | if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE% 181 | 182 | exit /B %ERROR_CODE% 183 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | 7 | org.springframework.boot 8 | spring-boot-starter-parent 9 | 2.2.1.RELEASE 10 | 11 | 12 | 13 | pom 14 | 15 | 16 | worker-paginator-calc 17 | worker-paginator 18 | core 19 | 20 | 21 | elasticsearch-paginator 22 | elasticsearch-paginator 23 | 0.0.1-SNAPSHOT 24 | 25 | 26 | 1.8 27 | 1.3.50 28 | 29 | 7.4.2 30 | 31 | 32 | 33 | 34 | org.springframework.boot 35 | spring-boot-starter-webflux 36 | 37 | 38 | 39 | org.springframework.boot 40 | spring-boot-starter-amqp 41 | 42 | 43 | io.projectreactor.rabbitmq 44 | reactor-rabbitmq 45 | 46 | 47 | 48 | org.elasticsearch.client 49 | elasticsearch-rest-high-level-client 50 | ${elasticsearch.version} 51 | 52 | 53 | 54 | org.springframework.boot 55 | spring-boot-starter-actuator 56 | 57 | 58 | io.micrometer 59 | micrometer-registry-prometheus 60 | runtime 61 | 62 | 63 | 64 | com.fasterxml.jackson.module 65 | jackson-module-kotlin 66 | 67 | 68 | org.jetbrains.kotlin 69 | kotlin-reflect 70 | 71 | 72 | org.jetbrains.kotlin 73 | kotlin-stdlib-jdk8 74 | 75 | 76 | org.jetbrains.kotlinx 77 | kotlinx-coroutines-reactor 78 | 79 | 80 | 81 | org.springframework.boot 82 | spring-boot-starter-test 83 | test 84 | 85 | 86 | org.junit.vintage 87 | junit-vintage-engine 88 | 89 | 90 | 91 | 92 | io.projectreactor 93 | reactor-test 94 | test 95 | 96 | 97 | org.testcontainers 98 | testcontainers 99 | 1.12.3 100 | test 101 | 102 | 103 | org.testcontainers 104 | rabbitmq 105 | 1.12.3 106 | 107 | 108 | org.testcontainers 109 | elasticsearch 110 | 1.12.3 111 | 112 | 113 | org.jeasy 114 | easy-random-core 115 | 4.1.0 116 | test 117 | 118 | 119 | 120 | 121 | ${project.basedir}/src/main/kotlin 122 | ${project.basedir}/src/test/kotlin 123 | 124 | 125 | org.jetbrains.kotlin 126 | kotlin-maven-plugin 127 | 128 | 129 | -Xjsr305=strict 130 | 131 | 132 | spring 133 | 134 | 135 | 136 | 137 | org.jetbrains.kotlin 138 | kotlin-maven-allopen 139 | ${kotlin.version} 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | -------------------------------------------------------------------------------- /worker-paginator-calc/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM openjdk:11 2 | 3 | EXPOSE 8080 4 | EXPOSE 8081 5 | 6 | ADD /target/*.jar /app/application.jar 7 | 8 | ENTRYPOINT java $JAVA_OPTS -jar /app/application.jar 9 | -------------------------------------------------------------------------------- /worker-paginator-calc/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | 8 | elasticsearch-paginator 9 | elasticsearch-paginator 10 | 0.0.1-SNAPSHOT 11 | 12 | 13 | worker-paginator-calc 14 | 15 | 16 | 17 | ${project.groupId} 18 | core 19 | ${project.version} 20 | 21 | 22 | 23 | 24 | 25 | 26 | org.springframework.boot 27 | spring-boot-maven-plugin 28 | 29 | 30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /worker-paginator-calc/src/main/kotlin/elasticsearchpaginator/workerpaginatorcalc/WorkerPaginatorCalcApplication.kt: -------------------------------------------------------------------------------- 1 | package elasticsearchpaginator.workerpaginatorcalc 2 | 3 | import org.springframework.boot.autoconfigure.SpringBootApplication 4 | import org.springframework.boot.runApplication 5 | 6 | @SpringBootApplication 7 | class WorkerPaginatorCalcApplication 8 | 9 | fun main(args: Array) { 10 | runApplication(*args) 11 | } 12 | -------------------------------------------------------------------------------- /worker-paginator-calc/src/main/kotlin/elasticsearchpaginator/workerpaginatorcalc/configuration/ElasticsearchConfiguration.kt: -------------------------------------------------------------------------------- 1 | package elasticsearchpaginator.workerpaginatorcalc.configuration 2 | 3 | import elasticsearchpaginator.workerpaginatorcalc.repository.PageRepository 4 | import org.springframework.beans.factory.InitializingBean 5 | import org.springframework.boot.context.properties.EnableConfigurationProperties 6 | import org.springframework.context.annotation.Configuration 7 | 8 | @Configuration 9 | @EnableConfigurationProperties(ElasticsearchProperties::class) 10 | class ElasticsearchConfiguration(private val pageRepository: PageRepository) : InitializingBean { 11 | 12 | /** 13 | * Create pages index if not exists 14 | */ 15 | override fun afterPropertiesSet() { 16 | this.pageRepository.existIndex() 17 | .filter { exists -> !exists } 18 | .flatMap { this.pageRepository.createIndex() } 19 | .block() 20 | } 21 | 22 | } 23 | -------------------------------------------------------------------------------- /worker-paginator-calc/src/main/kotlin/elasticsearchpaginator/workerpaginatorcalc/configuration/ElasticsearchProperties.kt: -------------------------------------------------------------------------------- 1 | package elasticsearchpaginator.workerpaginatorcalc.configuration 2 | 3 | import org.springframework.boot.context.properties.ConfigurationProperties 4 | import org.springframework.boot.context.properties.ConstructorBinding 5 | import java.time.Duration 6 | 7 | @ConstructorBinding 8 | @ConfigurationProperties("app.elasticsearch") 9 | data class ElasticsearchProperties( 10 | val scrollKeepAliveDuration: Duration, 11 | val pagesIndex: String, 12 | val pagesIndexNumberShards: Int, 13 | val pagesIndexNumberReplicas: Int, 14 | val pagesBulkSize: Int 15 | ) 16 | -------------------------------------------------------------------------------- /worker-paginator-calc/src/main/kotlin/elasticsearchpaginator/workerpaginatorcalc/configuration/RabbitmqConfiguration.kt: -------------------------------------------------------------------------------- 1 | package elasticsearchpaginator.workerpaginatorcalc.configuration 2 | 3 | import elasticsearchpaginator.core.util.RabbitmqUtils.createExchange 4 | import elasticsearchpaginator.core.util.RabbitmqUtils.createQueues 5 | import org.springframework.amqp.core.AmqpAdmin 6 | import org.springframework.beans.factory.InitializingBean 7 | import org.springframework.beans.factory.annotation.Value 8 | import org.springframework.boot.context.properties.EnableConfigurationProperties 9 | import org.springframework.context.annotation.Configuration 10 | 11 | @Configuration 12 | @EnableConfigurationProperties(RabbitmqProperties::class) 13 | class RabbitmqConfiguration(private val amqpAdmin: AmqpAdmin, 14 | private val rabbitmqProperties: RabbitmqProperties, 15 | @Value("\${spring.application.name}") private val applicationName: String) : InitializingBean { 16 | 17 | override fun afterPropertiesSet() { 18 | val exchange = this.amqpAdmin.createExchange(this.rabbitmqProperties.exchangeName) 19 | this.amqpAdmin.createQueues(this.computePagesQueueName(), this.computePagesDeadLetterQueueName(), this.rabbitmqProperties.computePagesKey, exchange) 20 | this.amqpAdmin.createQueues(this.deletePagesQueueName(), this.deletePagesDeadLetterQueueName(), this.rabbitmqProperties.deletePagesKey, exchange) 21 | } 22 | 23 | fun computePagesQueueName(): String { 24 | return "${this.applicationName}.${this.rabbitmqProperties.exchangeName}.${this.rabbitmqProperties.computePagesKey}" 25 | } 26 | 27 | fun computePagesDeadLetterQueueName(): String { 28 | return "${this.applicationName}${this.rabbitmqProperties.exchangeName}.${this.rabbitmqProperties.computePagesKey}.dead-letter" 29 | } 30 | 31 | fun deletePagesQueueName(): String { 32 | return "${this.applicationName}.${this.rabbitmqProperties.exchangeName}.${this.rabbitmqProperties.deletePagesKey}" 33 | } 34 | 35 | fun deletePagesDeadLetterQueueName(): String { 36 | return "${this.applicationName}.${this.rabbitmqProperties.exchangeName}.${this.rabbitmqProperties.deletePagesKey}.dead-letter" 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /worker-paginator-calc/src/main/kotlin/elasticsearchpaginator/workerpaginatorcalc/configuration/RabbitmqProperties.kt: -------------------------------------------------------------------------------- 1 | package elasticsearchpaginator.workerpaginatorcalc.configuration 2 | 3 | import org.springframework.boot.context.properties.ConfigurationProperties 4 | import org.springframework.boot.context.properties.ConstructorBinding 5 | 6 | @ConstructorBinding 7 | @ConfigurationProperties("app.rabbitmq") 8 | data class RabbitmqProperties( 9 | val exchangeName: String, 10 | val computePagesKey: String, 11 | val deletePagesKey: String, 12 | val deleteQueriesKey: String 13 | ) 14 | -------------------------------------------------------------------------------- /worker-paginator-calc/src/main/kotlin/elasticsearchpaginator/workerpaginatorcalc/exception/UnexpectedQueryException.kt: -------------------------------------------------------------------------------- 1 | package elasticsearchpaginator.workerpaginatorcalc.exception 2 | 3 | class UnexpectedQueryException(override val message: String) : Exception() 4 | -------------------------------------------------------------------------------- /worker-paginator-calc/src/main/kotlin/elasticsearchpaginator/workerpaginatorcalc/model/Page.kt: -------------------------------------------------------------------------------- 1 | package elasticsearchpaginator.workerpaginatorcalc.model 2 | 3 | data class Page( 4 | val queryId: String, 5 | val page: Long, 6 | val searchAfterQueryParameters: Any 7 | ) 8 | -------------------------------------------------------------------------------- /worker-paginator-calc/src/main/kotlin/elasticsearchpaginator/workerpaginatorcalc/repository/EntityElasticsearchRepository.kt: -------------------------------------------------------------------------------- 1 | package elasticsearchpaginator.workerpaginatorcalc.repository 2 | 3 | import elasticsearchpaginator.core.util.ElasticsearchUtils.async 4 | import elasticsearchpaginator.workerpaginatorcalc.configuration.ElasticsearchProperties 5 | import org.elasticsearch.action.search.* 6 | import org.elasticsearch.client.RequestOptions 7 | import org.elasticsearch.client.RestHighLevelClient 8 | import org.elasticsearch.common.settings.Settings 9 | import org.elasticsearch.common.unit.TimeValue 10 | import org.elasticsearch.common.xcontent.DeprecationHandler 11 | import org.elasticsearch.common.xcontent.NamedXContentRegistry 12 | import org.elasticsearch.common.xcontent.XContentFactory 13 | import org.elasticsearch.common.xcontent.XContentType 14 | import org.elasticsearch.search.SearchModule 15 | import org.elasticsearch.search.builder.SearchSourceBuilder 16 | import org.elasticsearch.search.sort.FieldSortBuilder 17 | import org.springframework.stereotype.Repository 18 | import reactor.core.publisher.Mono 19 | import java.util.* 20 | 21 | 22 | @Repository 23 | class EntityElasticsearchRepository(private val restHighLevelClient: RestHighLevelClient, 24 | private val elasticsearchProperties: ElasticsearchProperties) { 25 | 26 | private val scrollKeepAlive = TimeValue.timeValueMillis(this.elasticsearchProperties.scrollKeepAliveDuration.toMillis()) 27 | 28 | fun searchScroll(index: String, query: String, sort: String, size: Int): Mono { 29 | val payload = """ 30 | { 31 | "query": $query, 32 | "sort": $sort 33 | } 34 | """.trimIndent() 35 | 36 | return Mono.fromCallable { 37 | SearchSourceBuilder() 38 | .apply { 39 | val searchModule = SearchModule(Settings.EMPTY, false, Collections.emptyList()) 40 | val parser = XContentFactory.xContent(XContentType.JSON) 41 | .createParser( 42 | NamedXContentRegistry(searchModule.namedXContents), 43 | DeprecationHandler.THROW_UNSUPPORTED_OPERATION, 44 | payload 45 | ) 46 | this.parseXContent(parser) 47 | } 48 | .apply { 49 | val includes = this.sorts() 50 | .filterIsInstance() 51 | .map { fieldSortBuilder -> 52 | fieldSortBuilder.fieldName.removeSuffix(".keyword") 53 | } 54 | .toTypedArray() 55 | this.fetchSource(includes, null) 56 | } 57 | } 58 | .map { searchSourceBuilder -> 59 | SearchRequest() 60 | .indices(index) 61 | .source(searchSourceBuilder) 62 | .scroll(this.scrollKeepAlive) 63 | } 64 | .flatMap { searchRequest -> 65 | async { actionListener -> 66 | this.restHighLevelClient.searchAsync(searchRequest, RequestOptions.DEFAULT, actionListener) 67 | } 68 | } 69 | } 70 | 71 | fun scroll(scrollId: String): Mono { 72 | return Mono.just( 73 | SearchScrollRequest(scrollId) 74 | .scroll(this.scrollKeepAlive) 75 | ) 76 | .flatMap { searchScrollRequest -> 77 | async { actionListener -> 78 | this.restHighLevelClient.scrollAsync(searchScrollRequest, RequestOptions.DEFAULT, actionListener) 79 | } 80 | } 81 | } 82 | 83 | fun clearScroll(scrollId: String): Mono { 84 | return Mono.just( 85 | ClearScrollRequest() 86 | .apply { this.addScrollId(scrollId) } 87 | ) 88 | .flatMap { clearScrollRequest -> 89 | async { actionListener -> 90 | this.restHighLevelClient.clearScrollAsync(clearScrollRequest, RequestOptions.DEFAULT, actionListener) 91 | } 92 | } 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /worker-paginator-calc/src/main/kotlin/elasticsearchpaginator/workerpaginatorcalc/repository/PageRepository.kt: -------------------------------------------------------------------------------- 1 | package elasticsearchpaginator.workerpaginatorcalc.repository 2 | 3 | import com.fasterxml.jackson.databind.ObjectMapper 4 | import elasticsearchpaginator.core.util.ElasticsearchUtils.NUMBER_REPLICAS_SETTING 5 | import elasticsearchpaginator.core.util.ElasticsearchUtils.NUMBER_SHARDS_SETTING 6 | import elasticsearchpaginator.core.util.ElasticsearchUtils.async 7 | import elasticsearchpaginator.workerpaginatorcalc.configuration.ElasticsearchProperties 8 | import elasticsearchpaginator.workerpaginatorcalc.model.Page 9 | import org.elasticsearch.action.bulk.BulkRequest 10 | import org.elasticsearch.action.bulk.BulkResponse 11 | import org.elasticsearch.action.index.IndexRequest 12 | import org.elasticsearch.client.RequestOptions 13 | import org.elasticsearch.client.RestHighLevelClient 14 | import org.elasticsearch.client.indices.CreateIndexRequest 15 | import org.elasticsearch.client.indices.CreateIndexResponse 16 | import org.elasticsearch.client.indices.GetIndexRequest 17 | import org.elasticsearch.common.settings.Settings 18 | import org.elasticsearch.common.xcontent.XContentType 19 | import org.elasticsearch.index.query.TermQueryBuilder 20 | import org.elasticsearch.index.reindex.BulkByScrollResponse 21 | import org.elasticsearch.index.reindex.DeleteByQueryRequest 22 | import org.reactivestreams.Publisher 23 | import org.springframework.beans.factory.annotation.Value 24 | import org.springframework.core.io.Resource 25 | import org.springframework.stereotype.Repository 26 | import reactor.core.publisher.Flux 27 | import reactor.core.publisher.Mono 28 | import java.io.InputStreamReader 29 | 30 | @Repository 31 | class PageRepository(private val restHighLevelClient: RestHighLevelClient, 32 | private val mapper: ObjectMapper, 33 | private val elasticsearchProperties: ElasticsearchProperties, 34 | @Value("classpath:pages-mappings.json") private val pagesMappings: Resource) { 35 | 36 | fun createIndex(): Mono { 37 | return Mono.fromCallable { InputStreamReader(this.pagesMappings.inputStream).readText() } 38 | .map { mappings -> 39 | val settings = Settings.builder() 40 | .put(NUMBER_SHARDS_SETTING, this.elasticsearchProperties.pagesIndexNumberShards) 41 | .put(NUMBER_REPLICAS_SETTING, this.elasticsearchProperties.pagesIndexNumberReplicas) 42 | .build() 43 | 44 | CreateIndexRequest(this.elasticsearchProperties.pagesIndex) 45 | .settings(settings) 46 | .mapping(mappings, XContentType.JSON) 47 | } 48 | .flatMap { createIndexRequest -> 49 | async { actionListener -> 50 | this.restHighLevelClient.indices().createAsync(createIndexRequest, RequestOptions.DEFAULT, actionListener) 51 | } 52 | } 53 | .then() 54 | } 55 | 56 | fun existIndex(): Mono { 57 | return Mono.just( 58 | GetIndexRequest(this.elasticsearchProperties.pagesIndex) 59 | ) 60 | .flatMap { getIndexRequest -> 61 | async { actionListener -> 62 | this.restHighLevelClient.indices().existsAsync(getIndexRequest, RequestOptions.DEFAULT, actionListener) 63 | } 64 | } 65 | } 66 | 67 | fun save(pages: Publisher): Flux { 68 | return Flux.from(pages) 69 | .map { page -> 70 | Pair( 71 | IndexRequest(this.elasticsearchProperties.pagesIndex) 72 | .id("${page.queryId}_${page.page}") 73 | .source(this.mapper.writeValueAsBytes(page), XContentType.JSON), 74 | page 75 | ) 76 | } 77 | .buffer(this.elasticsearchProperties.pagesBulkSize) 78 | .filter(List>::isNotEmpty) 79 | .flatMap { indexRequests -> 80 | val bulkRequest = BulkRequest().add(indexRequests.map(Pair::first)) 81 | async { actionListener -> 82 | this.restHighLevelClient.bulkAsync(bulkRequest, RequestOptions.DEFAULT, actionListener) 83 | } 84 | .thenMany(Flux.fromIterable(indexRequests.map(Pair::second))) 85 | } 86 | } 87 | 88 | fun deleteByQueryId(id: String): Mono { 89 | return Mono.just( 90 | DeleteByQueryRequest(this.elasticsearchProperties.pagesIndex) 91 | .setQuery(TermQueryBuilder("queryId", id)) 92 | ) 93 | .flatMap { deleteByQueryRequest -> 94 | async { actionListener -> 95 | this.restHighLevelClient.deleteByQueryAsync(deleteByQueryRequest, RequestOptions.DEFAULT, actionListener) 96 | } 97 | } 98 | .then() 99 | } 100 | 101 | } 102 | -------------------------------------------------------------------------------- /worker-paginator-calc/src/main/kotlin/elasticsearchpaginator/workerpaginatorcalc/service/ComputePagesService.kt: -------------------------------------------------------------------------------- 1 | package elasticsearchpaginator.workerpaginatorcalc.service 2 | 3 | import com.fasterxml.jackson.databind.ObjectMapper 4 | import com.fasterxml.jackson.module.kotlin.readValue 5 | import elasticsearchpaginator.core.model.Query 6 | import elasticsearchpaginator.workerpaginatorcalc.model.Page 7 | import elasticsearchpaginator.workerpaginatorcalc.repository.EntityElasticsearchRepository 8 | import elasticsearchpaginator.workerpaginatorcalc.repository.PageRepository 9 | import org.elasticsearch.action.search.SearchResponse 10 | import org.springframework.stereotype.Service 11 | import reactor.core.publisher.Flux 12 | import reactor.core.publisher.Mono 13 | 14 | @Service 15 | class ComputePagesService(private val pageRepository: PageRepository, 16 | private val mapper: ObjectMapper, 17 | private val entityElasticsearchRepository: EntityElasticsearchRepository) { 18 | 19 | fun computePages(query: Query): Mono { 20 | return this.getLastEntityForEachPage(query) 21 | .index() 22 | .map { indexAndEntity -> 23 | Page( 24 | queryId = query.hash(), 25 | page = indexAndEntity.t1 + 1, 26 | searchAfterQueryParameters = indexAndEntity.t2 27 | ) 28 | } 29 | .transform(this.pageRepository::save) 30 | .then() 31 | } 32 | 33 | private fun getLastEntityForEachPage(query: Query): Flux { 34 | return this.findAllEntitiesWithTotalHits(query) 35 | .index() 36 | .filter { indexAndEntity -> 37 | this.matchesALastPosition( 38 | index = indexAndEntity.t1, 39 | firstPageSize = query.firstPageSize ?: query.size, 40 | size = query.size, 41 | totalHits =indexAndEntity.t2.second 42 | ) 43 | } 44 | .map { indexAndEnity -> indexAndEnity.t2.first } 45 | } 46 | 47 | private fun findAllEntitiesWithTotalHits(query: Query): Flux> { 48 | return this.entityElasticsearchRepository.searchScroll(query.index, query.query, query.sort, query.size) 49 | .expand { searchResponse -> 50 | if (searchResponse.hits.hits.isEmpty()) { 51 | this.entityElasticsearchRepository.clearScroll(searchResponse.scrollId) 52 | .then(Mono.empty()) 53 | } else { 54 | this.entityElasticsearchRepository.scroll(searchResponse.scrollId) 55 | } 56 | } 57 | .flatMapIterable { searchResponse -> 58 | searchResponse.hits.map { searchHit -> 59 | Pair(this.mapper.readValue(searchHit.sourceRef.streamInput()), searchResponse.hits.totalHits!!.value) 60 | } 61 | } 62 | } 63 | 64 | private fun matchesALastPosition(index: Long, firstPageSize: Int, size: Int, totalHits: Long): Boolean { 65 | return index == firstPageSize - 1L || (index - firstPageSize + 1L) % size == 0L && index != totalHits - 1L 66 | } 67 | 68 | } 69 | -------------------------------------------------------------------------------- /worker-paginator-calc/src/main/kotlin/elasticsearchpaginator/workerpaginatorcalc/service/DeletePagesService.kt: -------------------------------------------------------------------------------- 1 | package elasticsearchpaginator.workerpaginatorcalc.service 2 | 3 | import elasticsearchpaginator.core.model.DeleteQuery 4 | import elasticsearchpaginator.workerpaginatorcalc.repository.PageRepository 5 | import elasticsearchpaginator.workerpaginatorcalc.transport.DeleteQuerySender 6 | import org.springframework.stereotype.Service 7 | import reactor.core.publisher.Mono 8 | 9 | @Service 10 | class DeletePagesService(private val pageRepository: PageRepository, 11 | private val deleteQuerySender: DeleteQuerySender) { 12 | 13 | fun deletePages(queryId: String): Mono { 14 | return Mono.just(queryId) 15 | .flatMap(this.pageRepository::deleteByQueryId) 16 | .thenReturn( 17 | DeleteQuery( 18 | queryId = queryId 19 | ) 20 | ) 21 | .flatMap(this.deleteQuerySender::sendDeleteQueryEvent) 22 | } 23 | 24 | } 25 | -------------------------------------------------------------------------------- /worker-paginator-calc/src/main/kotlin/elasticsearchpaginator/workerpaginatorcalc/transport/ComputePagesReceiver.kt: -------------------------------------------------------------------------------- 1 | package elasticsearchpaginator.workerpaginatorcalc.transport 2 | 3 | import com.fasterxml.jackson.databind.ObjectMapper 4 | import elasticsearchpaginator.core.model.Query 5 | import elasticsearchpaginator.core.transport.AbstractRabbitmqReceiver 6 | import elasticsearchpaginator.workerpaginatorcalc.configuration.RabbitmqConfiguration 7 | import elasticsearchpaginator.workerpaginatorcalc.service.ComputePagesService 8 | import org.springframework.context.ApplicationContext 9 | import org.springframework.stereotype.Component 10 | import reactor.core.publisher.Mono 11 | import reactor.rabbitmq.Receiver 12 | 13 | @Component 14 | class ComputePagesReceiver(override val receiver: Receiver, 15 | override val mapper: ObjectMapper, 16 | override val applicationContext: ApplicationContext, 17 | private val computePagesService: ComputePagesService, 18 | private val rabbitmqConfiguration: RabbitmqConfiguration) : AbstractRabbitmqReceiver() { 19 | 20 | override val queueName = this.rabbitmqConfiguration.computePagesQueueName() 21 | override val eventClass = Query::class.java 22 | 23 | override fun eventHandler(event: Query): Mono { 24 | return this.computePagesService.computePages(event) 25 | } 26 | 27 | } 28 | -------------------------------------------------------------------------------- /worker-paginator-calc/src/main/kotlin/elasticsearchpaginator/workerpaginatorcalc/transport/DeletePagesReceiver.kt: -------------------------------------------------------------------------------- 1 | package elasticsearchpaginator.workerpaginatorcalc.transport 2 | 3 | import com.fasterxml.jackson.databind.ObjectMapper 4 | import elasticsearchpaginator.core.model.DeletePages 5 | import elasticsearchpaginator.core.transport.AbstractRabbitmqReceiver 6 | import elasticsearchpaginator.workerpaginatorcalc.configuration.RabbitmqConfiguration 7 | import elasticsearchpaginator.workerpaginatorcalc.service.DeletePagesService 8 | import org.springframework.context.ApplicationContext 9 | import org.springframework.stereotype.Component 10 | import reactor.core.publisher.Mono 11 | import reactor.rabbitmq.Receiver 12 | 13 | @Component 14 | class DeletePagesReceiver(override val receiver: Receiver, 15 | override val mapper: ObjectMapper, 16 | override val applicationContext: ApplicationContext, 17 | private val deletePagesService: DeletePagesService, 18 | private val rabbitmqConfiguration: RabbitmqConfiguration) : AbstractRabbitmqReceiver() { 19 | 20 | override val queueName = this.rabbitmqConfiguration.deletePagesQueueName() 21 | override val eventClass = DeletePages::class.java 22 | 23 | override fun eventHandler(event: DeletePages): Mono { 24 | return this.deletePagesService.deletePages(event.queryId) 25 | } 26 | 27 | } 28 | -------------------------------------------------------------------------------- /worker-paginator-calc/src/main/kotlin/elasticsearchpaginator/workerpaginatorcalc/transport/DeleteQuerySender.kt: -------------------------------------------------------------------------------- 1 | package elasticsearchpaginator.workerpaginatorcalc.transport 2 | 3 | import com.fasterxml.jackson.databind.ObjectMapper 4 | import elasticsearchpaginator.core.model.DeleteQuery 5 | import elasticsearchpaginator.workerpaginatorcalc.configuration.RabbitmqProperties 6 | import org.springframework.stereotype.Component 7 | import reactor.core.publisher.Mono 8 | import reactor.rabbitmq.OutboundMessage 9 | import reactor.rabbitmq.Sender 10 | 11 | @Component 12 | class DeleteQuerySender(private val sender: Sender, 13 | private val mapper: ObjectMapper, 14 | private val rabbitmqProperties: RabbitmqProperties) { 15 | 16 | fun sendDeleteQueryEvent(deleteQuery: DeleteQuery): Mono { 17 | return this.sender.send( 18 | Mono.just( 19 | OutboundMessage( 20 | this.rabbitmqProperties.exchangeName, 21 | this.rabbitmqProperties.deleteQueriesKey, 22 | this.mapper.writeValueAsBytes(deleteQuery) 23 | ) 24 | ) 25 | ) 26 | } 27 | 28 | } 29 | -------------------------------------------------------------------------------- /worker-paginator-calc/src/main/resources/application.yml: -------------------------------------------------------------------------------- 1 | management: 2 | server: 3 | port: 8081 4 | 5 | server: 6 | port: 8080 7 | 8 | spring: 9 | application: 10 | name: worker-paginator-calc 11 | rabbitmq: 12 | host: localhost 13 | port: 5672 14 | username: guest 15 | password: guest 16 | elasticsearch: 17 | rest: 18 | uris: http://locahost:9200 19 | 20 | app: 21 | elasticsearch: 22 | scroll-keep-alive-duration: 60s 23 | pages-index: pages 24 | pages-index-number-replicas: 2 25 | pages-index-number-shards: 5 26 | pages-bulk-size: 1000 27 | rabbitmq: 28 | exchange-name: paginator 29 | compute-pages-key: compute-pages 30 | delete-pages-key: delete-pages 31 | delete-queries-key: delete-queries 32 | -------------------------------------------------------------------------------- /worker-paginator-calc/src/main/resources/pages-mappings.json: -------------------------------------------------------------------------------- 1 | { 2 | "properties": { 3 | "queryId": { 4 | "type": "keyword" 5 | }, 6 | "page": { 7 | "type": "integer" 8 | }, 9 | "searchAfterQueryParameters": { 10 | "enabled": false 11 | } 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /worker-paginator-calc/src/test/kotlin/elasticsearchpaginator/workerpaginatorcalc/AbstractIntegrationTest.kt: -------------------------------------------------------------------------------- 1 | package elasticsearchpaginator.workerpaginatorcalc 2 | 3 | import com.fasterxml.jackson.databind.ObjectMapper 4 | import com.fasterxml.jackson.module.kotlin.readValue 5 | import elasticsearchpaginator.core.util.ElasticsearchUtils 6 | import elasticsearchpaginator.workerpaginatorcalc.configuration.ElasticsearchProperties 7 | import elasticsearchpaginator.workerpaginatorcalc.model.Page 8 | import org.elasticsearch.action.admin.indices.refresh.RefreshRequest 9 | import org.elasticsearch.action.admin.indices.refresh.RefreshResponse 10 | import org.elasticsearch.action.search.SearchRequest 11 | import org.elasticsearch.action.search.SearchResponse 12 | import org.elasticsearch.client.RequestOptions 13 | import org.elasticsearch.client.RestHighLevelClient 14 | import org.elasticsearch.index.query.QueryBuilders 15 | import org.elasticsearch.index.reindex.BulkByScrollResponse 16 | import org.elasticsearch.index.reindex.DeleteByQueryRequest 17 | import org.elasticsearch.search.builder.SearchSourceBuilder 18 | import org.elasticsearch.search.sort.SortBuilders 19 | import org.elasticsearch.search.sort.SortOrder 20 | import org.junit.jupiter.api.extension.ExtendWith 21 | import org.springframework.beans.factory.annotation.Autowired 22 | import org.springframework.boot.test.autoconfigure.web.reactive.AutoConfigureWebTestClient 23 | import org.springframework.boot.test.context.SpringBootTest 24 | import org.springframework.boot.test.util.TestPropertyValues 25 | import org.springframework.context.ApplicationContextInitializer 26 | import org.springframework.context.ConfigurableApplicationContext 27 | import org.springframework.test.context.ContextConfiguration 28 | import org.springframework.test.context.junit.jupiter.SpringExtension 29 | import org.testcontainers.containers.RabbitMQContainer 30 | import org.testcontainers.elasticsearch.ElasticsearchContainer 31 | import reactor.core.publisher.Flux 32 | import reactor.core.publisher.Mono 33 | import reactor.test.StepVerifier 34 | import java.time.Duration 35 | 36 | @ExtendWith(SpringExtension::class) 37 | @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) 38 | @ContextConfiguration(initializers = [AbstractIntegrationTest.Initializer::class]) 39 | @AutoConfigureWebTestClient 40 | abstract class AbstractIntegrationTest { 41 | 42 | init { 43 | StepVerifier.setDefaultTimeout(Duration.ofSeconds(5)) 44 | } 45 | 46 | @Autowired 47 | protected lateinit var restHighLevelClient: RestHighLevelClient 48 | 49 | @Autowired 50 | protected lateinit var mapper: ObjectMapper 51 | 52 | @Autowired 53 | protected lateinit var elasticsearchProperties: ElasticsearchProperties 54 | 55 | companion object { 56 | private val RABBITMQ_USERNAME = "guest" 57 | private val RABBITMQ_PASSWORD = "guest" 58 | 59 | val elasticsearchContainer = ElasticsearchContainer("docker.elastic.co/elasticsearch/elasticsearch-oss:7.4.2") 60 | val rabbitmqContainer = RabbitMQContainer("rabbitmq:3.8") 61 | .withUser(RABBITMQ_USERNAME, RABBITMQ_PASSWORD) 62 | } 63 | 64 | internal class Initializer : ApplicationContextInitializer { 65 | override fun initialize(configurableApplicationContext: ConfigurableApplicationContext) { 66 | rabbitmqContainer.start() 67 | elasticsearchContainer.start() 68 | 69 | TestPropertyValues.of( 70 | "spring.elasticsearch.rest.uris=${elasticsearchContainer.httpHostAddress}", 71 | "spring.rabbitmq.host=${rabbitmqContainer.containerIpAddress}", 72 | "spring.rabbitmq.port=${rabbitmqContainer.firstMappedPort}", 73 | "spring.rabbitmq.username=$RABBITMQ_USERNAME", 74 | "spring.rabbitmq.password=$RABBITMQ_PASSWORD" 75 | ) 76 | .applyTo(configurableApplicationContext.environment) 77 | } 78 | } 79 | 80 | protected fun refreshPages(): Mono { 81 | return Mono.just( 82 | RefreshRequest() 83 | .indices(this.elasticsearchProperties.pagesIndex) 84 | ) 85 | .flatMap { refreshRequest -> 86 | ElasticsearchUtils.async { actionListener -> 87 | this.restHighLevelClient.indices().refreshAsync(refreshRequest, RequestOptions.DEFAULT, actionListener) 88 | } 89 | } 90 | .then() 91 | } 92 | 93 | protected fun findAllPages(): Flux { 94 | return Mono.just( 95 | SearchSourceBuilder() 96 | .query( 97 | QueryBuilders.matchAllQuery() 98 | ) 99 | .sort( 100 | SortBuilders 101 | .fieldSort("page") 102 | .order(SortOrder.ASC) 103 | ) 104 | .size(10000) 105 | ) 106 | .map { searchSourceBuilder -> 107 | SearchRequest() 108 | .indices(this.elasticsearchProperties.pagesIndex) 109 | .source(searchSourceBuilder) 110 | } 111 | .flatMap { searchRequest -> 112 | ElasticsearchUtils.async { actionListener -> 113 | this.restHighLevelClient.searchAsync(searchRequest, RequestOptions.DEFAULT, actionListener) 114 | } 115 | } 116 | .flatMapIterable { searchResponse -> 117 | searchResponse.hits 118 | } 119 | .map { searchHit -> 120 | this.mapper.readValue(searchHit.sourceRef.streamInput()) 121 | } 122 | } 123 | 124 | protected fun clearPages(): Mono { 125 | return Mono.just( 126 | DeleteByQueryRequest(this.elasticsearchProperties.pagesIndex) 127 | .setQuery( 128 | QueryBuilders.matchAllQuery() 129 | ) 130 | .setRefresh(true) 131 | ) 132 | .flatMap { deleteByQueryRequest -> 133 | ElasticsearchUtils.async { actionListener -> 134 | this.restHighLevelClient.deleteByQueryAsync(deleteByQueryRequest, RequestOptions.DEFAULT, actionListener) 135 | } 136 | } 137 | .then() 138 | } 139 | 140 | } 141 | -------------------------------------------------------------------------------- /worker-paginator-calc/src/test/kotlin/elasticsearchpaginator/workerpaginatorcalc/ComputePagesIntegrationTest.kt: -------------------------------------------------------------------------------- 1 | package elasticsearchpaginator.workerpaginatorcalc 2 | 3 | import elasticsearchpaginator.core.model.Query 4 | import elasticsearchpaginator.core.util.ElasticsearchUtils 5 | import elasticsearchpaginator.workerpaginatorcalc.model.Page 6 | import elasticsearchpaginator.workerpaginatorcalc.service.ComputePagesService 7 | import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest 8 | import org.elasticsearch.action.bulk.BulkRequest 9 | import org.elasticsearch.action.bulk.BulkResponse 10 | import org.elasticsearch.action.index.IndexRequest 11 | import org.elasticsearch.action.support.WriteRequest 12 | import org.elasticsearch.action.support.master.AcknowledgedResponse 13 | import org.elasticsearch.client.RequestOptions 14 | import org.elasticsearch.common.xcontent.XContentType 15 | import org.elasticsearch.index.query.QueryBuilders 16 | import org.elasticsearch.search.sort.SortBuilders 17 | import org.elasticsearch.search.sort.SortOrder 18 | import org.jeasy.random.EasyRandom 19 | import org.junit.Assert 20 | import org.junit.jupiter.api.AfterAll 21 | import org.junit.jupiter.api.AfterEach 22 | import org.junit.jupiter.api.BeforeAll 23 | import org.junit.jupiter.api.Test 24 | import org.springframework.beans.factory.annotation.Autowired 25 | import reactor.core.publisher.Flux 26 | import reactor.core.publisher.Mono 27 | import reactor.test.StepVerifier 28 | import java.time.Instant 29 | import java.util.stream.Collectors 30 | 31 | class ComputePagesIntegrationTest : AbstractIntegrationTest() { 32 | 33 | private val indexName = "articles" 34 | 35 | val articles = EasyRandom().objects(Article::class.java, 22).collect(Collectors.toList()) 36 | 37 | @Autowired 38 | private lateinit var computePagesService: ComputePagesService 39 | 40 | @BeforeAll 41 | internal fun setUp() { 42 | StepVerifier.create(this.saveArticles(articles)) 43 | .verifyComplete() 44 | } 45 | 46 | @AfterAll 47 | internal fun tearDown() { 48 | StepVerifier.create(this.deleteArticlesIndex()) 49 | .verifyComplete() 50 | } 51 | 52 | @AfterEach 53 | internal fun cleanUp() { 54 | StepVerifier.create(this.clearPages()) 55 | .verifyComplete() 56 | } 57 | 58 | @Test 59 | fun `should compute pages for a basic query`() { 60 | val query = Query( 61 | index = indexName, 62 | size = 4, 63 | firstPageSize = 2, 64 | query = QueryBuilders.matchAllQuery().toString(), 65 | sort = listOf( 66 | SortBuilders 67 | .fieldSort("id.keyword") 68 | .order(SortOrder.ASC) 69 | .toString() 70 | ) 71 | .joinToString(",", "[", "]") 72 | ) 73 | val sortedArticles = articles 74 | .sortedBy { article -> article.id } 75 | 76 | StepVerifier.create(this.computePagesService.computePages(query)) 77 | .verifyComplete() 78 | 79 | StepVerifier.create(this.refreshPages().thenMany(this.findAllPages())) 80 | .assertNext { searchHit -> 81 | Assert.assertEquals( 82 | Page( 83 | queryId = query.hash(), 84 | page = 1, 85 | searchAfterQueryParameters = mapOf( 86 | "id" to sortedArticles[1].id 87 | ) 88 | ), 89 | searchHit 90 | ) 91 | } 92 | .assertNext { searchHit -> 93 | Assert.assertEquals( 94 | Page( 95 | queryId = query.hash(), 96 | page = 2, 97 | searchAfterQueryParameters = mapOf( 98 | "id" to sortedArticles[5].id 99 | ) 100 | ), 101 | searchHit 102 | ) 103 | } 104 | .assertNext { searchHit -> 105 | Assert.assertEquals( 106 | Page( 107 | queryId = query.hash(), 108 | page = 3, 109 | searchAfterQueryParameters = mapOf( 110 | "id" to sortedArticles[9].id 111 | ) 112 | ), 113 | searchHit 114 | ) 115 | } 116 | .assertNext { searchHit -> 117 | Assert.assertEquals( 118 | Page( 119 | queryId = query.hash(), 120 | page = 4, 121 | searchAfterQueryParameters = mapOf( 122 | "id" to sortedArticles[13].id 123 | ) 124 | ), 125 | searchHit 126 | ) 127 | } 128 | .assertNext { searchHit -> 129 | Assert.assertEquals( 130 | Page( 131 | queryId = query.hash(), 132 | page = 5, 133 | searchAfterQueryParameters = mapOf( 134 | "id" to sortedArticles[17].id 135 | ) 136 | ), 137 | searchHit 138 | ) 139 | } 140 | .verifyComplete() 141 | } 142 | 143 | @Test 144 | fun `should compute pages with sort on nested field`() { 145 | val query = Query( 146 | index = indexName, 147 | size = 4, 148 | firstPageSize = 2, 149 | query = QueryBuilders.matchAllQuery().toString(), 150 | sort = listOf( 151 | SortBuilders 152 | .fieldSort("content.title.keyword") 153 | .order(SortOrder.ASC) 154 | .toString() 155 | ) 156 | .joinToString(",", "[", "]") 157 | ) 158 | val sortedArticles = articles 159 | .sortedBy { article -> article.content.title } 160 | 161 | StepVerifier.create(this.computePagesService.computePages(query)) 162 | .verifyComplete() 163 | 164 | StepVerifier.create(this.refreshPages().thenMany(this.findAllPages())) 165 | .assertNext { searchHit -> 166 | Assert.assertEquals( 167 | Page( 168 | queryId = query.hash(), 169 | page = 1, 170 | searchAfterQueryParameters = mapOf( 171 | "content" to mapOf( 172 | "title" to sortedArticles[1].content.title 173 | ) 174 | ) 175 | ), 176 | searchHit 177 | ) 178 | 179 | } 180 | .assertNext { searchHit -> 181 | Assert.assertEquals( 182 | Page( 183 | queryId = query.hash(), 184 | page = 2, 185 | searchAfterQueryParameters = mapOf( 186 | "content" to mapOf( 187 | "title" to sortedArticles[5].content.title 188 | ) 189 | ) 190 | ), 191 | searchHit 192 | ) 193 | } 194 | .assertNext { searchHit -> 195 | Assert.assertEquals( 196 | Page( 197 | queryId = query.hash(), 198 | page = 3, 199 | searchAfterQueryParameters = mapOf( 200 | "content" to mapOf( 201 | "title" to sortedArticles[9].content.title 202 | ) 203 | ) 204 | ), 205 | searchHit 206 | ) 207 | } 208 | .assertNext { searchHit -> 209 | Assert.assertEquals( 210 | Page( 211 | queryId = query.hash(), 212 | page = 4, 213 | searchAfterQueryParameters = mapOf( 214 | "content" to mapOf( 215 | "title" to sortedArticles[13].content.title 216 | ) 217 | ) 218 | ), 219 | searchHit 220 | ) 221 | } 222 | .assertNext { searchHit -> 223 | Assert.assertEquals( 224 | Page( 225 | queryId = query.hash(), 226 | page = 5, 227 | searchAfterQueryParameters = mapOf( 228 | "content" to mapOf( 229 | "title" to sortedArticles[17].content.title 230 | ) 231 | ) 232 | ), 233 | searchHit 234 | ) 235 | } 236 | .verifyComplete() 237 | } 238 | 239 | private fun saveArticles(articles: List
): Mono { 240 | return Flux.fromIterable(articles) 241 | .map { article -> 242 | IndexRequest() 243 | .index(indexName) 244 | .id(article.id) 245 | .source(this.mapper.writeValueAsBytes(article), XContentType.JSON) 246 | } 247 | .collectList() 248 | .map { indexRequests -> 249 | BulkRequest() 250 | .add(indexRequests) 251 | .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) 252 | } 253 | .flatMap { bulkRequest -> 254 | ElasticsearchUtils.async { actionListener -> 255 | this.restHighLevelClient.bulkAsync(bulkRequest, RequestOptions.DEFAULT, actionListener) 256 | } 257 | } 258 | .then() 259 | } 260 | 261 | private fun deleteArticlesIndex(): Mono { 262 | return Mono.just( 263 | DeleteIndexRequest() 264 | .indices(indexName) 265 | ) 266 | .flatMap { deleteIndexRequest -> 267 | ElasticsearchUtils.async { actionListener -> 268 | this.restHighLevelClient.indices().deleteAsync(deleteIndexRequest, RequestOptions.DEFAULT, actionListener) 269 | } 270 | } 271 | .then() 272 | } 273 | 274 | data class Article( 275 | val id: String, 276 | val content: Content, 277 | val creationDate: Instant, 278 | val author: Author 279 | ) { 280 | 281 | data class Content( 282 | val title: String, 283 | val head: String, 284 | val body: String 285 | ) 286 | 287 | data class Author( 288 | val id: String, 289 | val firstName: String, 290 | val lastName: String 291 | ) 292 | 293 | } 294 | 295 | } 296 | -------------------------------------------------------------------------------- /worker-paginator-calc/src/test/kotlin/elasticsearchpaginator/workerpaginatorcalc/DeletePagesIntegrationTest.kt: -------------------------------------------------------------------------------- 1 | package elasticsearchpaginator.workerpaginatorcalc 2 | 3 | import elasticsearchpaginator.core.model.DeleteQuery 4 | import elasticsearchpaginator.core.util.RabbitmqUtils.createExchange 5 | import elasticsearchpaginator.core.util.RabbitmqUtils.createQueues 6 | import elasticsearchpaginator.workerpaginatorcalc.configuration.RabbitmqProperties 7 | import elasticsearchpaginator.workerpaginatorcalc.model.Page 8 | import elasticsearchpaginator.workerpaginatorcalc.repository.PageRepository 9 | import elasticsearchpaginator.workerpaginatorcalc.service.DeletePagesService 10 | import org.jeasy.random.EasyRandom 11 | import org.jeasy.random.EasyRandomParameters 12 | import org.jeasy.random.FieldPredicates 13 | import org.jeasy.random.api.Randomizer 14 | import org.junit.Assert 15 | import org.junit.jupiter.api.AfterEach 16 | import org.junit.jupiter.api.Test 17 | import org.springframework.amqp.core.AmqpAdmin 18 | import org.springframework.beans.factory.annotation.Autowired 19 | import reactor.core.publisher.Flux 20 | import reactor.rabbitmq.Receiver 21 | import reactor.test.StepVerifier 22 | import java.util.stream.Stream 23 | import kotlin.random.Random 24 | 25 | class DeletePagesIntegrationTest : AbstractIntegrationTest() { 26 | 27 | @Autowired 28 | private lateinit var deletePagesService: DeletePagesService 29 | 30 | @Autowired 31 | private lateinit var pageRepository: PageRepository 32 | 33 | @Autowired 34 | private lateinit var receiver: Receiver 35 | 36 | @Autowired 37 | private lateinit var amqpAdmin: AmqpAdmin 38 | 39 | @Autowired 40 | private lateinit var rabbitmqProperties: RabbitmqProperties 41 | 42 | @AfterEach 43 | internal fun cleanUp() { 44 | StepVerifier.create(this.clearPages()) 45 | .verifyComplete() 46 | } 47 | 48 | @Test 49 | fun `should delete all pages matching the given query id`() { 50 | val queryId = "queryId" 51 | val queryIdFieldPredicate = FieldPredicates 52 | .named("queryId") 53 | .and( 54 | FieldPredicates 55 | .ofType(String::class.java) 56 | ) 57 | .and( 58 | FieldPredicates 59 | .inClass(Page::class.java) 60 | ) 61 | val pageFieldPredicate = FieldPredicates 62 | .named("page") 63 | .and( 64 | FieldPredicates 65 | .ofType(Long::class.java) 66 | ) 67 | .and( 68 | FieldPredicates 69 | .inClass(Page::class.java) 70 | ) 71 | val searchAfterQueryParametersFieldPredicate = FieldPredicates 72 | .named("searchAfterQueryParameters") 73 | .and( 74 | FieldPredicates 75 | .ofType(Any::class.java) 76 | ) 77 | .and( 78 | FieldPredicates 79 | .inClass(Page::class.java) 80 | ) 81 | val parametersForPage = EasyRandomParameters() 82 | .randomize(pageFieldPredicate, Randomizer { Random.nextLong(0, 100) }) 83 | .randomize(searchAfterQueryParametersFieldPredicate, Randomizer { emptyMap() }) 84 | val parametersForPageWithGivenQueryId = EasyRandomParameters() 85 | .randomize(queryIdFieldPredicate, Randomizer { queryId }) 86 | .randomize(pageFieldPredicate, Randomizer { Random.nextLong(0, 100) }) 87 | .randomize(searchAfterQueryParametersFieldPredicate, Randomizer { emptyMap() }) 88 | 89 | val pages = Stream.concat( 90 | EasyRandom(parametersForPageWithGivenQueryId).objects(Page::class.java, 10), 91 | EasyRandom(parametersForPage).objects(Page::class.java, 10) 92 | ) 93 | 94 | StepVerifier.create(this.pageRepository.save(Flux.fromStream(pages)).then(this.refreshPages())) 95 | .verifyComplete() 96 | 97 | val rabbitMessage = this.consumeRabbitmqMessages() 98 | 99 | StepVerifier.create(this.deletePagesService.deletePages(queryId)) 100 | .verifyComplete() 101 | 102 | StepVerifier.create(this.refreshPages().thenMany(this.findAllPages())) 103 | .recordWith { mutableListOf() } 104 | .thenConsumeWhile { true } 105 | .expectRecordedMatches { savedPages -> 106 | savedPages.size == 10 && savedPages.all { page -> page.queryId != queryId } 107 | } 108 | .verifyComplete() 109 | 110 | StepVerifier.create(rabbitMessage) 111 | .assertNext { deleteQuery -> 112 | Assert.assertEquals( 113 | DeleteQuery( 114 | queryId = queryId 115 | ), 116 | deleteQuery 117 | ) 118 | } 119 | .verifyComplete() 120 | } 121 | 122 | private fun consumeRabbitmqMessages(): Flux { 123 | val queueName = "${this.rabbitmqProperties.exchangeName}.${this.rabbitmqProperties.deleteQueriesKey}" 124 | val deadLetterQueueName = "${this.rabbitmqProperties.exchangeName}.${this.rabbitmqProperties.deleteQueriesKey}.dead-letter" 125 | 126 | val exchange = this.amqpAdmin.createExchange(this.rabbitmqProperties.exchangeName) 127 | this.amqpAdmin.createQueues(queueName, deadLetterQueueName, this.rabbitmqProperties.deleteQueriesKey, exchange) 128 | 129 | return this.receiver.consumeAutoAck(queueName) 130 | .map { delivery -> this.mapper.readValue(delivery.body, DeleteQuery::class.java) } 131 | .take(1) 132 | } 133 | 134 | } 135 | -------------------------------------------------------------------------------- /worker-paginator-calc/src/test/resources/junit-platform.properties: -------------------------------------------------------------------------------- 1 | junit.jupiter.testinstance.lifecycle.default = per_class 2 | -------------------------------------------------------------------------------- /worker-paginator/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM openjdk:11 2 | 3 | EXPOSE 8080 4 | EXPOSE 8081 5 | 6 | ADD /target/*.jar /app/application.jar 7 | 8 | ENTRYPOINT java $JAVA_OPTS -jar /app/application.jar 9 | -------------------------------------------------------------------------------- /worker-paginator/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | 8 | elasticsearch-paginator 9 | elasticsearch-paginator 10 | 0.0.1-SNAPSHOT 11 | 12 | 13 | worker-paginator 14 | 15 | 16 | 17 | ${project.groupId} 18 | core 19 | ${project.version} 20 | 21 | 22 | 23 | 24 | 25 | 26 | org.springframework.boot 27 | spring-boot-maven-plugin 28 | 29 | 30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /worker-paginator/src/main/kotlin/elasticsearchpaginator/workerpaginator/WorkerPaginatorApplication.kt: -------------------------------------------------------------------------------- 1 | package elasticsearchpaginator.workerpaginator 2 | 3 | import org.springframework.boot.autoconfigure.SpringBootApplication 4 | import org.springframework.boot.runApplication 5 | 6 | @SpringBootApplication 7 | class WorkerPaginatorApplication 8 | 9 | fun main(args: Array) { 10 | runApplication(*args) 11 | } 12 | -------------------------------------------------------------------------------- /worker-paginator/src/main/kotlin/elasticsearchpaginator/workerpaginator/configuration/ElasticsearchConfiguration.kt: -------------------------------------------------------------------------------- 1 | package elasticsearchpaginator.workerpaginator.configuration 2 | 3 | import elasticsearchpaginator.workerpaginator.repository.QueryEntryRepository 4 | import org.springframework.beans.factory.InitializingBean 5 | import org.springframework.boot.context.properties.EnableConfigurationProperties 6 | import org.springframework.context.annotation.Configuration 7 | 8 | @Configuration 9 | @EnableConfigurationProperties(ElasticsearchProperties::class) 10 | class ElasticsearchConfiguration(private val queryEntryRepository: QueryEntryRepository) : InitializingBean { 11 | 12 | /** 13 | * Create query-entries index if not exists 14 | */ 15 | override fun afterPropertiesSet() { 16 | this.queryEntryRepository.existIndex() 17 | .filter { exists -> !exists } 18 | .flatMap { this.queryEntryRepository.createIndex() } 19 | .block() 20 | } 21 | 22 | } 23 | -------------------------------------------------------------------------------- /worker-paginator/src/main/kotlin/elasticsearchpaginator/workerpaginator/configuration/ElasticsearchProperties.kt: -------------------------------------------------------------------------------- 1 | package elasticsearchpaginator.workerpaginator.configuration 2 | 3 | import org.springframework.boot.context.properties.ConfigurationProperties 4 | import org.springframework.boot.context.properties.ConstructorBinding 5 | 6 | @ConstructorBinding 7 | @ConfigurationProperties("app.elasticsearch") 8 | data class ElasticsearchProperties( 9 | val queryEntriesIndex: String, 10 | val queryEntriesIndexNumberShards: Int, 11 | val queryEntriesIndexNumberReplicas: Int 12 | ) 13 | -------------------------------------------------------------------------------- /worker-paginator/src/main/kotlin/elasticsearchpaginator/workerpaginator/configuration/RabbitmqConfiguration.kt: -------------------------------------------------------------------------------- 1 | package elasticsearchpaginator.workerpaginator.configuration 2 | 3 | import elasticsearchpaginator.core.util.RabbitmqUtils.createExchange 4 | import elasticsearchpaginator.core.util.RabbitmqUtils.createQueues 5 | import org.springframework.amqp.core.AmqpAdmin 6 | import org.springframework.beans.factory.InitializingBean 7 | import org.springframework.beans.factory.annotation.Value 8 | import org.springframework.boot.context.properties.EnableConfigurationProperties 9 | import org.springframework.context.annotation.Configuration 10 | 11 | @Configuration 12 | @EnableConfigurationProperties(RabbitmqProperties::class) 13 | class RabbitmqConfiguration(private val amqpAdmin: AmqpAdmin, 14 | private val rabbitmqProperties: RabbitmqProperties, 15 | @Value("\${spring.application.name}") private val applicationName: String) : InitializingBean { 16 | 17 | override fun afterPropertiesSet() { 18 | val exchange = this.amqpAdmin.createExchange(this.rabbitmqProperties.exchangeName) 19 | this.amqpAdmin.createQueues(this.queryQueueName(), this.queryDeadLetterQueueName(), this.rabbitmqProperties.queriesKey, exchange) 20 | this.amqpAdmin.createQueues(this.deleteQueryQueueName(), this.deleteDeadLetterQueryQueueName(), this.rabbitmqProperties.deleteQueriesKey, exchange) 21 | } 22 | 23 | fun queryQueueName(): String { 24 | return "${this.applicationName}.${this.rabbitmqProperties.exchangeName}.${this.rabbitmqProperties.queriesKey}" 25 | } 26 | 27 | fun queryDeadLetterQueueName(): String { 28 | return "${this.applicationName}.${this.rabbitmqProperties.exchangeName}.${this.rabbitmqProperties.queriesKey}.dead-letter" 29 | } 30 | 31 | fun deleteQueryQueueName(): String { 32 | return "${this.applicationName}.${this.rabbitmqProperties.exchangeName}.${this.rabbitmqProperties.deleteQueriesKey}" 33 | } 34 | 35 | fun deleteDeadLetterQueryQueueName(): String { 36 | return "${this.applicationName}.${this.rabbitmqProperties.exchangeName}.${this.rabbitmqProperties.deleteQueriesKey}.dead-letter" 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /worker-paginator/src/main/kotlin/elasticsearchpaginator/workerpaginator/configuration/RabbitmqProperties.kt: -------------------------------------------------------------------------------- 1 | package elasticsearchpaginator.workerpaginator.configuration 2 | 3 | import org.springframework.boot.context.properties.ConfigurationProperties 4 | import org.springframework.boot.context.properties.ConstructorBinding 5 | 6 | @ConstructorBinding 7 | @ConfigurationProperties("app.rabbitmq") 8 | data class RabbitmqProperties( 9 | val exchangeName: String, 10 | val queriesKey: String, 11 | val deleteQueriesKey: String, 12 | val deletePagesKey: String, 13 | val computePagesKey: String 14 | ) 15 | -------------------------------------------------------------------------------- /worker-paginator/src/main/kotlin/elasticsearchpaginator/workerpaginator/model/QueryEntry.kt: -------------------------------------------------------------------------------- 1 | package elasticsearchpaginator.workerpaginator.model 2 | 3 | import elasticsearchpaginator.core.model.Query 4 | import java.time.Instant 5 | 6 | data class QueryEntry( 7 | val query: Query, 8 | val lastUseDate: Instant, 9 | val lastComputationDate: Instant 10 | ) 11 | -------------------------------------------------------------------------------- /worker-paginator/src/main/kotlin/elasticsearchpaginator/workerpaginator/repository/QueryEntryRepository.kt: -------------------------------------------------------------------------------- 1 | package elasticsearchpaginator.workerpaginator.repository 2 | 3 | import com.fasterxml.jackson.databind.ObjectMapper 4 | import com.fasterxml.jackson.module.kotlin.readValue 5 | import elasticsearchpaginator.core.util.ElasticsearchUtils.NUMBER_REPLICAS_SETTING 6 | import elasticsearchpaginator.core.util.ElasticsearchUtils.NUMBER_SHARDS_SETTING 7 | import elasticsearchpaginator.core.util.ElasticsearchUtils.async 8 | import elasticsearchpaginator.workerpaginator.configuration.ElasticsearchProperties 9 | import elasticsearchpaginator.workerpaginator.model.QueryEntry 10 | import org.elasticsearch.action.delete.DeleteRequest 11 | import org.elasticsearch.action.delete.DeleteResponse 12 | import org.elasticsearch.action.get.GetRequest 13 | import org.elasticsearch.action.get.GetResponse 14 | import org.elasticsearch.action.search.SearchRequest 15 | import org.elasticsearch.action.search.SearchResponse 16 | import org.elasticsearch.action.update.UpdateRequest 17 | import org.elasticsearch.action.update.UpdateResponse 18 | import org.elasticsearch.client.RequestOptions 19 | import org.elasticsearch.client.RestHighLevelClient 20 | import org.elasticsearch.client.indices.CreateIndexRequest 21 | import org.elasticsearch.client.indices.CreateIndexResponse 22 | import org.elasticsearch.client.indices.GetIndexRequest 23 | import org.elasticsearch.common.settings.Settings 24 | import org.elasticsearch.common.xcontent.XContentType 25 | import org.elasticsearch.index.query.QueryBuilders 26 | import org.elasticsearch.search.builder.SearchSourceBuilder 27 | import org.springframework.beans.factory.annotation.Value 28 | import org.springframework.core.io.Resource 29 | import org.springframework.stereotype.Repository 30 | import reactor.core.publisher.Flux 31 | import reactor.core.publisher.Mono 32 | import java.io.InputStreamReader 33 | import java.time.Instant 34 | 35 | @Repository 36 | class QueryEntryRepository(private val restHighLevelClient: RestHighLevelClient, 37 | private val mapper: ObjectMapper, 38 | private val elasticsearchProperties: ElasticsearchProperties, 39 | @Value("classpath:query-entries-mappings.json") private val queryEntriesMappings: Resource) { 40 | 41 | private val elasticsearchMaxResultWindow = 10000 42 | 43 | fun createIndex(): Mono { 44 | return Mono.fromCallable { InputStreamReader(this.queryEntriesMappings.inputStream).readText() } 45 | .map { mappings -> 46 | val settings = Settings.builder() 47 | .put(NUMBER_SHARDS_SETTING, this.elasticsearchProperties.queryEntriesIndexNumberShards) 48 | .put(NUMBER_REPLICAS_SETTING, this.elasticsearchProperties.queryEntriesIndexNumberReplicas) 49 | .build() 50 | 51 | CreateIndexRequest(this.elasticsearchProperties.queryEntriesIndex) 52 | .settings(settings) 53 | .mapping(mappings, XContentType.JSON) 54 | } 55 | .flatMap { createIndexRequest -> 56 | async { actionListener -> 57 | this.restHighLevelClient.indices().createAsync(createIndexRequest, RequestOptions.DEFAULT, actionListener) 58 | } 59 | } 60 | .then() 61 | } 62 | 63 | fun existIndex(): Mono { 64 | return Mono.just( 65 | GetIndexRequest(this.elasticsearchProperties.queryEntriesIndex) 66 | ) 67 | .flatMap { getIndexRequest -> 68 | async { actionListener -> 69 | this.restHighLevelClient.indices().existsAsync(getIndexRequest, RequestOptions.DEFAULT, actionListener) 70 | } 71 | } 72 | } 73 | 74 | fun updateLastUseDate(queryEntry: QueryEntry): Mono { 75 | return Mono.just( 76 | UpdateRequest() 77 | .index(this.elasticsearchProperties.queryEntriesIndex) 78 | .id(queryEntry.query.hash()) 79 | .doc(this.mapper.writeValueAsBytes(QueryEntryLastUseDateUpdate(lastUseDate = queryEntry.lastUseDate)), XContentType.JSON) 80 | .upsert(this.mapper.writeValueAsBytes(queryEntry), XContentType.JSON) 81 | ) 82 | .flatMap { updateRequest -> 83 | async { actionListener -> 84 | this.restHighLevelClient.updateAsync(updateRequest, RequestOptions.DEFAULT, actionListener) 85 | } 86 | } 87 | .then() 88 | } 89 | 90 | fun updateLastComputationDate(queryEntry: QueryEntry): Mono { 91 | return Mono.just( 92 | UpdateRequest() 93 | .index(this.elasticsearchProperties.queryEntriesIndex) 94 | .id(queryEntry.query.hash()) 95 | .doc(this.mapper.writeValueAsBytes(QueryEntryLastComputationDateUpdate(lastComputationDate = queryEntry.lastComputationDate)), XContentType.JSON) 96 | ) 97 | .flatMap { updateRequest -> 98 | async { actionListener -> 99 | this.restHighLevelClient.updateAsync(updateRequest, RequestOptions.DEFAULT, actionListener) 100 | } 101 | } 102 | .then() 103 | } 104 | 105 | fun findOne(id: String): Mono { 106 | return Mono.just( 107 | GetRequest() 108 | .index(this.elasticsearchProperties.queryEntriesIndex) 109 | .id(id) 110 | ) 111 | .flatMap { getRequest -> 112 | async { actionListener -> 113 | this.restHighLevelClient.getAsync(getRequest, RequestOptions.DEFAULT, actionListener) 114 | } 115 | } 116 | .filter { getResponse -> 117 | getResponse.isExists 118 | } 119 | .map { getResponse -> 120 | this.mapper.readValue(getResponse.sourceAsBytes) 121 | } 122 | } 123 | 124 | // TODO use scroll queries instead of limiting to the max result window 125 | fun findAll(): Flux { 126 | return Mono.just( 127 | SearchSourceBuilder() 128 | .query(QueryBuilders.matchAllQuery()) 129 | .size(elasticsearchMaxResultWindow) 130 | ) 131 | .map { searchSourceBuilder -> 132 | SearchRequest() 133 | .indices(this.elasticsearchProperties.queryEntriesIndex) 134 | .source(searchSourceBuilder) 135 | } 136 | .flatMap { searchRequest -> 137 | async { actionListener -> 138 | this.restHighLevelClient.searchAsync(searchRequest, RequestOptions.DEFAULT, actionListener) 139 | } 140 | } 141 | .flatMapIterable { searchResponse -> 142 | searchResponse.hits 143 | } 144 | .map { searchHit -> 145 | this.mapper.readValue(searchHit.sourceRef.streamInput()) 146 | } 147 | } 148 | 149 | // TODO use scroll queries instead of limiting to the max result window 150 | fun findAllWithLastUseDateOlderThan(instant: Instant): Flux { 151 | return Mono.just( 152 | SearchSourceBuilder() 153 | .query(QueryBuilders.rangeQuery("lastUseDate").lte(instant.toEpochMilli())) 154 | .size(elasticsearchMaxResultWindow) 155 | ) 156 | .map { searchSourceBuilder -> 157 | SearchRequest() 158 | .indices(this.elasticsearchProperties.queryEntriesIndex) 159 | .source(searchSourceBuilder) 160 | } 161 | .flatMap { searchRequest -> 162 | async { actionListener -> 163 | this.restHighLevelClient.searchAsync(searchRequest, RequestOptions.DEFAULT, actionListener) 164 | } 165 | } 166 | .flatMapIterable { searchResponse -> 167 | searchResponse.hits 168 | } 169 | .map { searchHit -> 170 | this.mapper.readValue(searchHit.sourceRef.streamInput()) 171 | } 172 | } 173 | 174 | fun deleteOne(id: String): Mono { 175 | return Mono.just( 176 | DeleteRequest() 177 | .index(this.elasticsearchProperties.queryEntriesIndex) 178 | .id(id) 179 | ) 180 | .flatMap { deleteRequest -> 181 | async { actionListener -> 182 | this.restHighLevelClient.deleteAsync(deleteRequest, RequestOptions.DEFAULT, actionListener) 183 | } 184 | } 185 | .then() 186 | } 187 | 188 | private data class QueryEntryLastUseDateUpdate( 189 | val lastUseDate: Instant 190 | ) 191 | 192 | private data class QueryEntryLastComputationDateUpdate( 193 | val lastComputationDate: Instant 194 | ) 195 | 196 | } 197 | -------------------------------------------------------------------------------- /worker-paginator/src/main/kotlin/elasticsearchpaginator/workerpaginator/service/CleaningQueriesService.kt: -------------------------------------------------------------------------------- 1 | package elasticsearchpaginator.workerpaginator.service 2 | 3 | import elasticsearchpaginator.core.model.DeletePages 4 | import elasticsearchpaginator.workerpaginator.repository.QueryEntryRepository 5 | import elasticsearchpaginator.workerpaginator.transport.DeletePagesSender 6 | import org.springframework.beans.factory.annotation.Value 7 | import org.springframework.stereotype.Service 8 | import reactor.core.publisher.Mono 9 | import java.time.Duration 10 | import java.time.Instant 11 | 12 | @Service 13 | class CleaningQueriesService(private val queryEntryRepository: QueryEntryRepository, 14 | private val deletePagesSender: DeletePagesSender, 15 | @Value("\${app.query-entries-ttl}") private val queryEntriesTtl: Duration) { 16 | 17 | fun deleteQuery(queryId: String): Mono { 18 | return this.queryEntryRepository.deleteOne(queryId) 19 | } 20 | 21 | fun getOutdatedQueriesThenDeleteRelatedPages(): Mono { 22 | return Mono.just(Instant.now().minus(this.queryEntriesTtl)) 23 | .flatMapMany(this.queryEntryRepository::findAllWithLastUseDateOlderThan) 24 | .map { queryEntry -> 25 | DeletePages( 26 | queryId = queryEntry.query.hash() 27 | ) 28 | } 29 | .flatMap(this.deletePagesSender::sendDeletePagesEvent) 30 | .then() 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /worker-paginator/src/main/kotlin/elasticsearchpaginator/workerpaginator/service/QueriesService.kt: -------------------------------------------------------------------------------- 1 | package elasticsearchpaginator.workerpaginator.service 2 | 3 | import elasticsearchpaginator.core.model.Query 4 | import elasticsearchpaginator.workerpaginator.model.QueryEntry 5 | import elasticsearchpaginator.workerpaginator.repository.QueryEntryRepository 6 | import elasticsearchpaginator.workerpaginator.transport.ComputePagesSender 7 | import org.springframework.beans.factory.annotation.Value 8 | import org.springframework.stereotype.Component 9 | import reactor.core.publisher.Mono 10 | import java.time.Duration 11 | import java.time.Instant 12 | 13 | @Component 14 | class QueriesService(private val queryEntryRepository: QueryEntryRepository, 15 | private val computePagesSender: ComputePagesSender, 16 | @Value("\${app.min-interval-between-pages-refresh}") private val minIntervalBetweenPagesRefresh: Duration) { 17 | 18 | fun upsertQueryAndAskForPagesComputation(query: Query): Mono { 19 | return this.queryEntryRepository.findOne(query.hash()) 20 | .map { queryEntry -> 21 | queryEntry.copy(lastUseDate = Instant.now()) 22 | } 23 | .defaultIfEmpty( 24 | QueryEntry( 25 | query = query, 26 | lastUseDate = Instant.now(), 27 | lastComputationDate = Instant.EPOCH 28 | ) 29 | ) 30 | .flatMap { queryEntry -> 31 | this.queryEntryRepository.updateLastUseDate(queryEntry) 32 | .then(Mono.just(queryEntry)) 33 | } 34 | .filter { queryEntry -> 35 | Instant.now().minus(minIntervalBetweenPagesRefresh) > queryEntry.lastComputationDate 36 | } 37 | .flatMapMany { queryEntry -> 38 | this.computePagesSender.sendComputePagesEvent(query) 39 | .then(this.queryEntryRepository.updateLastComputationDate(queryEntry.copy(lastComputationDate = Instant.now()))) 40 | } 41 | .then() 42 | } 43 | 44 | } 45 | -------------------------------------------------------------------------------- /worker-paginator/src/main/kotlin/elasticsearchpaginator/workerpaginator/service/RefreshQueriesService.kt: -------------------------------------------------------------------------------- 1 | package elasticsearchpaginator.workerpaginator.service 2 | 3 | import elasticsearchpaginator.workerpaginator.model.QueryEntry 4 | import elasticsearchpaginator.workerpaginator.repository.QueryEntryRepository 5 | import elasticsearchpaginator.workerpaginator.transport.ComputePagesSender 6 | import org.springframework.stereotype.Service 7 | import reactor.core.publisher.Mono 8 | 9 | @Service 10 | class RefreshQueriesService(private val queryEntryRepository: QueryEntryRepository, 11 | private val computePagesSender: ComputePagesSender) { 12 | 13 | fun refreshPagesForAllQueries(): Mono { 14 | return this.queryEntryRepository.findAll() 15 | .map(QueryEntry::query) 16 | .flatMap(this.computePagesSender::sendComputePagesEvent) 17 | .then() 18 | } 19 | 20 | } 21 | -------------------------------------------------------------------------------- /worker-paginator/src/main/kotlin/elasticsearchpaginator/workerpaginator/transport/ComputePagesSender.kt: -------------------------------------------------------------------------------- 1 | package elasticsearchpaginator.workerpaginator.transport 2 | 3 | import com.fasterxml.jackson.databind.ObjectMapper 4 | import elasticsearchpaginator.core.model.Query 5 | import elasticsearchpaginator.workerpaginator.configuration.RabbitmqProperties 6 | import org.springframework.stereotype.Component 7 | import reactor.core.publisher.Mono 8 | import reactor.rabbitmq.OutboundMessage 9 | import reactor.rabbitmq.Sender 10 | 11 | @Component 12 | class ComputePagesSender(private val sender: Sender, 13 | private val mapper: ObjectMapper, 14 | private val rabbitmqProperties: RabbitmqProperties) { 15 | 16 | fun sendComputePagesEvent(query: Query): Mono { 17 | return this.sender.send( 18 | Mono.just( 19 | OutboundMessage( 20 | this.rabbitmqProperties.exchangeName, 21 | this.rabbitmqProperties.computePagesKey, 22 | this.mapper.writeValueAsBytes(query) 23 | ) 24 | ) 25 | ) 26 | } 27 | 28 | } 29 | -------------------------------------------------------------------------------- /worker-paginator/src/main/kotlin/elasticsearchpaginator/workerpaginator/transport/DeletePagesSender.kt: -------------------------------------------------------------------------------- 1 | package elasticsearchpaginator.workerpaginator.transport 2 | 3 | import com.fasterxml.jackson.databind.ObjectMapper 4 | import elasticsearchpaginator.core.model.DeletePages 5 | import elasticsearchpaginator.core.model.DeleteQuery 6 | import elasticsearchpaginator.workerpaginator.configuration.RabbitmqProperties 7 | import org.springframework.stereotype.Component 8 | import reactor.core.publisher.Mono 9 | import reactor.rabbitmq.OutboundMessage 10 | import reactor.rabbitmq.Sender 11 | 12 | @Component 13 | class DeletePagesSender(private val sender: Sender, 14 | private val mapper: ObjectMapper, 15 | private val rabbitmqProperties: RabbitmqProperties) { 16 | 17 | fun sendDeletePagesEvent(deletePages: DeletePages): Mono { 18 | return this.sender.send( 19 | Mono.just( 20 | OutboundMessage( 21 | this.rabbitmqProperties.exchangeName, 22 | this.rabbitmqProperties.deletePagesKey, 23 | this.mapper.writeValueAsBytes(deletePages) 24 | ) 25 | ) 26 | ) 27 | } 28 | 29 | } 30 | -------------------------------------------------------------------------------- /worker-paginator/src/main/kotlin/elasticsearchpaginator/workerpaginator/transport/DeleteQueriesReceiver.kt: -------------------------------------------------------------------------------- 1 | package elasticsearchpaginator.workerpaginator.transport 2 | 3 | import com.fasterxml.jackson.databind.ObjectMapper 4 | import elasticsearchpaginator.core.model.DeleteQuery 5 | import elasticsearchpaginator.core.transport.AbstractRabbitmqReceiver 6 | import elasticsearchpaginator.workerpaginator.configuration.RabbitmqConfiguration 7 | import elasticsearchpaginator.workerpaginator.service.CleaningQueriesService 8 | import org.springframework.context.ApplicationContext 9 | import org.springframework.stereotype.Component 10 | import reactor.core.publisher.Mono 11 | import reactor.rabbitmq.Receiver 12 | 13 | @Component 14 | class DeleteQueriesReceiver(override val receiver: Receiver, 15 | override val mapper: ObjectMapper, 16 | override val applicationContext: ApplicationContext, 17 | private val cleaningQueriesService: CleaningQueriesService, 18 | private val rabbitmqConfiguration: RabbitmqConfiguration) : AbstractRabbitmqReceiver() { 19 | 20 | override val queueName = this.rabbitmqConfiguration.deleteQueryQueueName() 21 | override val eventClass = DeleteQuery::class.java 22 | 23 | override fun eventHandler(event: DeleteQuery): Mono { 24 | return this.cleaningQueriesService.deleteQuery(event.queryId) 25 | } 26 | 27 | } 28 | -------------------------------------------------------------------------------- /worker-paginator/src/main/kotlin/elasticsearchpaginator/workerpaginator/transport/QueriesReceiver.kt: -------------------------------------------------------------------------------- 1 | package elasticsearchpaginator.workerpaginator.transport 2 | 3 | import com.fasterxml.jackson.databind.ObjectMapper 4 | import elasticsearchpaginator.core.model.Query 5 | import elasticsearchpaginator.core.transport.AbstractRabbitmqReceiver 6 | import elasticsearchpaginator.workerpaginator.configuration.RabbitmqConfiguration 7 | import elasticsearchpaginator.workerpaginator.service.QueriesService 8 | import org.springframework.context.ApplicationContext 9 | import org.springframework.stereotype.Component 10 | import reactor.core.publisher.Mono 11 | import reactor.rabbitmq.Receiver 12 | 13 | @Component 14 | class QueriesReceiver(override val receiver: Receiver, 15 | override val mapper: ObjectMapper, 16 | override val applicationContext: ApplicationContext, 17 | private val queriesService: QueriesService, 18 | private val rabbitmqConfiguration: RabbitmqConfiguration) : AbstractRabbitmqReceiver() { 19 | 20 | override val queueName = this.rabbitmqConfiguration.queryQueueName() 21 | override val eventClass = Query::class.java 22 | 23 | override fun eventHandler(event: Query): Mono { 24 | return this.queriesService.upsertQueryAndAskForPagesComputation(event) 25 | } 26 | 27 | } 28 | -------------------------------------------------------------------------------- /worker-paginator/src/main/kotlin/elasticsearchpaginator/workerpaginator/web/QueriesController.kt: -------------------------------------------------------------------------------- 1 | package elasticsearchpaginator.workerpaginator.web 2 | 3 | import elasticsearchpaginator.workerpaginator.service.CleaningQueriesService 4 | import elasticsearchpaginator.workerpaginator.service.RefreshQueriesService 5 | import org.springframework.http.HttpStatus 6 | import org.springframework.web.bind.annotation.GetMapping 7 | import org.springframework.web.bind.annotation.RequestMapping 8 | import org.springframework.web.bind.annotation.ResponseStatus 9 | import org.springframework.web.bind.annotation.RestController 10 | import reactor.core.publisher.Mono 11 | 12 | @RestController 13 | @RequestMapping("/queries") 14 | class QueriesController(private val cleaningQueriesService: CleaningQueriesService, 15 | private val refreshQueriesService: RefreshQueriesService) { 16 | 17 | @GetMapping("/clear") 18 | @ResponseStatus(HttpStatus.NO_CONTENT) 19 | fun clearOutdatedQueries(): Mono { 20 | return this.cleaningQueriesService.getOutdatedQueriesThenDeleteRelatedPages() 21 | } 22 | 23 | @GetMapping("/refresh") 24 | @ResponseStatus(HttpStatus.NO_CONTENT) 25 | fun refreshAllQueries(): Mono { 26 | return this.refreshQueriesService.refreshPagesForAllQueries() 27 | } 28 | 29 | } 30 | -------------------------------------------------------------------------------- /worker-paginator/src/main/resources/application.yml: -------------------------------------------------------------------------------- 1 | management: 2 | server: 3 | port: 8081 4 | 5 | server: 6 | port: 8080 7 | 8 | spring: 9 | application: 10 | name: worker-paginator 11 | rabbitmq: 12 | username: user 13 | password: password 14 | elasticsearch: 15 | rest: 16 | uris: http://locahost:9200 17 | 18 | app: 19 | elasticsearch: 20 | query-entries-index: query-entries 21 | query-entries-index-number-replicas: 2 22 | query-entries-index-number-shards: 5 23 | rabbitmq: 24 | exchange-name: paginator 25 | queries-key: queries 26 | delete-queries-key: delete-queries 27 | delete-pages-key: delete-pages 28 | compute-pages-key: compute-pages 29 | min-interval-between-pages-refresh: 60m 30 | query-entries-ttl: 3d 31 | -------------------------------------------------------------------------------- /worker-paginator/src/main/resources/query-entries-mappings.json: -------------------------------------------------------------------------------- 1 | { 2 | "properties": { 3 | "query": { 4 | "enabled": false 5 | }, 6 | "lastUseDate": { 7 | "type": "date" 8 | }, 9 | "lastComputationDate": { 10 | "type": "date" 11 | } 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /worker-paginator/src/test/kotlin/elasticsearchpaginator/workerpaginator/AbstractIntegrationTest.kt: -------------------------------------------------------------------------------- 1 | package elasticsearchpaginator.workerpaginator 2 | 3 | import com.fasterxml.jackson.databind.ObjectMapper 4 | import com.fasterxml.jackson.module.kotlin.readValue 5 | import elasticsearchpaginator.core.util.ElasticsearchUtils 6 | import elasticsearchpaginator.core.util.RabbitmqUtils.createExchange 7 | import elasticsearchpaginator.core.util.RabbitmqUtils.createQueues 8 | import elasticsearchpaginator.workerpaginator.configuration.ElasticsearchProperties 9 | import elasticsearchpaginator.workerpaginator.configuration.RabbitmqProperties 10 | import elasticsearchpaginator.workerpaginator.model.QueryEntry 11 | import org.elasticsearch.action.admin.indices.refresh.RefreshRequest 12 | import org.elasticsearch.action.admin.indices.refresh.RefreshResponse 13 | import org.elasticsearch.action.bulk.BulkRequest 14 | import org.elasticsearch.action.bulk.BulkResponse 15 | import org.elasticsearch.action.index.IndexRequest 16 | import org.elasticsearch.action.search.SearchRequest 17 | import org.elasticsearch.action.search.SearchResponse 18 | import org.elasticsearch.action.support.WriteRequest 19 | import org.elasticsearch.client.RequestOptions 20 | import org.elasticsearch.client.RestHighLevelClient 21 | import org.elasticsearch.common.xcontent.XContentType 22 | import org.elasticsearch.index.query.QueryBuilders 23 | import org.elasticsearch.index.reindex.BulkByScrollResponse 24 | import org.elasticsearch.index.reindex.DeleteByQueryRequest 25 | import org.elasticsearch.search.builder.SearchSourceBuilder 26 | import org.junit.jupiter.api.extension.ExtendWith 27 | import org.springframework.amqp.core.AmqpAdmin 28 | import org.springframework.beans.factory.annotation.Autowired 29 | import org.springframework.boot.test.autoconfigure.web.reactive.AutoConfigureWebTestClient 30 | import org.springframework.boot.test.context.SpringBootTest 31 | import org.springframework.boot.test.util.TestPropertyValues 32 | import org.springframework.context.ApplicationContextInitializer 33 | import org.springframework.context.ConfigurableApplicationContext 34 | import org.springframework.test.context.ContextConfiguration 35 | import org.springframework.test.context.junit.jupiter.SpringExtension 36 | import org.testcontainers.containers.RabbitMQContainer 37 | import org.testcontainers.elasticsearch.ElasticsearchContainer 38 | import reactor.core.publisher.Flux 39 | import reactor.core.publisher.Mono 40 | import reactor.rabbitmq.Receiver 41 | import reactor.test.StepVerifier 42 | import java.time.Duration 43 | 44 | @ExtendWith(SpringExtension::class) 45 | @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) 46 | @ContextConfiguration(initializers = [AbstractIntegrationTest.Initializer::class]) 47 | @AutoConfigureWebTestClient 48 | abstract class AbstractIntegrationTest { 49 | 50 | init { 51 | StepVerifier.setDefaultTimeout(Duration.ofSeconds(5)) 52 | } 53 | 54 | @Autowired 55 | protected lateinit var restHighLevelClient: RestHighLevelClient 56 | 57 | @Autowired 58 | protected lateinit var mapper: ObjectMapper 59 | 60 | @Autowired 61 | protected lateinit var receiver: Receiver 62 | 63 | @Autowired 64 | protected lateinit var amqpAdmin: AmqpAdmin 65 | 66 | @Autowired 67 | protected lateinit var rabbitmqProperties: RabbitmqProperties 68 | 69 | @Autowired 70 | protected lateinit var elasticsearchProperties: ElasticsearchProperties 71 | 72 | companion object { 73 | private val RABBITMQ_USERNAME = "guest" 74 | private val RABBITMQ_PASSWORD = "guest" 75 | 76 | val elasticsearchContainer = ElasticsearchContainer("docker.elastic.co/elasticsearch/elasticsearch-oss:7.4.2") 77 | val rabbitmqContainer = RabbitMQContainer("rabbitmq:3.8") 78 | .withUser(RABBITMQ_USERNAME, RABBITMQ_PASSWORD) 79 | } 80 | 81 | internal class Initializer : ApplicationContextInitializer { 82 | override fun initialize(configurableApplicationContext: ConfigurableApplicationContext) { 83 | rabbitmqContainer.start() 84 | elasticsearchContainer.start() 85 | 86 | TestPropertyValues.of( 87 | "spring.elasticsearch.rest.uris=${elasticsearchContainer.httpHostAddress}", 88 | "spring.rabbitmq.host=${rabbitmqContainer.containerIpAddress}", 89 | "spring.rabbitmq.port=${rabbitmqContainer.firstMappedPort}", 90 | "spring.rabbitmq.username=$RABBITMQ_USERNAME", 91 | "spring.rabbitmq.password=$RABBITMQ_PASSWORD" 92 | ) 93 | .applyTo(configurableApplicationContext.environment) 94 | } 95 | } 96 | 97 | protected fun refreshQueryEntries(): Mono { 98 | return Mono.just( 99 | RefreshRequest() 100 | .indices(this.elasticsearchProperties.queryEntriesIndex) 101 | ) 102 | .flatMap { refreshRequest -> 103 | ElasticsearchUtils.async { actionListener -> 104 | this.restHighLevelClient.indices().refreshAsync(refreshRequest, RequestOptions.DEFAULT, actionListener) 105 | } 106 | } 107 | .then() 108 | } 109 | 110 | protected fun findAllQueryEntries(): Flux { 111 | return Mono.just( 112 | SearchSourceBuilder() 113 | .query( 114 | QueryBuilders.matchAllQuery() 115 | ) 116 | .size(10000) 117 | ) 118 | .map { searchSourceBuilder -> 119 | SearchRequest() 120 | .indices(this.elasticsearchProperties.queryEntriesIndex) 121 | .source(searchSourceBuilder) 122 | } 123 | .flatMap { searchRequest -> 124 | ElasticsearchUtils.async { actionListener -> 125 | this.restHighLevelClient.searchAsync(searchRequest, RequestOptions.DEFAULT, actionListener) 126 | } 127 | } 128 | .flatMapIterable { searchResponse -> 129 | searchResponse.hits 130 | } 131 | .map { searchHit -> 132 | this.mapper.readValue(searchHit.sourceRef.streamInput()) 133 | } 134 | } 135 | 136 | protected fun saveQueryEntries(queryEntries: List): Mono { 137 | return Flux.fromIterable(queryEntries) 138 | .map { queryEntry -> 139 | IndexRequest() 140 | .index(this.elasticsearchProperties.queryEntriesIndex) 141 | .id(queryEntry.query.hash()) 142 | .source(this.mapper.writeValueAsBytes(queryEntry), XContentType.JSON) 143 | } 144 | .collectList() 145 | .map { indexRequests -> 146 | BulkRequest() 147 | .add(indexRequests) 148 | .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) 149 | } 150 | .flatMap { bulkRequest -> 151 | ElasticsearchUtils.async { actionListener -> 152 | this.restHighLevelClient.bulkAsync(bulkRequest, RequestOptions.DEFAULT, actionListener) 153 | } 154 | } 155 | .then() 156 | } 157 | 158 | protected fun clearQueryEntries(): Mono { 159 | return Mono.just( 160 | DeleteByQueryRequest(this.elasticsearchProperties.queryEntriesIndex) 161 | .setQuery( 162 | QueryBuilders.matchAllQuery() 163 | ) 164 | .setRefresh(true) 165 | ) 166 | .flatMap { deleteByQueryRequest -> 167 | ElasticsearchUtils.async { actionListener -> 168 | this.restHighLevelClient.deleteByQueryAsync(deleteByQueryRequest, RequestOptions.DEFAULT, actionListener) 169 | } 170 | } 171 | .then() 172 | } 173 | 174 | protected fun consumeRabbitmqMessages(key: String, clazz: Class): Flux { 175 | val queueName = "${this.rabbitmqProperties.exchangeName}.$key" 176 | val deadLetterQueueName = "${this.rabbitmqProperties.exchangeName}.$key.dead-letter" 177 | 178 | val exchange = this.amqpAdmin.createExchange(this.rabbitmqProperties.exchangeName) 179 | this.amqpAdmin.createQueues(queueName, deadLetterQueueName, key, exchange) 180 | 181 | return this.receiver.consumeAutoAck(queueName) 182 | .map { delivery -> this.mapper.readValue(delivery.body, clazz) } 183 | .take(1) 184 | .timeout(Duration.ofSeconds(1)) 185 | } 186 | 187 | } 188 | -------------------------------------------------------------------------------- /worker-paginator/src/test/kotlin/elasticsearchpaginator/workerpaginator/ClearOutdatedQueriesIntegrationTest.kt: -------------------------------------------------------------------------------- 1 | package elasticsearchpaginator.workerpaginator 2 | 3 | import elasticsearchpaginator.core.model.DeletePages 4 | import elasticsearchpaginator.core.model.Query 5 | import elasticsearchpaginator.workerpaginator.model.QueryEntry 6 | import elasticsearchpaginator.workerpaginator.service.CleaningQueriesService 7 | import org.elasticsearch.index.query.QueryBuilders 8 | import org.elasticsearch.search.sort.SortBuilders 9 | import org.junit.Assert 10 | import org.junit.jupiter.api.AfterEach 11 | import org.junit.jupiter.api.Test 12 | import org.springframework.beans.factory.annotation.Autowired 13 | import org.springframework.beans.factory.annotation.Value 14 | import reactor.test.StepVerifier 15 | import java.time.Duration 16 | import java.time.Instant 17 | 18 | class ClearOutdatedQueriesIntegrationTest : AbstractIntegrationTest() { 19 | 20 | @Autowired 21 | private lateinit var cleaningQueriesService: CleaningQueriesService 22 | 23 | @Value("\${app.query-entries-ttl}") 24 | private lateinit var queryEntriesTtl: Duration 25 | 26 | @AfterEach 27 | internal fun cleanUp() { 28 | StepVerifier.create(this.clearQueryEntries()) 29 | .verifyComplete() 30 | } 31 | 32 | @Test 33 | fun `should retrieve outdated queries and ask for associated pages deletion`() { 34 | val queryEntry = QueryEntry( 35 | query = Query( 36 | index = "index1", 37 | query = QueryBuilders.matchAllQuery().toString(), 38 | sort = listOf( 39 | SortBuilders.fieldSort("field1") 40 | .toString() 41 | ) 42 | .joinToString(",", "[", "]"), 43 | firstPageSize = 2, 44 | size = 4 45 | ), 46 | lastComputationDate = Instant.now(), 47 | lastUseDate = Instant.now().minus(this.queryEntriesTtl) 48 | ) 49 | val anOtherQueryEntry = QueryEntry( 50 | query = Query( 51 | index = "index2", 52 | query = QueryBuilders.matchAllQuery().toString(), 53 | sort = listOf( 54 | SortBuilders.fieldSort("field2").toString() 55 | ) 56 | .joinToString(",", "[", "]"), 57 | firstPageSize = 2, 58 | size = 4 59 | ), 60 | lastComputationDate = Instant.now(), 61 | lastUseDate = Instant.now() 62 | ) 63 | 64 | StepVerifier.create(this.saveQueryEntries(listOf(queryEntry, anOtherQueryEntry))) 65 | .verifyComplete() 66 | 67 | val rabbitMessage = this.consumeRabbitmqMessages(this.rabbitmqProperties.deletePagesKey, DeletePages::class.java) 68 | 69 | StepVerifier.create(this.cleaningQueriesService.getOutdatedQueriesThenDeleteRelatedPages()) 70 | .verifyComplete() 71 | 72 | StepVerifier.create(this.refreshQueryEntries().thenMany(this.findAllQueryEntries())) 73 | .expectNextCount(2) 74 | .verifyComplete() 75 | 76 | StepVerifier.create(rabbitMessage) 77 | .assertNext { deleteQuery -> 78 | Assert.assertEquals( 79 | DeletePages( 80 | queryId = queryEntry.query.hash() 81 | ), 82 | deleteQuery 83 | ) 84 | } 85 | .verifyComplete() 86 | } 87 | 88 | } 89 | -------------------------------------------------------------------------------- /worker-paginator/src/test/kotlin/elasticsearchpaginator/workerpaginator/DeleteQueryIntegrationTest.kt: -------------------------------------------------------------------------------- 1 | package elasticsearchpaginator.workerpaginator 2 | 3 | import elasticsearchpaginator.core.model.Query 4 | import elasticsearchpaginator.core.util.ElasticsearchUtils 5 | import elasticsearchpaginator.workerpaginator.model.QueryEntry 6 | import elasticsearchpaginator.workerpaginator.service.CleaningQueriesService 7 | import org.elasticsearch.action.bulk.BulkRequest 8 | import org.elasticsearch.action.bulk.BulkResponse 9 | import org.elasticsearch.action.index.IndexRequest 10 | import org.elasticsearch.action.support.WriteRequest 11 | import org.elasticsearch.client.RequestOptions 12 | import org.elasticsearch.common.xcontent.XContentType 13 | import org.elasticsearch.index.query.QueryBuilders 14 | import org.elasticsearch.search.sort.SortBuilders 15 | import org.junit.Assert 16 | import org.junit.jupiter.api.AfterEach 17 | import org.junit.jupiter.api.Test 18 | import org.springframework.beans.factory.annotation.Autowired 19 | import reactor.core.publisher.Flux 20 | import reactor.core.publisher.Mono 21 | import reactor.test.StepVerifier 22 | import java.time.Instant 23 | 24 | class DeleteQueryIntegrationTest : AbstractIntegrationTest() { 25 | 26 | @Autowired 27 | private lateinit var cleaningQueriesService: CleaningQueriesService 28 | 29 | @AfterEach 30 | internal fun cleanUp() { 31 | StepVerifier.create(this.clearQueryEntries()) 32 | .verifyComplete() 33 | } 34 | 35 | @Test 36 | fun `should delete the query entry`() { 37 | val queryEntry = QueryEntry( 38 | query = Query( 39 | index = "index1", 40 | query = QueryBuilders.matchAllQuery().toString(), 41 | sort = listOf( 42 | SortBuilders.fieldSort("field1") 43 | .toString() 44 | ) 45 | .joinToString(",", "[", "]"), 46 | firstPageSize = 2, 47 | size = 4 48 | ), 49 | lastComputationDate = Instant.now(), 50 | lastUseDate = Instant.now() 51 | ) 52 | val anOtherQueryEntry = QueryEntry( 53 | query = Query( 54 | index = "index2", 55 | query = QueryBuilders.matchAllQuery().toString(), 56 | sort = listOf( 57 | SortBuilders.fieldSort("field2").toString() 58 | ) 59 | .joinToString(",", "[", "]"), 60 | firstPageSize = 2, 61 | size = 4 62 | ), 63 | lastComputationDate = Instant.now(), 64 | lastUseDate = Instant.now() 65 | ) 66 | 67 | StepVerifier.create(this.saveQueryEntries(listOf(queryEntry, anOtherQueryEntry))) 68 | .verifyComplete() 69 | 70 | StepVerifier.create(this.cleaningQueriesService.deleteQuery(queryEntry.query.hash())) 71 | .verifyComplete() 72 | 73 | StepVerifier.create(this.refreshQueryEntries().thenMany(this.findAllQueryEntries())) 74 | .assertNext { savedQueryEntry -> 75 | Assert.assertEquals(anOtherQueryEntry, savedQueryEntry) 76 | } 77 | .verifyComplete() 78 | } 79 | 80 | } 81 | -------------------------------------------------------------------------------- /worker-paginator/src/test/kotlin/elasticsearchpaginator/workerpaginator/HandleQueryIntegrationTest.kt: -------------------------------------------------------------------------------- 1 | package elasticsearchpaginator.workerpaginator 2 | 3 | import elasticsearchpaginator.core.model.Query 4 | import elasticsearchpaginator.workerpaginator.model.QueryEntry 5 | import elasticsearchpaginator.workerpaginator.service.QueriesService 6 | import org.elasticsearch.index.query.QueryBuilders 7 | import org.elasticsearch.search.sort.SortBuilders 8 | import org.junit.Assert 9 | import org.junit.jupiter.api.AfterEach 10 | import org.junit.jupiter.api.Test 11 | import org.springframework.beans.factory.annotation.Autowired 12 | import reactor.test.StepVerifier 13 | import java.time.Instant 14 | import java.util.concurrent.TimeoutException 15 | 16 | class HandleQueryIntegrationTest : AbstractIntegrationTest() { 17 | 18 | @Autowired 19 | private lateinit var queriesService: QueriesService 20 | 21 | @AfterEach 22 | internal fun cleanUp() { 23 | StepVerifier.create(this.clearQueryEntries()) 24 | .verifyComplete() 25 | } 26 | 27 | @Test 28 | fun `should create a new query entry and request pages computation for that query`() { 29 | val now = Instant.now() 30 | val query = Query( 31 | index = "index1", 32 | query = QueryBuilders.matchAllQuery().toString(), 33 | sort = listOf( 34 | SortBuilders.fieldSort("field1") 35 | .toString() 36 | ) 37 | .joinToString(",", "[", "]"), 38 | firstPageSize = 2, 39 | size = 4 40 | ) 41 | 42 | val rabbitMessage = this.consumeRabbitmqMessages(this.rabbitmqProperties.computePagesKey, Query::class.java) 43 | 44 | StepVerifier.create(this.queriesService.upsertQueryAndAskForPagesComputation(query)) 45 | .verifyComplete() 46 | 47 | StepVerifier.create(this.refreshQueryEntries().thenMany(this.findAllQueryEntries())) 48 | .assertNext { queryEntry -> 49 | Assert.assertEquals(query, queryEntry.query) 50 | Assert.assertTrue(queryEntry.lastUseDate > now) 51 | Assert.assertTrue(queryEntry.lastComputationDate > now) 52 | Assert.assertTrue(queryEntry.lastComputationDate > queryEntry.lastUseDate) 53 | } 54 | .verifyComplete() 55 | 56 | StepVerifier.create(rabbitMessage) 57 | .expectNext(query) 58 | .verifyComplete() 59 | } 60 | 61 | @Test 62 | fun `should request pages computation when last computation date is too recent`() { 63 | val now = Instant.now() 64 | val query = Query( 65 | index = "index1", 66 | query = QueryBuilders.matchAllQuery().toString(), 67 | sort = listOf( 68 | SortBuilders.fieldSort("field1") 69 | .toString() 70 | ) 71 | .joinToString(",", "[", "]"), 72 | firstPageSize = 2, 73 | size = 4 74 | ) 75 | val queryEntry = QueryEntry( 76 | query = query, 77 | lastUseDate = now, 78 | lastComputationDate = now 79 | ) 80 | 81 | StepVerifier.create(this.saveQueryEntries(listOf(queryEntry)).then(this.refreshQueryEntries())) 82 | .verifyComplete() 83 | 84 | val rabbitMessage = this.consumeRabbitmqMessages(this.rabbitmqProperties.computePagesKey, Query::class.java) 85 | 86 | StepVerifier.create(this.queriesService.upsertQueryAndAskForPagesComputation(query)) 87 | .verifyComplete() 88 | 89 | StepVerifier.create(this.refreshQueryEntries().thenMany(this.findAllQueryEntries())) 90 | .assertNext { entry -> 91 | Assert.assertEquals(query, entry.query) 92 | Assert.assertTrue(entry.lastUseDate > now) 93 | Assert.assertEquals(now, entry.lastComputationDate) 94 | } 95 | .verifyComplete() 96 | 97 | StepVerifier.create(rabbitMessage) 98 | .verifyError(TimeoutException::class.java) 99 | } 100 | 101 | } 102 | -------------------------------------------------------------------------------- /worker-paginator/src/test/resources/junit-platform.properties: -------------------------------------------------------------------------------- 1 | junit.jupiter.testinstance.lifecycle.default = per_class 2 | --------------------------------------------------------------------------------