├── .dockerignore ├── .github └── workflows │ └── ci.yml ├── .gitignore ├── .mvn └── wrapper │ ├── MavenWrapperDownloader.java │ ├── maven-wrapper.jar │ └── maven-wrapper.properties ├── LICENSE ├── README.md ├── mvnw ├── mvnw.cmd ├── pom.xml ├── roleStatements.yml ├── serverless.yml ├── src ├── assembly │ └── zip.xml ├── main │ ├── java │ │ └── dev │ │ │ └── arseny │ │ │ ├── RequestUtils.java │ │ │ ├── graalvm │ │ │ ├── ReflectionConfiguration.java │ │ │ └── substitutions │ │ │ │ ├── AttributeCreator.java │ │ │ │ ├── AttributeFactorySubstitution.java │ │ │ │ ├── CrtSubstitutions.java │ │ │ │ ├── DefaultAttributeFactorySubstitution.java │ │ │ │ ├── MMapDirectorySubstitutions.java │ │ │ │ ├── MappedByteBufferIndexInputProviderReplacement.java │ │ │ │ └── StaticImplementationAttributeFactorySubstitution.java │ │ │ ├── handler │ │ │ ├── DeleteIndex.java │ │ │ ├── EnqueueIndexHandler.java │ │ │ ├── IndexHandler.java │ │ │ └── QueryHandler.java │ │ │ ├── model │ │ │ ├── DeleteIndexRequest.java │ │ │ ├── ErrorResponse.java │ │ │ ├── IndexRequest.java │ │ │ ├── QueryRequest.java │ │ │ └── QueryResponse.java │ │ │ └── service │ │ │ ├── IndexConstants.java │ │ │ ├── IndexSearcherService.java │ │ │ ├── IndexWriterService.java │ │ │ └── SqsClientProducer.java │ └── resources │ │ └── application.properties └── test │ └── groovy │ └── dev.arseny │ └── HappyPathIntegrationTest.groovy └── target └── function.zip /.dockerignore: -------------------------------------------------------------------------------- 1 | * 2 | !target/*-runner 3 | !target/*-runner.jar 4 | !target/lib/* -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: ci 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | pull_request: 8 | 9 | env: 10 | JAVA_VERSION: 15 11 | 12 | jobs: 13 | build: 14 | runs-on: ubuntu-latest 15 | steps: 16 | - name: Checkout 17 | uses: actions/checkout@v2 18 | - name: Set up JDK 19 | uses: actions/setup-java@v1 20 | with: 21 | java-version: ${{env.JAVA_VERSION}} 22 | - name: Build 23 | run: ./mvnw package 24 | - name: Validate serverless 25 | uses: serverless/github-action@master 26 | with: 27 | args: package 28 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | **/target/* 2 | !**/target/function.zip 3 | .idea 4 | *.iml 5 | .serverless 6 | node_modules 7 | .DS_Store -------------------------------------------------------------------------------- /.mvn/wrapper/MavenWrapperDownloader.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2007-present the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | import java.net.*; 17 | import java.io.*; 18 | import java.nio.channels.*; 19 | import java.util.Properties; 20 | 21 | public class MavenWrapperDownloader { 22 | 23 | private static final String WRAPPER_VERSION = "0.5.6"; 24 | /** 25 | * Default URL to download the maven-wrapper.jar from, if no 'downloadUrl' is provided. 26 | */ 27 | private static final String DEFAULT_DOWNLOAD_URL = "https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/" 28 | + WRAPPER_VERSION + "/maven-wrapper-" + WRAPPER_VERSION + ".jar"; 29 | 30 | /** 31 | * Path to the maven-wrapper.properties file, which might contain a downloadUrl property to 32 | * use instead of the default one. 33 | */ 34 | private static final String MAVEN_WRAPPER_PROPERTIES_PATH = 35 | ".mvn/wrapper/maven-wrapper.properties"; 36 | 37 | /** 38 | * Path where the maven-wrapper.jar will be saved to. 39 | */ 40 | private static final String MAVEN_WRAPPER_JAR_PATH = 41 | ".mvn/wrapper/maven-wrapper.jar"; 42 | 43 | /** 44 | * Name of the property which should be used to override the default download url for the wrapper. 45 | */ 46 | private static final String PROPERTY_NAME_WRAPPER_URL = "wrapperUrl"; 47 | 48 | public static void main(String args[]) { 49 | System.out.println("- Downloader started"); 50 | File baseDirectory = new File(args[0]); 51 | System.out.println("- Using base directory: " + baseDirectory.getAbsolutePath()); 52 | 53 | // If the maven-wrapper.properties exists, read it and check if it contains a custom 54 | // wrapperUrl parameter. 55 | File mavenWrapperPropertyFile = new File(baseDirectory, MAVEN_WRAPPER_PROPERTIES_PATH); 56 | String url = DEFAULT_DOWNLOAD_URL; 57 | if(mavenWrapperPropertyFile.exists()) { 58 | FileInputStream mavenWrapperPropertyFileInputStream = null; 59 | try { 60 | mavenWrapperPropertyFileInputStream = new FileInputStream(mavenWrapperPropertyFile); 61 | Properties mavenWrapperProperties = new Properties(); 62 | mavenWrapperProperties.load(mavenWrapperPropertyFileInputStream); 63 | url = mavenWrapperProperties.getProperty(PROPERTY_NAME_WRAPPER_URL, url); 64 | } catch (IOException e) { 65 | System.out.println("- ERROR loading '" + MAVEN_WRAPPER_PROPERTIES_PATH + "'"); 66 | } finally { 67 | try { 68 | if(mavenWrapperPropertyFileInputStream != null) { 69 | mavenWrapperPropertyFileInputStream.close(); 70 | } 71 | } catch (IOException e) { 72 | // Ignore ... 73 | } 74 | } 75 | } 76 | System.out.println("- Downloading from: " + url); 77 | 78 | File outputFile = new File(baseDirectory.getAbsolutePath(), MAVEN_WRAPPER_JAR_PATH); 79 | if(!outputFile.getParentFile().exists()) { 80 | if(!outputFile.getParentFile().mkdirs()) { 81 | System.out.println( 82 | "- ERROR creating output directory '" + outputFile.getParentFile().getAbsolutePath() + "'"); 83 | } 84 | } 85 | System.out.println("- Downloading to: " + outputFile.getAbsolutePath()); 86 | try { 87 | downloadFileFromURL(url, outputFile); 88 | System.out.println("Done"); 89 | System.exit(0); 90 | } catch (Throwable e) { 91 | System.out.println("- Error downloading"); 92 | e.printStackTrace(); 93 | System.exit(1); 94 | } 95 | } 96 | 97 | private static void downloadFileFromURL(String urlString, File destination) throws Exception { 98 | if (System.getenv("MVNW_USERNAME") != null && System.getenv("MVNW_PASSWORD") != null) { 99 | String username = System.getenv("MVNW_USERNAME"); 100 | char[] password = System.getenv("MVNW_PASSWORD").toCharArray(); 101 | Authenticator.setDefault(new Authenticator() { 102 | @Override 103 | protected PasswordAuthentication getPasswordAuthentication() { 104 | return new PasswordAuthentication(username, password); 105 | } 106 | }); 107 | } 108 | URL website = new URL(urlString); 109 | ReadableByteChannel rbc; 110 | rbc = Channels.newChannel(website.openStream()); 111 | FileOutputStream fos = new FileOutputStream(destination); 112 | fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE); 113 | fos.close(); 114 | rbc.close(); 115 | } 116 | 117 | } 118 | -------------------------------------------------------------------------------- /.mvn/wrapper/maven-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ArsenyYankovsky/lucene-serverless/b75feec5af92759b888ca89f61d1fb55b8112fef/.mvn/wrapper/maven-wrapper.jar -------------------------------------------------------------------------------- /.mvn/wrapper/maven-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.6.3/apache-maven-3.6.3-bin.zip 2 | wrapperUrl=https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar 3 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021-2024 Arseny Yankovski, Thomas Barrasso 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Lucene Serverless 2 | 3 | This project demonstrates a proof-of-concept serverless full-text search solution built with Apache Lucene and Quarkus framework. 4 | 5 | ✔️ No servers 6 | 7 | ✔️ No fixed costs 8 | 9 | ✔️ Low (250-300ms) cold starts 10 | 11 | ⚠️ Better deletion policy is required. Right now old segment files are not deleted as a simple workaround to handle concurrent reads and writes 12 | 13 | ℹ️ Cost can be controlled via several factors: 14 | 15 | * EFS' [Elastic](https://docs.aws.amazon.com/efs/latest/ug/performance.html#throughput-modes) throughput mode cost more $$, but scales better up and down 16 | * Lambda's [Provisioned Concurrency](https://docs.aws.amazon.com/lambda/latest/dg/provisioned-concurrency.html) provides faster, more consistent first request times but costs more $$ 17 | * Lambda memory is kept ~256mb, tune based on index size and observed speeds. Note: vCPU is allocated proportional to memory. 18 | 19 | Please note that the project is not ready for production since I haven't tested it under a prolonged load and to be honest interfaces need to be nicer. 20 | 21 | Read the blog post about it [here](https://medium.com/@arsenyyankovski/serverless-full-text-search-with-aws-lambda-and-efs-cf24e1b6fe3b) 22 | 23 | ## Prerequisites 24 | - [Serverless framework >= 1.56.1](https://serverless.com/framework/docs/getting-started/) 25 | - AWS account 26 | 27 | ## Run it 28 | 1. Replace region, vpc id and subnets in the `serverless.yml` file 29 | 30 | 2. Deploy the stack 31 | `sls deploy` 32 | 33 | 3. Don't forget to remove it if you're not planning to use it 34 | `sls remove` 35 | 36 | ### Index a document 37 | 38 | URL: `https://.execute-api..amazonaws.com/dev/index` 39 | 40 | HTTP method: POST 41 | 42 | Example request body: 43 | 44 | ```json 45 | { 46 | "indexName": "books", 47 | "documents": [ 48 | { 49 | "name": "The Foundation", 50 | "author": "Isaac Asimov" 51 | } 52 | ] 53 | } 54 | ``` 55 | 56 | ### Query documents 57 | 58 | URL: `https://.execute-api..amazonaws.com/dev/query` 59 | 60 | HTTP method: POST 61 | 62 | Example request body: 63 | 64 | ```json 65 | { 66 | "indexName": "books", 67 | "query": "author:isaac" 68 | } 69 | ``` 70 | 71 | Example response body: 72 | 73 | ```json 74 | { 75 | "totalDocuments": "1", 76 | "documents": [ 77 | { 78 | "author": "Isaac Asimov", 79 | "name": "The Foundation" 80 | } 81 | ] 82 | } 83 | ``` 84 | 85 | ## Build native image 86 | `./mvnw clean package` 87 | -------------------------------------------------------------------------------- /mvnw: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # ---------------------------------------------------------------------------- 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | # ---------------------------------------------------------------------------- 20 | 21 | # ---------------------------------------------------------------------------- 22 | # Maven Start Up Batch script 23 | # 24 | # Required ENV vars: 25 | # ------------------ 26 | # JAVA_HOME - location of a JDK home dir 27 | # 28 | # Optional ENV vars 29 | # ----------------- 30 | # M2_HOME - location of maven2's installed home dir 31 | # MAVEN_OPTS - parameters passed to the Java VM when running Maven 32 | # e.g. to debug Maven itself, use 33 | # set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 34 | # MAVEN_SKIP_RC - flag to disable loading of mavenrc files 35 | # ---------------------------------------------------------------------------- 36 | 37 | if [ -z "$MAVEN_SKIP_RC" ] ; then 38 | 39 | if [ -f /etc/mavenrc ] ; then 40 | . /etc/mavenrc 41 | fi 42 | 43 | if [ -f "$HOME/.mavenrc" ] ; then 44 | . "$HOME/.mavenrc" 45 | fi 46 | 47 | fi 48 | 49 | # OS specific support. $var _must_ be set to either true or false. 50 | cygwin=false; 51 | darwin=false; 52 | mingw=false 53 | case "`uname`" in 54 | CYGWIN*) cygwin=true ;; 55 | MINGW*) mingw=true;; 56 | Darwin*) darwin=true 57 | # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home 58 | # See https://developer.apple.com/library/mac/qa/qa1170/_index.html 59 | if [ -z "$JAVA_HOME" ]; then 60 | if [ -x "/usr/libexec/java_home" ]; then 61 | export JAVA_HOME="`/usr/libexec/java_home`" 62 | else 63 | export JAVA_HOME="/Library/Java/Home" 64 | fi 65 | fi 66 | ;; 67 | esac 68 | 69 | if [ -z "$JAVA_HOME" ] ; then 70 | if [ -r /etc/gentoo-release ] ; then 71 | JAVA_HOME=`java-config --jre-home` 72 | fi 73 | fi 74 | 75 | if [ -z "$M2_HOME" ] ; then 76 | ## resolve links - $0 may be a link to maven's home 77 | PRG="$0" 78 | 79 | # need this for relative symlinks 80 | while [ -h "$PRG" ] ; do 81 | ls=`ls -ld "$PRG"` 82 | link=`expr "$ls" : '.*-> \(.*\)$'` 83 | if expr "$link" : '/.*' > /dev/null; then 84 | PRG="$link" 85 | else 86 | PRG="`dirname "$PRG"`/$link" 87 | fi 88 | done 89 | 90 | saveddir=`pwd` 91 | 92 | M2_HOME=`dirname "$PRG"`/.. 93 | 94 | # make it fully qualified 95 | M2_HOME=`cd "$M2_HOME" && pwd` 96 | 97 | cd "$saveddir" 98 | # echo Using m2 at $M2_HOME 99 | fi 100 | 101 | # For Cygwin, ensure paths are in UNIX format before anything is touched 102 | if $cygwin ; then 103 | [ -n "$M2_HOME" ] && 104 | M2_HOME=`cygpath --unix "$M2_HOME"` 105 | [ -n "$JAVA_HOME" ] && 106 | JAVA_HOME=`cygpath --unix "$JAVA_HOME"` 107 | [ -n "$CLASSPATH" ] && 108 | CLASSPATH=`cygpath --path --unix "$CLASSPATH"` 109 | fi 110 | 111 | # For Mingw, ensure paths are in UNIX format before anything is touched 112 | if $mingw ; then 113 | [ -n "$M2_HOME" ] && 114 | M2_HOME="`(cd "$M2_HOME"; pwd)`" 115 | [ -n "$JAVA_HOME" ] && 116 | JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`" 117 | fi 118 | 119 | if [ -z "$JAVA_HOME" ]; then 120 | javaExecutable="`which javac`" 121 | if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then 122 | # readlink(1) is not available as standard on Solaris 10. 123 | readLink=`which readlink` 124 | if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then 125 | if $darwin ; then 126 | javaHome="`dirname \"$javaExecutable\"`" 127 | javaExecutable="`cd \"$javaHome\" && pwd -P`/javac" 128 | else 129 | javaExecutable="`readlink -f \"$javaExecutable\"`" 130 | fi 131 | javaHome="`dirname \"$javaExecutable\"`" 132 | javaHome=`expr "$javaHome" : '\(.*\)/bin'` 133 | JAVA_HOME="$javaHome" 134 | export JAVA_HOME 135 | fi 136 | fi 137 | fi 138 | 139 | if [ -z "$JAVACMD" ] ; then 140 | if [ -n "$JAVA_HOME" ] ; then 141 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 142 | # IBM's JDK on AIX uses strange locations for the executables 143 | JAVACMD="$JAVA_HOME/jre/sh/java" 144 | else 145 | JAVACMD="$JAVA_HOME/bin/java" 146 | fi 147 | else 148 | JAVACMD="`which java`" 149 | fi 150 | fi 151 | 152 | if [ ! -x "$JAVACMD" ] ; then 153 | echo "Error: JAVA_HOME is not defined correctly." >&2 154 | echo " We cannot execute $JAVACMD" >&2 155 | exit 1 156 | fi 157 | 158 | if [ -z "$JAVA_HOME" ] ; then 159 | echo "Warning: JAVA_HOME environment variable is not set." 160 | fi 161 | 162 | CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher 163 | 164 | # traverses directory structure from process work directory to filesystem root 165 | # first directory with .mvn subdirectory is considered project base directory 166 | find_maven_basedir() { 167 | 168 | if [ -z "$1" ] 169 | then 170 | echo "Path not specified to find_maven_basedir" 171 | return 1 172 | fi 173 | 174 | basedir="$1" 175 | wdir="$1" 176 | while [ "$wdir" != '/' ] ; do 177 | if [ -d "$wdir"/.mvn ] ; then 178 | basedir=$wdir 179 | break 180 | fi 181 | # workaround for JBEAP-8937 (on Solaris 10/Sparc) 182 | if [ -d "${wdir}" ]; then 183 | wdir=`cd "$wdir/.."; pwd` 184 | fi 185 | # end of workaround 186 | done 187 | echo "${basedir}" 188 | } 189 | 190 | # concatenates all lines of a file 191 | concat_lines() { 192 | if [ -f "$1" ]; then 193 | echo "$(tr -s '\n' ' ' < "$1")" 194 | fi 195 | } 196 | 197 | BASE_DIR=`find_maven_basedir "$(pwd)"` 198 | if [ -z "$BASE_DIR" ]; then 199 | exit 1; 200 | fi 201 | 202 | ########################################################################################## 203 | # Extension to allow automatically downloading the maven-wrapper.jar from Maven-central 204 | # This allows using the maven wrapper in projects that prohibit checking in binary data. 205 | ########################################################################################## 206 | if [ -r "$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" ]; then 207 | if [ "$MVNW_VERBOSE" = true ]; then 208 | echo "Found .mvn/wrapper/maven-wrapper.jar" 209 | fi 210 | else 211 | if [ "$MVNW_VERBOSE" = true ]; then 212 | echo "Couldn't find .mvn/wrapper/maven-wrapper.jar, downloading it ..." 213 | fi 214 | if [ -n "$MVNW_REPOURL" ]; then 215 | jarUrl="$MVNW_REPOURL/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar" 216 | else 217 | jarUrl="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar" 218 | fi 219 | while IFS="=" read key value; do 220 | case "$key" in (wrapperUrl) jarUrl="$value"; break ;; 221 | esac 222 | done < "$BASE_DIR/.mvn/wrapper/maven-wrapper.properties" 223 | if [ "$MVNW_VERBOSE" = true ]; then 224 | echo "Downloading from: $jarUrl" 225 | fi 226 | wrapperJarPath="$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" 227 | if $cygwin; then 228 | wrapperJarPath=`cygpath --path --windows "$wrapperJarPath"` 229 | fi 230 | 231 | if command -v wget > /dev/null; then 232 | if [ "$MVNW_VERBOSE" = true ]; then 233 | echo "Found wget ... using wget" 234 | fi 235 | if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then 236 | wget "$jarUrl" -O "$wrapperJarPath" 237 | else 238 | wget --http-user=$MVNW_USERNAME --http-password=$MVNW_PASSWORD "$jarUrl" -O "$wrapperJarPath" 239 | fi 240 | elif command -v curl > /dev/null; then 241 | if [ "$MVNW_VERBOSE" = true ]; then 242 | echo "Found curl ... using curl" 243 | fi 244 | if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then 245 | curl -o "$wrapperJarPath" "$jarUrl" -f 246 | else 247 | curl --user $MVNW_USERNAME:$MVNW_PASSWORD -o "$wrapperJarPath" "$jarUrl" -f 248 | fi 249 | 250 | else 251 | if [ "$MVNW_VERBOSE" = true ]; then 252 | echo "Falling back to using Java to download" 253 | fi 254 | javaClass="$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.java" 255 | # For Cygwin, switch paths to Windows format before running javac 256 | if $cygwin; then 257 | javaClass=`cygpath --path --windows "$javaClass"` 258 | fi 259 | if [ -e "$javaClass" ]; then 260 | if [ ! -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then 261 | if [ "$MVNW_VERBOSE" = true ]; then 262 | echo " - Compiling MavenWrapperDownloader.java ..." 263 | fi 264 | # Compiling the Java class 265 | ("$JAVA_HOME/bin/javac" "$javaClass") 266 | fi 267 | if [ -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then 268 | # Running the downloader 269 | if [ "$MVNW_VERBOSE" = true ]; then 270 | echo " - Running MavenWrapperDownloader.java ..." 271 | fi 272 | ("$JAVA_HOME/bin/java" -cp .mvn/wrapper MavenWrapperDownloader "$MAVEN_PROJECTBASEDIR") 273 | fi 274 | fi 275 | fi 276 | fi 277 | ########################################################################################## 278 | # End of extension 279 | ########################################################################################## 280 | 281 | export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"} 282 | if [ "$MVNW_VERBOSE" = true ]; then 283 | echo $MAVEN_PROJECTBASEDIR 284 | fi 285 | MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" 286 | 287 | # For Cygwin, switch paths to Windows format before running java 288 | if $cygwin; then 289 | [ -n "$M2_HOME" ] && 290 | M2_HOME=`cygpath --path --windows "$M2_HOME"` 291 | [ -n "$JAVA_HOME" ] && 292 | JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"` 293 | [ -n "$CLASSPATH" ] && 294 | CLASSPATH=`cygpath --path --windows "$CLASSPATH"` 295 | [ -n "$MAVEN_PROJECTBASEDIR" ] && 296 | MAVEN_PROJECTBASEDIR=`cygpath --path --windows "$MAVEN_PROJECTBASEDIR"` 297 | fi 298 | 299 | # Provide a "standardized" way to retrieve the CLI args that will 300 | # work with both Windows and non-Windows executions. 301 | MAVEN_CMD_LINE_ARGS="$MAVEN_CONFIG $@" 302 | export MAVEN_CMD_LINE_ARGS 303 | 304 | WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain 305 | 306 | exec "$JAVACMD" \ 307 | $MAVEN_OPTS \ 308 | -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ 309 | "-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ 310 | "-Dorg.apache.lucene.store.MMapDirectory.enableUnmapHack=false" \ 311 | ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@" 312 | -------------------------------------------------------------------------------- /mvnw.cmd: -------------------------------------------------------------------------------- 1 | @REM ---------------------------------------------------------------------------- 2 | @REM Licensed to the Apache Software Foundation (ASF) under one 3 | @REM or more contributor license agreements. See the NOTICE file 4 | @REM distributed with this work for additional information 5 | @REM regarding copyright ownership. The ASF licenses this file 6 | @REM to you under the Apache License, Version 2.0 (the 7 | @REM "License"); you may not use this file except in compliance 8 | @REM with the License. You may obtain a copy of the License at 9 | @REM 10 | @REM http://www.apache.org/licenses/LICENSE-2.0 11 | @REM 12 | @REM Unless required by applicable law or agreed to in writing, 13 | @REM software distributed under the License is distributed on an 14 | @REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | @REM KIND, either express or implied. See the License for the 16 | @REM specific language governing permissions and limitations 17 | @REM under the License. 18 | @REM ---------------------------------------------------------------------------- 19 | 20 | @REM ---------------------------------------------------------------------------- 21 | @REM Maven Start Up Batch script 22 | @REM 23 | @REM Required ENV vars: 24 | @REM JAVA_HOME - location of a JDK home dir 25 | @REM 26 | @REM Optional ENV vars 27 | @REM M2_HOME - location of maven2's installed home dir 28 | @REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands 29 | @REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a keystroke before ending 30 | @REM MAVEN_OPTS - parameters passed to the Java VM when running Maven 31 | @REM e.g. to debug Maven itself, use 32 | @REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 33 | @REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files 34 | @REM ---------------------------------------------------------------------------- 35 | 36 | @REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on' 37 | @echo off 38 | @REM set title of command window 39 | title %0 40 | @REM enable echoing by setting MAVEN_BATCH_ECHO to 'on' 41 | @if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO% 42 | 43 | @REM set %HOME% to equivalent of $HOME 44 | if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%") 45 | 46 | @REM Execute a user defined script before this one 47 | if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre 48 | @REM check for pre script, once with legacy .bat ending and once with .cmd ending 49 | if exist "%HOME%\mavenrc_pre.bat" call "%HOME%\mavenrc_pre.bat" 50 | if exist "%HOME%\mavenrc_pre.cmd" call "%HOME%\mavenrc_pre.cmd" 51 | :skipRcPre 52 | 53 | @setlocal 54 | 55 | set ERROR_CODE=0 56 | 57 | @REM To isolate internal variables from possible post scripts, we use another setlocal 58 | @setlocal 59 | 60 | @REM ==== START VALIDATION ==== 61 | if not "%JAVA_HOME%" == "" goto OkJHome 62 | 63 | echo. 64 | echo Error: JAVA_HOME not found in your environment. >&2 65 | echo Please set the JAVA_HOME variable in your environment to match the >&2 66 | echo location of your Java installation. >&2 67 | echo. 68 | goto error 69 | 70 | :OkJHome 71 | if exist "%JAVA_HOME%\bin\java.exe" goto init 72 | 73 | echo. 74 | echo Error: JAVA_HOME is set to an invalid directory. >&2 75 | echo JAVA_HOME = "%JAVA_HOME%" >&2 76 | echo Please set the JAVA_HOME variable in your environment to match the >&2 77 | echo location of your Java installation. >&2 78 | echo. 79 | goto error 80 | 81 | @REM ==== END VALIDATION ==== 82 | 83 | :init 84 | 85 | @REM Find the project base dir, i.e. the directory that contains the folder ".mvn". 86 | @REM Fallback to current working directory if not found. 87 | 88 | set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR% 89 | IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir 90 | 91 | set EXEC_DIR=%CD% 92 | set WDIR=%EXEC_DIR% 93 | :findBaseDir 94 | IF EXIST "%WDIR%"\.mvn goto baseDirFound 95 | cd .. 96 | IF "%WDIR%"=="%CD%" goto baseDirNotFound 97 | set WDIR=%CD% 98 | goto findBaseDir 99 | 100 | :baseDirFound 101 | set MAVEN_PROJECTBASEDIR=%WDIR% 102 | cd "%EXEC_DIR%" 103 | goto endDetectBaseDir 104 | 105 | :baseDirNotFound 106 | set MAVEN_PROJECTBASEDIR=%EXEC_DIR% 107 | cd "%EXEC_DIR%" 108 | 109 | :endDetectBaseDir 110 | 111 | IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig 112 | 113 | @setlocal EnableExtensions EnableDelayedExpansion 114 | for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a 115 | @endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS% 116 | 117 | :endReadAdditionalConfig 118 | 119 | SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" 120 | set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar" 121 | set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain 122 | 123 | set DOWNLOAD_URL="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar" 124 | 125 | FOR /F "tokens=1,2 delims==" %%A IN ("%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties") DO ( 126 | IF "%%A"=="wrapperUrl" SET DOWNLOAD_URL=%%B 127 | ) 128 | 129 | @REM Extension to allow automatically downloading the maven-wrapper.jar from Maven-central 130 | @REM This allows using the maven wrapper in projects that prohibit checking in binary data. 131 | if exist %WRAPPER_JAR% ( 132 | if "%MVNW_VERBOSE%" == "true" ( 133 | echo Found %WRAPPER_JAR% 134 | ) 135 | ) else ( 136 | if not "%MVNW_REPOURL%" == "" ( 137 | SET DOWNLOAD_URL="%MVNW_REPOURL%/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar" 138 | ) 139 | if "%MVNW_VERBOSE%" == "true" ( 140 | echo Couldn't find %WRAPPER_JAR%, downloading it ... 141 | echo Downloading from: %DOWNLOAD_URL% 142 | ) 143 | 144 | powershell -Command "&{"^ 145 | "$webclient = new-object System.Net.WebClient;"^ 146 | "if (-not ([string]::IsNullOrEmpty('%MVNW_USERNAME%') -and [string]::IsNullOrEmpty('%MVNW_PASSWORD%'))) {"^ 147 | "$webclient.Credentials = new-object System.Net.NetworkCredential('%MVNW_USERNAME%', '%MVNW_PASSWORD%');"^ 148 | "}"^ 149 | "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; $webclient.DownloadFile('%DOWNLOAD_URL%', '%WRAPPER_JAR%')"^ 150 | "}" 151 | if "%MVNW_VERBOSE%" == "true" ( 152 | echo Finished downloading %WRAPPER_JAR% 153 | ) 154 | ) 155 | @REM End of extension 156 | 157 | @REM Provide a "standardized" way to retrieve the CLI args that will 158 | @REM work with both Windows and non-Windows executions. 159 | set MAVEN_CMD_LINE_ARGS=%* 160 | 161 | %MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %* 162 | if ERRORLEVEL 1 goto error 163 | goto end 164 | 165 | :error 166 | set ERROR_CODE=1 167 | 168 | :end 169 | @endlocal & set ERROR_CODE=%ERROR_CODE% 170 | 171 | if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost 172 | @REM check for post script, once with legacy .bat ending and once with .cmd ending 173 | if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat" 174 | if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd" 175 | :skipRcPost 176 | 177 | @REM pause the script if MAVEN_BATCH_PAUSE is set to 'on' 178 | if "%MAVEN_BATCH_PAUSE%" == "on" pause 179 | 180 | if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE% 181 | 182 | exit /B %ERROR_CODE% 183 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | dev.arseny 7 | lucene-serverless 8 | 1.0-SNAPSHOT 9 | 10 | 1.13.7.Final 11 | 9.9.1 12 | 3.2.1 13 | 3.8.1 14 | UTF-8 15 | 3.0.7 16 | 1.8 17 | 1.8 18 | true 19 | docker 20 | true 21 | native 22 | quay.io/quarkus/ubi-quarkus-native-image:20.3.1-java11 23 | 24 | 25 | 26 | 27 | 28 | io.quarkus 29 | quarkus-bom 30 | ${quarkus.version} 31 | pom 32 | import 33 | 34 | 35 | 36 | software.amazon.awssdk 37 | bom 38 | 2.21.46 39 | pom 40 | import 41 | 42 | 43 | 44 | org.spockframework 45 | spock-bom 46 | 2.0-M4-groovy-3.0 47 | pom 48 | import 49 | 50 | 51 | 52 | 53 | 54 | io.quarkus 55 | quarkus-amazon-lambda 56 | 57 | 58 | org.apache.lucene 59 | lucene-core 60 | ${lucene.version} 61 | 62 | 63 | org.apache.lucene 64 | lucene-queryparser 65 | ${lucene.version} 66 | 67 | 68 | software.amazon.awssdk 69 | sqs 70 | 71 | 72 | software.amazon.awssdk 73 | url-connection-client 74 | 75 | 76 | com.oracle.substratevm 77 | svm 78 | 19.2.1 79 | provided 80 | 81 | 82 | 83 | 84 | 85 | org.spockframework 86 | spock-core 87 | test 88 | 89 | 90 | org.spockframework 91 | spock-junit4 92 | test 93 | 94 | 95 | org.codehaus.groovy 96 | groovy 97 | ${groovy.version} 98 | test 99 | 100 | 101 | org.codehaus.groovy 102 | groovy-json 103 | ${groovy.version} 104 | test 105 | 106 | 107 | net.bytebuddy 108 | byte-buddy 109 | 1.10.10 110 | test 111 | 112 | 113 | org.objenesis 114 | objenesis 115 | 3.1 116 | test 117 | 118 | 119 | org.hamcrest 120 | hamcrest-core 121 | 2.2 122 | test 123 | 124 | 125 | commons-logging 126 | commons-logging 127 | 1.1.1 128 | test 129 | 130 | 131 | software.amazon.awssdk 132 | lambda 133 | test 134 | 135 | 136 | 137 | 138 | 139 | 140 | io.quarkus 141 | quarkus-maven-plugin 142 | ${quarkus.version} 143 | 144 | 145 | 146 | build 147 | 148 | 149 | 150 | 151 | 152 | org.apache.maven.plugins 153 | maven-assembly-plugin 154 | 3.1.0 155 | 156 | 157 | zip-assembly 158 | package 159 | 160 | single 161 | 162 | 163 | function 164 | 165 | src/assembly/zip.xml 166 | 167 | false 168 | false 169 | 170 | 171 | 172 | 173 | 174 | 175 | org.codehaus.gmavenplus 176 | gmavenplus-plugin 177 | 1.12.0 178 | 179 | 180 | 181 | compile 182 | compileTests 183 | 184 | 185 | 186 | 187 | 188 | maven-surefire-plugin 189 | 3.0.0-M4 190 | 191 | false 192 | 193 | **/*IntegrationTest.java 194 | 195 | 196 | 197 | 198 | integration-test 199 | 200 | test 201 | 202 | integration-test 203 | 204 | 205 | none 206 | 207 | 208 | **/*IntegrationTest.java 209 | 210 | 211 | 212 | 213 | 214 | 215 | 216 | 217 | 218 | 219 | central 220 | Central Repository 221 | https://repo.maven.apache.org/maven2 222 | default 223 | 224 | false 225 | 226 | 227 | never 228 | 229 | 230 | 231 | 232 | 233 | central 234 | Central Repository 235 | https://repo.maven.apache.org/maven2 236 | default 237 | 238 | false 239 | 240 | 241 | 242 | -------------------------------------------------------------------------------- /roleStatements.yml: -------------------------------------------------------------------------------- 1 | - Effect: Allow 2 | Action: 3 | - logs:* 4 | Resource: arn:aws:logs:*:*:* 5 | 6 | - Effect: Allow 7 | Action: 8 | - elasticfilesystem:ClientMount 9 | - elasticfilesystem:ClientRootAccess 10 | - elasticfilesystem:ClientWrite 11 | - elasticfilesystem:DescribeMountTargets 12 | Resource: "*" 13 | 14 | - Effect: Allow 15 | Action: 16 | - ec2:CreateNetworkInterface 17 | - ec2:DescribeNetworkInterfaces 18 | - ec2:DeleteNetworkInterface 19 | Resource: "*" 20 | 21 | 22 | - Effect: Allow 23 | Action: 24 | - sqs:ChangeMessageVisibility 25 | - sqs:DeleteMessage 26 | - sqs:ReceiveMessage 27 | - sqs:SendMessage 28 | - sqs:GetQueueAttributes 29 | Resource: 30 | - !GetAtt WriteQueue.Arn 31 | -------------------------------------------------------------------------------- /serverless.yml: -------------------------------------------------------------------------------- 1 | service: lucene-serverless 2 | variablesResolutionMode: 20210219 3 | 4 | # TODO: replace all variables below 5 | custom: 6 | name: ${sls:stage}-${self:service} 7 | region: ${opt:region, "us-east-1"} 8 | vpcId: vpc-00690f6e 9 | subnetId1: subnet-039f496e 10 | subnetId2: subnet-0c0af8ea 11 | 12 | frameworkVersion: ">=1.56.1" 13 | 14 | provider: 15 | name: aws 16 | region: ${self:custom.region} 17 | versionFunctions: false 18 | apiGateway: 19 | shouldStartNameWithService: true 20 | tracing: 21 | lambda: false 22 | timeout: 15 23 | environment: 24 | stage: prod 25 | DISABLE_SIGNAL_HANDLERS: true 26 | iam: 27 | role: 28 | statements: ${file(roleStatements.yml)} 29 | vpc: 30 | securityGroupIds: 31 | - Ref: EfsSecurityGroup 32 | subnetIds: 33 | - ${self:custom.subnetId1} 34 | - ${self:custom.subnetId2} 35 | 36 | package: 37 | individually: true 38 | 39 | functions: 40 | query: 41 | name: ${self:custom.name}-query 42 | runtime: provided 43 | handler: native.handler 44 | provisionedConcurrency: 1 # ~$2.5/m, lower cold start time 45 | memorySize: 256 46 | events: 47 | - http: POST /query 48 | dependsOn: 49 | - EfsMountTarget1 50 | - EfsMountTarget2 51 | - EfsAccessPoint 52 | fileSystemConfig: 53 | localMountPath: /mnt/data 54 | arn: 55 | Fn::GetAtt: [EfsAccessPoint, Arn] 56 | package: 57 | artifact: target/function.zip 58 | environment: 59 | QUARKUS_LAMBDA_HANDLER: query 60 | QUARKUS_PROFILE: prod 61 | 62 | index: 63 | name: ${self:custom.name}-index 64 | runtime: provided 65 | handler: native.handler 66 | reservedConcurrency: 1 67 | memorySize: 256 68 | timeout: 180 69 | dependsOn: 70 | - EfsMountTarget1 71 | - EfsMountTarget2 72 | - EfsAccessPoint 73 | fileSystemConfig: 74 | localMountPath: /mnt/data 75 | arn: 76 | Fn::GetAtt: [EfsAccessPoint, Arn] 77 | package: 78 | artifact: target/function.zip 79 | environment: 80 | QUARKUS_LAMBDA_HANDLER: index 81 | QUARKUS_PROFILE: prod 82 | events: 83 | - sqs: 84 | arn: 85 | Fn::GetAtt: [WriteQueue, Arn] 86 | batchSize: 5000 87 | maximumBatchingWindow: 5 88 | 89 | enqueue-index: 90 | name: ${self:custom.name}-enqueue-index 91 | runtime: provided 92 | handler: native.handler 93 | memorySize: 256 94 | package: 95 | artifact: target/function.zip 96 | vpc: 97 | securityGroupIds: [] 98 | subnetIds: [] 99 | events: 100 | - http: POST /index 101 | environment: 102 | QUARKUS_LAMBDA_HANDLER: enqueue-index 103 | QUARKUS_PROFILE: prod 104 | QUEUE_URL: 105 | Ref: WriteQueue 106 | 107 | delete-index: 108 | name: ${self:custom.name}-delete-index 109 | runtime: provided 110 | handler: native.handler 111 | memorySize: 256 112 | dependsOn: 113 | - EfsMountTarget1 114 | - EfsMountTarget2 115 | - EfsAccessPoint 116 | fileSystemConfig: 117 | localMountPath: /mnt/data 118 | arn: 119 | Fn::GetAtt: [EfsAccessPoint, Arn] 120 | package: 121 | artifact: target/function.zip 122 | environment: 123 | QUARKUS_LAMBDA_HANDLER: deleteIndex 124 | QUARKUS_PROFILE: prod 125 | 126 | resources: 127 | Resources: 128 | WriteQueue: 129 | Type: AWS::SQS::Queue 130 | Properties: 131 | QueueName: ${self:custom.name}-write-queue 132 | VisibilityTimeout: 900 133 | RedrivePolicy: 134 | deadLetterTargetArn: 135 | Fn::GetAtt: [WriteDLQ, Arn] 136 | maxReceiveCount: 5 137 | 138 | WriteDLQ: 139 | Type: AWS::SQS::Queue 140 | Properties: 141 | QueueName: ${self:custom.name}-write-dlq 142 | MessageRetentionPeriod: 1209600 # 14 days in seconds 143 | 144 | FileSystem: 145 | Type: AWS::EFS::FileSystem 146 | Properties: 147 | BackupPolicy: 148 | Status: DISABLED 149 | FileSystemTags: 150 | - Key: Name 151 | Value: ${self:custom.name}-fs 152 | PerformanceMode: generalPurpose 153 | ThroughputMode: elastic # faster scale up/down 154 | Encrypted: true 155 | FileSystemPolicy: 156 | Version: "2012-10-17" 157 | Statement: 158 | - Effect: "Allow" 159 | Action: 160 | - "elasticfilesystem:ClientMount" 161 | Principal: 162 | AWS: "*" 163 | 164 | EfsSecurityGroup: 165 | Type: AWS::EC2::SecurityGroup 166 | Properties: 167 | VpcId: ${self:custom.vpcId} 168 | GroupDescription: "mnt target sg" 169 | SecurityGroupIngress: 170 | - IpProtocol: -1 171 | CidrIp: "0.0.0.0/0" 172 | - IpProtocol: -1 173 | CidrIpv6: "::/0" 174 | SecurityGroupEgress: 175 | - IpProtocol: -1 176 | CidrIp: "0.0.0.0/0" 177 | - IpProtocol: -1 178 | CidrIpv6: "::/0" 179 | 180 | EfsMountTarget1: 181 | Type: AWS::EFS::MountTarget 182 | Properties: 183 | FileSystemId: !Ref FileSystem 184 | SubnetId: ${self:custom.subnetId1} 185 | SecurityGroups: 186 | - Ref: EfsSecurityGroup 187 | 188 | EfsMountTarget2: 189 | Type: AWS::EFS::MountTarget 190 | Properties: 191 | FileSystemId: !Ref FileSystem 192 | SubnetId: ${self:custom.subnetId2} 193 | SecurityGroups: 194 | - Ref: EfsSecurityGroup 195 | 196 | EfsAccessPoint: 197 | Type: "AWS::EFS::AccessPoint" 198 | Properties: 199 | FileSystemId: !Ref FileSystem 200 | PosixUser: 201 | Uid: "1000" 202 | Gid: "1000" 203 | RootDirectory: 204 | CreationInfo: 205 | OwnerGid: "1000" 206 | OwnerUid: "1000" 207 | Permissions: "0777" 208 | Path: "/mnt/data" 209 | -------------------------------------------------------------------------------- /src/assembly/zip.xml: -------------------------------------------------------------------------------- 1 | 4 | function-native-zip 5 | / 6 | 7 | zip 8 | 9 | 10 | 11 | ${project.build.directory}${file.separator}${artifactId}-${version}-runner 12 | ${file.separator} 13 | bootstrap 14 | 755 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /src/main/java/dev/arseny/RequestUtils.java: -------------------------------------------------------------------------------- 1 | package dev.arseny; 2 | 3 | import com.amazonaws.services.lambda.runtime.events.APIGatewayProxyRequestEvent; 4 | import com.amazonaws.services.lambda.runtime.events.APIGatewayProxyResponseEvent; 5 | import com.fasterxml.jackson.core.JsonProcessingException; 6 | import com.fasterxml.jackson.databind.ObjectMapper; 7 | import com.fasterxml.jackson.databind.ObjectReader; 8 | import com.fasterxml.jackson.databind.ObjectWriter; 9 | import dev.arseny.model.*; 10 | import org.jboss.logging.Logger; 11 | 12 | import java.io.IOException; 13 | 14 | public class RequestUtils { 15 | private static final Logger LOG = Logger.getLogger(RequestUtils.class); 16 | 17 | static ObjectWriter writer = new ObjectMapper().writerFor(ErrorResponse.class); 18 | static ObjectWriter queryResponseWriter = new ObjectMapper().writerFor(QueryResponse.class); 19 | static ObjectReader indexRequestReader = new ObjectMapper().readerFor(IndexRequest.class); 20 | static ObjectReader deleteIndexRequestReader = new ObjectMapper().readerFor(DeleteIndexRequest.class); 21 | static ObjectReader queryRequestReader = new ObjectMapper().readerFor(QueryRequest.class); 22 | 23 | public static APIGatewayProxyResponseEvent errorResponse(int errorCode, String message) { 24 | APIGatewayProxyResponseEvent response = new APIGatewayProxyResponseEvent(); 25 | try { 26 | return response.withStatusCode(errorCode).withBody(writer.writeValueAsString(new ErrorResponse(message, errorCode))); 27 | } catch (JsonProcessingException e) { 28 | LOG.error(e); 29 | return response.withStatusCode(500).withBody("Internal error"); 30 | } 31 | } 32 | 33 | public static IndexRequest parseIndexRequest(String eventBody) { 34 | try { 35 | return indexRequestReader.readValue(eventBody); 36 | } catch (IOException e) { 37 | throw new RuntimeException("Unable to parse list of Index Requests in body", e); 38 | } 39 | } 40 | 41 | public static DeleteIndexRequest parseDeleteIndexRequest(APIGatewayProxyRequestEvent event) { 42 | try { 43 | return deleteIndexRequestReader.readValue(event.getBody()); 44 | } catch (IOException e) { 45 | throw new RuntimeException("Unable to parse a delete index request in body", e); 46 | } 47 | } 48 | 49 | public static QueryRequest parseQueryRequest(APIGatewayProxyRequestEvent event) { 50 | try { 51 | return queryRequestReader.readValue(event.getBody()); 52 | } catch (IOException e) { 53 | throw new RuntimeException("Unable to parse a query request in body", e); 54 | } 55 | } 56 | 57 | public static APIGatewayProxyResponseEvent successResponse(QueryResponse queryResponse) throws JsonProcessingException { 58 | APIGatewayProxyResponseEvent response = new APIGatewayProxyResponseEvent(); 59 | try { 60 | return response.withStatusCode(200).withBody(queryResponseWriter.writeValueAsString(queryResponse)); 61 | } catch (JsonProcessingException e) { 62 | LOG.error(e); 63 | return response.withStatusCode(500).withBody("Internal error"); 64 | } 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /src/main/java/dev/arseny/graalvm/ReflectionConfiguration.java: -------------------------------------------------------------------------------- 1 | package dev.arseny.graalvm; 2 | 3 | import io.quarkus.runtime.annotations.RegisterForReflection; 4 | import org.apache.lucene.search.ScoreDoc; 5 | 6 | @RegisterForReflection(targets = ScoreDoc.class) 7 | public class ReflectionConfiguration { 8 | } 9 | -------------------------------------------------------------------------------- /src/main/java/dev/arseny/graalvm/substitutions/AttributeCreator.java: -------------------------------------------------------------------------------- 1 | package dev.arseny.graalvm.substitutions; 2 | 3 | import org.apache.lucene.analysis.tokenattributes.BytesTermAttribute; 4 | import org.apache.lucene.analysis.tokenattributes.BytesTermAttributeImpl; 5 | import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; 6 | import org.apache.lucene.analysis.tokenattributes.CharTermAttributeImpl; 7 | import org.apache.lucene.analysis.tokenattributes.FlagsAttribute; 8 | import org.apache.lucene.analysis.tokenattributes.FlagsAttributeImpl; 9 | import org.apache.lucene.analysis.tokenattributes.KeywordAttribute; 10 | import org.apache.lucene.analysis.tokenattributes.KeywordAttributeImpl; 11 | import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; 12 | import org.apache.lucene.analysis.tokenattributes.OffsetAttributeImpl; 13 | import org.apache.lucene.analysis.tokenattributes.PackedTokenAttributeImpl; 14 | import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; 15 | import org.apache.lucene.analysis.tokenattributes.PayloadAttributeImpl; 16 | import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; 17 | import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttributeImpl; 18 | import org.apache.lucene.analysis.tokenattributes.PositionLengthAttribute; 19 | import org.apache.lucene.analysis.tokenattributes.PositionLengthAttributeImpl; 20 | import org.apache.lucene.analysis.tokenattributes.SentenceAttributeImpl; 21 | import org.apache.lucene.analysis.tokenattributes.TermFrequencyAttribute; 22 | import org.apache.lucene.analysis.tokenattributes.TermFrequencyAttributeImpl; 23 | import org.apache.lucene.analysis.tokenattributes.TypeAttribute; 24 | import org.apache.lucene.analysis.tokenattributes.TypeAttributeImpl; 25 | import org.apache.lucene.search.BoostAttribute; 26 | import org.apache.lucene.search.BoostAttributeImpl; 27 | import org.apache.lucene.search.MaxNonCompetitiveBoostAttribute; 28 | import org.apache.lucene.search.MaxNonCompetitiveBoostAttributeImpl; 29 | import org.apache.lucene.util.Attribute; 30 | import org.apache.lucene.util.AttributeImpl; 31 | 32 | /** 33 | * Utility to create an {@link Attribute} based on a class avoiding usage of {@link java.lang.invoke.MethodHandle} 34 | */ 35 | public final class AttributeCreator { 36 | 37 | static AttributeImpl create(Class attClass) { 38 | if (attClass == PackedTokenAttributeImpl.class) { 39 | return new PackedTokenAttributeImpl(); 40 | } else if (attClass == CharTermAttribute.class) { 41 | return new CharTermAttributeImpl(); 42 | } else if (attClass == OffsetAttribute.class) { 43 | return new OffsetAttributeImpl(); 44 | } else if (attClass == PositionIncrementAttribute.class) { 45 | return new PositionIncrementAttributeImpl(); 46 | } else if (attClass == TypeAttribute.class) { 47 | return new TypeAttributeImpl(); 48 | } else if (attClass == TermFrequencyAttribute.class) { 49 | return new TermFrequencyAttributeImpl(); 50 | } else if (attClass == PositionLengthAttribute.class) { 51 | return new PositionLengthAttributeImpl(); 52 | } else if (attClass == BoostAttribute.class) { 53 | return new BoostAttributeImpl(); 54 | } else if (attClass == KeywordAttribute.class) { 55 | return new KeywordAttributeImpl(); 56 | } else if (attClass == PayloadAttribute.class) { 57 | return new PayloadAttributeImpl(); 58 | } else if (attClass == BytesTermAttribute.class) { 59 | return new BytesTermAttributeImpl(); 60 | } else if (attClass == FlagsAttribute.class) { 61 | return new FlagsAttributeImpl(); 62 | } else if (attClass == MaxNonCompetitiveBoostAttribute.class) { 63 | return new MaxNonCompetitiveBoostAttributeImpl(); 64 | } else if (attClass == SentenceAttributeImpl.class) { 65 | return new SentenceAttributeImpl(); 66 | } 67 | 68 | throw new UnsupportedOperationException( 69 | String.format("Attribute class '%s' not supported in the image", attClass)); 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /src/main/java/dev/arseny/graalvm/substitutions/AttributeFactorySubstitution.java: -------------------------------------------------------------------------------- 1 | package dev.arseny.graalvm.substitutions; 2 | 3 | import com.oracle.svm.core.annotate.Substitute; 4 | import com.oracle.svm.core.annotate.TargetClass; 5 | 6 | import org.apache.lucene.util.AttributeFactory; 7 | import org.apache.lucene.util.AttributeFactory.StaticImplementationAttributeFactory; 8 | import org.apache.lucene.util.AttributeImpl; 9 | 10 | import java.lang.reflect.UndeclaredThrowableException; 11 | 12 | /** 13 | * An AttributeFactory creates instances of {@link AttributeImpl}s. 14 | */ 15 | @TargetClass(AttributeFactory.class) 16 | public final class AttributeFactorySubstitution { 17 | @Substitute 18 | @SuppressWarnings("unchecked") 19 | public static AttributeFactory getStaticImplementation(AttributeFactory delegate, 20 | Class clazz) { 21 | return new StaticImplementationAttributeFactory(delegate, clazz) { 22 | @Override 23 | protected A createInstance() { 24 | try { 25 | return (A) AttributeCreator.create(clazz); 26 | } catch (Error | RuntimeException e) { 27 | throw e; 28 | } catch (Throwable e) { 29 | throw new UndeclaredThrowableException(e); 30 | } 31 | } 32 | }; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/dev/arseny/graalvm/substitutions/CrtSubstitutions.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package dev.arseny.graalvm.substitutions; 18 | 19 | import java.util.Arrays; 20 | import java.util.function.BooleanSupplier; 21 | 22 | /** 23 | * Based on the original implementation in quarkus-amazon-services: 24 | * 25 | * https://github.com/quarkiverse/quarkus-amazon-services/blob/main/common/runtime/src/main/java/io/quarkus/amazon/common/runtime/CrtSubstitutions.java 26 | */ 27 | public class CrtSubstitutions { 28 | public static final String QUARKUS_AWSSDK_CONFIG_CLASS_NAME = "io.quarkus.amazon.common.runtime.SdkConfig"; 29 | static final String SOFTWARE_AMAZON_AWSSDK_CRT_PACKAGE = "software.amazon.awssdk.crt"; 30 | static final String SOFTWARE_AMAZON_AWSSDK_HTTP_AUTH_AWS_CRT_PACKAGE = "software.amazon.awssdk.http.auth.aws.crt"; 31 | 32 | static final class IsCrtAbsent implements BooleanSupplier { 33 | @Override 34 | public boolean getAsBoolean() { 35 | return isQuarkusAwsAbsent() && Arrays.stream(Package.getPackages()) 36 | .map(Package::getName) 37 | .noneMatch(p -> p.equals(SOFTWARE_AMAZON_AWSSDK_CRT_PACKAGE)); 38 | } 39 | } 40 | 41 | static final class IsHttpAuthAwsCrtAbsent implements BooleanSupplier { 42 | @Override 43 | public boolean getAsBoolean() { 44 | return isQuarkusAwsAbsent() && Arrays.stream(Package.getPackages()) 45 | .map(Package::getName) 46 | .noneMatch(p -> p.equals(SOFTWARE_AMAZON_AWSSDK_HTTP_AUTH_AWS_CRT_PACKAGE)); 47 | } 48 | } 49 | 50 | static boolean isQuarkusAwsAbsent() { 51 | try { 52 | Thread.currentThread().getContextClassLoader().loadClass(QUARKUS_AWSSDK_CONFIG_CLASS_NAME); 53 | return false; 54 | } catch (ClassNotFoundException e) { 55 | return true; 56 | } 57 | } 58 | } 59 | 60 | -------------------------------------------------------------------------------- /src/main/java/dev/arseny/graalvm/substitutions/DefaultAttributeFactorySubstitution.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Gunnar Morling 3 | * 4 | * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | package dev.arseny.graalvm.substitutions; 7 | 8 | import com.oracle.svm.core.annotate.Substitute; 9 | import com.oracle.svm.core.annotate.TargetClass; 10 | import org.apache.lucene.util.Attribute; 11 | import org.apache.lucene.util.AttributeImpl; 12 | 13 | /** 14 | * An AttributeFactory creates instances of {@link AttributeImpl}s. 15 | */ 16 | @TargetClass(className = "org.apache.lucene.util.AttributeFactory$DefaultAttributeFactory") 17 | public final class DefaultAttributeFactorySubstitution { 18 | @Substitute 19 | public AttributeImpl createAttributeInstance(Class attClass) { 20 | return AttributeCreator.create(attClass); 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/main/java/dev/arseny/graalvm/substitutions/MMapDirectorySubstitutions.java: -------------------------------------------------------------------------------- 1 | package dev.arseny.graalvm.substitutions; 2 | 3 | import com.oracle.svm.core.annotate.Alias; 4 | import com.oracle.svm.core.annotate.RecomputeFieldValue; 5 | import com.oracle.svm.core.annotate.TargetClass; 6 | 7 | @TargetClass(org.apache.lucene.store.MMapDirectory.class) 8 | final class MMapDirectorySubstitutions { 9 | @Alias 10 | @RecomputeFieldValue(kind = RecomputeFieldValue.Kind.FromAlias, isFinal = true) 11 | private static boolean UNMAP_SUPPORTED = false; 12 | } 13 | -------------------------------------------------------------------------------- /src/main/java/dev/arseny/graalvm/substitutions/MappedByteBufferIndexInputProviderReplacement.java: -------------------------------------------------------------------------------- 1 | package dev.arseny.graalvm.substitutions; 2 | 3 | import com.oracle.svm.core.annotate.*; 4 | 5 | /** 6 | * Used to disable the workaround for bug JDK-4724038 7 | * 8 | * @author w.glanzer, 23.08.2023 9 | */ 10 | @TargetClass(className = "org.apache.lucene.store.MappedByteBufferIndexInputProvider") 11 | public final class MappedByteBufferIndexInputProviderReplacement { 12 | @Substitute 13 | private static boolean checkUnmapHackSysprop() { 14 | return false; // disable permanently 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/main/java/dev/arseny/graalvm/substitutions/StaticImplementationAttributeFactorySubstitution.java: -------------------------------------------------------------------------------- 1 | package dev.arseny.graalvm.substitutions; 2 | 3 | import com.oracle.svm.core.annotate.Substitute; 4 | import com.oracle.svm.core.annotate.TargetClass; 5 | import org.apache.lucene.util.Attribute; 6 | import org.apache.lucene.util.AttributeFactory.StaticImplementationAttributeFactory; 7 | import org.apache.lucene.util.AttributeImpl; 8 | 9 | /** 10 | * An AttributeFactory creates instances of {@link AttributeImpl}s. 11 | */ 12 | @TargetClass(StaticImplementationAttributeFactory.class) 13 | public final class StaticImplementationAttributeFactorySubstitution { 14 | @Substitute 15 | public AttributeImpl createAttributeInstance(Class attClass) { 16 | return AttributeCreator.create(attClass); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/main/java/dev/arseny/handler/DeleteIndex.java: -------------------------------------------------------------------------------- 1 | package dev.arseny.handler; 2 | 3 | import com.amazonaws.services.lambda.runtime.Context; 4 | import com.amazonaws.services.lambda.runtime.RequestHandler; 5 | import com.amazonaws.services.lambda.runtime.events.APIGatewayProxyRequestEvent; 6 | import com.amazonaws.services.lambda.runtime.events.APIGatewayProxyResponseEvent; 7 | import dev.arseny.RequestUtils; 8 | import dev.arseny.model.DeleteIndexRequest; 9 | import dev.arseny.service.IndexWriterService; 10 | import org.apache.lucene.index.IndexWriter; 11 | import org.jboss.logging.Logger; 12 | 13 | import javax.inject.Inject; 14 | import javax.inject.Named; 15 | import java.io.IOException; 16 | 17 | @Named("deleteIndex") 18 | public class DeleteIndex implements RequestHandler { 19 | private static final Logger LOG = Logger.getLogger(IndexHandler.class); 20 | 21 | @Inject 22 | protected IndexWriterService indexWriterService; 23 | 24 | @Override 25 | public APIGatewayProxyResponseEvent handleRequest(APIGatewayProxyRequestEvent event, Context context) { 26 | DeleteIndexRequest deleteIndexRequest = RequestUtils.parseDeleteIndexRequest(event); 27 | 28 | IndexWriter writer = indexWriterService.getIndexWriter(deleteIndexRequest.getIndexName()); 29 | 30 | try { 31 | writer.deleteAll(); 32 | writer.commit(); 33 | writer.close(); 34 | } catch (IOException e) { 35 | LOG.error(e); 36 | } 37 | 38 | return new APIGatewayProxyResponseEvent().withStatusCode(200); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/main/java/dev/arseny/handler/EnqueueIndexHandler.java: -------------------------------------------------------------------------------- 1 | package dev.arseny.handler; 2 | 3 | import com.amazonaws.services.lambda.runtime.Context; 4 | import com.amazonaws.services.lambda.runtime.RequestHandler; 5 | import com.amazonaws.services.lambda.runtime.events.APIGatewayProxyRequestEvent; 6 | import com.amazonaws.services.lambda.runtime.events.APIGatewayProxyResponseEvent; 7 | import software.amazon.awssdk.services.sqs.SqsClient; 8 | import software.amazon.awssdk.services.sqs.model.SendMessageRequest; 9 | 10 | import javax.inject.Inject; 11 | import javax.inject.Named; 12 | 13 | @Named("enqueue-index") 14 | public class EnqueueIndexHandler implements RequestHandler { 15 | protected String queueName = System.getenv("QUEUE_URL"); 16 | 17 | @Inject 18 | protected SqsClient sqsClient; 19 | 20 | @Override 21 | public APIGatewayProxyResponseEvent handleRequest(APIGatewayProxyRequestEvent event, Context context) { 22 | this.sqsClient.sendMessage(SendMessageRequest.builder() 23 | .messageBody(event.getBody()) 24 | .queueUrl(queueName).build()); 25 | 26 | return new APIGatewayProxyResponseEvent().withStatusCode(200); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/main/java/dev/arseny/handler/IndexHandler.java: -------------------------------------------------------------------------------- 1 | package dev.arseny.handler; 2 | 3 | import com.amazonaws.services.lambda.runtime.Context; 4 | import com.amazonaws.services.lambda.runtime.RequestHandler; 5 | import com.amazonaws.services.lambda.runtime.events.APIGatewayProxyResponseEvent; 6 | import com.amazonaws.services.lambda.runtime.events.SQSEvent; 7 | import dev.arseny.RequestUtils; 8 | import dev.arseny.model.IndexRequest; 9 | import dev.arseny.service.IndexWriterService; 10 | import org.apache.lucene.document.Document; 11 | import org.apache.lucene.document.Field; 12 | import org.apache.lucene.document.TextField; 13 | import org.apache.lucene.index.IndexWriter; 14 | import org.jboss.logging.Logger; 15 | 16 | import javax.inject.Inject; 17 | import javax.inject.Named; 18 | import java.io.IOException; 19 | import java.util.ArrayList; 20 | import java.util.HashMap; 21 | import java.util.List; 22 | import java.util.Map; 23 | 24 | @Named("index") 25 | public class IndexHandler implements RequestHandler { 26 | private static final Logger LOG = Logger.getLogger(IndexHandler.class); 27 | 28 | @Inject 29 | protected IndexWriterService indexWriterService; 30 | 31 | @Override 32 | public APIGatewayProxyResponseEvent handleRequest(SQSEvent event, Context context) { 33 | List records = event.getRecords(); 34 | 35 | List requests = new ArrayList<>(); 36 | 37 | for (SQSEvent.SQSMessage record : records) { 38 | requests.add(RequestUtils.parseIndexRequest(record.getBody())); 39 | } 40 | 41 | Map writerMap = new HashMap<>(); 42 | 43 | for (IndexRequest request : requests) { 44 | IndexWriter writer; 45 | if (writerMap.containsKey(request.getIndexName())) { 46 | writer = writerMap.get(request.getIndexName()); 47 | } else { 48 | writer = indexWriterService.getIndexWriter(request.getIndexName()); 49 | writerMap.put(request.getIndexName(), writer); 50 | } 51 | 52 | List documents = new ArrayList<>(); 53 | 54 | for (Map requestDocument : request.getDocuments()) { 55 | Document document = new Document(); 56 | for (Map.Entry entry : requestDocument.entrySet()) { 57 | document.add(new TextField(entry.getKey(), entry.getValue().toString(), Field.Store.YES)); 58 | } 59 | documents.add(document); 60 | } 61 | 62 | try { 63 | writer.addDocuments(documents); 64 | } catch (IOException e) { 65 | LOG.error(e); 66 | } 67 | } 68 | 69 | for (IndexWriter writer : writerMap.values()) { 70 | try { 71 | writer.commit(); 72 | writer.close(); 73 | } catch (IOException e) { 74 | LOG.error(e); 75 | } 76 | } 77 | 78 | return new APIGatewayProxyResponseEvent().withStatusCode(200); 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /src/main/java/dev/arseny/handler/QueryHandler.java: -------------------------------------------------------------------------------- 1 | package dev.arseny.handler; 2 | 3 | import com.amazonaws.services.lambda.runtime.Context; 4 | import com.amazonaws.services.lambda.runtime.RequestHandler; 5 | import com.amazonaws.services.lambda.runtime.events.APIGatewayProxyRequestEvent; 6 | import com.amazonaws.services.lambda.runtime.events.APIGatewayProxyResponseEvent; 7 | import dev.arseny.RequestUtils; 8 | import dev.arseny.model.QueryRequest; 9 | import dev.arseny.model.QueryResponse; 10 | import dev.arseny.service.IndexSearcherService; 11 | import org.apache.lucene.analysis.standard.StandardAnalyzer; 12 | import org.apache.lucene.document.Document; 13 | import org.apache.lucene.index.IndexableField; 14 | import org.apache.lucene.queryparser.classic.ParseException; 15 | import org.apache.lucene.queryparser.classic.QueryParser; 16 | import org.apache.lucene.search.*; 17 | import org.jboss.logging.Logger; 18 | 19 | import javax.inject.Inject; 20 | import javax.inject.Named; 21 | import java.io.IOException; 22 | import java.util.HashMap; 23 | import java.util.Map; 24 | 25 | @Named("query") 26 | public class QueryHandler implements RequestHandler { 27 | private static final Logger LOG = Logger.getLogger(QueryHandler.class); 28 | 29 | @Inject 30 | protected IndexSearcherService indexSearcherService; 31 | 32 | @Override 33 | public APIGatewayProxyResponseEvent handleRequest(APIGatewayProxyRequestEvent event, Context context) { 34 | QueryRequest queryRequest = RequestUtils.parseQueryRequest(event); 35 | 36 | QueryParser qp = new QueryParser("content", new StandardAnalyzer()); 37 | 38 | QueryResponse queryResponse = new QueryResponse(); 39 | 40 | try { 41 | Query query = qp.parse(queryRequest.getQuery()); 42 | 43 | IndexSearcher searcher = indexSearcherService.getIndexSearcher(queryRequest.getIndexName()); 44 | 45 | TopDocs topDocs = searcher.search(query, 10); 46 | 47 | for (ScoreDoc scoreDocs : topDocs.scoreDocs) { 48 | Document document = searcher.storedFields().document(scoreDocs.doc); 49 | 50 | Map result = new HashMap<>(); 51 | 52 | for (IndexableField field : document.getFields()) { 53 | result.put(field.name(), field.stringValue()); 54 | } 55 | 56 | queryResponse.getDocuments().add(result); 57 | } 58 | 59 | queryResponse.setTotalDocuments((topDocs.totalHits.relation == TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO ? "≥" : "") + topDocs.totalHits.value); 60 | 61 | return RequestUtils.successResponse(queryResponse); 62 | } catch (ParseException | IOException e) { 63 | LOG.error(e); 64 | 65 | return RequestUtils.errorResponse(500, "Error"); 66 | } 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /src/main/java/dev/arseny/model/DeleteIndexRequest.java: -------------------------------------------------------------------------------- 1 | package dev.arseny.model; 2 | 3 | public class DeleteIndexRequest { 4 | private String indexName; 5 | 6 | public DeleteIndexRequest() { 7 | } 8 | 9 | public String getIndexName() { 10 | return indexName; 11 | } 12 | 13 | public void setIndexName(String indexName) { 14 | this.indexName = indexName; 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/main/java/dev/arseny/model/ErrorResponse.java: -------------------------------------------------------------------------------- 1 | package dev.arseny.model; 2 | 3 | import io.quarkus.runtime.annotations.RegisterForReflection; 4 | 5 | @RegisterForReflection 6 | public class ErrorResponse { 7 | 8 | private String message; 9 | private int errorCode; 10 | 11 | public ErrorResponse() { 12 | } 13 | 14 | public ErrorResponse(String message, int errorCode) { 15 | this.message = message; 16 | this.errorCode = errorCode; 17 | } 18 | 19 | public String getMessage() { 20 | return message; 21 | } 22 | 23 | public int getErrorCode() { 24 | return errorCode; 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/dev/arseny/model/IndexRequest.java: -------------------------------------------------------------------------------- 1 | package dev.arseny.model; 2 | 3 | import java.util.List; 4 | import java.util.Map; 5 | 6 | public class IndexRequest { 7 | private String indexName; 8 | private List> documents; 9 | 10 | public IndexRequest() { 11 | } 12 | 13 | public String getIndexName() { 14 | return indexName; 15 | } 16 | 17 | public void setIndexName(String indexName) { 18 | this.indexName = indexName; 19 | } 20 | 21 | public List> getDocuments() { 22 | return documents; 23 | } 24 | 25 | public void setDocuments(List> documents) { 26 | this.documents = documents; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/main/java/dev/arseny/model/QueryRequest.java: -------------------------------------------------------------------------------- 1 | package dev.arseny.model; 2 | 3 | public class QueryRequest { 4 | private String indexName; 5 | private String query; 6 | 7 | public QueryRequest() { 8 | } 9 | 10 | public String getIndexName() { 11 | return indexName; 12 | } 13 | 14 | public void setIndexName(String indexName) { 15 | this.indexName = indexName; 16 | } 17 | 18 | public String getQuery() { 19 | return query; 20 | } 21 | 22 | public void setQuery(String query) { 23 | this.query = query; 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /src/main/java/dev/arseny/model/QueryResponse.java: -------------------------------------------------------------------------------- 1 | package dev.arseny.model; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | import java.util.Map; 6 | 7 | public class QueryResponse { 8 | private String totalDocuments; 9 | private List> documents = new ArrayList<>(); 10 | 11 | public QueryResponse() { 12 | } 13 | 14 | public String getTotalDocuments() { 15 | return totalDocuments; 16 | } 17 | 18 | public void setTotalDocuments(String totalDocuments) { 19 | this.totalDocuments = totalDocuments; 20 | } 21 | 22 | public List> getDocuments() { 23 | return documents; 24 | } 25 | 26 | public void setDocuments(List> documents) { 27 | this.documents = documents; 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/main/java/dev/arseny/service/IndexConstants.java: -------------------------------------------------------------------------------- 1 | package dev.arseny.service; 2 | 3 | public class IndexConstants { 4 | public static final String LUCENE_INDEX_ROOT_DIRECTORY = "/mnt/data/"; 5 | } 6 | -------------------------------------------------------------------------------- /src/main/java/dev/arseny/service/IndexSearcherService.java: -------------------------------------------------------------------------------- 1 | package dev.arseny.service; 2 | 3 | import dev.arseny.RequestUtils; 4 | import org.apache.lucene.index.DirectoryReader; 5 | import org.apache.lucene.search.IndexSearcher; 6 | import org.apache.lucene.store.FSDirectory; 7 | import org.jboss.logging.Logger; 8 | 9 | import javax.enterprise.context.ApplicationScoped; 10 | import java.io.IOException; 11 | import java.nio.file.Paths; 12 | 13 | @ApplicationScoped 14 | public class IndexSearcherService { 15 | private static final Logger LOG = Logger.getLogger(RequestUtils.class); 16 | 17 | private DirectoryReader directoryReader; 18 | 19 | public IndexSearcher getIndexSearcher(String indexName) { 20 | try { 21 | DirectoryReader newDirectoryReader; 22 | 23 | if (directoryReader == null) { 24 | newDirectoryReader = DirectoryReader.open(FSDirectory.open(Paths.get(IndexConstants.LUCENE_INDEX_ROOT_DIRECTORY + indexName))); 25 | } else { 26 | newDirectoryReader = DirectoryReader.openIfChanged(directoryReader); 27 | } 28 | 29 | if (newDirectoryReader != null) { 30 | this.directoryReader = newDirectoryReader; 31 | } 32 | 33 | return new IndexSearcher(this.directoryReader); 34 | } catch (IOException e) { 35 | LOG.error("Error while trying to create an index searcher for index " + indexName, e); 36 | throw new RuntimeException(e); 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/main/java/dev/arseny/service/IndexWriterService.java: -------------------------------------------------------------------------------- 1 | package dev.arseny.service; 2 | 3 | import dev.arseny.RequestUtils; 4 | import org.apache.lucene.analysis.standard.StandardAnalyzer; 5 | import org.apache.lucene.index.IndexWriter; 6 | import org.apache.lucene.index.IndexWriterConfig; 7 | import org.apache.lucene.index.NoDeletionPolicy; 8 | import org.apache.lucene.store.FSDirectory; 9 | import org.jboss.logging.Logger; 10 | 11 | import javax.enterprise.context.ApplicationScoped; 12 | import java.io.IOException; 13 | import java.nio.file.Paths; 14 | 15 | 16 | @ApplicationScoped 17 | public class IndexWriterService { 18 | private static final Logger LOG = Logger.getLogger(RequestUtils.class); 19 | 20 | public IndexWriter getIndexWriter(String indexName) { 21 | try { 22 | IndexWriter indexWriter = new IndexWriter( 23 | FSDirectory.open(Paths.get(IndexConstants.LUCENE_INDEX_ROOT_DIRECTORY + indexName)), 24 | new IndexWriterConfig(new StandardAnalyzer()) 25 | .setIndexDeletionPolicy(NoDeletionPolicy.INSTANCE) 26 | ); 27 | 28 | return indexWriter; 29 | } catch (IOException e) { 30 | LOG.error("Error while trying to create an index writer for index " + indexName, e); 31 | throw new RuntimeException(e); 32 | } 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/dev/arseny/service/SqsClientProducer.java: -------------------------------------------------------------------------------- 1 | package dev.arseny.service; 2 | 3 | import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient; 4 | import software.amazon.awssdk.services.sqs.SqsClient; 5 | 6 | import javax.enterprise.context.ApplicationScoped; 7 | import javax.enterprise.inject.Produces; 8 | 9 | @ApplicationScoped 10 | public class SqsClientProducer { 11 | protected SqsClient client = SqsClient.builder() 12 | .httpClientBuilder(UrlConnectionHttpClient.builder()) 13 | .build(); 14 | 15 | @Produces 16 | public SqsClient sqsClient() { 17 | return client; 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | quarkus.lambda.handler=fetchAll 2 | quarkus.package.type=native 3 | quarkus.native.additional-build-args=--allow-incomplete-classpath,--enable-url-protocols=https,--report-unsupported-elements-at-runtime 4 | quarkus.native.debug.enabled=false 5 | -------------------------------------------------------------------------------- /src/test/groovy/dev.arseny/HappyPathIntegrationTest.groovy: -------------------------------------------------------------------------------- 1 | package dev.arseny 2 | 3 | import groovy.json.JsonOutput 4 | import groovy.json.JsonSlurper 5 | import software.amazon.awssdk.core.SdkBytes 6 | import software.amazon.awssdk.http.apache.ApacheHttpClient 7 | import software.amazon.awssdk.regions.Region 8 | import software.amazon.awssdk.services.lambda.LambdaClient 9 | import software.amazon.awssdk.services.lambda.model.InvokeRequest 10 | import spock.lang.* 11 | 12 | class HappyPathIntegrationTest extends Specification { 13 | def jsonSlurper = new JsonSlurper() 14 | 15 | @Shared 16 | def client = LambdaClient.builder() 17 | .httpClient(ApacheHttpClient.builder().build()) 18 | .region(Region.EU_NORTH_1).build() 19 | 20 | private deleteTestIndex() { 21 | def response = client.invoke(InvokeRequest.builder() 22 | .functionName("dev-lucene-serverless-delete-index") 23 | .payload(SdkBytes.fromUtf8String(JsonOutput.toJson([ 24 | body: JsonOutput.toJson([ 25 | indexName: 'books', 26 | ]) 27 | ]))).build()) 28 | } 29 | 30 | def setupSpec() { 31 | deleteTestIndex() 32 | } 33 | 34 | def cleanupSpec() { 35 | deleteTestIndex() 36 | } 37 | 38 | def "simple write-read test"() { 39 | when: 40 | client.invoke(InvokeRequest.builder() 41 | .functionName("dev-lucene-serverless-enqueue-index") 42 | .payload(SdkBytes.fromUtf8String(JsonOutput.toJson([ 43 | body: JsonOutput.toJson([ 44 | indexName: 'books', 45 | documents: [[ 46 | name : 'Foundation', 47 | author: 'Isaac Asimov', 48 | ]] 49 | ]) 50 | ]))).build()) 51 | 52 | sleep(60000) 53 | 54 | def response = client.invoke(InvokeRequest.builder() 55 | .functionName("dev-lucene-serverless-query") 56 | .payload(SdkBytes.fromUtf8String(JsonOutput.toJson([ 57 | body: JsonOutput.toJson([ 58 | indexName: 'books', 59 | query : 'author:isaac', 60 | ]) 61 | ]))).build()) 62 | 63 | def result = jsonSlurper.parseText(response.payload().asUtf8String()) 64 | def body = jsonSlurper.parseText(result.body) 65 | 66 | then: 67 | response.statusCode() == 200 68 | body.documents[0].author == "Isaac Asimov" 69 | body.documents[0].name == "Foundation" 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /target/function.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ArsenyYankovsky/lucene-serverless/b75feec5af92759b888ca89f61d1fb55b8112fef/target/function.zip --------------------------------------------------------------------------------