├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ ├── documentation_issue.md │ └── enhancement.md └── workflows │ ├── deploy-docs-and-test.yml │ └── pullrequest.yml ├── .gitignore ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── SECURITY.md ├── build.gradle.kts ├── docker-compose-es-7.yml ├── docker-compose-es-8.yml ├── docker-compose-es-9.yml ├── docker-compose-os-1.yml ├── docker-compose-os-2.yml ├── docker-compose-os-3.yml ├── docs ├── build.gradle.kts └── src │ └── test │ ├── kotlin │ └── documentation │ │ ├── DocumentationTest.kt │ │ ├── extensions.kt │ │ ├── manual │ │ ├── bulk │ │ │ ├── bulk.kt │ │ │ └── indexmanagement │ │ │ │ ├── datastreams.kt │ │ │ │ └── index-management.kt │ │ ├── crud │ │ │ └── crud.kt │ │ ├── extending │ │ │ └── extending.kt │ │ ├── gettingstarted │ │ │ ├── client-customization.kt │ │ │ ├── getting-started.kt │ │ │ ├── migrating.md │ │ │ └── whatisktsearch.md │ │ ├── indexrepo │ │ │ └── indexrepo.kt │ │ ├── intro.md │ │ ├── jupyter │ │ │ └── jupyter.md │ │ ├── knn │ │ │ └── knn.kt │ │ ├── manual-index.kt │ │ ├── outro.md │ │ ├── scripting │ │ │ ├── sampleScript.md │ │ │ └── scripting.kt │ │ └── search │ │ │ ├── aggregations.kt │ │ │ ├── compound-queries.kt │ │ │ ├── deep-paging.kt │ │ │ ├── delete-by-query.kt │ │ │ ├── geo-queries.kt │ │ │ ├── helpers.kt │ │ │ ├── highlighting.kt │ │ │ ├── join-queries.kt │ │ │ ├── reusing-query-logic.kt │ │ │ ├── search.kt │ │ │ ├── specialized-queries.kt │ │ │ ├── term-level-queries.kt │ │ │ └── text-queries.kt │ │ └── projectreadme │ │ ├── gradle-maven.md │ │ ├── oneliner.md │ │ ├── readme-intro.md │ │ ├── readme-outro.md │ │ ├── readme.kt │ │ └── related.md │ └── resources │ ├── embeddings.tsv │ └── input.tsv ├── es_kibana ├── README.md └── docker-compose.yml ├── forbidden_signatures.txt ├── gradle.properties ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── jupyter-example ├── README.md ├── config.env └── kt-search-example.ipynb ├── kotlin-js-store └── yarn.lock ├── publish.sh ├── search-client ├── build.gradle.kts └── src │ ├── commonMain │ └── kotlin │ │ └── com │ │ └── jillesvangurp │ │ └── ktsearch │ │ ├── KtorRestClient.kt │ │ ├── Restclient.kt │ │ ├── RoundRobinNodeSelector.kt │ │ ├── SearchClient.kt │ │ ├── SearchResponse.kt │ │ ├── SniffingNodeSelector.kt │ │ ├── aliases.kt │ │ ├── analyze-api.kt │ │ ├── bulk-api.kt │ │ ├── cluster-api.kt │ │ ├── common.kt │ │ ├── delete-by-query.kt │ │ ├── documents-api.kt │ │ ├── ilm.kt │ │ ├── index-api.kt │ │ ├── index-templates.kt │ │ ├── reindex-api.kt │ │ ├── repository │ │ ├── IndexRepository.kt │ │ ├── KotlinxSerializationModelSerializationStrategy.kt │ │ └── ModelSerializationStrategy.kt │ │ ├── request-dsl.kt │ │ ├── root-api.kt │ │ ├── search-api.kt │ │ ├── snapshot-api.kt │ │ ├── tasks-api.kt │ │ └── update-api.kt │ ├── commonTest │ └── kotlin │ │ └── com │ │ └── jillesvangurp │ │ └── ktsearch │ │ ├── AggQueryTest.kt │ │ ├── AliasManagementTest.kt │ │ ├── AnalyzeTest.kt │ │ ├── AuthenticationTest.kt │ │ ├── BulkTest.kt │ │ ├── ClusterHealthTest.kt │ │ ├── CommonTestKt.kt │ │ ├── DeleteByQyeryTest.kt │ │ ├── DeleteIndexTest.kt │ │ ├── DocumentCRUDTest.kt │ │ ├── GeoSpatialQueriesTest.kt │ │ ├── IlmTest.kt │ │ ├── IndexCreateTest.kt │ │ ├── IndexTemplateTest.kt │ │ ├── KnnSearchTest.kt │ │ ├── NestedQueryTest.kt │ │ ├── ParentChildQueryTest.kt │ │ ├── ReindexTest.kt │ │ ├── ScrollTest.kt │ │ ├── SearchResponseDefaultsTest.kt │ │ ├── SearchTest.kt │ │ ├── SearchTestBase.kt │ │ ├── SniffingNodeSelectorTest.kt │ │ ├── SpecializedQueriesTest.kt │ │ ├── repository │ │ └── IndexRepositoryTest.kt │ │ └── test-fixture.kt │ ├── iosMain │ └── kotlin │ │ └── com │ │ └── jillesvangurp │ │ └── ktsearch │ │ ├── KtorRestClient.ios.kt │ │ ├── RoundRobinNodeSelector.ios.kt │ │ └── SniffingNodeSelector.ios.kt │ ├── jsMain │ └── kotlin │ │ └── com │ │ └── jillesvangurp │ │ └── ktsearch │ │ ├── defaultHttpClient.kt │ │ ├── nodeselectors.kt │ │ └── threadId.kt │ ├── jsTest │ └── kotlin │ │ └── com │ │ └── jillesvangurp │ │ └── ktsearch │ │ └── coTest.kt │ ├── jvmMain │ └── kotlin │ │ └── com │ │ └── jillesvangurp │ │ └── ktsearch │ │ ├── defaultHttpClient.kt │ │ ├── nodeselectors.kt │ │ └── threadId.kt │ ├── jvmTest │ └── kotlin │ │ └── com │ │ └── jillesvangurp │ │ └── ktsearch │ │ ├── AppLogTest.kt │ │ └── coTest.kt │ ├── linuxMain │ └── kotlin │ │ └── com │ │ └── jillesvangurp │ │ └── ktsearch │ │ ├── KtorRestClient.linux.kt │ │ ├── RoundRobinNodeSelector.linux.kt │ │ └── SniffingNodeSelector.linux.kt │ ├── macosMain │ └── kotlin │ │ └── com │ │ └── jillesvangurp │ │ └── ktsearch │ │ ├── KtorRestClient.macos.kt │ │ ├── RoundRobinNodeSelector.macos.kt │ │ └── SniffingNodeSelector.macos.kt │ ├── mingwMain │ └── kotlin │ │ └── com │ │ └── jillesvangurp │ │ └── ktsearch │ │ ├── KtorRestClient.mingw.kt │ │ ├── RoundRobinNodeSelector.mingw.kt │ │ └── SniffingNodeSelector.mingw.kt │ ├── nativeTest │ └── kotlin │ │ └── com │ │ └── jillesvangurp │ │ └── ktsearch │ │ └── SearchTestBase.native.kt │ ├── wasmJsMain │ └── kotlin │ │ └── com │ │ └── jillesvangurp │ │ └── ktsearch │ │ ├── KtorRestClient.wasmJs.kt │ │ ├── RoundRobinNodeSelector.wasmJs.kt │ │ └── SniffingNodeSelector.wasmJs.kt │ └── wasmJsTest │ └── kotlin │ └── com │ └── jillesvangurp │ └── ktsearch │ └── coTest.kt ├── search-dsls ├── build.gradle.kts └── src │ └── commonMain │ └── kotlin │ └── com │ └── jillesvangurp │ └── searchdsls │ ├── annotations.kt │ ├── mappingdsl │ └── IndexSettings.kt │ └── querydsl │ ├── QueryClauses.kt │ ├── aggregation-queries.kt │ ├── compound-queries.kt │ ├── filter-source.kt │ ├── full-text-queries.kt │ ├── geo-queries.kt │ ├── highlight-dsl.kt │ ├── join-queries.kt │ ├── reindex-dsl.kt │ ├── search-dsl.kt │ ├── specialized-queries.kt │ └── term-level-queries.kt ├── settings.gradle.kts └── versions.properties /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: "[BUG]" 5 | labels: bug 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | 12 | A clear and concise description of what the bug is. 13 | 14 | **To Reproduce** 15 | 16 | Steps to reproduce the behavior: 17 | 18 | **Expected behavior** 19 | 20 | A clear and concise description of what you expected to happen. 21 | 22 | **Your context** 23 | 24 | - what versions of software are you using 25 | - what operating system are you using 26 | - whatever else you think is relevant 27 | 28 | **Will you be able to help with a pull request?** 29 | 30 | Optional of course, but do let me know if you plan to do work. 31 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/documentation_issue.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Documentation issue 3 | about: Report an issue with the documentation. 4 | title: "[DOCS]" 5 | labels: documentation 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the issue** 11 | 12 | Which part of the documentation is wrong, unclear, or missing. 13 | 14 | **Will you be able to help with a pull request?** 15 | 16 | Optional of course, but do let me know if you plan to do work. 17 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/enhancement.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Enhancement/feature request 3 | about: Propose a chage 4 | title: "[FEAT]" 5 | labels: feature 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the enhancement** 11 | 12 | What would you like changed? 13 | 14 | **Why is this needed?** 15 | 16 | Describe why this is useful, important, nice, etc. 17 | 18 | **How do you think it should be done?** 19 | 20 | If you have some clear ideas, outline them here. 21 | 22 | **Will you be able to help with a pull request?** 23 | 24 | Optional, but do let me know if you plan to do work. 25 | -------------------------------------------------------------------------------- /.github/workflows/deploy-docs-and-test.yml: -------------------------------------------------------------------------------- 1 | name: matrix-test-and-deploy-docs 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | jobs: 8 | test: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - name: Checkout the code 12 | uses: actions/checkout@master 13 | - name: Install cURL Headers 14 | run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev -y 15 | - name: Setup Java 16 | uses: actions/setup-java@v4 17 | with: 18 | distribution: temurin 19 | java-version: '17' 20 | cache: gradle 21 | - name: Compose up 22 | run: ./gradlew :search-client:composeUp 23 | - name: Gradle Build 24 | run: ./gradlew check :search-client:dokkaHtml -x jsTest -x jsBrowserTest -x jsNodeTest -x wasmJsBrowserTest -x wasmJsNodeTest -x wasmJsD8Test 25 | - name: Compose down 26 | run: ./gradlew :search-client:composeDown 27 | - name: Deploy Manual 28 | uses: peaceiris/actions-gh-pages@v3 29 | with: 30 | github_token: ${{ secrets.GITHUB_TOKEN }} 31 | publish_dir: ./docs/build/manual 32 | enable_jekyll: true 33 | destination_dir: manual 34 | - name: Deploy Dokka 35 | uses: peaceiris/actions-gh-pages@v3 36 | with: 37 | github_token: ${{ secrets.GITHUB_TOKEN }} 38 | publish_dir: ./search-client/build/dokka/html 39 | enable_jekyll: true 40 | destination_dir: api 41 | matrix-test: 42 | runs-on: ubuntu-latest 43 | strategy: 44 | matrix: 45 | version: ["es-7","es-8","es-9","os-1", "os-2","os-3"] 46 | steps: 47 | - name: Checkout the code 48 | uses: actions/checkout@master 49 | - name: Install cURL Headers 50 | run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev -y 51 | - name: Setup Java 52 | uses: actions/setup-java@v4 53 | with: 54 | distribution: temurin 55 | java-version: '17' 56 | cache: gradle 57 | - name: Compose up 58 | run: ./gradlew :search-client:composeUp -PsearchEngine=${{ matrix.version }} 59 | - name: Gradle Build 60 | run: ./gradlew :search-client:jvmTest -PsearchEngine=${{ matrix.version }} 61 | - name: Compose down 62 | run: ./gradlew :search-client:composeDown -PsearchEngine=${{ matrix.version }} 63 | 64 | 65 | -------------------------------------------------------------------------------- /.github/workflows/pullrequest.yml: -------------------------------------------------------------------------------- 1 | name: pull-request-build 2 | 3 | on: 4 | pull_request: 5 | branches: 6 | - master 7 | jobs: 8 | test: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - name: Checkout the code 12 | uses: actions/checkout@master 13 | - name: Install cURL Headers 14 | run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev -y 15 | - name: Setup Java 16 | uses: actions/setup-java@v4 17 | with: 18 | distribution: temurin 19 | java-version: '17' 20 | cache: gradle 21 | - name: Compose up 22 | run: ./gradlew :search-client:composeUp 23 | - name: Gradle Build 24 | # skipping js/wasm targets because of browser related flakiness and random client failures 25 | run: ./gradlew check -x jsTest -x jsBrowserTest -x jsNodeTest -x wasmJsBrowserTest -x wasmJsNodeTest -x wasmJsD8Test 26 | - name: Compose down 27 | run: ./gradlew :search-client:composeDown 28 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .gradle 2 | bin 3 | build/ 4 | !gradle/wrapper/gradle-wrapper.jar 5 | .kotlin 6 | 7 | ### STS ### 8 | .apt_generated 9 | .classpath 10 | .factorypath 11 | .project 12 | .settings 13 | .springBeans 14 | .sts4-cache 15 | 16 | ### IntelliJ IDEA ### 17 | .idea 18 | *.iws 19 | *.iml 20 | *.ipr 21 | out/ 22 | 23 | # Cruft 24 | *.hprof 25 | .DS_Store 26 | .vscode 27 | epub 28 | book.epub 29 | 30 | # deploy secrets and local configuration 31 | secring.gpg 32 | local.properties 33 | localRepo 34 | .ipynb_checkpoints 35 | .direnv 36 | flake.nix 37 | flake.lock 38 | .envrc 39 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | Pull requests are very welcome. Also, filing issues, doing documentation, or providing general feedback, etc. is much appreciated. 2 | 3 | I try to be responsive processing PRs and critical issues. Otherwise, I have to balance working on this with other stuff so I may not respond right away. If this blocks you, reach out on twitter or via email jilles AT jillesvangurp.com so we can resolve things. 4 | 5 | Where possible, please stick with conventions visible in the code for naming things, formatting things, etc. Bear in mind that this is a kotlin multiplatform project and that code needs to work (ideally) on both Elasticsearch and Opensearch. Features specific to either fork need to be clearly marked as such using the annotations. 6 | 7 | Also, please consider updating the manual so it covers your new feature. 8 | 9 | As this is a multiplatform library, please be careful adding dependencies; especially platform specific ones. 10 | 11 | Before starting on any big pull requests, please file an issue and/or ping me on @jillesvangurp on twitter or several other platforms where I go by the same handle. This is so we can avoid conflicts/disappointment and coordinate changes. Especially for API changes, big refactorings, etc. 12 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2018-present, Jilles van Gurp 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining 4 | a copy of this software and associated documentation files (the 5 | "Software"), to deal in the Software without restriction, including 6 | without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to 8 | permit persons to whom the Software is furnished to do so, subject to 9 | the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be included 12 | in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 17 | IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 18 | CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 19 | TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 20 | SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | I update dependencies regularly (using the refresh versions plugin). This should ensure, this project is mostly reasonably up to date most of the time. I expect most security issues would typically relate to dependencies. 4 | 5 | ## Supported Versions 6 | 7 | The latest stable release is generally the only supported release. 8 | 9 | Previous versions of this library under the name es-kotlin-wrapper are no longer supported. 10 | 11 | ## Reporting a Vulnerability 12 | 13 | If you find any issues or suspect any issues, file an issue via the issue tracker. 14 | 15 | For serious/sensitive issues, contact me via email jilles AT jillesvangurp.com. 16 | -------------------------------------------------------------------------------- /build.gradle.kts: -------------------------------------------------------------------------------- 1 | import org.jetbrains.kotlin.gradle.dsl.jvm.JvmTargetValidationMode.WARNING 2 | import org.jetbrains.kotlin.gradle.tasks.KotlinJvmCompile 3 | 4 | plugins { 5 | kotlin("multiplatform") apply false 6 | id("org.jetbrains.dokka") apply false 7 | } 8 | 9 | println("project: $path") 10 | println("version: $version") 11 | println("group: $group") 12 | 13 | allprojects { 14 | repositories { 15 | mavenCentral() 16 | maven("https://maven.tryformation.com/releases") { 17 | content { 18 | includeGroup("com.jillesvangurp") 19 | } 20 | } 21 | } 22 | } 23 | 24 | subprojects { 25 | 26 | tasks.register("versionCheck") { 27 | doLast { 28 | if (rootProject.version == "unspecified") { 29 | error("call with -Pversion=x.y.z to set a version and make sure it lines up with the current tag") 30 | } 31 | } 32 | } 33 | 34 | tasks.withType { 35 | dependsOn("versionCheck") 36 | } 37 | 38 | tasks.withType { 39 | useJUnitPlatform() 40 | testLogging.exceptionFormat = org.gradle.api.tasks.testing.logging.TestExceptionFormat.FULL 41 | testLogging.events = setOf( 42 | org.gradle.api.tasks.testing.logging.TestLogEvent.FAILED, 43 | org.gradle.api.tasks.testing.logging.TestLogEvent.PASSED, 44 | org.gradle.api.tasks.testing.logging.TestLogEvent.SKIPPED, 45 | org.gradle.api.tasks.testing.logging.TestLogEvent.STANDARD_ERROR, 46 | org.gradle.api.tasks.testing.logging.TestLogEvent.STANDARD_OUT 47 | ) 48 | addTestListener(object : TestListener { 49 | val failures = mutableListOf() 50 | override fun beforeSuite(desc: TestDescriptor) { 51 | } 52 | 53 | override fun afterSuite(desc: TestDescriptor, result: TestResult) { 54 | } 55 | 56 | override fun beforeTest(desc: TestDescriptor) { 57 | } 58 | 59 | override fun afterTest(desc: TestDescriptor, result: TestResult) { 60 | if (result.resultType == TestResult.ResultType.FAILURE) { 61 | val report = 62 | """ 63 | TESTFAILURE ${desc.className} - ${desc.name} 64 | ${ 65 | result.exception?.let { e -> 66 | """ 67 | ${e::class.simpleName} ${e.message} 68 | """.trimIndent() 69 | } 70 | } 71 | ----------------- 72 | """.trimIndent() 73 | failures.add(report) 74 | } 75 | } 76 | }) 77 | } 78 | 79 | apply(plugin = "maven-publish") 80 | apply(plugin = "org.jetbrains.dokka") 81 | 82 | afterEvaluate { 83 | tasks.register("dokkaJar") { 84 | from(tasks["dokkaHtml"]) 85 | dependsOn(tasks["dokkaHtml"]) 86 | archiveClassifier.set("javadoc") 87 | } 88 | 89 | configure { 90 | publications { 91 | withType { 92 | pom { 93 | name.set("KtSearch") 94 | url.set("https://github.com/jillesvangurp/kt-search") 95 | 96 | licenses { 97 | license { 98 | name.set("MIT License") 99 | url.set("https://github.com/jillesvangurp/kt-search/blob/master/LICENSE") 100 | } 101 | } 102 | 103 | developers { 104 | developer { 105 | id.set("jillesvangurp") 106 | name.set("Jilles van Gurp") 107 | email.set("jilles@no-reply.github.com") 108 | } 109 | } 110 | 111 | scm { 112 | connection.set("scm:git:git://github.com/jillesvangurp/kt-search.git") 113 | developerConnection.set("scm:git:ssh://github.com:jillesvangurp/kt-search.git") 114 | url.set("https://github.com/jillesvangurp/kt-search") 115 | } 116 | } 117 | } 118 | } 119 | 120 | repositories { 121 | maven { 122 | // GOOGLE_APPLICATION_CREDENTIALS env var must be set for this to work 123 | // public repository is at https://maven.tryformation.com/releases 124 | url = uri("gcs://mvn-public-tryformation/releases") 125 | name = "FormationPublic" 126 | } 127 | } 128 | } 129 | } 130 | } 131 | 132 | 133 | -------------------------------------------------------------------------------- /docker-compose-es-7.yml: -------------------------------------------------------------------------------- 1 | services: 2 | es7: 3 | image: docker.elastic.co/elasticsearch/elasticsearch:7.17.22 4 | environment: 5 | - cluster.name=docker-test-cluster 6 | - bootstrap.memory_lock=true 7 | - discovery.type=single-node 8 | - "ES_JAVA_OPTS=-Xms1024m -Xmx1024m" 9 | - cluster.routing.allocation.disk.threshold_enabled=true 10 | # make sure it works on nearly full disk 11 | - cluster.routing.allocation.disk.watermark.low=3gb 12 | - cluster.routing.allocation.disk.watermark.high=2gb 13 | - cluster.routing.allocation.disk.watermark.flood_stage=1gb 14 | - cluster.routing.allocation.disk.threshold_enabled=false 15 | - xpack.security.enabled=false 16 | ulimits: 17 | memlock: 18 | soft: -1 19 | hard: -1 20 | ports: 21 | # give our test instance a port number that is for sure not going to write to some poor cluster listening on 9200 22 | - "9999:9200" 23 | -------------------------------------------------------------------------------- /docker-compose-es-8.yml: -------------------------------------------------------------------------------- 1 | services: 2 | es8: 3 | image: docker.elastic.co/elasticsearch/elasticsearch:8.18.1 4 | environment: 5 | - cluster.name=docker-test-cluster 6 | - bootstrap.memory_lock=true 7 | - discovery.type=single-node 8 | - "ES_JAVA_OPTS=-Xms2048m -Xmx2048m" 9 | - cluster.routing.allocation.disk.threshold_enabled=true 10 | # make sure it works on nearly full disk 11 | - cluster.routing.allocation.disk.watermark.low=3gb 12 | - cluster.routing.allocation.disk.watermark.high=2gb 13 | - cluster.routing.allocation.disk.watermark.flood_stage=1gb 14 | - cluster.routing.allocation.disk.threshold_enabled=false 15 | - xpack.security.enabled=false 16 | - network.host=127.0.0.1 17 | - http.host=0.0.0.0 18 | 19 | ulimits: 20 | memlock: 21 | soft: -1 22 | hard: -1 23 | ports: 24 | # give our test instance a port number that is for sure not going to write to some poor cluster listening on 9200 25 | - "9999:9200" 26 | -------------------------------------------------------------------------------- /docker-compose-es-9.yml: -------------------------------------------------------------------------------- 1 | services: 2 | es9: 3 | image: docker.elastic.co/elasticsearch/elasticsearch:9.0.1 4 | environment: 5 | - cluster.name=docker-test-cluster 6 | - bootstrap.memory_lock=true 7 | - discovery.type=single-node 8 | - "ES_JAVA_OPTS=-Xms2048m -Xmx2048m" 9 | - cluster.routing.allocation.disk.threshold_enabled=true 10 | # make sure it works on nearly full disk 11 | - cluster.routing.allocation.disk.watermark.low=3gb 12 | - cluster.routing.allocation.disk.watermark.high=2gb 13 | - cluster.routing.allocation.disk.watermark.flood_stage=1gb 14 | - cluster.routing.allocation.disk.threshold_enabled=false 15 | - xpack.security.enabled=false 16 | - network.host=127.0.0.1 17 | - http.host=0.0.0.0 18 | 19 | ulimits: 20 | memlock: 21 | soft: -1 22 | hard: -1 23 | ports: 24 | # give our test instance a port number that is for sure not going to write to some poor cluster listening on 9200 25 | - "9999:9200" 26 | -------------------------------------------------------------------------------- /docker-compose-os-1.yml: -------------------------------------------------------------------------------- 1 | services: 2 | os1: 3 | image: opensearchproject/opensearch:1 4 | environment: 5 | - cluster.name=docker-test-cluster 6 | - bootstrap.memory_lock=true 7 | - discovery.type=single-node 8 | - "ES_JAVA_OPTS=-Xms1024m -Xmx1024m" 9 | - cluster.routing.allocation.disk.threshold_enabled=true 10 | # make sure it works on nearly full disk 11 | - cluster.routing.allocation.disk.watermark.low=3gb 12 | - cluster.routing.allocation.disk.watermark.high=2gb 13 | - cluster.routing.allocation.disk.watermark.flood_stage=1gb 14 | - cluster.routing.allocation.disk.threshold_enabled=false 15 | - plugins.security.disabled=true 16 | ulimits: 17 | memlock: 18 | soft: -1 19 | hard: -1 20 | ports: 21 | # give our test instance a port number that is for sure not going to write to some poor cluster listening on 9200 22 | - "9999:9200" 23 | -------------------------------------------------------------------------------- /docker-compose-os-2.yml: -------------------------------------------------------------------------------- 1 | services: 2 | os2: 3 | image: opensearchproject/opensearch:2 4 | environment: 5 | - cluster.name=docker-test-cluster 6 | - bootstrap.memory_lock=true 7 | - discovery.type=single-node 8 | - "ES_JAVA_OPTS=-Xms1024m -Xmx1024m" 9 | - cluster.routing.allocation.disk.threshold_enabled=true 10 | # make sure it works on nearly full disk 11 | - cluster.routing.allocation.disk.watermark.low=3gb 12 | - cluster.routing.allocation.disk.watermark.high=2gb 13 | - cluster.routing.allocation.disk.watermark.flood_stage=1gb 14 | - cluster.routing.allocation.disk.threshold_enabled=false 15 | - DISABLE_SECURITY_PLUGIN=true 16 | ulimits: 17 | memlock: 18 | soft: -1 19 | hard: -1 20 | ports: 21 | # give our test instance a port number that is for sure not going to write to some poor cluster listening on 9200 22 | - "9999:9200" 23 | # os2-dashboards: 24 | # image: opensearchproject/opensearch-dashboards:2.18.0 25 | # environment: 26 | # - OPENSEARCH_HOSTS=http://os2:9200 27 | # - DISABLE_SECURITY_DASHBOARDS_PLUGIN=true 28 | # ports: 29 | # - "5601:5601" 30 | # depends_on: 31 | # - os2 -------------------------------------------------------------------------------- /docker-compose-os-3.yml: -------------------------------------------------------------------------------- 1 | services: 2 | os2: 3 | image: opensearchproject/opensearch:3 4 | environment: 5 | - cluster.name=docker-test-cluster 6 | - bootstrap.memory_lock=true 7 | - discovery.type=single-node 8 | - "ES_JAVA_OPTS=-Xms1024m -Xmx1024m" 9 | - cluster.routing.allocation.disk.threshold_enabled=true 10 | # make sure it works on nearly full disk 11 | - cluster.routing.allocation.disk.watermark.low=3gb 12 | - cluster.routing.allocation.disk.watermark.high=2gb 13 | - cluster.routing.allocation.disk.watermark.flood_stage=1gb 14 | - cluster.routing.allocation.disk.threshold_enabled=false 15 | - DISABLE_SECURITY_PLUGIN=true 16 | ulimits: 17 | memlock: 18 | soft: -1 19 | hard: -1 20 | ports: 21 | # give our test instance a port number that is for sure not going to write to some poor cluster listening on 9200 22 | - "9999:9200" 23 | # os2-dashboards: 24 | # image: opensearchproject/opensearch-dashboards:2.18.0 25 | # environment: 26 | # - OPENSEARCH_HOSTS=http://os2:9200 27 | # - DISABLE_SECURITY_DASHBOARDS_PLUGIN=true 28 | # ports: 29 | # - "5601:5601" 30 | # depends_on: 31 | # - os2 -------------------------------------------------------------------------------- /docs/build.gradle.kts: -------------------------------------------------------------------------------- 1 | import java.util.* 2 | 3 | val localProperties = Properties().apply { 4 | rootProject.file("local.properties").takeIf { it.exists() }?.reader()?.use { 5 | load(it) 6 | } 7 | } 8 | 9 | fun getProperty(propertyName: String, defaultValue: Any? = null) = 10 | (localProperties[propertyName] ?: project.properties[propertyName]) ?: defaultValue 11 | 12 | fun getBooleanProperty(propertyName: String) = getProperty(propertyName)?.toString().toBoolean() 13 | 14 | 15 | plugins { 16 | kotlin("jvm") 17 | kotlin("plugin.serialization") 18 | } 19 | 20 | repositories { 21 | mavenCentral() 22 | maven(url = "https://jitpack.io") { 23 | content { 24 | includeGroup("com.github.jillesvangurp") 25 | } 26 | } 27 | } 28 | 29 | kotlin { 30 | compilerOptions { 31 | freeCompilerArgs= listOf("-Xcontext-receivers") 32 | } 33 | } 34 | 35 | val searchEngine: String = getProperty("searchEngine", "es-7").toString() 36 | 37 | dependencies { 38 | testImplementation(project(":search-dsls")) 39 | testImplementation(project(":search-client")) 40 | testImplementation("com.jillesvangurp:json-dsl:_") 41 | testImplementation("com.jillesvangurp:kotlinx-serialization-extensions:_") 42 | 43 | testImplementation(Kotlin.stdlib.jdk8) 44 | testImplementation(KotlinX.coroutines.jdk8) 45 | testImplementation(KotlinX.datetime) 46 | testImplementation(Ktor.client.core) 47 | testImplementation(KotlinX.coroutines.core) 48 | 49 | testImplementation(KotlinX.serialization.json) 50 | testImplementation(Ktor.client.core) 51 | testImplementation(Ktor.client.logging) 52 | testImplementation(Ktor.client.serialization) 53 | testImplementation("io.ktor:ktor-client-logging:_") 54 | testImplementation("io.ktor:ktor-serialization-kotlinx:_") 55 | testImplementation("io.ktor:ktor-serialization-kotlinx-json:_") 56 | testImplementation("io.ktor:ktor-client-content-negotiation:_") 57 | 58 | 59 | // bring your own logging, but we need some in tests 60 | testImplementation("org.slf4j:slf4j-api:_") 61 | testImplementation("org.slf4j:jcl-over-slf4j:_") 62 | testImplementation("org.slf4j:log4j-over-slf4j:_") 63 | testImplementation("org.slf4j:jul-to-slf4j:_") 64 | testImplementation("org.apache.logging.log4j:log4j-to-slf4j:_") // es seems to insist on log4j2 65 | testImplementation("ch.qos.logback:logback-classic:_") 66 | 67 | testImplementation(kotlin("test-junit5")) 68 | testImplementation(Testing.junit.jupiter.api) 69 | testImplementation(Testing.junit.jupiter.engine) 70 | 71 | testImplementation("com.github.jillesvangurp:kotlin4example:_") 72 | testImplementation("com.github.doyaaaaaken:kotlin-csv:_") 73 | } 74 | -------------------------------------------------------------------------------- /docs/src/test/kotlin/documentation/DocumentationTest.kt: -------------------------------------------------------------------------------- 1 | package documentation 2 | 3 | import com.jillesvangurp.kotlin4example.SourceRepository 4 | import documentation.manual.manualIndexMd 5 | import documentation.manual.manualPages 6 | import documentation.projectreadme.projectReadme 7 | import org.junit.jupiter.api.Test 8 | import java.io.File 9 | 10 | const val githubLink = "https://github.com/jillesvangurp/kt-search" 11 | //const val jitpackLink = "[![](https://jitpack.io/v/jillesvangurp/kt-search.svg)](https://jitpack.io/#jillesvangurp/kt-search)" 12 | 13 | data class Page( 14 | val title: String, 15 | val fileName: String, 16 | val outputDir: String 17 | ) 18 | 19 | val Page.mdLink get() = "[$title]($fileName)" 20 | 21 | fun Page.write(content: String) { 22 | File(outputDir, fileName).writeText( 23 | """ 24 | # $title 25 | 26 | """.trimIndent().trimMargin() + "\n\n" + content 27 | ) 28 | } 29 | 30 | val sourceGitRepository = SourceRepository( 31 | repoUrl = githubLink, 32 | sourcePaths = setOf("src/main/kotlin", "src/test/kotlin") 33 | ) 34 | 35 | internal const val manualOutputDir = "build/manual" 36 | 37 | val manualRootPage = Page("KT Search Manual", "README.md", manualOutputDir) 38 | val readmePages = listOf( 39 | Page("KT Search Client", "README.md", "..") to projectReadme, 40 | manualRootPage to manualIndexMd, 41 | ) 42 | 43 | fun loadMd(fileName: String) = sourceGitRepository.md { 44 | includeMdFile(fileName) 45 | } 46 | 47 | class DocumentationTest { 48 | 49 | @Test 50 | fun documentation() { 51 | File(manualOutputDir).mkdirs() 52 | readmePages.forEach { (page, md) -> 53 | page.write(md.value) 54 | } 55 | val pagesWithNav = manualPages.indices.map { index -> 56 | val (previousPage,_)=if(index>0) { 57 | manualPages[index-1] 58 | } else { 59 | null to null 60 | } 61 | val (nextPage,_)=if(index< manualPages.size-1) { 62 | manualPages[index+1] 63 | } else { 64 | null to null 65 | } 66 | val navigation = """ 67 | | ${manualRootPage.mdLink} | ${previousPage?.let{ "Previous: ${it.mdLink}" } ?: "-"} | ${nextPage?.let{ "Next: ${it.mdLink}" } ?: "-"} | 68 | | [Github]($githubLink) | © Jilles van Gurp | | 69 | """.trimIndent() 70 | 71 | val (page,md) = manualPages[index] 72 | page to (""" 73 | $navigation 74 | 75 | --- 76 | 77 | ${md.value} 78 | 79 | --- 80 | 81 | $navigation 82 | """.trimIndent()) 83 | } 84 | pagesWithNav.forEach { (page, md) -> 85 | page.write(md) 86 | } 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /docs/src/test/kotlin/documentation/extensions.kt: -------------------------------------------------------------------------------- 1 | package documentation 2 | 3 | import com.jillesvangurp.kotlin4example.ExampleOutput 4 | import com.jillesvangurp.kotlin4example.Kotlin4Example 5 | 6 | context(Kotlin4Example) 7 | fun ExampleOutput<*>.printStdOut() { 8 | +""" 9 | This prints: 10 | """.trimIndent() 11 | 12 | mdCodeBlock(stdOut, type = "text", wrap = true) 13 | } 14 | -------------------------------------------------------------------------------- /docs/src/test/kotlin/documentation/manual/bulk/indexmanagement/datastreams.kt: -------------------------------------------------------------------------------- 1 | package documentation.manual.bulk.indexmanagement 2 | 3 | import com.jillesvangurp.jsondsl.withJsonDsl 4 | import com.jillesvangurp.ktsearch.* 5 | import documentation.manual.ManualPages 6 | import documentation.mdLink 7 | import documentation.sourceGitRepository 8 | import kotlinx.coroutines.runBlocking 9 | import kotlin.time.Duration.Companion.hours 10 | 11 | val dataStreamsMd = sourceGitRepository.md { 12 | val client = SearchClient(KtorRestClient(Node("localhost", 9999))) 13 | cleanup(client) 14 | +""" 15 | Particularly for large volume time series data, you can use data streams to make the management of 16 | indices a bit easier. Data streams allow you to automate a lot of the things you would otherwise do manually 17 | with manually ${ManualPages.IndexManagement.page.mdLink}. 18 | 19 | A data stream is simply a set of indices that are managed and controlled by Elasticsearch. 20 | To create a data stream, you need to define policies and templates that tell Elasticsearch how to do this. 21 | 22 | - Index Life-cycle managent is used to control how the index is managed and ultimately deleted over time. 23 | - Index templates and index component templates are used to control the mappings and settings for the indices. 24 | - Finally you can use the data stream API to control and introspect the data stream. 25 | 26 | """.trimIndent() 27 | 28 | 29 | section("Index Life Cycle Management") { 30 | +""" 31 | It is advisable to set up index life cycle management (ILM) with data streams. 32 | Using ILM, you can automatically roll over indices, shrink them and delete them. 33 | 34 | Index life cycle management is an Elastic only feature. However, Opensearch has a similar 35 | feature, called Index State Management. At this point we do not support this. But pull 36 | requests are welcome for this of course. 37 | 38 | For a full overview of ILM see the Elastic documentation for this. 39 | """.trimMargin() 40 | 41 | example(runExample = false) { 42 | client.setIlmPolicy("my-ilm") { 43 | hot { 44 | // this is where your data goes 45 | actions { 46 | rollOver(maxPrimaryShardSizeGb = 2) 47 | } 48 | } 49 | warm { 50 | // indices get rolled over to this 51 | // and are still queryable 52 | // of course we use Duration here 53 | minAge(24.hours) 54 | actions { 55 | shrink(numberOfShards = 1) 56 | forceMerge(numberOfSegments = 1) 57 | } 58 | } 59 | } 60 | } 61 | } 62 | 63 | 64 | section("Index Templates") { 65 | +""" 66 | Once you have defined an ILM policy, you can refer it in an index template. An index template 67 | consists of index component templates. So we have to define those first. 68 | """.trimIndent() 69 | example(runExample = true) { 70 | 71 | // using component templates is a good idea 72 | // note, Elastic bundles quite a few default ones that you can use 73 | client.updateComponentTemplate("my-logs-settings") { 74 | settings { 75 | replicas = 4 76 | indexLifeCycleName = "my-ilm" 77 | } 78 | } 79 | client.updateComponentTemplate("my-logs-mappings") { 80 | mappings { 81 | text("name") 82 | keyword("category") 83 | // note data streams require @timestamp 84 | date("@timestamp") 85 | } 86 | } 87 | // now create the template 88 | client.createIndexTemplate("my-logs-template") { 89 | indexPatterns = listOf("my-logs*") 90 | // make sure to specify an empty object for data_stream 91 | dataStream = withJsonDsl { 92 | // the elastic docs are a bit vague on what goes here 93 | } 94 | composedOf = listOf("my-logs-settings", "my-logs-mappings") 95 | 96 | // in case multiple templates can be applied, the ones 97 | // with the highest priority wins. The managed ones 98 | // that come with Elastic have a priority of 100 99 | priority = 200 100 | } 101 | 102 | client.createDataStream("my-logs") 103 | } 104 | cleanup(client) 105 | } 106 | } 107 | 108 | private fun cleanup(client: SearchClient) { 109 | runBlocking { 110 | runCatching { client.deleteDataStream("my-logs") } 111 | runCatching { client.deleteIndexTemplate("my-logs-template") } 112 | runCatching { client.deleteComponentTemplate("my-logs-settings") } 113 | runCatching { client.deleteComponentTemplate("my-logs-mappings") } 114 | runCatching { client.deleteIlmPolicy("my-ilm") } 115 | } 116 | } -------------------------------------------------------------------------------- /docs/src/test/kotlin/documentation/manual/crud/crud.kt: -------------------------------------------------------------------------------- 1 | @file:Suppress("UNUSED_VARIABLE") 2 | 3 | package documentation.manual.crud 4 | 5 | import com.jillesvangurp.ktsearch.* 6 | import com.jillesvangurp.searchdsls.querydsl.Script 7 | import documentation.manual.ManualPages 8 | import documentation.mdLink 9 | import documentation.printStdOut 10 | import documentation.sourceGitRepository 11 | import kotlinx.serialization.Serializable 12 | 13 | val crudMd = sourceGitRepository.md { 14 | val client = SearchClient(KtorRestClient(Node("localhost", 9999))) 15 | @Serializable 16 | data class TestDoc(val id: String, val name: String, val tags: List = listOf()) 17 | 18 | +""" 19 | Mostly, you will use bulk indexing to manipulate documents in Elasticsearch. However, 20 | sometimes it is useful to be able to manipulate individual documents with the 21 | Create, Read, Update, and Delete (CRUD) APIs. 22 | """.trimIndent() 23 | section("CRUD") { 24 | example { 25 | // create 26 | val resp = client.indexDocument( 27 | target = "myindex", 28 | document = TestDoc("1", "A Document"), 29 | // optional id, you can let elasticsearch assign one 30 | id = "1", 31 | // this is the default 32 | // fails if the id already exists 33 | opType = OperationType.Create 34 | ) 35 | 36 | // read 37 | val doc = client.getDocument("myindex", resp.id) 38 | // to get the TestDoc, use this extension function 39 | // this works on the _source field in the response 40 | .document() 41 | 42 | // update 43 | client.indexDocument( 44 | target = "myindex", 45 | document = TestDoc("1", "A Document"), 46 | id = "1", 47 | // will overwrite if the id already existed 48 | opType = OperationType.Index 49 | ) 50 | 51 | // delete 52 | client.deleteDocument("myindex", resp.id) 53 | }.printStdOut() 54 | } 55 | 56 | section("Updates") { 57 | +""" 58 | Elasticsearch also has a dedicated update API that you can use with either a partial document or a script. 59 | """.trimIndent() 60 | 61 | example { 62 | client.indexDocument( 63 | target = "myindex", 64 | document = TestDoc("42", "x"), 65 | id = "42" 66 | ) 67 | var resp = client.updateDocument( 68 | target = "myindex", 69 | id = "42", 70 | docJson = """{"name":"changed"}""", 71 | source = "true" 72 | ) 73 | println(resp.get?.source) 74 | 75 | resp = client.updateDocument( 76 | target = "myindex", 77 | id = "42", 78 | script = Script.create { 79 | source = """ctx._source.name = params.p1 """ 80 | params = mapOf( 81 | "p1" to "again" 82 | ) 83 | }, 84 | source = "true" 85 | ) 86 | println(resp.get?.source) 87 | 88 | }.printStdOut() 89 | } 90 | 91 | section("Bulk") { 92 | +""" 93 | The index API has a lot more parameters that are supported here as well 94 | via nullable parameters. You can also use a variant of the index API 95 | that accepts a json String instead of the TestDoc. 96 | 97 | Note, for inserting large amounts of documents you should of course use the bulk API. 98 | You can learn more about that here: ${ 99 | ManualPages.BulkIndexing.page.mdLink 100 | }. 101 | """.trimIndent() 102 | } 103 | section("Multi Get") { 104 | +""" 105 | To retrieve multiple documents, you can use the `mget` API> 106 | """.trimIndent() 107 | 108 | example { 109 | client.indexDocument( 110 | target = "myindex", 111 | document = TestDoc("1", "One"), 112 | id = "1" 113 | ) 114 | client.indexDocument( 115 | target = "myindex", 116 | document = TestDoc("2", "Two"), 117 | id = "2" 118 | ) 119 | 120 | // the simple way is to provide a list of ids 121 | client.mGet(index = "myindex") { 122 | ids= listOf("1","2") 123 | }.let { resp-> 124 | println("Found ${resp.docs.size} documents") 125 | } 126 | // or do it like this 127 | val resp = client.mGet { 128 | doc { 129 | index="myindex" 130 | id = "1" 131 | // default is true 132 | source=true 133 | } 134 | doc { 135 | index="myindex" 136 | id = "2" 137 | } 138 | } 139 | 140 | // you can of course get the deserialized TestDocs 141 | // from the response with an extension function 142 | resp.documents().forEach { 143 | println("${it.id}: ${it.name}") 144 | } 145 | }.printStdOut() 146 | } 147 | } -------------------------------------------------------------------------------- /docs/src/test/kotlin/documentation/manual/gettingstarted/getting-started.kt: -------------------------------------------------------------------------------- 1 | @file:Suppress("UNUSED_VARIABLE", "LocalVariableName") 2 | 3 | package documentation.manual.gettingstarted 4 | 5 | import com.jillesvangurp.ktsearch.* 6 | import com.jillesvangurp.searchdsls.querydsl.matchAll 7 | import com.jillesvangurp.searchdsls.querydsl.matchPhrase 8 | import com.jillesvangurp.searchdsls.querydsl.term 9 | import documentation.manual.ManualPages 10 | import documentation.mdLink 11 | import documentation.sourceGitRepository 12 | import kotlinx.coroutines.async 13 | import kotlinx.coroutines.coroutineScope 14 | import kotlinx.coroutines.runBlocking 15 | import kotlinx.serialization.ExperimentalSerializationApi 16 | import kotlinx.serialization.json.Json 17 | import kotlin.time.Duration.Companion.seconds 18 | 19 | val whatIsKtSearchMd = sourceGitRepository.md { 20 | includeMdFile("whatisktsearch.md") 21 | } 22 | 23 | @OptIn(ExperimentalSerializationApi::class) 24 | val gettingStartedMd = sourceGitRepository.md { 25 | +""" 26 | To get started, simply add the dependency to your project and create a client. 27 | The process is the same for both jvm and kotlin-js. 28 | """.trimIndent() 29 | includeMdFile("../../projectreadme/gradle-maven.md") 30 | // test server runs on 9999, so we need to override 31 | val client = SearchClient(KtorRestClient(Node("localhost", 9999))) 32 | 33 | section("Create a Client") { 34 | +""" 35 | To use `kt-search` you need a `SearchClient` instance. Similar to what the Elastic and Opensearch Java client do, there is a 36 | simple `RestClient` interface that currently has a default implementation based on `ktor-client`. This client 37 | takes care of sending HTTP calls to your search cluster. 38 | """.trimIndent() 39 | example { 40 | // creates a client with the default RestClient 41 | val client = SearchClient() 42 | } 43 | } 44 | 45 | section("Using the client") { 46 | +""" 47 | After creating the client, you can use it. Since kt-search uses non blocking IO via ktor client, all 48 | calls are suspending and have to be inside a co-routine. 49 | """.trimIndent() 50 | 51 | example { 52 | // use a simple runBlocking 53 | // normally you would get a co-routine via e.g. Spring's flux async framework. 54 | runBlocking { 55 | // call the root API with some version information 56 | client.root().let { resp -> 57 | println("${resp.variantInfo.variant}: ${resp.version.number}") 58 | } 59 | // get the cluster health 60 | client.clusterHealth().let { resp -> 61 | println(resp.clusterName + " is " + resp.status) 62 | } 63 | 64 | } 65 | } 66 | 67 | +""" 68 | The main purpose of kt-search is of course searching. This is how you do a simple search and work with 69 | data classes: 70 | """.trimIndent() 71 | example(runExample = false) { 72 | 73 | // define a model for your indexed json documents 74 | data class MyModelClass(val title: String, ) 75 | 76 | // a simple search 77 | val results = client.search("myindex") { 78 | query = matchPhrase( 79 | field = "title", 80 | query = "lorum ipsum") 81 | } 82 | 83 | // returns a list of MyModelClass 84 | val parsedHits = results.parseHits() 85 | 86 | // if you don't have a model class, you can just use a JsonObject 87 | val jsonObjects = results 88 | .hits 89 | ?.hits 90 | // extract the source from the hits (JsonObject) 91 | ?.map { it.source } 92 | // fall back to empty list 93 | ?: listOf() 94 | } 95 | } 96 | 97 | section("Next steps") { 98 | +""" 99 | - ${ManualPages.ClientConfiguration.page.mdLink}: Learn how to customize the client further. 100 | - ${ManualPages.Search.page.mdLink}: Learn more about how to use the query DSL. 101 | - ${ManualPages.IndexRepository.page.mdLink}: Configure an index repository to make working with a specific index easier. 102 | """.trimIndent() 103 | } 104 | } -------------------------------------------------------------------------------- /docs/src/test/kotlin/documentation/manual/gettingstarted/migrating.md: -------------------------------------------------------------------------------- 1 | Migrating from the old es-kotlin-client is quite easy but it is going to involve a bit of work: 2 | 3 | - Packages have been renamed. This allows you to use the old client side by side with the new client. 4 | - The search dsl has been extracted from the old client but is otherwise largely backwards compatible. So, your queries should mostly work with the new client after fixing the package names. 5 | - As the java `RestHighLevelClient` is no longer used, the kotlin APIs have changed. And that of course includes model classes for API responses that can no longer rely or depend on the old Java classes that came with the java client. 6 | - Unlike the es-kotlin-client which generated kotlin `suspend` functions close to 100 api end points in the `RestHighLevelClient`, Kt Search only includes a handful of APIs for indexing, document crud, and of course searching. We have no intention to support the REST API in its entirety at this point. However, it's really easy to support more of the REST API yourself using the `RestClient` and to define `JsonDsl` based model classes for their request payloads. 7 | - All APIs in `kt-search` are suspend only. Supporting blocking IO is not a priority and this gets rid of a lot of code duplication. 8 | 9 | For the migration of the FORMATION code base (this is the company that I am the CTO of), we managed to convert most 10 | of the code without too much problems. We ran with both the old and the new client for several months before getting rid 11 | of the old client completely. Most of our queries did not need changes. Though of course, we were able to make 12 | them a bit nicer using several of the new features. 13 | 14 | -------------------------------------------------------------------------------- /docs/src/test/kotlin/documentation/manual/intro.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | -------------------------------------------------------------------------------- /docs/src/test/kotlin/documentation/manual/jupyter/jupyter.md: -------------------------------------------------------------------------------- 1 | Using kt-search from jupyter and the kotlin kernel is easy! See the `jupyter-example` directory in the kt-search project. 2 | 3 | ## Install conda 4 | 5 | On a mac, use home brew of course. 6 | 7 | ```bash 8 | brew install miniconda 9 | ``` 10 | 11 | ## Install jupyter with conda 12 | 13 | Once you have conda installed, install jupyter and the kotlin kernel. 14 | 15 | ```bash 16 | conda create -n kjupyter 17 | conda activate kjupyter 18 | conda install jupyter 19 | conda install -c jetbrains kotlin-jupyter-kernel 20 | ``` 21 | 22 | ## Open the notebook 23 | 24 | Now you are ready to open the notebook! 25 | 26 | ```bash 27 | cd jupyter-example 28 | jupyter notebook kt-search-example.ipynb 29 | ``` 30 | 31 | ## Importing kt-search 32 | 33 | Create a cell in your notebook with something like this: 34 | 35 | ```kotlin 36 | @file:Repository("https://jitpack.io") 37 | @file:DependsOn("com.github.jillesvangurp.kt-search:search-client:1.99.14") 38 | 39 | import com.jillesvangurp.ktsearch.* 40 | import kotlinx.coroutines.runBlocking 41 | 42 | val client = SearchClient( 43 | KtorRestClient( 44 | host = "localhost", 45 | port = 9200 46 | ) 47 | ) 48 | 49 | runBlocking { 50 | val engineInfo = client.engineInfo() 51 | println(engineInfo.variantInfo.variant.name + ":" + engineInfo.version.number) 52 | } 53 | ``` 54 | 55 | Note, you need to use `runBlocking` to use suspending calls on the client. 56 | 57 | Otherwise, see the documentation for how to use the kotlin scripting support. 58 | -------------------------------------------------------------------------------- /docs/src/test/kotlin/documentation/manual/manual-index.kt: -------------------------------------------------------------------------------- 1 | package documentation.manual 2 | 3 | import documentation.* 4 | import documentation.manual.bulk.bulkMd 5 | import documentation.manual.bulk.indexmanagement.dataStreamsMd 6 | import documentation.manual.bulk.indexmanagement.indexManagementMd 7 | import documentation.manual.crud.crudMd 8 | import documentation.manual.extending.extendingMd 9 | import documentation.manual.gettingstarted.clientConfiguration 10 | import documentation.manual.gettingstarted.gettingStartedMd 11 | import documentation.manual.gettingstarted.whatIsKtSearchMd 12 | import documentation.manual.indexrepo.indexRepoMd 13 | import documentation.manual.knn.knnMd 14 | import documentation.manual.scripting.scriptingMd 15 | import documentation.manual.search.* 16 | 17 | enum class ManualPages(title: String = "") { 18 | WhatIsKtSearch("What is Kt-Search"), 19 | GettingStarted("Getting Started"), 20 | ClientConfiguration("Client Configuration"), 21 | Search("Search and Queries"), 22 | ReusableSearchQueries("Reusing your Query logic"), 23 | TextQueries("Text Queries"), 24 | TermLevelQueries("Term Level Queries"), 25 | CompoundQueries("Compound Queries"), 26 | GeoQueries("Geo Spatial Queries"), 27 | SpecializedQueries("Specialized Queries"), 28 | JoinQueries("Join Queries"), 29 | Highlighting("Highlighting"), 30 | Aggregations("Aggregations"), 31 | DeepPaging("Deep Paging Using search_after and scroll"), 32 | DeleteByQuery("Deleting by query"), 33 | DocumentManipulation("Document Manipulation"), 34 | IndexRepository("Index Repository"), 35 | BulkIndexing("Efficiently Ingest Content Using Bulk Indexing"), 36 | IndexManagement("Indices, Settings, Mappings, and Aliases"), 37 | DataStreams("Creating Data Streams"), 38 | KnnSearch("KNN Search"), 39 | Migrating("Migrating from the old Es Kotlin Client"), 40 | ExtendingTheDSL("Extending the Json DSLs"), 41 | Scripting("Using Kotlin Scripting"), 42 | Jupyter("Jupyter Notebooks"), 43 | ; 44 | 45 | val page by lazy { 46 | Page(title, "${name}.md", manualOutputDir) 47 | } 48 | val publicLink = "https://jillesvangurp.github.io/kt-search/manual/${name}.html" 49 | } 50 | 51 | data class Section(val title: String, val pages: List>>) 52 | 53 | val sections = listOf( 54 | Section( 55 | "Introduction", listOf( 56 | ManualPages.WhatIsKtSearch to whatIsKtSearchMd, 57 | ManualPages.GettingStarted to gettingStartedMd, 58 | ManualPages.ClientConfiguration to clientConfiguration, 59 | ManualPages.IndexManagement to indexManagementMd, 60 | ) 61 | ), 62 | Section( 63 | "Search", listOf( 64 | ManualPages.Search to searchMd, 65 | ManualPages.TextQueries to textQueriesMd, 66 | ManualPages.TermLevelQueries to termLevelQueriesMd, 67 | ManualPages.CompoundQueries to compoundQueriesMd, 68 | ManualPages.GeoQueries to geoQueriesMd, 69 | ManualPages.SpecializedQueries to specializedQueriesMd, 70 | ManualPages.Aggregations to aggregationsMd, 71 | ManualPages.DeepPaging to deepPagingMd, 72 | ManualPages.JoinQueries to joinQueriesMd, 73 | ManualPages.Highlighting to highlightingMd, 74 | ManualPages.ReusableSearchQueries to reusingQueryLogicMd 75 | ) 76 | ), 77 | Section("Indices and Documents", listOf( 78 | ManualPages.DeleteByQuery to deleteByQueryMd, 79 | ManualPages.DocumentManipulation to crudMd, 80 | ManualPages.IndexRepository to indexRepoMd, 81 | ManualPages.BulkIndexing to bulkMd, 82 | ManualPages.DataStreams to dataStreamsMd, 83 | )), 84 | Section("Advanced Topics", listOf( 85 | ManualPages.KnnSearch to knnMd, 86 | ManualPages.ExtendingTheDSL to extendingMd, 87 | ManualPages.Scripting to scriptingMd, 88 | ManualPages.Jupyter to loadMd("manual/jupyter/jupyter.md"), 89 | ManualPages.Migrating to loadMd("manual/gettingstarted/migrating.md"), 90 | )) 91 | ) 92 | 93 | val manualPages = sections.flatMap { it.pages }.map { (mp,md) -> mp.page to md } 94 | 95 | val manualIndexMd = sourceGitRepository.md { 96 | includeMdFile("../projectreadme/oneliner.md") 97 | 98 | section("Table of contents") { 99 | 100 | sections.forEach { 101 | +""" 102 | ### ${it.title} 103 | 104 | """.trimIndent() 105 | it.pages.forEach {(mp,_) -> 106 | +"${mp.page.mdLink}\n" 107 | } 108 | 109 | } 110 | } 111 | section("About this Manual") { 112 | includeMdFile("outro.md") 113 | } 114 | 115 | includeMdFile("../projectreadme/related.md") 116 | } -------------------------------------------------------------------------------- /docs/src/test/kotlin/documentation/manual/outro.md: -------------------------------------------------------------------------------- 1 | This manual documents how to use the kotlin search client and all of its features. As a manual like this contains a lot of code samples, I ended up writing a mini framework to allow me to generate markdown from Kotlin. 2 | 3 | The project for this is called [Kotlin4Example](https://github.com/jillesvangurp/kotlin4example). Most of the documentation you will find here has correct code samples that get tested and compiled whenever this project is built and whenever something gets refactored that would affect one of the code samples. 4 | 5 | -------------------------------------------------------------------------------- /docs/src/test/kotlin/documentation/manual/scripting/sampleScript.md: -------------------------------------------------------------------------------- 1 | ```kotlin 2 | #!/usr/bin/env kotlin 3 | 4 | @file:Repository("https://maven.tryformation.com/releases") 5 | @file:Repository(" https://repo.maven.apache.org/maven2/") 6 | @file:Repository("https://jitpack.io") 7 | @file:DependsOn("com.github.jillesvangurp:kt-search-kts:1.0.7") 8 | 9 | import com.jillesvangurp.ktsearch.ClusterStatus 10 | import com.jillesvangurp.ktsearch.clusterHealth 11 | import com.jillesvangurp.ktsearch.kts.addClientParams 12 | import com.jillesvangurp.ktsearch.kts.searchClient 13 | import com.jillesvangurp.ktsearch.root 14 | import kotlinx.cli.ArgParser 15 | import kotlinx.coroutines.runBlocking 16 | 17 | // ArgParser is included by kt-search-kts to allow you to configure the search endpoint 18 | val parser = ArgParser("script") 19 | // this adds the params for configuring search end point 20 | val searchClientParams = parser.addClientParams() 21 | parser.parse(args) 22 | 23 | // extension function in kt-search-kts that uses the params 24 | val client = searchClientParams.searchClient 25 | 26 | // now use the client as normally in a runBlocking block (creates a co-routine) 27 | runBlocking { 28 | val clusterStatus=client.clusterHealth() 29 | client.root().let { 30 | println( 31 | """ 32 | Cluster name: ${it.clusterName} 33 | Search Engine distribution: ${it.version.distribution} 34 | Version: ${it.version.number} 35 | """.trimIndent() 36 | ) 37 | } 38 | 39 | when(clusterStatus.status) { 40 | ClusterStatus.Green -> println("Relax, your cluster is green!") 41 | ClusterStatus.Yellow -> println("WARNING: cluster is yellow") 42 | ClusterStatus.Red -> error("OMG: cluster is red!!!!!") 43 | } 44 | } 45 | ``` 46 | -------------------------------------------------------------------------------- /docs/src/test/kotlin/documentation/manual/scripting/scripting.kt: -------------------------------------------------------------------------------- 1 | package documentation.manual.scripting 2 | 3 | import documentation.sourceGitRepository 4 | 5 | val scriptingMd = sourceGitRepository.md { 6 | +""" 7 | One interesting use of kt-search is to script common operations around Elasticsearch. 8 | 9 | For this we have a companion library [kt-search-kts](https://github.com/jillesvangurp/kt-search-kts/) 10 | that makes integrating `kt-search` into your `.main.kts` scripts very easy. 11 | 12 | You may find a few example scripts in the `scripts` directory of that project. To run these scripts, 13 | you need to have kotlin installed on your system of course. 14 | """.trimIndent() 15 | 16 | section("Example script") { 17 | includeMdFile("sampleScript.md") 18 | 19 | +""" 20 | The example script above adds the maven dependency and our two maven repositories 21 | via `@file:` directives. 22 | 23 | The `parser.addClientParams()` extension function adds a few parameters to the 24 | `kotlinx-cli` command line argument parser so we can create a `SearchClient` with the 25 | right parameters. You can add more parameters for your script of course. If you call the script with `-h` 26 | it will print all the parameters that we added: 27 | 28 | ``` 29 | Usage: script options_list 30 | Options: 31 | --host, -a [localhost] -> Host { String } 32 | --port, -p [9200] -> Port { Int } 33 | --user -> Basic authentication user name if using with cloud hosting { String } 34 | --password -> Basic authentication user name if using with cloud hosting { String } 35 | --protocol [false] -> Use https if true 36 | --help, -h -> Usage info 37 | ``` 38 | 39 | After parsing the `args`, you can get a `SearchClient` by simply calling 40 | `searchClientParams.searchClient`. Kotlinx-cli's commandline parser will populate the settings. 41 | 42 | You can use then import the client as normally. Because the client uses suspending 43 | functions, you have to surround your code with a `runBlocking {...}` 44 | 45 | Note, be sure to use the latest version of [kt-search-kts](https://github.com/jillesvangurp/kt-search-kts/). 46 | """.trimIndent() 47 | } 48 | section("Some ideas for using kt-search on the cli") { 49 | +""" 50 | Some ideas for using `kts` scripting with Kt-Search: 51 | 52 | - index creation and alias management 53 | - bulk indexing content 54 | - manage cluster settings 55 | - orchestrate rolling restarts 56 | - snapshot management 57 | """.trimIndent() 58 | } 59 | 60 | section("How to run `.main.kts` scripts") { 61 | +""" 62 | To be able to run the scripts, install kotlin 1.7 via your linux package manager, 63 | home-brew, sdkman, snap, etc. There are many ways to do this. 64 | 65 | Unfortunately, using kotlin script is a bit under-documented by Jetbrains and still has some issues. 66 | 67 | [kt-search-kts](https://github.com/jillesvangurp/kt-search-kts/) is there to get you started, of course. 68 | 69 | Limitations: 70 | 71 | - your script name **MUST** end in `.main.kts` 72 | - import and dependency handling is a bit limited especially for extension functions outside of 73 | intellij. So, you may have to add the right imports manually. 74 | - KTS and compiler plugins are tricky. Since kt-search uses kotlinx-serialization 75 | that means that defining new serializable data classes is not possible in 76 | KTS unless you can add the compiler plugin. There are some workarounds for that documented 77 | here: https://youtrack.jetbrains.com/issue/KT-47384. Alternatively, put your model classes in a separate library (that builds with the compiler plugin) and add a dependency to that. 78 | - KTS is a bit limited in with respect to handling multi-platform dependencies. 79 | Make sure to depend on the `-jvm` dependency for multi-platform dependencies 80 | (like kt-search). The `kt-search-kts` library has a transitive dependency and that 81 | works out fine. 82 | - if you add a custom repository, you also have to specify maven 83 | central as a repository explicitly if you need more dependencies 84 | 85 | ```kotlin 86 | @file:Repository("https://maven.tryformation.com/releases") 87 | @file:Repository(" https://repo.maven.apache.org/maven2/") 88 | @file:Repository("https://jitpack.io") 89 | @file:DependsOn("com.github.jillesvangurp:kt-search-kts:1.0.7") 90 | ``` 91 | - make sure to add the shebang to your script `#!/usr/bin/env kotlin` and of 92 | course make it executable `chmod 755 myscript.main.kts` 93 | this will direct linux/mac to use kotlin to run the script with kotlin 94 | - intellij does not reliably reload the script context when you 95 | modify the dependencies: closing and re-opening the IDE seems to work. 96 | """.trimIndent() 97 | 98 | } 99 | } -------------------------------------------------------------------------------- /docs/src/test/kotlin/documentation/manual/search/delete-by-query.kt: -------------------------------------------------------------------------------- 1 | package documentation.manual.search 2 | 3 | import com.jillesvangurp.ktsearch.Refresh 4 | import com.jillesvangurp.ktsearch.create 5 | import com.jillesvangurp.ktsearch.repository.repository 6 | import com.jillesvangurp.searchdsls.querydsl.match 7 | import documentation.sourceGitRepository 8 | 9 | val deleteByQueryMd = sourceGitRepository.md { 10 | val indexName = "docs-term-queries-demo" 11 | client.indexTestFixture(indexName) 12 | 13 | +""" 14 | Delete by query is supported both on the client and the repository. 15 | """.trimIndent() 16 | 17 | example { 18 | val repo = client.repository(indexName, TestDoc.serializer()) 19 | repo.bulk(refresh = Refresh.WaitFor) { 20 | create(TestDoc("1", "banana", price = 2.0)) 21 | create(TestDoc("1", "apple", price = 1.0)) 22 | } 23 | 24 | repo.deleteByQuery { 25 | query = match(TestDoc::name, "apple") 26 | }.deleted 27 | } 28 | +""" 29 | If you need the optional query parameters on this API, use `client.deleteByQuery` instead. 30 | """.trimIndent() 31 | } 32 | -------------------------------------------------------------------------------- /docs/src/test/kotlin/documentation/manual/search/helpers.kt: -------------------------------------------------------------------------------- 1 | package documentation.manual.search 2 | 3 | import com.jillesvangurp.ktsearch.* 4 | import kotlinx.coroutines.runBlocking 5 | import kotlinx.serialization.Serializable 6 | 7 | @Serializable 8 | data class TestDoc(val id: String, val name: String, val tags: List = listOf(), val price: Double) 9 | 10 | val client by lazy { SearchClient(KtorRestClient("localhost", 9999, logging = true)) } 11 | 12 | fun SearchClient.indexTestFixture(indexName: String) = runBlocking { 13 | // begin INITTESTFIXTURE 14 | // re-create the index 15 | deleteIndex(target = indexName, ignoreUnavailable = true) 16 | createIndex(indexName) { 17 | mappings { 18 | text(TestDoc::name) 19 | keyword(TestDoc::tags) { 20 | fields { 21 | text("txt") 22 | } 23 | } 24 | number(TestDoc::price) 25 | } 26 | } 27 | 28 | val docs = listOf( 29 | TestDoc( 30 | id = "1", 31 | name = "Apple", 32 | tags = listOf("fruit"), 33 | price = 0.50 34 | ), 35 | TestDoc( 36 | id = "2", 37 | name = "Banana", 38 | tags = listOf("fruit"), 39 | price = 0.80 40 | ), 41 | TestDoc( 42 | id = "3", 43 | name = "Green Beans", 44 | tags = listOf("legumes"), 45 | price = 1.20 46 | ) 47 | ) 48 | docs.forEach { d -> 49 | client.indexDocument( 50 | target = indexName, 51 | document = d, 52 | id = d.id, 53 | refresh = Refresh.WaitFor 54 | ) 55 | } 56 | // end INITTESTFIXTURE 57 | 58 | } 59 | 60 | 61 | // begin RESULTSPRETTYPRINT 62 | fun SearchResponse.pretty(message: String): String = 63 | // simple extension function to print the results 64 | "$message Found ${total} results:\n" + 65 | hits!!.hits.joinToString("\n") { h -> 66 | "- ${h.score} ${h.id} ${h.parseHit().name}" 67 | } 68 | // end RESULTSPRETTYPRINT 69 | 70 | -------------------------------------------------------------------------------- /docs/src/test/kotlin/documentation/manual/search/highlighting.kt: -------------------------------------------------------------------------------- 1 | package documentation.manual.search 2 | 3 | import com.jillesvangurp.serializationext.DEFAULT_PRETTY_JSON 4 | import com.jillesvangurp.ktsearch.search 5 | import com.jillesvangurp.searchdsls.querydsl.Fragmenter 6 | import com.jillesvangurp.searchdsls.querydsl.Type 7 | import com.jillesvangurp.searchdsls.querydsl.highlight 8 | import com.jillesvangurp.searchdsls.querydsl.match 9 | import documentation.sourceGitRepository 10 | import kotlinx.serialization.encodeToString 11 | 12 | val highlightingMd = sourceGitRepository.md { 13 | val indexName = "docs-term-highlighting-demo" 14 | client.indexTestFixture(indexName) 15 | 16 | +""" 17 | Highlighting allows you to show to your users why particular results are matching a query. 18 | """.trimIndent() 19 | 20 | example { 21 | client.search(indexName) { 22 | query=match(TestDoc::name,"bananana") { 23 | fuzziness="AUTO" 24 | } 25 | // create a highlight on the name field with default settings 26 | highlight { 27 | add(TestDoc::name) 28 | } 29 | } 30 | }.let { 31 | +DEFAULT_PRETTY_JSON.encodeToString(it.result.getOrThrow()) 32 | } 33 | +""" 34 | Of course you can customize how highlighting works: 35 | """.trimIndent() 36 | 37 | example { 38 | client.search(indexName) { 39 | query=match(TestDoc::name,"bananana") { 40 | fuzziness="AUTO" 41 | } 42 | // create a highlight on the name field with default settings 43 | highlight { 44 | // use some alternative tags instead of the defaults 45 | preTags="
"
46 |                 postTags="
" 47 | 48 | add(TestDoc::name) { 49 | // configure some per field settings 50 | type = Type.plain 51 | fragmenter=Fragmenter.span 52 | } 53 | } 54 | } 55 | }.let { 56 | +DEFAULT_PRETTY_JSON.encodeToString(it.result.getOrThrow()) 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /docs/src/test/kotlin/documentation/manual/search/term-level-queries.kt: -------------------------------------------------------------------------------- 1 | package documentation.manual.search 2 | 3 | import com.jillesvangurp.ktsearch.search 4 | import com.jillesvangurp.searchdsls.querydsl.* 5 | import documentation.printStdOut 6 | import documentation.sourceGitRepository 7 | 8 | val termLevelQueriesMd = sourceGitRepository.md { 9 | val indexName = "docs-term-queries-demo" 10 | client.indexTestFixture(indexName) 11 | 12 | +""" 13 | The most basic queries in Elasticsearch are queries on individual terms. 14 | """.trimIndent() 15 | 16 | section("Term query") { 17 | example { 18 | client.search(indexName) { 19 | query = term(TestDoc::tags, "fruit") 20 | }.pretty("Term Query.").let { println(it) } 21 | }.printStdOut() 22 | 23 | +""" 24 | You can also do terms queries using numbers or booleans. 25 | """.trimIndent() 26 | example { 27 | client.search(indexName) { 28 | query = term(TestDoc::price, 0.80) 29 | }.pretty("Term Query.").let { println(it) } 30 | } 31 | 32 | +""" 33 | By default term queries are case sensitive. But you can turn that off. 34 | """.trimIndent() 35 | example { 36 | client.search(indexName) { 37 | query = term(TestDoc::tags, "fRuIt") { 38 | caseInsensitive = true 39 | } 40 | }.pretty("Term Query.").let { println(it) } 41 | }.printStdOut() 42 | 43 | 44 | } 45 | section("Terms query") { 46 | example { 47 | client.search(indexName) { 48 | query = terms(TestDoc::tags, "fruit", "legumes") 49 | }.pretty("Terms Query.").let { println(it) } 50 | }.printStdOut() 51 | } 52 | section("Fuzzy query") { 53 | example { 54 | client.search(indexName) { 55 | query = fuzzy(TestDoc::tags, "friut") { 56 | fuzziness = "auto" 57 | } 58 | }.pretty("Fuzzy Query.").let { println(it) } 59 | } 60 | 61 | } 62 | section("Prefix query") { 63 | example { 64 | client.search(indexName) { 65 | query = prefix(TestDoc::tags, "fru") 66 | }.pretty("Prefix Query.").let { println(it) } 67 | }.printStdOut() 68 | 69 | } 70 | section("Wildcard query") { 71 | example { 72 | client.search(indexName) { 73 | query = wildcard(TestDoc::tags, "f*") 74 | }.pretty("Wildcard Query.").let { println(it) } 75 | }.printStdOut() 76 | 77 | } 78 | section("RegExp query") { 79 | example { 80 | client.search(indexName) { 81 | query = regExp(TestDoc::tags, "(fruit|legumes)") 82 | }.pretty("RegExp Query.").let { println(it) } 83 | }.printStdOut() 84 | 85 | } 86 | section("Ids query") { 87 | example { 88 | client.search(indexName) { 89 | query = ids("1", "2") 90 | 91 | }.pretty("Ids Query.").let { println(it) } 92 | }.printStdOut() 93 | 94 | } 95 | section("Exists query") { 96 | example { 97 | client.search(indexName) { 98 | query = ids("1", "2") 99 | }.pretty("Exists Query.").let { println(it) } 100 | }.printStdOut() 101 | 102 | } 103 | section("Range query") { 104 | example { 105 | client.search(indexName) { 106 | query = range(TestDoc::price) { 107 | gt=0 108 | lte=100.0 109 | } 110 | 111 | }.pretty("Range Query.").let { println(it) } 112 | }.printStdOut() 113 | } 114 | section("Terms Set query") { 115 | example { 116 | client.search(indexName) { 117 | query = termsSet(TestDoc::tags, "fruit","legumes","foo") { 118 | minimumShouldMatchScript = Script.create { 119 | source = "2" 120 | } 121 | } 122 | }.pretty("Terms Set Query").let { println(it) } 123 | }.printStdOut() 124 | } 125 | } -------------------------------------------------------------------------------- /docs/src/test/kotlin/documentation/projectreadme/gradle-maven.md: -------------------------------------------------------------------------------- 1 | ## Gradle 2 | 3 | Kt-search is published to the FORMATION maven repository. 4 | 5 | Add the `maven.tryformation.com` repository: 6 | 7 | ```kotlin 8 | repositories { 9 | mavenCentral() 10 | maven("https://maven.tryformation.com/releases") { 11 | content { 12 | includeGroup("com.jillesvangurp") 13 | } 14 | } 15 | } 16 | ``` 17 | And then add the dependency like this: 18 | 19 | ```kotlin 20 | // check the latest release tag for the latest version 21 | implementation("com.jillesvangurp:search-client:2.x.y") 22 | ``` 23 | Note, several of the search-client dependencies for ktor client are marked as implementation. This means you have to explicitly add those on your side. This is intentional as some people may want to use their own rest client with the kt-search search client. 24 | 25 | If you use the KtorRestClient that comes with kt-search you need to add the relevant ktor dependencies for the lates ktor-client 3.x: 26 | 27 | ```kotlin 28 | implementation("io.ktor:ktor-client-core:3.x.y") 29 | implementation("io.ktor:ktor-client-auth:3.x.y") 30 | implementation("io.ktor:ktor-client-logging:3.x.y") 31 | implementation("io.ktor:ktor-client-serialization:3.x.y") 32 | implementation("io.ktor:ktor-client-json:3.x.y") 33 | ``` 34 | 35 | ## Maven 36 | 37 | If you have maven based kotlin project targeting jvm and can't use kotlin multiplatform dependency, you will need to **append '-jvm' to the artifacts**. 38 | 39 | Add the `maven.tryformation.com` repository: 40 | 41 | ```xml 42 | 43 | 44 | try-formation 45 | kt search repository 46 | https://maven.tryformation.com/releases 47 | 48 | 49 | ``` 50 | 51 | And then add dependencies for jvm targets: 52 | 53 | ```xml 54 | 55 | 56 | com.jillesvangurp 57 | search-client-jvm 58 | 2.x.y 59 | 60 | 61 | com.jillesvangurp 62 | search-dsls-jvm 63 | 2.x.y 64 | 65 | 66 | com.jillesvangurp 67 | json-dsl-jvm 68 | 3.x.y 69 | 70 | 71 | ``` 72 | **Note:** The `json-dsl` is moved to separate repository. To find the latest version number, check releases: https://github.com/jillesvangurp/json-dsl/releases 73 | -------------------------------------------------------------------------------- /docs/src/test/kotlin/documentation/projectreadme/oneliner.md: -------------------------------------------------------------------------------- 1 | Kt-search is a Kotlin Multi Platform library to search across the Opensearch and Elasticsearch ecosystem on any platform that kotlin can compile to. It provides Kotlin DSLs for querying, defining mappings, bulk indexing, index templates, index life cycle management, index aliases, and much more. The key goal for this library is to provide a best in class developer experience for using Elasticsearch and Opensearch. 2 | -------------------------------------------------------------------------------- /docs/src/test/kotlin/documentation/projectreadme/readme-intro.md: -------------------------------------------------------------------------------- 1 | ## Why Kt-search? 2 | 3 | If you develop software in Kotlin and would like to use Opensearch or Elasticsearch, you have a few choices to make. There are multiple clients to choose from and not all of them work for each version. And then there is Kotlin multi platform to consider as well. Maybe you are running spring boot on the jvm. Or maybe you are using ktor compiled to native or WASM and using that to run lambda functions. 4 | 5 | Kt-search has you covered for all of those. The official Elastic or Opensearch clients are Java clients. You can use them from Kotlin but only on the JVM. And they are not source compatible with each other. The Opensearch client is based on a fork of the old Java client which after the fork was deprecated. On top of that, it uses opensearch specific package names. 6 | 7 | Kt-search solves a few important problems here: 8 | 9 | - It's Kotlin! You don't have to deal with all the Java idiomatic stuff that comes with the three Java libraries. You can write pure Kotlin code, use co-routines, and use Kotlin DSLs for everything. Simpler code, easier to debug, etc. 10 | - It's a multiplatform library. We use it on the jvm and in the browser (javascript). Targets for native, IOS, WASM, etc. are also there. So, your Kotlin code should be extremely portable. So, whether you are doing backend development, doing lambda functions, command line tools, mobile apps, or web apps, you can embed kt-search in each of those. 11 | - It doesn't force you to choose between Elasticsearch or Opensearch. Some features are specific to those products and will only work for those platforms but most of the baseline functionality is exactly the same for both. 12 | - It's future proof. Everything is extensible (DSLs) and modular. Even supporting custom plugins that add new features is pretty easy with the `json-dsl` library that is part of kt-search. 13 | 14 | ## License 15 | 16 | This project is [licensed](LICENSE) under the MIT license and will always be. 17 | 18 | ## Learn more 19 | 20 | - **[Manual](https://jillesvangurp.github.io/kt-search/manual)** - this is generated from the `docs` module. Just like this README.md file. The manual covers most of the extensive feature set of this library. Please provide feedback via the issue tracker if something is not clear to you. Or create a pull request to improve the manual. 21 | - [API Documentation](https://jillesvangurp.github.io/kt-search/api/). Dokka documentation. You can browse it, or access this in your IDE. 22 | - [Release Notes](https://github.com/jillesvangurp/kt-search/releases). 23 | - You can also learn a lot by looking at the integration tests in the `search-client` module. 24 | - There's a [full stack Kotlin demo project](https://github.com/formation-res/kt-fullstack-demo) that we built to show off this library and a few other things. 25 | - The code sample below should help you figure out the basics. 26 | 27 | ## Use cases 28 | 29 | Integrate **advanced search** capabilities in your Kotlin applications. Whether you want to build a web based dashboard, an advanced ETL pipeline or simply expose a search endpoint as a microservice, this library has you covered. 30 | 31 | - Add search functionality to your server applications. Kt-search works great with **Spring Boot**, Ktor, Quarkus, and other popular JVM based servers. Simply create your client as a singleton object and inject it wherever you need search. 32 | - Build complicated ETL functionality using the Bulk indexing DSL. 33 | - Use Kt-search in a **Kotlin-js** based web application to create **dashboards**, or web applications that don't need a separate server. See our [Full Stack at FORMATION](https://github.com/formation-res/kt-fullstack-demo) demo project for an example. 34 | - For dashboards and advanced querying, aggregation support is key and kt-search provides great support for that and makes it really easy to deal with complex nested aggregations. 35 | - Use **Kotlin Scripting** to operate and introspect your cluster. See the companion project [kt-search-kts](https://github.com/jillesvangurp/kt-search-kts/) for more on this as well as the scripting section in the [Manual](https://jillesvangurp.github.io/kt-search/manual/Scripting.html). The companion library combines `kt-search` with `kotlinx-cli` for command line argument parsing and provides some example scripts; all with the minimum of boiler plate. 36 | - Use kt-search from a **Jupyter Notebook** with the Kotlin kernel. See the `jupyter-example` directory for an example and check the [Manual](https://jillesvangurp.github.io/kt-search/manual/Jupyter.html) for instructions. 37 | 38 | The goal for kt-search is to be the **most convenient way to use opensearch and elasticsearch from Kotlin** on any platform where Kotlin is usable. 39 | 40 | Kt-search is extensible and modular. You can easily add your own custom DSLs for e.g. things not covered by this library or any custom plugins you use. And while it is opinionated about using e.g. kotlinx.serialization, you can also choose to use alternative serialization frameworks, or even use your own http client and just use the search-dsl. -------------------------------------------------------------------------------- /docs/src/test/kotlin/documentation/projectreadme/related.md: -------------------------------------------------------------------------------- 1 | ## Related projects 2 | 3 | There are several libraries that build on kt-search: 4 | 5 | - [kt-search-kts](https://github.com/jillesvangurp/kt-search-kts) - this library combines `kt-search` with `kotlinx-cli` to make scripting really easy. Combined with the out of the box support for managing snapshots, creating template mappings, bulk indexing, data-streams, etc. this is the perfect companion to script all your index operations. Additionally, it's a great tool to e.g. query your data, or build some health checks against your production indices. 6 | - [kt-search-logback-appender](https://github.com/jillesvangurp/kt-search-logback-appender) - this is a logback appender that bulk indexes log events straight to elasticsearch. We use this at FORMATION. 7 | - [full stack Kotlin demo project](https://github.com/formation-res/kt-fullstack-demo) A demo project that uses kt-search. 8 | - [es-kotlin-client](https://github.com/jillesvangurp/es-kotlin-client) - version 1 of this client; now no longer maintained. 9 | 10 | Additionally, I also maintain a few other search related projects that you might find interesting. 11 | 12 | - [Rankquest Studio](https://rankquest.jillesvangurp.com) - A user friendly tool that requires no installation process that helps you build and run test cases to measure search relevance for your search products. Rankquest Studio of course uses kt-search but it is also able to talk directly to your search API and is designed to work with any kind of search api or product that is able to return lists of results. 13 | - [querylight](https://github.com/jillesvangurp/querylight) - Sometimes Elasticsearch/Opensearch is just overkill. Query light is a tiny but capable in memory search engine that you can embed in your kotlin browser, server, or mobile applications. We use it at FORMATION to support e.g. in app icon search. Querylight comes with its own analyzers and query language. -------------------------------------------------------------------------------- /docs/src/test/resources/input.tsv: -------------------------------------------------------------------------------- 1 | id text 2 | input-1 banana muffin with chocolate chips 3 | input-2 apple crumble 4 | input-3 apple pie 5 | input-4 chocolate chip cookie 6 | input-5 the cookie monster 7 | input-6 pattiserie 8 | input-7 chicken teriyaki with rice 9 | input-8 tikka massala 10 | input-9 chicken 11 | q-1 rice 12 | q-2 gebak en taart 13 | q-3 muppets 14 | q-4 artisanal baker 15 | q-5 indian curry 16 | q-6 japanese food 17 | q-7 baked goods -------------------------------------------------------------------------------- /es_kibana/README.md: -------------------------------------------------------------------------------- 1 | Simple docker compose file to fire up es & kibana on their normal ports. Doesn't mount any volumes so shutting it down means data is gone. 2 | 3 | ``` 4 | # start 5 | docker-compose up -d 6 | 7 | # tail the logs 8 | docker logs -f es_kibana_elasticsearch_1 9 | 10 | # shut it down 11 | docker-compose down 12 | ``` 13 | 14 | -------------------------------------------------------------------------------- /es_kibana/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "2.3" 2 | # simple docker compose to play with Elasticsearch & Kibana locally 3 | services: 4 | es: 5 | image: docker.elastic.co/elasticsearch/elasticsearch:8.14.0 6 | environment: 7 | discovery.type: "single-node" 8 | cluster.name: "docker-cluster" 9 | bootstrap.memory_lock: "true" 10 | cluster.routing.allocation.disk.watermark.low: "3gb" 11 | cluster.routing.allocation.disk.watermark.high: "2gb" 12 | cluster.routing.allocation.disk.watermark.flood_stage: "1gb" 13 | cluster.routing.allocation.disk.threshold_enabled: "false" 14 | network.host: "127.0.0.1" 15 | http.host: "0.0.0.0" 16 | http.cors.enabled: "true" 17 | indices.id_field_data.enabled: true 18 | http.cors.allow-origin: |- 19 | "*" 20 | http.cors.allow-methods: "OPTIONS, HEAD, GET, POST, PUT, DELETE" 21 | http.cors.allow-headers: "X-Requested-With, X-Auth-Token, Content-Type, Content-Length, Authorization, Access-Control-Allow-Headers, Accept" 22 | xpack.security.enabled: "false" 23 | ES_JAVA_OPTS: "-Xms1024m -Xmx1024m" 24 | ES_TEMP: "/tmp" 25 | ports: 26 | - "9200:9200" 27 | - "9300:9300" 28 | ulimits: 29 | memlock: 30 | soft: -1 31 | hard: -1 32 | volumes: 33 | - data:/usr/share/elasticsearch/data 34 | kibana: 35 | image: docker.elastic.co/kibana/kibana:8.14.0 36 | environment: 37 | SERVER_NAME: localhost 38 | ELASTICSEARCH_URL: http://es:9200 39 | ports: 40 | - "5601:5601" 41 | volumes: 42 | data: 43 | driver: local 44 | -------------------------------------------------------------------------------- /forbidden_signatures.txt: -------------------------------------------------------------------------------- 1 | org.apache.commons.lang.* 2 | # use org.apache.commons.lang3.math.NumberUtils.isCreatable instead 3 | #org.apache.commons.lang3.math.NumberUtils#isParsable(java.lang.String) 4 | groovy.util.logging.* -------------------------------------------------------------------------------- /gradle.properties: -------------------------------------------------------------------------------- 1 | group=com.jillesvangurp 2 | 3 | org.gradle.parallel=true 4 | org.gradle.caching=true 5 | # dokka and kotlin compiler wants memory: https://dev.to/martinhaeusler/is-your-kotlin-compiler-slow-here-s-a-potential-fix-4if4 6 | org.gradle.jvmargs=-XX:MaxMetaspaceSize=512m -Xmx2g -Dkotlin.daemon.jvm.options=-Xmx2g 7 | 8 | # some isssues with this and dokka 9 | org.gradle.configuration-cache=false 10 | 11 | # on github actions I got this wonderful error: 12 | # e: /home/runner/.konan/dependencies/x86_64-unknown-linux-gnu-gcc-8.3.0-glibc-2.19-kernel-4.9-2/x86_64-unknown-linux-gnu/bin/ld.gold invocation reported errors 13 | # 14 | # Please try to disable compiler caches and rerun the build. To disable compiler caches, add the following line to the gradle.properties file in the project's root directory: 15 | # > Task :search-client:linkDebugTestLinuxX64 FAILED 16 | kotlin.native.cacheKind.linuxX64=none 17 | # matrix builds on gh actions ... 18 | kotlin.native.ignoreDisabledTargets=true 19 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jillesvangurp/kt-search/8fa48b59699bcc10ea01bcacc66370876cb9fcea/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-8.14-bin.zip 4 | networkTimeout=10000 5 | validateDistributionUrl=true 6 | zipStoreBase=GRADLE_USER_HOME 7 | zipStorePath=wrapper/dists 8 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @rem 2 | @rem Copyright 2015 the original author or authors. 3 | @rem 4 | @rem Licensed under the Apache License, Version 2.0 (the "License"); 5 | @rem you may not use this file except in compliance with the License. 6 | @rem You may obtain a copy of the License at 7 | @rem 8 | @rem https://www.apache.org/licenses/LICENSE-2.0 9 | @rem 10 | @rem Unless required by applicable law or agreed to in writing, software 11 | @rem distributed under the License is distributed on an "AS IS" BASIS, 12 | @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | @rem See the License for the specific language governing permissions and 14 | @rem limitations under the License. 15 | @rem 16 | @rem SPDX-License-Identifier: Apache-2.0 17 | @rem 18 | 19 | @if "%DEBUG%"=="" @echo off 20 | @rem ########################################################################## 21 | @rem 22 | @rem Gradle startup script for Windows 23 | @rem 24 | @rem ########################################################################## 25 | 26 | @rem Set local scope for the variables with windows NT shell 27 | if "%OS%"=="Windows_NT" setlocal 28 | 29 | set DIRNAME=%~dp0 30 | if "%DIRNAME%"=="" set DIRNAME=. 31 | @rem This is normally unused 32 | set APP_BASE_NAME=%~n0 33 | set APP_HOME=%DIRNAME% 34 | 35 | @rem Resolve any "." and ".." in APP_HOME to make it shorter. 36 | for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi 37 | 38 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 39 | set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" 40 | 41 | @rem Find java.exe 42 | if defined JAVA_HOME goto findJavaFromJavaHome 43 | 44 | set JAVA_EXE=java.exe 45 | %JAVA_EXE% -version >NUL 2>&1 46 | if %ERRORLEVEL% equ 0 goto execute 47 | 48 | echo. 1>&2 49 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 50 | echo. 1>&2 51 | echo Please set the JAVA_HOME variable in your environment to match the 1>&2 52 | echo location of your Java installation. 1>&2 53 | 54 | goto fail 55 | 56 | :findJavaFromJavaHome 57 | set JAVA_HOME=%JAVA_HOME:"=% 58 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 59 | 60 | if exist "%JAVA_EXE%" goto execute 61 | 62 | echo. 1>&2 63 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 64 | echo. 1>&2 65 | echo Please set the JAVA_HOME variable in your environment to match the 1>&2 66 | echo location of your Java installation. 1>&2 67 | 68 | goto fail 69 | 70 | :execute 71 | @rem Setup the command line 72 | 73 | set CLASSPATH= 74 | 75 | 76 | @rem Execute Gradle 77 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" -jar "%APP_HOME%\gradle\wrapper\gradle-wrapper.jar" %* 78 | 79 | :end 80 | @rem End local scope for the variables with windows NT shell 81 | if %ERRORLEVEL% equ 0 goto mainEnd 82 | 83 | :fail 84 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 85 | rem the _cmd.exe /c_ return code! 86 | set EXIT_CODE=%ERRORLEVEL% 87 | if %EXIT_CODE% equ 0 set EXIT_CODE=1 88 | if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% 89 | exit /b %EXIT_CODE% 90 | 91 | :mainEnd 92 | if "%OS%"=="Windows_NT" endlocal 93 | 94 | :omega 95 | -------------------------------------------------------------------------------- /jupyter-example/README.md: -------------------------------------------------------------------------------- 1 | Using kt-search from jupyter is easy! 2 | 3 | 4 | ## Install conda 5 | 6 | On a mac, use home brew of course. 7 | 8 | ```bash 9 | brew install miniconda 10 | ``` 11 | 12 | ## Install jupyter with conda 13 | 14 | Once you have conda installed, install jupyter and the kotlin kernel. 15 | 16 | ```bash 17 | conda install jupyter 18 | conda install -c jetbrains kotlin-jupyter-kernel 19 | ``` 20 | 21 | ## Open the notebook 22 | 23 | Now you are ready to open the notebook! 24 | 25 | ```bash 26 | cd jupyter-example 27 | jupyter notebook kt-search-example.ipynb 28 | ``` 29 | -------------------------------------------------------------------------------- /jupyter-example/config.env: -------------------------------------------------------------------------------- 1 | foo=bar 2 | -------------------------------------------------------------------------------- /jupyter-example/kt-search-example.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "714f31ce", 6 | "metadata": {}, 7 | "source": [ 8 | "This should get you started with kt-search. Kt-search has it's own maven repository; which we'll add. \n", 9 | "\n", 10 | "Note that we added `-jvm` to the search-client dependency. The Kotlin kernel does not handle multi platform dependencies so we have to add the platform for this to work properly." 11 | ] 12 | }, 13 | { 14 | "cell_type": "code", 15 | "execution_count": 3, 16 | "id": "c30c6f89", 17 | "metadata": {}, 18 | "outputs": [], 19 | "source": [ 20 | "@file:Repository(\"https://jitpack.io\")\n", 21 | "@file:DependsOn(\"com.github.jillesvangurp.kt-search:search-client-jvm:1.99.15\")\n" 22 | ] 23 | }, 24 | { 25 | "cell_type": "markdown", 26 | "id": "9ad559b7", 27 | "metadata": {}, 28 | "source": [ 29 | "Import everything in `com.jillesvangurp.ktsearch.*` and create a client." 30 | ] 31 | }, 32 | { 33 | "cell_type": "code", 34 | "execution_count": 4, 35 | "id": "c3f242ec", 36 | "metadata": {}, 37 | "outputs": [], 38 | "source": [ 39 | "import com.jillesvangurp.ktsearch.*\n", 40 | "\n", 41 | "val client = SearchClient(\n", 42 | " KtorRestClient(\n", 43 | " host = \"localhost\",\n", 44 | " port = 9200\n", 45 | " )\n", 46 | ")" 47 | ] 48 | }, 49 | { 50 | "cell_type": "markdown", 51 | "id": "65c58c4f", 52 | "metadata": {}, 53 | "source": [ 54 | "Now we're ready to start using kt-search! Make sure to import runBlocking because all kt-search interactions are suspending and require a co-routine.\n", 55 | "\n", 56 | "Of course make sure to have Elasticsearch (or Opensearch) running on port 9200. You can use docker for this or install it manually." 57 | ] 58 | }, 59 | { 60 | "cell_type": "code", 61 | "execution_count": 5, 62 | "id": "32a70f61", 63 | "metadata": {}, 64 | "outputs": [ 65 | { 66 | "name": "stdout", 67 | "output_type": "stream", 68 | "text": [ 69 | "ES8:8.3.2\n" 70 | ] 71 | } 72 | ], 73 | "source": [ 74 | "import kotlinx.coroutines.runBlocking\n", 75 | "\n", 76 | "runBlocking {\n", 77 | " val engineInfo = client.engineInfo()\n", 78 | " println(engineInfo.variantInfo.variant.name + \":\" + engineInfo.version.number)\n", 79 | "}\n" 80 | ] 81 | }, 82 | { 83 | "cell_type": "code", 84 | "execution_count": 6, 85 | "id": "6f7870b0", 86 | "metadata": {}, 87 | "outputs": [ 88 | { 89 | "name": "stdout", 90 | "output_type": "stream", 91 | "text": [ 92 | "[{\"health\":\"yellow\",\"status\":\"open\",\"index\":\".ds-applogs-2022.11.30-000001\",\"uuid\":\"mDrKx4CwTwGW9_zm4XdYOg\",\"pri\":\"1\",\"rep\":\"1\",\"docs.count\":\"0\",\"docs.deleted\":\"0\",\"store.size\":\"225b\",\"pri.store.size\":\"225b\"},{\"health\":\"green\",\"status\":\"open\",\"index\":\"recipes\",\"uuid\":\"igcNcjo-QJ21okWpyIGEXQ\",\"pri\":\"1\",\"rep\":\"0\",\"docs.count\":\"11\",\"docs.deleted\":\"0\",\"store.size\":\"73.3kb\",\"pri.store.size\":\"73.3kb\"}]\n" 93 | ] 94 | } 95 | ], 96 | "source": [ 97 | "runBlocking {\n", 98 | " client.restClient.get { \n", 99 | " path(\"_cat\",\"indices\")\n", 100 | "\n", 101 | " }.getOrThrow().let { resp -> \n", 102 | " println(resp.text)\n", 103 | " }\n", 104 | "}" 105 | ] 106 | }, 107 | { 108 | "cell_type": "code", 109 | "execution_count": 7, 110 | "id": "e8b711e0", 111 | "metadata": {}, 112 | "outputs": [ 113 | { 114 | "data": { 115 | "text/plain": [ 116 | "SearchResponse(took=3, shards=Shards(total=2, successful=2, failed=0, skipped=0), timedOut=false, hits=Hits(maxScore=null, total=Total(value=0, relation=Eq), hits=[]), aggregations={}, scrollId=null, pitId=null)" 117 | ] 118 | }, 119 | "execution_count": 7, 120 | "metadata": {}, 121 | "output_type": "execute_result" 122 | } 123 | ], 124 | "source": [ 125 | "import com.jillesvangurp.searchdsls.querydsl.*\n", 126 | "\n", 127 | "runBlocking {\n", 128 | " client.search(\"\") {\n", 129 | " query = bool { \n", 130 | " should(match(\"name\", \"test\"))\n", 131 | " }\n", 132 | " }\n", 133 | "}" 134 | ] 135 | } 136 | ], 137 | "metadata": { 138 | "kernelspec": { 139 | "display_name": "Kotlin", 140 | "language": "kotlin", 141 | "name": "kotlin" 142 | }, 143 | "language_info": { 144 | "codemirror_mode": "text/x-kotlin", 145 | "file_extension": ".kt", 146 | "mimetype": "text/x-kotlin", 147 | "name": "kotlin", 148 | "nbconvert_exporter": "", 149 | "pygments_lexer": "kotlin", 150 | "version": "1.8.0-dev-3517" 151 | } 152 | }, 153 | "nbformat": 4, 154 | "nbformat_minor": 5 155 | } 156 | -------------------------------------------------------------------------------- /publish.sh: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | set -e 3 | 4 | die () { 5 | echo >&2 "$@" 6 | exit 1 7 | } 8 | 9 | [ "$#" -eq 1 ] || die "1 argument required, $# provided" 10 | echo $1 | grep -E -q '^[0-9]+\.[0-9]+(\.[0-9]+).*?$' || die "Semantic Version argument required, $1 provided" 11 | 12 | [[ -z $(git status -s) ]] || die "git status is not clean" 13 | 14 | export TAG=$1 15 | 16 | gradle -Pversion="$TAG" publish 17 | 18 | #echo "upload to gcloud" 19 | #gsutil -m rsync -r localRepo gs://mvn-public-tryformation/releases 20 | 21 | echo "tagging" 22 | git tag "$TAG" 23 | 24 | echo "publishing $TAG" 25 | 26 | git push --tags 27 | -------------------------------------------------------------------------------- /search-client/src/commonMain/kotlin/com/jillesvangurp/ktsearch/RoundRobinNodeSelector.kt: -------------------------------------------------------------------------------- 1 | package com.jillesvangurp.ktsearch 2 | 3 | interface IndexProvider { 4 | fun get(): Int 5 | fun set(value: Int) 6 | } 7 | expect fun simpleIndexProvider(initialIndex: Int=0): IndexProvider 8 | 9 | class RoundRobinNodeSelector( 10 | private val nodes: Array 11 | ) : NodeSelector { 12 | private val index = simpleIndexProvider() 13 | override suspend fun selectNode(): Node { 14 | val result = nodes[index.get()] 15 | index.set((index.get() + 1).mod(nodes.size)) 16 | return result 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /search-client/src/commonMain/kotlin/com/jillesvangurp/ktsearch/SearchClient.kt: -------------------------------------------------------------------------------- 1 | @file:Suppress("MemberVisibilityCanBePrivate") 2 | 3 | package com.jillesvangurp.ktsearch 4 | 5 | import com.jillesvangurp.searchdsls.SearchEngineVariant 6 | import com.jillesvangurp.serializationext.DEFAULT_JSON 7 | import kotlinx.serialization.DeserializationStrategy 8 | import kotlinx.serialization.json.Json 9 | import kotlinx.serialization.json.JsonObject 10 | 11 | 12 | fun Result.parse(deserializationStrategy: DeserializationStrategy, json: Json = DEFAULT_JSON): T = 13 | json.decodeFromString(deserializationStrategy, this.getOrThrow().text) 14 | 15 | fun Result.parseJsonObject() = parse(JsonObject.serializer()) 16 | /** 17 | * Search client that you can use to talk to Elasticsearch or Opensearch. 18 | * 19 | * Most client api functions are implemented as extension functions. 20 | * 21 | * @param restClient rest client configured to talk to your search engine. 22 | * Defaults to [KtorRestClient] configured to talk to localhost:9200. 23 | * 24 | * @param json kotlinx.serialization Json used to deserialize responses. 25 | * Defaults to [DEFAULT_JSON] which is configured with some sane defaults. 26 | */ 27 | class SearchClient(val restClient: RestClient=KtorRestClient(), val json: Json = DEFAULT_JSON) { 28 | private lateinit var info: SearchEngineInformation 29 | 30 | /** 31 | * Cheap way to access the version information returned by [root] 32 | * 33 | * caches the response in a lateinit var so [root] is called only once 34 | */ 35 | suspend fun engineInfo(): SearchEngineInformation { 36 | if(!this::info.isInitialized) { 37 | info = root() 38 | } 39 | return info 40 | } 41 | 42 | suspend fun validateEngine(message: String, vararg supportedVariants: SearchEngineVariant) { 43 | val variant = engineInfo().variantInfo.variant 44 | if(!supportedVariants.contains(variant)) { 45 | throw UnsupportedOperationException("$variant is not supported; requires one of ${supportedVariants.joinToString(", ")}. $message") 46 | } 47 | } 48 | 49 | fun close() { 50 | restClient.close() 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /search-client/src/commonMain/kotlin/com/jillesvangurp/ktsearch/aliases.kt: -------------------------------------------------------------------------------- 1 | package com.jillesvangurp.ktsearch 2 | 3 | import com.jillesvangurp.jsondsl.JsonDsl 4 | import com.jillesvangurp.jsondsl.json 5 | import com.jillesvangurp.jsondsl.withJsonDsl 6 | import com.jillesvangurp.searchdsls.querydsl.ESQuery 7 | import kotlinx.serialization.Serializable 8 | import kotlinx.serialization.builtins.MapSerializer 9 | import kotlinx.serialization.builtins.serializer 10 | import kotlinx.serialization.json.JsonObject 11 | 12 | @Suppress("unused") 13 | class AliasAction: JsonDsl() { 14 | var alias by property() 15 | var aliases by property>() 16 | var index by property() 17 | var indices by property>() 18 | var filter by property() 19 | } 20 | 21 | class AliasUpdateRequest: JsonDsl() { 22 | private var actions by property("actions",mutableListOf()) 23 | fun add(block: AliasAction.()->Unit) { 24 | actions.add(withJsonDsl { 25 | this["add"] =AliasAction().apply(block) 26 | }) 27 | } 28 | fun remove(block: AliasAction.()->Unit) { 29 | actions.add(withJsonDsl { 30 | this["remove"] =AliasAction().apply(block) 31 | }) 32 | } 33 | fun removeIndex(block: AliasAction.()->Unit) { 34 | actions.add(withJsonDsl { 35 | this["remove_index"] =AliasAction().apply(block) 36 | }) 37 | } 38 | } 39 | 40 | suspend fun SearchClient.updateAliases(block: AliasUpdateRequest.()->Unit): AcknowledgedResponse { 41 | return restClient.post { 42 | path("_aliases") 43 | body = AliasUpdateRequest().apply(block).json(true) 44 | }.parse(AcknowledgedResponse.serializer()) 45 | } 46 | 47 | 48 | @Serializable 49 | data class AliasResponse(val aliases: Map) 50 | suspend fun SearchClient.getAliases(target: String?=null): Map { 51 | return restClient.get { 52 | path(target,"_alias") 53 | }.parse(MapSerializer(String.serializer(),AliasResponse.serializer())) 54 | } -------------------------------------------------------------------------------- /search-client/src/commonMain/kotlin/com/jillesvangurp/ktsearch/analyze-api.kt: -------------------------------------------------------------------------------- 1 | package com.jillesvangurp.ktsearch 2 | 3 | import com.jillesvangurp.jsondsl.JsonDsl 4 | import kotlinx.serialization.SerialName 5 | import kotlinx.serialization.Serializable 6 | 7 | 8 | @Suppress("unused") 9 | class AnalyzeRequest : JsonDsl() { 10 | var analyzer by property() 11 | var attributes by property>() 12 | var charFilter by property>() 13 | var field by property() 14 | var filter by property>() 15 | var normalizer by property() 16 | var text by property>() 17 | var tokenizer by property() 18 | } 19 | 20 | 21 | @Serializable 22 | data class AnalyzeResponse( 23 | @SerialName("tokens") val tokens: List, 24 | ) 25 | 26 | @Serializable 27 | data class AnalyzeToken( 28 | @SerialName("token") val token: String, 29 | @SerialName("start_offset") val startOffset: Int, 30 | @SerialName("end_offset") val endOffset: Int, 31 | @SerialName("type") val type: String, 32 | @SerialName("position") val position: Int, 33 | ) 34 | 35 | suspend fun SearchClient.analyze( 36 | target: String? = null, 37 | block: AnalyzeRequest.() -> Unit 38 | ): AnalyzeResponse { 39 | return restClient.post { 40 | path(*listOfNotNull(target.takeIf { !it.isNullOrBlank() }, "_analyze").toTypedArray()) 41 | json(AnalyzeRequest().apply(block)) 42 | }.parse(AnalyzeResponse.serializer()) 43 | } 44 | 45 | -------------------------------------------------------------------------------- /search-client/src/commonMain/kotlin/com/jillesvangurp/ktsearch/cluster-api.kt: -------------------------------------------------------------------------------- 1 | @file:Suppress("unused") 2 | 3 | package com.jillesvangurp.ktsearch 4 | 5 | import kotlinx.serialization.SerialName 6 | import kotlinx.serialization.Serializable 7 | 8 | @Serializable 9 | enum class ClusterStatus { 10 | @SerialName("red") 11 | Red, 12 | 13 | @SerialName("yellow") 14 | Yellow, 15 | 16 | @SerialName("green") 17 | Green 18 | } 19 | 20 | val ClusterStatus.usable: Boolean get() = this == ClusterStatus.Green || this == ClusterStatus.Yellow 21 | 22 | @Serializable 23 | data class ClusterHealthResponse( 24 | @SerialName("cluster_name") 25 | val clusterName: String, 26 | val status: ClusterStatus, 27 | @SerialName("timed_out") 28 | val timedOut: Boolean, 29 | ) 30 | 31 | suspend fun SearchClient.clusterHealth( 32 | extraParameters: Map? = null, 33 | ): ClusterHealthResponse { 34 | return restClient.get { 35 | path("_cluster", "health") 36 | parameters(extraParameters) 37 | }.parse(ClusterHealthResponse.serializer(), json) 38 | } 39 | 40 | -------------------------------------------------------------------------------- /search-client/src/commonMain/kotlin/com/jillesvangurp/ktsearch/common.kt: -------------------------------------------------------------------------------- 1 | @file:Suppress("unused") 2 | 3 | package com.jillesvangurp.ktsearch 4 | 5 | import com.jillesvangurp.jsondsl.camelCase2SnakeCase 6 | import kotlinx.datetime.Clock 7 | import kotlinx.datetime.TimeZone 8 | import kotlinx.datetime.toLocalDateTime 9 | import kotlinx.serialization.Serializable 10 | import kotlin.time.Duration 11 | 12 | enum class OperationType { 13 | Create, Index, Update, Delete 14 | } 15 | 16 | enum class Refresh { 17 | WaitFor, 18 | True, 19 | False 20 | } 21 | 22 | enum class VersionType { 23 | External, 24 | ExternalGte 25 | } 26 | 27 | fun Enum<*>.snakeCase() = this.name.camelCase2SnakeCase() 28 | 29 | @Serializable 30 | data class Shards(val total: Int, val successful: Int, val failed: Int, val skipped: Int? = null) 31 | 32 | private fun Int.formatTwoChars() = if (this < 10) "0$this" else this 33 | fun formatTimestamp(): String { 34 | val now = Clock.System.now().toLocalDateTime(TimeZone.UTC) 35 | return "${now.year}${now.monthNumber.formatTwoChars()}${now.dayOfMonth.formatTwoChars()}T${now.hour.formatTwoChars()}${now.minute.formatTwoChars()}${now.second.formatTwoChars()}" 36 | } 37 | 38 | @Serializable 39 | data class AcknowledgedResponse(val acknowledged: Boolean) 40 | 41 | fun Duration?.toElasticsearchTimeUnit(): String? = this?.toComponents { days, hours, minutes, seconds, nanoseconds -> 42 | when { 43 | this.isInfinite() -> null 44 | this.isNegative() -> null 45 | this.inWholeNanoseconds == 0L -> null 46 | nanoseconds != 0 -> "${this.inWholeNanoseconds}nanos" 47 | seconds != 0 -> "${this.inWholeSeconds}s" 48 | minutes != 0 -> "${this.inWholeMinutes}m" 49 | hours != 0 -> "${this.inWholeHours}h" 50 | days != 0L -> "${this.inWholeDays}d" 51 | else -> error("Unable to convert $this") 52 | } 53 | } -------------------------------------------------------------------------------- /search-client/src/commonMain/kotlin/com/jillesvangurp/ktsearch/ilm.kt: -------------------------------------------------------------------------------- 1 | @file:Suppress("unused", "MemberVisibilityCanBePrivate") 2 | 3 | package com.jillesvangurp.ktsearch 4 | 5 | import com.jillesvangurp.jsondsl.JsonDsl 6 | import com.jillesvangurp.jsondsl.json 7 | import com.jillesvangurp.jsondsl.withJsonDsl 8 | import com.jillesvangurp.searchdsls.SearchEngineVariant 9 | import kotlinx.serialization.json.JsonObject 10 | import kotlin.time.Duration 11 | 12 | class ILMActions : JsonDsl() { 13 | fun rollOver(maxPrimaryShardSizeGb: Int) { 14 | this["rollover"] = withJsonDsl { 15 | this["max_primary_shard_size"] = "${maxPrimaryShardSizeGb}gb" 16 | } 17 | } 18 | 19 | fun shrink(numberOfShards: Int) { 20 | this["shrink"] = withJsonDsl { 21 | this["number_of_shards"] = numberOfShards 22 | } 23 | } 24 | 25 | fun forceMerge(numberOfSegments: Int) { 26 | this["forcemerge"] = withJsonDsl { 27 | this["max_num_segments"] = numberOfSegments 28 | } 29 | } 30 | 31 | fun delete() { 32 | this["delete"] = JsonDsl() 33 | } 34 | } 35 | 36 | class ILMPhaseConfiguration : JsonDsl() { 37 | var minAge by property() 38 | fun minAge(duration: Duration) { 39 | val m = duration.inWholeMinutes 40 | minAge = when { 41 | m>60*24 -> { 42 | "${duration.inWholeDays}d" 43 | } 44 | m>60 -> { 45 | "${duration.inWholeHours}h" 46 | } 47 | else -> { 48 | "${duration.inWholeSeconds}s" 49 | } 50 | } 51 | } 52 | var actions by property("actions",defaultValue = ILMActions()) 53 | 54 | fun actions(block: ILMActions.() -> Unit) { 55 | actions.apply(block) 56 | } 57 | 58 | } 59 | 60 | class IMLPhases : JsonDsl() { 61 | fun hot(block: ILMPhaseConfiguration.()->Unit) { 62 | this["hot"] = ILMPhaseConfiguration().apply(block) 63 | } 64 | fun warm(block: ILMPhaseConfiguration.()->Unit) { 65 | this["warm"] = ILMPhaseConfiguration().apply(block) 66 | } 67 | fun cold(block: ILMPhaseConfiguration.()->Unit) { 68 | this["cold"] = ILMPhaseConfiguration().apply(block) 69 | } 70 | fun frozen(block: ILMPhaseConfiguration.()->Unit) { 71 | this["frozen"] = ILMPhaseConfiguration().apply(block) 72 | } 73 | fun delete(block: ILMPhaseConfiguration.()->Unit) { 74 | this["delete"] = ILMPhaseConfiguration().apply(block) 75 | } 76 | } 77 | 78 | class IMLPolicy : JsonDsl() { 79 | var phases by property("phases",defaultValue = IMLPhases()) 80 | } 81 | 82 | class ILMConfiguration: JsonDsl() { 83 | var policy by property("policy",defaultValue = IMLPolicy()) 84 | } 85 | 86 | suspend fun SearchClient.setIlmPolicy(policyId: String, block: IMLPhases.()->Unit): AcknowledgedResponse { 87 | validateEngine( 88 | "ilm only works on Elasticsearch", 89 | SearchEngineVariant.ES7, 90 | SearchEngineVariant.ES8, 91 | SearchEngineVariant.ES9 92 | ) 93 | 94 | val config = ILMConfiguration() 95 | config.policy.phases.apply(block) 96 | 97 | return restClient.put { 98 | path("_ilm","policy", policyId) 99 | body = config.json(true) 100 | }.parse(AcknowledgedResponse.serializer()) 101 | } 102 | 103 | suspend fun SearchClient.deleteIlmPolicy(policyId: String): JsonObject { 104 | return restClient.delete { 105 | path("_ilm","policy", policyId) 106 | }.parseJsonObject() 107 | } 108 | 109 | suspend fun SearchClient.getIlmPolicy(policyId: String): JsonObject { 110 | return restClient.get { 111 | path("_ilm","policy", policyId) 112 | }.parseJsonObject() 113 | } -------------------------------------------------------------------------------- /search-client/src/commonMain/kotlin/com/jillesvangurp/ktsearch/index-api.kt: -------------------------------------------------------------------------------- 1 | @file:Suppress("unused") 2 | 3 | package com.jillesvangurp.ktsearch 4 | 5 | import com.jillesvangurp.searchdsls.mappingdsl.IndexSettingsAndMappingsDSL 6 | import kotlinx.serialization.SerialName 7 | import kotlinx.serialization.Serializable 8 | import kotlinx.serialization.json.JsonObject 9 | import kotlin.time.Duration 10 | 11 | @Serializable 12 | data class IndexCreateResponse( 13 | val acknowledged: Boolean, 14 | @SerialName("shards_acknowledged") 15 | val shardsAcknowledged: Boolean, 16 | val index: String 17 | ) 18 | 19 | suspend fun SearchClient.createIndex( 20 | name: String, 21 | mappingAndSettings: String, 22 | waitForActiveShards: Int? = null, 23 | masterTimeOut: Duration? = null, 24 | timeout: Duration? = null, 25 | extraParameters: Map? = null, 26 | 27 | ): IndexCreateResponse { 28 | return restClient.put { 29 | path(name) 30 | 31 | parameter("wait_for_active_shards", waitForActiveShards) 32 | parameter("master_timeout", masterTimeOut) 33 | parameter("timeout", timeout) 34 | parameters(extraParameters) 35 | rawBody(mappingAndSettings) 36 | }.parse(IndexCreateResponse.serializer(), json) 37 | } 38 | 39 | suspend fun SearchClient.createIndex( 40 | name: String, 41 | mapping: IndexSettingsAndMappingsDSL, 42 | waitForActiveShards: Int? = null, 43 | masterTimeOut: Duration? = null, 44 | timeout: Duration? = null, 45 | extraParameters: Map? = null, 46 | 47 | ): IndexCreateResponse { 48 | return restClient.put { 49 | path(name) 50 | 51 | parameter("wait_for_active_shards", waitForActiveShards) 52 | parameter("master_timeout", masterTimeOut) 53 | parameter("timeout", timeout) 54 | parameters(extraParameters) 55 | json(mapping) 56 | }.parse(IndexCreateResponse.serializer(), json) 57 | } 58 | 59 | suspend fun SearchClient.createIndex( 60 | name: String, 61 | waitForActiveShards: Int? = null, 62 | masterTimeOut: Duration? = null, 63 | timeout: Duration? = null, 64 | extraParameters: Map? = null, 65 | block: (IndexSettingsAndMappingsDSL.() -> Unit)?=null 66 | ): IndexCreateResponse { 67 | val dsl = IndexSettingsAndMappingsDSL() 68 | block?.invoke(dsl) 69 | 70 | return createIndex( 71 | name = name, 72 | mapping = dsl, 73 | waitForActiveShards = waitForActiveShards, 74 | masterTimeOut = masterTimeOut, 75 | timeout = timeout, 76 | extraParameters = extraParameters 77 | ) 78 | } 79 | 80 | suspend fun SearchClient.deleteIndex( 81 | target: String, 82 | masterTimeOut: Duration? = null, 83 | timeout: Duration? = null, 84 | ignoreUnavailable: Boolean? = null, 85 | extraParameters: Map? = null, 86 | ): JsonObject = restClient.delete { 87 | path(target) 88 | 89 | parameter("master_timeout", masterTimeOut) 90 | parameter("timeout", timeout) 91 | parameter("ignore_unavailable", ignoreUnavailable) 92 | parameters(extraParameters) 93 | }.parseJsonObject() 94 | 95 | suspend fun SearchClient.getIndex(name: String): JsonObject { 96 | return restClient.get { 97 | path(name) 98 | }.parseJsonObject() 99 | } 100 | 101 | suspend fun SearchClient.getIndexMappings(name: String): JsonObject { 102 | return restClient.get { 103 | path(name,"_mappings") 104 | }.parseJsonObject() 105 | } 106 | 107 | -------------------------------------------------------------------------------- /search-client/src/commonMain/kotlin/com/jillesvangurp/ktsearch/index-templates.kt: -------------------------------------------------------------------------------- 1 | @file:Suppress("unused") 2 | 3 | package com.jillesvangurp.ktsearch 4 | 5 | import com.jillesvangurp.jsondsl.JsonDsl 6 | import com.jillesvangurp.jsondsl.json 7 | import com.jillesvangurp.searchdsls.mappingdsl.IndexSettingsAndMappingsDSL 8 | import kotlinx.serialization.json.JsonObject 9 | 10 | class ComponentTemplate: JsonDsl() { 11 | var template by property("template",defaultValue = IndexSettingsAndMappingsDSL()) 12 | } 13 | 14 | suspend fun SearchClient.updateComponentTemplate(templateId: String, block: IndexSettingsAndMappingsDSL.()->Unit): JsonObject { 15 | return restClient.put { 16 | path("_component_template",templateId) 17 | body = ComponentTemplate().also { 18 | it.template.apply(block) 19 | }.json(true) 20 | }.parseJsonObject() 21 | } 22 | 23 | suspend fun SearchClient.deleteComponentTemplate(templateId: String): JsonObject { 24 | return restClient.delete { 25 | path("_component_template",templateId) 26 | }.parseJsonObject() 27 | } 28 | 29 | class IndexTemplate: JsonDsl() { 30 | var indexPatterns by property("index_patterns",defaultValue = listOf()) 31 | var dataStream by property() 32 | var composedOf by property("composed_of",defaultValue = listOf()) 33 | var priority by property("priority",defaultValue = 300) 34 | var meta by property>(customPropertyName = "_meta") 35 | 36 | } 37 | 38 | suspend fun SearchClient.createIndexTemplate(templateId: String, block: IndexTemplate.() -> Unit): JsonObject { 39 | return restClient.put { 40 | path("_index_template", templateId) 41 | body = IndexTemplate().apply(block).json(true) 42 | }.parseJsonObject() 43 | } 44 | 45 | suspend fun SearchClient.deleteIndexTemplate(templateId: String): JsonObject { 46 | return restClient.delete { 47 | path("_index_template", templateId) 48 | }.parseJsonObject() 49 | } 50 | 51 | suspend fun SearchClient.createDataStream(name: String): JsonObject { 52 | return restClient.put { 53 | path("_data_stream", name) 54 | }.parseJsonObject() 55 | } 56 | 57 | suspend fun SearchClient.exists(name: String): Boolean { 58 | restClient.head { 59 | path(name) 60 | }.let { 61 | if(it.status<300) { 62 | return true 63 | } else if(it.status==404) { 64 | return false 65 | } else { 66 | throw it.asResult().exceptionOrNull()?: error("should have an exception") 67 | } 68 | } 69 | } 70 | suspend fun SearchClient.deleteDataStream(name: String): JsonObject { 71 | return restClient.delete { 72 | path("_data_stream", name) 73 | }.parseJsonObject() 74 | } 75 | -------------------------------------------------------------------------------- /search-client/src/commonMain/kotlin/com/jillesvangurp/ktsearch/reindex-api.kt: -------------------------------------------------------------------------------- 1 | package com.jillesvangurp.ktsearch 2 | 3 | import com.jillesvangurp.searchdsls.SearchEngineVariant.ES7 4 | import com.jillesvangurp.searchdsls.SearchEngineVariant.ES8 5 | import com.jillesvangurp.searchdsls.SearchEngineVariant.ES9 6 | import com.jillesvangurp.searchdsls.VariantRestriction 7 | import com.jillesvangurp.searchdsls.querydsl.ReindexDSL 8 | import kotlinx.serialization.SerialName 9 | import kotlinx.serialization.Serializable 10 | import kotlin.RequiresOptIn.Level.WARNING 11 | import kotlin.annotation.AnnotationRetention.BINARY 12 | import kotlin.annotation.AnnotationTarget.FUNCTION 13 | import kotlin.time.Duration 14 | 15 | @Serializable 16 | data class ReindexResponse( 17 | val took: Int, 18 | @SerialName("timed_out") 19 | val timedOut: Boolean, 20 | val total: Int, 21 | val updated: Int, 22 | val created: Int, 23 | val deleted: Int, 24 | val batches: Int, 25 | @SerialName("version_conflicts") 26 | val versionConflicts: Int, 27 | val noops: Int, 28 | val retries: ReindexRetries, 29 | @SerialName("throttled_millis") 30 | val throttledMillis: Int, 31 | @SerialName("requests_per_second") 32 | val requestsPerSecond: Double, 33 | @SerialName("throttled_until_millis") 34 | val throttledUntilMillis: Int, 35 | val failures: List, 36 | ) 37 | 38 | @Serializable 39 | data class ReindexRetries(val bulk: Int, val search: Int) 40 | 41 | @VariantRestriction(ES7, ES8,ES9) 42 | suspend fun SearchClient.reindex( 43 | refresh: Boolean? = null, 44 | timeout: Duration? = null, 45 | waitForActiveShards: String? = null, 46 | requestsPerSecond: Int? = null, 47 | requireAlias: Boolean? = null, 48 | scroll: Duration? = null, 49 | slices: Int? = null, 50 | maxDocs: Int? = null, 51 | block: ReindexDSL.() -> Unit, 52 | ): ReindexResponse = reindexGeneric( 53 | refresh, 54 | timeout, 55 | waitForActiveShards, 56 | true, 57 | requestsPerSecond, 58 | requireAlias, 59 | scroll, 60 | slices, 61 | maxDocs, 62 | block 63 | ).parse(ReindexResponse.serializer()) 64 | 65 | 66 | @VariantRestriction(ES7, ES8, ES9) 67 | suspend fun SearchClient.reindexAsync( 68 | refresh: Boolean? = null, 69 | timeout: Duration? = null, 70 | waitForActiveShards: String? = null, 71 | requestsPerSecond: Int? = null, 72 | requireAlias: Boolean? = null, 73 | scroll: Duration? = null, 74 | slices: Int? = null, 75 | maxDocs: Int? = null, 76 | block: ReindexDSL.() -> Unit, 77 | ): TaskId = reindexGeneric( 78 | refresh, 79 | timeout, 80 | waitForActiveShards, 81 | false, 82 | requestsPerSecond, 83 | requireAlias, 84 | scroll, 85 | slices, 86 | maxDocs, 87 | block 88 | ).parse(TaskResponse.serializer()).toTaskId() 89 | 90 | 91 | private suspend fun SearchClient.reindexGeneric( 92 | refresh: Boolean? = null, 93 | timeout: Duration? = null, 94 | waitForActiveShards: String? = null, 95 | waitForCompletion: Boolean? = null, 96 | requestsPerSecond: Int? = null, 97 | requireAlias: Boolean? = null, 98 | scroll: Duration? = null, 99 | slices: Int? = null, 100 | maxDocs: Int? = null, 101 | block: ReindexDSL.() -> Unit, 102 | ): Result { 103 | val reindexDSL = ReindexDSL() 104 | block(reindexDSL) 105 | 106 | return restClient.post { 107 | path("_reindex") 108 | parameter("refresh", refresh) 109 | parameter("timeout", timeout) 110 | parameter("wait_for_active_shards", waitForActiveShards) 111 | parameter("wait_for_completion", waitForCompletion) 112 | parameter("requests_per_second", requestsPerSecond) 113 | parameter("require_alias", requireAlias) 114 | parameter("scroll", scroll) 115 | parameter("slices", slices) 116 | parameter("max_docs", maxDocs) 117 | body = reindexDSL.toString() 118 | } 119 | } 120 | 121 | @Serializable 122 | data class TaskResponse(val task: String) { 123 | fun toTaskId() = TaskId(task) 124 | } 125 | 126 | data class TaskId(val value: String) -------------------------------------------------------------------------------- /search-client/src/commonMain/kotlin/com/jillesvangurp/ktsearch/repository/KotlinxSerializationModelSerializationStrategy.kt: -------------------------------------------------------------------------------- 1 | package com.jillesvangurp.ktsearch.repository 2 | 3 | import com.jillesvangurp.serializationext.DEFAULT_JSON 4 | import com.jillesvangurp.ktsearch.SearchClient 5 | import kotlinx.serialization.KSerializer 6 | import kotlinx.serialization.json.Json 7 | import kotlinx.serialization.json.JsonObject 8 | 9 | class KotlinxSerializationModelSerializationStrategy( 10 | private val serializer: KSerializer, 11 | private val json: Json = DEFAULT_JSON 12 | ) : ModelSerializationStrategy { 13 | override fun serialize(value: T): String { 14 | return json.encodeToString(serializer, value) 15 | } 16 | 17 | override fun deSerialize(value: JsonObject): T { 18 | return json.decodeFromJsonElement(serializer, value) 19 | } 20 | } 21 | 22 | fun SearchClient.ktorModelSerializer(serializer: KSerializer, customJson: Json? = null) = 23 | KotlinxSerializationModelSerializationStrategy(serializer, customJson ?: json) 24 | -------------------------------------------------------------------------------- /search-client/src/commonMain/kotlin/com/jillesvangurp/ktsearch/repository/ModelSerializationStrategy.kt: -------------------------------------------------------------------------------- 1 | package com.jillesvangurp.ktsearch.repository 2 | 3 | import kotlinx.serialization.json.JsonObject 4 | 5 | interface ModelSerializationStrategy { 6 | fun serialize(value: T): String 7 | fun deSerialize(value: JsonObject): T 8 | } -------------------------------------------------------------------------------- /search-client/src/commonMain/kotlin/com/jillesvangurp/ktsearch/request-dsl.kt: -------------------------------------------------------------------------------- 1 | @file:Suppress("unused") 2 | 3 | package com.jillesvangurp.ktsearch 4 | 5 | import com.jillesvangurp.jsondsl.JsonDsl 6 | import com.jillesvangurp.jsondsl.json 7 | import kotlin.time.Duration 8 | 9 | data class SearchAPIRequest( 10 | internal var body: String? = null, 11 | internal var contentType: String = "application/json", 12 | internal var pathComponents: List = listOf(), 13 | internal val parameters: MutableMap = mutableMapOf(), 14 | internal val headers: MutableMap = mutableMapOf() 15 | ) { 16 | fun path(vararg components: String?) { 17 | pathComponents = components.toList().filterNotNull() 18 | } 19 | 20 | fun parameter(key: String, value: String?) { 21 | value?.let { 22 | parameters[key] = value 23 | } 24 | } 25 | 26 | fun parameter(key: String, value: Duration?) { 27 | value.toElasticsearchTimeUnit()?.let { 28 | parameters[key] = it 29 | } 30 | } 31 | 32 | fun parameter(key: String, value: Number?) { 33 | value?.let { 34 | parameters[key] = "$value" 35 | } 36 | } 37 | 38 | fun parameter(key: String, value: Boolean?) { 39 | value?.let { 40 | parameters[key] = "$value" 41 | } 42 | } 43 | 44 | fun parameter(key: String, value: Enum<*>?) { 45 | value?.let { 46 | parameters[key] = value.snakeCase() 47 | } 48 | } 49 | 50 | fun parameters(params: Map?) { 51 | params?.let { 52 | parameters.putAll(params) 53 | } 54 | } 55 | fun header(key: String, value: String) { 56 | headers[key] = value 57 | } 58 | fun header(key: String, values: List) { 59 | headers[key] = values 60 | } 61 | 62 | fun json(dsl: JsonDsl, pretty: Boolean = false) { 63 | body = dsl.json(pretty) 64 | } 65 | 66 | fun rawBody(body: String, contentType: String = "application/json") { 67 | this.body = body 68 | this.contentType = contentType 69 | } 70 | } 71 | 72 | suspend fun RestClient.head(block: SearchAPIRequest.() -> Unit): RestResponse { 73 | val request = SearchAPIRequest() 74 | block.invoke(request) 75 | return doRequest( 76 | pathComponents = listOf("/" + request.pathComponents.joinToString("/")), 77 | httpMethod = HttpMethod.Head, 78 | parameters = request.parameters, 79 | headers = request.headers 80 | ) 81 | } 82 | 83 | suspend fun RestClient.post(block: SearchAPIRequest.() -> Unit): Result { 84 | val request = SearchAPIRequest() 85 | block.invoke(request) 86 | return doRequest( 87 | pathComponents = listOf(request.pathComponents.joinToString("/")), 88 | payload = request.body, 89 | httpMethod = HttpMethod.Post, 90 | parameters = request.parameters, 91 | headers = request.headers, 92 | contentType= request.contentType, 93 | 94 | ).asResult() 95 | } 96 | 97 | suspend fun RestClient.delete(block: SearchAPIRequest.() -> Unit): Result { 98 | val request = SearchAPIRequest() 99 | block.invoke(request) 100 | return doRequest( 101 | pathComponents = listOf("/" + request.pathComponents.joinToString("/")), 102 | payload = request.body, 103 | httpMethod = HttpMethod.Delete, 104 | parameters = request.parameters, 105 | headers = request.headers 106 | ).asResult() 107 | } 108 | 109 | suspend fun RestClient.get(block: SearchAPIRequest.() -> Unit): Result { 110 | val request = SearchAPIRequest() 111 | block.invoke(request) 112 | return doRequest( 113 | pathComponents = listOf("/" + request.pathComponents.joinToString("/")), 114 | httpMethod = HttpMethod.Get, 115 | parameters = request.parameters, 116 | headers = request.headers 117 | ).asResult() 118 | } 119 | 120 | suspend fun RestClient.put(block: SearchAPIRequest.() -> Unit): Result { 121 | val request = SearchAPIRequest() 122 | block.invoke(request) 123 | return doRequest( 124 | pathComponents = listOf("/" + request.pathComponents.joinToString("/")), 125 | payload = request.body, 126 | httpMethod = HttpMethod.Put, 127 | parameters = request.parameters, 128 | headers = request.headers, 129 | contentType= request.contentType, 130 | ).asResult() 131 | } 132 | -------------------------------------------------------------------------------- /search-client/src/commonMain/kotlin/com/jillesvangurp/ktsearch/root-api.kt: -------------------------------------------------------------------------------- 1 | package com.jillesvangurp.ktsearch 2 | 3 | import com.jillesvangurp.searchdsls.SearchEngineVariant 4 | import kotlinx.serialization.SerialName 5 | import kotlinx.serialization.Serializable 6 | 7 | @Serializable 8 | data class SearchEngineVersion( 9 | val distribution: String?, 10 | val number: String, 11 | @SerialName("build_flavor") 12 | val buildFlavor: String?, 13 | @SerialName("build_hash") 14 | val buildHash: String, 15 | @SerialName("build_date") 16 | val buildDate: String, 17 | @SerialName("build_snapshot") 18 | val buildSnapshot: Boolean, 19 | @SerialName("lucene_version") 20 | val luceneVersion: String, 21 | @SerialName("minimum_wire_compatibility_version") 22 | val minimumWireCompatibilityVersion: String, 23 | @SerialName("minimum_index_compatibility_version") 24 | val minimumIndexCompatibilityVersion: String, 25 | ) 26 | 27 | @Serializable 28 | data class SearchEngineInformation( 29 | val name: String, 30 | @SerialName("cluster_name") 31 | val clusterName: String, 32 | @SerialName("cluster_uuid") 33 | val clusterUUID: String, 34 | val version: SearchEngineVersion, 35 | ) { 36 | val variantInfo by lazy { 37 | VariantInfo( 38 | variant = 39 | when { 40 | // opensearch added the distribution 41 | this.version.distribution == "opensearch" && this.version.number.startsWith("1.") -> SearchEngineVariant.OS1 42 | this.version.distribution == "opensearch" && this.version.number.startsWith("2.") ->SearchEngineVariant.OS2 43 | this.version.distribution == "opensearch" && this.version.number.startsWith("3.") ->SearchEngineVariant.OS3 44 | this.version.number.startsWith("7.") -> SearchEngineVariant.ES7 45 | this.version.number.startsWith("8.") -> SearchEngineVariant.ES8 46 | this.version.number.startsWith("9.") -> SearchEngineVariant.ES9 47 | else -> error("version not recognized") 48 | }, 49 | versionString = version.number 50 | ) 51 | } 52 | } 53 | 54 | 55 | /** 56 | * Search engine variant meta data. 57 | * 58 | * You can get this from [SearchEngineInformation] to figure out which 59 | * search engine your client is connected to. 60 | */ 61 | @Suppress("unused") 62 | data class VariantInfo( 63 | val variant: SearchEngineVariant, 64 | val versionString: String, 65 | ) { 66 | val majorVersion by lazy { versionString.split('.')[0].toIntOrNull() } 67 | val minorVersion by lazy { versionString.split('.')[1].toIntOrNull() } 68 | val patchVersion by lazy { versionString.split('.')[2].toIntOrNull() } 69 | } 70 | 71 | /** 72 | * Http GET to `/` 73 | * 74 | * Note, you can may want to use [SearchClient.engineInfo], which 75 | * caches the response and avoid calling this multiple times. 76 | * 77 | * @return meta information about the search engine 78 | */ 79 | suspend fun SearchClient.root(): SearchEngineInformation { 80 | return restClient.get { 81 | 82 | }.parse(SearchEngineInformation.serializer(), json) 83 | } 84 | 85 | -------------------------------------------------------------------------------- /search-client/src/commonMain/kotlin/com/jillesvangurp/ktsearch/snapshot-api.kt: -------------------------------------------------------------------------------- 1 | @file:Suppress("unused") 2 | 3 | package com.jillesvangurp.ktsearch 4 | 5 | import com.jillesvangurp.jsondsl.JsonDsl 6 | import com.jillesvangurp.jsondsl.json 7 | import kotlin.time.Duration 8 | import kotlin.time.Duration.Companion.minutes 9 | 10 | class SnapshotRepository : JsonDsl() { 11 | class Settings : JsonDsl() { 12 | var bucket by property() 13 | var endPoint by property() 14 | var protocol by property() 15 | var pathStyleAccess by property() 16 | var region by property() 17 | } 18 | 19 | var type by property() 20 | var verify by property() 21 | fun settings(block: Settings.() -> Unit) { 22 | this["settings"] = Settings().apply(block) 23 | } 24 | } 25 | 26 | suspend fun SearchClient.registerSnapshotRepository(repositoryName: String, repository: SnapshotRepository) = 27 | restClient.put { 28 | path("_snapshot", repositoryName) 29 | body = repository.json(true) 30 | }.parseJsonObject() 31 | 32 | 33 | suspend fun SearchClient.registerSnapshotRepository(repositoryName: String, block: SnapshotRepository.() -> Unit) { 34 | val repo = SnapshotRepository().apply(block) 35 | registerSnapshotRepository(repositoryName, repo) 36 | } 37 | 38 | suspend fun SearchClient.verifySnapshotRepository(repositoryName: String) = restClient.post { 39 | path("_snapshot", repositoryName, "_verify") 40 | }.parseJsonObject() 41 | 42 | suspend fun SearchClient.getSnapshotRepository(repositoryName: String?) = restClient.get { 43 | path("_snapshot", repositoryName) 44 | }.parseJsonObject() 45 | 46 | suspend fun SearchClient.deleteSnapshotRepository(repositoryName: String) = restClient.delete { 47 | path("_snapshot", repositoryName) 48 | }.parseJsonObject() 49 | 50 | 51 | suspend fun SearchClient.listSnapshots(repositoryName: String, pattern: String = "_all") = restClient.get { 52 | path("_snapshot", repositoryName, pattern) 53 | }.parseJsonObject() 54 | 55 | 56 | suspend fun SearchClient.takeSnapshot( 57 | repositoryName: String, 58 | snapshotName: String = formatTimestamp(), // sane default 59 | waitForCompletion: Boolean = false, 60 | timeout: Duration = 1.minutes 61 | ) = restClient.put { 62 | path("_snapshot", repositoryName, snapshotName) 63 | if (waitForCompletion) { 64 | parameter("wait_for_completion", true) 65 | parameter("timeout", "${timeout.inWholeSeconds}s") 66 | } 67 | }.parseJsonObject() 68 | 69 | 70 | suspend fun SearchClient.restoreSnapshot( 71 | repositoryName: String, 72 | snapshotName: String, 73 | waitForCompletion: Boolean = false, 74 | timeout: Duration = 1.minutes 75 | ) = restClient.post { 76 | path("_snapshot", repositoryName, snapshotName, "_restore") 77 | if (waitForCompletion) { 78 | parameter("wait_for_completion", true) 79 | parameter("timeout", "${timeout.inWholeSeconds}s") 80 | } 81 | }.parseJsonObject() 82 | 83 | 84 | suspend fun SearchClient.deleteSnapshot( 85 | repositoryName: String, 86 | snapshotName: String, 87 | waitForCompletion: Boolean = false, 88 | timeout: Duration = 1.minutes 89 | 90 | ) = 91 | restClient.delete { 92 | path("_snapshot", repositoryName, snapshotName) 93 | if (waitForCompletion) { 94 | parameter("wait_for_completion", true) 95 | parameter("timeout", "${timeout.inWholeSeconds}s") 96 | } 97 | }.parseJsonObject() 98 | 99 | 100 | -------------------------------------------------------------------------------- /search-client/src/commonMain/kotlin/com/jillesvangurp/ktsearch/tasks-api.kt: -------------------------------------------------------------------------------- 1 | @file:Suppress("unused") 2 | 3 | package com.jillesvangurp.ktsearch 4 | 5 | import kotlinx.serialization.json.JsonObject 6 | import kotlin.time.Duration 7 | import kotlin.time.Duration.Companion.minutes 8 | 9 | suspend fun SearchClient.getTask(id: String?, waitForCompletion: Boolean = false, timeout: Duration = 1.minutes): JsonObject { 10 | return restClient.get { 11 | path("_tasks",id) 12 | if(waitForCompletion) { 13 | parameter("wait_for_completion",true) 14 | parameter("timeout","${timeout.inWholeSeconds}s") 15 | } 16 | }.parse(JsonObject.serializer()) 17 | } 18 | 19 | suspend fun SearchClient.cancelTask(id: String,waitForCompletion: Boolean = false, timeout: Duration = 1.minutes): JsonObject { 20 | return restClient.post { 21 | path("_tasks",id,"_cancel") 22 | if(waitForCompletion) { 23 | parameter("wait_for_completion",true) 24 | parameter("timeout","${timeout.inWholeSeconds}s") 25 | } 26 | }.parse(JsonObject.serializer()) 27 | } -------------------------------------------------------------------------------- /search-client/src/commonTest/kotlin/com/jillesvangurp/ktsearch/AliasManagementTest.kt: -------------------------------------------------------------------------------- 1 | package com.jillesvangurp.ktsearch 2 | 3 | import io.kotest.matchers.collections.shouldContain 4 | import kotlin.test.Test 5 | 6 | class AliasManagementTest: SearchTestBase() { 7 | 8 | @Test 9 | fun addRemoveAliases() = coRun{ 10 | 11 | val name = randomIndexName() 12 | val index1 = "$name-1" 13 | val index2 = "$name-2" 14 | val aliasName = "alias-$name" 15 | client.createIndex(index1) { 16 | 17 | } 18 | client.createIndex(index2) { 19 | 20 | } 21 | 22 | client.updateAliases { 23 | add { 24 | alias= aliasName 25 | indices= listOf(index1, index2) 26 | } 27 | } 28 | 29 | client.getAliases(index1).let { 30 | it[index1]!!.aliases.keys shouldContain aliasName 31 | } 32 | client.getAliases().let { 33 | it[index2]!!.aliases.keys shouldContain aliasName 34 | } 35 | client.updateAliases { 36 | remove { 37 | alias= aliasName 38 | indices=listOf(index1) 39 | } 40 | } 41 | client.updateAliases { 42 | removeIndex { 43 | indices=listOf(index1, index2) 44 | } 45 | } 46 | } 47 | } -------------------------------------------------------------------------------- /search-client/src/commonTest/kotlin/com/jillesvangurp/ktsearch/AnalyzeTest.kt: -------------------------------------------------------------------------------- 1 | package com.jillesvangurp.ktsearch 2 | 3 | import io.kotest.matchers.shouldBe 4 | import kotlin.test.Test 5 | 6 | class AnalyzeTest : SearchTestBase() { 7 | 8 | @Test 9 | fun shouldAnalyzeQuery() = coRun { 10 | val resp = client.analyze { 11 | text = listOf("foo bar") 12 | analyzer = "standard" 13 | } 14 | 15 | resp.tokens.size shouldBe 2 16 | } 17 | } 18 | 19 | -------------------------------------------------------------------------------- /search-client/src/commonTest/kotlin/com/jillesvangurp/ktsearch/AuthenticationTest.kt: -------------------------------------------------------------------------------- 1 | package com.jillesvangurp.ktsearch 2 | 3 | class AuthenticationTest { 4 | // @Test run manually with correct credentials to test if auth is working FIXME better auth test than this but seems to work now 5 | fun shouldAddBasicAuth() = coRun { 6 | 7 | val client = SearchClient( 8 | KtorRestClient( 9 | https = true, 10 | host = "xxxxxx.europe-west3.gcp.cloud.es.io", 11 | port = 9243, 12 | user = "elastic", 13 | password = "xxxxx", 14 | logging = false 15 | ) 16 | ) 17 | println(client.clusterHealth()) 18 | 19 | } 20 | } -------------------------------------------------------------------------------- /search-client/src/commonTest/kotlin/com/jillesvangurp/ktsearch/BulkTest.kt: -------------------------------------------------------------------------------- 1 | package com.jillesvangurp.ktsearch 2 | 3 | import com.jillesvangurp.searchdsls.querydsl.Script 4 | import io.kotest.assertions.throwables.shouldThrow 5 | import io.kotest.matchers.shouldBe 6 | import io.kotest.matchers.string.shouldContain 7 | import kotlin.test.Test 8 | 9 | class BulkTest : SearchTestBase() { 10 | 11 | @Test 12 | fun shouldBulkIndex() = coRun { 13 | testDocumentIndex { index -> 14 | 15 | 16 | client.bulk(refresh = Refresh.WaitFor, bulkSize = 4, target = index) { 17 | (1..20).forEach { 18 | create(TestDocument(name = "doc $it").json(false)) 19 | } 20 | } 21 | client.search(index) { 22 | 23 | }.total shouldBe 20 24 | } 25 | } 26 | 27 | @Test 28 | fun shouldCallback() = coRun { 29 | testDocumentIndex { index -> 30 | 31 | var success = 0 32 | var failed = 0 33 | client.bulk(target = index, bulkSize = 4, callBack = object : BulkItemCallBack { 34 | override fun itemFailed(operationType: OperationType, item: BulkResponse.ItemDetails) { 35 | failed++ 36 | } 37 | 38 | override fun itemOk(operationType: OperationType, item: BulkResponse.ItemDetails) { 39 | success++ 40 | } 41 | 42 | override fun bulkRequestFailed(e: Exception, ops: List>) { 43 | error("${e.message} on") 44 | } 45 | }) { 46 | (1..10).forEach { 47 | index(TestDocument(name = "doc $it").json()) 48 | } 49 | } 50 | 51 | success shouldBe 10 52 | failed shouldBe 0 53 | } 54 | } 55 | 56 | @Test 57 | fun shouldHandleSourceFieldOnUpdate() = coRun { 58 | testDocumentIndex { index -> 59 | client.bulk(target = index, source = "true") { 60 | update(TestDocument("bar"), id = "1", docAsUpsert = true) 61 | } 62 | } 63 | } 64 | 65 | @Test 66 | fun shouldHandleScriptUpdate() = coRun { 67 | testDocumentIndex { index -> 68 | client.bulk(target = index, source = "true") { 69 | // will get initialized to 0 by the upsert 70 | update(script = Script.create { 71 | source = "ctx._source.number += params.param1" 72 | params = mapOf( 73 | "param1" to 10 74 | ) 75 | }, id = "1", upsert = TestDocument("counter", number = 0)) 76 | // now the script runs 77 | update(script = Script.create { 78 | source = "ctx._source.number += params.param1" 79 | params = mapOf( 80 | "param1" to 10 81 | ) 82 | }, id = "1", upsert = TestDocument("counter", number = 0)) 83 | } 84 | } 85 | } 86 | 87 | @Test 88 | fun shouldAcceptSerializedUpdate() = coRun { 89 | testDocumentIndex { index -> 90 | client.bulk(target = index, source = "true") { 91 | // will get initialized to 0 by the upsert 92 | 93 | create(TestDocument("original"), id = "42") 94 | // now the script runs 95 | update(TestDocument("changed"), id = "42") 96 | } 97 | val resp = client.getDocument(index, "42").source!!.parse() 98 | resp.name shouldBe "changed" 99 | } 100 | } 101 | 102 | @Test 103 | fun shouldOverrideBulkRoutingForItemsWithRouting() = coRun { 104 | testDocumentIndex { index -> 105 | client.bulk(target = index, source = "true", routing = "1") { 106 | create(doc = TestDocument(name = "document with specific routing"), id = "42", routing = "2") 107 | create(doc = TestDocument(name = "document without specific routing"), id = "43") 108 | index(doc = TestDocument(name = "document to delete"), id = "44", routing = "3") 109 | 110 | update( 111 | id = "42", 112 | doc = TestDocument(name = "document with specific routing updated").json(), 113 | routing = "2" 114 | ) 115 | 116 | delete(id = "44", routing = "3") 117 | } 118 | 119 | val docWithSpecificRouting = client.getDocument(index, "42") 120 | val docWithoutSpecificRouting = client.getDocument(index, "43") 121 | 122 | shouldThrow { 123 | client.getDocument(index, "44") 124 | }.message shouldContain "RequestIsWrong 404" 125 | 126 | docWithSpecificRouting.routing shouldBe "2" 127 | docWithSpecificRouting.source!!.parse().name shouldBe "document with specific routing updated" 128 | docWithoutSpecificRouting.routing shouldBe "1" 129 | } 130 | } 131 | } -------------------------------------------------------------------------------- /search-client/src/commonTest/kotlin/com/jillesvangurp/ktsearch/ClusterHealthTest.kt: -------------------------------------------------------------------------------- 1 | package com.jillesvangurp.ktsearch 2 | 3 | import io.kotest.matchers.shouldNotBe 4 | import kotlin.test.Test 5 | 6 | class ClusterHealthTest : SearchTestBase() { 7 | 8 | @Test 9 | fun clusterShouldBeHealthy() = coRun { 10 | client.clusterHealth().status shouldNotBe ClusterStatus.Red 11 | } 12 | 13 | 14 | } -------------------------------------------------------------------------------- /search-client/src/commonTest/kotlin/com/jillesvangurp/ktsearch/CommonTestKt.kt: -------------------------------------------------------------------------------- 1 | package com.jillesvangurp.ktsearch 2 | 3 | import io.kotest.matchers.shouldBe 4 | import kotlin.test.Test 5 | import kotlin.time.Duration 6 | import kotlin.time.Duration.Companion.INFINITE 7 | import kotlin.time.Duration.Companion.days 8 | import kotlin.time.Duration.Companion.hours 9 | import kotlin.time.Duration.Companion.microseconds 10 | import kotlin.time.Duration.Companion.milliseconds 11 | import kotlin.time.Duration.Companion.minutes 12 | import kotlin.time.Duration.Companion.nanoseconds 13 | import kotlin.time.Duration.Companion.seconds 14 | 15 | class CommonTestKt { 16 | 17 | @Test 18 | fun nullCase() { 19 | val duration: Duration? = null 20 | duration.toElasticsearchTimeUnit() shouldBe null 21 | } 22 | 23 | @Test 24 | fun daysCase() { 25 | 2.days.toElasticsearchTimeUnit() shouldBe "2d" 26 | } 27 | 28 | @Test 29 | fun hoursCase() { 30 | 26.hours.toElasticsearchTimeUnit() shouldBe "26h" 31 | } 32 | 33 | @Test 34 | fun minutesCase() { 35 | 65.minutes.toElasticsearchTimeUnit() shouldBe "65m" 36 | } 37 | 38 | @Test 39 | fun secondsCase() { 40 | 65.seconds.toElasticsearchTimeUnit() shouldBe "65s" 41 | } 42 | @Test 43 | fun millisecondsCase() { 44 | 7887.milliseconds.toElasticsearchTimeUnit() shouldBe "7887000000nanos" 45 | } 46 | 47 | @Test 48 | fun microsecondsCase() { 49 | 1234.microseconds.toElasticsearchTimeUnit() shouldBe "1234000nanos" 50 | } 51 | 52 | @Test 53 | fun nanosecondsCase() { 54 | 54.nanoseconds.toElasticsearchTimeUnit() shouldBe "54nanos" 55 | } 56 | 57 | @Test 58 | fun infiniteCase() { 59 | INFINITE.toElasticsearchTimeUnit() shouldBe null 60 | } 61 | 62 | @Test 63 | fun negativeCase() { 64 | (-10).seconds.toElasticsearchTimeUnit() shouldBe null 65 | } 66 | 67 | @Test 68 | fun zeroCase() { 69 | 0.seconds.toElasticsearchTimeUnit() shouldBe null 70 | } 71 | } -------------------------------------------------------------------------------- /search-client/src/commonTest/kotlin/com/jillesvangurp/ktsearch/DeleteByQyeryTest.kt: -------------------------------------------------------------------------------- 1 | package com.jillesvangurp.ktsearch 2 | 3 | import com.jillesvangurp.searchdsls.querydsl.matchAll 4 | import io.kotest.matchers.shouldBe 5 | import kotlin.test.Test 6 | 7 | class DeleteByQyeryTest: SearchTestBase() { 8 | 9 | @Test 10 | fun shouldDeleteByQuery() = coRun { 11 | testDocumentIndex { index -> 12 | 13 | client.indexDocument(index, TestDocument("foo bar").json(false), refresh = Refresh.WaitFor) 14 | client.indexDocument(index, TestDocument("fooo").json(false), refresh = Refresh.WaitFor) 15 | 16 | val resp = client.deleteByQuery(index) { 17 | query = matchAll() 18 | } 19 | 20 | resp.deleted shouldBe 2 21 | } 22 | } 23 | } -------------------------------------------------------------------------------- /search-client/src/commonTest/kotlin/com/jillesvangurp/ktsearch/DeleteIndexTest.kt: -------------------------------------------------------------------------------- 1 | package com.jillesvangurp.ktsearch 2 | 3 | import io.kotest.assertions.throwables.shouldThrow 4 | import io.kotest.matchers.shouldBe 5 | import io.kotest.matchers.string.shouldContain 6 | import kotlinx.serialization.json.jsonPrimitive 7 | import kotlin.test.Test 8 | 9 | class DeleteIndexTest : SearchTestBase() { 10 | 11 | @Test 12 | fun deleteFailsWhenIndexIsNotAvailable() = coRun { 13 | val indexName = randomIndexName() 14 | val exception = shouldThrow { 15 | client.deleteIndex(indexName) 16 | } 17 | exception.status shouldBe 404 18 | exception.message shouldContain "index_not_found_exception" 19 | } 20 | 21 | @Test 22 | fun deleteFailuresCanBeIgnored() = coRun { 23 | val indexName = randomIndexName() 24 | val response = client.deleteIndex(target = indexName, ignoreUnavailable = true) 25 | response.getValue("acknowledged").jsonPrimitive.content shouldBe "true" 26 | } 27 | } -------------------------------------------------------------------------------- /search-client/src/commonTest/kotlin/com/jillesvangurp/ktsearch/DocumentCRUDTest.kt: -------------------------------------------------------------------------------- 1 | package com.jillesvangurp.ktsearch 2 | 3 | import com.jillesvangurp.searchdsls.querydsl.Script 4 | import com.jillesvangurp.serializationext.DEFAULT_JSON 5 | import io.kotest.matchers.shouldBe 6 | import io.kotest.matchers.shouldNotBe 7 | import kotlin.test.Test 8 | import kotlin.test.fail 9 | 10 | class DocumentCRUDTest: SearchTestBase() { 11 | 12 | @Test 13 | fun shouldDoDocumentCrud() = coRun { 14 | testDocumentIndex { index -> 15 | 16 | client.indexDocument(index, TestDocument("xx").json(false)).also { createResponse -> 17 | createResponse.index shouldBe index 18 | createResponse.shards.failed shouldBe 0 19 | client.getDocument(index, createResponse.id).also { getResponse -> 20 | val document = getResponse.document() 21 | getResponse.id shouldBe createResponse.id 22 | document.name shouldBe "xx" 23 | client.indexDocument(index, TestDocument(name = "yy").json(false), id = getResponse.id) 24 | .also { updateResponse -> 25 | updateResponse.id shouldBe createResponse.id 26 | 27 | } 28 | } 29 | client.getDocument(index, createResponse.id).also { getResponse -> 30 | val document = getResponse.document() 31 | document.name shouldBe "yy" 32 | } 33 | client.deleteDocument(index, createResponse.id) 34 | try { 35 | client.getDocument(index, createResponse.id) 36 | fail("should throw") 37 | } catch (e: RestException) { 38 | // not a problem 39 | } 40 | } 41 | } 42 | } 43 | 44 | @Test 45 | fun shouldSupportDocumentUpdates() = coRun { 46 | testDocumentIndex { index -> 47 | 48 | client.indexDocument(index, TestDocument("foo", id = 1), "1") 49 | 50 | client.updateDocument(index, "1", TestDocument("bar", id = 1)).let { resp -> 51 | resp.result shouldBe "updated" 52 | } 53 | client.updateDocument(index, "1", TestDocument("bar", id = 1), detectNoop = true).let { resp -> 54 | resp.result shouldBe "noop" 55 | } 56 | } 57 | } 58 | 59 | @Test 60 | fun shouldSupportScriptUpdates() = coRun { 61 | testDocumentIndex { index -> 62 | 63 | client.updateDocument( 64 | target = index, 65 | id = "1", 66 | script = Script.create { 67 | source = "ctx._source.number += params.param1" 68 | params = mapOf( 69 | "param1" to 1 70 | ) 71 | }, 72 | upsertJson = TestDocument("foo", number = 0), 73 | ) 74 | client.updateDocument( 75 | target = index, 76 | id = "1", 77 | script = Script.create { 78 | source = "ctx._source.number += params.param1" 79 | params = mapOf( 80 | "param1" to 1 81 | ) 82 | }, 83 | upsertJson = TestDocument("foo", number = 0), 84 | source = "true" 85 | ).let { resp -> 86 | resp.get?.source shouldNotBe null 87 | DEFAULT_JSON.decodeFromJsonElement(TestDocument.serializer(), resp.get?.source!!).let { 88 | it.number shouldBe 1 89 | } 90 | } 91 | } 92 | } 93 | 94 | @Test 95 | fun shouldMgetDocs() = coRun { 96 | testDocumentIndex { indexName -> 97 | 98 | client.indexDocument(indexName, TestDocument("foo", description = "Foo"), id = "1") 99 | client.indexDocument(indexName, TestDocument("bar", description = "Bar"), id = "2") 100 | 101 | client.mGet { 102 | doc { 103 | id = "1" 104 | index = indexName 105 | source = true 106 | } 107 | doc { 108 | id = "idontexist" 109 | index = indexName 110 | source = true 111 | } 112 | 113 | }.docs.let { 114 | it.firstOrNull { !it.found } shouldNotBe null 115 | it.size shouldBe 2 116 | } 117 | } 118 | } 119 | } -------------------------------------------------------------------------------- /search-client/src/commonTest/kotlin/com/jillesvangurp/ktsearch/GeoSpatialQueriesTest.kt: -------------------------------------------------------------------------------- 1 | package com.jillesvangurp.ktsearch 2 | 3 | import com.jillesvangurp.searchdsls.SearchEngineVariant 4 | import com.jillesvangurp.searchdsls.querydsl.* 5 | import io.kotest.matchers.shouldBe 6 | import kotlin.test.Test 7 | 8 | class GeoSpatialQueriesTest : SearchTestBase() { 9 | 10 | @Test 11 | fun shouldDoBoundingBox() = coRun { 12 | geoTestFixture { index -> 13 | client.search(index) { 14 | query = GeoBoundingBoxQuery(TestDocument::point) { 15 | topLeft(arrayOf(12.0, 53.0)) 16 | bottomRight(arrayOf(14.0, 51.0)) 17 | } 18 | }.total shouldBe 1 19 | 20 | client.search(index) { 21 | query = GeoBoundingBoxQuery(TestDocument::point) { 22 | topLeft(arrayOf(14.0, 53.0)) 23 | bottomRight(arrayOf(16.0, 51.0)) 24 | } 25 | }.total shouldBe 0 26 | } 27 | } 28 | 29 | @Test 30 | fun shouldDoDistanceSearch() = coRun { 31 | geoTestFixture { index -> 32 | client.search(index) { 33 | query = GeoDistanceQuery(TestDocument::point, "1000km", "POINT (13.0 51.0)") 34 | }.total shouldBe 1 35 | client.search(index) { 36 | query = GeoDistanceQuery(TestDocument::point, "1km", "POINT (13.0 51.0)") 37 | }.total shouldBe 0 38 | } 39 | } 40 | 41 | @Test 42 | fun shouldDoGridQuery() = coRun { 43 | onlyOn( 44 | "elasticsearch only feature that was introduced in recent 8.x releases", 45 | SearchEngineVariant.ES8, 46 | SearchEngineVariant.ES9, 47 | ) { 48 | geoTestFixture { index -> 49 | client.search(index) { 50 | query = GeoGridQuery(TestDocument::point) { 51 | geohash = "u33d" 52 | } 53 | }.total shouldBe 1 54 | client.search(index) { 55 | query = GeoGridQuery(TestDocument::point) { 56 | geohash = "sr3n" 57 | } 58 | }.total shouldBe 0 59 | } 60 | } 61 | } 62 | 63 | @Test 64 | fun shouldDoGeoShapeQuery() = coRun { 65 | geoTestFixture { index -> 66 | client.search(index) { 67 | query = GeoShapeQuery(TestDocument::point) { 68 | shape = Shape.Envelope(listOf(listOf(12.0, 53.0), listOf(14.0, 51.0))) 69 | } 70 | }.total shouldBe 1 71 | 72 | client.search(index) { 73 | query = GeoShapeQuery(TestDocument::point) { 74 | shape = Shape.Envelope(listOf(listOf(2.0, 53.0), listOf(4.0, 51.0))) 75 | } 76 | }.total shouldBe 0 77 | } 78 | } 79 | 80 | suspend fun geoTestFixture(block: suspend (String)->Unit) { 81 | testDocumentIndex { index -> 82 | 83 | client.bulk(target = index, refresh = Refresh.WaitFor) { 84 | // Fun fact, I contributed documentation 85 | // to Elasticsearch 1.x for geojson based searches back in 2013. The POI 86 | // used here refers to a coffee bar / vegan restaurant / and cocktail bar 87 | // that no longer exist that we used as our office while writing that documentation. 88 | // Somehow, the modern day Elastic documentation still uses this point. ;-) 89 | create(TestDocument("Wind und Wetter, Berlin, Germany", point = listOf(13.400544, 52.530286))) 90 | } 91 | block.invoke(index) 92 | } 93 | } 94 | } -------------------------------------------------------------------------------- /search-client/src/commonTest/kotlin/com/jillesvangurp/ktsearch/IlmTest.kt: -------------------------------------------------------------------------------- 1 | package com.jillesvangurp.ktsearch 2 | 3 | import com.jillesvangurp.searchdsls.SearchEngineVariant 4 | import io.kotest.assertions.throwables.shouldThrow 5 | import io.kotest.matchers.shouldBe 6 | import kotlin.test.Test 7 | import kotlin.time.Duration.Companion.hours 8 | 9 | class IlmTest: SearchTestBase() { 10 | @Test 11 | fun shouldSetUpIlmPolicy() = coRun { 12 | onlyOn("ilm only works on elasticsearch", 13 | SearchEngineVariant.ES7, 14 | SearchEngineVariant.ES8, 15 | SearchEngineVariant.ES9, 16 | ) { 17 | client.setIlmPolicy("my-ilm") { 18 | hot { 19 | actions { 20 | rollOver(2) 21 | } 22 | } 23 | warm { 24 | minAge(24.hours) 25 | actions { 26 | shrink(1) 27 | forceMerge(1) 28 | } 29 | } 30 | } 31 | println(client.getIlmPolicy("my-ilm")) 32 | client.deleteIlmPolicy("my-ilm") 33 | shouldThrow { 34 | client.getIlmPolicy("my-ilm") 35 | }.status shouldBe 404 36 | } 37 | } 38 | } -------------------------------------------------------------------------------- /search-client/src/commonTest/kotlin/com/jillesvangurp/ktsearch/IndexCreateTest.kt: -------------------------------------------------------------------------------- 1 | package com.jillesvangurp.ktsearch 2 | 3 | import io.kotest.matchers.shouldBe 4 | import kotlinx.serialization.json.* 5 | import kotlin.test.Test 6 | import kotlin.time.Duration.Companion.seconds 7 | 8 | class IndexCreateTest: SearchTestBase() { 9 | @Test 10 | fun createIndex() = coRun { 11 | val indexName = randomIndexName() 12 | val response = client.createIndex(indexName) { 13 | dynamicTemplate("test_fields") { 14 | match = "test*" 15 | mapping("text") { 16 | fields { 17 | keyword("keyword") 18 | } 19 | } 20 | } 21 | dynamicTemplate("more_fields") { 22 | match = "more*" 23 | mapping("keyword") 24 | } 25 | mappings(true) { 26 | keyword("foo") 27 | number("bar") 28 | objField("foo", dynamic = "true") 29 | } 30 | meta { 31 | this["foo"] = "bar" 32 | } 33 | settings { 34 | replicas = 0 35 | shards = 5 36 | refreshInterval = 31.seconds 37 | } 38 | } 39 | response.acknowledged shouldBe true 40 | client.getIndex(indexName).jsonObject(indexName).let { 41 | val mappings = it.jsonObject("mappings") 42 | mappings.jsonPrimitive("dynamic").booleanOrNull shouldBe true 43 | mappings.jsonObject("_meta").jsonPrimitive("foo").content shouldBe "bar" 44 | mappings.jsonArray("dynamic_templates").size shouldBe 2 45 | 46 | val settings = it.jsonObject("settings").jsonObject("index") 47 | settings.jsonPrimitive("number_of_replicas").intOrNull shouldBe 0 48 | settings.jsonPrimitive("number_of_shards").intOrNull shouldBe 5 49 | settings.jsonPrimitive("refresh_interval").content shouldBe "31s" 50 | } 51 | } 52 | 53 | private fun JsonObject.jsonPrimitive(key: String) = this.getValue(key).jsonPrimitive 54 | private fun JsonObject.jsonObject(key: String) = this.getValue(key).jsonObject 55 | private fun JsonObject.jsonArray(key: String) = this.getValue(key).jsonArray 56 | } 57 | 58 | -------------------------------------------------------------------------------- /search-client/src/commonTest/kotlin/com/jillesvangurp/ktsearch/IndexTemplateTest.kt: -------------------------------------------------------------------------------- 1 | package com.jillesvangurp.ktsearch 2 | 3 | import com.jillesvangurp.jsondsl.JsonDsl 4 | import io.kotest.matchers.shouldBe 5 | import kotlin.random.Random 6 | import kotlin.random.nextULong 7 | import kotlin.test.Test 8 | 9 | class IndexTemplateTest : SearchTestBase() { 10 | 11 | @Test 12 | fun shouldCreateDataStream() = coRun { 13 | 14 | val suffix = Random.nextULong() 15 | val settingsTemplateId = "test-settings-$suffix" 16 | val mappingsTemplateId = "test-mappings-$suffix" 17 | val templateId = "test-template-$suffix" 18 | val dataStreamName = "test-logs-$suffix" 19 | 20 | runCatching { client.deleteDataStream(dataStreamName) } 21 | runCatching { client.deleteIndexTemplate(templateId) } 22 | runCatching { client.deleteComponentTemplate(mappingsTemplateId) } 23 | runCatching { client.deleteComponentTemplate(settingsTemplateId) } 24 | 25 | client.updateComponentTemplate(settingsTemplateId) { 26 | settings { 27 | replicas = 4 28 | } 29 | } 30 | client.updateComponentTemplate(mappingsTemplateId) { 31 | mappings { 32 | text("name") 33 | keyword("category") 34 | } 35 | } 36 | client.createIndexTemplate(templateId) { 37 | indexPatterns = listOf("test-logs-$suffix*") 38 | dataStream = JsonDsl() 39 | composedOf = listOf(settingsTemplateId, mappingsTemplateId) 40 | } 41 | 42 | client.exists(dataStreamName) shouldBe false 43 | client.createDataStream(dataStreamName) 44 | client.exists(dataStreamName) shouldBe true 45 | client.deleteDataStream(dataStreamName) 46 | 47 | client.deleteIndexTemplate(templateId) 48 | client.deleteComponentTemplate(mappingsTemplateId) 49 | client.deleteComponentTemplate(settingsTemplateId) 50 | } 51 | } -------------------------------------------------------------------------------- /search-client/src/commonTest/kotlin/com/jillesvangurp/ktsearch/KnnSearchTest.kt: -------------------------------------------------------------------------------- 1 | package com.jillesvangurp.ktsearch 2 | 3 | import com.jillesvangurp.searchdsls.SearchEngineVariant 4 | import com.jillesvangurp.searchdsls.querydsl.KnnQuery 5 | import io.kotest.matchers.shouldBe 6 | import kotlinx.serialization.Serializable 7 | import kotlin.test.Test 8 | 9 | @Serializable 10 | data class KnnTestDoc(val name: String, val vector: List) 11 | 12 | class KnnSearchTest : SearchTestBase() { 13 | 14 | @Test 15 | fun shouldDoKnnSearch() = coRun { 16 | onlyOn( 17 | "knn only works with ES8", 18 | SearchEngineVariant.ES8, 19 | SearchEngineVariant.ES9, 20 | ) { 21 | val index = randomIndexName() 22 | client.createIndex(index) { 23 | mappings { 24 | keyword(KnnTestDoc::name) 25 | denseVector(KnnTestDoc::vector, 3, index = true) 26 | } 27 | } 28 | client.bulk(target = index) { 29 | create(KnnTestDoc("1", listOf(0.1, 0.3, 0.9))) 30 | create(KnnTestDoc("2", listOf(0.09, 0.31, 0.90))) 31 | create(KnnTestDoc("3", listOf(0.9, 0.1, 0.5))) 32 | create(KnnTestDoc("4", listOf(0.1, 0.1, 0.1))) 33 | } 34 | 35 | client.search(index) { 36 | knn = KnnQuery(KnnTestDoc::vector, listOf(0.93, 0.11, 0.48)) 37 | }.parseHits(KnnTestDoc.serializer()).map { it.name }.let { ids -> 38 | ids.size shouldBe 4 // it will rank all of them 39 | ids.first() shouldBe "3" // 3 is the closest 40 | } 41 | } 42 | } 43 | } -------------------------------------------------------------------------------- /search-client/src/commonTest/kotlin/com/jillesvangurp/ktsearch/NestedQueryTest.kt: -------------------------------------------------------------------------------- 1 | package com.jillesvangurp.ktsearch 2 | 3 | import com.jillesvangurp.ktsearch.repository.repository 4 | import com.jillesvangurp.searchdsls.mappingdsl.IndexSettingsAndMappingsDSL 5 | import com.jillesvangurp.searchdsls.querydsl.ScoreMode 6 | import com.jillesvangurp.searchdsls.querydsl.nested 7 | import com.jillesvangurp.searchdsls.querydsl.range 8 | import com.jillesvangurp.serializationext.DEFAULT_JSON 9 | import com.jillesvangurp.serializationext.DEFAULT_PRETTY_JSON 10 | import io.kotest.matchers.shouldBe 11 | import kotlinx.serialization.SerialName 12 | import kotlinx.serialization.Serializable 13 | import kotlin.test.Test 14 | 15 | class NestedQueryTest : SearchTestBase() { 16 | @Test 17 | fun shouldCreatedMappingWithNestedFieldAndQuery() = coRun { 18 | val index = randomIndexName() 19 | client.createIndex(index, NestedTestDocument.mapping) 20 | println(NestedTestDocument.mapping) 21 | 22 | val repo = client.repository(index,NestedTestDocument.serializer()) 23 | 24 | repo.index(NestedTestDocument("1", listOf( 25 | TestDocument("1.1", number=1), 26 | TestDocument("1.2", number=2), 27 | TestDocument("1.3", number=3), 28 | ))) 29 | repo.index(NestedTestDocument("2", listOf( 30 | TestDocument("2.3", number=3), 31 | TestDocument("2.4", number=4), 32 | ))) 33 | 34 | val r0 = repo.search { } 35 | r0.total shouldBe 2 36 | val r1 = repo.search { 37 | query = nested { 38 | path = "test_docs" 39 | query = range("test_docs.number") { 40 | gt = 3 41 | lt = 5 42 | } 43 | scoreMode = ScoreMode.none 44 | ignoreUnmapped = true 45 | } 46 | } 47 | r1.total shouldBe 1 48 | } 49 | } 50 | 51 | 52 | @Serializable 53 | data class NestedTestDocument( 54 | val name: String, 55 | @SerialName("test_docs") 56 | val testDocs: List 57 | ) { 58 | companion object { 59 | val mapping = IndexSettingsAndMappingsDSL().apply { 60 | mappings(dynamicEnabled = false) { 61 | text(NestedTestDocument::name) 62 | nestedField("test_docs") { 63 | text(TestDocument::name) 64 | number(TestDocument::number) 65 | } 66 | } 67 | } 68 | } 69 | 70 | @Suppress("unused") 71 | fun json(pretty: Boolean = false): String { 72 | return if (pretty) 73 | DEFAULT_PRETTY_JSON.encodeToString(serializer(), this) 74 | else 75 | DEFAULT_JSON.encodeToString(serializer(), this) 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /search-client/src/commonTest/kotlin/com/jillesvangurp/ktsearch/ParentChildQueryTest.kt: -------------------------------------------------------------------------------- 1 | package com.jillesvangurp.ktsearch 2 | 3 | import com.jillesvangurp.ktsearch.repository.IndexRepository 4 | import com.jillesvangurp.ktsearch.repository.repository 5 | import com.jillesvangurp.searchdsls.mappingdsl.IndexSettingsAndMappingsDSL 6 | import com.jillesvangurp.searchdsls.querydsl.* 7 | import io.kotest.matchers.shouldBe 8 | import kotlinx.serialization.Serializable 9 | import kotlin.test.Test 10 | 11 | class ParentChildQueryTest: SearchTestBase() { 12 | 13 | @Serializable 14 | data class JoinField(val name: String, val parent: String? = null) 15 | 16 | @Serializable 17 | data class TestDoc(val name: String, val joinField: JoinField) 18 | 19 | private val parent1 = "parent-1" 20 | private val parent2 = "parent-2" 21 | private val child1 = "child-1" 22 | 23 | @Test 24 | fun shouldCreateJoinMappingAndQueryForDocumentsWithChildren() = coRun { 25 | val repo = setupIndex() 26 | 27 | val emptySearch = repo.search { } 28 | emptySearch.total shouldBe 3 29 | 30 | val childrenSearch = repo.search { 31 | query = hasChild("child") { 32 | query = matchAll() 33 | } 34 | } 35 | childrenSearch.total shouldBe 1 36 | childrenSearch.hits?.hits?.first()?.id shouldBe parent1 37 | } 38 | 39 | @Test 40 | fun shouldCreateJoinMappingAndQueryForDocumentsWithParents() = coRun { 41 | val repo = setupIndex() 42 | 43 | val emptySearch = repo.search { } 44 | emptySearch.total shouldBe 3 45 | 46 | val parentSearch = repo.search { 47 | query = hasParent("parent") { 48 | query = matchAll() 49 | } 50 | } 51 | parentSearch.total shouldBe 1 52 | parentSearch.hits?.hits?.first()?.id shouldBe child1 53 | } 54 | 55 | @Test 56 | fun shouldQueryForAndReturnChildrenInInnerHits() = coRun { 57 | val repo = setupIndex() 58 | 59 | val emptySearch = repo.search { } 60 | emptySearch.total shouldBe 3 61 | 62 | val childrenSearch = repo.search { 63 | query = hasChild("child") { 64 | query = matchAll() 65 | innerHits() 66 | } 67 | } 68 | val firstHit = childrenSearch.hits?.hits?.first() 69 | val innerHits = firstHit?.innerHits?.get("child")?.hits 70 | 71 | childrenSearch.total shouldBe 1 72 | firstHit?.id shouldBe parent1 73 | innerHits?.total?.value shouldBe 1 74 | innerHits?.hits?.first()?.id shouldBe child1 75 | } 76 | 77 | @Test 78 | fun shouldReturnChildrenWithParentId() = coRun { 79 | val repo = setupIndex() 80 | 81 | val emptySearch = repo.search { } 82 | emptySearch.total shouldBe 3 83 | 84 | val childrenSearch = repo.search { 85 | query = parentId("child", parent1) 86 | } 87 | 88 | childrenSearch.total shouldBe 1 89 | childrenSearch.hits?.hits?.first()?.id shouldBe child1 90 | } 91 | 92 | private suspend fun setupIndex(): IndexRepository { 93 | val index = randomIndexName() 94 | val mapping = IndexSettingsAndMappingsDSL().apply { 95 | mappings(dynamicEnabled = false) { 96 | text("name") 97 | join("joinField") { 98 | relations("parent" to listOf("child")) 99 | } 100 | } 101 | } 102 | client.createIndex(index, mapping) 103 | 104 | val repo = client.repository(index, TestDoc.serializer()) 105 | 106 | repo.index(TestDoc(name = parent1, joinField = JoinField(name = "parent")), id = parent1) 107 | repo.index(TestDoc(name = parent2, joinField = JoinField(name = "parent")), id = parent2) 108 | repo.index(TestDoc(name = child1, joinField = JoinField(name = "child", parent = parent1)), id = child1, routing = parent1) 109 | 110 | return repo 111 | } 112 | } -------------------------------------------------------------------------------- /search-client/src/commonTest/kotlin/com/jillesvangurp/ktsearch/ScrollTest.kt: -------------------------------------------------------------------------------- 1 | package com.jillesvangurp.ktsearch 2 | 3 | import io.kotest.assertions.throwables.shouldThrow 4 | import io.kotest.matchers.shouldBe 5 | import io.kotest.matchers.string.shouldContain 6 | import kotlin.random.Random 7 | import kotlin.random.nextULong 8 | import kotlin.test.Test 9 | 10 | class ScrollTest : SearchTestBase() { 11 | 12 | @Test 13 | fun deleteFailsWhenIndexIsNotAvailable() = coRun { 14 | val scrollId = "scrollId-${Random.nextULong()}" 15 | val exception = shouldThrow { 16 | client.deleteScroll(scrollId) 17 | } 18 | exception.status shouldBe 400 19 | exception.message shouldContain "illegal_argument_exception" 20 | } 21 | 22 | } -------------------------------------------------------------------------------- /search-client/src/commonTest/kotlin/com/jillesvangurp/ktsearch/SearchResponseDefaultsTest.kt: -------------------------------------------------------------------------------- 1 | package com.jillesvangurp.ktsearch 2 | 3 | import com.jillesvangurp.serializationext.DEFAULT_JSON 4 | import io.kotest.matchers.shouldBe 5 | import kotlin.test.Test 6 | 7 | class SearchResponseDefaultsTest { 8 | 9 | @Test 10 | fun shouldParseDefaults() { 11 | val parsed = DEFAULT_JSON.decodeFromString(ExtendedStatsBucketResult.serializer(),"{}") 12 | parsed.min shouldBe 0.0 13 | parsed.stdDeviationBounds.lower shouldBe 0.0 14 | } 15 | } -------------------------------------------------------------------------------- /search-client/src/commonTest/kotlin/com/jillesvangurp/ktsearch/SearchTestBase.kt: -------------------------------------------------------------------------------- 1 | package com.jillesvangurp.ktsearch 2 | 3 | import com.jillesvangurp.ktsearch.repository.repository 4 | import com.jillesvangurp.searchdsls.SearchEngineVariant 5 | import io.github.oshai.kotlinlogging.KotlinLogging 6 | import kotlin.random.Random 7 | import kotlin.random.nextULong 8 | import kotlin.time.Duration 9 | import kotlin.time.Duration.Companion.hours 10 | import kotlin.time.Duration.Companion.seconds 11 | 12 | 13 | expect fun coRun(timeout: Duration = 30.seconds, block: suspend () -> Unit) 14 | 15 | private val logger = KotlinLogging.logger { } 16 | private var versionInfo: SearchEngineInformation?=null 17 | 18 | /** 19 | * Base class for search tests. Use together with the gradle compose plugin. 20 | * 21 | * It will talk to whatever runs on port 9999. That's intentionally different 22 | * from the default port, so you don't fill your production cluster with test data. 23 | */ 24 | open class SearchTestBase { 25 | // make sure we use the same client in all tests 26 | val client by lazy { SearchClient(sharedClient) } 27 | 28 | suspend fun onlyOn(message: String, vararg variants: SearchEngineVariant, block: suspend () -> Unit) { 29 | if(versionInfo==null) { 30 | versionInfo = client.root() 31 | } 32 | val variant = versionInfo!!.variantInfo.variant 33 | if(variants.contains(variant)) { 34 | block.invoke() 35 | } else { 36 | logger.info { "Skipping test active variant $variant is not supported. Supported [${variants.joinToString(",")}]. $message" } 37 | } 38 | } 39 | 40 | fun randomIndexName() = "index-${Random.nextULong()}" 41 | 42 | suspend fun testDocumentIndex(block: suspend (String) -> Unit) { 43 | val index = randomIndexName() 44 | TestDocument.mapping.let { 45 | client.createIndex(index,it) 46 | }.index 47 | block.invoke(index) 48 | client.deleteIndex(index) 49 | } 50 | 51 | val repo by lazy { 52 | client.repository(randomIndexName(),TestDocument.serializer()) 53 | } 54 | 55 | companion object { 56 | private val sharedClient by lazy { 57 | val nodes = arrayOf( 58 | Node("127.0.0.1", 9999), 59 | Node("localhost", 9999) 60 | ) 61 | KtorRestClient( 62 | nodes = nodes, 63 | client = defaultKtorHttpClient(true) {}, 64 | // sniffing is a bit weird in docker, publish address is not always reachable 65 | nodeSelector = SniffingNodeSelector(initialNodes = nodes, maxNodeAge = 5.hours) 66 | ) 67 | } 68 | } 69 | } 70 | 71 | -------------------------------------------------------------------------------- /search-client/src/commonTest/kotlin/com/jillesvangurp/ktsearch/SniffingNodeSelectorTest.kt: -------------------------------------------------------------------------------- 1 | package com.jillesvangurp.ktsearch 2 | 3 | import io.kotest.assertions.nondeterministic.eventually 4 | import io.kotest.matchers.shouldBe 5 | import kotlinx.coroutines.async 6 | import kotlinx.coroutines.awaitAll 7 | import kotlinx.coroutines.coroutineScope 8 | import kotlinx.coroutines.delay 9 | import kotlin.random.Random 10 | import kotlin.random.nextULong 11 | import kotlin.test.Test 12 | import kotlin.time.Duration.Companion.seconds 13 | 14 | class SniffingNodeSelectorTest : SearchTestBase() { 15 | 16 | @Test 17 | fun shouldPickSameNodeGivenSameAffinity() = coRun { 18 | val ids = (0..5).map { 19 | "thread-${Random.nextULong()}" 20 | } 21 | coroutineScope { 22 | eventually(duration = 10.seconds) { 23 | // this fails occasionally bug with sniffing kicking in and clearing the affinity map 24 | // so use eventually to work around this 25 | 26 | val firstSelectedHosts = ids.map { id -> 27 | async(AffinityId(id)) { 28 | id to client.restClient.nextNode().host 29 | } 30 | }.awaitAll().toMap() 31 | delay(1000) 32 | ids.map { id -> 33 | async(AffinityId(id)) { 34 | id to client.restClient.nextNode().host 35 | } 36 | }.awaitAll().forEach { (id, host) -> 37 | host shouldBe firstSelectedHosts[id] 38 | } 39 | } 40 | } 41 | } 42 | } -------------------------------------------------------------------------------- /search-client/src/commonTest/kotlin/com/jillesvangurp/ktsearch/SpecializedQueriesTest.kt: -------------------------------------------------------------------------------- 1 | package com.jillesvangurp.ktsearch 2 | 3 | import com.jillesvangurp.searchdsls.querydsl.distanceFeature 4 | import com.jillesvangurp.searchdsls.querydsl.rankFeature 5 | import io.kotest.assertions.assertSoftly 6 | import io.kotest.matchers.shouldBe 7 | import kotlinx.datetime.Clock 8 | import kotlin.test.Test 9 | import kotlin.time.Duration.Companion.days 10 | 11 | class SpecializedQueriesTest : SearchTestBase() { 12 | 13 | @Test 14 | fun shouldRankOnDistance() = coRun { 15 | testDocumentIndex { index -> 16 | 17 | client.bulk(target = index, refresh = Refresh.WaitFor) { 18 | create( 19 | TestDocument( 20 | name = "p1", 21 | point = listOf(12.0, 50.0), 22 | timestamp = Clock.System.now().minus(10.days) 23 | ) 24 | ) 25 | create( 26 | TestDocument( 27 | name = "p2", 28 | point = listOf(13.0, 52.0), 29 | timestamp = Clock.System.now().minus(5.days) 30 | ) 31 | ) 32 | } 33 | 34 | client.search(target = index) { 35 | query = distanceFeature(TestDocument::timestamp, "30d", "now-40d") 36 | }.hits?.hits?.first()?.parseHit()!!.name shouldBe "p1" 37 | client.search(target = index) { 38 | query = distanceFeature(TestDocument::point, "10km", listOf(14.0, 52.0)) 39 | }.hits?.hits?.first()?.parseHit()!!.name shouldBe "p2" 40 | } 41 | } 42 | 43 | @Test 44 | fun shouldRankFeature() = coRun { 45 | testDocumentIndex { index -> 46 | 47 | client.bulk(target = index, refresh = Refresh.WaitFor) { 48 | create( 49 | TestDocument( 50 | name = "p1", 51 | feature = 20 52 | ) 53 | ) 54 | create( 55 | TestDocument( 56 | name = "p2", 57 | feature = 100 58 | ) 59 | ) 60 | } 61 | 62 | client.search(target = index) { 63 | query = rankFeature(TestDocument::feature) 64 | }.hits?.hits?.first()?.parseHit()!!.name shouldBe "p2" 65 | client.search(target = index) { 66 | query = rankFeature(TestDocument::feature) { 67 | linear() 68 | } 69 | }.hits?.hits?.first()?.parseHit()!!.name shouldBe "p2" 70 | client.search(target = index) { 71 | query = rankFeature(TestDocument::feature) { 72 | saturation(pivot = 2.0) 73 | } 74 | }.hits?.hits?.first()?.parseHit()!!.name shouldBe "p2" 75 | client.search(target = index) { 76 | query = rankFeature(TestDocument::feature) { 77 | log(2.0) 78 | } 79 | }.hits?.hits?.first()?.parseHit()!!.name shouldBe "p2" 80 | client.search(target = index) { 81 | query = rankFeature(TestDocument::feature) { 82 | sigmoid(pivot = 2.0, exponent = 0.8) 83 | } 84 | }.hits?.hits?.first()?.parseHit()!!.name shouldBe "p2" 85 | 86 | } 87 | } 88 | } -------------------------------------------------------------------------------- /search-client/src/commonTest/kotlin/com/jillesvangurp/ktsearch/repository/IndexRepositoryTest.kt: -------------------------------------------------------------------------------- 1 | package com.jillesvangurp.ktsearch.repository 2 | 3 | import com.jillesvangurp.ktsearch.SearchTestBase 4 | import com.jillesvangurp.ktsearch.TestDocument 5 | import com.jillesvangurp.ktsearch.coRun 6 | import com.jillesvangurp.ktsearch.total 7 | import io.kotest.matchers.ints.shouldBeGreaterThan 8 | import io.kotest.matchers.shouldBe 9 | import kotlin.test.Test 10 | import kotlin.time.Duration.Companion.seconds 11 | 12 | class IndexRepositoryTest : SearchTestBase() { 13 | 14 | @Test 15 | fun shouldDoCrudWithRepo() = coRun { 16 | val d = repo.index(TestDocument("1")) 17 | d.shards.successful shouldBeGreaterThan 0 18 | val (doc,_)= repo.get(d.id) 19 | doc.name shouldBe "1" 20 | 21 | val (doc2,_) =repo.update(d.id, block = { 22 | it.copy(name="2") 23 | }, retryDelay = 1.seconds) 24 | doc2.name shouldBe "2" 25 | val (doc3,_)= repo.get(d.id) 26 | doc3 shouldBe doc2 27 | } 28 | 29 | @Test 30 | fun shouldNotFailOnWrongId() = coRun { 31 | repo.getDocument("idontexist") shouldBe null 32 | } 33 | 34 | @Test 35 | fun shouldDoBulkWithRepo() = coRun{ 36 | repo.createIndex {} 37 | repo.bulk { 38 | index(TestDocument("1").json()) 39 | index(TestDocument("2").json()) 40 | index(TestDocument("3").json()) 41 | } 42 | val r = repo.search { } 43 | r.total shouldBe 3 44 | } 45 | 46 | @Test 47 | fun shouldDoBulkUpdatesWithOptimisticLocking() = coRun{ 48 | repo.createIndex {} 49 | 50 | repo.bulk { 51 | create(TestDocument("1").json(), id = "1") 52 | create(TestDocument("2").json(), id = "2") 53 | } 54 | 55 | repo.bulk(callBack = null) { 56 | update(repo.get("1").second) { 57 | it.copy(name="Changed 1") 58 | } 59 | // this one should trigger a retry 60 | update("2", TestDocument("2"),42,42) { 61 | it.copy(name = "Changed 2") 62 | } 63 | } 64 | 65 | repo.get("1").first.name shouldBe "Changed 1" 66 | repo.get("2").first.name shouldBe "Changed 2" 67 | } 68 | } -------------------------------------------------------------------------------- /search-client/src/commonTest/kotlin/com/jillesvangurp/ktsearch/test-fixture.kt: -------------------------------------------------------------------------------- 1 | package com.jillesvangurp.ktsearch 2 | 3 | import com.jillesvangurp.searchdsls.mappingdsl.IndexSettingsAndMappingsDSL 4 | import com.jillesvangurp.serializationext.DEFAULT_JSON 5 | import com.jillesvangurp.serializationext.DEFAULT_PRETTY_JSON 6 | import kotlinx.datetime.Clock 7 | import kotlinx.datetime.Instant 8 | import kotlinx.serialization.EncodeDefault 9 | import kotlinx.serialization.ExperimentalSerializationApi 10 | import kotlinx.serialization.Serializable 11 | import kotlin.random.Random 12 | 13 | @OptIn(ExperimentalSerializationApi::class) 14 | @Serializable 15 | data class TestDocument( 16 | val name: String, 17 | val description: String? = null, 18 | val number: Long? = null, 19 | val tags: List? = null, 20 | val point: List? = null, 21 | val id : Long = Random.nextLong(), 22 | @EncodeDefault // default to same time so tests depending on document equality don't fail 23 | val timestamp: Instant = Instant.fromEpochMilliseconds(6666666666), 24 | val feature: Int = 42 25 | ) { 26 | companion object { 27 | val mapping = IndexSettingsAndMappingsDSL().apply { 28 | mappings(dynamicEnabled = false) { 29 | number(TestDocument::id) 30 | text(TestDocument::name) 31 | text(TestDocument::description) 32 | number(TestDocument::number) 33 | keyword(TestDocument::tags) 34 | geoPoint(TestDocument::point) 35 | date(TestDocument::timestamp) 36 | rankFeature(TestDocument::feature) 37 | } 38 | } 39 | } 40 | 41 | fun json(pretty: Boolean = false): String { 42 | return if (pretty) 43 | DEFAULT_PRETTY_JSON.encodeToString(serializer(), this) 44 | else 45 | DEFAULT_JSON.encodeToString(serializer(), this) 46 | } 47 | } 48 | 49 | -------------------------------------------------------------------------------- /search-client/src/iosMain/kotlin/com/jillesvangurp/ktsearch/KtorRestClient.ios.kt: -------------------------------------------------------------------------------- 1 | package com.jillesvangurp.ktsearch 2 | 3 | 4 | import io.ktor.client.HttpClient 5 | import io.ktor.client.HttpClientConfig 6 | import io.ktor.client.engine.darwin.Darwin 7 | import io.ktor.client.plugins.auth.Auth 8 | import io.ktor.client.plugins.auth.providers.BasicAuthCredentials 9 | import io.ktor.client.plugins.auth.providers.basic 10 | import io.ktor.client.plugins.logging.LogLevel 11 | import io.ktor.client.plugins.logging.Logging 12 | import io.ktor.http.headers 13 | 14 | actual fun defaultKtorHttpClient( 15 | logging: Boolean, 16 | user: String?, 17 | password: String?, 18 | elasticApiKey: String?, 19 | block: HttpClientConfig<*>.()->Unit 20 | ): HttpClient { 21 | return HttpClient(Darwin) { 22 | engine { 23 | pipelining = true 24 | configureRequest { 25 | setAllowsCellularAccess(true) 26 | } 27 | } 28 | block(this) 29 | } 30 | } -------------------------------------------------------------------------------- /search-client/src/iosMain/kotlin/com/jillesvangurp/ktsearch/RoundRobinNodeSelector.ios.kt: -------------------------------------------------------------------------------- 1 | package com.jillesvangurp.ktsearch 2 | 3 | actual fun simpleIndexProvider(initialIndex: Int): IndexProvider = object : IndexProvider { 4 | private var index = 0 5 | 6 | override fun get(): Int = index 7 | 8 | override fun set(value: Int) { 9 | index = value 10 | } 11 | } -------------------------------------------------------------------------------- /search-client/src/iosMain/kotlin/com/jillesvangurp/ktsearch/SniffingNodeSelector.ios.kt: -------------------------------------------------------------------------------- 1 | package com.jillesvangurp.ktsearch 2 | 3 | /** 4 | * On JVM this will return the current thread name, otherwise this will return null and pick a random node 5 | */ 6 | actual fun threadId(): String? = null -------------------------------------------------------------------------------- /search-client/src/jsMain/kotlin/com/jillesvangurp/ktsearch/defaultHttpClient.kt: -------------------------------------------------------------------------------- 1 | package com.jillesvangurp.ktsearch 2 | 3 | import io.ktor.client.HttpClient 4 | import io.ktor.client.HttpClientConfig 5 | import io.ktor.client.engine.js.Js 6 | 7 | actual fun defaultKtorHttpClient( 8 | logging: Boolean, 9 | user: String?, 10 | password: String?, 11 | elasticApiKey: String?, 12 | block: HttpClientConfig<*>.()->Unit 13 | ): HttpClient { 14 | return HttpClient(Js) { 15 | block(this) 16 | } 17 | } -------------------------------------------------------------------------------- /search-client/src/jsMain/kotlin/com/jillesvangurp/ktsearch/nodeselectors.kt: -------------------------------------------------------------------------------- 1 | package com.jillesvangurp.ktsearch 2 | 3 | actual fun simpleIndexProvider(initialIndex: Int): IndexProvider = object: IndexProvider { 4 | private var index: Int = initialIndex 5 | 6 | override fun get(): Int = index 7 | 8 | override fun set(value: Int) { 9 | index = value 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /search-client/src/jsMain/kotlin/com/jillesvangurp/ktsearch/threadId.kt: -------------------------------------------------------------------------------- 1 | package com.jillesvangurp.ktsearch 2 | 3 | actual fun threadId(): String? { 4 | // javascript has no threads, use AffinityId scope if you want different nodes 5 | return null 6 | } -------------------------------------------------------------------------------- /search-client/src/jsTest/kotlin/com/jillesvangurp/ktsearch/coTest.kt: -------------------------------------------------------------------------------- 1 | package com.jillesvangurp.ktsearch 2 | 3 | import kotlinx.coroutines.* 4 | import kotlin.time.Duration 5 | 6 | @OptIn(DelicateCoroutinesApi::class) 7 | actual fun coRun(timeout: Duration, block: suspend () -> Unit) { 8 | GlobalScope.async { 9 | withTimeout(timeout) { 10 | block.invoke() 11 | } 12 | }.asPromise() 13 | } 14 | 15 | 16 | -------------------------------------------------------------------------------- /search-client/src/jvmMain/kotlin/com/jillesvangurp/ktsearch/defaultHttpClient.kt: -------------------------------------------------------------------------------- 1 | package com.jillesvangurp.ktsearch 2 | 3 | import io.ktor.client.HttpClient 4 | import io.ktor.client.HttpClientConfig 5 | import io.ktor.client.engine.cio.CIO 6 | import io.ktor.client.engine.cio.endpoint 7 | import io.ktor.client.engine.java.Java 8 | import java.time.Duration 9 | 10 | 11 | actual fun defaultKtorHttpClient( 12 | logging: Boolean, 13 | user: String?, 14 | password: String?, 15 | elasticApiKey: String?, 16 | block: HttpClientConfig<*>.()->Unit 17 | ): HttpClient { 18 | // We experienced some threading issues with CIO. Java engine seems more stable currently. 19 | return ktorClientWithJavaEngine(logging, user, password, elasticApiKey, block) 20 | } 21 | fun ktorClientWithJavaEngine( 22 | logging: Boolean, 23 | user: String?, 24 | password: String?, 25 | elasticApiKey: String?, 26 | block: HttpClientConfig<*>.() -> Unit, 27 | ): HttpClient { 28 | return HttpClient(Java) { 29 | // note the Java engine uses the IO dispatcher 30 | // you may want to bump the number of threads 31 | // for that by setting the system property 32 | // kotlinx.coroutines.io.parallelism=128 33 | engine { 34 | config { 35 | connectTimeout(Duration.ofSeconds(5)) 36 | } 37 | pipelining = true 38 | } 39 | block(this) 40 | } 41 | } 42 | 43 | fun ktorClientWithCIOEngine( 44 | logging: Boolean, 45 | user: String?, 46 | password: String?, 47 | elasticApiKey: String?, 48 | block: HttpClientConfig<*>.() -> Unit, 49 | ) = HttpClient(CIO) { 50 | // there are some known issues with using CIO and weird EOF errors in combination with pipelining 51 | engine { 52 | maxConnectionsCount = 100 53 | endpoint { 54 | keepAliveTime = 100_000 55 | connectTimeout = 5_000 56 | requestTimeout = 30_000 57 | connectAttempts = 3 58 | } 59 | } 60 | block(this) 61 | } -------------------------------------------------------------------------------- /search-client/src/jvmMain/kotlin/com/jillesvangurp/ktsearch/nodeselectors.kt: -------------------------------------------------------------------------------- 1 | package com.jillesvangurp.ktsearch 2 | 3 | import java.util.concurrent.atomic.AtomicInteger 4 | 5 | actual fun simpleIndexProvider(initialIndex: Int): IndexProvider = object: IndexProvider { 6 | // use AtomicInteger for thread safety 7 | private val index = AtomicInteger(initialIndex) 8 | 9 | override fun get(): Int = index.get() 10 | 11 | override fun set(value: Int) = index.set(value) 12 | } 13 | -------------------------------------------------------------------------------- /search-client/src/jvmMain/kotlin/com/jillesvangurp/ktsearch/threadId.kt: -------------------------------------------------------------------------------- 1 | package com.jillesvangurp.ktsearch 2 | 3 | actual fun threadId(): String? = Thread.currentThread().name -------------------------------------------------------------------------------- /search-client/src/jvmTest/kotlin/com/jillesvangurp/ktsearch/AppLogTest.kt: -------------------------------------------------------------------------------- 1 | package com.jillesvangurp.ktsearch 2 | 3 | import io.kotest.matchers.shouldNotBe 4 | import kotlinx.coroutines.runBlocking 5 | import java.io.FileInputStream 6 | import java.util.* 7 | 8 | class AppLogTest { 9 | 10 | // don't talk to es cloud in default test runs 11 | // @Test 12 | fun shouldConnectToEsAppLoggCluster() { 13 | // use this to test connectivity with elastic cloud 14 | // we've had this break a few times after changes in ktor client 15 | // put your properties in local.properties, this file is git ignored and 16 | // should not be committed 17 | val properties = Properties() 18 | properties.load(FileInputStream("../local.properties")) 19 | 20 | val host=properties.getProperty("esHost") 21 | val port=properties.getProperty("esPort").toInt() 22 | val https=properties.getProperty("esHttps").toBoolean() 23 | val user=properties.getProperty("esUser") 24 | val password = properties.getProperty("esPassword") 25 | password shouldNotBe null 26 | val searchClient = SearchClient(KtorRestClient( 27 | host = host, 28 | port = port, 29 | user = user, 30 | password = password, 31 | https = https, 32 | logging = true 33 | )) 34 | runBlocking { 35 | // should not throw exceptions because of authorization 36 | println(searchClient.engineInfo().version) 37 | } 38 | } 39 | } -------------------------------------------------------------------------------- /search-client/src/jvmTest/kotlin/com/jillesvangurp/ktsearch/coTest.kt: -------------------------------------------------------------------------------- 1 | package com.jillesvangurp.ktsearch 2 | 3 | import kotlinx.coroutines.runBlocking 4 | import kotlinx.coroutines.withTimeout 5 | import kotlin.time.Duration 6 | 7 | actual fun coRun(timeout: Duration, block: suspend () -> Unit) { 8 | runBlocking { 9 | withTimeout(timeout) { 10 | block.invoke() 11 | } 12 | } 13 | } -------------------------------------------------------------------------------- /search-client/src/linuxMain/kotlin/com/jillesvangurp/ktsearch/KtorRestClient.linux.kt: -------------------------------------------------------------------------------- 1 | package com.jillesvangurp.ktsearch 2 | 3 | import io.ktor.client.HttpClient 4 | import io.ktor.client.HttpClientConfig 5 | import io.ktor.client.engine.curl.Curl 6 | 7 | // FIXME intellij weirdness with this class on mac; none of the ktor stuff is found somehow 8 | // of course it builds fine with gradle; no idea if this is actually usable 9 | actual fun defaultKtorHttpClient( 10 | logging: Boolean, 11 | user: String?, 12 | password: String?, 13 | elasticApiKey: String?, 14 | block: HttpClientConfig<*>.()->Unit 15 | ): HttpClient { 16 | return HttpClient(Curl) { 17 | block(this) 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /search-client/src/linuxMain/kotlin/com/jillesvangurp/ktsearch/RoundRobinNodeSelector.linux.kt: -------------------------------------------------------------------------------- 1 | package com.jillesvangurp.ktsearch 2 | 3 | actual fun simpleIndexProvider(initialIndex: Int): IndexProvider = object : IndexProvider { 4 | private var index = 0 5 | 6 | override fun get(): Int = index 7 | 8 | override fun set(value: Int) { 9 | index = value 10 | } 11 | } -------------------------------------------------------------------------------- /search-client/src/linuxMain/kotlin/com/jillesvangurp/ktsearch/SniffingNodeSelector.linux.kt: -------------------------------------------------------------------------------- 1 | package com.jillesvangurp.ktsearch 2 | 3 | /** 4 | * On JVM this will return the current thread name, otherwise this will return null and pick a random node 5 | */ 6 | actual fun threadId(): String? = null -------------------------------------------------------------------------------- /search-client/src/macosMain/kotlin/com/jillesvangurp/ktsearch/KtorRestClient.macos.kt: -------------------------------------------------------------------------------- 1 | package com.jillesvangurp.ktsearch 2 | 3 | 4 | import io.ktor.client.HttpClient 5 | import io.ktor.client.HttpClientConfig 6 | import io.ktor.client.engine.darwin.Darwin 7 | import io.ktor.client.plugins.auth.Auth 8 | import io.ktor.client.plugins.auth.providers.BasicAuthCredentials 9 | import io.ktor.client.plugins.auth.providers.basic 10 | import io.ktor.client.plugins.logging.LogLevel 11 | import io.ktor.client.plugins.logging.Logging 12 | import io.ktor.http.headers 13 | 14 | actual fun defaultKtorHttpClient( 15 | logging: Boolean, 16 | user: String?, 17 | password: String?, 18 | elasticApiKey: String?, 19 | block: HttpClientConfig<*>.()->Unit 20 | ): HttpClient { 21 | return HttpClient(Darwin) { 22 | engine { 23 | pipelining = true 24 | configureRequest { 25 | setAllowsCellularAccess(true) 26 | } 27 | } 28 | block(this) 29 | } 30 | } -------------------------------------------------------------------------------- /search-client/src/macosMain/kotlin/com/jillesvangurp/ktsearch/RoundRobinNodeSelector.macos.kt: -------------------------------------------------------------------------------- 1 | package com.jillesvangurp.ktsearch 2 | 3 | actual fun simpleIndexProvider(initialIndex: Int): IndexProvider = object : IndexProvider { 4 | private var index = 0 5 | 6 | override fun get(): Int = index 7 | 8 | override fun set(value: Int) { 9 | index = value 10 | } 11 | } -------------------------------------------------------------------------------- /search-client/src/macosMain/kotlin/com/jillesvangurp/ktsearch/SniffingNodeSelector.macos.kt: -------------------------------------------------------------------------------- 1 | package com.jillesvangurp.ktsearch 2 | 3 | /** 4 | * On JVM this will return the current thread name, otherwise this will return null and pick a random node 5 | */ 6 | actual fun threadId(): String? = null -------------------------------------------------------------------------------- /search-client/src/mingwMain/kotlin/com/jillesvangurp/ktsearch/KtorRestClient.mingw.kt: -------------------------------------------------------------------------------- 1 | package com.jillesvangurp.ktsearch 2 | 3 | import io.ktor.client.HttpClient 4 | import io.ktor.client.HttpClientConfig 5 | import io.ktor.client.engine.curl.Curl 6 | import io.ktor.client.plugins.auth.Auth 7 | import io.ktor.client.plugins.auth.providers.BasicAuthCredentials 8 | import io.ktor.client.plugins.auth.providers.basic 9 | import io.ktor.client.plugins.logging.LogLevel 10 | import io.ktor.client.plugins.logging.Logging 11 | import io.ktor.http.headers 12 | 13 | actual fun defaultKtorHttpClient( 14 | logging: Boolean, 15 | user: String?, 16 | password: String?, 17 | elasticApiKey: String?, 18 | block: HttpClientConfig<*>.()->Unit 19 | ): HttpClient { 20 | return HttpClient(Curl) { 21 | block(this) 22 | } 23 | } -------------------------------------------------------------------------------- /search-client/src/mingwMain/kotlin/com/jillesvangurp/ktsearch/RoundRobinNodeSelector.mingw.kt: -------------------------------------------------------------------------------- 1 | package com.jillesvangurp.ktsearch 2 | 3 | actual fun simpleIndexProvider(initialIndex: Int): IndexProvider = object : IndexProvider { 4 | private var index = 0 5 | 6 | override fun get(): Int = index 7 | 8 | override fun set(value: Int) { 9 | index = value 10 | } 11 | } -------------------------------------------------------------------------------- /search-client/src/mingwMain/kotlin/com/jillesvangurp/ktsearch/SniffingNodeSelector.mingw.kt: -------------------------------------------------------------------------------- 1 | package com.jillesvangurp.ktsearch 2 | 3 | /** 4 | * On JVM this will return the current thread name, otherwise this will return null and pick a random node 5 | */ 6 | actual fun threadId(): String? = null -------------------------------------------------------------------------------- /search-client/src/nativeTest/kotlin/com/jillesvangurp/ktsearch/SearchTestBase.native.kt: -------------------------------------------------------------------------------- 1 | package com.jillesvangurp.ktsearch 2 | 3 | import kotlinx.coroutines.runBlocking 4 | import kotlinx.coroutines.test.TestResult 5 | import kotlinx.coroutines.withTimeout 6 | import kotlin.time.Duration 7 | 8 | actual fun coRun(timeout: Duration, block: suspend () -> Unit) { 9 | runBlocking { 10 | withTimeout(timeout) { 11 | block.invoke() 12 | } 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /search-client/src/wasmJsMain/kotlin/com/jillesvangurp/ktsearch/KtorRestClient.wasmJs.kt: -------------------------------------------------------------------------------- 1 | package com.jillesvangurp.ktsearch 2 | 3 | import io.ktor.client.HttpClient 4 | import io.ktor.client.HttpClientConfig 5 | 6 | actual fun defaultKtorHttpClient( 7 | logging: Boolean, 8 | user: String?, 9 | password: String?, 10 | elasticApiKey: String?, 11 | block: HttpClientConfig<*>.()->Unit 12 | ): HttpClient { 13 | return HttpClient { 14 | block(this) 15 | } 16 | } -------------------------------------------------------------------------------- /search-client/src/wasmJsMain/kotlin/com/jillesvangurp/ktsearch/RoundRobinNodeSelector.wasmJs.kt: -------------------------------------------------------------------------------- 1 | package com.jillesvangurp.ktsearch 2 | 3 | actual fun simpleIndexProvider(initialIndex: Int): IndexProvider = object : IndexProvider { 4 | private var index = 0 5 | 6 | override fun get(): Int = index 7 | 8 | override fun set(value: Int) { 9 | index = value 10 | } 11 | } -------------------------------------------------------------------------------- /search-client/src/wasmJsMain/kotlin/com/jillesvangurp/ktsearch/SniffingNodeSelector.wasmJs.kt: -------------------------------------------------------------------------------- 1 | package com.jillesvangurp.ktsearch 2 | 3 | /** 4 | * On JVM this will return the current thread name, on kotlin-js this will return null 5 | */ 6 | actual fun threadId(): String? { 7 | return null 8 | } -------------------------------------------------------------------------------- /search-client/src/wasmJsTest/kotlin/com/jillesvangurp/ktsearch/coTest.kt: -------------------------------------------------------------------------------- 1 | package com.jillesvangurp.ktsearch 2 | 3 | import kotlin.time.Duration 4 | import kotlinx.coroutines.DelicateCoroutinesApi 5 | import kotlinx.coroutines.GlobalScope 6 | import kotlinx.coroutines.asPromise 7 | import kotlinx.coroutines.async 8 | import kotlinx.coroutines.withTimeout 9 | 10 | @OptIn(DelicateCoroutinesApi::class) 11 | actual fun coRun(timeout: Duration, block: suspend () -> Unit) { 12 | GlobalScope.async { 13 | withTimeout(timeout) { 14 | try { 15 | block.invoke() 16 | } catch (e: Exception) { 17 | println("${e::class.simpleName} ${e.message}") 18 | error("Block failing") 19 | } 20 | } 21 | }.asPromise() 22 | } 23 | 24 | 25 | -------------------------------------------------------------------------------- /search-dsls/build.gradle.kts: -------------------------------------------------------------------------------- 1 | @file:OptIn(ExperimentalWasmDsl::class) 2 | 3 | import org.jetbrains.kotlin.gradle.ExperimentalKotlinGradlePluginApi 4 | import org.jetbrains.kotlin.gradle.dsl.JvmTarget 5 | import org.jetbrains.kotlin.gradle.targets.js.dsl.ExperimentalWasmDsl 6 | 7 | plugins { 8 | kotlin("multiplatform") 9 | } 10 | 11 | java { 12 | sourceCompatibility = JavaVersion.VERSION_11 13 | targetCompatibility = JavaVersion.VERSION_11 14 | } 15 | kotlin { 16 | jvm { 17 | // should work for android as well 18 | @OptIn(ExperimentalKotlinGradlePluginApi::class) 19 | compilerOptions { 20 | jvmTarget = JvmTarget.JVM_11 21 | } 22 | } 23 | 24 | js(IR) { 25 | browser() 26 | nodejs { 27 | testTask(Action { 28 | useMocha { 29 | // javascript is a lot slower than Java, we hit the default timeout of 2000 30 | timeout = "20s" 31 | } 32 | }) 33 | } 34 | } 35 | linuxX64() 36 | linuxArm64() 37 | mingwX64() 38 | macosX64() 39 | macosArm64() 40 | iosArm64() 41 | iosX64() 42 | iosSimulatorArm64() 43 | // Blocked on json-dsl and kotlinx-serialization-extensions support 44 | // iosSimulatorArm64() 45 | wasmJs { 46 | browser() 47 | nodejs() 48 | d8() 49 | } 50 | // not supported by kotest yet 51 | // wasmWasi() 52 | sourceSets { 53 | commonMain { 54 | dependencies { 55 | implementation(kotlin("stdlib-common", "_")) 56 | implementation("com.jillesvangurp:json-dsl:_") 57 | } 58 | } 59 | commonTest { 60 | dependencies { 61 | implementation(kotlin("test-common", "_")) 62 | implementation(kotlin("test-annotations-common", "_")) 63 | implementation(Testing.kotest.assertions.core) 64 | } 65 | } 66 | jvmMain { 67 | dependencies { 68 | } 69 | } 70 | jvmTest { 71 | dependencies { 72 | implementation(kotlin("test-junit5", "_")) 73 | implementation("ch.qos.logback:logback-classic:_") 74 | 75 | implementation(Testing.junit.jupiter.api) 76 | implementation(Testing.junit.jupiter.engine) 77 | } 78 | } 79 | jsMain { 80 | } 81 | jsTest { 82 | dependencies { 83 | implementation(kotlin("test-js", "_")) 84 | } 85 | } 86 | 87 | all { 88 | languageSettings { 89 | languageVersion = "1.9" 90 | apiVersion = "1.9" 91 | } 92 | 93 | @OptIn(ExperimentalKotlinGradlePluginApi::class) 94 | compilerOptions { 95 | freeCompilerArgs.add("-Xexpect-actual-classes") 96 | } 97 | languageSettings.optIn("kotlin.RequiresOptIn") 98 | } 99 | } 100 | } 101 | 102 | tasks.named("iosSimulatorArm64Test") { 103 | // requires IOS simulator and tens of GB of other stuff to be installed 104 | // so keep it disabled 105 | enabled = false 106 | } 107 | 108 | 109 | -------------------------------------------------------------------------------- /search-dsls/src/commonMain/kotlin/com/jillesvangurp/searchdsls/annotations.kt: -------------------------------------------------------------------------------- 1 | package com.jillesvangurp.searchdsls 2 | 3 | enum class SearchEngineFamily {Elasticsearch, Opensearch} 4 | enum class SearchEngineVariant(val family: SearchEngineFamily) { 5 | ES7(SearchEngineFamily.Elasticsearch), 6 | ES8(SearchEngineFamily.Elasticsearch), 7 | ES9(SearchEngineFamily.Elasticsearch), 8 | OS1(SearchEngineFamily.Opensearch), 9 | OS2(SearchEngineFamily.Opensearch), 10 | OS3(SearchEngineFamily.Opensearch) 11 | } 12 | 13 | annotation class VariantRestriction(vararg val variant: SearchEngineVariant) 14 | 15 | 16 | -------------------------------------------------------------------------------- /search-dsls/src/commonMain/kotlin/com/jillesvangurp/searchdsls/querydsl/QueryClauses.kt: -------------------------------------------------------------------------------- 1 | package com.jillesvangurp.searchdsls.querydsl 2 | 3 | interface QueryClauses 4 | 5 | /** 6 | * Allows you to create a query without a search dsl context. Returns the [ESQuery] created by your [block]. 7 | * 8 | * This makes it possible to create utility functions that aren't extension functions 9 | * on QueryClauses that construct queries. The core use case is reusable functionality for 10 | * building queries or parts of queries. 11 | * 12 | * It works by creating an anonymous object implementing QueryClauses and then passing that to [block]. 13 | */ 14 | fun constructQueryClause(block: QueryClauses.() -> ESQuery): ESQuery { 15 | val obj = object : QueryClauses {} 16 | return block(obj) 17 | } -------------------------------------------------------------------------------- /search-dsls/src/commonMain/kotlin/com/jillesvangurp/searchdsls/querydsl/filter-source.kt: -------------------------------------------------------------------------------- 1 | @file:Suppress("unused") 2 | 3 | package com.jillesvangurp.searchdsls.querydsl 4 | 5 | import kotlin.reflect.KProperty 6 | 7 | class SourceBuilder { 8 | internal val sourceFilter = mutableMapOf() 9 | 10 | fun includes(vararg fields: KProperty<*>) = includes(*fields.map { it.name }.toTypedArray()) 11 | fun includes(vararg fields: String) = sourceFilter.set("includes", arrayOf(*fields)) 12 | 13 | fun excludes(vararg fields: KProperty<*>) = excludes(*fields.map { it.name }.toTypedArray()) 14 | fun excludes(vararg fields: String) = sourceFilter.set("excludes", arrayOf(*fields)) 15 | } 16 | 17 | fun SearchDSL.filterSource(returnSource: Boolean) { 18 | this["_source"] = returnSource 19 | } 20 | 21 | fun SearchDSL.filterSource(vararg fields: KProperty<*>) { 22 | this["_source"] = arrayOf(*fields.map { it.name }.toTypedArray()) 23 | } 24 | 25 | fun SearchDSL.filterSource(vararg fields: String) { 26 | this["_source"] = arrayOf(*fields) 27 | } 28 | 29 | fun SearchDSL.filterSource(block: SourceBuilder.() -> Unit) { 30 | val builder = SourceBuilder() 31 | block.invoke(builder) 32 | this["_source"] = builder.sourceFilter 33 | } 34 | 35 | -------------------------------------------------------------------------------- /search-dsls/src/commonMain/kotlin/com/jillesvangurp/searchdsls/querydsl/highlight-dsl.kt: -------------------------------------------------------------------------------- 1 | package com.jillesvangurp.searchdsls.querydsl 2 | 3 | import kotlin.reflect.KProperty 4 | 5 | @Suppress("EnumEntryName") 6 | enum class BoundaryScanner { 7 | chars, 8 | sentence, 9 | word, 10 | } 11 | 12 | @Suppress("EnumEntryName") 13 | enum class Encoder { 14 | default, 15 | html, 16 | } 17 | 18 | @Suppress("EnumEntryName") 19 | enum class Fragmenter { 20 | simple, 21 | span, 22 | } 23 | 24 | @Suppress("EnumEntryName") 25 | enum class Order { 26 | none, 27 | score, 28 | } 29 | 30 | @Suppress("EnumEntryName") 31 | enum class Type { 32 | unified, 33 | plain, 34 | fvh, 35 | } 36 | 37 | open class HighlightField(name: String) : ESQuery(name) { 38 | constructor(property: KProperty<*>) : this(property.name) 39 | 40 | var boundaryChars by queryDetails.property() 41 | var boundaryMaxScan by queryDetails.property() 42 | var boundaryScanner by queryDetails.property() 43 | var boundaryScannerLocale by queryDetails.property() 44 | var encoder by queryDetails.property() 45 | var fragmenter by queryDetails.property("fragmenter",Fragmenter.span) 46 | var fragmentOffset by queryDetails.property() 47 | var fragmentSize by queryDetails.property() 48 | 49 | var highlightQuery by queryDetails.esQueryProperty() 50 | 51 | var noMatchSize by queryDetails.property() 52 | var numberOfFragments by queryDetails.property() 53 | var phraseLimit by queryDetails.property() 54 | var maxAnalyzedOffset by queryDetails.property() 55 | var tagsSchema by queryDetails.property() 56 | var type by queryDetails.property() 57 | 58 | fun matchedFields(vararg matchedFields: String) = queryDetails.getOrCreateMutableList("matched_fields").also { 59 | it.addAll(matchedFields) 60 | } 61 | } 62 | 63 | class Highlight : ESQuery("highlight") { 64 | var preTags by queryDetails.property() 65 | var postTags by queryDetails.property() 66 | var requireFieldMatch by queryDetails.property() 67 | var order by queryDetails.property() 68 | 69 | fun fields(vararg fields: HighlightField) = 70 | queryDetails.getOrCreateMutableList("fields") 71 | .addAll(fields.map { it.wrapWithName() }) 72 | 73 | fun add( 74 | name: String, 75 | block: (HighlightField.() -> Unit)? = null, 76 | ) { 77 | val hf = HighlightField(name) 78 | block?.invoke(hf) 79 | fields(hf) 80 | } 81 | fun add( 82 | name: KProperty<*>, 83 | block: (HighlightField.() -> Unit)? = null, 84 | ) { 85 | val hf = HighlightField(name) 86 | block?.invoke(hf) 87 | fields(hf) 88 | } 89 | } 90 | 91 | fun SearchDSL.highlight( 92 | vararg fields: HighlightField, 93 | block: (Highlight.() -> Unit)?=null 94 | ) { 95 | val builder = Highlight() 96 | if(fields.isNotEmpty()) { 97 | builder.fields(*fields) 98 | } 99 | block?.invoke(builder) 100 | this["highlight"] = builder 101 | } -------------------------------------------------------------------------------- /search-dsls/src/commonMain/kotlin/com/jillesvangurp/searchdsls/querydsl/join-queries.kt: -------------------------------------------------------------------------------- 1 | @file:Suppress("unused", "UnusedReceiverParameter") 2 | 3 | package com.jillesvangurp.searchdsls.querydsl 4 | 5 | import com.jillesvangurp.jsondsl.JsonDsl 6 | import com.jillesvangurp.jsondsl.json 7 | 8 | enum class ScoreMode { 9 | avg,min,max,none,sum 10 | } 11 | 12 | @SearchDSLMarker 13 | class NestedQuery : ESQuery(name = "nested") { 14 | var path: String by queryDetails.property() 15 | var query: ESQuery by queryDetails.esQueryProperty() 16 | var scoreMode: ScoreMode by queryDetails.property() 17 | var ignoreUnmapped: Boolean by queryDetails.property() 18 | } 19 | 20 | fun QueryClauses.nested(block: NestedQuery.() -> Unit): NestedQuery { 21 | val q = NestedQuery() 22 | block.invoke(q) 23 | return q 24 | } 25 | 26 | @SearchDSLMarker 27 | class ParentIdQuery(val type: String, val id: String) : ESQuery("parent_id") { 28 | init { 29 | queryDetails["type"] = type 30 | queryDetails["id"] = id 31 | } 32 | 33 | var ignoreUnmapped: Boolean by queryDetails.property() 34 | } 35 | 36 | fun QueryClauses.parentId(type: String, id: String, block: ParentIdQuery.() -> Unit): ParentIdQuery { 37 | val q = ParentIdQuery(type = type, id = id) 38 | block.invoke(q) 39 | return q 40 | } 41 | 42 | fun QueryClauses.parentId(type: String, id: String) = ParentIdQuery(type = type, id = id) 43 | 44 | open class ParentChildQuery(queryType: ParentChildQueryType, val type: String) : ESQuery(name = "has_${queryType.name}") { 45 | init { 46 | queryDetails[queryType.typeFieldName] = type 47 | } 48 | 49 | enum class ParentChildQueryType(val typeFieldName: String) { 50 | parent("parent_type"), 51 | child("type") 52 | } 53 | 54 | var query: ESQuery by queryDetails.esQueryProperty() 55 | var scoreMode: ScoreMode by queryDetails.property() 56 | var ignoreUnmapped: Boolean by queryDetails.property() 57 | var minChildren: Int by queryDetails.property() 58 | var maxChildren: Int by queryDetails.property() 59 | 60 | fun innerHits(block: InnerHits.() -> Unit): InnerHits { 61 | val q = InnerHits() 62 | block.invoke(q) 63 | queryDetails["inner_hits"] = q 64 | return q 65 | } 66 | 67 | fun innerHits(): InnerHits { 68 | val q = InnerHits() 69 | queryDetails["inner_hits"] = q 70 | return q 71 | } 72 | 73 | class InnerHits : JsonDsl() { 74 | var from: Int by property() 75 | var innerHitSize: Int by property("size") 76 | var name: String by property() 77 | var sort: List by property() 78 | 79 | fun sort(block: SortBuilder.() -> Unit) { 80 | val builder = SortBuilder() 81 | block.invoke(builder) 82 | this["sort"] = builder.sortFields 83 | } 84 | } 85 | } 86 | 87 | @SearchDSLMarker 88 | class HasChildQuery(type: String) : ParentChildQuery(queryType = ParentChildQueryType.child, type = type) 89 | 90 | @SearchDSLMarker 91 | class HasParentQuery(type: String) : ParentChildQuery(queryType = ParentChildQueryType.parent, type = type) 92 | 93 | fun QueryClauses.hasChild(type: String, block: HasChildQuery.() -> Unit): HasChildQuery { 94 | val q = HasChildQuery(type) 95 | block.invoke(q) 96 | return q 97 | } 98 | 99 | fun QueryClauses.hasParent(type: String, block: HasParentQuery.() -> Unit): HasParentQuery { 100 | val q = HasParentQuery(type) 101 | block.invoke(q) 102 | return q 103 | } -------------------------------------------------------------------------------- /search-dsls/src/commonMain/kotlin/com/jillesvangurp/searchdsls/querydsl/reindex-dsl.kt: -------------------------------------------------------------------------------- 1 | package com.jillesvangurp.searchdsls.querydsl 2 | 3 | import com.jillesvangurp.jsondsl.CustomValue 4 | import com.jillesvangurp.jsondsl.JsonDsl 5 | import kotlin.time.Duration 6 | 7 | enum class Conflict(override val value: String) : CustomValue { 8 | ABORT("abort"), 9 | PROCEED("proceed"); 10 | } 11 | 12 | class ReindexDSL : JsonDsl() { 13 | var conflicts: Conflict by property() 14 | var maxDocs: Int by property(customPropertyName = "max_docs") 15 | 16 | fun source(block: ReindexSourceDSL.() -> Unit) { 17 | val sourceDSL = ReindexSourceDSL() 18 | block(sourceDSL) 19 | this["source"] = sourceDSL 20 | } 21 | 22 | fun destination(block: ReindexDestinationDSL.() -> Unit) { 23 | val destinationDSL = ReindexDestinationDSL() 24 | block(destinationDSL) 25 | this["dest"] = destinationDSL 26 | } 27 | 28 | fun script(block: ReindexScriptDSL.() -> Unit) { 29 | val scriptDSL = ReindexScriptDSL() 30 | block(scriptDSL) 31 | this["script"] = scriptDSL 32 | } 33 | } 34 | 35 | class ReindexSourceDSL : JsonDsl(), QueryClauses { 36 | var index: String by property() 37 | var batchSize: Int by property("size") 38 | 39 | var query: ESQuery 40 | get() { 41 | val map = this["query"] as Map<*, *> 42 | val (name, queryDetails) = map.entries.first() 43 | if(queryDetails is JsonDsl) { 44 | return ESQuery(name.toString(), queryDetails) 45 | } else { 46 | error("wrong type for queryDetails, should be JsonDSL") 47 | } 48 | } 49 | set(value) { 50 | this["query"] = value.wrapWithName() 51 | } 52 | 53 | fun fields(vararg names: String) { 54 | if (names.isNotEmpty()) { 55 | this["_source"] = names.toList() 56 | } 57 | } 58 | fun fields(names: Collection) { 59 | if (names.isNotEmpty()) { 60 | this["_source"] = names 61 | } 62 | } 63 | 64 | fun remote(block: ReindexRemoteDSL.() -> Unit) { 65 | val scriptDSL = ReindexRemoteDSL() 66 | block(scriptDSL) 67 | this["remote"] = scriptDSL 68 | } 69 | 70 | fun slice(block: ReindexSliceDSL.() -> Unit) { 71 | val scriptDSL = ReindexSliceDSL() 72 | block(scriptDSL) 73 | this["slice"] = scriptDSL 74 | } 75 | } 76 | 77 | class ReindexRemoteDSL : JsonDsl() { 78 | var host: String by property() 79 | var username: String by property() 80 | var password: String by property() 81 | var socketTimeout: Duration by property(customPropertyName = "socket_timeout") 82 | var connectTimeout: Duration by property(customPropertyName = "connect_timeout") 83 | var headers: Map by property() 84 | } 85 | 86 | class ReindexSliceDSL : JsonDsl() { 87 | var id: Int by property() 88 | var max: Int by property() 89 | } 90 | 91 | class ReindexDestinationDSL : JsonDsl() { 92 | var index: String by property() 93 | var versionType: ReindexVersionType by property(customPropertyName = "version_type") 94 | var operationType: ReindexOperationType by property(customPropertyName = "op_type") 95 | var pipeline: String by property() 96 | } 97 | 98 | enum class ReindexVersionType(override val value: String) : CustomValue { 99 | INTERNAL("internal"), 100 | EXTERNAL("external"), 101 | EXTERNAL_GT("external_gt"), 102 | EXTERNAL_GTE("external_gte") 103 | } 104 | 105 | enum class ReindexOperationType(override val value: String) : CustomValue { 106 | INDEX("index"), 107 | CREATE("create") 108 | } 109 | 110 | enum class Language(override val value: String) : CustomValue { 111 | PAINLESS("painless"), 112 | EXPRESSION("expression"), 113 | MUSTACHE("mustache"), 114 | JAVA("java") 115 | } 116 | 117 | class ReindexScriptDSL : JsonDsl() { 118 | var source: String by property() 119 | var language: Language by property(customPropertyName = "lang") 120 | } 121 | -------------------------------------------------------------------------------- /search-dsls/src/commonMain/kotlin/com/jillesvangurp/searchdsls/querydsl/specialized-queries.kt: -------------------------------------------------------------------------------- 1 | package com.jillesvangurp.searchdsls.querydsl 2 | 3 | import com.jillesvangurp.jsondsl.withJsonDsl 4 | import kotlin.reflect.KProperty 5 | 6 | class DistanceFeature() : ESQuery("distance_feature") { 7 | var field by property() 8 | var pivot by property() 9 | var origin by property() 10 | var boost by property() 11 | } 12 | 13 | fun QueryClauses.distanceFeature( 14 | field: KProperty<*>, 15 | pivot: String, 16 | origin: String, 17 | block: (DistanceFeature.() -> Unit)? = null 18 | ) = DistanceFeature().let { 19 | it.field = field.name 20 | it.pivot = pivot 21 | it.origin = origin 22 | block?.invoke(it) 23 | it 24 | } 25 | 26 | fun QueryClauses.distanceFeature( 27 | field: String, 28 | pivot: String, 29 | origin: String, 30 | block: (DistanceFeature.() -> Unit)? = null 31 | ) = DistanceFeature().let { 32 | it.field = field 33 | it.pivot = pivot 34 | it.origin = origin 35 | block?.invoke(it) 36 | it 37 | } 38 | 39 | fun QueryClauses.distanceFeature( 40 | field: KProperty<*>, 41 | pivot: String, 42 | origin: List, 43 | block: (DistanceFeature.() -> Unit)? = null 44 | ) = DistanceFeature().let { 45 | it.field = field.name 46 | it.pivot = pivot 47 | it.origin = origin 48 | block?.invoke(it) 49 | it 50 | } 51 | 52 | fun QueryClauses.distanceFeature( 53 | field: String, 54 | pivot: String, 55 | origin: List, 56 | block: (DistanceFeature.() -> Unit)? = null 57 | ) = DistanceFeature().let { 58 | it.field = field 59 | it.pivot = pivot 60 | it.origin = origin 61 | block?.invoke(it) 62 | it 63 | } 64 | 65 | fun QueryClauses.distanceFeature( 66 | field: KProperty<*>, 67 | pivot: String, 68 | origin: Array, 69 | block: (DistanceFeature.() -> Unit)? = null 70 | ) = DistanceFeature().let { 71 | it.field = field.name 72 | it.pivot = pivot 73 | it.origin = origin 74 | block?.invoke(it) 75 | it 76 | } 77 | 78 | fun QueryClauses.distanceFeature( 79 | field: String, 80 | pivot: String, 81 | origin: Array, 82 | block: (DistanceFeature.() -> Unit)? = null 83 | ) = DistanceFeature().let { 84 | it.field = field 85 | it.pivot = pivot 86 | it.origin = origin 87 | block?.invoke(it) 88 | it 89 | } 90 | 91 | fun QueryClauses.distanceFeature( 92 | field: KProperty<*>, 93 | pivot: String, 94 | origin: DoubleArray, 95 | block: (DistanceFeature.() -> Unit)? = null 96 | ) = DistanceFeature().let { 97 | it.field = field.name 98 | it.pivot = pivot 99 | it.origin = origin 100 | block?.invoke(it) 101 | it 102 | } 103 | 104 | fun QueryClauses.distanceFeature( 105 | field: String, 106 | pivot: String, 107 | origin: DoubleArray, 108 | block: (DistanceFeature.() -> Unit)? = null 109 | ) = DistanceFeature().let { 110 | it.field = field 111 | it.pivot = pivot 112 | it.origin = origin 113 | block?.invoke(it) 114 | it 115 | } 116 | 117 | class RankFeature(val field: String, block: (RankFeature.() -> Unit)? = null) : ESQuery("rank_feature") { 118 | constructor(field: KProperty<*>, block: (RankFeature.() -> Unit)? = null) : this(field.name, block) 119 | 120 | var boost by property() 121 | 122 | init { 123 | this["field"] = field 124 | block?.invoke(this) 125 | } 126 | 127 | /** 128 | * Configure the saturation function with an optional [pivot]. If omitted, 129 | * it uses the mean value of the feature that you are ranking on. 130 | */ 131 | fun saturation(pivot: Double? = null) { 132 | this["saturation"] = withJsonDsl { 133 | if (pivot != null) { 134 | this["pivot"] = pivot 135 | } 136 | } 137 | } 138 | 139 | /** 140 | * Configure the logarithmic function. 141 | */ 142 | fun log(scalingFactor: Double) { 143 | this["log"] = withJsonDsl { 144 | this["scaling_factor"] = scalingFactor 145 | } 146 | } 147 | 148 | /** 149 | * Variant of [saturation] with a configurable [exponent]. 150 | * The [exponent] should generally be between 0.5 and 1.0 151 | * and is typically calculated via some training method. 152 | * Use saturation if you are unable to do this. 153 | */ 154 | fun sigmoid(pivot: Double, exponent: Double) { 155 | this["sigmoid"] = withJsonDsl { 156 | this["pivot"] = pivot 157 | this["exponent"] = exponent 158 | } 159 | } 160 | 161 | /** 162 | * Configure linear function. 163 | */ 164 | fun linear() { 165 | this["linear"] = withJsonDsl { } 166 | } 167 | } 168 | 169 | fun QueryClauses.rankFeature(field: String, block: (RankFeature.() -> Unit)? = null) = RankFeature(field, block) 170 | fun QueryClauses.rankFeature(field: KProperty<*>, block: (RankFeature.() -> Unit)? = null) = RankFeature(field, block) 171 | -------------------------------------------------------------------------------- /settings.gradle.kts: -------------------------------------------------------------------------------- 1 | 2 | pluginManagement { 3 | repositories { 4 | mavenCentral() 5 | gradlePluginPortal() 6 | } 7 | } 8 | 9 | plugins { 10 | id("de.fayard.refreshVersions") version "0.60.5" 11 | } 12 | 13 | refreshVersions { 14 | } 15 | 16 | include("search-dsls") 17 | include("search-client") 18 | include("docs") 19 | rootProject.name = "kt-search" 20 | -------------------------------------------------------------------------------- /versions.properties: -------------------------------------------------------------------------------- 1 | #### Dependencies and Plugin versions with their available updates. 2 | #### Generated by `./gradlew refreshVersions` version 0.60.5 3 | #### 4 | #### Don't manually edit or split the comments that start with four hashtags (####), 5 | #### they will be overwritten by refreshVersions. 6 | #### 7 | #### suppress inspection "SpellCheckingInspection" for whole file 8 | #### suppress inspection "UnusedProperty" for whole file 9 | 10 | plugin.com.avast.gradle.docker-compose=0.17.12 11 | 12 | plugin.org.jetbrains.dokka=2.0.0 13 | 14 | version.ch.qos.logback..logback-classic=1.5.18 15 | 16 | version.com.github.doyaaaaaken..kotlin-csv=1.10.0 17 | 18 | version.com.github.jillesvangurp..kotlin4example=1.1.6 19 | 20 | version.com.jillesvangurp..json-dsl=3.0.7 21 | 22 | version.com.jillesvangurp..kotlinx-serialization-extensions=1.0.6 23 | 24 | version.io.github.oshai..kotlin-logging=7.0.7 25 | 26 | version.junit.jupiter=5.12.2 27 | ## # available=5.13.0-M1 28 | ## # available=5.13.0-M2 29 | ## # available=5.13.0-M3 30 | ## # available=5.13.0-RC1 31 | 32 | version.kotest=5.9.1 33 | ## # available=6.0.0.M1 34 | ## # available=6.0.0.M2 35 | ## # available=6.0.0.M3 36 | ## # available=6.0.0.M4 37 | 38 | version.kotlin=2.1.21 39 | ## # available=2.2.0-Beta1 40 | ## # available=2.2.0-Beta2 41 | ## # available=2.2.0-RC 42 | 43 | version.kotlinx.coroutines=1.10.2 44 | 45 | version.kotlinx.datetime=0.6.2 46 | 47 | version.kotlinx.serialization=1.8.1 48 | 49 | version.ktor=3.1.3 50 | 51 | version.org.apache.logging.log4j..log4j-to-slf4j=2.24.3 52 | ## # available=3.0.0-alpha1 53 | ## # available=3.0.0-beta1 54 | ## # available=3.0.0-beta2 55 | 56 | version.org.slf4j..jcl-over-slf4j=2.0.17 57 | ## # available=2.1.0-alpha0 58 | ## # available=2.1.0-alpha1 59 | 60 | version.org.slf4j..jul-to-slf4j=2.0.17 61 | ## # available=2.1.0-alpha0 62 | ## # available=2.1.0-alpha1 63 | 64 | version.org.slf4j..log4j-over-slf4j=2.0.17 65 | ## # available=2.1.0-alpha0 66 | ## # available=2.1.0-alpha1 67 | 68 | version.org.slf4j..slf4j-api=2.0.17 69 | ## # available=2.1.0-alpha0 70 | ## # available=2.1.0-alpha1 71 | --------------------------------------------------------------------------------