├── .github ├── CODEOWNERS └── workflows │ ├── add-untriaged.yml │ ├── build.yml │ └── maven-publish.yml ├── .gitignore ├── .whitesource ├── CHANGELOG-3.0.md ├── CHANGELOG.md ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE ├── MAINTAINERS.md ├── NOTICE ├── README.md ├── RELEASING.md ├── RESPONSIBILITIES.md ├── SECURITY.md ├── build.gradle ├── dataprepper ├── data-prepper-config.yaml └── pipelines.yaml ├── docker-compose.yaml ├── gradle.properties ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── licenses ├── commons-logging-1.2.jar.sha1 ├── commons-logging-LICENSE.txt ├── commons-logging-NOTICE.txt ├── httpclient-4.5.14.jar.sha1 ├── httpclient-LICENSE.txt ├── httpclient-NOTICE.txt ├── httpcore-4.4.16.jar.sha1 ├── httpcore-LICENSE.txt ├── httpcore-NOTICE.txt ├── jackson-annotations-2.18.2.jar.sha1 ├── jackson-annotations-LICENSE.txt ├── jackson-annotations-NOTICE.txt ├── jackson-databind-2.18.2.jar.sha1 ├── jackson-databind-LICENSE.txt └── jackson-databind-NOTICE.txt ├── scripts ├── get-indexed-queries.sh ├── initialize-ubi.sh ├── msearch.sh └── search.sh ├── search-quality-evaluation-dashboards ├── README.md ├── install_dashboards.sh ├── sample_search_quality_evaluation_data.ndjson └── search_quality_evaluation_dashboard.ndjson ├── settings.gradle ├── src ├── main │ ├── java │ │ └── org │ │ │ └── opensearch │ │ │ └── ubi │ │ │ ├── QueryRequest.java │ │ │ ├── QueryResponse.java │ │ │ ├── UbiActionFilter.java │ │ │ ├── UbiPlugin.java │ │ │ ├── UbiRestHandler.java │ │ │ ├── UbiSearchResponse.java │ │ │ ├── UbiSettings.java │ │ │ └── ext │ │ │ ├── UbiParameters.java │ │ │ └── UbiParametersExtBuilder.java │ └── resources │ │ ├── events-mapping.json │ │ └── queries-mapping.json ├── test │ └── java │ │ └── org │ │ └── opensearch │ │ └── ubi │ │ ├── UbIParametersTests.java │ │ ├── UbiActionFilterTests.java │ │ ├── UbiModulePluginTests.java │ │ └── UbiParametersExtBuilderTests.java └── yamlRestTest │ ├── java │ └── org │ │ └── opensearch │ │ └── ubi │ │ └── rest │ │ └── action │ │ └── UserBehaviorInsightsClientYamlTestSuiteIT.java │ └── resources │ └── rest-api-spec │ ├── api │ └── ubi.initialize.json │ └── test │ └── _plugins.ubi │ ├── 10_initialize_ubi.yml │ ├── 20_queries_with_ubi.yml │ ├── 30_queries_without_ubi.yml │ └── 40_msearch_queries_with_ubi.yml ├── ubi-dashboards ├── README.md ├── install_dashboards.sh └── ubi_dashboard.ndjson ├── ubi-data-generator ├── README.md ├── requirements.txt └── ubi_data_generator.py └── ubi-javascript-collector └── ubi.js /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @epugh @jzonthemtn @msfroh 2 | -------------------------------------------------------------------------------- /.github/workflows/add-untriaged.yml: -------------------------------------------------------------------------------- 1 | name: Apply 'untriaged' label during issue lifecycle 2 | 3 | on: 4 | issues: 5 | types: [opened, reopened, transferred] 6 | 7 | jobs: 8 | apply-label: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - uses: actions/github-script@v6 12 | with: 13 | script: | 14 | github.rest.issues.addLabels({ 15 | issue_number: context.issue.number, 16 | owner: context.repo.owner, 17 | repo: context.repo.repo, 18 | labels: ['untriaged'] 19 | }) 20 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: Build and Test 2 | on: [push, pull_request, workflow_dispatch] 3 | jobs: 4 | build: 5 | runs-on: ${{ matrix.os }} 6 | continue-on-error: ${{ matrix.experimental }} 7 | strategy: 8 | matrix: 9 | os: [ubuntu-latest, macOS-latest, windows-latest] 10 | jdk: [ 21 ] 11 | experimental: [false] 12 | steps: 13 | - uses: actions/checkout@v4 14 | - name: Set up JDK ${{ matrix.jdk }} 15 | uses: actions/setup-java@v4 16 | with: 17 | distribution: temurin 18 | java-version: ${{ matrix.jdk }} 19 | cache: gradle 20 | - name: Assemble target plugin 21 | uses: gradle/gradle-build-action@v3 22 | with: 23 | cache-disabled: true 24 | arguments: -Dtests.security.manager=false assemble 25 | -------------------------------------------------------------------------------- /.github/workflows/maven-publish.yml: -------------------------------------------------------------------------------- 1 | name: Publish snapshots to maven 2 | 3 | on: 4 | workflow_dispatch: 5 | push: 6 | branches: 7 | - main 8 | - '[0-9]+.[0-9]+' 9 | - '[0-9]+.x' 10 | 11 | jobs: 12 | build-and-publish-snapshots: 13 | runs-on: ubuntu-latest 14 | 15 | permissions: 16 | id-token: write 17 | contents: write 18 | 19 | steps: 20 | - uses: actions/checkout@v4 21 | - name: Set up JDK 21 22 | uses: actions/setup-java@v3 23 | with: 24 | distribution: temurin # Temurin is a distribution of adoptium 25 | java-version: 21 26 | 27 | - name: Configure AWS credentials 28 | uses: aws-actions/configure-aws-credentials@v4 29 | with: 30 | role-to-assume: ${{ secrets.PUBLISH_SNAPSHOTS_ROLE }} 31 | aws-region: us-east-1 32 | 33 | - name: Publish snapshots to maven 34 | run: | 35 | export SONATYPE_USERNAME=$(aws secretsmanager get-secret-value --secret-id maven-snapshots-username --query SecretString --output text) 36 | export SONATYPE_PASSWORD=$(aws secretsmanager get-secret-value --secret-id maven-snapshots-password --query SecretString --output text) 37 | echo "::add-mask::$SONATYPE_USERNAME" 38 | echo "::add-mask::$SONATYPE_PASSWORD" 39 | ./gradlew publishPluginZipPublicationToSnapshotsRepository -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore Gradle project-specific cache directory 2 | .gradle 3 | 4 | # Ignore Gradle build output directory 5 | build 6 | 7 | # intellij files 8 | .idea/ 9 | *.iml 10 | *.ipr 11 | *.iws 12 | build-idea/ 13 | out/ 14 | 15 | volumes/ 16 | -------------------------------------------------------------------------------- /.whitesource: -------------------------------------------------------------------------------- 1 | { 2 | "scanSettings": { 3 | "configMode": "AUTO", 4 | "configExternalURL": "", 5 | "projectToken": "", 6 | "baseBranches": [] 7 | }, 8 | "scanSettingsSAST": { 9 | "enableScan": false, 10 | "scanPullRequests": false, 11 | "incrementalScan": true, 12 | "baseBranches": [], 13 | "snippetSize": 10 14 | }, 15 | "checkRunSettings": { 16 | "vulnerableCheckRunConclusionLevel": "failure", 17 | "displayMode": "diff", 18 | "useMendCheckNames": true 19 | }, 20 | "checkRunSettingsSAST": { 21 | "checkRunConclusionLevel": "failure", 22 | "severityThreshold": "high" 23 | }, 24 | "issueSettings": { 25 | "minSeverityLevel": "LOW", 26 | "issueType": "DEPENDENCY" 27 | }, 28 | "issueSettingsSAST": { 29 | "minSeverityLevel": "high", 30 | "issueType": "repo" 31 | }, 32 | "remediateSettings": { 33 | "workflowRules": { 34 | "enabled": true 35 | } 36 | }, 37 | "imageSettings":{ 38 | "imageTracing":{ 39 | "enableImageTracingPR": false, 40 | "addRepositoryCoordinate": false, 41 | "addDockerfilePath": false, 42 | "addMendIdentifier": false 43 | } 44 | } 45 | } -------------------------------------------------------------------------------- /CHANGELOG-3.0.md: -------------------------------------------------------------------------------- 1 | # CHANGELOG 2 | All notable changes to this project are documented in this file. 3 | 4 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). See the [CONTRIBUTING guide](./CONTRIBUTING.md#Changelog) for instructions on how to add changelog entries. 5 | 6 | ## [Unreleased 3.0] 7 | ### Added 8 | 9 | ### Dependencies 10 | 11 | ### Changed 12 | 13 | ### Deprecated 14 | 15 | ### Removed 16 | 17 | ### Fixed 18 | 19 | ### Security 20 | 21 | [Unreleased 3.0]: https://github.com/opensearch-project/user-behavior-insights/compare/2.x...HEAD 22 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # CHANGELOG 2 | All notable changes to this project are documented in this file. 3 | 4 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). See the [CONTRIBUTING guide](./CONTRIBUTING.md#Changelog) for instructions on how to add changelog entries. 5 | 6 | ## [Unreleased 2.x] 7 | ### Added 8 | 9 | ### Dependencies 10 | 11 | ### Changed 12 | 13 | ### Deprecated 14 | 15 | ### Removed 16 | 17 | ### Fixed 18 | 19 | ### Security 20 | 21 | [Unreleased 2.x]: https://github.com/opensearch-project/user-behavior-insights/compare/2.14...2.x 22 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | 2 | This code of conduct applies to all spaces provided by the OpenSource project including in code, documentation, issue trackers, mailing lists, chat channels, wikis, blogs, social media, events, conferences, meetings, and any other communication channels used by the project. 3 | 4 | **Our open source communities endeavor to:** 5 | 6 | * Be Inclusive: We are committed to being a community where everyone can join and contribute. This means using inclusive and welcoming language. 7 | * Be Welcoming: We are committed to maintaining a safe space for everyone to be able to contribute. 8 | * Be Respectful: We are committed to encouraging differing viewpoints, accepting constructive criticism and work collaboratively towards decisions that help the project grow. Disrespectful and unacceptable behavior will not be tolerated. 9 | * Be Collaborative: We are committed to supporting what is best for our community and users. When we build anything for the benefit of the project, we should document the work we do and communicate to others on how this affects their work. 10 | 11 | **Our Responsibility. As contributors, members, or bystanders we each individually have the responsibility to behave professionally and respectfully at all times. Disrespectful and unacceptable behaviors include, but are not limited to:** 12 | 13 | * The use of violent threats, abusive, discriminatory, or derogatory language; 14 | * Offensive comments related to gender, gender identity and expression, sexual orientation, disability, mental illness, race, political or religious affiliation; 15 | * Posting of sexually explicit or violent content; 16 | * The use of sexualized language and unwelcome sexual attention or advances; 17 | * Public or private harassment of any kind; 18 | * Publishing private information, such as physical or electronic address, without permission; 19 | * Other conduct which could reasonably be considered inappropriate in a professional setting; 20 | * Advocating for or encouraging any of the above behaviors. 21 | 22 | **Enforcement and Reporting Code of Conduct Issues:** 23 | 24 | Instances of abusive, harassing, or otherwise unacceptable behavior may be reported. [Contact us](mailto:opensource-codeofconduct@amazon.com). All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. 25 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM opensearchstaging/opensearch:3.1.0-SNAPSHOT 2 | 3 | ARG UBI_VERSION="3.1.0.0-SNAPSHOT" 4 | 5 | COPY ./build/distributions/opensearch-ubi-${UBI_VERSION}.zip /tmp/ 6 | 7 | # Required for OTel capabilities. 8 | #RUN /usr/share/opensearch/bin/opensearch-plugin install --batch telemetry-otel 9 | 10 | RUN /usr/share/opensearch/bin/opensearch-plugin install --batch file:/tmp/opensearch-ubi-${UBI_VERSION}.zip -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | -------------------------------------------------------------------------------- /MAINTAINERS.md: -------------------------------------------------------------------------------- 1 | - [Overview](#overview) 2 | - [Current Maintainers](#current-maintainers) 3 | - [Emeritus](#emeritus) 4 | 5 | ## Overview 6 | 7 | This document contains a list of maintainers in this repo. See [opensearch-project/user-behavior-insights/RESPONSIBILITIES.md](https://github.com/opensearch-project/user-behavior-insights/blob/main/RESPONSIBILITIES.md#maintainer-responsibilities) that explains what the role of maintainer means, what maintainers do in this and other repos, and how they should be doing it. If you're interested in contributing, and becoming a maintainer, see [CONTRIBUTING](CONTRIBUTING.md). 8 | 9 | ## Current Maintainers 10 | 11 | | Maintainer | GitHub ID | Affiliation | 12 | | ------------------ | --------------------------------------------------------- | ----------- | 13 | | Eric Pugh | [epugh](https://github.com/epugh) | OpenSource Connections | 14 | | Jeff Zemerick | [jzonthemtn](https://github.com/jzonthemtn) | Mountain Fog | 15 | | Michael Froh | [msfroh](https://github.com/msfroh) | Amazon | 16 | 17 | ## Emeritus 18 | 19 | | Maintainer | GitHub ID | Affiliation | 20 | | ------------------ | --------------------------------------------------------- | ----------- | 21 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | -------------------------------------------------------------------------------- /RELEASING.md: -------------------------------------------------------------------------------- 1 | - [Overview](#overview) 2 | - [Branching](#branching) 3 | - [OpenSearch Branching](#opensearch-branching) 4 | - [Plugin Branching](#plugin-branching) 5 | - [Feature Branches](#feature-branches) 6 | - [Versioning](#versioning) 7 | - [Version Numbers](#version-numbers) 8 | - [Incrementing Versions](#incrementing-versions) 9 | - [Tagging](#tagging) 10 | - [Release Labels](#release-labels) 11 | - [Releasing](#releasing) 12 | - [Release Process for OpenSearch Bundle Minor Releases and Patch Releases](#Release-Process-for-OpenSearch-Bundle-Minor-Releases-and-Patch-Releases) 13 | - [Entrance Criteria to Start Release Window](#Entrance-Criteria-to-Start-Release-Window) 14 | - [Exit Criteria to Close Release Window](#Exit-Criteria-to-Close-Release-Window) 15 | - [Security Reviews](#security-reviews) 16 | - [Backporting](#backporting) 17 | 18 | ## Overview 19 | 20 | This document explains the release strategy for artifacts in this organization. 21 | 22 | ## Branching 23 | 24 | Projects create a new branch when they need to start working on 2 separate versions of the product, with the `main` branch being the furthermost release. 25 | 26 | ### OpenSearch Branching 27 | 28 | [OpenSearch](https://github.com/opensearch-project/OpenSearch) typically tracks 3 releases in parallel. For example, given the last major release of 2.0, OpenSearch in this organization maintains the following active branches. 29 | 30 | * **main**: The _next major_ release, currently 3.0. This is the branch where all merges take place, and code moves fast. 31 | * * **2.x**: The _next minor_ release. Once a change is merged into `main`, decide whether to backport it to `2.x`. 32 | * **1.3**: The _next patch_ release. Once a change is merged into `main`, decide whether to backport it to `1.3`. 33 | 34 | Label PRs with the next major version label (e.g. `3.0.0`) and merge changes into `main`. Label PRs that you believe need to be backported as `2.x`, `1.x` and `1.3`. Backport PRs by checking out the versioned branch, cherry-pick changes and open a PR against each target backport branch. 35 | 36 | ### Plugin Branching 37 | 38 | Plugins are bundled and shiped together along with OpenSearch for every release. Plugin branching follows OpenSearch core branching that will allow working on 3 releases at the same time. 39 | 40 | ### Feature Branches 41 | 42 | Do not creating branches in the upstream repo, use your fork, for the exception of long lasting feature branches that require active collaboration from multiple developers. Name feature branches `feature/`. Once the work is merged to `main`, please make sure to delete the feature branch. 43 | 44 | ## Versioning 45 | 46 | OpenSearch versioning [follows semver](https://opensearch.org/blog/technical-post/2021/08/what-is-semver/). 47 | 48 | ### Version Numbers 49 | 50 | The build number of the engine is 3-digit `major.minor.patch` (e.g. `2.9.0`), while plugins use 4 digits (`2.9.0.45`). See [OpenSearch#1093](https://github.com/opensearch-project/OpenSearch/issues/1093) for a proposal to remove this difference. 51 | 52 | ### Incrementing Versions 53 | 54 | Versions are incremented as soon as development starts on a given version to avoid confusion. In the examples above versions are as follows. 55 | 56 | * OpenSearch: `main` = 3.0.0, `2.x` = 2.10.0 `1.x` = 1.4.0, and `1.2` = 1.2.5 57 | * job-scheduler: `main` = 3.0.0.0, `2.x` = 2.10.0.0 , `1.x` = 1.4.0.0 and `1.2` = 1.2.5.0 58 | 59 | ## Tagging 60 | 61 | Create tags after a release that match the version number, `major.minor.patch`, without a `v` prefix. 62 | 63 | * [OpenSearch tags](https://github.com/opensearch-project/OpenSearch/tags): [1.0.0](https://github.com/opensearch-project/OpenSearch/releases/tag/1.0.0) 64 | * [job-scheduler tags](https://github.com/opensearch-project/job-scheduler/tags): [1.0.0.0](https://github.com/opensearch-project/job-scheduler/releases/tag/1.0.0.0) 65 | 66 | For a discussion on whether to add a prefixing `v` to release tags, see [#35](https://github.com/opensearch-project/.github/issues/35). 67 | 68 | ## Release Labels 69 | 70 | Repositories create consistent release labels, such as `v2.9.0`, `v1.0.0`, `v1.1.0` and `v2.0.0`, as well as `patch` and `backport`. Use release labels to target an issue or a PR for a given release. See [MAINTAINERS](MAINTAINERS.md#triage-open-issues) for more information on triaging issues. 71 | 72 | ## Releasing 73 | 74 | OpenSearch follows semver, which means we will only release breaking changes in major versions. All minor versions are compatible with every other minor version for that major. For example, 1.2.0 will work with 1.3.2, 1.4.1, etc, but may not work with 2.0. 75 | 76 | OpenSearch uses a “release-train” model. For minor version, that train leaves approximately every six weeks we release a new minor version which includes all the new features and fixes that are ready to go. Having a set release schedule makes sure OpenSearch is releasing in a predictable way and prevents a backlog of unreleased changes. 77 | 78 | In contrast, OpenSearch releases new major versions only when there are a critical mass of breaking changes (e.g. changes that are incompatible with existing APIs). These tend to be tied to Lucene major version releases, and will be announced in the forums at least 4 weeks prior to the release date. 79 | 80 | ### Release Process for OpenSearch Bundle Minor Releases and Patch Releases 81 | 82 | At the beginning of every year, the project will publish on [OpenSearch.org](https://opensearch.org/releases.html) the “release windows start” dates for the year. These dates will be spaced out ~6 weeks. 83 | 84 | On release window start date: 85 | 1. We generate the first candidate with all plug-ins/components that have met the entrance criteria. If a plug-in/component hasn't met all of the criteria, we'll version bump the last released version and release that. Once the release window opens and the first candidate is generated, no additioanl features can be added, and we will not delay the start of a release window for any plug-in/component. 86 | 1. During the release window we will do final quality testing, documentation and performance testing. Bug fixes can be added in during this time, but no new features will be added. 87 | 1. We will generate a new candidate every day and post on the release issue about the status of the exit criteria. When all the exit criteria have been met, we'll announce that the candidate is ready and release it 2 days later (unless that falls on Friday, in which case we'll release on Monday) 88 | 1. If we cannot clear the exit criteria within 2 weeks from the start of the window (1 week for patch releases), we will cancel that release window and try again in the next window. 89 | 90 | Please note: This process is for regularly scheduled minor and patch releases. For "hot" patches (patches required for security or other urgent issues) we will build, test and release as quickly as possible. 91 | 92 | #### Entrance Criteria to Start Release Window 93 | * Documentation draft PRs are up and in tech review for all component changes. 94 | * Sanity testing is done for all components. 95 | * Code coverage has not decreased (all new code has tests). 96 | * Release notes are ready and available for all components. 97 | * Roadmap is up-to-date (information is available to create release highlights). 98 | * Release ticket is cut, and there's a forum post announcing the start of the window. 99 | * [Any necessary security reviews](##Security-Reviews) are complete. 100 | 101 | #### Exit Criteria to Close Release Window 102 | * Performance tests are run, results are posted to the release ticket and there no unexpected regressions 103 | * Documentation has been fully reviewed and signed off by the documentation community. 104 | * All integration tests are passing 105 | * Release blog is ready 106 | 107 | ### Release Process for OpenSearch Major Releases 108 | 109 | OpenSearch only does major releases when there are significant breaking changes that are ready for release. Once we become aware of the need for a major version, we will start a major version release window which will be similar to a minor release, except for two things: Participation is mandatory for all components and the release window will be at least 4 weeks long to accommodate the testing required. 110 | 111 | For the actual steps to build a release, please see [Releasing OpenSearch](https://github.com/opensearch-project/opensearch-build/blob/main/README.md#releasing-opensearch). 112 | 113 | 114 | ### Security Reviews 115 | 116 | If you discover a potential security issue in this project we ask that you notify the OpenSearch Security Team by email at opensearch-security@amazon.com. Please do not create a public GitHub issue. See [SECURITY.md](SECURITY.md) for more information on the security response process. 117 | 118 | The OpenSearch Project currently performs security reviews before releasing signed artifacts. These are typically conducted for any of the following: 119 | 120 | * Releases from a new GitHub repository, such as a new plugin or extension, client, or tool. 121 | * Major new features added to an existing application, including significant UX or API changes. 122 | * Changes to authentication, authorization, cryptography, or other security-impacting functions. 123 | * New software or infrastructure deployed to support the project, such as CI/CD. 124 | * New 3rd-party providers or vendors being used by the project. 125 | 126 | The review process consists of building a threat model for the proposed change and optionally engaging a specialist to perform additional testing, such as a penetration testing. This process is done in parallel and in private within the project, during development, and usually takes 4-10 weeks. A repository maintainer will assess the scope of the new changes, initiate and manage a security review, provide public updates, and, if needed, communicate privately by email with the contributors. Please add a note in your pull request if you believe a security review is warranted. 127 | 128 | Please see [opensearch-project/.github#81](https://github.com/opensearch-project/.github/issues/81) for a discussion on improving this and other security-related processes.​ 129 | 130 | ## Backporting 131 | 132 | This project follows [semantic versioning](https://semver.org/spec/v2.0.0.html). Backwards-incompatible changes always result in a new major version and will __never__ be backported. Small improvements and features will be backported to a new minor version (e.g. `2.9.0`). Security fixes will be backported to a new patch version (e.g. `2.9.1`). 133 | 134 | Here are the commands we typically run to backport changes to release branches: 135 | 136 | 1. Checkout the target release branch and pull the latest changes from `upstream`. In the examples below, our target release branch is `2.x`. 137 | 138 | ``` 139 | git checkout 2.x 140 | git pull upstream 2.x 141 | ``` 142 | 143 | 2. Create a local branch for the backport. A convenient naming convention is _backport-\[PR-id\]-\[target-release-branch\]_. 144 | 145 | ``` 146 | git checkout -b backport-pr-xyz-1.x 147 | ``` 148 | 149 | 3. Cherry-pick the commit to backport. Remember to include [DCO signoff](CONTRIBUTING.md#developer-certificate-of-origin). 150 | 151 | ``` 152 | git cherry-pick -s 153 | ``` 154 | 155 | 4. Push the local branch to your fork. 156 | 157 | ``` 158 | git push origin backport-pr-xyz-1.x 159 | ``` 160 | 161 | 5. Create a pull request for the change. 162 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Issue Response Process 2 | 3 | ## Introduction 4 | 5 | Security issues happen as part of the normal lifecycle of software development of OpenSearch. This document describes the process for reporting these issues, responding to these issues, and ensuring we treat them not only with the appropriate priority but also with respect for the discoverers, software providers and their users, and the OpenSearch community in general. 6 | 7 | If you discover a potential security issue in this project we ask that you notify the OpenSearch Security Team via email to security@opensearch.org. Please do **not** create a public GitHub issue. 8 | 9 | *Giving credit where credit is due, this policy is heavily influenced by the [Xen Project’s security response process](https://xenproject.org/developers/security-policy/), that was put to the test during the [embargo period for XSA-108 back in 2014](https://xenproject.org/2014/10/22/xen-project-security-policy-improvements-get-involved/) and improved its clarity around managing the pre-disclosure list and the deployment of fixes during embargo. We are standing on the shoulders of these battle-tested giants.* 10 | 11 | ## The Security Team 12 | 13 | The OpenSearch Security Team is a subset of the project’s maintainers responsible for looking after the project’s security, including the security issue response process outlined below. Maintainers can be added to the Security Team by submitting a PR updating this document and adding their name to the list below (the process for becoming a maintaner can be found [here](https://github.com/opensearch-project/.github/blob/main/MAINTAINERS.md#becoming-a-maintainer)). 14 | 15 | The OpenSearch Security Team will address reported issues on a best effort basis, prioritizing them based on several factors, including severity. 16 | 17 | ### Current Members 18 | 19 | | Security Team member | GitHub Alias | Affiliation | 20 | | ------------------------ | --------------------------------------------- | ----------- | 21 | | Kunal Khatua | [kkhatua](https://github.com/kkhatua) | Amazon | 22 | | Daniel (dB.) Doubrovkine | [dblock](https://github.com/dblock) | Amazon | 23 | | Anirudha (Ani) Jadhav | [anirudha](https://github.com/anirudha) | Amazon | 24 | | Anan Zhuang | [ananzh](https://github.com/ananzh) | Amazon | 25 | | Eli Fisher | [elfisher](https://github.com/elfisher) | Amazon | 26 | 27 | ## Process 28 | 29 | Anyone finding an issue that is already publicly disclosed (for example, a CVE in one of the project’s dependencies) should feel free to create an issue and discuss openly on GitHub. The process below is only intended for issues that have not been publicly disclosed yet. 30 | 31 | 1. We request that instead of opening a GitHub issue, it is reported via email at security@opensearch.org. Please include a description of the issue, and any other information that could help in the reproduction and creation of a fix (version numbers, configuration values, reproduction steps...) 32 | 2. The OpenSearch Security Team will negotiate the conditions for an embargo period and a disclosure timeline with the discoverer (see [Embargo schedule](#embargo-schedule)). 33 | 3. After the vulnerability is confirmed, if no CVE number is already reserved, the OpenSearch Security Team will reserve one, and communicate it to the discoverer and all parties in the pre-disclosure list (see [Pre-disclosure list](#pre-disclosure-list)). 34 | 4. As soon as our advisory is available, we will send it, including patches, to members of the pre-disclosure list. In the event that we do not have a patch available 2 working weeks before the disclosure date, we aim to send an advisory that reflects the current state of knowledge to the pre-disclosure list. An updated advisory will be published as soon as available. At this stage, the advisory will be clearly marked with the embargo date. 35 | 5. On the day the embargo is lifted, we will publish the advisory and release the new versions containing the fix. 36 | 6. If new information and/or better fixes become available, new advisory versions will be released. 37 | 38 | During an embargo period, the OpenSearch Security Team may be required to make potentially controversial decisions in private, since they cannot confer with the community without breaking the embargo. The team will attempt to make such decisions following the guidance of this document and, where necessary, their own best judgement. Following the embargo period, on a best effort basis, the Security Team will disclose any such decisions, including the reasoning behind them, in the interests of transparency and to help provide guidance should a similar decision be required in the future. 39 | 40 | ## Embargo Schedule 41 | 42 | Embargo periods will be negotiated on a case-by-case basis depending on the severity of the issue and availability of the fix, where a general starting point is to release an advisory to the pre-disclosure list within 2 weeks of the initial notification, and publicly releasing the advisory within 4 weeks of the advisory pre-release. 43 | 44 | When a discoverer reports a problem to us and requests longer delays than we would consider ideal, we will honor such a request if reasonable. If a discoverer wants an accelerated disclosure compared to what we would prefer, we naturally do not have the power to insist that a discoverer waits for us to be ready and will honor the date specified by the discoverer. 45 | Naturally, if a vulnerability is being exploited in the wild, we will make immediately public release of the patch(es.) 46 | 47 | ## Pre-disclosure List 48 | 49 | We maintain a pre-disclosure list of contributors, vendors and operators of OpenSearch for two main reasons: 50 | 51 | 1. To apply the fixes to large user populations requiring a significant amount of work post-disclosure 52 | 2. To privately collaborate with those who can help write and test the fixes so we can release them as soon as possible and with high confidence in their quality 53 | 54 | If there is an embargo, the pre-disclosure list will receive copies of the advisory and patches, with a clearly marked embargo date, as soon as they are available. The pre-disclosure list will also receive copies of public advisories when they are first issued or updated. 55 | 56 | We expect list members to maintain the confidentiality of the vulnerability up to the embargo date. Specifically, prior to the embargo date, pre-disclosure list members should not make available, even to their own customers and partners: 57 | 58 | * The OpenSearch advisory 59 | * Their own advisory 60 | * The impact, scope, set of vulnerable systems or the nature of the vulnerability 61 | * Revision control commits which are a fix for the problem 62 | * Patched software (even in binary form) 63 | 64 | Without prior consultation with the OpenSearch Security Team, list members may make available to their users only: 65 | 66 | * The existence of an issue 67 | * The assigned OpenSearch advisory number 68 | * The planned disclosure date 69 | 70 | List members may, if (and only if) the OpenSearch Security Team grants permission, deploy fixed versions during the embargo. Permission for deployment, and any restrictions, will be stated in the embargoed advisory text. Where the list member is a service provider who intends to take disruptive action such as rebooting as part of deploying a fix: the list member’s communications to its users about the service disruption may mention that the disruption is to correct a security issue, and relate it to the public information about the issue (as listed above). This applies whether the deployment occurs during the embargo (with permission–see above) or is planned for after the end of the embargo. 71 | 72 | Pre-disclosure list members are allowed to share fixes to embargoed issues, analysis, etc., with the OpenSearch Security Team and security teams of other list members. Technical measures must be taken to prevent non-list-member organizations, or unauthorized staff in list-member organizations, from obtaining the embargoed materials. 73 | 74 | ## Pre-disclosure list application process 75 | 76 | Organizations who meet the criteria above (i.e. significant work needed post-disclosure to remediate the issue and/or ability to help create or test the potential fixes) should contact the OpenSearch Security Team via email at opensearch-security@amazon.com if they wish to be added to the pre-disclosure list. In the email, you must include: 77 | 78 | * The name of your organization 79 | * How you’re using OpenSearch 80 | * A description of why you fit the criteria (number of users, amount of work needed to remediate, ability to collaborate on fixes...) 81 | * Information about your handling of security problems 82 | * Your invitation to members of the public, who discover security problems with your products/services, to report them in confidence to you 83 | * Specifically, the contact information (email addresses or other contact instructions) which such a member of the public should use 84 | * A statement to the effect that you have read this policy and agree to abide by the terms for inclusion in the list, specifically the requirements to regarding confidentiality during an embargo period 85 | * The email(s) you wish added to the pre-disclosure list 86 | 87 | 88 | The OpenSearch Security Team will review your application and get back to you with their decision. 89 | -------------------------------------------------------------------------------- /build.gradle: -------------------------------------------------------------------------------- 1 | import org.opensearch.gradle.test.RestIntegTestTask 2 | 3 | import java.util.concurrent.Callable 4 | 5 | apply plugin: 'java' 6 | apply plugin: 'idea' 7 | apply plugin: 'opensearch.opensearchplugin' 8 | apply plugin: 'opensearch.pluginzip' 9 | apply plugin: 'opensearch.yaml-rest-test' 10 | apply plugin: 'maven-publish' 11 | 12 | ext.opensearch_tmp_dir = rootProject.file('build/private/opensearch_tmp').absoluteFile 13 | opensearch_tmp_dir.mkdirs() 14 | 15 | configurations { 16 | zipArchive 17 | secureIntegTestPluginArchive 18 | } 19 | 20 | opensearchplugin { 21 | name 'opensearch-ubi' 22 | description 'OpenSearch User Behavior Insights Plugin' 23 | classname 'org.opensearch.ubi.UbiPlugin' 24 | licenseFile rootProject.file('LICENSE') 25 | noticeFile rootProject.file('NOTICE') 26 | } 27 | 28 | group = 'org.opensearch' 29 | version = "${ubiVersion}" 30 | 31 | // disabling some unnecessary validations for this plugin 32 | testingConventions.enabled = false 33 | loggerUsageCheck.enabled = false 34 | validateNebulaPom.enabled = false 35 | forbiddenApisTest.enabled = false 36 | 37 | buildscript { 38 | ext { 39 | opensearch_version = System.getProperty("opensearch.version", "3.1.0-SNAPSHOT") 40 | buildVersionQualifier = System.getProperty("build.version_qualifier", "") 41 | isSnapshot = "true" == System.getProperty("build.snapshot", "true") 42 | version_tokens = opensearch_version.tokenize('-') 43 | opensearch_build = version_tokens[0] + '.0' 44 | plugin_no_snapshot = opensearch_build 45 | if (buildVersionQualifier) { 46 | opensearch_build += "-${buildVersionQualifier}" 47 | plugin_no_snapshot += "-${buildVersionQualifier}" 48 | } 49 | if (isSnapshot) { 50 | opensearch_build += "-SNAPSHOT" 51 | } 52 | opensearch_group = "org.opensearch" 53 | opensearch_no_snapshot = opensearch_build.replace("-SNAPSHOT","") 54 | } 55 | 56 | repositories { 57 | mavenLocal() 58 | maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } 59 | mavenCentral() 60 | maven { url "https://plugins.gradle.org/m2/" } 61 | } 62 | 63 | dependencies { 64 | classpath "org.opensearch.gradle:build-tools:${opensearchVersion}" 65 | } 66 | } 67 | 68 | repositories { 69 | mavenLocal() 70 | maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } 71 | mavenCentral() 72 | maven { url "https://plugins.gradle.org/m2/" } 73 | } 74 | 75 | thirdPartyAudit.enabled = false 76 | 77 | dependencies { 78 | runtimeOnly "org.apache.logging.log4j:log4j-core:${versions.log4j}" 79 | api "com.fasterxml.jackson.core:jackson-core:${versions.jackson}" 80 | api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}" 81 | api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}" 82 | api "org.apache.httpcomponents:httpcore:${versions.httpcore}" 83 | api "org.apache.httpcomponents:httpclient:${versions.httpclient}" 84 | api "commons-logging:commons-logging:${versions.commonslogging}" 85 | zipArchive group: 'org.opensearch.plugin', name:'opensearch-job-scheduler', version: "${opensearch_build}" 86 | 87 | testImplementation("org.opensearch:opensearch-agent-bootstrap:${opensearchVersion}") 88 | 89 | yamlRestTestImplementation("org.opensearch:opensearch-agent-bootstrap:${opensearchVersion}") 90 | yamlRestTestImplementation "org.apache.logging.log4j:log4j-core:${versions.log4j}" 91 | } 92 | 93 | test { 94 | systemProperty 'tests.security.manager', 'false' 95 | } 96 | 97 | yamlRestTest { 98 | systemProperty 'tests.security.manager', 'false' 99 | } 100 | 101 | publishing { 102 | repositories { 103 | maven { 104 | name = "Snapshots" 105 | url = "https://aws.oss.sonatype.org/content/repositories/snapshots" 106 | credentials { 107 | username "$System.env.SONATYPE_USERNAME" 108 | password "$System.env.SONATYPE_PASSWORD" 109 | } 110 | } 111 | } 112 | publications { 113 | pluginZip(MavenPublication) { publication -> 114 | pom { 115 | name = "opensearch-ubi" 116 | description = "OpenSearch User Behavior Insights plugin" 117 | groupId = "org.opensearch.plugin" 118 | licenses { 119 | license { 120 | name = "The Apache License, Version 2.0" 121 | url = "http://www.apache.org/licenses/LICENSE-2.0.txt" 122 | } 123 | } 124 | developers { 125 | developer { 126 | name = "OpenSearch" 127 | url = "https://github.com/opensearch-project/user-behavior-insights" 128 | } 129 | } 130 | } 131 | } 132 | } 133 | } 134 | 135 | def _numNodes = findProperty('numNodes') as Integer ?: 1 136 | 137 | // Setting up Integration Tests 138 | task integTest(type: RestIntegTestTask) { 139 | description = "Run tests against a cluster" 140 | testClassesDirs = sourceSets.test.output.classesDirs 141 | classpath = sourceSets.test.runtimeClasspath 142 | } 143 | tasks.named("check").configure { dependsOn(integTest) } 144 | 145 | integTest { 146 | systemProperty 'tests.security.manager', 'false' 147 | systemProperty 'java.io.tmpdir', opensearch_tmp_dir.absolutePath 148 | // allows integration test classes to access test resource from project root path 149 | systemProperty('project.root', project.rootDir.absolutePath) 150 | systemProperty 'security.enabled', System.getProperty('security.enabled') 151 | var is_https = System.getProperty("https") 152 | var user = System.getProperty("user") 153 | var password = System.getProperty("password") 154 | if (System.getProperty("security.enabled") != null) { 155 | // If security is enabled, set is_https/user/password defaults 156 | is_https = is_https == null ? "true" : is_https 157 | user = user == null ? "admin" : user 158 | password = password == null ? "admin" : password 159 | } 160 | systemProperty("https", is_https) 161 | systemProperty("user", user) 162 | systemProperty("password", password) 163 | 164 | doFirst { 165 | // Tell the test JVM if the cluster JVM is running under a debugger so that tests can 166 | // use longer timeouts for requests. 167 | def isDebuggingCluster = getDebug() || System.getProperty("test.debug") != null 168 | systemProperty 'cluster.debug', isDebuggingCluster 169 | // Set number of nodes system property to be used in tests 170 | systemProperty 'cluster.number_of_nodes', "${_numNodes}" 171 | // There seems to be an issue when running multi node run or integ tasks with unicast_hosts 172 | // not being written, the waitForAllConditions ensures it's written 173 | getClusters().forEach { cluster -> 174 | cluster.waitForAllConditions() 175 | } 176 | } 177 | 178 | // The --debug-jvm command-line option makes the cluster debuggable; this makes the tests debuggable 179 | if (System.getProperty("test.debug") != null) { 180 | jvmArgs '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005' 181 | } 182 | } 183 | 184 | testClusters.integTest { 185 | testDistribution = "ARCHIVE" 186 | 187 | // Optionally install security 188 | if (System.getProperty("security.enabled") != null && System.getProperty("security.enabled") == "true") { 189 | configureSecurityPlugin(testClusters.integTest) 190 | } 191 | 192 | // Install plugins on the integTest cluster nodes 193 | configurations.zipArchive.asFileTree.each { 194 | plugin(provider(new Callable(){ 195 | @Override 196 | RegularFile call() throws Exception { 197 | return new RegularFile() { 198 | @Override 199 | File getAsFile() { 200 | return it 201 | } 202 | } 203 | } 204 | })) 205 | } 206 | 207 | plugin(project.tasks.bundlePlugin.archiveFile) 208 | // Cluster shrink exception thrown if we try to set numberOfNodes to 1, so only apply if > 1 209 | if (_numNodes > 1) numberOfNodes = _numNodes 210 | // When running integration tests it doesn't forward the --debug-jvm to the cluster anymore 211 | // i.e. we have to use a custom property to flag when we want to debug opensearch JVM 212 | // since we also support multi node integration tests we increase debugPort per node 213 | if (System.getProperty("cluster.debug") != null) { 214 | def debugPort = 5005 215 | nodes.forEach { node -> 216 | node.jvmArgs("-agentlib:jdwp=transport=dt_socket,server=n,suspend=y,address=${debugPort}") 217 | debugPort += 1 218 | } 219 | } 220 | } -------------------------------------------------------------------------------- /dataprepper/data-prepper-config.yaml: -------------------------------------------------------------------------------- 1 | ssl: false 2 | -------------------------------------------------------------------------------- /dataprepper/pipelines.yaml: -------------------------------------------------------------------------------- 1 | chorus-ubi-pipeline: 2 | source: 3 | http: 4 | port: 2021 5 | ssl: false 6 | sink: 7 | - opensearch: 8 | hosts: [ "http://ubi-dev-os:9200" ] 9 | index_type: custom 10 | index: ubi_queries 11 | -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | 3 | # Uncomment to use OTel or Data Prepper -> OpenSearch pipelines. 4 | # dataprepper-dev-os: 5 | # depends_on: 6 | # - ubi-dev-os 7 | # container_name: dataprepper 8 | # image: opensearchproject/data-prepper:2.8.0 9 | # ports: 10 | # - 4900:4900 11 | # - 2021:2021 12 | # volumes: 13 | # - ./dataprepper/pipelines.yaml:/usr/share/data-prepper/pipelines/pipelines.yaml 14 | # - ./dataprepper/data-prepper-config.yaml:/usr/share/data-prepper/config/data-prepper-config.yaml 15 | # networks: 16 | # - ubi-dev-os-net 17 | 18 | ubi-dev-os: 19 | build: ./ 20 | container_name: ubi-dev-os 21 | environment: 22 | discovery.type: single-node 23 | node.name: opensearch 24 | plugins.security.disabled: "true" 25 | logger.level: info 26 | OPENSEARCH_INITIAL_ADMIN_PASSWORD: SuperSecretPassword_123 27 | # Requires the Data Prepper container: 28 | # ubi.dataprepper.url: "http://dataprepper-dev-os:2021/log/ingest" 29 | # Requires the OTel plugin to be installed. 30 | #telemetry.feature.tracer.enabled: true 31 | #telemetry.tracer.enabled: true 32 | #telemetry.tracer.sampler.probability: 1.0 33 | #opensearch.experimental.feature.telemetry.enabled: true 34 | #telemetry.otel.tracer.span.exporter.class: io.opentelemetry.exporter.logging.LoggingSpanExporter 35 | #telemetry.otel.tracer.exporter.batch_size: 1 36 | #telemetry.otel.tracer.exporter.max_queue_size: 3 37 | ulimits: 38 | memlock: 39 | soft: -1 40 | hard: -1 41 | nofile: 42 | soft: 65536 43 | hard: 65536 44 | ports: 45 | - 9200:9200 46 | - 9600:9600 47 | expose: 48 | - 9200 49 | - 9600 50 | networks: 51 | - ubi-dev-os-net 52 | 53 | networks: 54 | ubi-dev-os-net: 55 | driver: bridge 56 | -------------------------------------------------------------------------------- /gradle.properties: -------------------------------------------------------------------------------- 1 | opensearchVersion = 3.1.0-SNAPSHOT 2 | ubiVersion = 3.1.0.0-SNAPSHOT 3 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opensearch-project/user-behavior-insights/f33eb4adae45cf38ea33b347fd977f91904947aa/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-8.5-bin.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists -------------------------------------------------------------------------------- /gradlew: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | # 4 | # Copyright 2015 the original author or authors. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # https://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | ############################################################################## 20 | ## 21 | ## Gradle start up script for UN*X 22 | ## 23 | ############################################################################## 24 | 25 | # Attempt to set APP_HOME 26 | # Resolve links: $0 may be a link 27 | PRG="$0" 28 | # Need this for relative symlinks. 29 | while [ -h "$PRG" ] ; do 30 | ls=`ls -ld "$PRG"` 31 | link=`expr "$ls" : '.*-> \(.*\)$'` 32 | if expr "$link" : '/.*' > /dev/null; then 33 | PRG="$link" 34 | else 35 | PRG=`dirname "$PRG"`"/$link" 36 | fi 37 | done 38 | SAVED="`pwd`" 39 | cd "`dirname \"$PRG\"`/" >/dev/null 40 | APP_HOME="`pwd -P`" 41 | cd "$SAVED" >/dev/null 42 | 43 | APP_NAME="Gradle" 44 | APP_BASE_NAME=`basename "$0"` 45 | 46 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 47 | DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' 48 | 49 | # Use the maximum available, or set MAX_FD != -1 to use that value. 50 | MAX_FD="maximum" 51 | 52 | warn () { 53 | echo "$*" 54 | } 55 | 56 | die () { 57 | echo 58 | echo "$*" 59 | echo 60 | exit 1 61 | } 62 | 63 | # OS specific support (must be 'true' or 'false'). 64 | cygwin=false 65 | msys=false 66 | darwin=false 67 | nonstop=false 68 | case "`uname`" in 69 | CYGWIN* ) 70 | cygwin=true 71 | ;; 72 | Darwin* ) 73 | darwin=true 74 | ;; 75 | MINGW* ) 76 | msys=true 77 | ;; 78 | NONSTOP* ) 79 | nonstop=true 80 | ;; 81 | esac 82 | 83 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 84 | 85 | 86 | # Determine the Java command to use to start the JVM. 87 | if [ -n "$JAVA_HOME" ] ; then 88 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 89 | # IBM's JDK on AIX uses strange locations for the executables 90 | JAVACMD="$JAVA_HOME/jre/sh/java" 91 | else 92 | JAVACMD="$JAVA_HOME/bin/java" 93 | fi 94 | if [ ! -x "$JAVACMD" ] ; then 95 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 96 | 97 | Please set the JAVA_HOME variable in your environment to match the 98 | location of your Java installation." 99 | fi 100 | else 101 | JAVACMD="java" 102 | which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 103 | 104 | Please set the JAVA_HOME variable in your environment to match the 105 | location of your Java installation." 106 | fi 107 | 108 | # Increase the maximum file descriptors if we can. 109 | if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then 110 | MAX_FD_LIMIT=`ulimit -H -n` 111 | if [ $? -eq 0 ] ; then 112 | if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then 113 | MAX_FD="$MAX_FD_LIMIT" 114 | fi 115 | ulimit -n $MAX_FD 116 | if [ $? -ne 0 ] ; then 117 | warn "Could not set maximum file descriptor limit: $MAX_FD" 118 | fi 119 | else 120 | warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" 121 | fi 122 | fi 123 | 124 | # For Darwin, add options to specify how the application appears in the dock 125 | if $darwin; then 126 | GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" 127 | fi 128 | 129 | # For Cygwin or MSYS, switch paths to Windows format before running java 130 | if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then 131 | APP_HOME=`cygpath --path --mixed "$APP_HOME"` 132 | CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` 133 | 134 | JAVACMD=`cygpath --unix "$JAVACMD"` 135 | 136 | # We build the pattern for arguments to be converted via cygpath 137 | ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` 138 | SEP="" 139 | for dir in $ROOTDIRSRAW ; do 140 | ROOTDIRS="$ROOTDIRS$SEP$dir" 141 | SEP="|" 142 | done 143 | OURCYGPATTERN="(^($ROOTDIRS))" 144 | # Add a user-defined pattern to the cygpath arguments 145 | if [ "$GRADLE_CYGPATTERN" != "" ] ; then 146 | OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" 147 | fi 148 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 149 | i=0 150 | for arg in "$@" ; do 151 | CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` 152 | CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option 153 | 154 | if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition 155 | eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` 156 | else 157 | eval `echo args$i`="\"$arg\"" 158 | fi 159 | i=`expr $i + 1` 160 | done 161 | case $i in 162 | 0) set -- ;; 163 | 1) set -- "$args0" ;; 164 | 2) set -- "$args0" "$args1" ;; 165 | 3) set -- "$args0" "$args1" "$args2" ;; 166 | 4) set -- "$args0" "$args1" "$args2" "$args3" ;; 167 | 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; 168 | 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; 169 | 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; 170 | 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; 171 | 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; 172 | esac 173 | fi 174 | 175 | # Escape application args 176 | save () { 177 | for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done 178 | echo " " 179 | } 180 | APP_ARGS=`save "$@"` 181 | 182 | # Collect all arguments for the java command, following the shell quoting and substitution rules 183 | eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" 184 | 185 | exec "$JAVACMD" "$@" 186 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @rem 2 | @rem Copyright 2015 the original author or authors. 3 | @rem 4 | @rem Licensed under the Apache License, Version 2.0 (the "License"); 5 | @rem you may not use this file except in compliance with the License. 6 | @rem You may obtain a copy of the License at 7 | @rem 8 | @rem https://www.apache.org/licenses/LICENSE-2.0 9 | @rem 10 | @rem Unless required by applicable law or agreed to in writing, software 11 | @rem distributed under the License is distributed on an "AS IS" BASIS, 12 | @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | @rem See the License for the specific language governing permissions and 14 | @rem limitations under the License. 15 | @rem 16 | 17 | @if "%DEBUG%" == "" @echo off 18 | @rem ########################################################################## 19 | @rem 20 | @rem Gradle startup script for Windows 21 | @rem 22 | @rem ########################################################################## 23 | 24 | @rem Set local scope for the variables with windows NT shell 25 | if "%OS%"=="Windows_NT" setlocal 26 | 27 | set DIRNAME=%~dp0 28 | if "%DIRNAME%" == "" set DIRNAME=. 29 | set APP_BASE_NAME=%~n0 30 | set APP_HOME=%DIRNAME% 31 | 32 | @rem Resolve any "." and ".." in APP_HOME to make it shorter. 33 | for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi 34 | 35 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 36 | set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" 37 | 38 | @rem Find java.exe 39 | if defined JAVA_HOME goto findJavaFromJavaHome 40 | 41 | set JAVA_EXE=java.exe 42 | %JAVA_EXE% -version >NUL 2>&1 43 | if "%ERRORLEVEL%" == "0" goto execute 44 | 45 | echo. 46 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 47 | echo. 48 | echo Please set the JAVA_HOME variable in your environment to match the 49 | echo location of your Java installation. 50 | 51 | goto fail 52 | 53 | :findJavaFromJavaHome 54 | set JAVA_HOME=%JAVA_HOME:"=% 55 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 56 | 57 | if exist "%JAVA_EXE%" goto execute 58 | 59 | echo. 60 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 61 | echo. 62 | echo Please set the JAVA_HOME variable in your environment to match the 63 | echo location of your Java installation. 64 | 65 | goto fail 66 | 67 | :execute 68 | @rem Setup the command line 69 | 70 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 71 | 72 | 73 | @rem Execute Gradle 74 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* 75 | 76 | :end 77 | @rem End local scope for the variables with windows NT shell 78 | if "%ERRORLEVEL%"=="0" goto mainEnd 79 | 80 | :fail 81 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 82 | rem the _cmd.exe /c_ return code! 83 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 84 | exit /b 1 85 | 86 | :mainEnd 87 | if "%OS%"=="Windows_NT" endlocal 88 | 89 | :omega 90 | -------------------------------------------------------------------------------- /licenses/commons-logging-1.2.jar.sha1: -------------------------------------------------------------------------------- 1 | 4bfc12adfe4842bf07b657f0369c4cb522955686 -------------------------------------------------------------------------------- /licenses/commons-logging-LICENSE.txt: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | -------------------------------------------------------------------------------- /licenses/commons-logging-NOTICE.txt: -------------------------------------------------------------------------------- 1 | Apache HttpComponents Core 2 | Copyright 2005-2024 The Apache Software Foundation 3 | 4 | This product includes software developed at 5 | The Apache Software Foundation (http://www.apache.org/). -------------------------------------------------------------------------------- /licenses/httpclient-4.5.14.jar.sha1: -------------------------------------------------------------------------------- 1 | 1194890e6f56ec29177673f2f12d0b8e627dec98 -------------------------------------------------------------------------------- /licenses/httpclient-LICENSE.txt: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | -------------------------------------------------------------------------------- /licenses/httpclient-NOTICE.txt: -------------------------------------------------------------------------------- 1 | Apache HttpComponents Core 2 | Copyright 2005-2024 The Apache Software Foundation 3 | 4 | This product includes software developed at 5 | The Apache Software Foundation (http://www.apache.org/). -------------------------------------------------------------------------------- /licenses/httpcore-4.4.16.jar.sha1: -------------------------------------------------------------------------------- 1 | 51cf043c87253c9f58b539c9f7e44c8894223850 -------------------------------------------------------------------------------- /licenses/httpcore-LICENSE.txt: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | -------------------------------------------------------------------------------- /licenses/httpcore-NOTICE.txt: -------------------------------------------------------------------------------- 1 | Apache HttpComponents Core 2 | Copyright 2005-2024 The Apache Software Foundation 3 | 4 | This product includes software developed at 5 | The Apache Software Foundation (http://www.apache.org/). -------------------------------------------------------------------------------- /licenses/jackson-annotations-2.18.2.jar.sha1: -------------------------------------------------------------------------------- 1 | 985d77751ebc7fce5db115a986bc9aa82f973f4a -------------------------------------------------------------------------------- /licenses/jackson-annotations-LICENSE.txt: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. -------------------------------------------------------------------------------- /licenses/jackson-annotations-NOTICE.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opensearch-project/user-behavior-insights/f33eb4adae45cf38ea33b347fd977f91904947aa/licenses/jackson-annotations-NOTICE.txt -------------------------------------------------------------------------------- /licenses/jackson-databind-2.18.2.jar.sha1: -------------------------------------------------------------------------------- 1 | deef8697b92141fb6caf7aa86966cff4eec9b04f -------------------------------------------------------------------------------- /licenses/jackson-databind-LICENSE.txt: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. -------------------------------------------------------------------------------- /licenses/jackson-databind-NOTICE.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opensearch-project/user-behavior-insights/f33eb4adae45cf38ea33b347fd977f91904947aa/licenses/jackson-databind-NOTICE.txt -------------------------------------------------------------------------------- /scripts/get-indexed-queries.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | # Get the indexed queries. 4 | 5 | curl http://localhost:9200/ubi_queries/_search | jq 6 | -------------------------------------------------------------------------------- /scripts/initialize-ubi.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | curl -s -X DELETE http://localhost:9200/ubi_queries,ubi_events | jq 4 | 5 | curl -s -X POST http://localhost:9200/_plugins/ubi/initialize | jq 6 | -------------------------------------------------------------------------------- /scripts/msearch.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | curl -s -X GET "http://localhost:9200/_msearch" -H 'Content-Type: application/json' -d' 4 | { "index": "ecommerce"} 5 | { "query": { "match_all": {} }, "ext": { "ubi": { "query_id": "11111" } } } 6 | { "index": "ecommerce"} 7 | { "query": { "match_all": {} }, "ext": { "ubi": { "query_id": "22222" } } } 8 | ' -------------------------------------------------------------------------------- /scripts/search.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | # An example UBI search of the ecommerce index. 4 | 5 | curl -s http://localhost:9200/ecommerce/_search -H "Content-Type: application/json" -d' 6 | { 7 | "ext": { 8 | "ubi": { 9 | } 10 | }, 11 | "query": { 12 | "match": { 13 | "name": "toner" 14 | } 15 | } 16 | }' | jq 17 | -------------------------------------------------------------------------------- /search-quality-evaluation-dashboards/README.md: -------------------------------------------------------------------------------- 1 | 2 | # How to load dashboards into your OpenSearch installation 3 | 4 | Run the following shell script: `./install_dashboards.sh http://localhost:9200 http://localhost:5601` 5 | 6 | * `sample_search_quality_evaluation_data.ndjson` represents the data produced by running the search quality evaluation framework. 7 | * `search_quality_evaluation_dashboard.ndjson` represents the Search Quality Evaluation Dashboards needed to analyze and understand the data. 8 | -------------------------------------------------------------------------------- /search-quality-evaluation-dashboards/install_dashboards.sh: -------------------------------------------------------------------------------- 1 | 2 | # Ansi color code variables 3 | ERROR='\033[0;31m' 4 | MAJOR='\033[0;34m' 5 | MINOR='\033[0;37m ' 6 | RESET='\033[0m' # No Color 7 | 8 | opensearch=$1 9 | opensearch_dashboard=$2 10 | set -eo pipefail 11 | 12 | if [ -z "$opensearch" ]; then 13 | echo "Error: please pass in both the opensearch url and the opensearch dashboard url" 14 | exit 1 15 | fi 16 | if [ -z "$opensearch_dashboard" ]; then 17 | echo "Error: please pass in both the opensearch url and the opensearch dashboard url" 18 | exit 1 19 | fi 20 | 21 | echo "${MAJOR}Using Open Search and Open Search Dashboards at $opensearch and $opensearch_dashboard respectively.${RESET}" 22 | 23 | echo "${MAJOR}Deleting index sqe_metrics_sample_data${RESET}" 24 | curl -s -X DELETE $opensearch/sqe_metrics_sample_data/ 25 | 26 | echo "${MAJOR}Populating index sqe_metrics_sample_data with sample metric data${RESET}" 27 | curl -s -H 'Content-Type: application/x-ndjson' -XPOST "$opensearch/sqe_metrics_sample_data/_bulk?pretty=false&filter_path=-items" --data-binary @sample_search_quality_evaluation_data.ndjson 28 | 29 | echo "${MAJOR}\nInstalling Quality Evaluation Framework Dashboards${RESET}" 30 | curl -X POST "$opensearch_dashboard/api/saved_objects/_import?overwrite=true" -H "osd-xsrf: true" --form file=@search_quality_evaluation_dashboard.ndjson > /dev/null 31 | 32 | echo "${MAJOR}The Search Quality Evaluation Framework Dashboards were successfully installed${RESET}" 33 | -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * This file was generated by the Gradle 'init' task. 3 | * 4 | * The settings file is used to specify which projects to include in your build. 5 | * 6 | * Detailed information about configuring a multi-project build in Gradle can be found 7 | * in the user manual at https://docs.gradle.org/6.5.1/userguide/multi_project_builds.html 8 | */ 9 | 10 | rootProject.name = 'user-behavior-insights-plugin' 11 | -------------------------------------------------------------------------------- /src/main/java/org/opensearch/ubi/QueryRequest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * SPDX-License-Identifier: Apache-2.0 3 | * 4 | * The OpenSearch Contributors require contributions made to 5 | * this file be licensed under the Apache-2.0 license or a 6 | * compatible open source license. 7 | */ 8 | 9 | package org.opensearch.ubi; 10 | 11 | import com.fasterxml.jackson.core.JsonProcessingException; 12 | import com.fasterxml.jackson.databind.ObjectMapper; 13 | 14 | import java.security.AccessController; 15 | import java.security.PrivilegedAction; 16 | import java.text.SimpleDateFormat; 17 | import java.util.Date; 18 | import java.util.Locale; 19 | import java.util.Map; 20 | 21 | /** 22 | * A query received by OpenSearch. 23 | */ 24 | public class QueryRequest { 25 | 26 | private final String timestamp; 27 | private final String queryId; 28 | private final String clientId; 29 | private final String userQuery; 30 | private final String query; 31 | private final String application; 32 | private final Map queryAttributes; 33 | private final QueryResponse queryResponse; 34 | 35 | private static final SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ", Locale.getDefault()); 36 | 37 | /** 38 | * Creates a query request. 39 | * @param queryId The ID of the query. 40 | * @param userQuery The user-entered query. 41 | * @param clientId The ID of the client that initiated the query. 42 | * @param query The raw query. 43 | * @param queryAttributes An optional map of additional attributes for the query. 44 | * @param queryResponse The {@link QueryResponse} for this query request. 45 | */ 46 | public QueryRequest(final String queryId, final String userQuery, final String clientId, final String query, 47 | final String application, final Map queryAttributes, 48 | final QueryResponse queryResponse) { 49 | 50 | this.timestamp = sdf.format(new Date()); 51 | this.queryId = queryId; 52 | this.clientId = clientId; 53 | this.userQuery = userQuery; 54 | this.query = query; 55 | this.application = application; 56 | this.queryAttributes = queryAttributes; 57 | this.queryResponse = queryResponse; 58 | 59 | } 60 | 61 | @Override 62 | public String toString() { 63 | 64 | final ObjectMapper objectMapper = new ObjectMapper(); 65 | 66 | final String json = AccessController.doPrivileged((PrivilegedAction) () -> { 67 | try { 68 | return objectMapper.writeValueAsString(this); 69 | } catch (JsonProcessingException ex) { 70 | throw new RuntimeException(ex); 71 | } 72 | }); 73 | 74 | return "[" + json + "]"; 75 | 76 | } 77 | 78 | /** 79 | * Gets the query attributes. 80 | * @return The query attributes. 81 | */ 82 | public Map getQueryAttributes() { 83 | return queryAttributes; 84 | } 85 | 86 | /** 87 | * Gets the timestamp. 88 | * @return The timestamp. 89 | */ 90 | public String getTimestamp() { 91 | return timestamp; 92 | } 93 | 94 | /** 95 | * Gets the application. 96 | * @return The application. 97 | */ 98 | public String getApplication() { 99 | return application; 100 | } 101 | 102 | /** 103 | * Gets the query ID. 104 | * @return The query ID. 105 | */ 106 | public String getQueryId() { 107 | return queryId; 108 | } 109 | 110 | /** 111 | * Gets the user query. 112 | * @return The user query. 113 | */ 114 | public String getUserQuery() { 115 | if(userQuery == null) { 116 | return ""; 117 | } 118 | return userQuery; 119 | } 120 | 121 | /** 122 | * Gets the client ID. 123 | * @return The client ID. 124 | */ 125 | public String getClientId() { 126 | if(clientId == null) { 127 | return ""; 128 | } 129 | return clientId; 130 | } 131 | 132 | /** 133 | * Gets the raw query. 134 | * @return The raw query. 135 | */ 136 | public String getQuery() { 137 | if(query == null) { 138 | return ""; 139 | } 140 | return query; 141 | } 142 | 143 | /** 144 | * Gets the query response for this query request. 145 | * @return The {@link QueryResponse} for this query request. 146 | */ 147 | public QueryResponse getQueryResponse() { 148 | return queryResponse; 149 | } 150 | 151 | } 152 | -------------------------------------------------------------------------------- /src/main/java/org/opensearch/ubi/QueryResponse.java: -------------------------------------------------------------------------------- 1 | /* 2 | * SPDX-License-Identifier: Apache-2.0 3 | * 4 | * The OpenSearch Contributors require contributions made to 5 | * this file be licensed under the Apache-2.0 license or a 6 | * compatible open source license. 7 | */ 8 | 9 | package org.opensearch.ubi; 10 | 11 | import java.util.List; 12 | 13 | /** 14 | * A query response. 15 | */ 16 | public class QueryResponse { 17 | 18 | private final String queryId; 19 | private final String queryResponseId; 20 | private final List queryResponseHitIds; 21 | 22 | /** 23 | * Creates a query response. 24 | * @param queryId The ID of the query. 25 | * @param queryResponseId The ID of the query response. 26 | * @param queryResponseHitIds A list of IDs for the hits in the query. 27 | */ 28 | public QueryResponse(final String queryId, final String queryResponseId, final List queryResponseHitIds) { 29 | this.queryId = queryId; 30 | this.queryResponseId = queryResponseId; 31 | this.queryResponseHitIds = queryResponseHitIds; 32 | } 33 | 34 | /** 35 | * Gets the query ID. 36 | * @return The query ID. 37 | */ 38 | public String getQueryId() { 39 | return queryId; 40 | } 41 | 42 | /** 43 | * Gets the query response ID. 44 | * @return The query response ID. 45 | */ 46 | public String getQueryResponseId() { 47 | return queryResponseId; 48 | } 49 | 50 | /** 51 | * Gets the list of query response hit IDs. 52 | * @return A list of query response hit IDs. 53 | */ 54 | public List getQueryResponseHitIds() { 55 | return queryResponseHitIds; 56 | } 57 | 58 | } 59 | -------------------------------------------------------------------------------- /src/main/java/org/opensearch/ubi/UbiActionFilter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * SPDX-License-Identifier: Apache-2.0 3 | * 4 | * The OpenSearch Contributors require contributions made to 5 | * this file be licensed under the Apache-2.0 license or a 6 | * compatible open source license. 7 | */ 8 | 9 | package org.opensearch.ubi; 10 | 11 | import org.apache.http.client.methods.CloseableHttpResponse; 12 | import org.apache.http.client.methods.HttpPost; 13 | import org.apache.http.entity.StringEntity; 14 | import org.apache.http.impl.client.CloseableHttpClient; 15 | import org.apache.http.impl.client.HttpClients; 16 | import org.apache.logging.log4j.LogManager; 17 | import org.apache.logging.log4j.Logger; 18 | import org.opensearch.action.ActionRequest; 19 | import org.opensearch.action.index.IndexRequest; 20 | import org.opensearch.action.index.IndexResponse; 21 | import org.opensearch.action.search.MultiSearchRequest; 22 | import org.opensearch.action.search.SearchRequest; 23 | import org.opensearch.action.search.SearchResponse; 24 | import org.opensearch.action.support.ActionFilter; 25 | import org.opensearch.action.support.ActionFilterChain; 26 | import org.opensearch.common.xcontent.XContentType; 27 | import org.opensearch.core.action.ActionListener; 28 | import org.opensearch.core.action.ActionResponse; 29 | import org.opensearch.env.Environment; 30 | import org.opensearch.search.SearchHit; 31 | import org.opensearch.tasks.Task; 32 | import org.opensearch.telemetry.tracing.Span; 33 | import org.opensearch.telemetry.tracing.SpanBuilder; 34 | import org.opensearch.telemetry.tracing.Tracer; 35 | import org.opensearch.transport.client.Client; 36 | import org.opensearch.ubi.ext.UbiParameters; 37 | 38 | import java.io.IOException; 39 | import java.util.HashMap; 40 | import java.util.LinkedList; 41 | import java.util.List; 42 | import java.util.Map; 43 | import java.util.UUID; 44 | 45 | import static org.opensearch.ubi.UbiPlugin.UBI_QUERIES_INDEX; 46 | 47 | /** 48 | * An implementation of {@link ActionFilter} that listens for OpenSearch 49 | * queries and persists the queries to the UBI store. 50 | */ 51 | public class UbiActionFilter implements ActionFilter { 52 | 53 | private static final Logger LOGGER = LogManager.getLogger(UbiActionFilter.class); 54 | 55 | private final Client client; 56 | private final Environment environment; 57 | private final Tracer tracer; 58 | 59 | /** 60 | * Creates a new filter. 61 | * @param client An OpenSearch {@link Client}. 62 | * @param environment The OpenSearch {@link Environment}. 63 | * @param tracer An Open Telemetry {@link Tracer tracer}. 64 | */ 65 | public UbiActionFilter(Client client, Environment environment, Tracer tracer) { 66 | 67 | this.client = client; 68 | this.environment = environment; 69 | this.tracer = tracer; 70 | } 71 | 72 | @Override 73 | public int order() { 74 | return Integer.MAX_VALUE; 75 | } 76 | 77 | @Override 78 | public void apply( 79 | Task task, 80 | String action, 81 | Request request, 82 | ActionListener listener, 83 | ActionFilterChain chain 84 | ) { 85 | 86 | if (!(request instanceof SearchRequest || request instanceof MultiSearchRequest)) { 87 | chain.proceed(task, action, request, listener); 88 | return; 89 | } 90 | 91 | chain.proceed(task, action, request, new ActionListener<>() { 92 | 93 | @Override 94 | public void onResponse(Response response) { 95 | 96 | if (request instanceof MultiSearchRequest) { 97 | 98 | final MultiSearchRequest multiSearchRequest = (MultiSearchRequest) request; 99 | 100 | for(final SearchRequest searchRequest : multiSearchRequest.requests()) { 101 | handleSearchRequest(searchRequest, response); 102 | } 103 | 104 | } 105 | 106 | if(request instanceof SearchRequest) { 107 | response = (Response) handleSearchRequest((SearchRequest) request, response); 108 | } 109 | 110 | listener.onResponse(response); 111 | 112 | } 113 | 114 | @Override 115 | public void onFailure(Exception ex) { 116 | listener.onFailure(ex); 117 | } 118 | 119 | }); 120 | 121 | } 122 | 123 | private ActionResponse handleSearchRequest(final SearchRequest searchRequest, ActionResponse response) { 124 | 125 | if (response instanceof SearchResponse) { 126 | 127 | final UbiParameters ubiParameters = UbiParameters.getUbiParameters(searchRequest); 128 | 129 | if (ubiParameters != null) { 130 | 131 | final String queryId = ubiParameters.getQueryId(); 132 | final String userQuery = ubiParameters.getUserQuery(); 133 | final String userId = ubiParameters.getClientId(); 134 | final String objectIdField = ubiParameters.getObjectIdField(); 135 | final String application = ubiParameters.getApplication(); 136 | final Map queryAttributes = ubiParameters.getQueryAttributes(); 137 | 138 | final String query = searchRequest.source().toString(); 139 | 140 | final List queryResponseHitIds = new LinkedList<>(); 141 | 142 | for (final SearchHit hit : ((SearchResponse) response).getHits()) { 143 | 144 | if (objectIdField == null || objectIdField.isEmpty()) { 145 | // Use the result's docId since no object_id was given for the search. 146 | queryResponseHitIds.add(String.valueOf(hit.docId())); 147 | } else { 148 | final Map source = hit.getSourceAsMap(); 149 | queryResponseHitIds.add((String) source.get(objectIdField)); 150 | } 151 | 152 | } 153 | 154 | final String queryResponseId = UUID.randomUUID().toString(); 155 | final QueryResponse queryResponse = new QueryResponse(queryId, queryResponseId, queryResponseHitIds); 156 | final QueryRequest queryRequest = new QueryRequest(queryId, userQuery, userId, query, application, queryAttributes, queryResponse); 157 | 158 | final String dataPrepperUrl = environment.settings().get(UbiSettings.DATA_PREPPER_URL); 159 | if (dataPrepperUrl != null) { 160 | sendToDataPrepper(dataPrepperUrl, queryRequest); 161 | } else { 162 | indexQuery(queryRequest); 163 | } 164 | 165 | final SearchResponse searchResponse = (SearchResponse) response; 166 | 167 | response = new UbiSearchResponse( 168 | searchResponse.getInternalResponse(), 169 | searchResponse.getScrollId(), 170 | searchResponse.getTotalShards(), 171 | searchResponse.getSuccessfulShards(), 172 | searchResponse.getSkippedShards(), 173 | searchResponse.getTook().millis(), 174 | searchResponse.getShardFailures(), 175 | searchResponse.getClusters(), 176 | queryId 177 | ); 178 | 179 | } 180 | 181 | } 182 | 183 | return response; 184 | 185 | } 186 | 187 | private void sendToDataPrepper(final String dataPrepperUrl, final QueryRequest queryRequest) { 188 | 189 | LOGGER.debug("Sending query to DataPrepper at {}", dataPrepperUrl); 190 | 191 | // TODO: Do this in a background thread? 192 | try { 193 | 194 | try (CloseableHttpClient httpClient = HttpClients.createDefault()) { 195 | 196 | final HttpPost httpPost = new HttpPost(dataPrepperUrl); 197 | 198 | httpPost.setEntity(new StringEntity(queryRequest.toString())); 199 | httpPost.setHeader("Content-type", "application/json"); 200 | 201 | try (CloseableHttpResponse response = httpClient.execute(httpPost)) { 202 | final int status = response.getStatusLine().getStatusCode(); 203 | if (status != 200) { 204 | LOGGER.error("Unexpected response status from Data Prepper: " + status); 205 | } 206 | } catch (Exception ex) { 207 | LOGGER.error("Unable to send query to Data Prepper", ex); 208 | } 209 | 210 | } 211 | 212 | } catch (IOException ex) { 213 | LOGGER.error("Failed to send query to Data Prepper", ex); 214 | } 215 | 216 | } 217 | 218 | private void indexQuery(final QueryRequest queryRequest) { 219 | 220 | LOGGER.debug( 221 | "Indexing query ID {} with response ID {}", 222 | queryRequest.getQueryId(), 223 | queryRequest.getQueryResponse().getQueryResponseId() 224 | ); 225 | 226 | // What will be indexed - adheres to the queries-mapping.json 227 | final Map source = new HashMap<>(); 228 | source.put("timestamp", queryRequest.getTimestamp()); 229 | source.put("query_id", queryRequest.getQueryId()); 230 | source.put("query_response_id", queryRequest.getQueryResponse().getQueryResponseId()); 231 | source.put("query_response_hit_ids", queryRequest.getQueryResponse().getQueryResponseHitIds()); 232 | source.put("client_id", queryRequest.getClientId()); 233 | source.put("application", queryRequest.getApplication()); 234 | source.put("user_query", queryRequest.getUserQuery()); 235 | source.put("query_attributes", queryRequest.getQueryAttributes()); 236 | 237 | // The query can be null for some types of queries. 238 | if(queryRequest.getQuery() != null) { 239 | source.put("query", queryRequest.getQuery()); 240 | } 241 | 242 | final IndexRequest indexRequest = new IndexRequest(); 243 | indexRequest.index(UBI_QUERIES_INDEX); 244 | indexRequest.source(source, XContentType.JSON); 245 | 246 | client.index(indexRequest, new ActionListener<>() { 247 | 248 | @Override 249 | public void onResponse(IndexResponse indexResponse) {} 250 | 251 | @Override 252 | public void onFailure(Exception e) { 253 | LOGGER.error("Unable to index query into " + UBI_QUERIES_INDEX + ".", e); 254 | } 255 | 256 | }); 257 | 258 | } 259 | 260 | private void sendOtelTrace(final Task task, final Tracer tracer, final QueryRequest queryRequest) { 261 | 262 | final Span span = tracer.startSpan(SpanBuilder.from(task, "ubi_search")); 263 | 264 | span.addAttribute("ubi.user_id", queryRequest.getQueryId()); 265 | span.addAttribute("ubi.query", queryRequest.getQuery()); 266 | span.addAttribute("ubi.user_query", queryRequest.getUserQuery()); 267 | span.addAttribute("ubi.client_id", queryRequest.getClientId()); 268 | span.addAttribute("ubi.timestamp", queryRequest.getTimestamp()); 269 | 270 | for (final String key : queryRequest.getQueryAttributes().keySet()) { 271 | span.addAttribute("ubi.attribute." + key, queryRequest.getQueryAttributes().get(key)); 272 | } 273 | 274 | span.addAttribute("ubi.query_response.response_id", queryRequest.getQueryResponse().getQueryResponseId()); 275 | span.addAttribute("ubi.query_response.query_id", queryRequest.getQueryResponse().getQueryId()); 276 | span.addAttribute("ubi.query_response.response_id", String.join(",", queryRequest.getQueryResponse().getQueryResponseHitIds())); 277 | 278 | span.endSpan(); 279 | 280 | } 281 | 282 | } 283 | -------------------------------------------------------------------------------- /src/main/java/org/opensearch/ubi/UbiPlugin.java: -------------------------------------------------------------------------------- 1 | /* 2 | * SPDX-License-Identifier: Apache-2.0 3 | * 4 | * The OpenSearch Contributors require contributions made to 5 | * this file be licensed under the Apache-2.0 license or a 6 | * compatible open source license. 7 | */ 8 | 9 | package org.opensearch.ubi; 10 | 11 | import org.opensearch.action.support.ActionFilter; 12 | import org.opensearch.cluster.metadata.IndexNameExpressionResolver; 13 | import org.opensearch.cluster.node.DiscoveryNodes; 14 | import org.opensearch.cluster.service.ClusterService; 15 | import org.opensearch.common.settings.ClusterSettings; 16 | import org.opensearch.common.settings.IndexScopedSettings; 17 | import org.opensearch.common.settings.Setting; 18 | import org.opensearch.common.settings.Settings; 19 | import org.opensearch.common.settings.SettingsFilter; 20 | import org.opensearch.core.common.io.stream.NamedWriteableRegistry; 21 | import org.opensearch.core.xcontent.NamedXContentRegistry; 22 | import org.opensearch.env.Environment; 23 | import org.opensearch.env.NodeEnvironment; 24 | import org.opensearch.plugins.ActionPlugin; 25 | import org.opensearch.plugins.Plugin; 26 | import org.opensearch.plugins.SearchPlugin; 27 | import org.opensearch.plugins.TelemetryAwarePlugin; 28 | import org.opensearch.repositories.RepositoriesService; 29 | import org.opensearch.rest.RestController; 30 | import org.opensearch.rest.RestHandler; 31 | import org.opensearch.script.ScriptService; 32 | import org.opensearch.telemetry.metrics.MetricsRegistry; 33 | import org.opensearch.telemetry.tracing.Tracer; 34 | import org.opensearch.threadpool.ThreadPool; 35 | import org.opensearch.transport.client.Client; 36 | import org.opensearch.ubi.ext.UbiParametersExtBuilder; 37 | import org.opensearch.watcher.ResourceWatcherService; 38 | 39 | import java.util.ArrayList; 40 | import java.util.Collection; 41 | import java.util.Collections; 42 | import java.util.List; 43 | import java.util.function.Supplier; 44 | 45 | import static java.util.Collections.singletonList; 46 | 47 | /** 48 | * OpenSearch User Behavior Insights 49 | */ 50 | public class UbiPlugin extends Plugin implements ActionPlugin, SearchPlugin, TelemetryAwarePlugin { 51 | 52 | public static final String UBI_QUERIES_INDEX = "ubi_queries"; 53 | public static final String UBI_EVENTS_INDEX = "ubi_events"; 54 | 55 | public static final String EVENTS_MAPPING_FILE = "/events-mapping.json"; 56 | public static final String QUERIES_MAPPING_FILE = "/queries-mapping.json"; 57 | 58 | private ActionFilter ubiActionFilter; 59 | 60 | /** 61 | * Creates a new instance of {@link UbiPlugin}. 62 | */ 63 | public UbiPlugin() {} 64 | 65 | @Override 66 | public List> getSettings() { 67 | return UbiSettings.getSettings(); 68 | } 69 | 70 | @Override 71 | public List getActionFilters() { 72 | return singletonList(ubiActionFilter); 73 | } 74 | 75 | @Override 76 | public Collection createComponents( 77 | Client client, 78 | ClusterService clusterService, 79 | ThreadPool threadPool, 80 | ResourceWatcherService resourceWatcherService, 81 | ScriptService scriptService, 82 | NamedXContentRegistry xContentRegistry, 83 | Environment environment, 84 | NodeEnvironment nodeEnvironment, 85 | NamedWriteableRegistry namedWriteableRegistry, 86 | IndexNameExpressionResolver indexNameExpressionResolver, 87 | Supplier repositoriesServiceSupplier, 88 | Tracer tracer, 89 | MetricsRegistry metricsRegistry 90 | ) { 91 | 92 | this.ubiActionFilter = new UbiActionFilter(client, environment, tracer); 93 | return Collections.emptyList(); 94 | 95 | } 96 | 97 | @Override 98 | public List> getSearchExts() { 99 | 100 | final List> searchExts = new ArrayList<>(); 101 | 102 | searchExts.add( 103 | new SearchExtSpec<>(UbiParametersExtBuilder.UBI_PARAMETER_NAME, UbiParametersExtBuilder::new, UbiParametersExtBuilder::parse) 104 | ); 105 | 106 | return searchExts; 107 | 108 | } 109 | 110 | @Override 111 | public List getRestHandlers( 112 | final Settings settings, 113 | final RestController restController, 114 | final ClusterSettings clusterSettings, 115 | final IndexScopedSettings indexScopedSettings, 116 | final SettingsFilter settingsFilter, 117 | final IndexNameExpressionResolver indexNameExpressionResolver, 118 | final Supplier nodesInCluster 119 | ) { 120 | return Collections.singletonList(new UbiRestHandler()); 121 | } 122 | 123 | } 124 | -------------------------------------------------------------------------------- /src/main/java/org/opensearch/ubi/UbiRestHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * SPDX-License-Identifier: Apache-2.0 3 | * 4 | * The OpenSearch Contributors require contributions made to 5 | * this file be licensed under the Apache-2.0 license or a 6 | * compatible open source license. 7 | */ 8 | 9 | package org.opensearch.ubi; 10 | 11 | import org.apache.logging.log4j.LogManager; 12 | import org.apache.logging.log4j.Logger; 13 | import org.opensearch.action.admin.indices.create.CreateIndexRequest; 14 | import org.opensearch.action.admin.indices.create.CreateIndexResponse; 15 | import org.opensearch.action.admin.indices.exists.indices.IndicesExistsRequest; 16 | import org.opensearch.action.admin.indices.exists.indices.IndicesExistsResponse; 17 | import org.opensearch.cluster.metadata.IndexMetadata; 18 | import org.opensearch.common.settings.Settings; 19 | import org.opensearch.common.util.io.Streams; 20 | import org.opensearch.core.action.ActionListener; 21 | import org.opensearch.core.rest.RestStatus; 22 | import org.opensearch.rest.BaseRestHandler; 23 | import org.opensearch.rest.BytesRestResponse; 24 | import org.opensearch.rest.RestRequest; 25 | import org.opensearch.transport.client.node.NodeClient; 26 | 27 | import java.io.ByteArrayOutputStream; 28 | import java.io.IOException; 29 | import java.io.InputStream; 30 | import java.nio.charset.StandardCharsets; 31 | import java.util.List; 32 | 33 | import static org.opensearch.ubi.UbiPlugin.EVENTS_MAPPING_FILE; 34 | import static org.opensearch.ubi.UbiPlugin.QUERIES_MAPPING_FILE; 35 | import static org.opensearch.ubi.UbiPlugin.UBI_EVENTS_INDEX; 36 | import static org.opensearch.ubi.UbiPlugin.UBI_QUERIES_INDEX; 37 | 38 | public class UbiRestHandler extends BaseRestHandler { 39 | 40 | private static final Logger LOGGER = LogManager.getLogger(UbiRestHandler.class); 41 | 42 | /** 43 | * URL for initializing the plugin and the index mappings. 44 | */ 45 | public static final String INITIALIZE_URL = "/_plugins/ubi/initialize"; 46 | 47 | @Override 48 | public String getName() { 49 | return "User Behavior Insights"; 50 | } 51 | 52 | @Override 53 | public List routes() { 54 | return List.of(new Route(RestRequest.Method.POST, INITIALIZE_URL)); 55 | } 56 | 57 | @Override 58 | protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { 59 | 60 | if(request.path().equalsIgnoreCase(INITIALIZE_URL)) { 61 | 62 | if (request.method().equals(RestRequest.Method.POST)) { 63 | 64 | final IndicesExistsRequest indicesExistsRequest = new IndicesExistsRequest(UBI_EVENTS_INDEX, UBI_QUERIES_INDEX); 65 | 66 | client.admin().indices().exists(indicesExistsRequest, new ActionListener<>() { 67 | 68 | @Override 69 | public void onResponse(IndicesExistsResponse indicesExistsResponse) { 70 | 71 | if (!indicesExistsResponse.isExists()) { 72 | 73 | final Settings indexSettings = Settings.builder() 74 | .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1) 75 | .put(IndexMetadata.INDEX_AUTO_EXPAND_REPLICAS_SETTING.getKey(), "0-2") 76 | .put(IndexMetadata.SETTING_PRIORITY, Integer.MAX_VALUE) 77 | .build(); 78 | 79 | // Create the UBI events index. 80 | final CreateIndexRequest createEventsIndexRequest = new CreateIndexRequest(UBI_EVENTS_INDEX).mapping( 81 | getResourceFile(EVENTS_MAPPING_FILE) 82 | ).settings(indexSettings); 83 | 84 | client.admin().indices().create(createEventsIndexRequest, new ActionListener<>() { 85 | @Override 86 | public void onResponse(CreateIndexResponse createIndexResponse) { 87 | LOGGER.debug("ubi_queries index created."); 88 | } 89 | 90 | @Override 91 | public void onFailure(Exception e) { 92 | LOGGER.error("Unable to create ubi_queries index", e); 93 | } 94 | }); 95 | 96 | // Create the UBI queries index. 97 | final CreateIndexRequest createQueriesIndexRequest = new CreateIndexRequest(UBI_QUERIES_INDEX).mapping( 98 | getResourceFile(QUERIES_MAPPING_FILE) 99 | ).settings(indexSettings); 100 | 101 | client.admin().indices().create(createQueriesIndexRequest, new ActionListener<>() { 102 | @Override 103 | public void onResponse(CreateIndexResponse createIndexResponse) { 104 | LOGGER.debug("ubi_events index created."); 105 | } 106 | 107 | @Override 108 | public void onFailure(Exception e) { 109 | LOGGER.error("Unable to create ubi_events index", e); 110 | } 111 | }); 112 | 113 | } else { 114 | LOGGER.debug("UBI indexes already exist."); 115 | } 116 | 117 | } 118 | 119 | @Override 120 | public void onFailure(Exception ex) { 121 | LOGGER.error("Error determining if UBI indexes exist.", ex); 122 | } 123 | 124 | }); 125 | 126 | return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.OK, "{\"message\": \"UBI indexes created.\"}")); 127 | 128 | } else { 129 | return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.METHOD_NOT_ALLOWED, "{\"error\": \"" + request.method() + " is not allowed.\"}")); 130 | } 131 | 132 | } else { 133 | return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.NOT_FOUND, "{\"error\": \"" + request.path() + " was not found.\"}")); 134 | } 135 | 136 | } 137 | 138 | private static String getResourceFile(final String fileName) { 139 | try (InputStream is = UbiActionFilter.class.getResourceAsStream(fileName)) { 140 | ByteArrayOutputStream out = new ByteArrayOutputStream(); 141 | Streams.copy(is, out); 142 | return out.toString(StandardCharsets.UTF_8); 143 | } catch (IOException e) { 144 | throw new IllegalStateException("Unable to get mapping from resource [" + fileName + "]", e); 145 | } 146 | } 147 | 148 | } 149 | -------------------------------------------------------------------------------- /src/main/java/org/opensearch/ubi/UbiSearchResponse.java: -------------------------------------------------------------------------------- 1 | /* 2 | * SPDX-License-Identifier: Apache-2.0 3 | * 4 | * The OpenSearch Contributors require contributions made to 5 | * this file be licensed under the Apache-2.0 license or a 6 | * compatible open source license. 7 | */ 8 | 9 | package org.opensearch.ubi; 10 | 11 | import org.opensearch.action.search.SearchResponse; 12 | import org.opensearch.action.search.SearchResponseSections; 13 | import org.opensearch.action.search.ShardSearchFailure; 14 | import org.opensearch.core.xcontent.XContentBuilder; 15 | import org.opensearch.ubi.ext.UbiParametersExtBuilder; 16 | 17 | import java.io.IOException; 18 | 19 | /** 20 | * A UBI search response. 21 | */ 22 | public class UbiSearchResponse extends SearchResponse { 23 | 24 | private static final String EXT_SECTION_NAME = "ext"; 25 | private static final String UBI_QUERY_ID_FIELD_NAME = "query_id"; 26 | 27 | private final String queryId; 28 | 29 | /** 30 | * Creates a new UBI search response. 31 | * @param internalResponse The internal response. 32 | * @param scrollId The scroll ID. 33 | * @param totalShards The count total shards. 34 | * @param successfulShards The count of successful shards. 35 | * @param skippedShards The count of skipped shards. 36 | * @param tookInMillis The time took in milliseconds. 37 | * @param shardFailures An array of {@link ShardSearchFailure}. 38 | * @param clusters The {@link Clusters}. 39 | * @param queryId The query ID. 40 | */ 41 | public UbiSearchResponse( 42 | SearchResponseSections internalResponse, 43 | String scrollId, 44 | int totalShards, 45 | int successfulShards, 46 | int skippedShards, 47 | long tookInMillis, 48 | ShardSearchFailure[] shardFailures, 49 | Clusters clusters, 50 | String queryId 51 | ) { 52 | super(internalResponse, scrollId, totalShards, successfulShards, skippedShards, tookInMillis, shardFailures, clusters); 53 | this.queryId = queryId; 54 | } 55 | 56 | @Override 57 | public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { 58 | 59 | builder.startObject(); 60 | innerToXContent(builder, params); 61 | 62 | builder.startObject(EXT_SECTION_NAME); 63 | builder.startObject(UbiParametersExtBuilder.UBI_PARAMETER_NAME); 64 | builder.field(UBI_QUERY_ID_FIELD_NAME, this.queryId); 65 | builder.endObject(); 66 | builder.endObject(); 67 | builder.endObject(); 68 | 69 | return builder; 70 | 71 | } 72 | 73 | } -------------------------------------------------------------------------------- /src/main/java/org/opensearch/ubi/UbiSettings.java: -------------------------------------------------------------------------------- 1 | /* 2 | * SPDX-License-Identifier: Apache-2.0 3 | * 4 | * The OpenSearch Contributors require contributions made to 5 | * this file be licensed under the Apache-2.0 license or a 6 | * compatible open source license. 7 | */ 8 | 9 | package org.opensearch.ubi; 10 | 11 | import org.opensearch.common.settings.Setting; 12 | 13 | import java.util.Collections; 14 | import java.util.List; 15 | 16 | /** 17 | * The UBI settings. 18 | */ 19 | public class UbiSettings { 20 | 21 | /** 22 | * The name of the Data Prepper http_source URL for receiving queries. 23 | */ 24 | public static final String DATA_PREPPER_URL = "ubi.dataprepper.url"; 25 | 26 | private static final Setting DATA_PREPPER_URL_SETTING = Setting.simpleString( 27 | DATA_PREPPER_URL, 28 | Setting.Property.Dynamic, 29 | Setting.Property.NodeScope); 30 | 31 | /** 32 | * Gets a list of the UBI plugin settings. 33 | * @return A list of the UBI plugin settings. 34 | */ 35 | public static List> getSettings() { 36 | return Collections.singletonList( 37 | DATA_PREPPER_URL_SETTING 38 | ); 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /src/main/java/org/opensearch/ubi/ext/UbiParameters.java: -------------------------------------------------------------------------------- 1 | /* 2 | * SPDX-License-Identifier: Apache-2.0 3 | * 4 | * The OpenSearch Contributors require contributions made to 5 | * this file be licensed under the Apache-2.0 license or a 6 | * compatible open source license. 7 | */ 8 | 9 | package org.opensearch.ubi.ext; 10 | 11 | import org.opensearch.action.search.SearchRequest; 12 | import org.opensearch.core.ParseField; 13 | import org.opensearch.core.common.io.stream.StreamInput; 14 | import org.opensearch.core.common.io.stream.StreamOutput; 15 | import org.opensearch.core.common.io.stream.Writeable; 16 | import org.opensearch.core.xcontent.ObjectParser; 17 | import org.opensearch.core.xcontent.ToXContentObject; 18 | import org.opensearch.core.xcontent.XContentBuilder; 19 | import org.opensearch.core.xcontent.XContentParser; 20 | import org.opensearch.search.SearchExtBuilder; 21 | 22 | import java.io.IOException; 23 | import java.util.HashMap; 24 | import java.util.Map; 25 | import java.util.Objects; 26 | import java.util.Optional; 27 | import java.util.UUID; 28 | 29 | /** 30 | * The UBI parameters available in the ext. 31 | */ 32 | public class UbiParameters implements Writeable, ToXContentObject { 33 | 34 | private static final ObjectParser PARSER; 35 | private static final ParseField QUERY_ID = new ParseField("query_id"); 36 | private static final ParseField USER_QUERY = new ParseField("user_query"); 37 | private static final ParseField CLIENT_ID = new ParseField("client_id"); 38 | private static final ParseField APPLICATION = new ParseField("application"); 39 | private static final ParseField OBJECT_ID_FIELD = new ParseField("object_id_field"); 40 | private static final ParseField QUERY_ATTRIBUTES = new ParseField("query_attributes"); 41 | 42 | static { 43 | PARSER = new ObjectParser<>(UbiParametersExtBuilder.UBI_PARAMETER_NAME, UbiParameters::new); 44 | PARSER.declareString(UbiParameters::setQueryId, QUERY_ID); 45 | PARSER.declareString(UbiParameters::setUserQuery, USER_QUERY); 46 | PARSER.declareString(UbiParameters::setClientId, CLIENT_ID); 47 | PARSER.declareString(UbiParameters::setApplication, APPLICATION); 48 | PARSER.declareString(UbiParameters::setObjectIdField, OBJECT_ID_FIELD); 49 | PARSER.declareObject(UbiParameters::setQueryAttributes, (p, c) -> p.mapStrings(), QUERY_ATTRIBUTES); 50 | } 51 | 52 | /** 53 | * Get the {@link UbiParameters} from a {@link SearchRequest}. 54 | * @param request A {@link SearchRequest}, 55 | * @return The UBI {@link UbiParameters parameters}. 56 | */ 57 | public static UbiParameters getUbiParameters(final SearchRequest request) { 58 | 59 | UbiParametersExtBuilder builder = null; 60 | 61 | if (request.source() != null && request.source().ext() != null && !request.source().ext().isEmpty()) { 62 | final Optional b = request.source() 63 | .ext() 64 | .stream() 65 | .filter(bldr -> UbiParametersExtBuilder.UBI_PARAMETER_NAME.equals(bldr.getWriteableName())) 66 | .findFirst(); 67 | if (b.isPresent()) { 68 | builder = (UbiParametersExtBuilder) b.get(); 69 | } 70 | } 71 | 72 | if (builder != null) { 73 | return builder.getParams(); 74 | } else { 75 | return null; 76 | } 77 | 78 | } 79 | 80 | private String queryId; 81 | private String userQuery; 82 | private String clientId; 83 | private String application; 84 | private String objectIdField; 85 | private Map queryAttributes; 86 | 87 | /** 88 | * Creates a new instance. 89 | */ 90 | public UbiParameters() {} 91 | 92 | /** 93 | * Creates a new instance. 94 | * @param input The {@link StreamInput} to read parameters from. 95 | * @throws IOException Thrown if the parameters cannot be read. 96 | */ 97 | @SuppressWarnings("unchecked") 98 | public UbiParameters(StreamInput input) throws IOException { 99 | this.queryId = input.readString(); 100 | this.userQuery = input.readOptionalString(); 101 | this.clientId = input.readOptionalString(); 102 | this.application = input.readOptionalString(); 103 | this.objectIdField = input.readOptionalString(); 104 | this.queryAttributes = (Map) input.readGenericValue(); 105 | } 106 | 107 | /** 108 | * Creates a new instance. 109 | * @param queryId The query ID. 110 | * @param userQuery The user-entered search query. 111 | * @param clientId The client ID. 112 | * @param application The name of the application making the query. 113 | * @param objectIdField The object ID field. 114 | * @param queryAttributes Optional attributes for UBI. 115 | */ 116 | public UbiParameters(String queryId, String userQuery, String clientId, String application, String objectIdField, Map queryAttributes) { 117 | this.queryId = queryId; 118 | this.userQuery = userQuery; 119 | this.clientId = clientId; 120 | this.application = application; 121 | this.objectIdField = objectIdField; 122 | this.queryAttributes = queryAttributes; 123 | } 124 | 125 | @Override 126 | public XContentBuilder toXContent(XContentBuilder xContentBuilder, Params params) throws IOException { 127 | return xContentBuilder 128 | .field(QUERY_ID.getPreferredName(), this.queryId) 129 | .field(USER_QUERY.getPreferredName(), this.userQuery) 130 | .field(CLIENT_ID.getPreferredName(), this.clientId) 131 | .field(APPLICATION.getPreferredName(), this.application) 132 | .field(OBJECT_ID_FIELD.getPreferredName(), this.objectIdField) 133 | .field(QUERY_ATTRIBUTES.getPreferredName(), this.queryAttributes); 134 | } 135 | 136 | @Override 137 | public void writeTo(StreamOutput out) throws IOException { 138 | out.writeString(getQueryId()); 139 | out.writeOptionalString(userQuery); 140 | out.writeOptionalString(clientId); 141 | out.writeOptionalString(application); 142 | out.writeOptionalString(objectIdField); 143 | out.writeGenericValue(queryAttributes); 144 | } 145 | 146 | /** 147 | * Create the {@link UbiParameters} from a {@link XContentParser}. 148 | * @param parser An {@link XContentParser}. 149 | * @return The {@link UbiParameters}. 150 | * @throws IOException Thrown if the parameters cannot be read. 151 | */ 152 | public static UbiParameters parse(XContentParser parser) throws IOException { 153 | return PARSER.parse(parser, null); 154 | } 155 | 156 | @Override 157 | public boolean equals(Object o) { 158 | if (this == o) { 159 | return true; 160 | } 161 | if (o == null || getClass() != o.getClass()) { 162 | return false; 163 | } 164 | 165 | final UbiParameters other = (UbiParameters) o; 166 | return Objects.equals(this.queryId, other.getQueryId()) 167 | && Objects.equals(this.userQuery, other.getUserQuery()) 168 | && Objects.equals(this.clientId, other.getClientId()) 169 | && Objects.equals(this.application, other.getApplication()) 170 | && Objects.equals(this.objectIdField, other.getObjectIdField()) 171 | && Objects.equals(this.queryAttributes, other.getQueryAttributes()); 172 | } 173 | 174 | @Override 175 | public int hashCode() { 176 | return Objects.hash(this.getClass(), this.queryId); 177 | } 178 | 179 | /** 180 | * Get the query ID. 181 | * @return The query ID, or a random UUID if the query ID is null. 182 | */ 183 | public String getQueryId() { 184 | if(queryId == null) { 185 | queryId = UUID.randomUUID().toString(); 186 | } 187 | return queryId; 188 | } 189 | 190 | /** 191 | * Set the query ID. 192 | * @param queryId The query ID. 193 | */ 194 | public void setQueryId(String queryId) { 195 | this.queryId = queryId; 196 | } 197 | 198 | /** 199 | * Get the client ID. 200 | * @return The client ID. 201 | */ 202 | public String getClientId() { 203 | return clientId; 204 | } 205 | 206 | /** 207 | * Set the client ID. 208 | * @param clientId The client ID. 209 | */ 210 | public void setClientId(String clientId) { 211 | this.clientId = clientId; 212 | } 213 | 214 | /** 215 | * Gets the application. 216 | * @return The application. 217 | */ 218 | public String getApplication() { 219 | return application; 220 | } 221 | 222 | /** 223 | * Set the application. 224 | * @param application The application. 225 | */ 226 | public void setApplication(String application) { 227 | this.application = application; 228 | } 229 | 230 | /** 231 | * Get the object ID field. 232 | * @return The object ID field. 233 | */ 234 | public String getObjectIdField() { 235 | return objectIdField; 236 | } 237 | 238 | /** 239 | * Set the object ID field. 240 | * @param objectIdField The object ID field. 241 | */ 242 | public void setObjectIdField(String objectIdField) { 243 | this.objectIdField = objectIdField; 244 | } 245 | 246 | /** 247 | * Get the user query. 248 | * @return The user query. 249 | */ 250 | public String getUserQuery() { 251 | return userQuery; 252 | } 253 | 254 | /** 255 | * Set the user query. 256 | * @param userQuery The user query. 257 | */ 258 | public void setUserQuery(String userQuery) { 259 | this.userQuery = userQuery; 260 | } 261 | 262 | /** 263 | * Get the attributes. 264 | * @return A map of attributes. 265 | */ 266 | public Map getQueryAttributes() { 267 | if(queryAttributes == null) { 268 | queryAttributes = new HashMap<>(); 269 | } 270 | return queryAttributes; 271 | } 272 | 273 | /** 274 | * Sets the attributes. 275 | * @param queryAttributes A map of attributes. 276 | */ 277 | public void setQueryAttributes(Map queryAttributes) { 278 | this.queryAttributes = queryAttributes; 279 | } 280 | 281 | } 282 | -------------------------------------------------------------------------------- /src/main/java/org/opensearch/ubi/ext/UbiParametersExtBuilder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * SPDX-License-Identifier: Apache-2.0 3 | * 4 | * The OpenSearch Contributors require contributions made to 5 | * this file be licensed under the Apache-2.0 license or a 6 | * compatible open source license. 7 | */ 8 | 9 | package org.opensearch.ubi.ext; 10 | 11 | import org.opensearch.core.common.io.stream.StreamInput; 12 | import org.opensearch.core.common.io.stream.StreamOutput; 13 | import org.opensearch.core.xcontent.XContentBuilder; 14 | import org.opensearch.core.xcontent.XContentParser; 15 | import org.opensearch.search.SearchExtBuilder; 16 | 17 | import java.io.IOException; 18 | import java.util.Objects; 19 | 20 | /** 21 | * Subclass of {@link SearchExtBuilder} to access UBI parameters. 22 | */ 23 | public class UbiParametersExtBuilder extends SearchExtBuilder { 24 | 25 | /** 26 | * The name of the "ext" section containing UBI parameters. 27 | */ 28 | public static final String UBI_PARAMETER_NAME = "ubi"; 29 | 30 | private UbiParameters params; 31 | 32 | /** 33 | * Creates a new instance. 34 | */ 35 | public UbiParametersExtBuilder() {} 36 | 37 | /** 38 | * Creates a new instance from a {@link StreamInput}. 39 | * @param input A {@link StreamInput} containing the parameters. 40 | * @throws IOException Thrown if the stream cannot be read. 41 | */ 42 | public UbiParametersExtBuilder(StreamInput input) throws IOException { 43 | this.params = new UbiParameters(input); 44 | } 45 | 46 | @Override 47 | public int hashCode() { 48 | return Objects.hash(this.getClass(), this.params); 49 | } 50 | 51 | @Override 52 | public boolean equals(Object obj) { 53 | if (this == obj) { 54 | return true; 55 | } 56 | 57 | if (!(obj instanceof UbiParametersExtBuilder)) { 58 | return false; 59 | } 60 | 61 | return this.params.equals(((UbiParametersExtBuilder) obj).getParams()); 62 | } 63 | 64 | @Override 65 | public String getWriteableName() { 66 | return UBI_PARAMETER_NAME; 67 | } 68 | 69 | @Override 70 | public void writeTo(StreamOutput out) throws IOException { 71 | this.params.writeTo(out); 72 | } 73 | 74 | @Override 75 | public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { 76 | return builder.value(this.params); 77 | } 78 | 79 | /** 80 | * Parses the ubi section of the ext block. 81 | * @param parser A {@link XContentParser parser}. 82 | * @return The {@link UbiParameters paramers}. 83 | * @throws IOException Thrown if the UBI parameters cannot be read. 84 | */ 85 | public static UbiParametersExtBuilder parse(XContentParser parser) throws IOException { 86 | final UbiParametersExtBuilder builder = new UbiParametersExtBuilder(); 87 | builder.setParams(UbiParameters.parse(parser)); 88 | return builder; 89 | } 90 | 91 | /** 92 | * Gets the {@link UbiParameters params}. 93 | * @return The {@link UbiParameters params}. 94 | */ 95 | public UbiParameters getParams() { 96 | return params; 97 | } 98 | 99 | /** 100 | * Set the {@link UbiParameters params}. 101 | * @param params The {@link UbiParameters params}. 102 | */ 103 | public void setParams(UbiParameters params) { 104 | this.params = params; 105 | } 106 | 107 | } 108 | -------------------------------------------------------------------------------- /src/main/resources/events-mapping.json: -------------------------------------------------------------------------------- 1 | { 2 | "properties": { 3 | "application": { "type": "keyword", "ignore_above": 256 }, 4 | "action_name": { "type": "keyword", "ignore_above": 100 }, 5 | "client_id": { "type": "keyword", "ignore_above": 100 }, 6 | "query_id": { "type": "keyword", "ignore_above": 100 }, 7 | "message": { "type": "keyword", "ignore_above": 1024 }, 8 | "message_type": { "type": "keyword", "ignore_above": 100 }, 9 | "user_query": { "type": "keyword" }, 10 | "timestamp": { 11 | "type": "date", 12 | "format":"strict_date_time", 13 | "ignore_malformed": true, 14 | "doc_values": true 15 | }, 16 | "event_attributes": { 17 | "dynamic": true, 18 | "properties": { 19 | "position": { 20 | "properties": { 21 | "ordinal": { "type": "integer" }, 22 | "x": { "type": "integer" }, 23 | "y": { "type": "integer" }, 24 | "page_depth": { "type": "integer" }, 25 | "scroll_depth": { "type": "integer" }, 26 | "trail": { "type": "text", 27 | "fields": { "keyword": { "type": "keyword", "ignore_above": 256 } 28 | } 29 | } 30 | } 31 | }, 32 | "object": { 33 | "properties": { 34 | "internal_id": { "type": "keyword" }, 35 | "object_id": { "type": "keyword", "ignore_above": 256 }, 36 | "object_id_field": { "type": "keyword", "ignore_above": 100 }, 37 | "name": { "type": "keyword", "ignore_above": 256 }, 38 | "description": { "type": "text", 39 | "fields": { "keyword": { "type": "keyword", "ignore_above": 256 } } 40 | }, 41 | "object_detail": { "type": "object" } 42 | } 43 | } 44 | } 45 | } 46 | } 47 | } -------------------------------------------------------------------------------- /src/main/resources/queries-mapping.json: -------------------------------------------------------------------------------- 1 | { 2 | "dynamic": false, 3 | "properties": { 4 | "timestamp": { "type": "date", "format": "strict_date_time" }, 5 | "query_id": { "type": "keyword", "ignore_above": 100 }, 6 | "query": { "type": "text" }, 7 | "query_response_id": { "type": "keyword", "ignore_above": 100 }, 8 | "query_response_hit_ids": { "type": "keyword" }, 9 | "user_query": { "type": "keyword" }, 10 | "query_attributes": { "type": "flat_object" }, 11 | "client_id": { "type": "keyword", "ignore_above": 100 }, 12 | "application": { "type": "keyword", "ignore_above": 100 } 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/test/java/org/opensearch/ubi/UbIParametersTests.java: -------------------------------------------------------------------------------- 1 | /* 2 | * SPDX-License-Identifier: Apache-2.0 3 | * 4 | * The OpenSearch Contributors require contributions made to 5 | * this file be licensed under the Apache-2.0 license or a 6 | * compatible open source license. 7 | */ 8 | 9 | package org.opensearch.ubi; 10 | 11 | import org.opensearch.action.search.SearchRequest; 12 | import org.opensearch.core.common.io.stream.StreamOutput; 13 | import org.opensearch.core.xcontent.XContent; 14 | import org.opensearch.core.xcontent.XContentBuilder; 15 | import org.opensearch.core.xcontent.XContentGenerator; 16 | import org.opensearch.search.builder.SearchSourceBuilder; 17 | import org.opensearch.test.OpenSearchTestCase; 18 | import org.opensearch.ubi.ext.UbiParameters; 19 | import org.opensearch.ubi.ext.UbiParametersExtBuilder; 20 | 21 | import java.io.IOException; 22 | import java.io.OutputStream; 23 | import java.util.ArrayList; 24 | import java.util.Collections; 25 | import java.util.List; 26 | 27 | import static org.mockito.ArgumentMatchers.any; 28 | import static org.mockito.Mockito.mock; 29 | import static org.mockito.Mockito.when; 30 | 31 | public class UbIParametersTests extends OpenSearchTestCase { 32 | 33 | static class DummyStreamOutput extends StreamOutput { 34 | 35 | List list = new ArrayList<>(); 36 | List intValues = new ArrayList<>(); 37 | 38 | @Override 39 | public void writeString(String str) { 40 | list.add(str); 41 | } 42 | 43 | public List getList() { 44 | return list; 45 | } 46 | 47 | @Override 48 | public void writeInt(int i) { 49 | intValues.add(i); 50 | } 51 | 52 | public List getIntValues() { 53 | return this.intValues; 54 | } 55 | 56 | @Override 57 | public void writeByte(byte b) throws IOException { 58 | 59 | } 60 | 61 | @Override 62 | public void writeBytes(byte[] b, int offset, int length) throws IOException { 63 | 64 | } 65 | 66 | @Override 67 | public void flush() throws IOException { 68 | 69 | } 70 | 71 | @Override 72 | public void close() throws IOException { 73 | 74 | } 75 | 76 | @Override 77 | public void reset() throws IOException { 78 | 79 | } 80 | } 81 | 82 | public void testUbiParameters() { 83 | final UbiParameters params = new UbiParameters("query_id", "user_query", "client_id", "app", "object_id", Collections.emptyMap()); 84 | UbiParametersExtBuilder extBuilder = new UbiParametersExtBuilder(); 85 | extBuilder.setParams(params); 86 | SearchSourceBuilder srcBulder = SearchSourceBuilder.searchSource().ext(List.of(extBuilder)); 87 | SearchRequest request = new SearchRequest("my_index").source(srcBulder); 88 | UbiParameters actual = UbiParameters.getUbiParameters(request); 89 | assertEquals(params, actual); 90 | } 91 | 92 | public void testWriteTo() throws IOException { 93 | final UbiParameters params = new UbiParameters("query_id", "user_query", "client_id", "app", "object_id", Collections.emptyMap()); 94 | StreamOutput output = new DummyStreamOutput(); 95 | params.writeTo(output); 96 | List actual = ((DummyStreamOutput) output).getList(); 97 | assertEquals("query_id", actual.get(0)); 98 | assertEquals("user_query", actual.get(1)); 99 | assertEquals("client_id", actual.get(2)); 100 | assertEquals("app", actual.get(3)); 101 | assertEquals("object_id", actual.get(4)); 102 | } 103 | 104 | public void testToXContent() throws IOException { 105 | final UbiParameters params = new UbiParameters("query_id", "user_query", "client_id", "app", "object_id", Collections.emptyMap()); 106 | XContent xc = mock(XContent.class); 107 | OutputStream os = mock(OutputStream.class); 108 | XContentGenerator generator = mock(XContentGenerator.class); 109 | when(xc.createGenerator(any(), any(), any())).thenReturn(generator); 110 | XContentBuilder builder = new XContentBuilder(xc, os); 111 | assertNotNull(params.toXContent(builder, null)); 112 | } 113 | 114 | public void testToXContentAllOptionalParameters() throws IOException { 115 | final UbiParameters params = new UbiParameters("query_id", "user_query", "client_id", "app", "object_id", Collections.emptyMap()); 116 | XContent xc = mock(XContent.class); 117 | OutputStream os = mock(OutputStream.class); 118 | XContentGenerator generator = mock(XContentGenerator.class); 119 | when(xc.createGenerator(any(), any(), any())).thenReturn(generator); 120 | XContentBuilder builder = new XContentBuilder(xc, os); 121 | assertNotNull(params.toXContent(builder, null)); 122 | } 123 | 124 | } 125 | -------------------------------------------------------------------------------- /src/test/java/org/opensearch/ubi/UbiActionFilterTests.java: -------------------------------------------------------------------------------- 1 | /* 2 | * SPDX-License-Identifier: Apache-2.0 3 | * 4 | * The OpenSearch Contributors require contributions made to 5 | * this file be licensed under the Apache-2.0 license or a 6 | * compatible open source license. 7 | */ 8 | 9 | package org.opensearch.ubi; 10 | 11 | import org.apache.lucene.search.TotalHits; 12 | import org.opensearch.action.admin.indices.exists.indices.IndicesExistsRequest; 13 | import org.opensearch.action.admin.indices.exists.indices.IndicesExistsResponse; 14 | import org.opensearch.action.search.SearchRequest; 15 | import org.opensearch.action.search.SearchResponse; 16 | import org.opensearch.action.support.ActionFilterChain; 17 | import org.opensearch.common.action.ActionFuture; 18 | import org.opensearch.common.settings.Settings; 19 | import org.opensearch.common.unit.TimeValue; 20 | import org.opensearch.core.action.ActionListener; 21 | import org.opensearch.env.Environment; 22 | import org.opensearch.search.SearchExtBuilder; 23 | import org.opensearch.search.SearchHit; 24 | import org.opensearch.search.SearchHits; 25 | import org.opensearch.search.builder.SearchSourceBuilder; 26 | import org.opensearch.tasks.Task; 27 | import org.opensearch.telemetry.tracing.noop.NoopTracer; 28 | import org.opensearch.test.OpenSearchTestCase; 29 | import org.opensearch.transport.client.AdminClient; 30 | import org.opensearch.transport.client.Client; 31 | import org.opensearch.transport.client.IndicesAdminClient; 32 | import org.opensearch.ubi.ext.UbiParameters; 33 | import org.opensearch.ubi.ext.UbiParametersExtBuilder; 34 | 35 | import java.util.ArrayList; 36 | import java.util.List; 37 | 38 | import static org.mockito.ArgumentMatchers.any; 39 | import static org.mockito.ArgumentMatchers.anyString; 40 | import static org.mockito.ArgumentMatchers.eq; 41 | import static org.mockito.Mockito.atMostOnce; 42 | import static org.mockito.Mockito.doAnswer; 43 | import static org.mockito.Mockito.mock; 44 | import static org.mockito.Mockito.never; 45 | import static org.mockito.Mockito.verify; 46 | import static org.mockito.Mockito.when; 47 | 48 | public class UbiActionFilterTests extends OpenSearchTestCase { 49 | 50 | @SuppressWarnings("unchecked") 51 | public void testApplyWithoutUbiBlock() { 52 | 53 | final Client client = mock(Client.class); 54 | final AdminClient adminClient = mock(AdminClient.class); 55 | final IndicesAdminClient indicesAdminClient = mock(IndicesAdminClient.class); 56 | 57 | when(client.admin()).thenReturn(adminClient); 58 | when(adminClient.indices()).thenReturn(indicesAdminClient); 59 | 60 | final Environment environment = mock(Environment.class); 61 | when(environment.settings()).thenReturn(Settings.EMPTY); 62 | 63 | final ActionFuture actionFuture = mock(ActionFuture.class); 64 | when(indicesAdminClient.exists(any(IndicesExistsRequest.class))).thenReturn(actionFuture); 65 | 66 | final UbiActionFilter ubiActionFilter = new UbiActionFilter(client, environment, NoopTracer.INSTANCE); 67 | final ActionListener listener = mock(ActionListener.class); 68 | 69 | final SearchRequest request = mock(SearchRequest.class); 70 | SearchHit[] searchHit = {}; 71 | final SearchHits searchHits = new SearchHits(searchHit, new TotalHits(0, TotalHits.Relation.EQUAL_TO), 0); 72 | 73 | final SearchResponse response = mock(SearchResponse.class); 74 | when(response.getHits()).thenReturn(searchHits); 75 | 76 | final TimeValue timeValue = new TimeValue(System.currentTimeMillis()); 77 | when(response.getTook()).thenReturn(timeValue); 78 | 79 | final Task task = mock(Task.class); 80 | 81 | final ActionFilterChain chain = mock(ActionFilterChain.class); 82 | 83 | doAnswer(invocation -> { 84 | ActionListener actionListener = invocation.getArgument(3); 85 | actionListener.onResponse(response); 86 | return null; 87 | }).when(chain).proceed(eq(task), anyString(), eq(request), any()); 88 | 89 | UbiParametersExtBuilder builder = mock(UbiParametersExtBuilder.class); 90 | final List builders = new ArrayList<>(); 91 | builders.add(builder); 92 | 93 | when(builder.getWriteableName()).thenReturn(UbiParametersExtBuilder.UBI_PARAMETER_NAME); 94 | when(builder.getParams()).thenReturn(null); 95 | 96 | SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); 97 | searchSourceBuilder.ext(builders); 98 | 99 | when(request.source()).thenReturn(searchSourceBuilder); 100 | 101 | ubiActionFilter.apply(task, "ubi", request, listener, chain); 102 | 103 | verify(client, never()).index(any(), any()); 104 | 105 | } 106 | 107 | @SuppressWarnings("unchecked") 108 | public void testApplyWithUbiBlockWithoutQueryId() { 109 | 110 | final Client client = mock(Client.class); 111 | final AdminClient adminClient = mock(AdminClient.class); 112 | final IndicesAdminClient indicesAdminClient = mock(IndicesAdminClient.class); 113 | 114 | when(client.admin()).thenReturn(adminClient); 115 | when(adminClient.indices()).thenReturn(indicesAdminClient); 116 | 117 | final Environment environment = mock(Environment.class); 118 | when(environment.settings()).thenReturn(Settings.EMPTY); 119 | 120 | final ActionFuture actionFuture = mock(ActionFuture.class); 121 | when(indicesAdminClient.exists(any(IndicesExistsRequest.class))).thenReturn(actionFuture); 122 | 123 | final UbiActionFilter ubiActionFilter = new UbiActionFilter(client, environment, NoopTracer.INSTANCE); 124 | final ActionListener listener = mock(ActionListener.class); 125 | 126 | final SearchRequest request = mock(SearchRequest.class); 127 | SearchHit[] searchHit = {}; 128 | final SearchHits searchHits = new SearchHits(searchHit, new TotalHits(0, TotalHits.Relation.EQUAL_TO), 0); 129 | 130 | final SearchResponse response = mock(SearchResponse.class); 131 | when(response.getHits()).thenReturn(searchHits); 132 | 133 | final TimeValue timeValue = new TimeValue(System.currentTimeMillis()); 134 | when(response.getTook()).thenReturn(timeValue); 135 | 136 | final Task task = mock(Task.class); 137 | 138 | final ActionFilterChain chain = mock(ActionFilterChain.class); 139 | 140 | doAnswer(invocation -> { 141 | ActionListener actionListener = invocation.getArgument(3); 142 | actionListener.onResponse(response); 143 | return null; 144 | }).when(chain).proceed(eq(task), anyString(), eq(request), any()); 145 | 146 | UbiParametersExtBuilder builder = mock(UbiParametersExtBuilder.class); 147 | final List builders = new ArrayList<>(); 148 | builders.add(builder); 149 | 150 | final UbiParameters ubiParameters = new UbiParameters(); 151 | 152 | when(builder.getWriteableName()).thenReturn(UbiParametersExtBuilder.UBI_PARAMETER_NAME); 153 | when(builder.getParams()).thenReturn(ubiParameters); 154 | 155 | SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); 156 | searchSourceBuilder.ext(builders); 157 | 158 | when(request.source()).thenReturn(searchSourceBuilder); 159 | 160 | ubiActionFilter.apply(task, "ubi", request, listener, chain); 161 | 162 | verify(client, atMostOnce()).index(any(), any()); 163 | 164 | } 165 | 166 | } 167 | -------------------------------------------------------------------------------- /src/test/java/org/opensearch/ubi/UbiModulePluginTests.java: -------------------------------------------------------------------------------- 1 | /* 2 | * SPDX-License-Identifier: Apache-2.0 3 | * 4 | * The OpenSearch Contributors require contributions made to 5 | * this file be licensed under the Apache-2.0 license or a 6 | * compatible open source license. 7 | */ 8 | 9 | package org.opensearch.ubi; 10 | 11 | import org.junit.Assert; 12 | import org.junit.Before; 13 | import org.opensearch.plugins.SearchPlugin; 14 | import org.opensearch.test.OpenSearchTestCase; 15 | import org.opensearch.transport.client.Client; 16 | 17 | import java.util.List; 18 | 19 | import static org.mockito.Mockito.mock; 20 | 21 | public class UbiModulePluginTests extends OpenSearchTestCase { 22 | 23 | private UbiPlugin ubiModulePlugin; 24 | 25 | private final Client client = mock(Client.class); 26 | 27 | @Before 28 | public void setup() { 29 | ubiModulePlugin = new UbiPlugin(); 30 | } 31 | 32 | public void testCreateComponent() { 33 | List components = (List) ubiModulePlugin.createComponents( 34 | client, 35 | null, 36 | null, 37 | null, 38 | null, 39 | null, 40 | null, 41 | null, 42 | null, 43 | null, 44 | null 45 | ); 46 | Assert.assertEquals(0, components.size()); 47 | } 48 | 49 | public void testGetSearchExts() { 50 | 51 | final List> searchExts = ubiModulePlugin.getSearchExts(); 52 | Assert.assertEquals(1, searchExts.size()); 53 | 54 | } 55 | 56 | } 57 | -------------------------------------------------------------------------------- /src/test/java/org/opensearch/ubi/UbiParametersExtBuilderTests.java: -------------------------------------------------------------------------------- 1 | /* 2 | * SPDX-License-Identifier: Apache-2.0 3 | * 4 | * The OpenSearch Contributors require contributions made to 5 | * this file be licensed under the Apache-2.0 license or a 6 | * compatible open source license. 7 | */ 8 | 9 | package org.opensearch.ubi; 10 | 11 | import org.opensearch.common.io.stream.BytesStreamOutput; 12 | import org.opensearch.common.xcontent.XContentType; 13 | import org.opensearch.core.common.bytes.BytesReference; 14 | import org.opensearch.core.common.io.stream.StreamInput; 15 | import org.opensearch.core.xcontent.XContentHelper; 16 | import org.opensearch.core.xcontent.XContentParser; 17 | import org.opensearch.test.OpenSearchTestCase; 18 | import org.opensearch.ubi.ext.UbiParameters; 19 | import org.opensearch.ubi.ext.UbiParametersExtBuilder; 20 | 21 | import java.io.EOFException; 22 | import java.io.IOException; 23 | import java.util.Collections; 24 | import java.util.HashMap; 25 | import java.util.Map; 26 | 27 | import static org.mockito.Mockito.mock; 28 | import static org.mockito.Mockito.when; 29 | 30 | public class UbiParametersExtBuilderTests extends OpenSearchTestCase { 31 | 32 | public void testCtor() { 33 | 34 | final Map queryAttributes = new HashMap<>(); 35 | 36 | final UbiParametersExtBuilder builder = new UbiParametersExtBuilder(); 37 | final UbiParameters parameters = new UbiParameters("query_id", "user_query", "client_id", "app", "object_id_field", queryAttributes); 38 | builder.setParams(parameters); 39 | assertEquals(parameters, builder.getParams()); 40 | 41 | } 42 | 43 | public void testParse() throws IOException { 44 | XContentParser xcParser = mock(XContentParser.class); 45 | when(xcParser.nextToken()).thenReturn(XContentParser.Token.START_OBJECT).thenReturn(XContentParser.Token.END_OBJECT); 46 | UbiParametersExtBuilder builder = UbiParametersExtBuilder.parse(xcParser); 47 | assertNotNull(builder); 48 | assertNotNull(builder.getParams()); 49 | } 50 | 51 | public void testXContentRoundTrip() throws IOException { 52 | UbiParameters param1 = new UbiParameters("query_id", "user_query", "client_id", "app", "object_id_field", Collections.emptyMap()); 53 | UbiParametersExtBuilder extBuilder = new UbiParametersExtBuilder(); 54 | extBuilder.setParams(param1); 55 | XContentType xContentType = randomFrom(XContentType.values()); 56 | BytesReference serialized = XContentHelper.toXContent(extBuilder, xContentType, true); 57 | XContentParser parser = createParser(xContentType.xContent(), serialized); 58 | UbiParametersExtBuilder deserialized = UbiParametersExtBuilder.parse(parser); 59 | assertEquals(extBuilder, deserialized); 60 | UbiParameters parameters = deserialized.getParams(); 61 | assertEquals("query_id", parameters.getQueryId()); 62 | assertEquals("user_query", parameters.getUserQuery()); 63 | assertEquals("client_id", parameters.getClientId()); 64 | assertEquals("app", parameters.getApplication()); 65 | assertEquals("object_id_field", parameters.getObjectIdField()); 66 | } 67 | 68 | public void testXContentRoundTripAllValues() throws IOException { 69 | UbiParameters param1 = new UbiParameters("query_id", "user_query", "client_id", "app","object_id_field", Collections.emptyMap()); 70 | UbiParametersExtBuilder extBuilder = new UbiParametersExtBuilder(); 71 | extBuilder.setParams(param1); 72 | XContentType xContentType = randomFrom(XContentType.values()); 73 | BytesReference serialized = XContentHelper.toXContent(extBuilder, xContentType, true); 74 | XContentParser parser = createParser(xContentType.xContent(), serialized); 75 | UbiParametersExtBuilder deserialized = UbiParametersExtBuilder.parse(parser); 76 | assertEquals(extBuilder, deserialized); 77 | } 78 | 79 | public void testStreamRoundTrip() throws IOException { 80 | UbiParameters param1 = new UbiParameters("query_id", "user_query", "client_id", "app","object_id_field", Collections.emptyMap()); 81 | UbiParametersExtBuilder extBuilder = new UbiParametersExtBuilder(); 82 | extBuilder.setParams(param1); 83 | BytesStreamOutput bso = new BytesStreamOutput(); 84 | extBuilder.writeTo(bso); 85 | UbiParametersExtBuilder deserialized = new UbiParametersExtBuilder(bso.bytes().streamInput()); 86 | assertEquals(extBuilder, deserialized); 87 | UbiParameters parameters = deserialized.getParams(); 88 | assertEquals("query_id", parameters.getQueryId()); 89 | assertEquals("user_query", parameters.getUserQuery()); 90 | assertEquals("client_id", parameters.getClientId()); 91 | assertEquals("app", parameters.getApplication()); 92 | assertEquals("object_id_field", parameters.getObjectIdField()); 93 | } 94 | 95 | public void testStreamRoundTripAllValues() throws IOException { 96 | UbiParameters param1 = new UbiParameters("query_id", "user_query", "client_id", "app","object_id_field", Collections.emptyMap()); 97 | UbiParametersExtBuilder extBuilder = new UbiParametersExtBuilder(); 98 | extBuilder.setParams(param1); 99 | BytesStreamOutput bso = new BytesStreamOutput(); 100 | extBuilder.writeTo(bso); 101 | UbiParametersExtBuilder deserialized = new UbiParametersExtBuilder(bso.bytes().streamInput()); 102 | assertEquals(extBuilder, deserialized); 103 | } 104 | 105 | } 106 | -------------------------------------------------------------------------------- /src/yamlRestTest/java/org/opensearch/ubi/rest/action/UserBehaviorInsightsClientYamlTestSuiteIT.java: -------------------------------------------------------------------------------- 1 | /* 2 | * SPDX-License-Identifier: Apache-2.0 3 | * 4 | * The OpenSearch Contributors require contributions made to 5 | * this file be licensed under the Apache-2.0 license or a 6 | * compatible open source license. 7 | */ 8 | package org.opensearch.ubi.rest.action; 9 | 10 | import com.carrotsearch.randomizedtesting.annotations.Name; 11 | import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; 12 | import org.opensearch.test.rest.yaml.ClientYamlTestCandidate; 13 | import org.opensearch.test.rest.yaml.OpenSearchClientYamlSuiteTestCase; 14 | 15 | public class UserBehaviorInsightsClientYamlTestSuiteIT extends OpenSearchClientYamlSuiteTestCase { 16 | 17 | public UserBehaviorInsightsClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { 18 | super(testCandidate); 19 | } 20 | 21 | @ParametersFactory 22 | public static Iterable parameters() throws Exception { 23 | return OpenSearchClientYamlSuiteTestCase.createParameters(); 24 | } 25 | 26 | } 27 | -------------------------------------------------------------------------------- /src/yamlRestTest/resources/rest-api-spec/api/ubi.initialize.json: -------------------------------------------------------------------------------- 1 | { 2 | "ubi.initialize": { 3 | "stability": "stable", 4 | "url": { 5 | "paths": [ 6 | { 7 | "path": "/_plugins/ubi/initialize", 8 | "methods": [ 9 | "POST" 10 | ] 11 | } 12 | ] 13 | }, 14 | "body": null 15 | } 16 | } -------------------------------------------------------------------------------- /src/yamlRestTest/resources/rest-api-spec/test/_plugins.ubi/10_initialize_ubi.yml: -------------------------------------------------------------------------------- 1 | --- 2 | "Initialize the UBI indexes": 3 | 4 | - do: 5 | ubi.initialize: {} 6 | 7 | - do: 8 | cluster.health: 9 | index: [ubi_queries] 10 | wait_for_no_initializing_shards: true 11 | 12 | - do: 13 | indices.exists: 14 | index: ubi_queries 15 | 16 | - is_true: '' 17 | 18 | - do: 19 | indices.exists: 20 | index: ubi_events 21 | 22 | - is_true: '' 23 | -------------------------------------------------------------------------------- /src/yamlRestTest/resources/rest-api-spec/test/_plugins.ubi/20_queries_with_ubi.yml: -------------------------------------------------------------------------------- 1 | --- 2 | "Query": 3 | 4 | - do: 5 | ubi.initialize: {} 6 | 7 | - do: 8 | cluster.health: 9 | index: [ubi_queries] 10 | wait_for_no_initializing_shards: true 11 | 12 | - do: 13 | indices.exists: 14 | index: ubi_queries 15 | 16 | - is_true: '' 17 | 18 | - do: 19 | indices.create: 20 | index: ecommerce 21 | body: 22 | mappings: 23 | { "properties": { "category": { "type": "text" } } } 24 | 25 | - match: { acknowledged: true } 26 | - match: { index: "ecommerce"} 27 | 28 | - do: 29 | index: 30 | index: ecommerce 31 | id: 1 32 | body: { category: notebook } 33 | 34 | - match: { result: created } 35 | 36 | - do: 37 | indices.refresh: 38 | index: [ "ecommerce" ] 39 | 40 | - do: 41 | search: 42 | rest_total_hits_as_int: true 43 | index: ecommerce 44 | body: "{\"query\": {\"match\": {\"category\": \"notebook\"}}, \"ext\": {\"ubi\": {\"query_id\": \"wertwert\", \"client_id\": \"abcabc\", \"user_query\": \"notebook\", \"query_attributes\": {\"experiment\": \"number_1\"}}}}" 45 | 46 | - gte: { hits.total: 1 } 47 | 48 | - do: 49 | search: 50 | rest_total_hits_as_int: true 51 | index: ecommerce 52 | body: "{\"query\": {\"match\": {\"category\": \"notebook\"}}, \"ext\": {\"ubi\": {\"query_id\": \"1234512345\", \"client_id\": \"abcabc\", \"user_query\": \"notebook\"}}}" 53 | 54 | - gte: { hits.total: 1 } 55 | 56 | - do: 57 | search: 58 | rest_total_hits_as_int: true 59 | index: ecommerce 60 | body: "{\"query\": {\"match\": {\"category\": \"notebook\"}}, \"ext\": {\"ubi\": {\"query_id\": \"abcdef\", \"client_id\": \"abcabc\", \"user_query\": \"notebook\", \"application\": \"app1\"}}}" 61 | 62 | - gte: { hits.total: 1 } 63 | 64 | - do: 65 | search: 66 | rest_total_hits_as_int: true 67 | index: ecommerce 68 | body: "{\"query\": {\"match\": {\"category\": \"notebook\"}}, \"ext\": {\"ubi\": {}}}" 69 | 70 | - gte: { hits.total: 1 } 71 | 72 | - do: 73 | search: 74 | rest_total_hits_as_int: true 75 | index: ecommerce 76 | body: "{\"query\": {\"match\": {\"category\": \"notebook\"}}}" 77 | 78 | - gte: { hits.total: 1 } 79 | 80 | - do: 81 | indices.refresh: 82 | index: [ubi_queries] 83 | 84 | - do: 85 | cluster.health: 86 | index: [ubi_queries] 87 | wait_for_no_initializing_shards: true 88 | 89 | - do: 90 | count: 91 | index: ubi_queries 92 | 93 | - match: { count: 4 } 94 | 95 | - do: 96 | search: 97 | rest_total_hits_as_int: true 98 | index: ubi_queries 99 | body: "{\"query\": {\"match\": {\"application\": \"app1\"}}}" 100 | 101 | - gte: { hits.total: 1 } 102 | -------------------------------------------------------------------------------- /src/yamlRestTest/resources/rest-api-spec/test/_plugins.ubi/30_queries_without_ubi.yml: -------------------------------------------------------------------------------- 1 | --- 2 | "Query without ubi block": 3 | 4 | - do: 5 | indices.create: 6 | index: ecommerce 7 | body: 8 | mappings: 9 | { "properties": { "category": { "type": "text" } } } 10 | 11 | - match: { acknowledged: true } 12 | - match: { index: "ecommerce"} 13 | 14 | - do: 15 | index: 16 | index: ecommerce 17 | id: 1 18 | body: { category: notebook } 19 | 20 | - match: { result: created } 21 | 22 | - do: 23 | indices.refresh: 24 | index: [ "ecommerce" ] 25 | 26 | - do: 27 | search: 28 | rest_total_hits_as_int: true 29 | index: ecommerce 30 | body: "{\"query\": {\"match\": {\"category\": \"notebook\"}}}" 31 | 32 | - gte: { hits.total: 1 } 33 | 34 | - do: 35 | indices.exists: 36 | index: ubi_queries 37 | 38 | - is_false: '' 39 | 40 | --- 41 | "Query without query_id": 42 | 43 | - do: 44 | indices.create: 45 | index: ecommerce 46 | body: 47 | mappings: 48 | { "properties": { "category": { "type": "text" } } } 49 | 50 | - match: { acknowledged: true } 51 | - match: { index: "ecommerce"} 52 | 53 | - do: 54 | index: 55 | index: ecommerce 56 | id: 1 57 | body: { category: notebook } 58 | 59 | - match: { result: created } 60 | 61 | - do: 62 | indices.refresh: 63 | index: [ "ecommerce" ] 64 | 65 | - do: 66 | search: 67 | rest_total_hits_as_int: true 68 | index: ecommerce 69 | body: "{\"query\": {\"match\": {\"category\": \"notebook\"}}, \"ext\": {\"ubi\": {\"user_query\": \"notebook\"}}}" 70 | 71 | - gte: { hits.total: 1 } 72 | 73 | - do: 74 | indices.exists: 75 | index: ubi_queries 76 | 77 | - is_false: '' 78 | -------------------------------------------------------------------------------- /src/yamlRestTest/resources/rest-api-spec/test/_plugins.ubi/40_msearch_queries_with_ubi.yml: -------------------------------------------------------------------------------- 1 | --- 2 | "msearch Queries": 3 | 4 | - do: 5 | ubi.initialize: {} 6 | 7 | - do: 8 | cluster.health: 9 | index: [ubi_queries] 10 | wait_for_no_initializing_shards: true 11 | 12 | - do: 13 | indices.exists: 14 | index: ubi_queries 15 | 16 | - is_true: '' 17 | 18 | - do: 19 | indices.create: 20 | index: ecommerce1 21 | body: 22 | mappings: 23 | { "properties": { "category": { "type": "text" } } } 24 | 25 | - match: { acknowledged: true } 26 | - match: { index: "ecommerce1"} 27 | 28 | - do: 29 | indices.create: 30 | index: ecommerce2 31 | body: 32 | mappings: 33 | { "properties": { "category": { "type": "text" } } } 34 | 35 | - match: { acknowledged: true } 36 | - match: { index: "ecommerce2"} 37 | 38 | - do: 39 | index: 40 | index: ecommerce1 41 | id: 1 42 | body: { category: notebook } 43 | 44 | - match: { result: created } 45 | 46 | - do: 47 | index: 48 | index: ecommerce2 49 | id: 1 50 | body: { category: notebook } 51 | 52 | - match: { result: created } 53 | 54 | - do: 55 | indices.refresh: 56 | index: [ "ecommerce1", "ecommerce2" ] 57 | 58 | - do: 59 | msearch: 60 | rest_total_hits_as_int: true 61 | body: 62 | - index: ecommerce1 63 | - {query: {match_all: {} }, ext: {ubi: {query_id: "12345"}}} 64 | - index: ecommerce2 65 | - {query: {match_all: {} }, ext: {ubi: {query_id: "09876"}}} 66 | 67 | - match: { responses.0.hits.total: 1} 68 | - match: { responses.1.hits.total: 1} 69 | 70 | - do: 71 | indices.refresh: 72 | index: [ "ubi_queries" ] 73 | - do: 74 | count: 75 | index: ubi_queries 76 | 77 | - match: { count: 2 } 78 | -------------------------------------------------------------------------------- /ubi-dashboards/README.md: -------------------------------------------------------------------------------- 1 | 2 | # How to load dashboards into your OpenSearch installation 3 | 4 | This assumes that you have already have UBI data and the `ubi_queries` index created. 5 | See the `ubi-data-generator` tool to generate data. 6 | 7 | Run the following shell script: `./install_dashboards.sh localhost:9200 localhost:5601` 8 | 9 | * `ubi_dashboard.ndjson` represents the UBI Dashboards needed to analyze and understand the data. 10 | -------------------------------------------------------------------------------- /ubi-dashboards/install_dashboards.sh: -------------------------------------------------------------------------------- 1 | 2 | # Ansi color code variables 3 | ERROR='\033[0;31m' 4 | MAJOR='\033[0;34m' 5 | MINOR='\033[0;37m ' 6 | RESET='\033[0m' # No Color 7 | 8 | opensearch=$1 9 | opensearch_dashboard=$2 10 | set -eo pipefail 11 | 12 | if [ -z "$opensearch" ]; then 13 | echo "Error: please pass in both the opensearch url and the opensearch dashboard url" 14 | exit 1 15 | fi 16 | if [ -z "$opensearch_dashboard" ]; then 17 | echo "Error: please pass in both the opensearch url and the opensearch dashboard url" 18 | exit 1 19 | fi 20 | 21 | echo "${MAJOR}Using Open Search and Open Search Dashboards at $opensearch and $opensearch_dashboard respectively.${RESET}" 22 | 23 | echo "${MAJOR}\nInstalling User Behavior Insights Dashboards${RESET}" 24 | curl -X POST "http://$opensearch_dashboard/api/saved_objects/_import?overwrite=true" -H "osd-xsrf: true" --form file=@ubi_dashboard.ndjson > /dev/null 25 | 26 | echo "${MAJOR}The UBI Dashboards were successfully installed${RESET}" 27 | -------------------------------------------------------------------------------- /ubi-data-generator/README.md: -------------------------------------------------------------------------------- 1 | 2 | # UBI tracking data generation 3 | 4 | The `ubi_data_generator.py` script generates UBI tracking events that can be used to test 5 | UBI related infrastructure such as dashboards and evaluation frameworks. 6 | 7 | The script requires input judgments for query and document pairs. 8 | At the moment it only supports the [Amazon ESCI dataset](https://github.com/amazon-science/esci-data) as an input. Using the script requires you to reference a local copy of this dataset. 9 | 10 | The script uses a click model that facilitates the testing of judgment calculations 11 | based on Clicks over Expected Clicks (COEC). 12 | We plan to release additional click models in the future. 13 | 14 | ## Installation 15 | 16 | ``` 17 | pip install -r requirements.txt 18 | ``` 19 | 20 | ## Usage 21 | 22 | Generate query, view and click events for the [Amazon ESCI dataset](https://github.com/amazon-science/esci-data) for the top 1000 queries, altogether 23 | generate 100k query events and populate into OpenSearch. 24 | 25 | ``` 26 | python ubi_data_generator.py --esci-dataset ../../esci-data/shopping_queries_dataset --num-unique-queries 1000 --num-query-events 100000 --generate-opensearch 27 | ``` 28 | 29 | Alternatively you can save the generated events into an ndjson file for later ingestion through the bulk endpoint: 30 | 31 | ``` 32 | python ubi_data_generator.py --esci-dataset ../../esci-data/shopping_queries_dataset --num-unique-queries 1000 --num-query-events 100000 --generate-ndjson 33 | ``` 34 | 35 | ## Interpreting the output 36 | 37 | ### Expected CTR per rank 38 | This is the CTR per rank that is expected from the generated events. 39 | This quantity is relevant for the calculation of COEC. 40 | 41 | ### Expected judgment under COEC for 5 documents over top 3 queries 42 | Here the script shows the documents that it has selected as the top 5 search results for the top 3 queries. 43 | The `p_click` column shows the click probability with which clicks are generated. 44 | The script attempts to have the expected COEC score, that is `p_click` divided by the CTR at that rank which is shown in `exp_rating`, 45 | to be the same as the original rating (column `rating`), but this does not always work. 46 | 47 | # Goal and design choices 48 | 49 | The goal of the script is to generate events that can be used for judgment 50 | calculation based on implicit feedback. The calculated judgments are designed 51 | to match the input judgments. 52 | 53 | Goals: 54 | * Test calculation of judgments based on implicit feedback. 55 | * Test calculation of judgments at scale. 56 | * End to end testing of the Search Quality Evaluation Framework. 57 | 58 | Non-goals: 59 | * At the moment the script does not aim to generate realistic events (which could for example help test corner 60 | cases that arise in real world data distributions). 61 | 62 | The design choices are: 63 | * Events are only generated for query document pairs that have a judgment. 64 | * The query sampling distribution is simplistic (proportional to the number of judgments). 65 | * The current click generation assumes that judgments are calculated using COEC 66 | (clicks over expected clicks) and as such the click generation aims to reproduce the input 67 | judgments. This in particular implies: 68 | * The expected judgment for a query document is near the original judgment value. That is 69 | `ctr(q, d) / ctr_at_pos(p) = orig_judgment(q, d)` where p is the position of the document. 70 | * To achieve this, the script assigns products as query results such that the average expected 71 | judgment is 1.0 at that rank. This simplifies giving a click rate to a query document pair 72 | where the equation above holds. 73 | 74 | # Future Work 75 | 76 | * Enable other data sources besides Amazon ESCI. 77 | * More realistic data generation: 78 | * More realistic query frequency distribution 79 | * Top results of a query have a higher relevance 80 | * More realistic click generation 81 | -------------------------------------------------------------------------------- /ubi-data-generator/requirements.txt: -------------------------------------------------------------------------------- 1 | opensearch-py==2.7.1 2 | numpy 3 | pandas 4 | pyarrow 5 | rich 6 | tqdm 7 | -------------------------------------------------------------------------------- /ubi-javascript-collector/ubi.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright OpenSearch Contributors 3 | * SPDX-License-Identifier: Apache-2.0 4 | * 5 | * The OpenSearch Contributors require contributions made to 6 | * this file be licensed under the Apache-2.0 license or a 7 | * compatible open source license. 8 | * 9 | */ 10 | 11 | /* 12 | ubi.js is sourced from https://github.com/opensearch-project/user-behavior-insights/tree/main/ubi-javascript-collector. 13 | */ 14 | 15 | import axios from 'axios'; 16 | 17 | /** 18 | * Methods and client to manage tracking queries and events that follow the User Behavior Insights specification. 19 | */ 20 | export class UbiClient { 21 | constructor(baseUrl) { 22 | // point to the specific middleware endpoint for receiving events 23 | this.eventUrl = `${baseUrl}/ubi_events`; 24 | this.queryUrl = `${baseUrl}/ubi_queries`; 25 | 26 | this.rest_config = { 27 | headers: { 28 | 'Content-type': 'application/json', 29 | 'Accept': 'application/json' 30 | } 31 | }; 32 | 33 | this.rest_client = axios.create({ 34 | baseURL: baseUrl, 35 | headers: { 36 | 'Content-type': 'application/json', 37 | 'Accept': 'application/json' 38 | }, 39 | withCredentials: false 40 | }); 41 | 42 | this.verbose = 0; // Default value for verbose 43 | } 44 | 45 | async trackEvent(event, message = null, message_type = null) { 46 | if (message) { 47 | if (event.message) { 48 | event['extra_info'] = message; 49 | if (message_type) { 50 | event['extra_info_type'] = message_type; 51 | } 52 | } else { 53 | event.message = message; 54 | event.message_type = message_type; 55 | } 56 | } 57 | 58 | // Data prepper wants an array of JSON. 59 | let json = JSON.stringify([event]); 60 | if (this.verbose > 0) { 61 | console.log('POSTing event: ' + json); 62 | } 63 | 64 | return this._post(json, this.eventUrl); 65 | } 66 | 67 | async trackQuery(query, message = null, message_type = null) { 68 | if (message) { 69 | if (query.message) { 70 | query['extra_info'] = message; 71 | if (message_type) { 72 | query['extra_info_type'] = message_type; 73 | } 74 | } else { 75 | query.message = message; 76 | query.message_type = message_type; 77 | } 78 | } 79 | 80 | // Data prepper wants an array of JSON. 81 | let json = JSON.stringify([query]); 82 | if (this.verbose > 0) { 83 | console.log('POSTing query: ' + json); 84 | } 85 | 86 | return this._post(json,this.queryUrl); 87 | } 88 | 89 | async _post(data,url) { 90 | try { 91 | const response = await this.rest_client.post(url, data, this.rest_config); 92 | return response.data; 93 | } catch (error) { 94 | console.error(error); 95 | } 96 | } 97 | } 98 | 99 | export class UbiEvent { 100 | /** 101 | * This maps to the UBI Event Specification at https://github.com/o19s/ubi 102 | */ 103 | constructor(application, action_name, client_id, session_id, query_id, event_attributes, message = null) { 104 | this.application = application; 105 | this.action_name = action_name; 106 | this.query_id = query_id; 107 | this.session_id = session_id; 108 | this.client_id = client_id; 109 | this.user_id = ''; 110 | this.timestamp = new Date().toISOString(); 111 | this.message_type = 'INFO'; 112 | this.message = message || ''; // Default to an empty string if no message 113 | this.event_attributes = event_attributes 114 | } 115 | 116 | setMessage(message, message_type = 'INFO') { 117 | this.message = message; 118 | this.message_type = message_type; 119 | } 120 | 121 | /** 122 | * Use to suppress null objects in the json output 123 | * @param key 124 | * @param value 125 | * @returns 126 | */ 127 | static replacer(key, value) { 128 | return (value == null) ? undefined : value; // Return undefined for null values 129 | } 130 | 131 | /** 132 | * 133 | * @returns json string 134 | */ 135 | toJson() { 136 | return JSON.stringify(this, UbiEvent.replacer); 137 | } 138 | } 139 | 140 | export class UbiEventAttributes { 141 | constructor(idField, id = null, description = null, details = null, position = null) { 142 | this.object = { 143 | object_id: id, 144 | object_id_field: idField, 145 | description: description, 146 | 147 | } 148 | 149 | // merge in the details, but make sure to filter out any defined properties 150 | // since details is a free form structure that could be anything. 151 | var { object_id, object_id_field, description, ...filteredDetails } = details; 152 | 153 | this.object = { ...this.object, ...filteredDetails }; 154 | 155 | // validate positional information 156 | // either ordinal or xy have to be set with ordinal, x and y being numbers 157 | if (!position) { 158 | throw new Error("The 'position' parameter is required."); 159 | } 160 | const hasOrdinal = 'ordinal' in position && Number.isInteger(position.ordinal); 161 | const hasXY = 162 | 'xy' in position && 163 | typeof position.xy === 'object' && 164 | 'x' in position.xy && 165 | 'y' in position.xy && 166 | typeof position.xy.x === 'number' && 167 | typeof position.xy.y === 'number'; 168 | 169 | if (!hasOrdinal && !hasXY) { 170 | throw new Error( 171 | "The 'position' object must have either an 'ordinal' property (integer) or an 'xy' property (object with 'x' and 'y' as numbers)." + position.ordinal 172 | ); 173 | } 174 | 175 | this.position = { ...position }; // Merge position into top-level property 176 | } 177 | } 178 | 179 | export class UbiQueryRequest { 180 | /** 181 | * This maps to the UBI Query Request Specification at https://github.com/o19s/ubi 182 | */ 183 | constructor(application, client_id, query_id, user_query, object_id_field, query_attributes = {}) { 184 | this.application = application; 185 | this.query_id = query_id; 186 | this.client_id = client_id; 187 | this.user_query = user_query; 188 | this.query_attributes = query_attributes 189 | this.object_id_field = object_id_field; 190 | this.timestamp = new Date().toISOString(); 191 | } 192 | 193 | /** 194 | * Use to suppress null objects in the json output 195 | * @param key 196 | * @param value 197 | * @returns 198 | */ 199 | static replacer(key, value) { 200 | return (value == null) ? undefined : value; // Return undefined for null values 201 | } 202 | 203 | /** 204 | * 205 | * @returns json string 206 | */ 207 | toJson() { 208 | return JSON.stringify(this, UbiEvent.replacer); 209 | } 210 | } 211 | --------------------------------------------------------------------------------