├── .codecov.yml ├── .github └── workflows │ ├── build.yml │ ├── multi-node-test-workflow.yml │ └── release-workflow.yml ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── NOTICE ├── README.md ├── build-tools ├── pkgbuild.gradle └── plugin-coverage.gradle ├── build.gradle ├── checkstyle └── checkstyle.xml ├── gradle.properties ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── release-notes ├── opendistro-for-elasticsearch-asynchronous-search.release-notes-1.13.0.0.md └── opendistro-for-elasticsearch-asynchronous-search.release-notes-1.13.0.1.md ├── settings.gradle └── src ├── main ├── java │ └── com │ │ └── amazon │ │ └── opendistroforelasticsearch │ │ └── search │ │ └── asynchronous │ │ ├── action │ │ ├── AsynchronousSearchStatsAction.java │ │ ├── DeleteAsynchronousSearchAction.java │ │ ├── GetAsynchronousSearchAction.java │ │ └── SubmitAsynchronousSearchAction.java │ │ ├── context │ │ ├── AsynchronousSearchContext.java │ │ ├── AsynchronousSearchContextId.java │ │ ├── active │ │ │ ├── AsynchronousSearchActiveContext.java │ │ │ ├── AsynchronousSearchActiveStore.java │ │ │ └── AsynchronousSearchContextClosedException.java │ │ ├── permits │ │ │ ├── AsynchronousSearchContextPermits.java │ │ │ └── NoopAsynchronousSearchContextPermits.java │ │ ├── persistence │ │ │ ├── AsynchronousSearchPersistenceContext.java │ │ │ └── AsynchronousSearchPersistenceModel.java │ │ └── state │ │ │ ├── AsynchronousSearchContextEvent.java │ │ │ ├── AsynchronousSearchState.java │ │ │ ├── AsynchronousSearchStateMachine.java │ │ │ ├── AsynchronousSearchStateMachineClosedException.java │ │ │ ├── AsynchronousSearchTransition.java │ │ │ ├── StateMachine.java │ │ │ ├── Transition.java │ │ │ └── event │ │ │ ├── BeginPersistEvent.java │ │ │ ├── SearchDeletedEvent.java │ │ │ ├── SearchFailureEvent.java │ │ │ ├── SearchResponsePersistFailedEvent.java │ │ │ ├── SearchResponsePersistedEvent.java │ │ │ ├── SearchStartedEvent.java │ │ │ └── SearchSuccessfulEvent.java │ │ ├── id │ │ ├── AsynchronousSearchId.java │ │ └── AsynchronousSearchIdConverter.java │ │ ├── listener │ │ ├── AsynchronousSearchContextEventListener.java │ │ ├── AsynchronousSearchProgressListener.java │ │ ├── AsynchronousSearchTimeoutWrapper.java │ │ ├── CompositeSearchProgressActionListener.java │ │ ├── PartialResponseProvider.java │ │ └── PrioritizedActionListener.java │ │ ├── management │ │ └── AsynchronousSearchManagementService.java │ │ ├── plugin │ │ └── AsynchronousSearchPlugin.java │ │ ├── processor │ │ └── AsynchronousSearchPostProcessor.java │ │ ├── request │ │ ├── AsynchronousSearchRoutingRequest.java │ │ ├── AsynchronousSearchStatsRequest.java │ │ ├── DeleteAsynchronousSearchRequest.java │ │ ├── GetAsynchronousSearchRequest.java │ │ └── SubmitAsynchronousSearchRequest.java │ │ ├── response │ │ ├── AcknowledgedResponse.java │ │ ├── AsynchronousSearchResponse.java │ │ └── AsynchronousSearchStatsResponse.java │ │ ├── rest │ │ ├── RestAsynchronousSearchStatsAction.java │ │ ├── RestDeleteAsynchronousSearchAction.java │ │ ├── RestGetAsynchronousSearchAction.java │ │ └── RestSubmitAsynchronousSearchAction.java │ │ ├── service │ │ ├── AsynchronousSearchPersistenceService.java │ │ └── AsynchronousSearchService.java │ │ ├── stats │ │ ├── AsynchronousSearchCountStats.java │ │ ├── AsynchronousSearchStats.java │ │ └── InternalAsynchronousSearchStats.java │ │ ├── task │ │ ├── AsynchronousSearchTask.java │ │ └── SubmitAsynchronousSearchTask.java │ │ ├── transport │ │ ├── TransportAsynchronousSearchRoutingAction.java │ │ ├── TransportAsynchronousSearchStatsAction.java │ │ ├── TransportDeleteAsynchronousSearchAction.java │ │ ├── TransportGetAsynchronousSearchAction.java │ │ └── TransportSubmitAsynchronousSearchAction.java │ │ └── utils │ │ ├── AsynchronousSearchExceptionUtils.java │ │ └── UserAuthUtils.java └── plugin-metadata │ └── plugin-security.policy └── test └── java └── com └── amazon └── opendistroforelasticsearch └── search └── asynchronous ├── commons ├── AsynchronousSearchIntegTestCase.java ├── AsynchronousSearchSingleNodeTestCase.java └── AsynchronousSearchTestCase.java ├── context ├── active │ ├── AsynchronousSearchActiveContextTests.java │ └── AsynchronousSearchActiveStoreTests.java ├── permits │ ├── AsynchronousSearchContextPermitsTests.java │ └── NoopAsynchronousSearchContextPermitsTests.java └── persistence │ └── AsynchronousSearchPersistenceContextTests.java ├── id └── AsynchronousSearchIdTests.java ├── integTests ├── AsynchronousSearchQueryIT.java ├── AsynchronousSearchRejectionIT.java ├── AsynchronousSearchStatsIT.java ├── AsynchronousSearchTaskCancellationIT.java ├── DeleteAsynchronousSearchSingleNodeIT.java ├── GetAsynchronousSearchSingleNodeIT.java ├── MixedOperationSingleNodeIT.java └── SubmitAsynchronousSearchSingleNodeIT.java ├── listener ├── AsynchronousSearchCancellationIT.java ├── AsynchronousSearchPartialResponseIT.java ├── AsynchronousSearchProgressListenerIT.java ├── AsynchronousSearchTimeoutWrapperTests.java └── SearchProgressActionListenerTests.java ├── management ├── AsynchronousSearchManagementServiceIT.java └── AsynchronousSearchManagementServiceTests.java ├── request ├── AsynchronousSearchRequestRoutingIT.java └── SubmitAsynchronousSearchRequestTests.java ├── response └── AsynchronousSearchResponseTests.java ├── restIT ├── ApiParamsValidationIT.java ├── AsynchronousSearchRestIT.java ├── AsynchronousSearchRestTestCase.java └── AsynchronousSearchSettingsIT.java ├── service ├── AsynchronousSearchPostProcessorTests.java ├── AsynchronousSearchServiceFreeContextTests.java ├── AsynchronousSearchServiceTests.java ├── AsynchronousSearchServiceUpdateContextTests.java ├── AsynchronousSearchStateMachineTests.java └── persistence │ └── AsynchronousSearchPersistenceServiceIT.java └── utils ├── AsynchronousSearchAssertions.java ├── QuadConsumer.java ├── RestTestUtils.java ├── TestClientUtils.java └── TestUtils.java /.codecov.yml: -------------------------------------------------------------------------------- 1 | codecov: 2 | require_ci_to_pass: yes 3 | 4 | coverage: 5 | precision: 2 6 | round: down 7 | range: "80...100" 8 | status: 9 | project: 10 | default: 11 | target: 80% # the required coverage value 12 | threshold: 1% # the leniency in hitting the target 13 | patch: off 14 | 15 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: Test and Build Workflow 2 | # This workflow is triggered on pull requests to master or a opendistro release branch 3 | on: 4 | pull_request: 5 | branches: 6 | - main 7 | - opendistro-* 8 | - plugin-dev 9 | push: 10 | branches: 11 | - main 12 | - opendistro-* 13 | - plugin-dev 14 | 15 | jobs: 16 | build: 17 | # Job name 18 | name: Build Asynchronous Search 19 | # This job runs on Linux. TODO Make it work for other OS distributions 20 | runs-on: ubuntu-latest 21 | steps: 22 | # This step uses the checkout Github action: https://github.com/actions/checkout 23 | - name: Checkout Branch 24 | uses: actions/checkout@v2 25 | # This step uses the setup-java Github action: https://github.com/actions/setup-java 26 | - name: Set Up JDK 14 27 | uses: actions/setup-java@v1 28 | with: 29 | java-version: 14 30 | - name: Build with Gradle 31 | run: ./gradlew build 32 | - name: Upload failed logs 33 | uses: actions/upload-artifact@v2 34 | if: failure() 35 | with: 36 | name: logs 37 | path: build/testclusters/integTest-*/logs/* 38 | - name: Create Artifact Path 39 | run: | 40 | mkdir -p asynchronous-search-artifacts 41 | cp ./build/distributions/*.zip asynchronous-search-artifacts 42 | - name: Uploads coverage 43 | uses: codecov/codecov-action@v1.2.1 44 | with: 45 | token: ${{ secrets.CODECOV_TOKEN }} 46 | # This step uses the upload-artifact Github action: https://github.com/actions/upload-artifact 47 | - name: Upload Artifacts 48 | uses: actions/upload-artifact@v1 49 | with: 50 | name: asynchronous-search-plugin 51 | path: asynchronous-search-artifacts 52 | -------------------------------------------------------------------------------- /.github/workflows/multi-node-test-workflow.yml: -------------------------------------------------------------------------------- 1 | name: Multi node test workflow 2 | 3 | env: 4 | java_version: 14 5 | # This workflow is triggered on pull requests to master 6 | on: 7 | pull_request: 8 | branches: 9 | - main 10 | push: 11 | branches: 12 | - main 13 | 14 | jobs: 15 | build: 16 | # Job name 17 | name: Build Asynchronous Search 18 | # This job runs on Linux 19 | runs-on: ubuntu-latest 20 | steps: 21 | # This step uses the checkout Github action: https://github.com/actions/checkout 22 | - name: Checkout Branch 23 | uses: actions/checkout@v2 24 | # This step uses the setup-java Github action: https://github.com/actions/setup-java 25 | - name: Set Up JDK 14 26 | uses: actions/setup-java@v1 27 | with: 28 | java-version: ${{ env.java_version }} 29 | - name: Run integration tests with multi node config 30 | run: ./gradlew integTest -PnumNodes=5 31 | - name: Upload failed logs 32 | uses: actions/upload-artifact@v2 33 | if: failure() 34 | with: 35 | name: logs 36 | path: build/testclusters/integTest-*/logs/* 37 | -------------------------------------------------------------------------------- /.github/workflows/release-workflow.yml: -------------------------------------------------------------------------------- 1 | name: Release workflow 2 | 3 | env: 4 | aws_staging_region: us-west-2 5 | 6 | # This workflow is triggered on creating tags to master or a opendistro release branch 7 | on: 8 | push: 9 | tags: 10 | - 'v*' 11 | 12 | jobs: 13 | build: 14 | strategy: 15 | matrix: 16 | java: [14] 17 | # Job name 18 | name: Build Asynchronous Search with JDK ${{ matrix.java }} 19 | # This job runs on Linux 20 | runs-on: ubuntu-latest 21 | steps: 22 | # This step uses the checkout Github action: https://github.com/actions/checkout 23 | - name: Checkout Branch 24 | uses: actions/checkout@v2 25 | # This step uses the setup-java Github action: https://github.com/actions/setup-java 26 | - name: Set Up JDK ${{ matrix.java }} 27 | uses: actions/setup-java@v1 28 | with: 29 | java-version: ${{ matrix.java }} 30 | 31 | # Building zip, deb and rpm files 32 | - name: Build with Gradle 33 | run: ./gradlew build buildDeb buildRpm --no-daemon -Dbuild.snapshot=false 34 | 35 | # Creating artifact path as well as individual folders for rpm, zip and deb assets 36 | - name: Create Artifact Path 37 | run: | 38 | artifacts_dir="asynchronous-search-artifacts" 39 | build_dir="./build/distributions" 40 | mkdir -p ${artifacts_dir} 41 | cp ${build_dir}/*.deb ${artifacts_dir} 42 | cp ${build_dir}/*.deb ${artifacts_dir}_deb 43 | cp ${build_dir}/*.rpm ${artifacts_dir} 44 | cp ${build_dir}/*.rpm ${artifacts_dir}_rpm 45 | cp ${build_dir}/*.zip ${artifacts_dir} 46 | cp ${build_dir}/*.zip ${artifacts_dir}_zip 47 | echo "TAG_VERSION=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_ENV 48 | # AWS authentication 49 | - name: Configure AWS Credentials 50 | uses: aws-actions/configure-aws-credentials@v1 51 | with: 52 | aws-access-key-id: ${{ secrets.AWS_STAGING_ACCESS_KEY_ID }} 53 | aws-secret-access-key: ${{ secrets.AWS_STAGING_SECRET_ACCESS_KEY }} 54 | aws-region: ${{ env.aws_staging_region }} 55 | 56 | # This step uses the upload-artifact Github action: https://github.com/actions/upload-artifact 57 | - name: Upload Artifacts to S3 58 | run: | 59 | zip=`ls asynchronous-search-artifacts/*.zip` 60 | rpm=`ls asynchronous-search-artifacts/*.rpm` 61 | deb=`ls asynchronous-search-artifacts/*.deb` 62 | # Inject the build number before the suffix 63 | zip_outfile=`basename ${zip%.zip}-build-${GITHUB_RUN_NUMBER}.zip` 64 | rpm_outfile=`basename ${rpm%.rpm}-build-${GITHUB_RUN_NUMBER}.rpm` 65 | deb_outfile=`basename ${deb%.deb}-build-${GITHUB_RUN_NUMBER}.deb` 66 | s3_prefix="s3://staging.artifacts.opendistroforelasticsearch.amazon.com/snapshots/elasticsearch-plugins/asynchronous-search/" 67 | echo "Copying ${zip} to ${s3_prefix}${zip_outfile}" 68 | aws s3 cp --quiet $zip ${s3_prefix}${zip_outfile} 69 | echo "Copying ${rpm} to ${s3_prefix}${rpm_outfile}" 70 | aws s3 cp --quiet $rpm ${s3_prefix}${rpm_outfile} 71 | echo "Copying ${deb} to ${s3_prefix}${deb_outfile}" 72 | aws s3 cp --quiet $deb ${s3_prefix}${deb_outfile} 73 | 74 | - name: Create Github Draft Release 75 | id: create_release 76 | uses: actions/create-release@v1.0.0 77 | env: 78 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 79 | with: 80 | tag_name: ${{ github.ref }} 81 | release_name: Release ${{ env.TAG_VERSION }} 82 | draft: true 83 | prerelease: false 84 | 85 | # Upload the release with .zip as asset 86 | - name: Upload Release Asset 87 | uses: actions/upload-release-asset@v1.0.1 88 | env: 89 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 90 | with: 91 | upload_url: ${{ steps.create_release.outputs.upload_url }} 92 | asset_name: asynchronous-search.zip 93 | asset_path: asynchronous-search-artifacts_zip 94 | asset_content_type: application/zip 95 | 96 | # Upload the release with .rpm as asset 97 | - name: Upload Release Asset 98 | uses: actions/upload-release-asset@v1.0.1 99 | env: 100 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 101 | with: 102 | upload_url: ${{ steps.create_release.outputs.upload_url }} 103 | asset_name: asynchronous-search.rpm 104 | asset_path: asynchronous-search-artifacts_rpm 105 | asset_content_type: application/zip 106 | 107 | # Upload the release with .deb as asset 108 | - name: Upload Release Asset 109 | uses: actions/upload-release-asset@v1.0.1 110 | env: 111 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 112 | with: 113 | upload_url: ${{ steps.create_release.outputs.upload_url }} 114 | asset_name: asynchronous-search.deb 115 | asset_path: asynchronous-search-artifacts_deb 116 | asset_content_type: application/zip 117 | 118 | - name: Upload Workflow Artifacts 119 | uses: actions/upload-artifact@v1 120 | with: 121 | name: asynchronous-search-plugin 122 | path: asynchronous-search-artifacts 123 | 124 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | ## Code of Conduct 2 | This project has adopted an [Open Source Code of Conduct](https://opendistro.github.io/for-elasticsearch/codeofconduct.html). -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing Guidelines 2 | 3 | Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional 4 | documentation, we greatly value feedback and contributions from our community. 5 | 6 | Please read through this document before submitting any issues or pull requests to ensure we have all the necessary 7 | information to effectively respond to your bug report or contribution. 8 | 9 | 10 | ## Reporting Bugs/Feature Requests 11 | 12 | We welcome you to use the GitHub issue tracker to report bugs or suggest features. 13 | 14 | When filing an issue, please check existing open, or recently closed, issues to make sure somebody else hasn't already 15 | reported the issue. Please try to include as much information as you can. Details like these are incredibly useful: 16 | 17 | * A reproducible test case or series of steps 18 | * The version of our code being used 19 | * Any modifications you've made relevant to the bug 20 | * Anything unusual about your environment or deployment 21 | 22 | ## Sign your work 23 | The sign-off is a simple line at the end of each commit, which certifies that you wrote it or otherwise have the right to pass it on as an open-source patch. if you can certify the below 24 | ``` 25 | By making a contribution to this project, I certify that: 26 | (a) The contribution was created in whole or in part by me and I 27 | have the right to submit it under the open source license 28 | indicated in the file; or 29 | (b) The contribution is based upon previous work that, to the best 30 | of my knowledge, is covered under an appropriate open source 31 | license and I have the right under that license to submit that 32 | work with modifications, whether created in whole or in part 33 | by me, under the same open source license (unless I am 34 | permitted to submit under a different license), as indicated 35 | in the file; or 36 | (c) The contribution was provided directly to me by some other 37 | person who certified (a), (b) or (c) and I have not modified 38 | it. 39 | (d) I understand and agree that this project and the contribution 40 | are public and that a record of the contribution (including all 41 | personal information I submit with it, including my sign-off) is 42 | maintained indefinitely and may be redistributed consistent with 43 | this project or the open source license(s) involved. 44 | ``` 45 | then you just add a line to every git commit message: 46 | ``` 47 | Signed-off-by: Bob Sanders 48 | ``` 49 | You can sign off your work easily by adding the configuration in github 50 | ``` 51 | git config user.name "Bob Sanders" 52 | git config user.email "bob.sanders@email.com" 53 | ``` 54 | Then, you could sign off commits automatically by adding `-s` or `-=signoff` parameter to your usual git commits commands. e.g. 55 | ``` 56 | git commit -s -m "my first commit" 57 | ``` 58 | 59 | ## Contributing via Pull Requests 60 | Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that: 61 | 62 | 1. You are working against the latest source on the *main* branch. 63 | 2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already. 64 | 3. You open an issue to discuss any significant work - we would hate for your time to be wasted. 65 | 66 | To send us a pull request, please: 67 | 68 | 1. Fork the repository. 69 | 2. Modify the source; please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change. 70 | 3. Ensure local tests pass. 71 | 4. Commit to your fork using clear commit messages. 72 | 5. Send us a pull request, answering any default questions in the pull request interface. 73 | 6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation. 74 | 75 | GitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and 76 | [creating a pull request](https://help.github.com/articles/creating-a-pull-request/). 77 | 78 | 79 | ## Finding contributions to work on 80 | Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels (enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any 'help wanted' issues is a great place to start. 81 | 82 | 83 | ## Code of Conduct 84 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 85 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 86 | opensource-codeofconduct@amazon.com with any additional questions or comments. 87 | 88 | 89 | ## Security issue notifications 90 | If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public github issue. 91 | 92 | 93 | ## Licensing 94 | 95 | See the [LICENSE](LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution. 96 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Test and Build Workflow](https://github.com/opendistro-for-elasticsearch/asynchronous-search/workflows/Test%20and%20Build%20Workflow/badge.svg)](https://github.com/opendistro-for-elasticsearch/asynchronous-search/actions) 2 | [![codecov](https://codecov.io/gh/opendistro-for-elasticsearch/asynchronous-search/branch/main/graph/badge.svg)](https://codecov.io/gh/opendistro-for-elasticsearch/asynchronous-search) 3 | 4 | # Open Distro for Elasticsearch asynchronous search 5 | Asynchronous search makes it possible for users to run such queries without worrying about the query timing out. 6 | These queries run in the background, and users can track the progress, and retrieve partial results as they become available. 7 | 8 | The asynchronous search plugin supports the below operations 9 | 10 | **1. Submit asynchronous search** 11 | ``` 12 | POST /_opendistro/_asynchronous_search?wait_for_completion_timeout=500ms&keep_on_completion=true&keep_alive=3d 13 | { "aggs": { 14 | "city": { 15 | "terms": { 16 | "field": "city", "size": 50 17 | } 18 | } 19 | } 20 | } 21 | 22 | ``` 23 | 24 | **2. Retrieve asynchronous search results** 25 | ``` 26 | GET /_opendistro/_asynchronous_search/FjdITFhYbC1zVFdHVVV1MUd3UkxkMFEFMjQ1MzYUWHRrZjhuWUJXdFhxMmlCSW5HTE8BMQ==?keep_alive=3d 27 | ``` 28 | 29 | **3. Delete an asynchronous search** 30 | 31 | ``` 32 | DELETE /_opendistro/_asynchronous_search/FjdITFhYbC1zVFdHVVV1MUd3UkxkMFEFMjQ1MzYUWHRrZjhuWUJXdFhxMmlCSW5HTE8BMQ== 33 | ``` 34 | 35 | **4. Stats for asynchronous search** 36 | 37 | ``` 38 | GET /_opendistro/_asynchronous_search/stats 39 | ``` 40 | 41 | **Tunable Settings** 42 | 1. `opendistro.asynchronous_search.max_search_running_time` : Maximum running time for the search beyond which the search would be terminated 43 | 2. `opendistro.asynchronous_search.node_concurrent_running_searches` : Concurrent searches running per coordinator node 44 | 3. `opendistro.asynchronous_search.max_keep_alive` : Maximum keep alive for search which dictates how long the search is allowed to be present in the cluster 45 | 4. `opendistro.asynchronous_search.max_wait_for_completion_timeout` : Maximum keep on completion to block for the search response 46 | 5. `opendistro.asynchronous_search.persist_search_failures` : Persist asynchronous search result ending with search failure in system index 47 | 48 | ## Setup 49 | 50 | 1. Check out this package from version control. 51 | 2. Launch Intellij IDEA, choose **Import Project**, and select the `settings.gradle` file in the root of this package. 52 | 3. To build from the command line, set `JAVA_HOME` to point to a JDK >= 14 before running `./gradlew`. 53 | - Unix System 54 | 1. `export JAVA_HOME=jdk-install-dir`: Replace `jdk-install-dir` with the JAVA_HOME directory of your system. 55 | 2. `export PATH=$JAVA_HOME/bin:$PATH` 56 | 57 | - Windows System 58 | 1. Find **My Computers** from file directory, right click and select **properties**. 59 | 2. Select the **Advanced** tab, select **Environment variables**. 60 | 3. Edit **JAVA_HOME** to path of where JDK software is installed. 61 | 62 | ## Build 63 | 64 | The project in this package uses the [Gradle](https://docs.gradle.org/current/userguide/userguide.html) build system. Gradle comes with excellent documentation that should be your first stop when trying to figure out how to operate or modify the build. 65 | 66 | 67 | ### Building from the command line 68 | 69 | 1. `./gradlew build` builds and tests project. 70 | 2. `./gradlew run` launches a single node cluster with the asynchronous search plugin installed. 71 | 3. `./gradlew run -PnumNodes=3` launches a multi-node cluster with the asynchronous search plugin installed. 72 | 4. `./gradlew integTest` launches a single node cluster with the asynchronous search plugin installed and runs all integ tests. 73 | 5. `./gradlew integTest -PnumNodes=3` launches a multi-node cluster with the asynchronous search plugin installed and runs all integ tests. 74 | 6. `./gradlew integTest -Dtests.class=*AsynchronousSearchRestIT` runs a single integ class 75 | 7. `./gradlew integTest -Dtests.class=*AsynchronousSearchRestIT -Dtests.method="testSubmitWithRetainedResponse"` runs a single integ test method (remember to quote the test method name if it contains spaces) 76 | 77 | When launching a cluster using one of the above commands, logs are placed in `build/testclusters/integTest-0/logs`. Though the logs are teed to the console, in practices it's best to check the actual log file. 78 | 79 | ### Debugging 80 | 81 | Sometimes it is useful to attach a debugger to either the Elasticsearch cluster or the integ tests to see what's going on. When running unit tests, hit **Debug** from the IDE's gutter to debug the tests. For the Elasticsearch cluster or the integ tests, first, make sure start a debugger listening on port `5005`. 82 | 83 | To debug the server code, run: 84 | 85 | ``` 86 | ./gradlew :integTest -Dcluster.debug # to start a cluster with debugger and run integ tests 87 | ``` 88 | 89 | OR 90 | 91 | ``` 92 | ./gradlew run --debug-jvm # to just start a cluster that can be debugged 93 | ``` 94 | 95 | The Elasticsearch server JVM will connect to a debugger attached to `localhost:5005`. 96 | 97 | The IDE needs to listen for the remote JVM. If using Intellij you must set your debug configuration to "Listen to remote JVM" and make sure "Auto Restart" is checked. 98 | You must start your debugger to listen for remote JVM before running the commands. 99 | 100 | To debug code running in an integration test (which exercises the server from a separate JVM), first, setup a remote debugger listening on port `8000`, and then run: 101 | 102 | ``` 103 | ./gradlew :integTest -Dtest.debug 104 | ``` 105 | 106 | The test runner JVM will connect to a debugger attached to `localhost:8000` before running the tests. 107 | 108 | Additionally, it is possible to attach one debugger to the cluster JVM and another debugger to the test runner. First, make sure one debugger is listening on port `5005` and the other is listening on port `8000`. Then, run: 109 | ``` 110 | ./gradlew :integTest -Dtest.debug -Dcluster.debug 111 | ``` 112 | 113 | 114 | 115 | ## Security 116 | 117 | See [CONTRIBUTING](CONTRIBUTING.md#security-issue-notifications) for more information. 118 | 119 | ## License 120 | 121 | This project is licensed under the Apache-2.0 License. 122 | -------------------------------------------------------------------------------- /build-tools/pkgbuild.gradle: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"). 6 | * You may not use this file except in compliance with the License. 7 | * A copy of the License is located at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * or in the "license" file accompanying this file. This file is distributed 12 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 13 | * express or implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | 17 | apply plugin: 'nebula.ospackage' 18 | 19 | // This is afterEvaluate because the bundlePlugin ZIP task is updated afterEvaluate and changes the ZIP name to match the plugin name 20 | afterEvaluate { 21 | ospackage { 22 | packageName = "opendistro-${name}" 23 | release = isSnapshot ? "0.1" : '1' 24 | version = "${project.version}" - "-SNAPSHOT" 25 | 26 | into '/usr/share/elasticsearch/plugins' 27 | from(zipTree(bundlePlugin.archivePath)) { 28 | into esplugin.name 29 | } 30 | 31 | user 'root' 32 | permissionGroup 'root' 33 | fileMode 0644 34 | dirMode 0755 35 | 36 | requires('elasticsearch-oss', versions.elasticsearch, EQUAL) 37 | packager = 'Amazon' 38 | vendor = 'Amazon' 39 | os = 'LINUX' 40 | prefix '/usr' 41 | 42 | license 'ASL-2.0' 43 | maintainer 'OpenDistro for Elasticsearch Team ' 44 | url 'https://opendistro.github.io/for-elasticsearch/downloads.html' 45 | summary ''' 46 | Open Distro for Elasticsearch Asynchronous Search. 47 | Reference documentation can be found at https://opendistro.github.io/for-elasticsearch-docs/. 48 | '''.stripIndent().replace('\n', ' ').trim() 49 | } 50 | 51 | buildRpm { 52 | arch = 'NOARCH' 53 | dependsOn 'assemble' 54 | finalizedBy 'renameRpm' 55 | task renameRpm(type: Copy) { 56 | from("$buildDir/distributions") 57 | into("$buildDir/distributions") 58 | include archiveName 59 | rename archiveName, "${packageName}-${version}.rpm" 60 | doLast { delete file("$buildDir/distributions/$archiveName") } 61 | } 62 | } 63 | buildDeb { 64 | arch = 'all' 65 | dependsOn 'assemble' 66 | finalizedBy 'renameDeb' 67 | task renameDeb(type: Copy) { 68 | from("$buildDir/distributions") 69 | into("$buildDir/distributions") 70 | include archiveName 71 | rename archiveName, "${packageName}-${version}.deb" 72 | doLast { delete file("$buildDir/distributions/$archiveName") } 73 | } 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /build-tools/plugin-coverage.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | apply plugin: 'jacoco' 17 | 18 | /** 19 | * ES Plugin build tools don't work with the Gradle Jacoco Plugin to report coverage out of the box. 20 | * https://github.com/elastic/elasticsearch/issues/28867. 21 | * 22 | * This code sets up coverage reporting manually for ES plugin tests. This is complicated because: 23 | * 1. The ES integTest Task doesn't implement Gradle's JavaForkOptions so we have to manually start the jacoco agent with the test JVM 24 | * 2. The cluster nodes are stopped using 'kill -9' which means jacoco can't dump it's execution output to a file on VM shutdown 25 | * 3. The Java Security Manager prevents JMX from writing execution output to the file. 26 | * 27 | * To workaround these we start the cluster with jmx enabled and then use Jacoco's JMX MBean to get the execution data before the 28 | * cluster is stopped and dump it to a file. Luckily our current security policy seems to allow this. This will also probably 29 | * break if there are multiple nodes in the integTestCluster. But for now... it sorta works. 30 | */ 31 | 32 | // Get gradle to generate the required jvm agent arg for us using a dummy tasks of type Test. Unfortunately Elastic's 33 | // testing tasks don't derive from Test so the jacoco plugin can't do this automatically. 34 | def jacocoDir = "${buildDir}/jacoco" 35 | 36 | task dummyTest(type: Test) { 37 | enabled = false 38 | workingDir = file("/") // Force absolute path to jacoco agent jar 39 | jacoco { 40 | destinationFile = file("${jacocoDir}/test.exec") 41 | destinationFile.parentFile.mkdirs() 42 | jmx = true 43 | } 44 | } 45 | 46 | task dummyIntegTest(type: Test) { 47 | enabled = false 48 | workingDir = file("/") // Force absolute path to jacoco agent jar 49 | jacoco { 50 | destinationFile = file("${jacocoDir}/integTest.exec") 51 | destinationFile.parentFile.mkdirs() 52 | jmx = true 53 | } 54 | } 55 | task dummyIntegTestRunner(type: Test) { 56 | enabled = false 57 | workingDir = file("/") // Force absolute path to jacoco agent jar 58 | jacoco { 59 | destinationFile = file("${jacocoDir}/integTestRunner.exec") 60 | destinationFile.parentFile.mkdirs() 61 | jmx = true 62 | } 63 | } 64 | 65 | integTest { 66 | systemProperty 'jacoco.dir', "${jacocoDir}" 67 | } 68 | 69 | jacocoTestReport { 70 | dependsOn integTest, test 71 | executionData.from dummyTest.jacoco.destinationFile, dummyIntegTest.jacoco.destinationFile, dummyIntegTestRunner.jacoco.destinationFile 72 | sourceDirectories.from = "src/main/java" 73 | classDirectories.from = sourceSets.main.output 74 | reports { 75 | html.enabled = true // human readable 76 | csv.enabled = true 77 | xml.enabled = true // for coverlay 78 | } 79 | } 80 | 81 | 82 | allprojects { 83 | afterEvaluate { 84 | jacocoTestReport.dependsOn integTest 85 | 86 | testClusters.integTest { 87 | jvmArgs " ${dummyIntegTest.jacoco.getAsJvmArg()}".replace('javaagent:', 'javaagent:/') 88 | systemProperty 'com.sun.management.jmxremote', "true" 89 | systemProperty 'com.sun.management.jmxremote.authenticate', "false" 90 | systemProperty 'com.sun.management.jmxremote.port', "7777" 91 | systemProperty 'com.sun.management.jmxremote.ssl', "false" 92 | systemProperty 'java.rmi.server.hostname', "127.0.0.1" 93 | } 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /checkstyle/checkstyle.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 31 | 32 | 33 | 34 | 35 | 36 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 67 | 68 | 69 | 70 | 71 | 88 | 89 | 90 | 93 | 94 | 96 | 97 | 98 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | -------------------------------------------------------------------------------- /gradle.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"). 5 | # You may not use this file except in compliance with the License. 6 | # A copy of the License is located at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # or in the "license" file accompanying this file. This file is distributed 11 | # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | # express or implied. See the License for the specific language governing 13 | # permissions and limitations under the License. 14 | # 15 | 16 | version = 1.9.0 -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendistro-for-elasticsearch/asynchronous-search/4020b9d91de1635da07611f8011f71ba7851ba3c/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-6.7.1-bin.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /gradlew: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | # 4 | # Copyright 2015 the original author or authors. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # https://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | ############################################################################## 20 | ## 21 | ## Gradle start up script for UN*X 22 | ## 23 | ############################################################################## 24 | 25 | # Attempt to set APP_HOME 26 | # Resolve links: $0 may be a link 27 | PRG="$0" 28 | # Need this for relative symlinks. 29 | while [ -h "$PRG" ] ; do 30 | ls=`ls -ld "$PRG"` 31 | link=`expr "$ls" : '.*-> \(.*\)$'` 32 | if expr "$link" : '/.*' > /dev/null; then 33 | PRG="$link" 34 | else 35 | PRG=`dirname "$PRG"`"/$link" 36 | fi 37 | done 38 | SAVED="`pwd`" 39 | cd "`dirname \"$PRG\"`/" >/dev/null 40 | APP_HOME="`pwd -P`" 41 | cd "$SAVED" >/dev/null 42 | 43 | APP_NAME="Gradle" 44 | APP_BASE_NAME=`basename "$0"` 45 | 46 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 47 | DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' 48 | 49 | # Use the maximum available, or set MAX_FD != -1 to use that value. 50 | MAX_FD="maximum" 51 | 52 | warn () { 53 | echo "$*" 54 | } 55 | 56 | die () { 57 | echo 58 | echo "$*" 59 | echo 60 | exit 1 61 | } 62 | 63 | # OS specific support (must be 'true' or 'false'). 64 | cygwin=false 65 | msys=false 66 | darwin=false 67 | nonstop=false 68 | case "`uname`" in 69 | CYGWIN* ) 70 | cygwin=true 71 | ;; 72 | Darwin* ) 73 | darwin=true 74 | ;; 75 | MINGW* ) 76 | msys=true 77 | ;; 78 | NONSTOP* ) 79 | nonstop=true 80 | ;; 81 | esac 82 | 83 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 84 | 85 | 86 | # Determine the Java command to use to start the JVM. 87 | if [ -n "$JAVA_HOME" ] ; then 88 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 89 | # IBM's JDK on AIX uses strange locations for the executables 90 | JAVACMD="$JAVA_HOME/jre/sh/java" 91 | else 92 | JAVACMD="$JAVA_HOME/bin/java" 93 | fi 94 | if [ ! -x "$JAVACMD" ] ; then 95 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 96 | 97 | Please set the JAVA_HOME variable in your environment to match the 98 | location of your Java installation." 99 | fi 100 | else 101 | JAVACMD="java" 102 | which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 103 | 104 | Please set the JAVA_HOME variable in your environment to match the 105 | location of your Java installation." 106 | fi 107 | 108 | # Increase the maximum file descriptors if we can. 109 | if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then 110 | MAX_FD_LIMIT=`ulimit -H -n` 111 | if [ $? -eq 0 ] ; then 112 | if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then 113 | MAX_FD="$MAX_FD_LIMIT" 114 | fi 115 | ulimit -n $MAX_FD 116 | if [ $? -ne 0 ] ; then 117 | warn "Could not set maximum file descriptor limit: $MAX_FD" 118 | fi 119 | else 120 | warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" 121 | fi 122 | fi 123 | 124 | # For Darwin, add options to specify how the application appears in the dock 125 | if $darwin; then 126 | GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" 127 | fi 128 | 129 | # For Cygwin or MSYS, switch paths to Windows format before running java 130 | if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then 131 | APP_HOME=`cygpath --path --mixed "$APP_HOME"` 132 | CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` 133 | 134 | JAVACMD=`cygpath --unix "$JAVACMD"` 135 | 136 | # We build the pattern for arguments to be converted via cygpath 137 | ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` 138 | SEP="" 139 | for dir in $ROOTDIRSRAW ; do 140 | ROOTDIRS="$ROOTDIRS$SEP$dir" 141 | SEP="|" 142 | done 143 | OURCYGPATTERN="(^($ROOTDIRS))" 144 | # Add a user-defined pattern to the cygpath arguments 145 | if [ "$GRADLE_CYGPATTERN" != "" ] ; then 146 | OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" 147 | fi 148 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 149 | i=0 150 | for arg in "$@" ; do 151 | CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` 152 | CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option 153 | 154 | if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition 155 | eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` 156 | else 157 | eval `echo args$i`="\"$arg\"" 158 | fi 159 | i=`expr $i + 1` 160 | done 161 | case $i in 162 | 0) set -- ;; 163 | 1) set -- "$args0" ;; 164 | 2) set -- "$args0" "$args1" ;; 165 | 3) set -- "$args0" "$args1" "$args2" ;; 166 | 4) set -- "$args0" "$args1" "$args2" "$args3" ;; 167 | 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; 168 | 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; 169 | 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; 170 | 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; 171 | 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; 172 | esac 173 | fi 174 | 175 | # Escape application args 176 | save () { 177 | for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done 178 | echo " " 179 | } 180 | APP_ARGS=`save "$@"` 181 | 182 | # Collect all arguments for the java command, following the shell quoting and substitution rules 183 | eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" 184 | 185 | exec "$JAVACMD" "$@" 186 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @rem 2 | @rem Copyright 2015 the original author or authors. 3 | @rem 4 | @rem Licensed under the Apache License, Version 2.0 (the "License"); 5 | @rem you may not use this file except in compliance with the License. 6 | @rem You may obtain a copy of the License at 7 | @rem 8 | @rem https://www.apache.org/licenses/LICENSE-2.0 9 | @rem 10 | @rem Unless required by applicable law or agreed to in writing, software 11 | @rem distributed under the License is distributed on an "AS IS" BASIS, 12 | @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | @rem See the License for the specific language governing permissions and 14 | @rem limitations under the License. 15 | @rem 16 | 17 | @if "%DEBUG%" == "" @echo off 18 | @rem ########################################################################## 19 | @rem 20 | @rem Gradle startup script for Windows 21 | @rem 22 | @rem ########################################################################## 23 | 24 | @rem Set local scope for the variables with windows NT shell 25 | if "%OS%"=="Windows_NT" setlocal 26 | 27 | set DIRNAME=%~dp0 28 | if "%DIRNAME%" == "" set DIRNAME=. 29 | set APP_BASE_NAME=%~n0 30 | set APP_HOME=%DIRNAME% 31 | 32 | @rem Resolve any "." and ".." in APP_HOME to make it shorter. 33 | for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi 34 | 35 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 36 | set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" 37 | 38 | @rem Find java.exe 39 | if defined JAVA_HOME goto findJavaFromJavaHome 40 | 41 | set JAVA_EXE=java.exe 42 | %JAVA_EXE% -version >NUL 2>&1 43 | if "%ERRORLEVEL%" == "0" goto execute 44 | 45 | echo. 46 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 47 | echo. 48 | echo Please set the JAVA_HOME variable in your environment to match the 49 | echo location of your Java installation. 50 | 51 | goto fail 52 | 53 | :findJavaFromJavaHome 54 | set JAVA_HOME=%JAVA_HOME:"=% 55 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 56 | 57 | if exist "%JAVA_EXE%" goto execute 58 | 59 | echo. 60 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 61 | echo. 62 | echo Please set the JAVA_HOME variable in your environment to match the 63 | echo location of your Java installation. 64 | 65 | goto fail 66 | 67 | :execute 68 | @rem Setup the command line 69 | 70 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 71 | 72 | 73 | @rem Execute Gradle 74 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* 75 | 76 | :end 77 | @rem End local scope for the variables with windows NT shell 78 | if "%ERRORLEVEL%"=="0" goto mainEnd 79 | 80 | :fail 81 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 82 | rem the _cmd.exe /c_ return code! 83 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 84 | exit /b 1 85 | 86 | :mainEnd 87 | if "%OS%"=="Windows_NT" endlocal 88 | 89 | :omega 90 | -------------------------------------------------------------------------------- /release-notes/opendistro-for-elasticsearch-asynchronous-search.release-notes-1.13.0.0.md: -------------------------------------------------------------------------------- 1 | ## 2021-02-03 Version 1.13.0.0 2 | Initial launch of Asynchronous Search on opendistro-for-elasticsearch ([RFC](https://github.com/opendistro-for-elasticsearch/community/files/5618042/RFC_.Asynchronous.Search.with.Elasticsearch.1.pdf)) 3 | 4 | Occasionally business would like to run queries on vast amount of data that can take very long to return results. 5 | 6 | Asynchronous search makes it possible for users to run such queries without worrying about the query timing out. These queries run in the background, and users can track the progress, and retrieve results as they become available. 7 | 8 | The asynchronous search APIs let you asynchronously execute a search request, monitor its progress, and retrieve partial results as they become available. 9 | 10 | ### Maintenance 11 | * Supports Elasticsearch 7.10.2 ([#32](https://github.com/opendistro-for-elasticsearch/asynchronous-search/pull/32)) 12 | 13 | 14 | ### Features 15 | * Listener framework for asynchronous search ([#2](https://github.com/opendistro-for-elasticsearch/asynchronous-search/pull/2)) 16 | * Service layer and Transport Handlers ([#8](https://github.com/opendistro-for-elasticsearch/asynchronous-search/pull/8)) 17 | * Rest Layer and Asynchronous Search Cleanup Management ([#9](https://github.com/opendistro-for-elasticsearch/asynchronous-search/pull/9)) 18 | * Integrates security with asynchronous search ([#11](https://github.com/opendistro-for-elasticsearch/asynchronous-search/pull/11)) 19 | -------------------------------------------------------------------------------- /release-notes/opendistro-for-elasticsearch-asynchronous-search.release-notes-1.13.0.1.md: -------------------------------------------------------------------------------- 1 | ## 2021-02-13 Version 1.13.0.1 2 | Compatible with Elasticsearch 7.10.2 3 | ### Maintenance 4 | * Renamed settings for consistency with other ODFE plugins ([#35](https://github.com/opendistro-for-elasticsearch/asynchronous-search 5 | /pull/35)) 6 | -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"). 6 | * You may not use this file except in compliance with the License. 7 | * A copy of the License is located at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * or in the "license" file accompanying this file. This file is distributed 12 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 13 | * express or implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | 17 | rootProject.name = 'asynchronous-search' 18 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/opendistroforelasticsearch/search/asynchronous/action/AsynchronousSearchStatsAction.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.opendistroforelasticsearch.search.asynchronous.action; 17 | 18 | import com.amazon.opendistroforelasticsearch.search.asynchronous.response.AsynchronousSearchStatsResponse; 19 | import org.elasticsearch.action.ActionType; 20 | import org.elasticsearch.common.io.stream.Writeable; 21 | 22 | public class AsynchronousSearchStatsAction extends ActionType { 23 | 24 | 25 | public static final AsynchronousSearchStatsAction INSTANCE = new AsynchronousSearchStatsAction(); 26 | public static final String NAME = "cluster:admin/opendistro/asynchronous_search/stats"; 27 | 28 | private AsynchronousSearchStatsAction() { 29 | super(NAME, AsynchronousSearchStatsResponse::new); 30 | } 31 | 32 | @Override 33 | public Writeable.Reader getResponseReader() { 34 | return AsynchronousSearchStatsResponse::new; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/opendistroforelasticsearch/search/asynchronous/action/DeleteAsynchronousSearchAction.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.opendistroforelasticsearch.search.asynchronous.action; 17 | 18 | import com.amazon.opendistroforelasticsearch.search.asynchronous.response.AcknowledgedResponse; 19 | import org.elasticsearch.action.ActionType; 20 | 21 | public class DeleteAsynchronousSearchAction extends ActionType { 22 | 23 | public static final DeleteAsynchronousSearchAction INSTANCE = new DeleteAsynchronousSearchAction(); 24 | public static final String NAME = "cluster:admin/opendistro/asynchronous_search/delete"; 25 | 26 | private DeleteAsynchronousSearchAction() { 27 | super(NAME, AcknowledgedResponse::new); 28 | } 29 | 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/opendistroforelasticsearch/search/asynchronous/action/GetAsynchronousSearchAction.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.opendistroforelasticsearch.search.asynchronous.action; 17 | 18 | import com.amazon.opendistroforelasticsearch.search.asynchronous.response.AsynchronousSearchResponse; 19 | import org.elasticsearch.action.ActionType; 20 | 21 | public class GetAsynchronousSearchAction extends ActionType { 22 | 23 | public static final GetAsynchronousSearchAction INSTANCE = new GetAsynchronousSearchAction(); 24 | public static final String NAME = "cluster:admin/opendistro/asynchronous_search/get"; 25 | 26 | private GetAsynchronousSearchAction() { 27 | super(NAME, AsynchronousSearchResponse::new); 28 | } 29 | 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/opendistroforelasticsearch/search/asynchronous/action/SubmitAsynchronousSearchAction.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | package com.amazon.opendistroforelasticsearch.search.asynchronous.action; 16 | 17 | import com.amazon.opendistroforelasticsearch.search.asynchronous.response.AsynchronousSearchResponse; 18 | import org.elasticsearch.action.ActionType; 19 | 20 | public class SubmitAsynchronousSearchAction extends ActionType { 21 | 22 | public static final SubmitAsynchronousSearchAction INSTANCE = new SubmitAsynchronousSearchAction(); 23 | public static final String NAME = "cluster:admin/opendistro/asynchronous_search/submit"; 24 | 25 | private SubmitAsynchronousSearchAction() { 26 | super(NAME, AsynchronousSearchResponse::new); 27 | } 28 | 29 | } 30 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/opendistroforelasticsearch/search/asynchronous/context/AsynchronousSearchContext.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.opendistroforelasticsearch.search.asynchronous.context; 17 | 18 | import com.amazon.opendistroforelasticsearch.commons.authuser.User; 19 | import com.amazon.opendistroforelasticsearch.search.asynchronous.context.state.AsynchronousSearchState; 20 | import com.amazon.opendistroforelasticsearch.search.asynchronous.id.AsynchronousSearchId; 21 | import com.amazon.opendistroforelasticsearch.search.asynchronous.listener.AsynchronousSearchProgressListener; 22 | import com.amazon.opendistroforelasticsearch.search.asynchronous.response.AsynchronousSearchResponse; 23 | import org.elasticsearch.action.search.SearchResponse; 24 | import org.elasticsearch.common.Nullable; 25 | 26 | import java.util.function.LongSupplier; 27 | 28 | 29 | /** 30 | * Wrapper around information that needs to stay around when an asynchronous search has been submitted. 31 | * This class encapsulates the details of the various elements pertaining to an asynchronous search, including the 32 | * {@linkplain AsynchronousSearchId}, the start time, the updatable expiration time, the search response - completed or partial, the 33 | * error(if the underlying search request fails), the {@linkplain AsynchronousSearchProgressListener} and the current 34 | * {@linkplain AsynchronousSearchState} that the asynchronous search execution has reached in its lifecycle. 35 | */ 36 | public abstract class AsynchronousSearchContext { 37 | 38 | protected final AsynchronousSearchContextId asynchronousSearchContextId; 39 | protected final LongSupplier currentTimeSupplier; 40 | protected volatile AsynchronousSearchState currentStage = AsynchronousSearchState.INIT; 41 | protected volatile AsynchronousSearchProgressListener asynchronousSearchProgressListener; 42 | 43 | public AsynchronousSearchContext(AsynchronousSearchContextId asynchronousSearchContextId, LongSupplier currentTimeSupplier) { 44 | this.asynchronousSearchContextId = asynchronousSearchContextId; 45 | this.currentTimeSupplier = currentTimeSupplier; 46 | } 47 | 48 | public @Nullable 49 | AsynchronousSearchProgressListener getAsynchronousSearchProgressListener() { 50 | return asynchronousSearchProgressListener; 51 | } 52 | 53 | public AsynchronousSearchState getAsynchronousSearchState() { 54 | return currentStage; 55 | } 56 | 57 | public boolean isRunning() { 58 | return getAsynchronousSearchState() == AsynchronousSearchState.RUNNING; 59 | } 60 | 61 | public AsynchronousSearchContextId getContextId() { 62 | return asynchronousSearchContextId; 63 | } 64 | 65 | public abstract String getAsynchronousSearchId(); 66 | 67 | public abstract long getExpirationTimeMillis(); 68 | 69 | public abstract long getStartTimeMillis(); 70 | 71 | public abstract @Nullable 72 | SearchResponse getSearchResponse(); 73 | 74 | public abstract @Nullable 75 | Exception getSearchError(); 76 | 77 | public abstract @Nullable 78 | User getUser(); 79 | 80 | public boolean isExpired() { 81 | return getExpirationTimeMillis() < currentTimeSupplier.getAsLong(); 82 | } 83 | 84 | public AsynchronousSearchResponse getAsynchronousSearchResponse() { 85 | return new AsynchronousSearchResponse(getAsynchronousSearchId(), getAsynchronousSearchState(), getStartTimeMillis(), 86 | getExpirationTimeMillis(), getSearchResponse(), getSearchError()); 87 | } 88 | 89 | public void setState(AsynchronousSearchState targetState) { 90 | this.currentStage = targetState; 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/opendistroforelasticsearch/search/asynchronous/context/AsynchronousSearchContextId.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.opendistroforelasticsearch.search.asynchronous.context; 17 | 18 | import org.elasticsearch.common.io.stream.StreamInput; 19 | import org.elasticsearch.common.io.stream.StreamOutput; 20 | import org.elasticsearch.common.io.stream.Writeable; 21 | 22 | import java.io.IOException; 23 | import java.util.Objects; 24 | 25 | public class AsynchronousSearchContextId implements Writeable { 26 | 27 | private long id; 28 | private String contextId; 29 | 30 | public AsynchronousSearchContextId(String contextId, long id) { 31 | this.id = id; 32 | this.contextId = contextId; 33 | } 34 | 35 | public AsynchronousSearchContextId(StreamInput in) throws IOException { 36 | this.id = in.readLong(); 37 | this.contextId = in.readString(); 38 | } 39 | 40 | @Override 41 | public void writeTo(StreamOutput out) throws IOException { 42 | out.writeLong(id); 43 | out.writeString(contextId); 44 | } 45 | 46 | public String getContextId() { 47 | return contextId; 48 | } 49 | 50 | public long getId() { 51 | return id; 52 | } 53 | 54 | @Override 55 | public boolean equals(Object o) { 56 | if (this == o) return true; 57 | if (o == null || getClass() != o.getClass()) return false; 58 | AsynchronousSearchContextId other = (AsynchronousSearchContextId) o; 59 | return id == other.id && contextId.equals(other.contextId); 60 | } 61 | 62 | @Override 63 | public int hashCode() { 64 | return Objects.hash(contextId, id); 65 | } 66 | 67 | @Override 68 | public String toString() { 69 | return "[" + contextId + "][" + id + "]"; 70 | } 71 | 72 | } 73 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/opendistroforelasticsearch/search/asynchronous/context/active/AsynchronousSearchActiveStore.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | package com.amazon.opendistroforelasticsearch.search.asynchronous.context.active; 16 | 17 | import com.amazon.opendistroforelasticsearch.search.asynchronous.context.AsynchronousSearchContextId; 18 | import com.amazon.opendistroforelasticsearch.search.asynchronous.context.state.AsynchronousSearchStateMachine; 19 | import com.amazon.opendistroforelasticsearch.search.asynchronous.context.state.event.SearchDeletedEvent; 20 | import org.apache.logging.log4j.LogManager; 21 | import org.apache.logging.log4j.Logger; 22 | import org.elasticsearch.cluster.service.ClusterService; 23 | import org.elasticsearch.common.settings.Setting; 24 | import org.elasticsearch.common.settings.Settings; 25 | import org.elasticsearch.common.util.CollectionUtils; 26 | import org.elasticsearch.common.util.concurrent.ConcurrentMapLong; 27 | import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; 28 | 29 | import java.util.Map; 30 | import java.util.Optional; 31 | import java.util.function.Consumer; 32 | import java.util.stream.Stream; 33 | 34 | import static org.elasticsearch.common.util.concurrent.ConcurrentCollections.newConcurrentMapLongWithAggressiveConcurrency; 35 | 36 | 37 | public class AsynchronousSearchActiveStore { 38 | 39 | private static Logger logger = LogManager.getLogger(AsynchronousSearchActiveStore.class); 40 | private volatile int nodeConcurrentRunningSearches; 41 | public static final int NODE_CONCURRENT_RUNNING_SEARCHES = 20; 42 | public static final Setting NODE_CONCURRENT_RUNNING_SEARCHES_SETTING = Setting.intSetting( 43 | "opendistro.asynchronous_search.node_concurrent_running_searches", NODE_CONCURRENT_RUNNING_SEARCHES, 0, 44 | Setting.Property.Dynamic, Setting.Property.NodeScope); 45 | 46 | private final ConcurrentMapLong activeContexts = newConcurrentMapLongWithAggressiveConcurrency(); 47 | 48 | public AsynchronousSearchActiveStore(ClusterService clusterService) { 49 | Settings settings = clusterService.getSettings(); 50 | nodeConcurrentRunningSearches = NODE_CONCURRENT_RUNNING_SEARCHES_SETTING.get(settings); 51 | clusterService.getClusterSettings().addSettingsUpdateConsumer(NODE_CONCURRENT_RUNNING_SEARCHES_SETTING, 52 | this::setNodeConcurrentRunningSearches); 53 | } 54 | 55 | private void setNodeConcurrentRunningSearches(int nodeConcurrentRunningSearches) { 56 | this.nodeConcurrentRunningSearches = nodeConcurrentRunningSearches; 57 | } 58 | 59 | public synchronized void putContext(AsynchronousSearchContextId asynchronousSearchContextId, 60 | AsynchronousSearchActiveContext asynchronousSearchContext, 61 | Consumer contextRejectionEventConsumer) { 62 | if (activeContexts.size() >= nodeConcurrentRunningSearches) { 63 | contextRejectionEventConsumer.accept(asynchronousSearchContextId); 64 | throw new EsRejectedExecutionException("Trying to create too many concurrent searches. Must be less than or equal to: [" 65 | + nodeConcurrentRunningSearches + "]. This limit can be set by changing the [" 66 | + NODE_CONCURRENT_RUNNING_SEARCHES_SETTING.getKey() + "] settings."); 67 | } 68 | activeContexts.put(asynchronousSearchContextId.getId(), asynchronousSearchContext); 69 | } 70 | 71 | public Optional getContext(AsynchronousSearchContextId contextId) { 72 | AsynchronousSearchActiveContext context = activeContexts.get(contextId.getId()); 73 | if (context == null) { 74 | return Optional.empty(); 75 | } 76 | if (context.getContextId().getContextId().equals(contextId.getContextId())) { 77 | return Optional.of(context); 78 | } 79 | return Optional.empty(); 80 | } 81 | 82 | 83 | public Map getAllContexts() { 84 | return CollectionUtils.copyMap(activeContexts); 85 | } 86 | 87 | /** 88 | * Should strictly be executed from within the state machine for a {@link SearchDeletedEvent} 89 | * 90 | * @param asynchronousSearchContextId the context Id to be DELETED 91 | * @return if the context could be DELETED 92 | */ 93 | public boolean freeContext(AsynchronousSearchContextId asynchronousSearchContextId) { 94 | assert calledFromAsynchronousSearchStateMachine() : "Method should only ever be invoked by the state machine"; 95 | AsynchronousSearchActiveContext asynchronousSearchContext = activeContexts.get(asynchronousSearchContextId.getId()); 96 | if (asynchronousSearchContext != null) { 97 | logger.debug("Removing asynchronous search [{}] from active store", asynchronousSearchContext.getAsynchronousSearchId()); 98 | asynchronousSearchContext.close(); 99 | activeContexts.remove(asynchronousSearchContextId.getId()); 100 | return true; 101 | } 102 | return false; 103 | } 104 | 105 | private static boolean calledFromAsynchronousSearchStateMachine() { 106 | return Stream.of(Thread.currentThread().getStackTrace()). 107 | skip(1). //skip getStackTrace 108 | limit(10). //limit depth of analysis to 10 frames, it should be enough 109 | anyMatch(f -> 110 | { 111 | try { 112 | boolean isTestMethodInvocation = f.getClassName().contains("AsynchronousSearchActiveStoreTests"); 113 | boolean isStateMachineTriggerMethodInvocation = AsynchronousSearchStateMachine.class 114 | .isAssignableFrom(Class.forName(f.getClassName())) && f.getMethodName().equals("trigger"); 115 | return isTestMethodInvocation || isStateMachineTriggerMethodInvocation; 116 | } catch (Exception ignored) { 117 | return false; 118 | } 119 | } 120 | ); 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/opendistroforelasticsearch/search/asynchronous/context/active/AsynchronousSearchContextClosedException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.opendistroforelasticsearch.search.asynchronous.context.active; 17 | 18 | import com.amazon.opendistroforelasticsearch.search.asynchronous.context.AsynchronousSearchContextId; 19 | 20 | public class AsynchronousSearchContextClosedException extends Exception { 21 | 22 | private final AsynchronousSearchContextId asynchronousSearchContextId; 23 | 24 | public AsynchronousSearchContextClosedException(AsynchronousSearchContextId asynchronousSearchContextId) { 25 | super("message"); 26 | this.asynchronousSearchContextId = asynchronousSearchContextId; 27 | } 28 | 29 | public AsynchronousSearchContextId getAsynchronousSearchContextId() { 30 | return asynchronousSearchContextId; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/opendistroforelasticsearch/search/asynchronous/context/permits/NoopAsynchronousSearchContextPermits.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.opendistroforelasticsearch.search.asynchronous.context.permits; 17 | 18 | import com.amazon.opendistroforelasticsearch.search.asynchronous.context.AsynchronousSearchContextId; 19 | import com.amazon.opendistroforelasticsearch.search.asynchronous.context.active.AsynchronousSearchContextClosedException; 20 | import org.elasticsearch.action.ActionListener; 21 | import org.elasticsearch.common.lease.Releasable; 22 | import org.elasticsearch.common.unit.TimeValue; 23 | 24 | /** 25 | * NOOP context permit that responds with a NOOP {@linkplain Releasable} to release 26 | */ 27 | public class NoopAsynchronousSearchContextPermits extends AsynchronousSearchContextPermits { 28 | 29 | public NoopAsynchronousSearchContextPermits(AsynchronousSearchContextId asynchronousSearchContextId) { 30 | super(asynchronousSearchContextId, null, null); 31 | } 32 | 33 | @Override 34 | public void asyncAcquirePermit(final ActionListener onAcquired, final TimeValue timeout, String reason) { 35 | if (closed) { 36 | logger.debug("Trying to acquire permit for closed context [{}]", asynchronousSearchContextId); 37 | onAcquired.onFailure(new AsynchronousSearchContextClosedException(asynchronousSearchContextId)); 38 | } else { 39 | onAcquired.onResponse(() -> {}); 40 | } 41 | } 42 | 43 | @Override 44 | public void asyncAcquireAllPermits(ActionListener onAcquired, TimeValue timeout, String reason) { 45 | throw new IllegalStateException("Acquiring all permits is not allowed for asynchronous search id" + asynchronousSearchContextId); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/opendistroforelasticsearch/search/asynchronous/context/persistence/AsynchronousSearchPersistenceContext.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.opendistroforelasticsearch.search.asynchronous.context.persistence; 17 | 18 | import com.amazon.opendistroforelasticsearch.commons.authuser.User; 19 | import com.amazon.opendistroforelasticsearch.search.asynchronous.context.AsynchronousSearchContext; 20 | import com.amazon.opendistroforelasticsearch.search.asynchronous.context.AsynchronousSearchContextId; 21 | import com.amazon.opendistroforelasticsearch.search.asynchronous.context.state.AsynchronousSearchState; 22 | import org.apache.logging.log4j.LogManager; 23 | import org.apache.logging.log4j.Logger; 24 | import org.apache.logging.log4j.message.ParameterizedMessage; 25 | import org.elasticsearch.Version; 26 | import org.elasticsearch.action.search.SearchResponse; 27 | import org.elasticsearch.common.bytes.BytesReference; 28 | import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; 29 | import org.elasticsearch.common.io.stream.NamedWriteableRegistry; 30 | 31 | import java.io.IOException; 32 | import java.nio.ByteBuffer; 33 | import java.util.Base64; 34 | import java.util.Objects; 35 | import java.util.function.LongSupplier; 36 | 37 | /** 38 | * Represents a persisted version of {@link AsynchronousSearchContext} through a backing {@link AsynchronousSearchPersistenceModel} 39 | */ 40 | public class AsynchronousSearchPersistenceContext extends AsynchronousSearchContext { 41 | 42 | private static final Logger logger = LogManager.getLogger(AsynchronousSearchPersistenceContext.class); 43 | 44 | private final String asynchronousSearchId; 45 | private final AsynchronousSearchPersistenceModel asynchronousSearchPersistenceModel; 46 | private final NamedWriteableRegistry namedWriteableRegistry; 47 | 48 | public AsynchronousSearchPersistenceContext(String asynchronousSearchId, AsynchronousSearchContextId asynchronousSearchContextId, 49 | AsynchronousSearchPersistenceModel asynchronousSearchPersistenceModel, 50 | LongSupplier currentTimeSupplier, 51 | NamedWriteableRegistry namedWriteableRegistry) { 52 | super(asynchronousSearchContextId, currentTimeSupplier); 53 | Objects.requireNonNull(asynchronousSearchId); 54 | Objects.requireNonNull(asynchronousSearchContextId); 55 | Objects.requireNonNull(asynchronousSearchPersistenceModel); 56 | this.asynchronousSearchId = asynchronousSearchId; 57 | this.asynchronousSearchPersistenceModel = asynchronousSearchPersistenceModel; 58 | this.namedWriteableRegistry = namedWriteableRegistry; 59 | } 60 | 61 | public AsynchronousSearchPersistenceModel getAsynchronousSearchPersistenceModel() { 62 | return asynchronousSearchPersistenceModel; 63 | } 64 | 65 | @Override 66 | public String getAsynchronousSearchId() { 67 | return asynchronousSearchId; 68 | } 69 | 70 | @Override 71 | public boolean isRunning() { 72 | return false; 73 | } 74 | 75 | @Override 76 | public long getExpirationTimeMillis() { 77 | return asynchronousSearchPersistenceModel.getExpirationTimeMillis(); 78 | } 79 | 80 | @Override 81 | public long getStartTimeMillis() { 82 | return asynchronousSearchPersistenceModel.getStartTimeMillis(); 83 | } 84 | 85 | @Override 86 | public SearchResponse getSearchResponse() { 87 | if (asynchronousSearchPersistenceModel.getResponse() == null) { 88 | return null; 89 | } else { 90 | BytesReference bytesReference = 91 | BytesReference.fromByteBuffer(ByteBuffer.wrap(Base64.getUrlDecoder().decode( 92 | asynchronousSearchPersistenceModel.getResponse()))); 93 | try (NamedWriteableAwareStreamInput wrapperStreamInput = new NamedWriteableAwareStreamInput(bytesReference.streamInput(), 94 | namedWriteableRegistry)) { 95 | wrapperStreamInput.setVersion(Version.readVersion(wrapperStreamInput)); 96 | return new SearchResponse(wrapperStreamInput); 97 | } catch (IOException e) { 98 | logger.error(() -> new ParameterizedMessage("Failed to parse search response for asynchronous search [{}] Response : [{}] ", 99 | asynchronousSearchId, asynchronousSearchPersistenceModel.getResponse()), e); 100 | return null; 101 | } 102 | } 103 | } 104 | 105 | @Override 106 | public Exception getSearchError() { 107 | if (asynchronousSearchPersistenceModel.getError() == null) { 108 | return null; 109 | } 110 | BytesReference bytesReference = 111 | BytesReference.fromByteBuffer(ByteBuffer.wrap(Base64.getUrlDecoder() 112 | .decode(asynchronousSearchPersistenceModel.getError()))); 113 | try (NamedWriteableAwareStreamInput wrapperStreamInput = new NamedWriteableAwareStreamInput(bytesReference.streamInput(), 114 | namedWriteableRegistry)) { 115 | wrapperStreamInput.setVersion(Version.readVersion(wrapperStreamInput)); 116 | return wrapperStreamInput.readException(); 117 | } catch (IOException e) { 118 | logger.error(() -> new ParameterizedMessage("Failed to parse search error for asynchronous search [{}] Error : [{}] ", 119 | asynchronousSearchId, asynchronousSearchPersistenceModel.getResponse()), e); 120 | return null; 121 | } 122 | } 123 | 124 | @Override 125 | public User getUser() { 126 | return asynchronousSearchPersistenceModel.getUser(); 127 | } 128 | 129 | @Override 130 | public AsynchronousSearchState getAsynchronousSearchState() { 131 | return AsynchronousSearchState.STORE_RESIDENT; 132 | } 133 | 134 | @Override 135 | public int hashCode() { 136 | return Objects.hash(asynchronousSearchId, asynchronousSearchPersistenceModel); 137 | } 138 | 139 | @Override 140 | public boolean equals(Object o) { 141 | if (this == o) 142 | return true; 143 | if (o == null || getClass() != o.getClass()) 144 | return false; 145 | AsynchronousSearchPersistenceContext asynchronousSearchPersistenceContext = (AsynchronousSearchPersistenceContext) o; 146 | return asynchronousSearchPersistenceContext.getAsynchronousSearchId() 147 | .equals(this.asynchronousSearchId) && asynchronousSearchPersistenceContext.getAsynchronousSearchPersistenceModel() 148 | .equals(this.asynchronousSearchPersistenceModel); 149 | } 150 | 151 | } 152 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/opendistroforelasticsearch/search/asynchronous/context/persistence/AsynchronousSearchPersistenceModel.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.opendistroforelasticsearch.search.asynchronous.context.persistence; 17 | 18 | import com.amazon.opendistroforelasticsearch.commons.authuser.User; 19 | import org.elasticsearch.ElasticsearchException; 20 | import org.elasticsearch.Version; 21 | import org.elasticsearch.action.search.SearchResponse; 22 | import org.elasticsearch.common.bytes.BytesReference; 23 | import org.elasticsearch.common.io.stream.BytesStreamOutput; 24 | 25 | import java.io.IOException; 26 | import java.util.Base64; 27 | 28 | /** 29 | * The model for persisting asynchronous search response to an index for retrieval after the search is complete 30 | */ 31 | public class AsynchronousSearchPersistenceModel { 32 | 33 | private final long expirationTimeMillis; 34 | private final long startTimeMillis; 35 | private final String response; 36 | private final String error; 37 | private final User user; 38 | 39 | public AsynchronousSearchPersistenceModel(long startTimeMillis, long expirationTimeMillis, String response, String error, User user) { 40 | this.startTimeMillis = startTimeMillis; 41 | this.expirationTimeMillis = expirationTimeMillis; 42 | this.response = response; 43 | this.error = error; 44 | this.user = user; 45 | } 46 | 47 | public AsynchronousSearchPersistenceModel(long startTimeMillis, long expirationTimeMillis, SearchResponse response, 48 | Exception error, User user) throws IOException { 49 | this.startTimeMillis = startTimeMillis; 50 | this.expirationTimeMillis = expirationTimeMillis; 51 | this.response = serializeResponse(response); 52 | this.error = serializeError(error); 53 | this.user = user; 54 | } 55 | 56 | private String serializeResponse(SearchResponse response) throws IOException { 57 | if (response == null) { 58 | return null; 59 | } 60 | try (BytesStreamOutput out = new BytesStreamOutput()) { 61 | Version.writeVersion(Version.CURRENT, out); 62 | response.writeTo(out); 63 | byte[] bytes = BytesReference.toBytes(out.bytes()); 64 | return Base64.getUrlEncoder().withoutPadding().encodeToString(bytes); 65 | } 66 | } 67 | 68 | /** 69 | * Serializes exception in string format 70 | * 71 | * @param error the exception 72 | * @return Serialized error 73 | * @throws IOException when serialization fails. 74 | * String type is required to store binary field in index 75 | */ 76 | private String serializeError(Exception error) throws IOException { 77 | if (error == null) { 78 | return null; 79 | } 80 | try (BytesStreamOutput out = new BytesStreamOutput()) { 81 | Version.writeVersion(Version.CURRENT, out); 82 | out.writeException(error instanceof ElasticsearchException ? error : new ElasticsearchException(error)); 83 | byte[] bytes = BytesReference.toBytes(out.bytes()); 84 | return Base64.getUrlEncoder().withoutPadding().encodeToString(bytes); 85 | } 86 | } 87 | 88 | public long getStartTimeMillis() { 89 | return startTimeMillis; 90 | } 91 | 92 | public String getResponse() { 93 | return response; 94 | } 95 | 96 | @Override 97 | public int hashCode() { 98 | return super.hashCode(); 99 | } 100 | 101 | public String getError() { 102 | return error; 103 | } 104 | 105 | public long getExpirationTimeMillis() { 106 | return expirationTimeMillis; 107 | } 108 | 109 | public User getUser() { 110 | return user; 111 | } 112 | 113 | @Override 114 | public boolean equals(Object o) { 115 | if (this == o) { 116 | return true; 117 | } 118 | if (o == null || getClass() != o.getClass()) { 119 | return false; 120 | } 121 | AsynchronousSearchPersistenceModel other = (AsynchronousSearchPersistenceModel) o; 122 | return 123 | startTimeMillis == other.startTimeMillis && expirationTimeMillis == other.expirationTimeMillis 124 | && ((response == null && other.response == null) || 125 | (response != null && other.response != null && response.equals(other.response))) 126 | && ((error == null && other.error == null) || 127 | (error != null && other.error != null && error.equals(other.error))) 128 | && ((user == null && other.user == null) || 129 | (user != null && other.user != null && user.equals(other.user))); 130 | 131 | } 132 | } 133 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/opendistroforelasticsearch/search/asynchronous/context/state/AsynchronousSearchContextEvent.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.opendistroforelasticsearch.search.asynchronous.context.state; 17 | 18 | import com.amazon.opendistroforelasticsearch.search.asynchronous.context.AsynchronousSearchContext; 19 | 20 | import java.util.Objects; 21 | 22 | /** 23 | * The AsynchronousSearchContextEvent on which the transitions take place 24 | */ 25 | public abstract class AsynchronousSearchContextEvent { 26 | 27 | protected final AsynchronousSearchContext asynchronousSearchContext; 28 | 29 | protected AsynchronousSearchContextEvent(AsynchronousSearchContext asynchronousSearchContext) { 30 | Objects.requireNonNull(asynchronousSearchContext); 31 | this.asynchronousSearchContext = asynchronousSearchContext; 32 | } 33 | 34 | public AsynchronousSearchContext asynchronousSearchContext() { 35 | return asynchronousSearchContext; 36 | } 37 | 38 | } 39 | 40 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/opendistroforelasticsearch/search/asynchronous/context/state/AsynchronousSearchState.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.opendistroforelasticsearch.search.asynchronous.context.state; 17 | 18 | import org.elasticsearch.action.search.SearchTask; 19 | 20 | /** 21 | * The state of the asynchronous search. 22 | */ 23 | public enum AsynchronousSearchState { 24 | 25 | /** 26 | * At the start of the search, before the {@link SearchTask} starts to run 27 | */ 28 | INIT, 29 | 30 | /** 31 | * The search state actually has been started 32 | */ 33 | RUNNING, 34 | 35 | /** 36 | * The search has completed successfully 37 | */ 38 | SUCCEEDED, 39 | 40 | /** 41 | * The search execution has failed 42 | */ 43 | FAILED, 44 | 45 | /** 46 | * The response is starting to get persisted 47 | */ 48 | PERSISTING, 49 | 50 | /** 51 | * The context has been persisted to system index 52 | */ 53 | PERSIST_SUCCEEDED, 54 | 55 | /** 56 | * The context has failed to persist to system index 57 | */ 58 | PERSIST_FAILED, 59 | 60 | /** 61 | * The context has been deleted 62 | */ 63 | CLOSED, 64 | 65 | /** 66 | * The context has been retrieved from asynchronous search response system index 67 | */ 68 | STORE_RESIDENT 69 | } 70 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/opendistroforelasticsearch/search/asynchronous/context/state/AsynchronousSearchStateMachineClosedException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.opendistroforelasticsearch.search.asynchronous.context.state; 17 | 18 | import java.util.Locale; 19 | 20 | public class AsynchronousSearchStateMachineClosedException extends Exception { 21 | 22 | private final AsynchronousSearchState currentState; 23 | private final AsynchronousSearchContextEvent contextEvent; 24 | 25 | public AsynchronousSearchStateMachineClosedException(AsynchronousSearchState currentState, 26 | AsynchronousSearchContextEvent contextEvent) { 27 | super(String.format(Locale.ROOT, "Invalid transition for CLOSED context [%s] from source state [%s] on event [%s]", 28 | contextEvent.asynchronousSearchContext.getAsynchronousSearchId(), currentState, contextEvent.getClass().getName())); 29 | this.currentState = currentState; 30 | this.contextEvent = contextEvent; 31 | } 32 | 33 | public AsynchronousSearchState getCurrentState() { 34 | return currentState; 35 | } 36 | 37 | public AsynchronousSearchContextEvent getContextEvent() { 38 | return contextEvent; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/opendistroforelasticsearch/search/asynchronous/context/state/AsynchronousSearchTransition.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.opendistroforelasticsearch.search.asynchronous.context.state; 17 | 18 | import com.amazon.opendistroforelasticsearch.search.asynchronous.context.AsynchronousSearchContextId; 19 | import com.amazon.opendistroforelasticsearch.search.asynchronous.listener.AsynchronousSearchContextEventListener; 20 | 21 | import java.util.function.BiConsumer; 22 | 23 | public class AsynchronousSearchTransition 24 | implements Transition { 25 | 26 | private final AsynchronousSearchState sourceState; 27 | private final AsynchronousSearchState targetState; 28 | private final BiConsumer onEvent; 29 | private final BiConsumer eventListener; 30 | private final Class eventType; 31 | 32 | public AsynchronousSearchTransition(AsynchronousSearchState sourceState, AsynchronousSearchState targetState, 33 | BiConsumer onEvent, 34 | BiConsumer eventListener, 35 | Class eventName) { 36 | this.sourceState = sourceState; 37 | this.targetState = targetState; 38 | this.onEvent = onEvent; 39 | this.eventListener = eventListener; 40 | this.eventType = eventName; 41 | } 42 | 43 | @Override 44 | public AsynchronousSearchState sourceState() { 45 | return sourceState; 46 | } 47 | 48 | @Override 49 | public AsynchronousSearchState targetState() { 50 | return targetState; 51 | } 52 | 53 | @Override 54 | public Class eventType() { 55 | return eventType; 56 | } 57 | 58 | @Override 59 | public BiConsumer onEvent() { 60 | return onEvent; 61 | } 62 | 63 | @Override 64 | public BiConsumer eventListener() { 65 | return eventListener; 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/opendistroforelasticsearch/search/asynchronous/context/state/StateMachine.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.opendistroforelasticsearch.search.asynchronous.context.state; 17 | 18 | import java.util.Map; 19 | import java.util.Set; 20 | 21 | /** 22 | * {@linkplain StateMachine} provides APIs for generic finite state machine needed 23 | * for basic operations like working with states, events and a lifecycle. 24 | * 25 | * @param the type of state 26 | * @param the type of event 27 | */ 28 | interface StateMachine { 29 | 30 | /** 31 | * Return FSM initial state. 32 | * 33 | * @return FSM initial state 34 | */ 35 | State getInitialState(); 36 | 37 | /** 38 | * Return FSM final states. 39 | * 40 | * @return FSM final states 41 | */ 42 | Set getFinalStates(); 43 | 44 | /** 45 | * Return FSM registered states. 46 | * 47 | * @return FSM registered states 48 | */ 49 | Set getStates(); 50 | 51 | /** 52 | * Return FSM registered transitions. 53 | * 54 | * @return FSM registered transitions 55 | */ 56 | Map> getTransitions(); 57 | 58 | /** 59 | * Fire an event. According to event type, the FSM will make the right transition. 60 | * 61 | * @param event to fire 62 | * @return The next FSM state defined by the transition to make 63 | * @throws Exception thrown if an exception occurs during event handling 64 | */ 65 | State trigger(Event event) throws Exception; 66 | 67 | } 68 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/opendistroforelasticsearch/search/asynchronous/context/state/Transition.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.opendistroforelasticsearch.search.asynchronous.context.state; 17 | 18 | import java.util.function.BiConsumer; 19 | 20 | public interface Transition { 21 | 22 | /** 23 | * Return transition source state. 24 | * 25 | * @return transition source state 26 | */ 27 | State sourceState(); 28 | 29 | /** 30 | * Return transition target state. 31 | * 32 | * @return transition target state 33 | */ 34 | State targetState(); 35 | 36 | /** 37 | * the event type 38 | * @return event type 39 | */ 40 | 41 | Class eventType(); 42 | 43 | /** 44 | * Return a consumer to execute when an event is fired. 45 | * 46 | * @return a bi-consumer 47 | */ 48 | BiConsumer onEvent(); 49 | 50 | /** 51 | * Event listener to be invoked on transition 52 | * 53 | * @return the event listener 54 | */ 55 | 56 | BiConsumer eventListener(); 57 | 58 | } 59 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/opendistroforelasticsearch/search/asynchronous/context/state/event/BeginPersistEvent.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.opendistroforelasticsearch.search.asynchronous.context.state.event; 17 | 18 | import com.amazon.opendistroforelasticsearch.search.asynchronous.context.AsynchronousSearchContext; 19 | import com.amazon.opendistroforelasticsearch.search.asynchronous.context.persistence.AsynchronousSearchPersistenceModel; 20 | import com.amazon.opendistroforelasticsearch.search.asynchronous.context.state.AsynchronousSearchContextEvent; 21 | import org.apache.logging.log4j.LogManager; 22 | import org.apache.logging.log4j.Logger; 23 | import org.apache.logging.log4j.message.ParameterizedMessage; 24 | 25 | import java.io.IOException; 26 | 27 | public class BeginPersistEvent extends AsynchronousSearchContextEvent { 28 | 29 | private static final Logger logger = LogManager.getLogger(BeginPersistEvent.class); 30 | 31 | public BeginPersistEvent(AsynchronousSearchContext asynchronousSearchContext) { 32 | super(asynchronousSearchContext); 33 | } 34 | 35 | public AsynchronousSearchPersistenceModel getAsynchronousSearchPersistenceModel() { 36 | try { 37 | return new AsynchronousSearchPersistenceModel(asynchronousSearchContext.getStartTimeMillis(), 38 | asynchronousSearchContext.getExpirationTimeMillis(), asynchronousSearchContext.getSearchResponse(), 39 | asynchronousSearchContext.getSearchError(), asynchronousSearchContext.getUser()); 40 | } catch (IOException e) { 41 | logger.error(() -> new ParameterizedMessage("Failed to create asynchronous search persistence model" + 42 | " for asynchronous search [{}]", asynchronousSearchContext.getAsynchronousSearchId()), e); 43 | return null; 44 | } 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/opendistroforelasticsearch/search/asynchronous/context/state/event/SearchDeletedEvent.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.opendistroforelasticsearch.search.asynchronous.context.state.event; 17 | 18 | import com.amazon.opendistroforelasticsearch.search.asynchronous.context.active.AsynchronousSearchActiveContext; 19 | import com.amazon.opendistroforelasticsearch.search.asynchronous.context.state.AsynchronousSearchContextEvent; 20 | 21 | public class SearchDeletedEvent extends AsynchronousSearchContextEvent { 22 | 23 | public SearchDeletedEvent(AsynchronousSearchActiveContext asynchronousSearchActiveContext) { 24 | super(asynchronousSearchActiveContext); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/opendistroforelasticsearch/search/asynchronous/context/state/event/SearchFailureEvent.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.opendistroforelasticsearch.search.asynchronous.context.state.event; 17 | 18 | import com.amazon.opendistroforelasticsearch.search.asynchronous.context.AsynchronousSearchContext; 19 | import com.amazon.opendistroforelasticsearch.search.asynchronous.context.state.AsynchronousSearchContextEvent; 20 | 21 | /** 22 | * Event to trigger when an asynchronous search completes with an error. 23 | */ 24 | public class SearchFailureEvent extends AsynchronousSearchContextEvent { 25 | 26 | private final Exception exception; 27 | 28 | public SearchFailureEvent(AsynchronousSearchContext asynchronousSearchContext, Exception exception) { 29 | super(asynchronousSearchContext); 30 | this.exception = exception; 31 | } 32 | 33 | public Exception getException() { 34 | return exception; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/opendistroforelasticsearch/search/asynchronous/context/state/event/SearchResponsePersistFailedEvent.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.opendistroforelasticsearch.search.asynchronous.context.state.event; 17 | 18 | import com.amazon.opendistroforelasticsearch.search.asynchronous.context.AsynchronousSearchContext; 19 | import com.amazon.opendistroforelasticsearch.search.asynchronous.context.state.AsynchronousSearchContextEvent; 20 | 21 | /** 22 | * Event to trigger when failure occurs while storing asynchronous search response in system index. 23 | */ 24 | public class SearchResponsePersistFailedEvent extends AsynchronousSearchContextEvent { 25 | 26 | public SearchResponsePersistFailedEvent(AsynchronousSearchContext asynchronousSearchContext) { 27 | super(asynchronousSearchContext); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/opendistroforelasticsearch/search/asynchronous/context/state/event/SearchResponsePersistedEvent.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.opendistroforelasticsearch.search.asynchronous.context.state.event; 17 | 18 | import com.amazon.opendistroforelasticsearch.search.asynchronous.context.AsynchronousSearchContext; 19 | import com.amazon.opendistroforelasticsearch.search.asynchronous.context.state.AsynchronousSearchContextEvent; 20 | 21 | /** 22 | * Event triggered when asynchronous search response is successfully stored in system index. 23 | */ 24 | public class SearchResponsePersistedEvent extends AsynchronousSearchContextEvent { 25 | 26 | public SearchResponsePersistedEvent(AsynchronousSearchContext asynchronousSearchContext) { 27 | super(asynchronousSearchContext); 28 | } 29 | } 30 | 31 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/opendistroforelasticsearch/search/asynchronous/context/state/event/SearchStartedEvent.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.opendistroforelasticsearch.search.asynchronous.context.state.event; 17 | 18 | import com.amazon.opendistroforelasticsearch.search.asynchronous.context.AsynchronousSearchContext; 19 | import com.amazon.opendistroforelasticsearch.search.asynchronous.context.state.AsynchronousSearchContextEvent; 20 | import org.elasticsearch.action.ActionListener; 21 | import org.elasticsearch.action.ActionRequest; 22 | import org.elasticsearch.action.search.SearchTask; 23 | 24 | /** 25 | * Event triggered when 26 | * {@linkplain org.elasticsearch.action.search.TransportSearchAction#execute(ActionRequest, ActionListener)} is fired, to 27 | * signal the search has begun. 28 | */ 29 | public class SearchStartedEvent extends AsynchronousSearchContextEvent { 30 | 31 | private final SearchTask searchTask; 32 | 33 | public SearchStartedEvent(AsynchronousSearchContext asynchronousSearchContext, SearchTask searchTask) { 34 | super(asynchronousSearchContext); 35 | this.searchTask = searchTask; 36 | } 37 | 38 | @Override 39 | public AsynchronousSearchContext asynchronousSearchContext() { 40 | return asynchronousSearchContext; 41 | } 42 | 43 | public SearchTask getSearchTask() { 44 | return searchTask; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/opendistroforelasticsearch/search/asynchronous/context/state/event/SearchSuccessfulEvent.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.opendistroforelasticsearch.search.asynchronous.context.state.event; 17 | 18 | import com.amazon.opendistroforelasticsearch.search.asynchronous.context.AsynchronousSearchContext; 19 | import com.amazon.opendistroforelasticsearch.search.asynchronous.context.state.AsynchronousSearchContextEvent; 20 | import org.elasticsearch.action.search.SearchResponse; 21 | 22 | /** 23 | * Event triggered when asynchronous search completes with a successful search response. 24 | */ 25 | public class SearchSuccessfulEvent extends AsynchronousSearchContextEvent { 26 | 27 | private SearchResponse searchResponse; 28 | 29 | public SearchSuccessfulEvent(AsynchronousSearchContext asynchronousSearchContext, SearchResponse searchResponse) { 30 | super(asynchronousSearchContext); 31 | this.searchResponse = searchResponse; 32 | } 33 | 34 | public SearchResponse getSearchResponse() { 35 | return searchResponse; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/opendistroforelasticsearch/search/asynchronous/id/AsynchronousSearchId.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.opendistroforelasticsearch.search.asynchronous.id; 17 | 18 | import com.amazon.opendistroforelasticsearch.search.asynchronous.context.AsynchronousSearchContextId; 19 | import java.util.Objects; 20 | 21 | /** 22 | * Holds the following details pertaining to a submitted asynchronous search: 23 | * - the associated asynchronous search context 24 | * - id of the associated search task registered with the task manager 25 | * - the id of node on which acts as coordinator for the asynchronous search 26 | */ 27 | public final class AsynchronousSearchId { 28 | 29 | // UUID + ID generator for uniqueness 30 | private final AsynchronousSearchContextId asynchronousSearchContextId; 31 | // coordinator node id 32 | private final String node; 33 | // the search task id 34 | private final long taskId; 35 | 36 | public AsynchronousSearchId(String node, long taskId, AsynchronousSearchContextId asynchronousSearchContextId) { 37 | this.node = node; 38 | this.taskId = taskId; 39 | this.asynchronousSearchContextId = asynchronousSearchContextId; 40 | } 41 | 42 | public AsynchronousSearchContextId getAsynchronousSearchContextId() { 43 | return asynchronousSearchContextId; 44 | } 45 | 46 | public String getNode() { 47 | return node; 48 | } 49 | 50 | public long getTaskId() { 51 | return taskId; 52 | } 53 | 54 | @Override 55 | public String toString() { 56 | return "[" + node + "][" + taskId + "][" + asynchronousSearchContextId + "]"; 57 | } 58 | 59 | 60 | @Override 61 | public int hashCode() { 62 | return Objects.hash(this.asynchronousSearchContextId, this.node, this.taskId); 63 | } 64 | 65 | @Override 66 | public boolean equals(Object o) { 67 | if (this == o) 68 | return true; 69 | if (o == null || getClass() != o.getClass()) 70 | return false; 71 | AsynchronousSearchId asynchronousSearchId = (AsynchronousSearchId) o; 72 | return asynchronousSearchId.asynchronousSearchContextId.equals(this.asynchronousSearchContextId) 73 | && asynchronousSearchId.node.equals(this.node) 74 | && asynchronousSearchId.taskId == this.taskId; 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/opendistroforelasticsearch/search/asynchronous/id/AsynchronousSearchIdConverter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.opendistroforelasticsearch.search.asynchronous.id; 17 | 18 | import com.amazon.opendistroforelasticsearch.search.asynchronous.context.AsynchronousSearchContextId; 19 | import org.apache.lucene.store.ByteArrayDataInput; 20 | import org.elasticsearch.common.bytes.BytesReference; 21 | import org.elasticsearch.common.io.stream.BytesStreamOutput; 22 | 23 | import java.io.IOException; 24 | import java.util.Base64; 25 | 26 | public class AsynchronousSearchIdConverter { 27 | /** 28 | * Encodes the {@linkplain AsynchronousSearchId} in base64 encoding and returns an identifier for the submitted asynchronous search 29 | * 30 | * @param asynchronousSearchId The object to be encoded 31 | * @return The id which is used to access the submitted asynchronous search 32 | */ 33 | public static String buildAsyncId(AsynchronousSearchId asynchronousSearchId) { 34 | try (BytesStreamOutput out = new BytesStreamOutput()) { 35 | out.writeString(asynchronousSearchId.getNode()); 36 | out.writeString(String.valueOf(asynchronousSearchId.getTaskId())); 37 | out.writeString(asynchronousSearchId.getAsynchronousSearchContextId().getContextId()); 38 | out.writeString(String.valueOf(asynchronousSearchId.getAsynchronousSearchContextId().getId())); 39 | return Base64.getUrlEncoder().encodeToString(BytesReference.toBytes(out.bytes())); 40 | } catch (IOException e) { 41 | throw new IllegalArgumentException("Cannot build asynchronous search id", e); 42 | } 43 | } 44 | 45 | /** 46 | * Attempts to decode a base64 encoded string into an {@linkplain AsynchronousSearchId} which contains the details pertaining to 47 | * an asynchronous search being accessed. 48 | * 49 | * @param asynchronousSearchId The string to be decoded 50 | * @return The parsed {@linkplain AsynchronousSearchId} 51 | */ 52 | public static AsynchronousSearchId parseAsyncId(String asynchronousSearchId) { 53 | try { 54 | byte[] bytes = Base64.getUrlDecoder().decode(asynchronousSearchId); 55 | ByteArrayDataInput in = new ByteArrayDataInput(bytes); 56 | String node = in.readString(); 57 | long taskId = Long.parseLong(in.readString()); 58 | String contextId = in.readString(); 59 | long id = Long.parseLong(in.readString()); 60 | if (in.getPosition() != bytes.length) { 61 | throw new IllegalArgumentException("Not all bytes were read"); 62 | } 63 | return new AsynchronousSearchId(node, taskId, new AsynchronousSearchContextId(contextId, id)); 64 | } catch (Exception e) { 65 | throw new IllegalArgumentException("Cannot parse asynchronous search id", e); 66 | } 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/opendistroforelasticsearch/search/asynchronous/listener/AsynchronousSearchContextEventListener.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.opendistroforelasticsearch.search.asynchronous.listener; 17 | 18 | import com.amazon.opendistroforelasticsearch.search.asynchronous.context.AsynchronousSearchContextId; 19 | 20 | /** 21 | * An listener for asynchronous search context events. 22 | */ 23 | public interface AsynchronousSearchContextEventListener { 24 | 25 | /** 26 | * @param contextId Executed when a new asynchronous search context was created 27 | */ 28 | default void onNewContext(AsynchronousSearchContextId contextId) { 29 | } 30 | 31 | /** 32 | * @param contextId Executed when a previously created asynchronous search context completes. 33 | */ 34 | default void onContextCompleted(AsynchronousSearchContextId contextId) { 35 | } 36 | 37 | /** 38 | * @param contextId Executed when a previously created asynchronous search context fails. 39 | */ 40 | default void onContextFailed(AsynchronousSearchContextId contextId) { 41 | } 42 | 43 | /** 44 | * @param contextId Executed when a previously created asynchronous search context is persisted. 45 | */ 46 | default void onContextPersisted(AsynchronousSearchContextId contextId) { 47 | } 48 | 49 | /** 50 | * @param contextId Executed when a previously created asynchronous search context fails persisting. 51 | */ 52 | default void onContextPersistFailed(AsynchronousSearchContextId contextId) { 53 | } 54 | 55 | /** 56 | * @param contextId Executed when a previously created asynchronous search context is deleted. 57 | */ 58 | default void onContextDeleted(AsynchronousSearchContextId contextId) { 59 | } 60 | 61 | /** 62 | * @param contextId Executed when a previously created asynchronous search context is running. 63 | */ 64 | default void onContextRunning(AsynchronousSearchContextId contextId) { 65 | } 66 | 67 | /** 68 | * @param contextId Executed when asynchronous search context creation is rejected 69 | */ 70 | default void onContextRejected(AsynchronousSearchContextId contextId) { 71 | } 72 | 73 | /** 74 | * @param contextId Executed when a running asynchronous search context is deleted and has bypassed succeeded/failed state 75 | */ 76 | default void onRunningContextDeleted(AsynchronousSearchContextId contextId) { 77 | } 78 | 79 | /** 80 | * @param contextId Executed when an asynchronous search context is cancelled 81 | */ 82 | default void onContextCancelled(AsynchronousSearchContextId contextId) { 83 | } 84 | 85 | /** 86 | * @param contextId Executed when an asynchronous search context is initialized 87 | */ 88 | default void onContextInitialized(AsynchronousSearchContextId contextId) { 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/opendistroforelasticsearch/search/asynchronous/listener/AsynchronousSearchTimeoutWrapper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.opendistroforelasticsearch.search.asynchronous.listener; 17 | 18 | import org.apache.logging.log4j.LogManager; 19 | import org.apache.logging.log4j.Logger; 20 | import org.elasticsearch.action.ActionListener; 21 | import org.elasticsearch.common.unit.TimeValue; 22 | import org.elasticsearch.threadpool.Scheduler; 23 | import org.elasticsearch.threadpool.ThreadPool; 24 | 25 | import java.util.concurrent.atomic.AtomicBoolean; 26 | import java.util.function.Consumer; 27 | 28 | /*** 29 | * The wrapper that schedules a timeout and wraps it in an actual {@link ActionListener}. The {@link AsynchronousSearchTimeoutWrapper} 30 | * guarantees that only one of timeout/response/failure gets executed so that responses are not sent over a closed channel 31 | * The wrap and actual scheduling has been split to avoid races in listeners as they get attached. 32 | */ 33 | public class AsynchronousSearchTimeoutWrapper { 34 | 35 | private static final Logger logger = LogManager.getLogger(AsynchronousSearchTimeoutWrapper.class); 36 | 37 | /*** 38 | * Wraps the listener and schedules the timeout 39 | * @param threadPool threadPool 40 | * @param timeout timeout 41 | * @param executor executor 42 | * @param actionListener actionListener 43 | * @param timeoutConsumer timeoutConsumer 44 | * @param the response 45 | * @return PrioritizedListener 46 | */ 47 | public static PrioritizedActionListener wrapScheduledTimeout(ThreadPool threadPool, TimeValue timeout, 48 | String executor, 49 | ActionListener actionListener, 50 | Consumer> timeoutConsumer) { 51 | return scheduleTimeout(threadPool, timeout, executor, initListener(actionListener, timeoutConsumer)); 52 | } 53 | 54 | /** 55 | * 56 | * @param actionListener actionListener 57 | * @param timeoutConsumer timeoutConsumer 58 | * @param Response 59 | * @return PrioritizedListener 60 | */ 61 | public static PrioritizedActionListener initListener(ActionListener actionListener, 62 | Consumer> timeoutConsumer) { 63 | CompletionPrioritizedActionListener completionTimeoutListener = 64 | new CompletionPrioritizedActionListener<>(actionListener, timeoutConsumer); 65 | return completionTimeoutListener; 66 | } 67 | 68 | /** 69 | * 70 | * @param threadPool the thread pool instance 71 | * @param timeout the configured timeout 72 | * @param executor the thread pool name 73 | * @param completionTimeoutListener the listener 74 | * @param Response 75 | * @return PrioritizedListener 76 | */ 77 | public static PrioritizedActionListener scheduleTimeout(ThreadPool threadPool, TimeValue timeout, String executor, 78 | PrioritizedActionListener completionTimeoutListener) { 79 | ((CompletionPrioritizedActionListener)completionTimeoutListener).cancellable = threadPool.schedule( 80 | (Runnable) completionTimeoutListener, timeout, executor); 81 | return completionTimeoutListener; 82 | } 83 | 84 | static class CompletionPrioritizedActionListener implements PrioritizedActionListener, Runnable { 85 | private final ActionListener actionListener; 86 | private volatile Scheduler.ScheduledCancellable cancellable; 87 | private final AtomicBoolean complete = new AtomicBoolean(false); 88 | private final Consumer> timeoutConsumer; 89 | 90 | CompletionPrioritizedActionListener(ActionListener actionListener, Consumer> timeoutConsumer) { 91 | this.actionListener = actionListener; 92 | this.timeoutConsumer = timeoutConsumer; 93 | } 94 | 95 | void cancel() { 96 | if (cancellable != null && cancellable.isCancelled() == false) { 97 | cancellable.cancel(); 98 | } 99 | } 100 | 101 | @Override 102 | public void run() { 103 | executeImmediately(); 104 | } 105 | 106 | @Override 107 | public void executeImmediately() { 108 | if (complete.compareAndSet(false, true)) { 109 | cancel(); 110 | timeoutConsumer.accept(this); 111 | } 112 | } 113 | 114 | @Override 115 | public void onResponse(Response response) { 116 | if (complete.compareAndSet(false, true)) { 117 | cancel(); 118 | actionListener.onResponse(response); 119 | } 120 | } 121 | 122 | @Override 123 | public void onFailure(Exception e) { 124 | if (complete.compareAndSet(false, true)) { 125 | cancel(); 126 | actionListener.onFailure(e); 127 | } 128 | } 129 | } 130 | } 131 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/opendistroforelasticsearch/search/asynchronous/listener/CompositeSearchProgressActionListener.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.opendistroforelasticsearch.search.asynchronous.listener; 17 | 18 | import org.elasticsearch.action.ActionListener; 19 | import org.elasticsearch.action.search.SearchProgressActionListener; 20 | 21 | import java.util.ArrayList; 22 | import java.util.List; 23 | 24 | /*** 25 | * The implementation of {@link SearchProgressActionListener} responsible for maintaining a list of {@link PrioritizedActionListener} 26 | * to be invoked when a full response is available. The implementation guarantees that the listener once added will exactly be 27 | * invoked once. If the search completes before the listener was added, the listener is immediately invoked 28 | **/ 29 | 30 | public class CompositeSearchProgressActionListener implements ActionListener { 31 | 32 | private final List> actionListeners; 33 | private volatile boolean complete; 34 | 35 | CompositeSearchProgressActionListener() { 36 | this.actionListeners = new ArrayList<>(1); 37 | } 38 | 39 | /*** 40 | * Adds a prioritized listener to listen on to the progress of the search started by a previous request. If the search 41 | * has completed the timeout consumer is immediately invoked. 42 | * @param listener the listener 43 | */ 44 | public void addOrExecuteListener(PrioritizedActionListener listener) { 45 | if (addListener(listener) == false) { 46 | listener.executeImmediately(); 47 | } 48 | } 49 | 50 | public synchronized void removeListener(ActionListener listener) { 51 | this.actionListeners.remove(listener); 52 | } 53 | 54 | private synchronized boolean addListener(ActionListener listener) { 55 | if (complete == false) { 56 | this.actionListeners.add(listener); 57 | return true; 58 | } 59 | return false; 60 | } 61 | 62 | 63 | @Override 64 | public void onResponse(T response) { 65 | Iterable> actionListenersToBeInvoked = finalizeListeners(); 66 | if (actionListenersToBeInvoked != null) { 67 | ActionListener.onResponse(actionListenersToBeInvoked, response); 68 | } 69 | } 70 | 71 | @Override 72 | public void onFailure(Exception exception) { 73 | Iterable> actionListenersToBeInvoked = finalizeListeners(); 74 | if (actionListenersToBeInvoked != null) { 75 | ActionListener.onFailure(actionListenersToBeInvoked, exception); 76 | } 77 | } 78 | 79 | private Iterable> finalizeListeners() { 80 | List> actionListenersToBeInvoked = null; 81 | synchronized (this) { 82 | if (complete == false) { 83 | actionListenersToBeInvoked = new ArrayList<>(actionListeners); 84 | actionListeners.clear(); 85 | complete = true; 86 | } 87 | } 88 | return actionListenersToBeInvoked; 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/opendistroforelasticsearch/search/asynchronous/listener/PartialResponseProvider.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.opendistroforelasticsearch.search.asynchronous.listener; 17 | 18 | import org.elasticsearch.action.search.SearchResponse; 19 | 20 | @FunctionalInterface 21 | public interface PartialResponseProvider { 22 | 23 | SearchResponse partialResponse(); 24 | } 25 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/opendistroforelasticsearch/search/asynchronous/listener/PrioritizedActionListener.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.opendistroforelasticsearch.search.asynchronous.listener; 17 | 18 | 19 | import org.elasticsearch.action.ActionListener; 20 | 21 | public interface PrioritizedActionListener extends ActionListener { 22 | 23 | void executeImmediately(); 24 | } 25 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/opendistroforelasticsearch/search/asynchronous/request/AsynchronousSearchRoutingRequest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.opendistroforelasticsearch.search.asynchronous.request; 17 | 18 | import org.elasticsearch.action.ActionRequest; 19 | import org.elasticsearch.common.io.stream.StreamInput; 20 | import org.elasticsearch.common.io.stream.StreamOutput; 21 | 22 | import java.io.IOException; 23 | 24 | /** 25 | * A base request for routing asynchronous search request to the right node by id. 26 | */ 27 | public abstract class AsynchronousSearchRoutingRequest> extends ActionRequest { 28 | 29 | private final String id; 30 | 31 | protected AsynchronousSearchRoutingRequest(String id) { 32 | this.id = id; 33 | } 34 | 35 | protected AsynchronousSearchRoutingRequest(StreamInput in) throws IOException { 36 | super(in); 37 | id = in.readString(); 38 | } 39 | 40 | @Override 41 | public void writeTo(StreamOutput out) throws IOException { 42 | super.writeTo(out); 43 | out.writeString(id); 44 | } 45 | 46 | public String getId() { 47 | return this.id; 48 | } 49 | 50 | } 51 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/opendistroforelasticsearch/search/asynchronous/request/AsynchronousSearchStatsRequest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.opendistroforelasticsearch.search.asynchronous.request; 17 | 18 | import org.elasticsearch.action.support.nodes.BaseNodesRequest; 19 | import org.elasticsearch.common.io.stream.StreamInput; 20 | import org.elasticsearch.common.io.stream.StreamOutput; 21 | 22 | import java.io.IOException; 23 | 24 | /** 25 | * A request to get Async Search stats 26 | */ 27 | public class AsynchronousSearchStatsRequest extends BaseNodesRequest { 28 | 29 | /** 30 | * Empty constructor needed for AsynchronousSearchStatsTransportAction 31 | */ 32 | public AsynchronousSearchStatsRequest() { 33 | super((String[]) null); 34 | } 35 | 36 | /** 37 | * Constructor 38 | * 39 | * @param in input stream 40 | * @throws IOException in case of I/O errors 41 | */ 42 | public AsynchronousSearchStatsRequest(StreamInput in) throws IOException { 43 | super(in); 44 | } 45 | 46 | public AsynchronousSearchStatsRequest(String... nodeIds) { 47 | super(nodeIds); 48 | } 49 | 50 | @Override 51 | public void writeTo(StreamOutput out) throws IOException { 52 | super.writeTo(out); 53 | } 54 | 55 | } 56 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/opendistroforelasticsearch/search/asynchronous/request/DeleteAsynchronousSearchRequest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.opendistroforelasticsearch.search.asynchronous.request; 17 | 18 | import org.elasticsearch.action.ActionRequestValidationException; 19 | import org.elasticsearch.common.io.stream.StreamInput; 20 | 21 | import java.io.IOException; 22 | 23 | /** 24 | * A request to delete an asynchronous search response based on its id 25 | */ 26 | public class DeleteAsynchronousSearchRequest extends AsynchronousSearchRoutingRequest { 27 | 28 | public DeleteAsynchronousSearchRequest(String id) { 29 | super(id); 30 | } 31 | 32 | public DeleteAsynchronousSearchRequest(StreamInput streamInput) throws IOException { 33 | super(streamInput); 34 | } 35 | 36 | @Override 37 | public ActionRequestValidationException validate() { 38 | return null; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/opendistroforelasticsearch/search/asynchronous/request/GetAsynchronousSearchRequest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.opendistroforelasticsearch.search.asynchronous.request; 17 | 18 | import org.elasticsearch.action.ActionRequestValidationException; 19 | import org.elasticsearch.common.Nullable; 20 | import org.elasticsearch.common.io.stream.StreamInput; 21 | import org.elasticsearch.common.io.stream.StreamOutput; 22 | import org.elasticsearch.common.unit.TimeValue; 23 | 24 | import java.io.IOException; 25 | 26 | /** 27 | * A request to fetch an asynchronous search response by id. 28 | */ 29 | public class GetAsynchronousSearchRequest extends AsynchronousSearchRoutingRequest { 30 | 31 | @Nullable 32 | private TimeValue waitForCompletionTimeout; 33 | 34 | @Nullable 35 | private TimeValue keepAlive; 36 | 37 | public GetAsynchronousSearchRequest(String id) { 38 | super(id); 39 | } 40 | 41 | public TimeValue getWaitForCompletionTimeout() { 42 | return waitForCompletionTimeout; 43 | } 44 | 45 | public void setWaitForCompletionTimeout(TimeValue waitForCompletionTimeout) { 46 | this.waitForCompletionTimeout = waitForCompletionTimeout; 47 | } 48 | 49 | public TimeValue getKeepAlive() { 50 | return keepAlive; 51 | } 52 | 53 | public void setKeepAlive(TimeValue keepAlive) { 54 | this.keepAlive = keepAlive; 55 | } 56 | 57 | 58 | public GetAsynchronousSearchRequest(StreamInput streamInput) throws IOException { 59 | super(streamInput); 60 | keepAlive = streamInput.readOptionalTimeValue(); 61 | waitForCompletionTimeout = streamInput.readOptionalTimeValue(); 62 | } 63 | 64 | @Override 65 | public void writeTo(StreamOutput out) throws IOException { 66 | super.writeTo(out); 67 | out.writeOptionalTimeValue(keepAlive); 68 | out.writeOptionalTimeValue(waitForCompletionTimeout); 69 | } 70 | 71 | @Override 72 | public ActionRequestValidationException validate() { 73 | return null; 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/opendistroforelasticsearch/search/asynchronous/response/AcknowledgedResponse.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.opendistroforelasticsearch.search.asynchronous.response; 17 | 18 | import org.elasticsearch.action.ActionResponse; 19 | import org.elasticsearch.common.ParseField; 20 | import org.elasticsearch.common.io.stream.StreamInput; 21 | import org.elasticsearch.common.io.stream.StreamOutput; 22 | import org.elasticsearch.common.xcontent.ConstructingObjectParser; 23 | import org.elasticsearch.common.xcontent.ObjectParser; 24 | import org.elasticsearch.common.xcontent.StatusToXContentObject; 25 | import org.elasticsearch.common.xcontent.XContentBuilder; 26 | import org.elasticsearch.rest.RestStatus; 27 | 28 | import java.io.IOException; 29 | import java.util.Objects; 30 | 31 | import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; 32 | import static org.elasticsearch.rest.RestStatus.NOT_FOUND; 33 | import static org.elasticsearch.rest.RestStatus.OK; 34 | 35 | /** 36 | * A response that indicates that a request has been acknowledged 37 | */ 38 | public class AcknowledgedResponse extends ActionResponse implements StatusToXContentObject { 39 | 40 | private static final ParseField ACKNOWLEDGED = new ParseField("acknowledged"); 41 | 42 | protected boolean acknowledged; 43 | 44 | public AcknowledgedResponse(StreamInput in) throws IOException { 45 | super(in); 46 | acknowledged = in.readBoolean(); 47 | } 48 | 49 | public AcknowledgedResponse(StreamInput in, boolean readAcknowledged) throws IOException { 50 | super(in); 51 | if (readAcknowledged) { 52 | acknowledged = in.readBoolean(); 53 | } 54 | } 55 | 56 | public AcknowledgedResponse(boolean acknowledged) { 57 | this.acknowledged = acknowledged; 58 | } 59 | 60 | /** 61 | * Returns whether the response is acknowledged or not 62 | * 63 | * @return true if the response is acknowledged, false otherwise 64 | */ 65 | public final boolean isAcknowledged() { 66 | return acknowledged; 67 | } 68 | 69 | @Override 70 | public void writeTo(StreamOutput out) throws IOException { 71 | out.writeBoolean(acknowledged); 72 | } 73 | 74 | @Override 75 | public final XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { 76 | builder.startObject(); 77 | builder.field(ACKNOWLEDGED.getPreferredName(), isAcknowledged()); 78 | addCustomFields(builder, params); 79 | builder.endObject(); 80 | return builder; 81 | } 82 | 83 | protected void addCustomFields(XContentBuilder builder, Params params) throws IOException { 84 | 85 | } 86 | 87 | /** 88 | * A generic parser that simply parses the acknowledged flag 89 | */ 90 | private static final ConstructingObjectParser ACKNOWLEDGED_FLAG_PARSER = new ConstructingObjectParser<>( 91 | "acknowledged_flag", true, args -> (Boolean) args[0]); 92 | 93 | static { 94 | ACKNOWLEDGED_FLAG_PARSER.declareField(constructorArg(), (parser, context) -> parser.booleanValue(), ACKNOWLEDGED, 95 | ObjectParser.ValueType.BOOLEAN); 96 | } 97 | 98 | @Override 99 | public boolean equals(Object o) { 100 | if (this == o) { 101 | return true; 102 | } 103 | if (o == null || getClass() != o.getClass()) { 104 | return false; 105 | } 106 | AcknowledgedResponse that = (AcknowledgedResponse) o; 107 | return isAcknowledged() == that.isAcknowledged(); 108 | } 109 | 110 | @Override 111 | public int hashCode() { 112 | return Objects.hash(isAcknowledged()); 113 | } 114 | 115 | @Override 116 | public RestStatus status() { 117 | return acknowledged ? OK : NOT_FOUND; 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/opendistroforelasticsearch/search/asynchronous/response/AsynchronousSearchStatsResponse.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.opendistroforelasticsearch.search.asynchronous.response; 17 | 18 | import com.amazon.opendistroforelasticsearch.search.asynchronous.stats.AsynchronousSearchStats; 19 | import org.elasticsearch.action.FailedNodeException; 20 | import org.elasticsearch.action.support.nodes.BaseNodesResponse; 21 | import org.elasticsearch.cluster.ClusterName; 22 | import org.elasticsearch.common.Strings; 23 | import org.elasticsearch.common.io.stream.StreamInput; 24 | import org.elasticsearch.common.io.stream.StreamOutput; 25 | import org.elasticsearch.common.xcontent.ToXContentObject; 26 | import org.elasticsearch.common.xcontent.XContentBuilder; 27 | import org.elasticsearch.common.xcontent.XContentFactory; 28 | 29 | import java.io.IOException; 30 | import java.util.List; 31 | 32 | /** 33 | * Contains aggregation of asynchronous search stats from all the nodes 34 | */ 35 | public class AsynchronousSearchStatsResponse extends BaseNodesResponse implements ToXContentObject { 36 | public AsynchronousSearchStatsResponse(StreamInput in) throws IOException { 37 | super(in); 38 | } 39 | 40 | public AsynchronousSearchStatsResponse(ClusterName clusterName, List nodes, 41 | List failures) { 42 | super(clusterName, nodes, failures); 43 | } 44 | 45 | @Override 46 | protected List readNodesFrom(StreamInput in) throws IOException { 47 | return in.readList(new Reader() { 48 | @Override 49 | public AsynchronousSearchStats read(StreamInput in1) throws IOException { 50 | return new AsynchronousSearchStats(in1); 51 | } 52 | }); 53 | } 54 | 55 | @Override 56 | protected void writeNodesTo(StreamOutput out, List nodes) throws IOException { 57 | out.writeList(nodes); 58 | } 59 | 60 | @Override 61 | public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { 62 | builder.startObject("nodes"); 63 | for (AsynchronousSearchStats stats : getNodes()) { 64 | builder.startObject(stats.getNode().getId()); 65 | stats.toXContent(builder, params); 66 | builder.endObject(); 67 | } 68 | builder.endObject(); 69 | return builder; 70 | } 71 | 72 | @Override 73 | public String toString() { 74 | try { 75 | XContentBuilder builder = XContentFactory.jsonBuilder().prettyPrint(); 76 | builder.startObject(); 77 | toXContent(builder, EMPTY_PARAMS); 78 | builder.endObject(); 79 | return Strings.toString(builder); 80 | } catch (IOException e) { 81 | return "{ \"error\" : \"" + e.getMessage() + "\"}"; 82 | } 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/opendistroforelasticsearch/search/asynchronous/rest/RestAsynchronousSearchStatsAction.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.opendistroforelasticsearch.search.asynchronous.rest; 17 | 18 | import com.amazon.opendistroforelasticsearch.search.asynchronous.action.AsynchronousSearchStatsAction; 19 | import com.amazon.opendistroforelasticsearch.search.asynchronous.plugin.AsynchronousSearchPlugin; 20 | import com.amazon.opendistroforelasticsearch.search.asynchronous.request.AsynchronousSearchStatsRequest; 21 | import org.apache.logging.log4j.LogManager; 22 | import org.apache.logging.log4j.Logger; 23 | import org.elasticsearch.client.node.NodeClient; 24 | import org.elasticsearch.common.Strings; 25 | import org.elasticsearch.rest.BaseRestHandler; 26 | import org.elasticsearch.rest.RestRequest; 27 | import org.elasticsearch.rest.action.RestActions; 28 | 29 | import java.util.Arrays; 30 | import java.util.List; 31 | 32 | import static org.elasticsearch.rest.RestRequest.Method.GET; 33 | 34 | public class RestAsynchronousSearchStatsAction extends BaseRestHandler { 35 | 36 | private static final Logger LOG = LogManager.getLogger(RestAsynchronousSearchStatsAction.class); 37 | 38 | private static final String NAME = "asynchronous_search_stats_action"; 39 | 40 | public RestAsynchronousSearchStatsAction() { 41 | } 42 | 43 | @Override 44 | public String getName() { 45 | return NAME; 46 | } 47 | 48 | 49 | @Override 50 | public List routes() { 51 | return Arrays.asList( 52 | new Route(GET, AsynchronousSearchPlugin.BASE_URI + "/_nodes/{nodeId}/stats"), 53 | new Route(GET, AsynchronousSearchPlugin.BASE_URI + "/stats") 54 | ); 55 | } 56 | 57 | @Override 58 | protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) { 59 | AsynchronousSearchStatsRequest asynchronousSearchStatsRequest = getRequest(request); 60 | 61 | return channel -> client.execute(AsynchronousSearchStatsAction.INSTANCE, asynchronousSearchStatsRequest, 62 | new RestActions.NodesResponseRestListener<>(channel)); 63 | } 64 | 65 | private AsynchronousSearchStatsRequest getRequest(RestRequest request) { 66 | // parse the nodes the user wants to query 67 | String[] nodesIds = Strings.splitStringByCommaToArray(request.param("nodeId")); 68 | 69 | AsynchronousSearchStatsRequest asynchronousSearchStatsRequest = new AsynchronousSearchStatsRequest(nodesIds); 70 | asynchronousSearchStatsRequest.timeout(request.param("timeout")); 71 | return asynchronousSearchStatsRequest; 72 | } 73 | } 74 | 75 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/opendistroforelasticsearch/search/asynchronous/rest/RestDeleteAsynchronousSearchAction.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.opendistroforelasticsearch.search.asynchronous.rest; 17 | 18 | import com.amazon.opendistroforelasticsearch.search.asynchronous.action.DeleteAsynchronousSearchAction; 19 | import com.amazon.opendistroforelasticsearch.search.asynchronous.plugin.AsynchronousSearchPlugin; 20 | import com.amazon.opendistroforelasticsearch.search.asynchronous.request.DeleteAsynchronousSearchRequest; 21 | import org.elasticsearch.client.node.NodeClient; 22 | import org.elasticsearch.rest.BaseRestHandler; 23 | import org.elasticsearch.rest.RestRequest; 24 | import org.elasticsearch.rest.action.RestStatusToXContentListener; 25 | 26 | import java.util.Collections; 27 | import java.util.List; 28 | 29 | import static org.elasticsearch.rest.RestRequest.Method.DELETE; 30 | 31 | public class RestDeleteAsynchronousSearchAction extends BaseRestHandler { 32 | 33 | @Override 34 | public String getName() { 35 | return "delete_asynchronous_search"; 36 | } 37 | 38 | @Override 39 | public List routes() { 40 | return Collections.singletonList(new Route(DELETE, AsynchronousSearchPlugin.BASE_URI + "/{id}")); 41 | } 42 | 43 | @Override 44 | protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) { 45 | DeleteAsynchronousSearchRequest deleteRequest = new DeleteAsynchronousSearchRequest(request.param("id")); 46 | return channel -> { 47 | client.execute(DeleteAsynchronousSearchAction.INSTANCE, deleteRequest, new RestStatusToXContentListener<>(channel)); 48 | }; 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/opendistroforelasticsearch/search/asynchronous/rest/RestGetAsynchronousSearchAction.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.opendistroforelasticsearch.search.asynchronous.rest; 17 | 18 | import com.amazon.opendistroforelasticsearch.search.asynchronous.action.GetAsynchronousSearchAction; 19 | import com.amazon.opendistroforelasticsearch.search.asynchronous.plugin.AsynchronousSearchPlugin; 20 | import com.amazon.opendistroforelasticsearch.search.asynchronous.request.GetAsynchronousSearchRequest; 21 | import org.elasticsearch.client.node.NodeClient; 22 | import org.elasticsearch.rest.BaseRestHandler; 23 | import org.elasticsearch.rest.RestRequest; 24 | import org.elasticsearch.rest.action.RestStatusToXContentListener; 25 | 26 | import java.io.IOException; 27 | import java.util.Arrays; 28 | import java.util.List; 29 | 30 | import static org.elasticsearch.rest.RestRequest.Method.GET; 31 | 32 | public class RestGetAsynchronousSearchAction extends BaseRestHandler { 33 | 34 | @Override 35 | public String getName() { 36 | return "get_asynchronous_search"; 37 | } 38 | 39 | @Override 40 | public List routes() { 41 | return Arrays.asList(new Route(GET, AsynchronousSearchPlugin.BASE_URI + "/{id}")); 42 | } 43 | 44 | @Override 45 | protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { 46 | GetAsynchronousSearchRequest getRequest = new GetAsynchronousSearchRequest(request.param("id")); 47 | if (request.hasParam("wait_for_completion_timeout")) { 48 | getRequest.setWaitForCompletionTimeout(request.paramAsTime("wait_for_completion_timeout", null)); 49 | } 50 | if (request.hasParam("keep_alive")) { 51 | getRequest.setKeepAlive(request.paramAsTime("keep_alive", null)); 52 | } 53 | return channel -> { 54 | client.execute(GetAsynchronousSearchAction.INSTANCE, getRequest, new RestStatusToXContentListener<>(channel)); 55 | }; 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/opendistroforelasticsearch/search/asynchronous/rest/RestSubmitAsynchronousSearchAction.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | package com.amazon.opendistroforelasticsearch.search.asynchronous.rest; 16 | 17 | import com.amazon.opendistroforelasticsearch.search.asynchronous.action.SubmitAsynchronousSearchAction; 18 | import com.amazon.opendistroforelasticsearch.search.asynchronous.plugin.AsynchronousSearchPlugin; 19 | import com.amazon.opendistroforelasticsearch.search.asynchronous.request.SubmitAsynchronousSearchRequest; 20 | import org.elasticsearch.action.search.SearchRequest; 21 | import org.elasticsearch.client.node.NodeClient; 22 | import org.elasticsearch.rest.BaseRestHandler; 23 | import org.elasticsearch.rest.RestRequest; 24 | import org.elasticsearch.rest.action.RestCancellableNodeClient; 25 | import org.elasticsearch.rest.action.RestStatusToXContentListener; 26 | import org.elasticsearch.rest.action.search.RestSearchAction; 27 | 28 | import java.io.IOException; 29 | import java.util.Arrays; 30 | import java.util.Collections; 31 | import java.util.HashSet; 32 | import java.util.List; 33 | import java.util.Set; 34 | import java.util.function.IntConsumer; 35 | 36 | import static org.elasticsearch.rest.RestRequest.Method.POST; 37 | 38 | public class RestSubmitAsynchronousSearchAction extends BaseRestHandler { 39 | /** 40 | * Indicates whether hits.total should be rendered as an integer or an object 41 | * in the rest search response. 42 | */ 43 | public static final String TOTAL_HITS_AS_INT_PARAM = "rest_total_hits_as_int"; 44 | public static final String TYPED_KEYS_PARAM = "typed_keys"; 45 | private static final Set RESPONSE_PARAMS; 46 | 47 | static { 48 | final Set responseParams = new HashSet<>(Arrays.asList(TYPED_KEYS_PARAM, TOTAL_HITS_AS_INT_PARAM)); 49 | RESPONSE_PARAMS = Collections.unmodifiableSet(responseParams); 50 | } 51 | 52 | @Override 53 | public String getName() { 54 | return "submit_asynchronous_search_action"; 55 | } 56 | 57 | @Override 58 | public List routes() { 59 | return Arrays.asList( 60 | new Route(POST, AsynchronousSearchPlugin.BASE_URI), 61 | new Route(POST, "/{index}" + AsynchronousSearchPlugin.BASE_URI)); 62 | } 63 | 64 | @Override 65 | public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { 66 | SearchRequest searchRequest = new SearchRequest(); 67 | 68 | IntConsumer setSize = size -> searchRequest.source().size(size); 69 | request.withContentOrSourceParamParserOrNull(parser -> 70 | RestSearchAction.parseSearchRequest(searchRequest, request, parser, client.getNamedWriteableRegistry(), setSize)); 71 | SubmitAsynchronousSearchRequest submitAsynchronousSearchRequest = new SubmitAsynchronousSearchRequest(searchRequest); 72 | 73 | submitAsynchronousSearchRequest.waitForCompletionTimeout(request.paramAsTime("wait_for_completion_timeout", 74 | SubmitAsynchronousSearchRequest.DEFAULT_WAIT_FOR_COMPLETION_TIMEOUT)); 75 | 76 | submitAsynchronousSearchRequest.keepAlive(request.paramAsTime("keep_alive", SubmitAsynchronousSearchRequest.DEFAULT_KEEP_ALIVE)); 77 | 78 | submitAsynchronousSearchRequest.keepOnCompletion(request.paramAsBoolean("keep_on_completion", 79 | SubmitAsynchronousSearchRequest.DEFAULT_KEEP_ON_COMPLETION)); 80 | 81 | searchRequest.requestCache(request.paramAsBoolean("request_cache", SubmitAsynchronousSearchRequest.DEFAULT_REQUEST_CACHE)); 82 | 83 | searchRequest.setBatchedReduceSize(request.paramAsInt("batched_reduce_size", 84 | SubmitAsynchronousSearchRequest.DEFAULT_BATCHED_REDUCE_SIZE)); 85 | 86 | return channel -> { 87 | RestCancellableNodeClient cancellableClient = new RestCancellableNodeClient(client, request.getHttpChannel()); 88 | cancellableClient.execute(SubmitAsynchronousSearchAction.INSTANCE, submitAsynchronousSearchRequest, 89 | new RestStatusToXContentListener<>(channel)); 90 | }; 91 | } 92 | 93 | @Override 94 | protected Set responseParams() { 95 | return RESPONSE_PARAMS; 96 | } 97 | 98 | @Override 99 | public boolean allowsUnsafeBuffers() { 100 | return true; 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/opendistroforelasticsearch/search/asynchronous/stats/AsynchronousSearchCountStats.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.opendistroforelasticsearch.search.asynchronous.stats; 17 | 18 | import org.elasticsearch.common.io.stream.StreamInput; 19 | import org.elasticsearch.common.io.stream.StreamOutput; 20 | import org.elasticsearch.common.io.stream.Writeable; 21 | import org.elasticsearch.common.xcontent.ToXContentFragment; 22 | import org.elasticsearch.common.xcontent.XContentBuilder; 23 | 24 | import java.io.IOException; 25 | 26 | /** 27 | * Accumulates Async Search State-wise Counters stats on a single node 28 | */ 29 | public class AsynchronousSearchCountStats implements Writeable, ToXContentFragment { 30 | 31 | private final long runningCount; 32 | private final long persistedCount; 33 | private final long persistFailedCount; 34 | private final long completedCount; 35 | private final long failedCount; 36 | private final long throttledCount; 37 | private final long initializedCount; 38 | private final long cancelledCount; 39 | private final long submittedCount; 40 | 41 | public AsynchronousSearchCountStats(long runningCount, long persistedCount, long completedCount, long failedCount, long throttledCount, 42 | long persistFailedCount, long initializedCount, long submittedCount, long cancelledCount) { 43 | this.runningCount = runningCount; 44 | this.persistedCount = persistedCount; 45 | this.persistFailedCount = persistFailedCount; 46 | this.completedCount = completedCount; 47 | this.failedCount = failedCount; 48 | this.throttledCount = throttledCount; 49 | this.initializedCount = initializedCount; 50 | this.cancelledCount = cancelledCount; 51 | this.submittedCount = submittedCount; 52 | } 53 | 54 | public AsynchronousSearchCountStats(StreamInput in) throws IOException { 55 | this.runningCount = in.readVLong(); 56 | this.persistedCount = in.readVLong(); 57 | this.completedCount = in.readVLong(); 58 | this.failedCount = in.readVLong(); 59 | this.throttledCount = in.readVLong(); 60 | this.persistFailedCount = in.readVLong(); 61 | this.initializedCount = in.readVLong(); 62 | this.cancelledCount = in.readVLong(); 63 | this.submittedCount = in.readVLong(); 64 | } 65 | 66 | @Override 67 | public void writeTo(StreamOutput out) throws IOException { 68 | out.writeVLong(this.runningCount); 69 | out.writeVLong(this.persistedCount); 70 | out.writeVLong(this.completedCount); 71 | out.writeVLong(this.failedCount); 72 | out.writeVLong(this.throttledCount); 73 | out.writeVLong(this.persistFailedCount); 74 | out.writeVLong(this.initializedCount); 75 | out.writeVLong(this.cancelledCount); 76 | out.writeVLong(this.submittedCount); 77 | } 78 | 79 | @Override 80 | public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { 81 | builder.startObject(Fields.ASYNC_SEARCH_STATS); 82 | builder.field(Fields.SUBMITTED, submittedCount); 83 | builder.field(Fields.INITIALIZED, initializedCount); 84 | builder.field(Fields.RUNNING, runningCount); 85 | builder.field(Fields.PERSISTED, persistedCount); 86 | builder.field(Fields.FAILED, failedCount); 87 | builder.field(Fields.COMPLETED, completedCount); 88 | builder.field(Fields.REJECTED, throttledCount); 89 | builder.field(Fields.PERSIST_FAILED, persistFailedCount); 90 | builder.field(Fields.CANCELLED, cancelledCount); 91 | builder.endObject(); 92 | return builder; 93 | } 94 | 95 | static final class Fields { 96 | private static final String ASYNC_SEARCH_STATS = "asynchronous_search_stats"; 97 | private static final String RUNNING = "running_current"; 98 | private static final String PERSISTED = "persisted"; 99 | private static final String PERSIST_FAILED = "persist_failed"; 100 | private static final String FAILED = "search_failed"; 101 | private static final String COMPLETED = "search_completed"; 102 | private static final String REJECTED = "rejected"; 103 | private static final String SUBMITTED = "submitted"; 104 | private static final String INITIALIZED = "initialized"; 105 | private static final String CANCELLED = "cancelled"; 106 | } 107 | 108 | public long getRunningCount() { 109 | return runningCount; 110 | } 111 | 112 | public long getPersistedCount() { 113 | return persistedCount; 114 | } 115 | 116 | public long getCompletedCount() { 117 | return completedCount; 118 | } 119 | 120 | public long getFailedCount() { 121 | return failedCount; 122 | } 123 | 124 | public long getThrottledCount() { 125 | return throttledCount; 126 | } 127 | 128 | public long getInitializedCount() { 129 | return initializedCount; 130 | } 131 | 132 | public long getCancelledCount() { 133 | return cancelledCount; 134 | } 135 | 136 | public long getSubmittedCount() { 137 | return submittedCount; 138 | } 139 | } 140 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/opendistroforelasticsearch/search/asynchronous/stats/AsynchronousSearchStats.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.opendistroforelasticsearch.search.asynchronous.stats; 17 | 18 | import org.elasticsearch.action.support.nodes.BaseNodeResponse; 19 | import org.elasticsearch.cluster.node.DiscoveryNode; 20 | import org.elasticsearch.common.Nullable; 21 | import org.elasticsearch.common.io.stream.StreamInput; 22 | import org.elasticsearch.common.io.stream.StreamOutput; 23 | import org.elasticsearch.common.xcontent.ToXContentFragment; 24 | import org.elasticsearch.common.xcontent.XContentBuilder; 25 | 26 | import java.io.IOException; 27 | 28 | /** 29 | * Class represents all stats the plugin keeps track of on a single node 30 | */ 31 | public class AsynchronousSearchStats extends BaseNodeResponse implements ToXContentFragment { 32 | 33 | private AsynchronousSearchCountStats asynchronousSearchCountStats; 34 | 35 | public AsynchronousSearchCountStats getAsynchronousSearchCountStats() { 36 | return asynchronousSearchCountStats; 37 | } 38 | 39 | public AsynchronousSearchStats(StreamInput in) throws IOException { 40 | super(in); 41 | asynchronousSearchCountStats = in.readOptionalWriteable(in1 -> new AsynchronousSearchCountStats(in1)); 42 | } 43 | 44 | public AsynchronousSearchStats(DiscoveryNode node, @Nullable AsynchronousSearchCountStats asynchronousSearchCountStats) { 45 | super(node); 46 | this.asynchronousSearchCountStats = asynchronousSearchCountStats; 47 | } 48 | 49 | @Override 50 | public void writeTo(StreamOutput out) throws IOException { 51 | super.writeTo(out); 52 | out.writeOptionalWriteable(asynchronousSearchCountStats); 53 | } 54 | 55 | @Override 56 | public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { 57 | if (asynchronousSearchCountStats != null) { 58 | asynchronousSearchCountStats.toXContent(builder, params); 59 | } 60 | return builder; 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/opendistroforelasticsearch/search/asynchronous/stats/InternalAsynchronousSearchStats.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.opendistroforelasticsearch.search.asynchronous.stats; 17 | 18 | import com.amazon.opendistroforelasticsearch.search.asynchronous.context.AsynchronousSearchContextId; 19 | import com.amazon.opendistroforelasticsearch.search.asynchronous.listener.AsynchronousSearchContextEventListener; 20 | import org.elasticsearch.cluster.node.DiscoveryNode; 21 | import org.elasticsearch.common.metrics.CounterMetric; 22 | 23 | public class InternalAsynchronousSearchStats implements AsynchronousSearchContextEventListener { 24 | 25 | private final CountStatsHolder countStatsHolder = new CountStatsHolder(); 26 | 27 | @Override 28 | public void onContextFailed(AsynchronousSearchContextId contextId) { 29 | countStatsHolder.failedAsynchronousSearchCount.inc(); 30 | countStatsHolder.runningAsynchronousSearchCount.dec(); 31 | } 32 | 33 | @Override 34 | public void onContextPersisted(AsynchronousSearchContextId asynchronousSearchContextId) { 35 | countStatsHolder.persistedAsynchronousSearchCount.inc(); 36 | } 37 | 38 | @Override 39 | public void onContextPersistFailed(AsynchronousSearchContextId contextId) { 40 | countStatsHolder.persistFailedAsynchronousSearchCount.inc(); 41 | } 42 | 43 | @Override 44 | public void onContextRunning(AsynchronousSearchContextId context) { 45 | countStatsHolder.runningAsynchronousSearchCount.inc(); 46 | } 47 | 48 | @Override 49 | public void onContextRejected(AsynchronousSearchContextId contextId) { 50 | countStatsHolder.rejectedAsynchronousSearchCount.inc(); 51 | } 52 | 53 | 54 | @Override 55 | public void onNewContext(AsynchronousSearchContextId contextId) { 56 | countStatsHolder.submittedAsynchronousSearchCount.inc(); 57 | } 58 | 59 | @Override 60 | public void onContextCancelled(AsynchronousSearchContextId contextId) { 61 | countStatsHolder.cancelledAsynchronousSearchCount.inc(); 62 | } 63 | 64 | @Override 65 | public void onContextInitialized(AsynchronousSearchContextId contextId) { 66 | countStatsHolder.initializedAsynchronousSearchCount.inc(); 67 | } 68 | 69 | @Override 70 | public void onRunningContextDeleted(AsynchronousSearchContextId contextId) { 71 | countStatsHolder.runningAsynchronousSearchCount.dec(); 72 | } 73 | 74 | @Override 75 | public void onContextCompleted(AsynchronousSearchContextId context) { 76 | countStatsHolder.completedAsynchronousSearchCount.inc(); 77 | countStatsHolder.runningAsynchronousSearchCount.dec(); 78 | } 79 | 80 | public AsynchronousSearchStats stats(DiscoveryNode node) { 81 | return new AsynchronousSearchStats(node, countStatsHolder.countStats()); 82 | } 83 | 84 | static final class CountStatsHolder { 85 | final CounterMetric runningAsynchronousSearchCount = new CounterMetric(); 86 | final CounterMetric persistedAsynchronousSearchCount = new CounterMetric(); 87 | final CounterMetric persistFailedAsynchronousSearchCount = new CounterMetric(); 88 | final CounterMetric failedAsynchronousSearchCount = new CounterMetric(); 89 | final CounterMetric completedAsynchronousSearchCount = new CounterMetric(); 90 | final CounterMetric rejectedAsynchronousSearchCount = new CounterMetric(); 91 | final CounterMetric submittedAsynchronousSearchCount = new CounterMetric(); 92 | final CounterMetric cancelledAsynchronousSearchCount = new CounterMetric(); 93 | final CounterMetric initializedAsynchronousSearchCount = new CounterMetric(); 94 | 95 | 96 | public AsynchronousSearchCountStats countStats() { 97 | return new AsynchronousSearchCountStats(runningAsynchronousSearchCount.count(), persistedAsynchronousSearchCount.count(), 98 | completedAsynchronousSearchCount.count(), failedAsynchronousSearchCount.count(), 99 | rejectedAsynchronousSearchCount.count(), persistFailedAsynchronousSearchCount.count(), 100 | initializedAsynchronousSearchCount.count(), submittedAsynchronousSearchCount.count(), 101 | cancelledAsynchronousSearchCount.count()); 102 | } 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/opendistroforelasticsearch/search/asynchronous/task/AsynchronousSearchTask.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.opendistroforelasticsearch.search.asynchronous.task; 17 | 18 | import com.amazon.opendistroforelasticsearch.search.asynchronous.context.active.AsynchronousSearchActiveContext; 19 | import com.amazon.opendistroforelasticsearch.search.asynchronous.request.SubmitAsynchronousSearchRequest; 20 | import org.apache.logging.log4j.LogManager; 21 | import org.apache.logging.log4j.Logger; 22 | import org.elasticsearch.action.search.SearchRequest; 23 | import org.elasticsearch.action.search.SearchTask; 24 | import org.elasticsearch.common.Strings; 25 | import org.elasticsearch.tasks.TaskId; 26 | 27 | import java.util.Map; 28 | import java.util.Objects; 29 | import java.util.function.Consumer; 30 | 31 | /** 32 | * Task storing information about a currently running {@link SearchRequest}. 33 | */ 34 | public class AsynchronousSearchTask extends SearchTask { 35 | 36 | private static final Logger logger = LogManager.getLogger(AsynchronousSearchTask.class); 37 | 38 | private final Consumer freeActiveContextConsumer; 39 | private final AsynchronousSearchActiveContext asynchronousSearchActiveContext; 40 | private final SubmitAsynchronousSearchRequest request; 41 | 42 | public static final String NAME = "indices:data/read/opendistro/asynchronous_search"; 43 | 44 | public AsynchronousSearchTask(long id, String type, String action, TaskId parentTaskId, Map headers, 45 | AsynchronousSearchActiveContext asynchronousSearchContext, SubmitAsynchronousSearchRequest request, 46 | Consumer freeActiveContextConsumer) { 47 | super(id, type, action, () -> description(request), parentTaskId, headers); 48 | Objects.requireNonNull(asynchronousSearchContext); 49 | Objects.requireNonNull(freeActiveContextConsumer); 50 | this.freeActiveContextConsumer = freeActiveContextConsumer; 51 | this.asynchronousSearchActiveContext = asynchronousSearchContext; 52 | this.request = request; 53 | } 54 | 55 | @Override 56 | protected void onCancelled() { 57 | logger.debug("On Cancelled event received for asynchronous search context [{}] due to [{}]", 58 | asynchronousSearchActiveContext.getAsynchronousSearchId(), getReasonCancelled()); 59 | freeActiveContextConsumer.accept(asynchronousSearchActiveContext); 60 | } 61 | 62 | private static String description(SubmitAsynchronousSearchRequest request) { 63 | StringBuilder sb = new StringBuilder("[asynchronous search] :"); 64 | sb.append("indices["); 65 | Strings.arrayToDelimitedString(request.getSearchRequest().indices(), ",", sb); 66 | sb.append("], "); 67 | sb.append("types["); 68 | Strings.arrayToDelimitedString(request.getSearchRequest().types(), ",", sb); 69 | sb.append("], "); 70 | sb.append("search_type[").append(request.getSearchRequest().searchType()).append("], "); 71 | sb.append("keep_on_completion[").append(request.getKeepOnCompletion()).append("], "); 72 | sb.append("keep_alive[").append(request.getKeepAlive()).append("], "); 73 | if (request.getSearchRequest().source() != null) { 74 | sb.append("source[").append(request.getSearchRequest().source() 75 | .toString(SearchRequest.FORMAT_PARAMS)).append("]"); 76 | } else { 77 | sb.append("source[]"); 78 | } 79 | return sb.toString(); 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/opendistroforelasticsearch/search/asynchronous/task/SubmitAsynchronousSearchTask.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.opendistroforelasticsearch.search.asynchronous.task; 17 | 18 | import com.amazon.opendistroforelasticsearch.search.asynchronous.request.SubmitAsynchronousSearchRequest; 19 | import org.elasticsearch.tasks.CancellableTask; 20 | import org.elasticsearch.tasks.TaskId; 21 | 22 | import java.util.Map; 23 | 24 | /** 25 | * Task storing information about a currently running {@link SubmitAsynchronousSearchRequest}. 26 | */ 27 | public class SubmitAsynchronousSearchTask extends CancellableTask { 28 | 29 | public SubmitAsynchronousSearchTask( 30 | long id, String type, String action, String description, TaskId parentTaskId, Map headers) { 31 | super(id, type, action, description, parentTaskId, headers); 32 | } 33 | 34 | @Override 35 | public boolean shouldCancelChildrenOnCancellation() { 36 | return true; 37 | } 38 | 39 | } 40 | 41 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/opendistroforelasticsearch/search/asynchronous/transport/TransportAsynchronousSearchStatsAction.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.opendistroforelasticsearch.search.asynchronous.transport; 17 | 18 | import com.amazon.opendistroforelasticsearch.search.asynchronous.action.AsynchronousSearchStatsAction; 19 | import com.amazon.opendistroforelasticsearch.search.asynchronous.request.AsynchronousSearchStatsRequest; 20 | import com.amazon.opendistroforelasticsearch.search.asynchronous.response.AsynchronousSearchStatsResponse; 21 | import com.amazon.opendistroforelasticsearch.search.asynchronous.service.AsynchronousSearchService; 22 | import com.amazon.opendistroforelasticsearch.search.asynchronous.stats.AsynchronousSearchStats; 23 | import org.elasticsearch.action.FailedNodeException; 24 | import org.elasticsearch.action.support.ActionFilters; 25 | import org.elasticsearch.action.support.nodes.BaseNodeRequest; 26 | import org.elasticsearch.action.support.nodes.TransportNodesAction; 27 | import org.elasticsearch.cluster.service.ClusterService; 28 | import org.elasticsearch.common.inject.Inject; 29 | import org.elasticsearch.common.io.stream.StreamInput; 30 | import org.elasticsearch.common.io.stream.StreamOutput; 31 | import org.elasticsearch.threadpool.ThreadPool; 32 | import org.elasticsearch.transport.TransportService; 33 | 34 | import java.io.IOException; 35 | import java.util.List; 36 | 37 | public class TransportAsynchronousSearchStatsAction 38 | extends TransportNodesAction { 40 | 41 | private final AsynchronousSearchService asynchronousSearchService; 42 | 43 | @Inject 44 | public TransportAsynchronousSearchStatsAction(ThreadPool threadPool, ClusterService clusterService, TransportService transportService, 45 | ActionFilters actionFilters, AsynchronousSearchService asynchronousSearchService) { 46 | 47 | super(AsynchronousSearchStatsAction.NAME, threadPool, clusterService, transportService, actionFilters, 48 | AsynchronousSearchStatsRequest::new, AsynchronousSearchStatsNodeRequest::new, ThreadPool.Names.MANAGEMENT, 49 | AsynchronousSearchStats.class); 50 | this.asynchronousSearchService = asynchronousSearchService; 51 | 52 | } 53 | 54 | @Override 55 | protected AsynchronousSearchStatsResponse newResponse(AsynchronousSearchStatsRequest request, List responses, 56 | List failures) { 57 | return new AsynchronousSearchStatsResponse(clusterService.getClusterName(), responses, failures); 58 | } 59 | 60 | @Override 61 | protected AsynchronousSearchStatsNodeRequest newNodeRequest(AsynchronousSearchStatsRequest request) { 62 | return new AsynchronousSearchStatsNodeRequest(request); 63 | } 64 | 65 | @Override 66 | protected AsynchronousSearchStats newNodeResponse(StreamInput in) throws IOException { 67 | return new AsynchronousSearchStats(in); 68 | } 69 | 70 | @Override 71 | protected AsynchronousSearchStats nodeOperation(AsynchronousSearchStatsNodeRequest asynchronousSearchStatsNodeRequest) { 72 | return asynchronousSearchService.stats(); 73 | 74 | } 75 | 76 | /** 77 | * Request to fetch asynchronous search stats on a single node 78 | */ 79 | public static class AsynchronousSearchStatsNodeRequest extends BaseNodeRequest { 80 | 81 | AsynchronousSearchStatsRequest request; 82 | 83 | public AsynchronousSearchStatsNodeRequest(StreamInput in) throws IOException { 84 | super(in); 85 | request = new AsynchronousSearchStatsRequest(in); 86 | } 87 | 88 | AsynchronousSearchStatsNodeRequest(AsynchronousSearchStatsRequest request) { 89 | this.request = request; 90 | } 91 | 92 | @Override 93 | public void writeTo(StreamOutput out) throws IOException { 94 | super.writeTo(out); 95 | request.writeTo(out); 96 | } 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/opendistroforelasticsearch/search/asynchronous/transport/TransportDeleteAsynchronousSearchAction.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.opendistroforelasticsearch.search.asynchronous.transport; 17 | 18 | import com.amazon.opendistroforelasticsearch.commons.authuser.User; 19 | import com.amazon.opendistroforelasticsearch.search.asynchronous.action.DeleteAsynchronousSearchAction; 20 | import com.amazon.opendistroforelasticsearch.search.asynchronous.id.AsynchronousSearchId; 21 | import com.amazon.opendistroforelasticsearch.search.asynchronous.request.DeleteAsynchronousSearchRequest; 22 | import com.amazon.opendistroforelasticsearch.search.asynchronous.response.AcknowledgedResponse; 23 | import com.amazon.opendistroforelasticsearch.search.asynchronous.service.AsynchronousSearchService; 24 | import org.apache.logging.log4j.LogManager; 25 | import org.apache.logging.log4j.Logger; 26 | import org.apache.logging.log4j.message.ParameterizedMessage; 27 | import org.elasticsearch.action.ActionListener; 28 | import org.elasticsearch.action.support.ActionFilters; 29 | import org.elasticsearch.client.Client; 30 | import org.elasticsearch.cluster.service.ClusterService; 31 | import org.elasticsearch.common.inject.Inject; 32 | import org.elasticsearch.threadpool.ThreadPool; 33 | import org.elasticsearch.transport.TransportService; 34 | 35 | /** 36 | * Transport action for deleting an asynchronous search request 37 | */ 38 | public class TransportDeleteAsynchronousSearchAction extends TransportAsynchronousSearchRoutingAction { 40 | 41 | private static final Logger logger = LogManager.getLogger(TransportAsynchronousSearchRoutingAction.class); 42 | 43 | private final AsynchronousSearchService asynchronousSearchService; 44 | 45 | @Inject 46 | public TransportDeleteAsynchronousSearchAction(ThreadPool threadPool, TransportService transportService, ClusterService clusterService, 47 | ActionFilters actionFilters, AsynchronousSearchService asynchronousSearchService, 48 | Client client) { 49 | super(transportService, clusterService, threadPool, client, DeleteAsynchronousSearchAction.NAME, actionFilters, 50 | asynchronousSearchService, DeleteAsynchronousSearchRequest::new, AcknowledgedResponse::new); 51 | this.asynchronousSearchService = asynchronousSearchService; 52 | } 53 | 54 | @Override 55 | public void handleRequest(AsynchronousSearchId asynchronousSearchId, DeleteAsynchronousSearchRequest request, 56 | ActionListener listener, User user) { 57 | try { 58 | asynchronousSearchService.freeContext(request.getId(), asynchronousSearchId.getAsynchronousSearchContextId(), user, 59 | ActionListener.wrap((complete) -> listener.onResponse(new AcknowledgedResponse(complete)), listener::onFailure)); 60 | } catch (Exception e) { 61 | logger.error(() -> new ParameterizedMessage("Unable to delete asynchronous search [{}]", request.getId()), e); 62 | listener.onFailure(e); 63 | } 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/opendistroforelasticsearch/search/asynchronous/utils/AsynchronousSearchExceptionUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.opendistroforelasticsearch.search.asynchronous.utils; 17 | 18 | import org.elasticsearch.ResourceNotFoundException; 19 | 20 | import java.util.Locale; 21 | 22 | public class AsynchronousSearchExceptionUtils { 23 | 24 | public static ResourceNotFoundException buildResourceNotFoundException(String id) { 25 | return new ResourceNotFoundException(String.format( 26 | Locale.ROOT, "Either the resource [%s] does not exist or you do not have access", id)); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/opendistroforelasticsearch/search/asynchronous/utils/UserAuthUtils.java: -------------------------------------------------------------------------------- 1 | package com.amazon.opendistroforelasticsearch.search.asynchronous.utils; 2 | 3 | import com.amazon.opendistroforelasticsearch.commons.authuser.User; 4 | import org.elasticsearch.common.Nullable; 5 | 6 | import java.io.IOException; 7 | import java.util.ArrayList; 8 | import java.util.List; 9 | import java.util.Map; 10 | 11 | public class UserAuthUtils { 12 | 13 | @SuppressWarnings("unchecked") 14 | public static User parseUser(Map userDetails) throws IOException { 15 | if(userDetails == null) { 16 | return null; 17 | } 18 | String name = ""; 19 | List backendRoles = new ArrayList<>(); 20 | List roles = new ArrayList<>(); 21 | List customAttNames = new ArrayList<>(); 22 | for (Map.Entry userDetail : userDetails.entrySet()) 23 | { 24 | String fieldName = userDetail.getKey(); 25 | switch (fieldName) { 26 | case User.NAME_FIELD: 27 | name = (String) userDetail.getValue(); 28 | break; 29 | case User.BACKEND_ROLES_FIELD: 30 | if(userDetail.getValue()!= null) 31 | backendRoles = (List) userDetail.getValue(); 32 | break; 33 | case User.ROLES_FIELD: 34 | if(userDetail.getValue()!= null) 35 | roles = (List) userDetail.getValue(); 36 | break; 37 | case User.CUSTOM_ATTRIBUTE_NAMES_FIELD: 38 | if(userDetail.getValue()!= null) 39 | customAttNames = (List) userDetail.getValue(); 40 | break; 41 | default: 42 | break; 43 | } 44 | } 45 | return new User(name, backendRoles, roles, customAttNames); 46 | } 47 | 48 | public static boolean isUserValid(@Nullable User currentUser, @Nullable User originalUser) { 49 | if(originalUser == null || currentUser == null) { 50 | return true; 51 | } 52 | if(currentUser.getBackendRoles() == null) { 53 | return originalUser.getBackendRoles() == null; 54 | } 55 | return currentUser.getBackendRoles().containsAll(originalUser.getBackendRoles()); 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/main/plugin-metadata/plugin-security.policy: -------------------------------------------------------------------------------- 1 | grant { 2 | // needed to find the classloader to load whitelisted classes. 3 | permission java.lang.RuntimePermission "createClassLoader"; 4 | permission java.lang.RuntimePermission "getClassLoader"; 5 | 6 | permission java.net.SocketPermission "*", "connect,resolve"; 7 | permission java.net.NetPermission "getProxySelector"; 8 | }; -------------------------------------------------------------------------------- /src/test/java/com/amazon/opendistroforelasticsearch/search/asynchronous/commons/AsynchronousSearchTestCase.java: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"). 6 | * You may not use this file except in compliance with the License. 7 | * A copy of the License is located at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * or in the "license" file accompanying this file. This file is distributed 12 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 13 | * express or implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | 17 | package com.amazon.opendistroforelasticsearch.search.asynchronous.commons; 18 | 19 | import com.amazon.opendistroforelasticsearch.search.asynchronous.listener.AsynchronousSearchProgressListener; 20 | import com.amazon.opendistroforelasticsearch.search.asynchronous.response.AsynchronousSearchResponse; 21 | import org.elasticsearch.action.search.SearchResponse; 22 | import org.elasticsearch.common.util.BigArrays; 23 | import org.elasticsearch.search.aggregations.InternalAggregation; 24 | import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; 25 | import org.elasticsearch.test.ESTestCase; 26 | import org.elasticsearch.threadpool.ThreadPool; 27 | 28 | import java.util.function.Function; 29 | 30 | public abstract class AsynchronousSearchTestCase extends ESTestCase { 31 | 32 | public static AsynchronousSearchProgressListener mockAsynchronousSearchProgressListener(ThreadPool threadPool) { 33 | return new AsynchronousSearchProgressListener(threadPool.absoluteTimeInMillis(), r -> null, e -> null, threadPool.generic(), 34 | threadPool::relativeTimeInMillis, () -> getReduceContextBuilder()); 35 | } 36 | 37 | public static AsynchronousSearchProgressListener mockAsynchronousSearchProgressListener(ThreadPool threadPool, 38 | Function successFunction, 39 | Function failureFunction) { 40 | return new AsynchronousSearchProgressListener(threadPool.absoluteTimeInMillis(), successFunction, failureFunction, 41 | threadPool.generic(), 42 | threadPool::relativeTimeInMillis, () -> getReduceContextBuilder()); 43 | } 44 | 45 | private static InternalAggregation.ReduceContextBuilder getReduceContextBuilder() { 46 | return new InternalAggregation.ReduceContextBuilder() { 47 | @Override 48 | public InternalAggregation.ReduceContext forPartialReduction() { 49 | return InternalAggregation.ReduceContext.forPartialReduction(BigArrays.NON_RECYCLING_INSTANCE, null, 50 | null); 51 | } 52 | 53 | public InternalAggregation.ReduceContext forFinalReduction() { 54 | return InternalAggregation.ReduceContext.forFinalReduction( 55 | BigArrays.NON_RECYCLING_INSTANCE, null, b -> {}, PipelineAggregator.PipelineTree.EMPTY); 56 | }}; 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /src/test/java/com/amazon/opendistroforelasticsearch/search/asynchronous/context/permits/NoopAsynchronousSearchContextPermitsTests.java: -------------------------------------------------------------------------------- 1 | package com.amazon.opendistroforelasticsearch.search.asynchronous.context.permits; 2 | 3 | import com.amazon.opendistroforelasticsearch.search.asynchronous.context.AsynchronousSearchContextId; 4 | import com.amazon.opendistroforelasticsearch.search.asynchronous.context.active.AsynchronousSearchContextClosedException; 5 | import org.elasticsearch.action.ActionListener; 6 | import org.elasticsearch.action.LatchedActionListener; 7 | import org.elasticsearch.common.unit.TimeValue; 8 | import org.elasticsearch.test.ESTestCase; 9 | import org.junit.Assert; 10 | 11 | import java.util.UUID; 12 | import java.util.concurrent.CountDownLatch; 13 | 14 | public class NoopAsynchronousSearchContextPermitsTests extends ESTestCase { 15 | 16 | public void testAcquireAllPermits() { 17 | NoopAsynchronousSearchContextPermits permits = new NoopAsynchronousSearchContextPermits( 18 | new AsynchronousSearchContextId(UUID.randomUUID().toString(), randomNonNegativeLong())); 19 | expectThrows(IllegalStateException.class, 20 | () -> permits.asyncAcquireAllPermits(ActionListener.wrap(Assert::fail), TimeValue.ZERO, "reason")); 21 | } 22 | 23 | public void testAcquireSinglePermit() throws InterruptedException { 24 | NoopAsynchronousSearchContextPermits permits = new NoopAsynchronousSearchContextPermits( 25 | new AsynchronousSearchContextId(UUID.randomUUID().toString(), randomNonNegativeLong())); 26 | CountDownLatch countDownLatch = new CountDownLatch(1); 27 | permits.asyncAcquirePermit(new LatchedActionListener<>(ActionListener.wrap(r -> {}, 28 | e -> fail("expected permit acquisition to succeed")), countDownLatch), TimeValue.ZERO, "reason"); 29 | countDownLatch.await(); 30 | } 31 | 32 | public void testAcquireSinglePermitAfterClosure() throws InterruptedException { 33 | AsynchronousSearchContextId contextId = new AsynchronousSearchContextId(UUID.randomUUID().toString(), randomNonNegativeLong()); 34 | NoopAsynchronousSearchContextPermits permits = new NoopAsynchronousSearchContextPermits( 35 | contextId); 36 | permits.close(); 37 | CountDownLatch countDownLatch = new CountDownLatch(1); 38 | permits.asyncAcquirePermit(new LatchedActionListener<>(ActionListener.wrap( 39 | r -> fail("expected permit acquisition to fail due to permit closure"), 40 | e -> { 41 | assertTrue("expected context closed exception, got " + e.getClass(), 42 | e instanceof AsynchronousSearchContextClosedException); 43 | assertTrue(((AsynchronousSearchContextClosedException) e).getAsynchronousSearchContextId().equals(contextId)); 44 | }), countDownLatch), TimeValue.ZERO, "reason"); 45 | countDownLatch.await(); 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /src/test/java/com/amazon/opendistroforelasticsearch/search/asynchronous/id/AsynchronousSearchIdTests.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.opendistroforelasticsearch.search.asynchronous.id; 17 | 18 | import com.amazon.opendistroforelasticsearch.search.asynchronous.context.AsynchronousSearchContextId; 19 | import org.elasticsearch.common.UUIDs; 20 | import org.elasticsearch.test.ESTestCase; 21 | 22 | import java.util.UUID; 23 | 24 | public class AsynchronousSearchIdTests extends ESTestCase { 25 | 26 | public void testAsynchronousSearchIdParsing() { 27 | String node = "node" + randomIntBetween(1, 50); 28 | long taskId = randomNonNegativeLong(); 29 | AsynchronousSearchContextId asContextId = new AsynchronousSearchContextId( 30 | UUIDs.base64UUID(), randomNonNegativeLong()); 31 | 32 | AsynchronousSearchId original = new AsynchronousSearchId(node, taskId, asContextId); 33 | //generate identifier to access the submitted asynchronous search 34 | String id = AsynchronousSearchIdConverter.buildAsyncId(original); 35 | //parse the AsynchronousSearchId object which will contain information regarding node running the search, the associated task and 36 | //context id. 37 | AsynchronousSearchId parsed = AsynchronousSearchIdConverter.parseAsyncId(id); 38 | assertEquals(original, parsed); 39 | assertEquals(parsed.toString(), "[" + node + "][" + taskId + "][" + asContextId + "]"); 40 | } 41 | 42 | 43 | public void testAsynchronousSearchIdParsingFailure() { 44 | String id = UUID.randomUUID().toString(); 45 | expectThrows(IllegalArgumentException.class, () -> AsynchronousSearchIdConverter.parseAsyncId(id)); 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /src/test/java/com/amazon/opendistroforelasticsearch/search/asynchronous/request/SubmitAsynchronousSearchRequestTests.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.opendistroforelasticsearch.search.asynchronous.request; 17 | 18 | import com.amazon.opendistroforelasticsearch.search.asynchronous.action.SubmitAsynchronousSearchAction; 19 | import org.elasticsearch.action.search.SearchRequest; 20 | import org.elasticsearch.common.ValidationException; 21 | import org.elasticsearch.common.unit.TimeValue; 22 | import org.elasticsearch.search.builder.SearchSourceBuilder; 23 | import org.elasticsearch.search.suggest.SuggestBuilder; 24 | import org.elasticsearch.tasks.TaskId; 25 | import org.elasticsearch.test.ESTestCase; 26 | 27 | import java.util.HashMap; 28 | 29 | import static org.hamcrest.Matchers.containsString; 30 | 31 | public class SubmitAsynchronousSearchRequestTests extends ESTestCase { 32 | 33 | public void testValidRequest() { 34 | SearchSourceBuilder source = new SearchSourceBuilder(); 35 | SearchRequest searchRequest = new SearchRequest(new String[]{"test"}, source); 36 | searchRequest.setCcsMinimizeRoundtrips(false); 37 | SubmitAsynchronousSearchRequest request = new SubmitAsynchronousSearchRequest(searchRequest); 38 | ValidationException validationException = request.validate(); 39 | assertNull(validationException); 40 | assertEquals(request, new SubmitAsynchronousSearchRequest(searchRequest)); 41 | String description = request.createTask(randomNonNegativeLong(), "type", SubmitAsynchronousSearchAction.NAME, 42 | new TaskId("test", -1), new HashMap<>()).getDescription(); 43 | assertThat(description, containsString("indices[test]")); 44 | } 45 | 46 | public void testSuggestOnlyQueryFailsValidation() { 47 | SearchSourceBuilder source = new SearchSourceBuilder().suggest(new SuggestBuilder()); 48 | SearchRequest searchRequest = new SearchRequest(new String[]{"test"}, source); 49 | searchRequest.setCcsMinimizeRoundtrips(false); 50 | SubmitAsynchronousSearchRequest request = new SubmitAsynchronousSearchRequest(searchRequest); 51 | ValidationException validationException = request.validate(); 52 | assertNotNull(validationException); 53 | assertEquals(1, validationException.validationErrors().size()); 54 | assertEquals("suggest-only queries are not supported", validationException.validationErrors().get(0)); 55 | } 56 | 57 | public void testSearchScrollFailsValidation() { 58 | SearchSourceBuilder source = new SearchSourceBuilder(); 59 | SearchRequest searchRequest = new SearchRequest(new String[]{"test"}, source); 60 | searchRequest.setCcsMinimizeRoundtrips(false); 61 | searchRequest.scroll(randomTimeValue()); 62 | SubmitAsynchronousSearchRequest request = new SubmitAsynchronousSearchRequest(searchRequest); 63 | ValidationException validationException = request.validate(); 64 | assertNotNull(validationException); 65 | assertEquals(1, validationException.validationErrors().size()); 66 | assertEquals("scrolls are not supported", validationException.validationErrors().get(0)); 67 | } 68 | 69 | public void testCcsMinimizeRoundtripsFlagFailsValidation() { 70 | SearchSourceBuilder source = new SearchSourceBuilder(); 71 | SearchRequest searchRequest = new SearchRequest(new String[]{"test"}, source); 72 | SubmitAsynchronousSearchRequest request = new SubmitAsynchronousSearchRequest( 73 | searchRequest); 74 | searchRequest.setCcsMinimizeRoundtrips(true); 75 | ValidationException validationException = request.validate(); 76 | assertNotNull(validationException); 77 | assertEquals(1, validationException.validationErrors().size()); 78 | assertThat(validationException.validationErrors().get(0), containsString("[ccs_minimize_roundtrips] must be false")); 79 | } 80 | 81 | public void testInvalidKeepAliveFailsValidation() { 82 | SearchSourceBuilder source = new SearchSourceBuilder(); 83 | SearchRequest searchRequest = new SearchRequest(new String[]{"test"}, source); 84 | searchRequest.setCcsMinimizeRoundtrips(false); 85 | SubmitAsynchronousSearchRequest request = new SubmitAsynchronousSearchRequest(searchRequest); 86 | request.keepAlive(TimeValue.timeValueSeconds(59)); 87 | ValidationException validationException = request.validate(); 88 | assertEquals(1, validationException.validationErrors().size()); 89 | } 90 | 91 | public void testInvalidWaitForCompletionFailsValidation() { 92 | SearchSourceBuilder source = new SearchSourceBuilder(); 93 | SearchRequest searchRequest = new SearchRequest(new String[]{"test"}, source); 94 | searchRequest.setCcsMinimizeRoundtrips(false); 95 | SubmitAsynchronousSearchRequest request = new SubmitAsynchronousSearchRequest(searchRequest); 96 | request.waitForCompletionTimeout(TimeValue.timeValueMillis(-1)); 97 | ValidationException validationException = request.validate(); 98 | assertEquals(1, validationException.validationErrors().size()); 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /src/test/java/com/amazon/opendistroforelasticsearch/search/asynchronous/response/AsynchronousSearchResponseTests.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.opendistroforelasticsearch.search.asynchronous.response; 17 | 18 | import com.amazon.opendistroforelasticsearch.search.asynchronous.context.state.AsynchronousSearchState; 19 | import org.apache.lucene.search.TotalHits; 20 | import org.elasticsearch.action.search.SearchResponse; 21 | import org.elasticsearch.action.search.ShardSearchFailure; 22 | import org.elasticsearch.client.Requests; 23 | import org.elasticsearch.common.Randomness; 24 | import org.elasticsearch.common.bytes.BytesReference; 25 | import org.elasticsearch.common.io.stream.Writeable; 26 | import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; 27 | import org.elasticsearch.common.xcontent.NamedXContentRegistry; 28 | import org.elasticsearch.common.xcontent.XContentHelper; 29 | import org.elasticsearch.common.xcontent.XContentParser; 30 | import org.elasticsearch.common.xcontent.XContentType; 31 | import org.elasticsearch.search.SearchHit; 32 | import org.elasticsearch.search.SearchHits; 33 | import org.elasticsearch.search.aggregations.InternalAggregations; 34 | import org.elasticsearch.search.internal.InternalSearchResponse; 35 | import org.elasticsearch.search.profile.SearchProfileShardResults; 36 | import org.elasticsearch.search.suggest.Suggest; 37 | import org.elasticsearch.test.AbstractSerializingTestCase; 38 | 39 | import java.io.IOException; 40 | import java.util.Collections; 41 | import java.util.UUID; 42 | 43 | import static org.elasticsearch.common.xcontent.XContentHelper.toXContent; 44 | 45 | public class AsynchronousSearchResponseTests extends AbstractSerializingTestCase { 46 | 47 | @Override 48 | protected AsynchronousSearchResponse doParseInstance(XContentParser parser) throws IOException { 49 | return AsynchronousSearchResponse.fromXContent(parser); 50 | } 51 | 52 | @Override 53 | protected Writeable.Reader instanceReader() { 54 | return AsynchronousSearchResponse::new; 55 | } 56 | 57 | @Override 58 | protected AsynchronousSearchResponse createTestInstance() { 59 | return new AsynchronousSearchResponse(UUID.randomUUID().toString(), 60 | getRandomAsynchronousSearchState(), 61 | randomNonNegativeLong(), 62 | randomNonNegativeLong(), getMockSearchResponse(), null); 63 | 64 | } 65 | 66 | @Override 67 | protected AsynchronousSearchResponse mutateInstance(AsynchronousSearchResponse instance) { 68 | return new AsynchronousSearchResponse(randomBoolean() ? instance.getId() : UUID.randomUUID().toString(), 69 | getRandomAsynchronousSearchState(), 70 | randomBoolean() ? instance.getStartTimeMillis() : randomNonNegativeLong(), 71 | randomBoolean() ? instance.getExpirationTimeMillis() : randomNonNegativeLong(), 72 | getMockSearchResponse(), 73 | instance.getError()); 74 | } 75 | 76 | private AsynchronousSearchState getRandomAsynchronousSearchState() { 77 | return AsynchronousSearchState.values()[Randomness.get().nextInt(AsynchronousSearchState.values().length)]; 78 | } 79 | 80 | private SearchResponse getMockSearchResponse() { 81 | int totalShards = randomInt(100); 82 | int successfulShards = totalShards - randomInt(100); 83 | return new SearchResponse(new InternalSearchResponse( 84 | new SearchHits(new SearchHit[0], new TotalHits(0L, TotalHits.Relation.EQUAL_TO), 0.0f), 85 | InternalAggregations.from(Collections.emptyList()), 86 | new Suggest(Collections.emptyList()), 87 | new SearchProfileShardResults(Collections.emptyMap()), false, false, randomInt(5)), 88 | "", totalShards, successfulShards, 0, randomNonNegativeLong(), 89 | ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY); 90 | } 91 | 92 | /* 93 | * we cannot compare the cause, because it will be wrapped and serialized in an outer 94 | * ElasticSearchException best effort: try to check that the original 95 | * message appears somewhere in the rendered xContent. 96 | */ 97 | public void testXContentRoundTripForAsynchronousSearchResponseContainingError() throws IOException { 98 | AsynchronousSearchResponse asResponse = new AsynchronousSearchResponse(UUID.randomUUID().toString(), 99 | getRandomAsynchronousSearchState(), 100 | randomNonNegativeLong(), randomNonNegativeLong(), null, new RuntimeException("test")); 101 | 102 | BytesReference serializedResponse; 103 | XContentType xContentType = Requests.INDEX_CONTENT_TYPE; 104 | serializedResponse = toXContent(asResponse, xContentType, true); 105 | try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, 106 | LoggingDeprecationHandler.INSTANCE, serializedResponse, xContentType)) { 107 | AsynchronousSearchResponse asResponse1 = AsynchronousSearchResponse.fromXContent(parser); 108 | String originalMsg = asResponse1.getError().getCause().getMessage(); 109 | assertEquals(originalMsg, 110 | "Elasticsearch exception [type=runtime_exception, reason=test]"); 111 | } 112 | } 113 | } 114 | 115 | -------------------------------------------------------------------------------- /src/test/java/com/amazon/opendistroforelasticsearch/search/asynchronous/utils/AsynchronousSearchAssertions.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.opendistroforelasticsearch.search.asynchronous.utils; 17 | 18 | import org.elasticsearch.action.search.SearchResponse; 19 | 20 | import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; 21 | 22 | import static org.junit.Assert.assertEquals; 23 | 24 | public class AsynchronousSearchAssertions { 25 | 26 | public static void assertSearchResponses(SearchResponse expected, SearchResponse actual) { 27 | assertEquals(expected.getNumReducePhases(), actual.getNumReducePhases()); 28 | assertEquals(expected.getClusters(), actual.getClusters()); 29 | assertEquals(expected.getSkippedShards(), actual.getSkippedShards()); 30 | assertEquals(expected.getTotalShards(), actual.getTotalShards()); 31 | assertEquals(expected.getSuccessfulShards(), actual.getSuccessfulShards()); 32 | assertEquals(expected.getAggregations(), actual.getAggregations()); 33 | assertEquals(expected.getHits().getTotalHits(), actual.getHits().getTotalHits()); 34 | assertNoFailures(actual); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/test/java/com/amazon/opendistroforelasticsearch/search/asynchronous/utils/QuadConsumer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package com.amazon.opendistroforelasticsearch.search.asynchronous.utils; 17 | 18 | 19 | /** 20 | * Represents an operation that accepts four arguments and returns no result. 21 | * 22 | * @param the type of the first argument 23 | * @param the type of the second argument 24 | * @param the type of the third argument 25 | * @param the type of the fourth argument 26 | */ 27 | @FunctionalInterface 28 | public interface QuadConsumer { 29 | /** 30 | * Applies this function to the given arguments. 31 | * 32 | * @param s the first function argument 33 | * @param t the second function argument 34 | * @param u the third function argument 35 | * @param v the fourth function argument 36 | */ 37 | void apply(S s, T t, U u, V v); 38 | } 39 | -------------------------------------------------------------------------------- /src/test/java/com/amazon/opendistroforelasticsearch/search/asynchronous/utils/TestUtils.java: -------------------------------------------------------------------------------- 1 | package com.amazon.opendistroforelasticsearch.search.asynchronous.utils; 2 | 3 | import org.elasticsearch.action.search.SearchResponse; 4 | import org.elasticsearch.client.Requests; 5 | import org.elasticsearch.cluster.ClusterName; 6 | import org.elasticsearch.cluster.ClusterState; 7 | import org.elasticsearch.cluster.block.ClusterBlocks; 8 | import org.elasticsearch.cluster.node.DiscoveryNode; 9 | import org.elasticsearch.cluster.node.DiscoveryNodes; 10 | import org.elasticsearch.cluster.service.ClusterService; 11 | import org.elasticsearch.common.bytes.BytesReference; 12 | import org.elasticsearch.common.settings.ClusterSettings; 13 | import org.elasticsearch.common.settings.Settings; 14 | import org.elasticsearch.common.xcontent.XContentHelper; 15 | import org.elasticsearch.threadpool.ThreadPool; 16 | 17 | import java.io.IOException; 18 | import java.util.Map; 19 | 20 | import static java.util.Collections.emptyMap; 21 | import static org.elasticsearch.common.xcontent.XContentHelper.convertToMap; 22 | import static org.elasticsearch.test.ClusterServiceUtils.createClusterStatePublisher; 23 | import static org.elasticsearch.test.ClusterServiceUtils.createNoOpNodeConnectionsService; 24 | 25 | public class TestUtils { 26 | 27 | public static ClusterService createClusterService(Settings settings, ThreadPool threadPool, DiscoveryNode localNode, 28 | ClusterSettings clusterSettings) { 29 | ClusterService clusterService = new ClusterService(settings, clusterSettings, threadPool); 30 | clusterService.setNodeConnectionsService(createNoOpNodeConnectionsService()); 31 | ClusterState initialClusterState = ClusterState.builder(new ClusterName(TestUtils.class.getSimpleName())) 32 | .nodes(DiscoveryNodes.builder() 33 | .add(localNode) 34 | .localNodeId(localNode.getId()) 35 | .masterNodeId(localNode.getId())) 36 | .blocks(ClusterBlocks.EMPTY_CLUSTER_BLOCK).build(); 37 | clusterService.getClusterApplierService().setInitialState(initialClusterState); 38 | clusterService.getMasterService().setClusterStatePublisher( 39 | createClusterStatePublisher(clusterService.getClusterApplierService())); 40 | clusterService.getMasterService().setClusterStateSupplier(clusterService.getClusterApplierService()::state); 41 | clusterService.start(); 42 | return clusterService; 43 | } 44 | 45 | public static Map getResponseAsMap(SearchResponse searchResponse) throws IOException { 46 | if (searchResponse != null) { 47 | BytesReference response = XContentHelper.toXContent(searchResponse, Requests.INDEX_CONTENT_TYPE, true); 48 | if (response == null) { 49 | return emptyMap(); 50 | } 51 | return convertToMap(response, false, Requests.INDEX_CONTENT_TYPE).v2(); 52 | } else { 53 | return null; 54 | } 55 | } 56 | } 57 | --------------------------------------------------------------------------------