├── CHANGELOG.md
├── CODE_OF_CONDUCT.md
├── CONTRIBUTING.md
├── LICENSE.txt
├── NOTICE.txt
├── README.md
├── SECURITY.md
├── deployment
├── build-s3-dist.sh
└── run-unit-tests.sh
└── source
├── .DS_Store
└── solutions-infrastructure
├── .gitignore
├── .npmignore
├── README.md
├── bin
└── app.ts
├── cdk.json
├── initBootstrap.sh
├── jest.config.js
├── lib
└── solutions-infrastructure-stack.ts
├── package.json
├── test
└── solutions-infrastructure-stack.test.ts
└── tsconfig.json
/CHANGELOG.md:
--------------------------------------------------------------------------------
1 | # Change Log
2 |
3 | All notable changes to this project will be documented in this file.
4 |
5 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
6 | and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
7 |
8 | ## [1.0.0] - 2023-11-17
9 |
10 | ### Added
11 |
12 | - All files, initial release
13 |
--------------------------------------------------------------------------------
/CODE_OF_CONDUCT.md:
--------------------------------------------------------------------------------
1 | ## Code of Conduct
2 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct).
3 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact
4 | opensource-codeofconduct@amazon.com with any additional questions or comments.
--------------------------------------------------------------------------------
/CONTRIBUTING.md:
--------------------------------------------------------------------------------
1 | # Contributing Guidelines
2 |
3 | Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional
4 | documentation, we greatly value feedback and contributions from our community.
5 |
6 | Please read through this document before submitting any issues or pull requests to ensure we have all the necessary
7 | information to effectively respond to your bug report or contribution.
8 |
9 |
10 | ## Reporting Bugs/Feature Requests
11 |
12 | We welcome you to use the GitHub issue tracker to report bugs or suggest features.
13 |
14 | When filing an issue, please check [existing open](https://github.com/opensearch-project/opensearch-migrations), or [recently closed](https://github.com/opensearch-project/opensearch-migrations/issues?utf8=%E2%9C%93&q=is%3Aissue%20is%3Aclosed%20), issues to make sure somebody else hasn't already
15 | reported the issue. Please try to include as much information as you can. Details like these are incredibly useful:
16 |
17 | * A reproducible test case or series of steps
18 | * The version of our code being used
19 | * Any modifications you've made relevant to the bug
20 | * Anything unusual about your environment or deployment
21 |
22 |
23 | ## Contributing via Pull Requests
24 | Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that:
25 |
26 | 1. You are working against the latest source on the *master* branch.
27 | 2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already.
28 | 3. You open an issue to discuss any significant work - we would hate for your time to be wasted.
29 |
30 | To send us a pull request, please:
31 |
32 | 1. Fork the repository.
33 | 2. Modify the source; please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change.
34 | 3. Ensure all build processes execute successfully (see README.md for additional guidance).
35 | 4. Ensure all unit, integration, and/or snapshot tests pass, as applicable.
36 | 5. Commit to your fork using clear commit messages.
37 | 6. Send us a pull request, answering any default questions in the pull request interface.
38 | 7. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation.
39 |
40 | GitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and
41 | [creating a pull request](https://help.github.com/articles/creating-a-pull-request/).
42 |
43 |
44 | ## Finding contributions to work on
45 | Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels ((enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any ['help wanted'](https://github.com/awslabs/opensearch-migrations/labels/help%20wanted) issues is a great place to start.
46 |
47 |
48 | ## Code of Conduct
49 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct).
50 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact
51 | opensource-codeofconduct@amazon.com with any additional questions or comments.
52 |
53 |
54 | ## Security issue notifications
55 | If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public GitHub issue.
56 |
57 |
58 | ## Licensing
59 |
60 | See the [LICENSE](https://github.com/aws-solutions/migration-assistant-for-amazon-opensearch/blob/main/LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution.
61 |
62 | We may ask you to sign a [Contributor License Agreement (CLA)](http://en.wikipedia.org/wiki/Contributor_License_Agreement) for larger changes.
63 |
--------------------------------------------------------------------------------
/LICENSE.txt:
--------------------------------------------------------------------------------
1 |
2 | Apache License
3 | Version 2.0, January 2004
4 | http://www.apache.org/licenses/
5 |
6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
7 |
8 | 1. Definitions.
9 |
10 | "License" shall mean the terms and conditions for use, reproduction,
11 | and distribution as defined by Sections 1 through 9 of this document.
12 |
13 | "Licensor" shall mean the copyright owner or entity authorized by
14 | the copyright owner that is granting the License.
15 |
16 | "Legal Entity" shall mean the union of the acting entity and all
17 | other entities that control, are controlled by, or are under common
18 | control with that entity. For the purposes of this definition,
19 | "control" means (i) the power, direct or indirect, to cause the
20 | direction or management of such entity, whether by contract or
21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the
22 | outstanding shares, or (iii) beneficial ownership of such entity.
23 |
24 | "You" (or "Your") shall mean an individual or Legal Entity
25 | exercising permissions granted by this License.
26 |
27 | "Source" form shall mean the preferred form for making modifications,
28 | including but not limited to software source code, documentation
29 | source, and configuration files.
30 |
31 | "Object" form shall mean any form resulting from mechanical
32 | transformation or translation of a Source form, including but
33 | not limited to compiled object code, generated documentation,
34 | and conversions to other media types.
35 |
36 | "Work" shall mean the work of authorship, whether in Source or
37 | Object form, made available under the License, as indicated by a
38 | copyright notice that is included in or attached to the work
39 | (an example is provided in the Appendix below).
40 |
41 | "Derivative Works" shall mean any work, whether in Source or Object
42 | form, that is based on (or derived from) the Work and for which the
43 | editorial revisions, annotations, elaborations, or other modifications
44 | represent, as a whole, an original work of authorship. For the purposes
45 | of this License, Derivative Works shall not include works that remain
46 | separable from, or merely link (or bind by name) to the interfaces of,
47 | the Work and Derivative Works thereof.
48 |
49 | "Contribution" shall mean any work of authorship, including
50 | the original version of the Work and any modifications or additions
51 | to that Work or Derivative Works thereof, that is intentionally
52 | submitted to Licensor for inclusion in the Work by the copyright owner
53 | or by an individual or Legal Entity authorized to submit on behalf of
54 | the copyright owner. For the purposes of this definition, "submitted"
55 | means any form of electronic, verbal, or written communication sent
56 | to the Licensor or its representatives, including but not limited to
57 | communication on electronic mailing lists, source code control systems,
58 | and issue tracking systems that are managed by, or on behalf of, the
59 | Licensor for the purpose of discussing and improving the Work, but
60 | excluding communication that is conspicuously marked or otherwise
61 | designated in writing by the copyright owner as "Not a Contribution."
62 |
63 | "Contributor" shall mean Licensor and any individual or Legal Entity
64 | on behalf of whom a Contribution has been received by Licensor and
65 | subsequently incorporated within the Work.
66 |
67 | 2. Grant of Copyright License. Subject to the terms and conditions of
68 | this License, each Contributor hereby grants to You a perpetual,
69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
70 | copyright license to reproduce, prepare Derivative Works of,
71 | publicly display, publicly perform, sublicense, and distribute the
72 | Work and such Derivative Works in Source or Object form.
73 |
74 | 3. Grant of Patent License. Subject to the terms and conditions of
75 | this License, each Contributor hereby grants to You a perpetual,
76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
77 | (except as stated in this section) patent license to make, have made,
78 | use, offer to sell, sell, import, and otherwise transfer the Work,
79 | where such license applies only to those patent claims licensable
80 | by such Contributor that are necessarily infringed by their
81 | Contribution(s) alone or by combination of their Contribution(s)
82 | with the Work to which such Contribution(s) was submitted. If You
83 | institute patent litigation against any entity (including a
84 | cross-claim or counterclaim in a lawsuit) alleging that the Work
85 | or a Contribution incorporated within the Work constitutes direct
86 | or contributory patent infringement, then any patent licenses
87 | granted to You under this License for that Work shall terminate
88 | as of the date such litigation is filed.
89 |
90 | 4. Redistribution. You may reproduce and distribute copies of the
91 | Work or Derivative Works thereof in any medium, with or without
92 | modifications, and in Source or Object form, provided that You
93 | meet the following conditions:
94 |
95 | (a) You must give any other recipients of the Work or
96 | Derivative Works a copy of this License; and
97 |
98 | (b) You must cause any modified files to carry prominent notices
99 | stating that You changed the files; and
100 |
101 | (c) You must retain, in the Source form of any Derivative Works
102 | that You distribute, all copyright, patent, trademark, and
103 | attribution notices from the Source form of the Work,
104 | excluding those notices that do not pertain to any part of
105 | the Derivative Works; and
106 |
107 | (d) If the Work includes a "NOTICE" text file as part of its
108 | distribution, then any Derivative Works that You distribute must
109 | include a readable copy of the attribution notices contained
110 | within such NOTICE file, excluding those notices that do not
111 | pertain to any part of the Derivative Works, in at least one
112 | of the following places: within a NOTICE text file distributed
113 | as part of the Derivative Works; within the Source form or
114 | documentation, if provided along with the Derivative Works; or,
115 | within a display generated by the Derivative Works, if and
116 | wherever such third-party notices normally appear. The contents
117 | of the NOTICE file are for informational purposes only and
118 | do not modify the License. You may add Your own attribution
119 | notices within Derivative Works that You distribute, alongside
120 | or as an addendum to the NOTICE text from the Work, provided
121 | that such additional attribution notices cannot be construed
122 | as modifying the License.
123 |
124 | You may add Your own copyright statement to Your modifications and
125 | may provide additional or different license terms and conditions
126 | for use, reproduction, or distribution of Your modifications, or
127 | for any such Derivative Works as a whole, provided Your use,
128 | reproduction, and distribution of the Work otherwise complies with
129 | the conditions stated in this License.
130 |
131 | 5. Submission of Contributions. Unless You explicitly state otherwise,
132 | any Contribution intentionally submitted for inclusion in the Work
133 | by You to the Licensor shall be under the terms and conditions of
134 | this License, without any additional terms or conditions.
135 | Notwithstanding the above, nothing herein shall supersede or modify
136 | the terms of any separate license agreement you may have executed
137 | with Licensor regarding such Contributions.
138 |
139 | 6. Trademarks. This License does not grant permission to use the trade
140 | names, trademarks, service marks, or product names of the Licensor,
141 | except as required for reasonable and customary use in describing the
142 | origin of the Work and reproducing the content of the NOTICE file.
143 |
144 | 7. Disclaimer of Warranty. Unless required by applicable law or
145 | agreed to in writing, Licensor provides the Work (and each
146 | Contributor provides its Contributions) on an "AS IS" BASIS,
147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
148 | implied, including, without limitation, any warranties or conditions
149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
150 | PARTICULAR PURPOSE. You are solely responsible for determining the
151 | appropriateness of using or redistributing the Work and assume any
152 | risks associated with Your exercise of permissions under this License.
153 |
154 | 8. Limitation of Liability. In no event and under no legal theory,
155 | whether in tort (including negligence), contract, or otherwise,
156 | unless required by applicable law (such as deliberate and grossly
157 | negligent acts) or agreed to in writing, shall any Contributor be
158 | liable to You for damages, including any direct, indirect, special,
159 | incidental, or consequential damages of any character arising as a
160 | result of this License or out of the use or inability to use the
161 | Work (including but not limited to damages for loss of goodwill,
162 | work stoppage, computer failure or malfunction, or any and all
163 | other commercial damages or losses), even if such Contributor
164 | has been advised of the possibility of such damages.
165 |
166 | 9. Accepting Warranty or Additional Liability. While redistributing
167 | the Work or Derivative Works thereof, You may choose to offer,
168 | and charge a fee for, acceptance of support, warranty, indemnity,
169 | or other liability obligations and/or rights consistent with this
170 | License. However, in accepting such obligations, You may act only
171 | on Your own behalf and on Your sole responsibility, not on behalf
172 | of any other Contributor, and only if You agree to indemnify,
173 | defend, and hold each Contributor harmless for any liability
174 | incurred by, or claims asserted against, such Contributor by reason
175 | of your accepting any such warranty or additional liability.
--------------------------------------------------------------------------------
/NOTICE.txt:
--------------------------------------------------------------------------------
1 | Migration Assistant for Amazon OpenSearch Service
2 |
3 | Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
4 | Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except
5 | in compliance with the License. A copy of the License is located at http://www.apache.org/licenses/
6 | or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS,
7 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied. See the License for the
8 | specific language governing permissions and limitations under the License.
9 |
10 | **********************
11 | THIRD PARTY COMPONENTS
12 | **********************
13 | This software includes third party software subject to the following copyrights:
14 |
15 | AWS SDK under the Apache License Version 2.0
16 | source-map-support under the MIT License
17 | attrs under the MIT License
18 | boolean.py under the BSD-2-Clause
19 | coverage under the Apache License Version 2.0
20 | iniconfig under the MIT License
21 | jsondiff under the MIT License
22 | license-expression under the Apache License Version 2.0
23 | pluggy under the MIT License
24 | prometheus-client under the Apache License Version 2.0
25 | pytest under the MIT License
26 | responses under the Apache License Version 2.0
27 | types-PyYAML under the Apache License Version 2.0
28 | @aws-cdk/aws-servicecatalogappregistry-alpha under the Apache License Version 2.0
29 | Jinja2 under the BSD License (BSD-3-Clause)
30 | MarkupSafe under the BSD License (BSD-3-Clause)
31 | Werkzeug under the BSD License
32 | awscli under the Apache License Version 2.0
33 | boto3 under the Apache License Version 2.0
34 | botocore under the Apache License Version 2.0
35 | cffi under the MIT License
36 | click under the BSD 3-Clause "New"
37 | colorama under the BSD License
38 | cryptography under the BSD License
39 | docutils under the BSD License
40 | jmespath under MIT License (MIT)
41 | jsonpath-ng under Apache Software License Apache 2.0
42 | moto under Apache Software License (Apache License 2.0)
43 | ply under the BSD License
44 | pur under the BSD License
45 | pyasn1 under the BSD License (BSD-2-Clause)
46 | pycparser under the BSD License (BSD)
47 | python-dateutil under the Apache Software License, BSD License (Dual License)
48 | requests-aws under the BSD License
49 | requests-aws4auth is under MIT License (MIT License)
50 | rsa under the Apache Software License (Apache-2.0)
51 | s3transfer under the Apache Software License (Apache License 2.0)
52 | xmltodict under the MIT License (MIT)
53 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Migration Assistant for Amazon OpenSearch
2 |
3 | The Migration Assistant for Amazon OpenSearch Service solution is a toolkit designed to ease the transition to OpenSearch, facilitate upgrades to the latest OpenSearch versions, and refine cluster configurations based on observed traffic patterns. Whether you're looking to set up a proof-of-concept in AWS, transition production workloads with confidence, or enhance your current OpenSearch clusters, this guide provides references to step-by-step instructions, best practices, and insights to leverage the full potential of the OpenSearch migrations package.
4 |
5 | ## Table of content
6 |
7 | - [Migration Assistant for Amazon OpenSearch](#migration-assistant-for-amazon-opensearch)
8 | - [Table of content](#table-of-content)
9 | - [Use Cases](#use-cases)
10 | - [Solution Overview](#solution-overview)
11 | - [Architecture](#architecture)
12 | - [Deployment](#deployment)
13 | - [OpenSearch Migrations Repo](#opensearch-migrations-repo)
14 | - [Collection of Operational Metrics](#collection-of-operational-metrics)
15 | - [License](#license)
16 |
17 |
18 | ## Solution Overview
19 |
20 | The solution has the following features:
21 |
22 | ### Historical backfill with capture and restore
23 | This solution guides users through the process of transferring data from an originating (source) cluster to a designated (target) cluster.
24 |
25 | ### Live traffic capture and replay
26 | The solution offers guidance and tools to intercept traffic intended for an original cluster and archive it for future replay on a destination cluster. Typically, the replay occurs at the same rate and concurrency as the original traffic to precisely mimic the workload experienced by the source cluster. Users can choose to replay the recorded traffic subsequently or adjust the replay speed. This flexibility enables users to fine-tune the target cluster, enhancing its performance to suit their requirements.
27 |
28 | ### Traffic verification
29 | The solution records requests and responses between the source and destination clusters for comparison. It then forwards the latency metrics and response codes to an analytics platform, enabling users to analyze the data essential for transitioning their traffic from a legacy system to a new Amazon OpenSearch destination.
30 |
31 | ## Use Cases
32 | ### Migrating historical data
33 | Migration Assistant for Amazon OpenSearch Service offers various options for migrating historical data, including detailed guidance on running a historical migration applicable across all supported migration routes, such as from Elasticsearch 7.10.2 to OpenSearch 1.0.
34 |
35 | ### Near real-time migration of HTTP traffic between clusters
36 | The solution offers you the option to capture data destined for a source cluster and store this data for reuse. A user can replay this data to a target cluster in near real-time to migrate as soon as possible, or replay at a later time.
37 |
38 | ### Replay traffic to multiple targets
39 | The solution allows you to capture traffic for replay through multiple instances or in sequential runs, facilitating the validation of diverse cluster workloads and configurations.
40 |
41 | ### Precise simulation of your cluster workloads
42 | The solution allows users to capture and replay traffic either simultaneously with multiple instances, or in separate sequential runs. This feature aids in validating different cluster workloads and configurations. By default, the Replayer preserves the original concurrency and request rate to accurately simulate production loads, ensuring a fair like-for-like comparison.
43 |
44 | ### Verify target cluster results
45 | The solution facilitates user comparisons of source and target traffic in terms of accuracy and performance. It captures metrics and logs for analysis, providing users with the necessary confidence to migrate their production traffic to a new target.
46 |
47 |
48 |
49 | ## Architecture
50 |
51 | Deploying this solution with the default parameters builds the following environment in the AWS Cloud.
52 |
53 |
54 |
55 |
56 | The high-level process flow for the solution components deployed with the AWS CloudFormation template is as follows:
57 |
58 | 1. Client traffic is directed to the existing cluster.
59 | 2. An ALB with Capture Proxies relaying traffic to source while replicating to Amazon MSK.
60 | 3. With continuous traffic capture in place, a Reindex-from-Snapshot (RFS) is initiated by the user via Migration Console.
61 | 4. Once Reindex-from-Snapshot is complete, traffic captured is replayed from MSK by Traffic Replayer.
62 | 5. Performance and behavior of traffic sent to source and target clusters are compared by reviewing logs and metrics.
63 | 6. After confirming the target cluster’s functionality meets expectations the use redirects clients to new target.
64 |
65 |
66 | ## Deployment
67 |
68 | Please follow the [Implementation Guide](https://docs.aws.amazon.com/solutions/latest/migration-assistant-for-amazon-opensearch/) to deploy the solution in your AWS account.
69 |
70 | ## OpenSearch Migrations Repo
71 | This solution is open-source. The tools developed in this solution can be found in the [OpenSearch Project Migration Repo](https://github.com/opensearch-project/opensearch-migrations).
72 |
73 | Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
74 |
75 | ## Collection of Operational Metrics
76 | This solution collects anonymous operational metrics to help AWS improve the quality and features of the solution. For more information, including how to disable this capability, please see the [implementation guide](https://docs.aws.amazon.com/solutions/latest/migration-assistant-for-amazon-opensearch-service/).
77 |
78 | ## License
79 | Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at
80 | [https://www.apache.org/licenses/](https://www.apache.org/licenses/)
81 | or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions and limitations under the License.
82 |
--------------------------------------------------------------------------------
/SECURITY.md:
--------------------------------------------------------------------------------
1 | ## Reporting Security Issues
2 |
3 | We take all security reports seriously. When we receive such reports, we will investigate and
4 | subsequently address any potential vulnerabilities as quickly as possible. If you discover a potential
5 | security issue in this project, please notify AWS/Amazon Security via our [vulnerability reporting page](https://aws.amazon.com/security/vulnerability-reporting/) or directly via email to [AWS Security](aws-security@amazon.com). Please do not create a public GitHub issue in this project.
6 |
7 |
--------------------------------------------------------------------------------
/deployment/build-s3-dist.sh:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env bash
2 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3 | # SPDX-License-Identifier: Apache-2.0
4 |
5 | # Check to see if input has been provided:
6 | if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then
7 | echo "Please provide the base source bucket name, trademark approved solution name and version where the lambda code will eventually reside."
8 | echo "For example: ./build-s3-dist.sh solutions trademarked-solution-name v1.0.0"
9 | exit 1
10 | fi
11 |
12 | set -e
13 |
14 | # Get reference for all important folders
15 | template_dir="$PWD"
16 | template_dist_dir="${template_dir}/global-s3-assets"
17 | build_dist_dir="${template_dir}/regional-s3-assets"
18 | source_dir="${template_dir}/../source"
19 |
20 | echo "------------------------------------------------------------------------------"
21 | echo "Rebuild distribution"
22 | echo "------------------------------------------------------------------------------"
23 | rm -rf "${template_dist_dir}"
24 | mkdir -p "${template_dist_dir}"
25 | rm -rf "${build_dist_dir}"
26 | mkdir -p "${build_dist_dir}"
27 |
28 | [ -e "${template_dist_dir}" ] && rm -r "${template_dist_dir}"
29 | [ -e "${build_dist_dir}" ] && rm -r "${build_dist_dir}"
30 | mkdir -p "${template_dist_dir}" "${build_dist_dir}"
31 | touch "$build_dist_dir"/test.txt
32 |
33 | echo "--------------------------------------------------------------------------------------"
34 | echo "CloudFormation Template generation"
35 | echo "--------------------------------------------------------------------------------------"
36 | export CODE_BUCKET=$1
37 | export SOLUTION_NAME=$2
38 | export CODE_VERSION=$3
39 |
40 | cd "$source_dir"/solutions-infrastructure
41 | npm install
42 | node_modules/aws-cdk/bin/cdk synth --asset-metadata false --path-metadata false > "$template_dist_dir"/"$SOLUTION_NAME".template
43 |
44 | if [ $? -eq 0 ]
45 | then
46 | echo "Build for $SOLUTION_NAME.template succeeded"
47 | else
48 | echo "******************************************************************************"
49 | echo "Build FAILED for $SOLUTION_NAME.template"
50 | echo "******************************************************************************"
51 | exit 1
52 | fi
--------------------------------------------------------------------------------
/deployment/run-unit-tests.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | # This script runs all tests for the root CDK project,
3 |
4 | [ "$DEBUG" == 'true' ] && set -x
5 | set -e
6 |
7 |
8 | prepare_jest_coverage_report() {
9 | local component_name=$1
10 |
11 | if [ ! -d "coverage" ]; then
12 | echo "ValidationError: Missing required directory coverage after running unit tests"
13 | exit 129
14 | fi
15 |
16 | # prepare coverage reports
17 | rm -fr coverage/lcov-report
18 | mkdir -p "$coverage_reports_top_path"/jest
19 | coverage_report_path=$coverage_reports_top_path/jest/$component_name
20 | rm -fr "$coverage_report_path"
21 | mv coverage "$coverage_report_path"
22 | }
23 |
24 | run_python_tests() {
25 | local component_path=$1
26 | local component_name=$2
27 |
28 | echo "------------------------------------------------------------------------------"
29 | echo "[Test] Run unit test with coverage for $component_name"
30 | echo "------------------------------------------------------------------------------"
31 | echo "cd $component_path"
32 | cd "$component_path"
33 |
34 | python3 -m venv .venv
35 | source .venv/bin/activate
36 | pip install -r requirements.txt
37 | pip install -r dev-requirements.txt
38 | python3 -m coverage run -m unittest
39 | python3 -m coverage xml --omit "*/tests/*"
40 | # The coverage module uses absolutes paths in its coverage output. To avoid dependencies of tools (such as SonarQube)
41 | # on different absolute paths for source directories, this substitution is used to convert each absolute source
42 | # directory path to the corresponding project relative path. The $source_dir holds the absolute path for source
43 | # directory.
44 | sed -i -e "s,$source_dir,source,g" coverage.xml
45 | deactivate
46 | rm -rf .venv
47 | }
48 |
49 |
50 | run_gradle_tests() {
51 | local component_path=$1
52 | local component_name=$2
53 |
54 | echo "------------------------------------------------------------------------------"
55 | echo "[Test] Run unit test with coverage for $component_name"
56 | echo "------------------------------------------------------------------------------"
57 | echo "cd $component_path"
58 | cd "$component_path"
59 |
60 | ./gradlew build copyDependencies jacocoTestReport
61 | }
62 |
63 | run_npm_tests() {
64 | local component_path=$1
65 | local component_name=$2
66 |
67 | echo "------------------------------------------------------------------------------"
68 | echo "[Test] Run unit test with coverage for $component_name"
69 | echo "------------------------------------------------------------------------------"
70 | echo "cd $component_path"
71 | cd "$component_path"
72 |
73 | # install dependencies
74 | npm install
75 |
76 | # run unit tests
77 | npm test
78 |
79 | # prepare coverage reports
80 | prepare_jest_coverage_report "$component_name"
81 | rm -rf coverage node_modules package-lock.json
82 | }
83 |
84 | check_test_failure() {
85 | local component_name=$1
86 |
87 | # Check the result of the test and exit if a failure is identified
88 | if [ $? -eq 0 ]
89 | then
90 | echo "Test for $component_name passed"
91 | else
92 | echo "******************************************************************************"
93 | echo "Test FAILED for $component_name"
94 | echo "******************************************************************************"
95 | exit 1
96 | fi
97 | }
98 |
99 | # Run unit tests
100 | echo "Running unit tests"
101 |
102 | # Get reference for source folder
103 | source_dir="$(cd $PWD/../source; pwd -P)"
104 | coverage_reports_top_path=$source_dir/test/coverage-reports
105 |
106 | run_gradle_tests "$source_dir/opensearch-migrations/TrafficCapture" "TrafficCapture"
107 | check_test_failure "TrafficCapture"
108 |
109 | run_python_tests "$source_dir/opensearch-migrations/FetchMigration/python" "FetchMigration"
110 | check_test_failure "FetchMigration"
111 |
112 | # Test packages from /source directory
113 | declare -a packages=(
114 | "solutions-infrastructure" "opensearch-migrations/deployment/cdk/opensearch-service-migration"
115 | )
116 | for package in "${packages[@]}"; do
117 | package_name=$(echo "$package" | sed 's/.*\///')
118 | run_npm_tests "$source_dir"/"$package" "$package_name"
119 | check_test_failure "$package_name"
120 | done
121 |
--------------------------------------------------------------------------------
/source/.DS_Store:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/aws-solutions/migration-assistant-for-amazon-opensearch/26979a765c98012629bfdb7e0005a3a8479f287e/source/.DS_Store
--------------------------------------------------------------------------------
/source/solutions-infrastructure/.gitignore:
--------------------------------------------------------------------------------
1 | *.js
2 | !jest.config.js
3 | *.d.ts
4 | node_modules
5 |
6 | # CDK asset staging directory
7 | .cdk.staging
8 | cdk.out
9 |
--------------------------------------------------------------------------------
/source/solutions-infrastructure/.npmignore:
--------------------------------------------------------------------------------
1 | *.ts
2 | !*.d.ts
3 |
4 | # CDK asset staging directory
5 | .cdk.staging
6 | cdk.out
7 |
--------------------------------------------------------------------------------
/source/solutions-infrastructure/README.md:
--------------------------------------------------------------------------------
1 | # Welcome to your CDK TypeScript project
2 |
3 | This is a blank project for CDK development with TypeScript.
4 |
5 | The `cdk.json` file tells the CDK Toolkit how to execute your app.
6 |
7 | ## Useful commands
8 |
9 | * `npm run build` compile typescript to js
10 | * `npm run watch` watch for changes and compile
11 | * `npm run test` perform the jest unit tests
12 | * `cdk deploy` deploy this stack to your default AWS account/region
13 | * `cdk diff` compare deployed stack with current state
14 | * `cdk synth` emits the synthesized CloudFormation template
15 |
--------------------------------------------------------------------------------
/source/solutions-infrastructure/bin/app.ts:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env node
2 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3 | // SPDX-License-Identifier: Apache-2.0
4 |
5 | import 'source-map-support/register';
6 | import { App, DefaultStackSynthesizer } from 'aws-cdk-lib';
7 | import { SolutionsInfrastructureStack, SolutionsInfrastructureStackProps } from '../lib/solutions-infrastructure-stack';
8 |
9 | const getProps = (): SolutionsInfrastructureStackProps => {
10 | const { CODE_BUCKET, SOLUTION_NAME, CODE_VERSION } = process.env;
11 | if (typeof CODE_BUCKET !== 'string' || CODE_BUCKET.trim() === '') {
12 | throw new Error('Missing required environment variable: CODE_BUCKET');
13 | }
14 |
15 | if (typeof SOLUTION_NAME !== 'string' || SOLUTION_NAME.trim() === '') {
16 | throw new Error('Missing required environment variable: SOLUTION_NAME');
17 | }
18 |
19 | if (typeof CODE_VERSION !== 'string' || CODE_VERSION.trim() === '') {
20 | throw new Error('Missing required environment variable: CODE_VERSION');
21 | }
22 |
23 | const codeBucket = CODE_BUCKET;
24 | const solutionVersion = CODE_VERSION;
25 | const solutionId = 'SO0290';
26 | const solutionName = SOLUTION_NAME;
27 | const description = `(${solutionId}) - The AWS CloudFormation template for deployment of the ${solutionName}. Version ${solutionVersion}`;
28 |
29 | // Uncomment for local testing
30 | // const codeBucket = 'unknown';
31 | // const solutionVersion = "1.0.0";
32 | // const solutionId = 'SO0290';
33 | // const solutionName = 'migration-assistant-for-amazon-opensearch';
34 | // const description = `(${solutionId}) - The AWS CloudFormation template for deployment of the ${solutionName}. Version ${solutionVersion}`;
35 |
36 | return {
37 | codeBucket,
38 | solutionVersion,
39 | solutionId,
40 | solutionName,
41 | description
42 | };
43 | };
44 |
45 | const app = new App();
46 | const infraProps = getProps()
47 | new SolutionsInfrastructureStack(app, 'OSMigrations-Bootstrap', {
48 | synthesizer: new DefaultStackSynthesizer({
49 | generateBootstrapVersionRule: false
50 | }),
51 | ...infraProps
52 | });
--------------------------------------------------------------------------------
/source/solutions-infrastructure/cdk.json:
--------------------------------------------------------------------------------
1 | {
2 | "app": "npx ts-node --prefer-ts-exts bin/app.ts",
3 | "watch": {
4 | "include": [
5 | "**"
6 | ],
7 | "exclude": [
8 | "README.md",
9 | "cdk*.json",
10 | "**/*.d.ts",
11 | "**/*.js",
12 | "tsconfig.json",
13 | "package*.json",
14 | "yarn.lock",
15 | "node_modules",
16 | "test"
17 | ]
18 | },
19 | "context": {
20 | "@aws-cdk/aws-apigateway:usagePlanKeyOrderInsensitiveId": true,
21 | "@aws-cdk/core:stackRelativeExports": true,
22 | "@aws-cdk/aws-rds:lowercaseDbIdentifier": true,
23 | "@aws-cdk/aws-lambda:recognizeVersionProps": true,
24 | "@aws-cdk/aws-lambda:recognizeLayerVersion": true,
25 | "@aws-cdk/aws-cloudfront:defaultSecurityPolicyTLSv1.2_2021": true,
26 | "@aws-cdk-containers/ecs-service-extensions:enableDefaultLogDriver": true,
27 | "@aws-cdk/aws-ec2:uniqueImdsv2TemplateName": true,
28 | "@aws-cdk/core:checkSecretUsage": true,
29 | "@aws-cdk/aws-iam:minimizePolicies": true,
30 | "@aws-cdk/aws-ecs:arnFormatIncludesClusterName": true,
31 | "@aws-cdk/core:validateSnapshotRemovalPolicy": true,
32 | "@aws-cdk/aws-codepipeline:crossAccountKeyAliasStackSafeResourceName": true,
33 | "@aws-cdk/aws-s3:createDefaultLoggingPolicy": true,
34 | "@aws-cdk/aws-sns-subscriptions:restrictSqsDescryption": true,
35 | "@aws-cdk/aws-apigateway:disableCloudWatchRole": true,
36 | "@aws-cdk/core:enablePartitionLiterals": true,
37 | "@aws-cdk/core:target-partitions": [
38 | "aws",
39 | "aws-cn"
40 | ]
41 | }
42 | }
43 |
--------------------------------------------------------------------------------
/source/solutions-infrastructure/initBootstrap.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | yum update && yum install -y git java-11-amazon-corretto-devel docker nodejs https://s3.amazonaws.com/session-manager-downloads/plugin/latest/linux_64bit/session-manager-plugin.rpm
4 | systemctl start docker
5 | git init
6 | release_tag=$(curl -s https://api.github.com/repos/opensearch-project/opensearch-migrations/releases/latest | jq -r ".tag_name")
7 | git remote add -f origin https://github.com/opensearch-project/opensearch-migrations.git
8 | git checkout tags/$release_tag
9 |
10 | cd deployment/cdk/opensearch-service-migration || exit
11 | npm install -g aws-cdk
12 | npm install
13 | ./buildDockerImages.sh
14 |
--------------------------------------------------------------------------------
/source/solutions-infrastructure/jest.config.js:
--------------------------------------------------------------------------------
1 | module.exports = {
2 | testEnvironment: 'node',
3 | roots: ['/test'],
4 | testMatch: ['**/*.test.ts'],
5 | transform: {
6 | '^.+\\.tsx?$': 'ts-jest'
7 | }
8 | };
9 |
--------------------------------------------------------------------------------
/source/solutions-infrastructure/lib/solutions-infrastructure-stack.ts:
--------------------------------------------------------------------------------
1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2 | // SPDX-License-Identifier: Apache-2.0
3 |
4 | import {Aws, CfnMapping, CfnParameter, Fn, Stack, StackProps, Tags} from 'aws-cdk-lib';
5 | import {Construct} from 'constructs';
6 | import {
7 | BlockDeviceVolume,
8 | CloudFormationInit,
9 | InitCommand,
10 | InitElement,
11 | InitFile,
12 | Instance,
13 | InstanceClass,
14 | InstanceSize,
15 | InstanceType,
16 | IpAddresses,
17 | MachineImage,
18 | SubnetType,
19 | Vpc
20 | } from "aws-cdk-lib/aws-ec2";
21 | import {InstanceProfile, ManagedPolicy, Role, ServicePrincipal} from "aws-cdk-lib/aws-iam";
22 | import {CfnDocument} from "aws-cdk-lib/aws-ssm";
23 | import {Application, AttributeGroup} from "@aws-cdk/aws-servicecatalogappregistry-alpha";
24 |
25 | export interface SolutionsInfrastructureStackProps extends StackProps {
26 | readonly solutionId: string;
27 | readonly solutionName: string;
28 | readonly solutionVersion: string;
29 | readonly codeBucket: string;
30 | }
31 |
32 | export function applyAppRegistry(stack: Stack, stage: string, infraProps: SolutionsInfrastructureStackProps): string {
33 | const application = new Application(stack, "AppRegistry", {
34 | applicationName: Fn.join("-", [
35 | infraProps.solutionName,
36 | Aws.REGION,
37 | Aws.ACCOUNT_ID,
38 | stage // If your solution supports multiple deployments in the same region, add stage to the application name to make it unique.
39 | ]),
40 | description: `Service Catalog application to track and manage all your resources for the solution ${infraProps.solutionName}`,
41 | });
42 | application.associateApplicationWithStack(stack);
43 | Tags.of(application).add("Solutions:SolutionID", infraProps.solutionId);
44 | Tags.of(application).add("Solutions:SolutionName", infraProps.solutionName);
45 | Tags.of(application).add("Solutions:SolutionVersion", infraProps.solutionVersion);
46 | Tags.of(application).add("Solutions:ApplicationType", "AWS-Solutions");
47 |
48 | const attributeGroup = new AttributeGroup(
49 | stack,
50 | "DefaultApplicationAttributes",
51 | {
52 | attributeGroupName: Fn.join("-", [
53 | Aws.REGION,
54 | stage,
55 | "attributes"
56 | ]),
57 | description: "Attribute group for solution information",
58 | attributes: {
59 | applicationType: "AWS-Solutions",
60 | version: infraProps.solutionVersion,
61 | solutionID: infraProps.solutionId,
62 | solutionName: infraProps.solutionName,
63 | },
64 | }
65 | );
66 | attributeGroup.associateWith(application)
67 | return application.applicationArn
68 | }
69 |
70 | export class SolutionsInfrastructureStack extends Stack {
71 |
72 | constructor (scope: Construct, id: string, props: SolutionsInfrastructureStackProps) {
73 | super(scope, id, props);
74 |
75 | // CFN template format version
76 | this.templateOptions.templateFormatVersion = '2010-09-09';
77 |
78 | // CFN Mappings
79 | new CfnMapping(this, 'Solution', {
80 | mapping: {
81 | Config: {
82 | CodeVersion: props.solutionVersion,
83 | KeyPrefix: `${props.solutionName}/${props.solutionVersion}`,
84 | S3Bucket: props.codeBucket,
85 | SendAnonymousUsage: 'No',
86 | SolutionId: props.solutionId
87 | }
88 | }
89 | });
90 |
91 | const stageParameter = new CfnParameter(this, 'Stage', {
92 | type: 'String',
93 | description: 'Specify the stage identifier which will be used in naming resources, e.g. dev,gamma,wave1',
94 | default: 'dev',
95 | noEcho: false
96 | });
97 |
98 | const appRegistryAppARN = applyAppRegistry(this, stageParameter.valueAsString, props)
99 |
100 | // Ideally we would have an option to import an existing VPC but unfortunately without being in control of the
101 | // imported vpc we can not get the needed values at synthesis time and VPC lookup() does not allow token values.
102 | // More details can be found here: https://github.com/aws/aws-cdk/issues/3600
103 | const vpc = new Vpc(this, 'BootstrapVPC', {
104 | // IP space should be customized for use cases that have specific IP range needs
105 | ipAddresses: IpAddresses.cidr('10.0.0.0/16'),
106 | maxAzs: 1,
107 | subnetConfiguration: [
108 | // Outbound internet access for private subnets require a NAT Gateway which must live in
109 | // a public subnet
110 | {
111 | name: 'public-subnet',
112 | subnetType: SubnetType.PUBLIC,
113 | cidrMask: 24,
114 | },
115 | {
116 | name: 'private-subnet',
117 | subnetType: SubnetType.PRIVATE_WITH_EGRESS,
118 | cidrMask: 24,
119 | },
120 | ],
121 | });
122 |
123 | new CfnDocument(this, "BootstrapShellDoc", {
124 | name: `SSM-${stageParameter.valueAsString}-BootstrapShell`,
125 | documentType: "Session",
126 | content: {
127 | "schemaVersion": "1.0",
128 | "description": "Document to hold regional settings for Session Manager",
129 | "sessionType": "Standard_Stream",
130 | "inputs": {
131 | "cloudWatchLogGroupName": "",
132 | "cloudWatchEncryptionEnabled": true,
133 | "cloudWatchStreamingEnabled": false,
134 | "kmsKeyId": "",
135 | "runAsEnabled": false,
136 | "runAsDefaultUser": "",
137 | "idleSessionTimeout": "60",
138 | "maxSessionDuration": "",
139 | "shellProfile": {
140 | "linux": "cd /opensearch-migrations && sudo -s"
141 | }
142 | }
143 | }
144 | })
145 |
146 | const bootstrapFile = InitFile.fromFileInline("/opensearch-migrations/initBootstrap.sh", './initBootstrap.sh', {
147 | mode: "000744"
148 | })
149 | const solutionsUserAgent = `AwsSolution/${props.solutionId}/${props.solutionVersion}`
150 | const cfnInitConfig : InitElement[] = [
151 | InitCommand.shellCommand(`echo "export MIGRATIONS_APP_REGISTRY_ARN=${appRegistryAppARN}; export CUSTOM_REPLAYER_USER_AGENT=${solutionsUserAgent}" > /etc/profile.d/solutionsEnv.sh`),
152 | bootstrapFile
153 | ]
154 |
155 | const bootstrapRole = new Role(this, 'BootstrapRole', {
156 | assumedBy: new ServicePrincipal('ec2.amazonaws.com'),
157 | description: 'EC2 Bootstrap Role'
158 | });
159 | bootstrapRole.addManagedPolicy(ManagedPolicy.fromAwsManagedPolicyName('AdministratorAccess'))
160 |
161 | new InstanceProfile(this, 'BootstrapInstanceProfile', {
162 | instanceProfileName: `bootstrap-${stageParameter.valueAsString}-instance-profile`,
163 | role: bootstrapRole
164 | })
165 |
166 | new Instance(this, 'BootstrapEC2Instance', {
167 | vpc: vpc,
168 | instanceName: `bootstrap-${stageParameter.valueAsString}-instance`,
169 | instanceType: InstanceType.of(InstanceClass.T2, InstanceSize.LARGE),
170 | machineImage: MachineImage.latestAmazonLinux2023(),
171 | role: bootstrapRole,
172 | blockDevices: [
173 | {
174 | deviceName: "/dev/xvda",
175 | volume: BlockDeviceVolume.ebs(50)
176 | }
177 | ],
178 | init: CloudFormationInit.fromElements(...cfnInitConfig)
179 | });
180 |
181 | }
182 | }
183 |
--------------------------------------------------------------------------------
/source/solutions-infrastructure/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "migration-assistant-for-amazon-opensearch",
3 | "version": "1.0.0",
4 | "description": "Migration Assistant for Amazon OpenSearch Service",
5 | "license": "Apache-2.0",
6 | "author": {
7 | "name": "Amazon Web Services",
8 | "url": "https://aws.amazon.com/solutions"
9 | },
10 | "scripts": {
11 | "build": "tsc",
12 | "watch": "tsc -w",
13 | "test": "export overrideWarningsEnabled=false && jest --coverage",
14 | "cdk": "cdk"
15 | },
16 | "devDependencies": {
17 | "@aws-cdk/assert": "2.68.0",
18 | "aws-cdk-lib": "2.105.0",
19 | "constructs": "10.3.0",
20 | "@types/jest": "^29.5.5",
21 | "@types/node": "20.9.0",
22 | "@types/prettier": "3.0.0",
23 | "jest": "^29.7.0",
24 | "ts-jest": "^29.1.1",
25 | "aws-cdk": "2.105.0",
26 | "ts-node": "^10.9.1",
27 | "typescript": "~5.2.2"
28 | },
29 | "dependencies": {
30 | "@aws-cdk/aws-servicecatalogappregistry-alpha": "2.105.0-alpha.0",
31 | "source-map-support": "^0.5.21"
32 | }
33 | }
34 |
--------------------------------------------------------------------------------
/source/solutions-infrastructure/test/solutions-infrastructure-stack.test.ts:
--------------------------------------------------------------------------------
1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2 | // SPDX-License-Identifier: Apache-2.0
3 |
4 | import '@aws-cdk/assert/jest';
5 | import { Template } from 'aws-cdk-lib/assertions';
6 | import { App } from 'aws-cdk-lib';
7 | import { SolutionsInfrastructureStack } from '../lib/solutions-infrastructure-stack';
8 |
9 | test('EC2 bootstrap instance is created', () => {
10 | const app = new App();
11 | const stack = new SolutionsInfrastructureStack(app, 'TestBootstrapStack', {
12 | solutionId: 'SO0000',
13 | solutionName: 'test-solution',
14 | solutionVersion: '0.0.1',
15 | codeBucket: 'test-bucket'
16 | });
17 | const template = Template.fromStack(stack);
18 | template.hasResourceProperties('AWS::EC2::Instance', {
19 | InstanceType: "t2.large"
20 | });
21 | });
22 |
--------------------------------------------------------------------------------
/source/solutions-infrastructure/tsconfig.json:
--------------------------------------------------------------------------------
1 | {
2 | "compilerOptions": {
3 | "target": "ES2018",
4 | "module": "commonjs",
5 | "lib": [
6 | "es2018"
7 | ],
8 | "declaration": true,
9 | "strict": true,
10 | "noImplicitAny": true,
11 | "strictNullChecks": true,
12 | "noImplicitThis": true,
13 | "alwaysStrict": true,
14 | "noUnusedLocals": false,
15 | "noUnusedParameters": false,
16 | "noImplicitReturns": true,
17 | "noFallthroughCasesInSwitch": false,
18 | "inlineSourceMap": true,
19 | "inlineSources": true,
20 | "experimentalDecorators": true,
21 | "strictPropertyInitialization": false,
22 | "typeRoots": [
23 | "./node_modules/@types"
24 | ]
25 | },
26 | "exclude": [
27 | "node_modules",
28 | "cdk.out"
29 | ]
30 | }
31 |
--------------------------------------------------------------------------------