├── .github ├── solutionid_validator.sh └── workflows │ └── maintainer_workflows.yml ├── .gitignore ├── .npmignore ├── CODEOWNERS ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── bin ├── cross-region-dask-aws.ts ├── interface.ts └── variables.ts ├── cdk.json ├── images ├── Architecture.png └── ResultsPredictive.png ├── lib ├── ClientConstructs │ ├── client-region-stack.ts │ ├── client-region-tgw-route.ts │ └── secure-bucket.ts ├── DaskImage │ └── Dockerfile ├── LustreRepoTrigger │ └── index.mjs ├── NotebookRequirements.txt ├── SagemakerCode │ ├── get_catalog_inputs.ipynb │ ├── get_data.ipynb │ ├── get_historical_data.ipynb │ ├── get_predictive_data.ipynb │ ├── get_variables.ipynb │ ├── helpers.py │ ├── publish_research.ipynb │ └── ux_notebook.ipynb ├── ScriptsToUpdateOpenSearch │ ├── triggerScan.sh │ └── updateOpenSearch.py ├── SdkConstructs │ ├── accept-tgw-request-client.ts │ ├── create-data-repo-link-lustre.ts │ ├── default-transit-route-table-id.ts │ └── ssm-param-reader.ts └── WorkerConstructs │ ├── interregion-worker-tgw-route.ts │ ├── sync-lustre-to-opensearch.ts │ ├── worker-region-stack.ts │ ├── worker-region-tgw-route.ts │ └── worker-to-worker-tgw.ts ├── package-lock.json ├── package.json └── tsconfig.json /.github/solutionid_validator.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | #set -e 3 | 4 | echo "checking solution id $1" 5 | echo "grep -nr --exclude-dir='.github' "$1" ./.." 6 | result=$(grep -nr --exclude-dir='.github' "$1" ./..) 7 | if [ $? -eq 0 ] 8 | then 9 | echo "Solution ID $1 found\n" 10 | echo "$result" 11 | exit 0 12 | else 13 | echo "Solution ID $1 not found" 14 | exit 1 15 | fi 16 | 17 | export result 18 | -------------------------------------------------------------------------------- /.github/workflows/maintainer_workflows.yml: -------------------------------------------------------------------------------- 1 | # Workflows managed by aws-solutions-library-samples maintainers 2 | name: Maintainer Workflows 3 | on: 4 | # Triggers the workflow on push or pull request events but only for the "main" branch 5 | push: 6 | branches: [ "main" ] 7 | pull_request: 8 | branches: [ "main" ] 9 | types: [opened, reopened, edited] 10 | 11 | jobs: 12 | CheckSolutionId: 13 | runs-on: ubuntu-latest 14 | steps: 15 | - uses: actions/checkout@v4 16 | - name: Run solutionid validator 17 | run: | 18 | chmod u+x ./.github/solutionid_validator.sh 19 | ./.github/solutionid_validator.sh ${{ vars.SOLUTIONID }} -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.js 2 | !jest.config.js 3 | !LustreRepoTrigger/index.js 4 | *.d.ts 5 | node_modules 6 | .ipynb_checkpoints 7 | 8 | # CDK asset staging directory 9 | .cdk.staging 10 | cdk.out -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | *.ts 2 | !*.d.ts 3 | 4 | # CDK asset staging directory 5 | .cdk.staging 6 | cdk.out -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | CODEOWNERS @aws-solutions-library-samples/maintainers 2 | /.github/workflows/maintainer_workflows.yml @aws-solutions-library-samples/maintainers 3 | /.github/solutionid_validator.sh @aws-solutions-library-samples/maintainers 4 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | ## Code of Conduct 2 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 3 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 4 | opensource-codeofconduct@amazon.com with any additional questions or comments. 5 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing Guidelines 2 | 3 | Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional 4 | documentation, we greatly value feedback and contributions from our community. 5 | 6 | Please read through this document before submitting any issues or pull requests to ensure we have all the necessary 7 | information to effectively respond to your bug report or contribution. 8 | 9 | ## Reporting Bugs/Feature Requests 10 | 11 | We welcome you to use the GitHub issue tracker to report bugs or suggest features. 12 | 13 | When filing an issue, please check existing open, or recently closed, issues to make sure somebody else hasn't already 14 | reported the issue. Please try to include as much information as you can. Details like these are incredibly useful: 15 | 16 | - A reproducible test case or series of steps 17 | - The version of our code being used 18 | - Any modifications you've made relevant to the bug 19 | - Anything unusual about your environment or deployment 20 | 21 | ## Contributing via Pull Requests 22 | 23 | Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that: 24 | 25 | 1. You are working against the latest source on the _main_ branch. 26 | 2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already. 27 | 3. You open an issue to discuss any significant work - we would hate for your time to be wasted. 28 | 29 | To send us a pull request, please: 30 | 31 | 1. Fork the repository. 32 | 2. Modify the source; please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change. 33 | 3. Ensure local tests pass. 34 | 4. Commit to your fork using clear commit messages. 35 | 5. Send us a pull request, answering any default questions in the pull request interface. 36 | 6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation. 37 | 38 | GitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and 39 | [creating a pull request](https://help.github.com/articles/creating-a-pull-request/). 40 | 41 | ## Finding contributions to work on 42 | 43 | Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels (enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any 'help wanted' issues is a great place to start. 44 | 45 | ## Code of Conduct 46 | 47 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 48 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 49 | opensource-codeofconduct@amazon.com with any additional questions or comments. 50 | 51 | ## Security issue notifications 52 | 53 | If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public github issue. 54 | 55 | ## Licensing 56 | 57 | See the [LICENSE](LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution. 58 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of 4 | this software and associated documentation files (the "Software"), to deal in 5 | the Software without restriction, including without limitation the rights to 6 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 7 | the Software, and to permit persons to whom the Software is furnished to do so. 8 | 9 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 10 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 11 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 12 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 13 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 14 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 15 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Distributed Compute on AWS with Cross Regional Dask 2 | 3 | The packaged solution presented in this repository proposes a distributed computing solution which uses AWS's global network to not only scale on storage, but minimise data replication across the globe. Extending the capabilities of [Dask](https://www.dask.org/) in combination with libraries such as [dask-worker-pools](https://github.com/gjoseph92/dask-worker-pools). No longer are data scientist's having to migrate petabytes of data interregionally, but instead now have capability in querying/interacting with data at the edge. 4 | 5 | ## Architecture 6 | 7 | ![Architecture](images/Architecture.png) 8 | 9 | This architecture leverages the [Dask framework](https://docs.dask.org) deployed across multiple AWS regions. Dask is a widely adopted data science framework granting users the capabilities of running complex calculations on large datasets, however now with the added benefit of minimising the movement of data through this sample. No longer are users having to migrate mass public S3 datasets to their location, but rather leveraging [Lustre’s ability to integrate seamlessly with public S3 datasets](https://aws.amazon.com/blogs/aws/enhanced-amazon-s3-integration-for-amazon-fsx-for-lustre). 10 | 11 | Through the use of Dask & Lustre, data scientists can perform high I/O intensive workloads on data sparsely located across the globe. Instead of having data replicated from its source region to the user's region, we can deploy a compute solution that strategically positions its workers as close as possible to the dataset. This sample loads in two default datasets which you can **customise in bin/variables.ts** 12 | One is in _us-east-1_ region and the other in _us-west-2_. Which aligns in our architecture as Region B & C. 13 | 14 | The user interacting with these datasets is located in _eu-west-2_ region, or as illustrated in the diagram, Region A. Each region communicates to each other via the Amazon Transit Gateway. 15 | 16 | The bigger picture is that the solution has at the helm a client region containing all the necessary resources for the user to connect into a notebook. That notebook in turn is able to command a scheduler and query a catalogue listing the datasets currently loaded on each of the worker regions. 17 | 18 | Each of the worker regions are copies of themselves, containing Dask workers that connect into the scheduler and have a local Lustre filesystem which syncs to public datasets. The pattern of these regions is generic to allow the user to scale out, include more regions, or scale in, using fewer regions. 19 | 20 | ## Installation 21 | 22 | In order to install all the relevant libraries, execute the following code from the root of the project. 23 | 24 | ```bash 25 | npm install 26 | ``` 27 | 28 | ## CDK Deployment Readiness 29 | 30 | The sample takes advantage of [AWS Cloud Development Kit (CDK)](https://docs.aws.amazon.com/cdk/index.html) to define infrastructure as code. This section focuses on some of the tools and configurations required in order to successfully deploy the CDK. It highlights steps such as ensuring the deployment machine has the correct version of CDK, bootstrapping each region, quota limits to be able to deploy the resources, and a need for docker to deploy the image, otherwise an alternative if Docker is not possible. 31 | 32 | ### CDK Version 33 | 34 | Ensure that the cdk version you are using from your cli is up to date or more **strictly >= 21.0**. You can update your cli by running the below command for all users: 35 | 36 | ```bash 37 | sudo npm i -g aws-cdk 38 | cdk --version 39 | ``` 40 | 41 | ### CDK Bootstrapping Regions 42 | 43 | Make sure you bootstrap each region you plan on deploying to. Information on regions configured are stored in the **bin/variables.ts**. By default we deploy to eu-west-2, us-east-1, and us-west-2. You can change these and/or add more if you please. 44 | 45 | ```bash 46 | cdk bootstrap / / ... 47 | ``` 48 | 49 | ### Account Quotas 50 | 51 | You’ll need to ensure you have sufficient quota in each of the regions for the below, otherwise the cdk might fail to deploy: 52 | 53 | - Worker regions require a minimum of 50 vCPUs Running On-Demand Standard (A, C, D, H, I, M, R, T, Z) instances in order to reasonably deploy and scale up to 30 workers. More will allow it to scale higher. 54 | - All regions will require at least 2 Elastic Ips to assume for the public subnets deployed in each region. 55 | 56 | ### Docker On Deployment Machine 57 | 58 | In order to deploy the public Dask image and have inspector scan it for vulnerabilities, the deployment machine will have to have docker running in the background. This is because the CDK will attempt to build the image and upload it to ECR, to which the container will then use. If the machine is not able to have docker running you can pull directly from the public image on Dockerhub with the below code changes in the client-region-stack.ts and worker-region-stack.ts. 59 | 60 | ```javascript 61 | image: ContainerImage.fromRegistry('daskdev/dask:2022.10.0'), 62 | // image: ContainerImage.fromDockerImageAsset( 63 | // new DockerImageAsset(this, 'Worker Image Repo', { 64 | // directory: path.join(__dirname, 'DaskImage'), 65 | // platform: Platform.LINUX_AMD64, 66 | // }) 67 | // ), 68 | ``` 69 | 70 | ## Deployment 71 | 72 | Once all libraries have been installed, you can proceed in deploying all stacks with the following command. 73 | 74 | ```bash 75 | cdk deploy --all 76 | ``` 77 | 78 | Or if you'd like to skip all manual approvals run: 79 | 80 | ```bash 81 | cdk deploy --all --require-approval never 82 | ``` 83 | 84 | > Stack can take some time to deploy. Expect more than 1 $\frac{1}{2}$ hours to deploy. 85 | 86 | ## Testing 87 | 88 | Once all resources have successfully deployed, a Sagemaker Notebook in the _eu-west-2_ should be available. On that notebook should be a series of files; the one of interest is the **ux_notebook.ipynb**. The notebook contains details on it's purpose, however feel free to use the below inputs for a first run. 89 | 90 | > Q: Data type
91 | > A: both 92 | 93 | > Q: Which of the above would you like to select?
94 | > A: temperature 95 | 96 | > Q: Below are the start and end dates
97 | > A: Start Date 1st Jan 2022, End Date 28th Nov 2022 98 | 99 | > Q: Select all that you would like to be included in the dataset
100 | > A: ScenarioMIP, CMIP 101 | 102 | Any parameters not mentioned above, keep them as default and you should see after running the notebook a graph similiar to the below. 103 | 104 | ![ResultsPredictive](images/ResultsPredictive.png) 105 | 106 | ## Clean up 107 | 108 | Clean up is simple and easy when you develop using the CDK. In order to clean up your account run the following command to destroy resources relating to this project: 109 | 110 | ```bash 111 | cdk destroy --all 112 | ``` 113 | 114 | ## Security 115 | 116 | See [CONTRIBUTING](CONTRIBUTING.md#security-issue-notifications) for more information. 117 | 118 | ## License 119 | 120 | This library is licensed under the MIT-0 License. See the LICENSE file. 121 | 122 | ## Content Security Legal Disclaimer 123 | 124 | The sample code; software libraries; command line tools; proofs of concept; templates; or other related technology (including any of the foregoing that are provided by our personnel) is provided to you as AWS Content under the AWS Customer Agreement, or the relevant written agreement between you and AWS (whichever applies). You should not use this AWS Content in your production accounts, or on production or other critical data. You are responsible for testing, securing, and optimizing the AWS Content, such as sample code, as appropriate for production grade use based on your specific quality control practices and standards. Deploying AWS Content may incur AWS charges for creating or using AWS chargeable resources, such as running Amazon EC2 instances or using Amazon S3 storage. 125 | 126 | ## Operational Metrics Collection 127 | 128 | This solution collects anonymous operational metrics to help AWS improve the quality and features of the solution. Data collection is subject to the AWS Privacy Policy (https://aws.amazon.com/privacy/). To opt out of this feature, simply remove the tag(s) starting with “uksb-” or “SO” from the description(s) in any CloudFormation templates or CDK TemplateOptions. 129 | -------------------------------------------------------------------------------- /bin/cross-region-dask-aws.ts: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: MIT-0 3 | 4 | import { App, Aspects } from "aws-cdk-lib"; 5 | import { ClientRegion } from "../lib/ClientConstructs/client-region-stack"; 6 | import { ClientToWorkerTransitGatewayRoute } from "../lib/ClientConstructs/client-region-tgw-route"; 7 | import { InterRegionTransitGatewayRoute } from "../lib/WorkerConstructs/interregion-worker-tgw-route"; 8 | import { WorkerRegion } from "../lib/WorkerConstructs/worker-region-stack"; 9 | import { WorkerRegionTransitGatewayRoute } from "../lib/WorkerConstructs/worker-region-tgw-route"; 10 | import { WorkerToWorkerTGW } from "../lib/WorkerConstructs/worker-to-worker-tgw"; 11 | import { AwsSolutionsChecks } from "cdk-nag"; 12 | import { client, workers } from "./variables"; 13 | import { SyncLustreToOpenSearch } from "../lib/WorkerConstructs/sync-lustre-to-opensearch"; 14 | 15 | const app = new App(); 16 | // The clients configuration which includes the region and cidr range to which the notebook and scheduler will sit 17 | 18 | /** 19 | * Create the clients region where the notebook and scheduler will be located 20 | */ 21 | const clientStack = new ClientRegion(app, "Client-Region", { 22 | env: { 23 | region: client.region, 24 | }, 25 | clientCidr: client.cidr, 26 | workers, 27 | description: "Guidance for Distributed Compute on AWS with Cross Regional Dask (SO9190)", 28 | }); 29 | 30 | // This array will contain the contruct classes of workers which allow us to interconnect them later dynamically 31 | const WorkerStacks: WorkerRegion[] = []; 32 | // Load through each worker connecting each 33 | for (const worker of workers) { 34 | // Create the base infrastucture for workers, not yet connecting them 35 | const Worker = new WorkerRegion(app, `Worker-Region-${worker.region}`, { 36 | env: { 37 | region: worker.region, 38 | }, 39 | client, 40 | worker, 41 | }); 42 | Worker.addDependency(clientStack); 43 | // Wait until the peer to the client region has been established and then add to the tgw route table 44 | // a route from the worker region to the tgw 45 | new WorkerRegionTransitGatewayRoute( 46 | app, 47 | `Worker-Region-TGW-Route-${worker.region}`, 48 | { 49 | env: { 50 | region: worker.region, 51 | }, 52 | client, 53 | tgw: Worker.tgw, 54 | attachmentId: Worker.attachmentID, 55 | } 56 | ).addDependency(Worker); 57 | // Subsequently we must now add on the client side the same route to their TGW route table, same process 58 | new ClientToWorkerTransitGatewayRoute( 59 | app, 60 | `Client-Region-TGW-Route-${worker.region}`, 61 | { 62 | env: { 63 | region: client.region, 64 | }, 65 | clientTgw: clientStack.clientTGW, 66 | worker, 67 | } 68 | ).addDependency(Worker); 69 | WorkerStacks.push(Worker); 70 | } 71 | 72 | // Connect each worker to each other in a dynamic format 73 | const index = [...Array(workers.length).keys()]; 74 | // Loop each worker by each worker in a form where each worker's connection is visited only once 75 | for (const x in workers) { 76 | const StackWait: WorkerToWorkerTGW[] = []; 77 | for (const y of index.slice(parseInt(x) + 1, index.length)) { 78 | // First we create the neccessary peer connection adding what we can at this early point in time 79 | const W2WTransitGateway = new WorkerToWorkerTGW( 80 | app, 81 | `TGW-Peer-Region-${workers[x].region}-to-${workers[y].region}`, 82 | { 83 | env: { 84 | region: workers[x].region, 85 | }, 86 | peerWorker: workers[y], 87 | vpc: WorkerStacks[x].vpc, 88 | } 89 | ); 90 | StackWait.push(W2WTransitGateway); 91 | 92 | // We then want to add from worker to worker the transit gateway route 93 | const WRTGWRoute = new WorkerRegionTransitGatewayRoute( 94 | app, 95 | `Worker-Region-TGW-Route-${workers[x].region}-to-${workers[y].region}`, 96 | { 97 | env: { 98 | region: workers[x].region, 99 | }, 100 | client: workers[y], 101 | tgw: WorkerStacks[x].tgw, 102 | attachmentId: W2WTransitGateway.attachmentID, 103 | } 104 | ).addDependency(W2WTransitGateway); 105 | 106 | // And finally the inverse of what's done above 107 | new InterRegionTransitGatewayRoute( 108 | app, 109 | `Inter-Region-TGW-Route-${workers[y].region}-to-${workers[x].region}`, 110 | { 111 | env: { 112 | region: workers[y].region, 113 | }, 114 | peerWorker: workers[x], 115 | tgw: WorkerStacks[y].tgw, 116 | vpc: WorkerStacks[y].vpc, 117 | } 118 | ).addDependency(W2WTransitGateway); 119 | } 120 | // Give as much time as possible for lustre to sync to public s3, and then launch the autoscaling 121 | // instance to publish the results to opensearch 122 | const lustreToOS = new SyncLustreToOpenSearch( 123 | app, 124 | `ZyncLustreToOpenSearch-${workers[x].region}`, 125 | { 126 | env: { 127 | region: workers[x].region, 128 | }, 129 | client: client, 130 | worker: workers[x], 131 | vpc: WorkerStacks[x].vpc, 132 | lustre: WorkerStacks[x].lustre, 133 | } 134 | ); 135 | } 136 | // CDK nag reports are outputted into the dist folder as csv files 137 | Aspects.of(app).add(new AwsSolutionsChecks({ reports: true })); 138 | -------------------------------------------------------------------------------- /bin/interface.ts: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: MIT-0 3 | 4 | export interface IClient { 5 | region: string; 6 | cidr: string; 7 | } 8 | export interface IWorker { 9 | region: string; 10 | cidr: string; 11 | dataset: string; 12 | lustreFileSystemPath: string; 13 | } 14 | -------------------------------------------------------------------------------- /bin/variables.ts: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: MIT-0 3 | 4 | import { IClient, IWorker } from "./interface"; 5 | 6 | export const client: IClient = { region: "eu-west-2", cidr: "10.0.0.0/16" }; 7 | // The worker regions you wish to deploy to, which the respective datasets you want to connect to 8 | // NOTE: Between the client and workers these cidr ranges cannot overlap 9 | 10 | export const workers: IWorker[] = [ 11 | { 12 | region: "us-east-1", 13 | cidr: "10.1.0.0/16", 14 | dataset: "s3://era5-pds", 15 | // The public s3 dataset on https://registry.opendata.aws/ you wish to connect to 16 | lustreFileSystemPath: "era5-pds", 17 | }, 18 | { 19 | region: "us-west-2", 20 | cidr: "10.2.0.0/16", 21 | dataset: "s3://cmip6-pds/CMIP6/ScenarioMIP/MOHC", 22 | // The mapping you wish to have set up on the worker. 23 | // E.g. this mapping will be saved as /fsx/us-west-2/CMIP6/ScenarioMIP/MOHC 24 | lustreFileSystemPath: "CMIP6/ScenarioMIP/MOHC", 25 | }, 26 | ]; 27 | -------------------------------------------------------------------------------- /cdk.json: -------------------------------------------------------------------------------- 1 | { 2 | "app": "npx ts-node --prefer-ts-exts bin/cross-region-dask-aws.ts", 3 | "watch": { 4 | "include": ["**"], 5 | "exclude": [ 6 | "README.md", 7 | "cdk*.json", 8 | "**/*.d.ts", 9 | "**/*.js", 10 | "tsconfig.json", 11 | "package*.json", 12 | "yarn.lock", 13 | "node_modules", 14 | "test" 15 | ] 16 | }, 17 | "context": { 18 | "@aws-cdk/aws-apigateway:usagePlanKeyOrderInsensitiveId": true, 19 | "@aws-cdk/core:stackRelativeExports": true, 20 | "@aws-cdk/aws-rds:lowercaseDbIdentifier": true, 21 | "@aws-cdk/aws-lambda:recognizeVersionProps": true, 22 | "@aws-cdk/aws-lambda:recognizeLayerVersion": true, 23 | "@aws-cdk/aws-cloudfront:defaultSecurityPolicyTLSv1.2_2021": true, 24 | "@aws-cdk-containers/ecs-service-extensions:enableDefaultLogDriver": true, 25 | "@aws-cdk/aws-ec2:uniqueImdsv2TemplateName": true, 26 | "@aws-cdk/core:checkSecretUsage": true, 27 | "@aws-cdk/aws-iam:minimizePolicies": true, 28 | "@aws-cdk/aws-ecs:arnFormatIncludesClusterName": true, 29 | "@aws-cdk/core:validateSnapshotRemovalPolicy": true, 30 | "@aws-cdk/aws-codepipeline:crossAccountKeyAliasStackSafeResourceName": true, 31 | "@aws-cdk/aws-s3:createDefaultLoggingPolicy": true, 32 | "@aws-cdk/aws-sns-subscriptions:restrictSqsDescryption": true, 33 | "@aws-cdk/aws-apigateway:disableCloudWatchRole": true, 34 | "@aws-cdk/core:enablePartitionLiterals": true, 35 | "@aws-cdk/core:target-partitions": ["aws", "aws-cn"] 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /images/Architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions-library-samples/distributed-compute-on-aws-with-cross-regional-dask/1d5231cdd9fbd7835bfb84ffc4178f70ddb7c3a8/images/Architecture.png -------------------------------------------------------------------------------- /images/ResultsPredictive.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions-library-samples/distributed-compute-on-aws-with-cross-regional-dask/1d5231cdd9fbd7835bfb84ffc4178f70ddb7c3a8/images/ResultsPredictive.png -------------------------------------------------------------------------------- /lib/ClientConstructs/client-region-stack.ts: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: MIT-0 3 | 4 | import { Stack, App, StackProps, RemovalPolicy } from "aws-cdk-lib"; 5 | import { Code, Repository } from "aws-cdk-lib/aws-codecommit"; 6 | import { 7 | CfnRoute, 8 | CfnTransitGateway, 9 | CfnTransitGatewayAttachment, 10 | FlowLogDestination, 11 | FlowLogTrafficType, 12 | IpAddresses, 13 | Peer, 14 | Port, 15 | SecurityGroup, 16 | SubnetType, 17 | Vpc, 18 | } from "aws-cdk-lib/aws-ec2"; 19 | import { 20 | Cluster, 21 | ContainerImage, 22 | FargateTaskDefinition, 23 | LogDriver, 24 | } from "aws-cdk-lib/aws-ecs"; 25 | import { ApplicationLoadBalancedFargateService } from "aws-cdk-lib/aws-ecs-patterns"; 26 | import { 27 | CfnServiceLinkedRole, 28 | ManagedPolicy, 29 | PolicyDocument, 30 | PolicyStatement, 31 | Role, 32 | ServicePrincipal, 33 | } from "aws-cdk-lib/aws-iam"; 34 | import { Key } from "aws-cdk-lib/aws-kms"; 35 | import { LogGroup } from "aws-cdk-lib/aws-logs"; 36 | import { Domain, EngineVersion } from "aws-cdk-lib/aws-opensearchservice"; 37 | import { 38 | CfnNotebookInstance, 39 | CfnNotebookInstanceLifecycleConfig, 40 | } from "aws-cdk-lib/aws-sagemaker"; 41 | import { PrivateDnsNamespace, Service } from "aws-cdk-lib/aws-servicediscovery"; 42 | import { StringParameter } from "aws-cdk-lib/aws-ssm"; 43 | import { ApplicationLoadBalancer } from "aws-cdk-lib/aws-elasticloadbalancingv2"; 44 | import { NagSuppressions } from "cdk-nag"; 45 | import { secureBucket } from "./secure-bucket"; 46 | import { DockerImageAsset, Platform } from "aws-cdk-lib/aws-ecr-assets"; 47 | import { 48 | AwsCustomResource, 49 | AwsCustomResourcePolicy, 50 | } from "aws-cdk-lib/custom-resources"; 51 | import { readFileSync } from "fs"; 52 | import { IWorker } from "../../bin/interface"; 53 | import path = require("path"); 54 | 55 | export interface ClientRegionProps extends StackProps { 56 | clientCidr: string; 57 | workers: IWorker[]; 58 | } 59 | 60 | /** 61 | * The Client Region's primary function is in setting up all the relevant resources in the region 62 | * besides connecting each of the regions (We must wait until the other regions are up to do so) 63 | */ 64 | export class ClientRegion extends Stack { 65 | public clientTGW: CfnTransitGateway; 66 | vpc: Vpc; 67 | cluster: Cluster; 68 | schedulerDisovery: Service; 69 | openSearchDomain: StringParameter; 70 | openSearchArn: StringParameter; 71 | 72 | constructor(scope: App, id: string, props: ClientRegionProps) { 73 | super(scope, id, props); 74 | const { clientCidr, workers } = props; 75 | 76 | this.setupEnvironment(clientCidr, workers); 77 | this.setupDaskScheduler(clientCidr, workers); 78 | this.setupOpenSearch(workers); 79 | this.setupSagemaker(); 80 | NagSuppressions.addStackSuppressions(this, [ 81 | { 82 | id: "AwsSolutions-IAM4", 83 | reason: 84 | "Lambda execution policy for custom resources created by higher level CDK constructs", 85 | appliesTo: [ 86 | "Policy::arn::iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", 87 | ], 88 | }, 89 | { 90 | id: "AwsSolutions-L1", 91 | reason: "AWS CDK custom resources uses node 14.x environment", 92 | }, 93 | ]); 94 | } 95 | 96 | /** Environment Setup 97 | * 98 | * The function sets up the VPC, along with transit gateway, attachments, cfn routes 99 | * PrivateNamespace to publish the schedulers IP and empty cluster in which the scheduler will sit 100 | * 101 | * @param clientCidr - The cidr range of client where the notebook sits 102 | * @param workers - An array of worker regions 103 | */ 104 | setupEnvironment(clientCidr: string, workers: IWorker[]) { 105 | // Client VPC 106 | this.vpc = new Vpc(this, "Scheduler VPC", { 107 | ipAddresses: IpAddresses.cidr(clientCidr), 108 | flowLogs: { 109 | cloudwatch: { 110 | destination: FlowLogDestination.toCloudWatchLogs( 111 | new LogGroup(this, "SchedulerVpcFlowLogs") 112 | ), 113 | trafficType: FlowLogTrafficType.ALL, 114 | }, 115 | }, 116 | }); 117 | 118 | // Transit Gateway 119 | this.clientTGW = new CfnTransitGateway(this, "TGW"); 120 | // We will need this parameter in other regions to connect to 121 | new StringParameter(this, "TGW Param", { 122 | parameterName: `tgw-param-${this.region}`, 123 | stringValue: this.clientTGW.ref, 124 | }); 125 | // Base attachment for connecting the local VPC to TGW 126 | const attach = new CfnTransitGatewayAttachment(this, "tgw-attachment", { 127 | subnetIds: this.vpc.privateSubnets.map(({ subnetId }) => subnetId), 128 | transitGatewayId: this.clientTGW.ref, 129 | vpcId: this.vpc.vpcId, 130 | }); 131 | attach.addDependsOn(this.clientTGW); 132 | 133 | // At this early point we can add the routes to the private subnets indiciating that for 134 | // worker region cidrs they should use the TGW 135 | for (const worker of workers) { 136 | for (let i = 0; i < this.vpc.privateSubnets.length; i++) { 137 | new CfnRoute( 138 | this, 139 | `Subnet to TGW - ${this.vpc.privateSubnets[i]} - ${worker.region}`, 140 | { 141 | routeTableId: this.vpc.privateSubnets[i].routeTable.routeTableId, 142 | destinationCidrBlock: worker.cidr, 143 | transitGatewayId: this.clientTGW.ref, 144 | } 145 | ).addDependsOn(attach); 146 | } 147 | } 148 | 149 | /** 150 | * Below we initialise a private namespace which will keep track of the changing schedulers IP 151 | * The workers will need this IP to connect to, so instead of tracking it statically, they can 152 | * simply reference the DNS which will resolve to the IP everytime 153 | */ 154 | const PrivateNP = new PrivateDnsNamespace(this, "local-dask", { 155 | name: "local-dask", 156 | vpc: this.vpc, 157 | }); 158 | // Other regions will have to associate-vpc-with-hosted-zone to access this namespace 159 | new StringParameter(this, "PrivateNP Param", { 160 | parameterName: `privatenp-hostedid-param-${this.region}`, 161 | stringValue: PrivateNP.namespaceHostedZoneId, 162 | }); 163 | this.schedulerDisovery = new Service(this, "Scheduler Discovery", { 164 | name: "Dask-Scheduler", 165 | namespace: PrivateNP, 166 | }); 167 | 168 | // Scheduler Cluster initialised as empty for later 169 | this.cluster = new Cluster(this, "Scheduler Cluster", { 170 | clusterName: "DaskScheduler", 171 | containerInsights: true, 172 | vpc: this.vpc, 173 | }); 174 | } 175 | 176 | /** Dask Scheduler 177 | * 178 | * This function focuses on the setup of the Dask Scheduler. The scheduler is setup as a fargate task 179 | * operating on high cpu and memory given it's critical purpose in orchestrating the dask. 180 | * We manually publish the dashboard to 8787 which will be viewable from within the VPC. E.g. load from 181 | * a browser from cloud9 that sits inside the VPC 182 | * 183 | * @param workers - An array of the worker regions 184 | */ 185 | setupDaskScheduler(clientCidr: string, workers: IWorker[]) { 186 | // Fargate Definition 187 | const schedulerDefinition = new FargateTaskDefinition( 188 | this, 189 | "Scheduler Definition", 190 | { family: "Dask-Scheduler", memoryLimitMiB: 32768, cpu: 16384 } 191 | ); 192 | // Container loads in from a versioned dask image on a fixed 8787 dashboard address 193 | schedulerDefinition.addContainer("Container", { 194 | containerName: "Dask", 195 | image: ContainerImage.fromDockerImageAsset( 196 | new DockerImageAsset(this, "Scheduler Image Repo", { 197 | directory: path.join(__dirname, "..", "DaskImage"), 198 | platform: Platform.LINUX_AMD64, 199 | }) 200 | ), 201 | command: [ 202 | "dask", 203 | "scheduler", 204 | "--dashboard", 205 | "--dashboard-address", 206 | "8787", 207 | ], 208 | essential: true, 209 | logging: LogDriver.awsLogs({ 210 | streamPrefix: "ecs", 211 | logGroup: new LogGroup(this, "Scheduler Log Group"), 212 | }), 213 | portMappings: [{ containerPort: 8787 }, { containerPort: 8786 }], 214 | }); 215 | NagSuppressions.addResourceSuppressions( 216 | new AwsCustomResource(this, "Enable Scanning on Repo", { 217 | onCreate: { 218 | service: "ECR", 219 | action: "putRegistryScanningConfiguration", 220 | physicalResourceId: { id: "putRegistryScanningConfiguration" }, 221 | parameters: { 222 | scanType: "ENHANCED", 223 | }, 224 | }, 225 | policy: AwsCustomResourcePolicy.fromStatements([ 226 | new PolicyStatement({ 227 | actions: [ 228 | "iam:CreateServiceLinkedRole", 229 | "inspector2:Enable", 230 | "ecr:PutRegistryScanningConfiguration", 231 | ], 232 | resources: AwsCustomResourcePolicy.ANY_RESOURCE, 233 | }), 234 | ]), 235 | }), 236 | [ 237 | { 238 | id: "AwsSolutions-IAM5", 239 | reason: "Call needs access to all resources", 240 | }, 241 | ], 242 | true 243 | ); 244 | 245 | // Only worker region cidr ranges should be allowed to connect on port 8786 246 | const SchedulerSecurityGroup = new SecurityGroup( 247 | this, 248 | "Scheduler Security Group", 249 | { vpc: this.vpc } 250 | ); 251 | SchedulerSecurityGroup.addIngressRule( 252 | Peer.ipv4(clientCidr), 253 | Port.tcp(8786), 254 | `Allow the home VPC to connect ${clientCidr} to the scheduler` 255 | ); 256 | for (const worker of workers) { 257 | SchedulerSecurityGroup.addIngressRule( 258 | Peer.ipv4(worker.cidr), 259 | Port.tcp(8786), 260 | `Workers in ${worker.cidr} connect to Scheduler on this port` 261 | ); 262 | } 263 | 264 | // Configuring the ALB for our fargate service, we get added control this way 265 | const albSecurityGroup = new SecurityGroup(this, "Scheduler ALB SG", { 266 | vpc: this.vpc, 267 | }); 268 | // Restricting it so that only those within the VPC have access 269 | albSecurityGroup.addIngressRule( 270 | Peer.ipv4(this.vpc.vpcCidrBlock), 271 | Port.tcp(80), 272 | "Scheduler dashboard access" 273 | ); 274 | // Offered construct that launches and manages the connectivity of an ALB to Fargate 275 | const DaskService = new ApplicationLoadBalancedFargateService( 276 | this, 277 | "Scheduler with Load Balancer", 278 | { 279 | serviceName: "Dask-Scheduler", 280 | taskDefinition: schedulerDefinition, 281 | cluster: this.cluster, 282 | enableExecuteCommand: true, 283 | taskSubnets: { 284 | subnetType: SubnetType.PRIVATE_WITH_EGRESS, 285 | }, 286 | loadBalancer: new ApplicationLoadBalancer(this, "Dask-Scheduler-ALB", { 287 | vpc: this.vpc, 288 | securityGroup: albSecurityGroup, 289 | internetFacing: true, 290 | }), 291 | securityGroups: [SchedulerSecurityGroup], 292 | } 293 | ); 294 | DaskService.targetGroup.configureHealthCheck({ 295 | path: "/status", 296 | }); 297 | DaskService.service.associateCloudMapService({ 298 | service: this.schedulerDisovery, 299 | }); 300 | DaskService.loadBalancer.logAccessLogs( 301 | secureBucket(this, "Dask-Scheduler-ALB-Access-Logs") 302 | ); 303 | NagSuppressions.addResourceSuppressions( 304 | schedulerDefinition, 305 | [ 306 | { 307 | id: "AwsSolutions-IAM5", 308 | reason: 309 | "Task role created by CDK has specific cloudwatch and ssm messages actions", 310 | }, 311 | ], 312 | true 313 | ); 314 | } 315 | 316 | /** OpenSearch 317 | * 318 | * OpenSearch will index the metadata of datasource we are connecting to from each region. We expose them 319 | * to the client region and worker regions over HTTPS only. 320 | * 321 | * @param workers - An array of the worker regions 322 | */ 323 | setupOpenSearch(workers: IWorker[]) { 324 | // The security group is restricted to only allow HTTPS connectivity to the index 325 | const openSearchSecurityGroup = new SecurityGroup( 326 | this, 327 | "Open search Security Group", 328 | { vpc: this.vpc } 329 | ); 330 | openSearchSecurityGroup.addIngressRule( 331 | Peer.ipv4(this.vpc.vpcCidrBlock), 332 | Port.tcp(443), 333 | "Allow access from client" 334 | ); 335 | for (const worker of workers) { 336 | openSearchSecurityGroup.addIngressRule( 337 | Peer.ipv4(worker.cidr), 338 | Port.tcp(443), 339 | "Allow updates from worker/regional instances" 340 | ); 341 | } 342 | 343 | // OpenSearch requires a service linked role, if you experience errors here, it may be that 344 | // you already have a service linked role for opensearch, so you can comment and redpeloy below 345 | const serviceLinkedRole = new CfnServiceLinkedRole(this, "OpenSearch SLR", { 346 | awsServiceName: "opensearchservice.amazonaws.com", 347 | }); 348 | const openSearchDomain = new Domain(this, "OpenSearch Domain", { 349 | version: EngineVersion.OPENSEARCH_1_3, 350 | removalPolicy: RemovalPolicy.DESTROY, 351 | enableVersionUpgrade: true, 352 | nodeToNodeEncryption: true, 353 | capacity: { 354 | masterNodes: 3, 355 | dataNodes: 2, 356 | }, 357 | zoneAwareness: { 358 | availabilityZoneCount: 2, 359 | }, 360 | encryptionAtRest: { 361 | enabled: true, 362 | }, 363 | logging: { 364 | slowSearchLogEnabled: true, 365 | slowIndexLogEnabled: true, 366 | }, 367 | enforceHttps: true, 368 | vpc: this.vpc, 369 | useUnsignedBasicAuth: false, 370 | securityGroups: [openSearchSecurityGroup], 371 | }); 372 | openSearchDomain.node.addDependency(serviceLinkedRole); 373 | 374 | NagSuppressions.addResourceSuppressions( 375 | openSearchDomain, 376 | [ 377 | { 378 | id: "AwsSolutions-IAM5", 379 | reason: "CloudWatch logging role with specific actions", 380 | }, 381 | { 382 | id: "AwsSolutions-OS3", 383 | reason: 384 | "Domain is within a VPC, and uses security groups for allow-listing an IP block", 385 | }, 386 | { 387 | id: "AwsSolutions-OS5", 388 | reason: 389 | "Domain is within a VPC, and uses security groups for allow-listing an IP block", 390 | }, 391 | ], 392 | true 393 | ); 394 | 395 | this.openSearchDomain = new StringParameter(this, "OpenSearch HostName", { 396 | parameterName: `client-opensearch-domain-${this.region}`, 397 | stringValue: openSearchDomain.domainEndpoint, 398 | }); 399 | this.openSearchArn = new StringParameter(this, "OpenSearch ARN", { 400 | parameterName: `client-opensearch-arn-${this.region}`, 401 | stringValue: openSearchDomain.domainArn, 402 | }); 403 | } 404 | 405 | /** Jupyter Notebook 406 | * 407 | * The notebook acts as the interfacing body from the user to the scheduler. Below loads in the relevant 408 | * installations for the notebook to connect with the relevant permissions 409 | */ 410 | setupSagemaker() { 411 | // The notebook requires access to not only access full access but specific access to the opensearch to post requests 412 | const role = new Role(this, "Sagemaker Role", { 413 | assumedBy: new ServicePrincipal("sagemaker.amazonaws.com"), 414 | managedPolicies: [ 415 | ManagedPolicy.fromAwsManagedPolicyName("AmazonSageMakerFullAccess"), 416 | ManagedPolicy.fromAwsManagedPolicyName("AmazonECS_FullAccess"), 417 | ], 418 | inlinePolicies: { 419 | RetrieveOpenSearchDomain: new PolicyDocument({ 420 | statements: [ 421 | new PolicyStatement({ 422 | resources: [ 423 | this.openSearchArn.stringValue, 424 | `${this.openSearchArn.stringValue}/*`, 425 | ], 426 | actions: ["es:ESHttpPost", "es:ESHttpPut"], 427 | }), 428 | ], 429 | }), 430 | }, 431 | }); 432 | this.openSearchDomain.grantRead(role); 433 | 434 | NagSuppressions.addResourceSuppressions(role, [ 435 | { 436 | id: "AwsSolutions-IAM4", 437 | reason: 438 | "Sagemaker Notebook policies need to be broad to allow access to ", 439 | }, 440 | { 441 | id: "AwsSolutions-IAM5", 442 | reason: "Role requires access to all indicies", 443 | }, 444 | ]); 445 | 446 | const SagemakerSec = new SecurityGroup(this, "Sagemaker security group", { 447 | vpc: this.vpc, 448 | }); 449 | 450 | // Life Cycles allow us to install all the neccessary libraries so that when the noteook starts 451 | // the user need not worry about installing the required packages at the correct version 452 | const lifecycle = new CfnNotebookInstanceLifecycleConfig( 453 | this, 454 | "Life Cycle Config", 455 | { 456 | notebookInstanceLifecycleConfigName: "LibraryforDaskNotebook", 457 | onStart: [ 458 | { 459 | content: readFileSync( 460 | path.join(__dirname, "..", "NotebookRequirements.txt") 461 | ).toString("base64"), 462 | }, 463 | ], 464 | } 465 | ); 466 | 467 | // Preloaded code brought into the notebook at launch 468 | const repo = new Repository(this, "Sagemaker Code", { 469 | repositoryName: "Sagemaker_Dask", 470 | code: Code.fromDirectory(path.join(__dirname, "..", "SagemakerCode")), 471 | }); 472 | repo.grantRead(role); 473 | 474 | NagSuppressions.addResourceSuppressions( 475 | role, 476 | [ 477 | { 478 | id: "AwsSolutions-IAM5", 479 | reason: "Permissions added by grant read on CodeCommit repo", 480 | appliesTo: [ 481 | "Action::codecommit:Describe*", 482 | "Action::codecommit:Get*", 483 | ], 484 | }, 485 | ], 486 | true 487 | ); 488 | 489 | // Adding encryption is a best practise on the notebook 490 | const nbKey = new Key(this, "Notebook Key", { 491 | enableKeyRotation: true, 492 | }); 493 | 494 | // The Sagemaker Notebook 495 | new CfnNotebookInstance(this, "Dask Notebook", { 496 | notebookInstanceName: "Dask-Notebook", 497 | rootAccess: "Disabled", 498 | directInternetAccess: "Disabled", 499 | defaultCodeRepository: repo.repositoryCloneUrlHttp, 500 | instanceType: "ml.t3.2xlarge", 501 | roleArn: role.roleArn, 502 | subnetId: this.vpc.privateSubnets[0].subnetId, 503 | securityGroupIds: [SagemakerSec.securityGroupId], 504 | lifecycleConfigName: lifecycle.notebookInstanceLifecycleConfigName, 505 | kmsKeyId: nbKey.keyId, 506 | platformIdentifier: "notebook-al2-v1", 507 | volumeSizeInGb: 50, 508 | }); 509 | } 510 | } 511 | -------------------------------------------------------------------------------- /lib/ClientConstructs/client-region-tgw-route.ts: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: MIT-0 3 | 4 | import { Stack, App, StackProps } from "aws-cdk-lib"; 5 | import { 6 | CfnTransitGateway, 7 | CfnTransitGatewayRoute, 8 | Vpc, 9 | } from "aws-cdk-lib/aws-ec2"; 10 | import { NagSuppressions } from "cdk-nag"; 11 | import { IWorker } from "../../bin/interface"; 12 | import { TransitGatewayRouteTable } from "../SdkConstructs/default-transit-route-table-id"; 13 | import { SSMParameterReader } from "../SdkConstructs/ssm-param-reader"; 14 | 15 | export interface ClientRegionProps extends StackProps { 16 | clientTgw: CfnTransitGateway; 17 | worker: IWorker; 18 | } 19 | 20 | /** 21 | * Pull all the relevant information for the client in order to add to it's default TGW route table 22 | * a routing of local traffic to the peer attachment 23 | */ 24 | export class ClientToWorkerTransitGatewayRoute extends Stack { 25 | vpc: Vpc; 26 | 27 | constructor(scope: App, id: string, props: ClientRegionProps) { 28 | super(scope, id, props); 29 | const { clientTgw, worker } = props; 30 | 31 | NagSuppressions.addStackSuppressions(this, [ 32 | { 33 | id: "AwsSolutions-IAM4", 34 | reason: 35 | "Lambda execution policy for custom resources created by higher level CDK constructs", 36 | appliesTo: [ 37 | "Policy::arn::iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", 38 | ], 39 | }, 40 | { 41 | id: "AwsSolutions-L1", 42 | reason: "AWS CDK custom resources uses node 14.x environment", 43 | }, 44 | ]); 45 | 46 | const transitGatewayConstruct = new TransitGatewayRouteTable( 47 | this, 48 | "Transit Gateway", 49 | { 50 | parameterName: clientTgw.ref, 51 | region: this.region, 52 | } 53 | ); 54 | const transitGatewayRouteTableId = 55 | transitGatewayConstruct.getParameterValue(); 56 | NagSuppressions.addResourceSuppressions( 57 | transitGatewayConstruct, 58 | [ 59 | { 60 | id: "AwsSolutions-IAM5", 61 | reason: "By default the action specified supports all resource types", 62 | }, 63 | ], 64 | true 65 | ); 66 | 67 | const transitGatewayAttachmentId = new SSMParameterReader( 68 | this, 69 | `Transit Attachment ID - ${worker.region}`, 70 | { 71 | parameterName: `tgw-attachmentid-${worker.region}`, 72 | region: worker.region, 73 | account: this.account, 74 | } 75 | ).getParameterValue(); 76 | 77 | // Append the route 78 | new CfnTransitGatewayRoute(this, `TGW Route - ${worker.region}`, { 79 | transitGatewayRouteTableId, 80 | destinationCidrBlock: worker.cidr, 81 | transitGatewayAttachmentId, 82 | }); 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /lib/ClientConstructs/secure-bucket.ts: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: MIT-0 3 | 4 | import { Stack } from "aws-cdk-lib"; 5 | import { 6 | BlockPublicAccess, 7 | Bucket, 8 | BucketEncryption, 9 | } from "aws-cdk-lib/aws-s3"; 10 | import { PolicyStatement } from "aws-cdk-lib/aws-iam"; 11 | import { NagSuppressions } from "cdk-nag"; 12 | 13 | // Bucket is for launching a secure bucket for logging access to 14 | export const secureBucket = ( 15 | stack: Stack, 16 | bucketId: string, 17 | accessLogsBucket?: Bucket, 18 | resourcePolicy?: PolicyStatement 19 | ) => { 20 | const bucket = new Bucket(stack, bucketId, { 21 | blockPublicAccess: BlockPublicAccess.BLOCK_ALL, 22 | encryption: BucketEncryption.S3_MANAGED, 23 | enforceSSL: true, 24 | serverAccessLogsBucket: accessLogsBucket, 25 | }); 26 | if (!accessLogsBucket) { 27 | NagSuppressions.addResourceSuppressions( 28 | bucket, 29 | [{ id: "AwsSolutions-S1", reason: "Access logs bucket" }], 30 | true 31 | ); 32 | } 33 | if (resourcePolicy) { 34 | resourcePolicy.addResources(bucket.bucketArn, `${bucket.bucketArn}/*`); 35 | bucket.addToResourcePolicy(resourcePolicy); 36 | } 37 | return bucket; 38 | }; 39 | -------------------------------------------------------------------------------- /lib/DaskImage/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM daskdev/dask:2022.10.0 2 | -------------------------------------------------------------------------------- /lib/LustreRepoTrigger/index.mjs: -------------------------------------------------------------------------------- 1 | import { 2 | FSxClient, 3 | CreateDataRepositoryTaskCommand, 4 | } from "@aws-sdk/client-fsx"; 5 | 6 | const fsx = new FSxClient(); 7 | 8 | export const handler = async (event) => { 9 | const command = new CreateDataRepositoryTaskCommand({ 10 | FileSystemId: process.env.FileSystemId, 11 | Type: "IMPORT_METADATA_FROM_REPOSITORY", 12 | Report: { 13 | Enabled: false, 14 | }, 15 | }); 16 | const response = await fsx.send(command); 17 | 18 | console.log(response); 19 | return; 20 | }; 21 | -------------------------------------------------------------------------------- /lib/NotebookRequirements.txt: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | sudo -u ec2-user -i <<'EOF' 6 | 7 | # Install nodejs 8 | curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash 9 | . ~/.nvm/nvm.sh 10 | nvm install 16 11 | node -e "console.log('Running Node.js ' + process.version)" 12 | 13 | ## Install PIP deps on all jupyter envs 14 | env_name=python3 15 | source /home/ec2-user/anaconda3/bin/activate $env_name 16 | 17 | pip3 install --upgrade pip 18 | pip3 install xarray[complete] 19 | pip3 install distributed==2022.10.0 dask==2022.10.0 lz4==4.0.0 numpy==1.23.3 pandas==1.5.0 20 | pip3 install cloudpickle msgpack toolz --upgrade 21 | pip3 install git+https://github.com/gjoseph92/dask-worker-pools.git@main 22 | pip3 install boto3 botocore opensearch-py requests requests-aws4auth ipympl nodejs npm intake-esm --upgrade 23 | jupyter labextension install @jupyter-widgets/jupyterlab-manager 24 | jupyter lab build 25 | 26 | source /home/ec2-user/anaconda3/bin/deactivate 27 | 28 | ## Install jupyter-widgets in JupyterSystemEnv 29 | env_name1=JupyterSystemEnv 30 | source /home/ec2-user/anaconda3/bin/activate $env_name1 31 | jupyter labextension install @jupyter-widgets/jupyterlab-manager 32 | jupyter lab build 33 | source /home/ec2-user/anaconda3/bin/deactivate 34 | 35 | EOF 36 | -------------------------------------------------------------------------------- /lib/SagemakerCode/get_catalog_inputs.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "id": "e6ab8ba2", 7 | "metadata": {}, 8 | "outputs": [], 9 | "source": [ 10 | "from helpers import print_list\n", 11 | "import ipywidgets as widgets" 12 | ] 13 | }, 14 | { 15 | "cell_type": "code", 16 | "execution_count": null, 17 | "id": "18422499", 18 | "metadata": {}, 19 | "outputs": [], 20 | "source": [ 21 | "from helpers import print_list\n", 22 | "# from helpers import get_desired_value_dict" 23 | ] 24 | }, 25 | { 26 | "cell_type": "code", 27 | "execution_count": null, 28 | "id": "7b1dcb75", 29 | "metadata": {}, 30 | "outputs": [], 31 | "source": [ 32 | "%store -r data_type\n", 33 | "if data_type == \"historical\":\n", 34 | " print(\"Note: because you are doing historical, you don't need to set these inputs, but we are displaying them for UI demo. \")" 35 | ] 36 | }, 37 | { 38 | "cell_type": "code", 39 | "execution_count": null, 40 | "id": "8bb64fd1", 41 | "metadata": {}, 42 | "outputs": [], 43 | "source": [ 44 | "activity_ids = ['DCPP', # different activities, with multiple members\n", 45 | " 'ScenarioMIP',\n", 46 | " 'CMIP',\n", 47 | " 'GMMIP',\n", 48 | " 'CDRMIP',\n", 49 | " 'C4MIP']\n", 50 | "activity_id = [activity_ids[0]]\n", 51 | "%store activity_id" 52 | ] 53 | }, 54 | { 55 | "cell_type": "code", 56 | "execution_count": null, 57 | "id": "e7d63b81", 58 | "metadata": {}, 59 | "outputs": [], 60 | "source": [ 61 | "w = widgets.SelectMultiple(\n", 62 | " options=activity_ids,\n", 63 | " value=[activity_ids[0]],\n", 64 | " description=\"activity: \",\n", 65 | ")\n", 66 | "\n", 67 | "out = widgets.Output()\n", 68 | "# display(out)\n", 69 | "\n", 70 | "\n", 71 | "@out.capture()\n", 72 | "def on_change(change):\n", 73 | " print('method is called when printing this')\n", 74 | " if change['type'] == 'change' and change['name'] == 'value':\n", 75 | " global activity_id\n", 76 | " activity_id = list(change.new)\n", 77 | " %store activity_id\n", 78 | "w.observe(on_change)\n", 79 | "\n", 80 | "\n", 81 | "whats_below = \"all activity IDs. \\n\\nSelect all that you would like to be included in the dataset.\"\n", 82 | "print_list(whats_below, [])\n", 83 | "display(w)" 84 | ] 85 | }, 86 | { 87 | "cell_type": "code", 88 | "execution_count": null, 89 | "id": "8f6125af", 90 | "metadata": {}, 91 | "outputs": [], 92 | "source": [ 93 | "%%capture\n", 94 | "whats_below = \"a subset of the table IDs. Select one. \\nNote that there are over 30, so we chose a subset\"\n", 95 | "table_ids = [\n", 96 | " \"Amon\",\n", 97 | " \"day\",\n", 98 | " \"6hrPlev\",\n", 99 | " \"EmonZ\",\n", 100 | " \"3hr\"\n", 101 | "]\n", 102 | "table_id_dict = {\n", 103 | " \"monthly atmostphere\": \"Amon\",\n", 104 | " \"daily\": \"day\",\n", 105 | " \"three hours\": \"3hr\",\n", 106 | " \"pressure level value 6 hours\": \"6hrPlev\",\n", 107 | "}\n", 108 | "table_ids_widget = []\n", 109 | "for table_id_key in table_id_dict:\n", 110 | " table_ids_widget.append((table_id_key, table_id_dict[table_id_key]))\n", 111 | "# table_ids_widget\n", 112 | "table_id = table_ids_widget[0][1]\n", 113 | "%store table_id" 114 | ] 115 | }, 116 | { 117 | "cell_type": "code", 118 | "execution_count": null, 119 | "id": "ad2a09f4", 120 | "metadata": {}, 121 | "outputs": [], 122 | "source": [ 123 | "w = widgets.Dropdown(\n", 124 | " options=table_ids_widget,\n", 125 | " value=table_ids_widget[0][1],\n", 126 | " description=\"table ids: \",\n", 127 | ")\n", 128 | "\n", 129 | "out = widgets.Output()\n", 130 | "display(out)\n", 131 | "# table_id = table_ids_widget[0][1]\n", 132 | "\n", 133 | "@out.capture()\n", 134 | "def on_change(change):\n", 135 | "# print('method is called when printing this')\n", 136 | " if change['type'] == 'change' and change['name'] == 'value':\n", 137 | " global table_id\n", 138 | " table_id = change.new\n", 139 | " %store table_id\n", 140 | " print(\"table id value changed to: \" + table_id)\n", 141 | "w.observe(on_change)\n", 142 | "\n", 143 | "\n", 144 | "print_list(whats_below, [])\n", 145 | "display(w)\n" 146 | ] 147 | }, 148 | { 149 | "cell_type": "code", 150 | "execution_count": null, 151 | "id": "ec24cf44", 152 | "metadata": {}, 153 | "outputs": [], 154 | "source": [ 155 | "%%capture\n", 156 | "# variable id\n", 157 | "whats_below = \"a subset of the variable IDs. \\nNote that there are over 700 so we are only displaying a subset. \\\n", 158 | "\\n\\nSelect all that apply.\"\n", 159 | "variable_ids = [\n", 160 | " \"tas\", # tas is air temperature at 2m above surface\n", 161 | " \"ta\",\n", 162 | " \"tauv\",\n", 163 | " \"zg\",\n", 164 | " \"rsut\"\n", 165 | "]\n", 166 | "\n", 167 | "\n", 168 | "variable_id = [variable_ids[0]]\n", 169 | "%store variable_id" 170 | ] 171 | }, 172 | { 173 | "cell_type": "code", 174 | "execution_count": null, 175 | "id": "f12d7c55", 176 | "metadata": {}, 177 | "outputs": [], 178 | "source": [ 179 | "w = widgets.SelectMultiple(\n", 180 | " options=variable_ids,\n", 181 | " value=[variable_ids[0]],\n", 182 | " description=\"activity: \",\n", 183 | ")\n", 184 | "out = widgets.Output()\n", 185 | "# display(out)\n", 186 | "\n", 187 | "@out.capture()\n", 188 | "def on_change(change):\n", 189 | " print('method is called when printing this')\n", 190 | " if change['type'] == 'change' and change['name'] == 'value':\n", 191 | " global variable_id\n", 192 | " variable_id = list(change.new)\n", 193 | " %store variable_id\n", 194 | "w.observe(on_change)\n", 195 | "\n", 196 | "\n", 197 | "# whats_below = \"all activity IDs\"\n", 198 | "print_list(whats_below, [])\n", 199 | "display(w)" 200 | ] 201 | } 202 | ], 203 | "metadata": { 204 | "kernelspec": { 205 | "display_name": "conda_python3", 206 | "language": "python", 207 | "name": "conda_python3" 208 | }, 209 | "language_info": { 210 | "codemirror_mode": { 211 | "name": "ipython", 212 | "version": 3 213 | }, 214 | "file_extension": ".py", 215 | "mimetype": "text/x-python", 216 | "name": "python", 217 | "nbconvert_exporter": "python", 218 | "pygments_lexer": "ipython3", 219 | "version": "3.8.12" 220 | } 221 | }, 222 | "nbformat": 4, 223 | "nbformat_minor": 5 224 | } 225 | -------------------------------------------------------------------------------- /lib/SagemakerCode/get_data.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "e9984a95", 6 | "metadata": {}, 7 | "source": [ 8 | "# Get the data\n", 9 | "This notebook retrieves the user defined inputs, and executes the correct notebook depending on what the user wants" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "id": "085dd87d", 16 | "metadata": {}, 17 | "outputs": [], 18 | "source": [ 19 | "%store -r data_type\n", 20 | "print(data_type)" 21 | ] 22 | }, 23 | { 24 | "cell_type": "code", 25 | "execution_count": null, 26 | "id": "e1116548", 27 | "metadata": {}, 28 | "outputs": [], 29 | "source": [ 30 | "cost_information = '\\033[1m' + \"The below information is for demonstration only.\" + '\\033[0m' + \" \\nThe total cost for this run was: $X \\nThe amount you have remaining in your account is: $Y \\nTo upgrade, click this link\"\n", 31 | "%store cost_information" 32 | ] 33 | }, 34 | { 35 | "cell_type": "code", 36 | "execution_count": null, 37 | "id": "54dac374", 38 | "metadata": {}, 39 | "outputs": [], 40 | "source": [ 41 | "# %%capture\n", 42 | "import boto3\n", 43 | "client = boto3.client('ssm')\n", 44 | "\n", 45 | "openSearch = client.get_parameter(\n", 46 | " Name='client-opensearch-domain-' + client.meta.region_name,\n", 47 | ")\n", 48 | "host = openSearch['Parameter']['Value']\n", 49 | "%store host" 50 | ] 51 | }, 52 | { 53 | "cell_type": "code", 54 | "execution_count": null, 55 | "id": "f73f271d", 56 | "metadata": {}, 57 | "outputs": [], 58 | "source": [ 59 | "if data_type == \"historical\":\n", 60 | " %run get_historical_data.ipynb\n", 61 | "elif data_type == \"predictive\":\n", 62 | " %run get_predictive_data.ipynb\n", 63 | "elif data_type == \"both\":\n", 64 | " %run get_historical_data.ipynb\n", 65 | " %run get_predictive_data.ipynb" 66 | ] 67 | } 68 | ], 69 | "metadata": { 70 | "kernelspec": { 71 | "display_name": "conda_python3", 72 | "language": "python", 73 | "name": "conda_python3" 74 | }, 75 | "language_info": { 76 | "codemirror_mode": { 77 | "name": "ipython", 78 | "version": 3 79 | }, 80 | "file_extension": ".py", 81 | "mimetype": "text/x-python", 82 | "name": "python", 83 | "nbconvert_exporter": "python", 84 | "pygments_lexer": "ipython3", 85 | "version": "3.8.12" 86 | } 87 | }, 88 | "nbformat": 4, 89 | "nbformat_minor": 5 90 | } 91 | -------------------------------------------------------------------------------- /lib/SagemakerCode/get_historical_data.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "c5084494", 6 | "metadata": {}, 7 | "source": [ 8 | "# Processing Example for AWS illustrating a simple Algorithm to Test Infrastucture\n", 9 | "\n", 10 | "This is based on using an environment simliar to the one that is created from the ASDI CMPI example: https://github.com/awslabs/amazon-asdi/tree/main/examples/cmip6 (this needs to run in us-east-1 , I think as the CMPI references failed from London)\n", 11 | "\n", 12 | "If you want to run this you may need to install some addtional libraries using ```conda install``` from a Terminal" 13 | ] 14 | }, 15 | { 16 | "cell_type": "code", 17 | "execution_count": null, 18 | "id": "c662e3d2", 19 | "metadata": {}, 20 | "outputs": [], 21 | "source": [ 22 | "%matplotlib inline\n", 23 | "import xarray as xr\n", 24 | "import pandas as pd\n", 25 | "import numpy as np\n", 26 | "import matplotlib.pyplot as plt\n", 27 | "import matplotlib\n", 28 | "import intake\n", 29 | "import boto3\n", 30 | "import botocore\n", 31 | "import datetime\n", 32 | "import s3fs\n", 33 | "import fsspec\n", 34 | "import dask\n", 35 | "#import sys\n", 36 | "import lz4\n", 37 | "from opensearchpy import OpenSearch, RequestsHttpConnection, AWSV4SignerAuth\n", 38 | "from dask.distributed import performance_report, Client, progress, LocalCluster\n", 39 | "\n", 40 | "font = {'family' : 'sans-serif',\n", 41 | " 'weight' : 'normal',\n", 42 | " 'size' : 18}\n", 43 | "matplotlib.rc('font', **font)" 44 | ] 45 | }, 46 | { 47 | "cell_type": "code", 48 | "execution_count": null, 49 | "id": "f6f90e26", 50 | "metadata": {}, 51 | "outputs": [], 52 | "source": [ 53 | "import json\n", 54 | "import sys\n", 55 | "import boto3\n", 56 | "import os\n", 57 | "from boto3.dynamodb.conditions import Key, Attr" 58 | ] 59 | }, 60 | { 61 | "cell_type": "code", 62 | "execution_count": null, 63 | "id": "b089a11a", 64 | "metadata": {}, 65 | "outputs": [], 66 | "source": [ 67 | "#some paramters for regridding\n", 68 | "regrid_lon=np.arange(0.0,360.0,0.1)\n", 69 | "regrid_lat=np.arange(-90.0,90.0,0.1)\n", 70 | "regrid_method='slinear'" 71 | ] 72 | }, 73 | { 74 | "cell_type": "markdown", 75 | "id": "b51e0073", 76 | "metadata": {}, 77 | "source": [ 78 | "# Connect to Dask cluster scheduler\n" 79 | ] 80 | }, 81 | { 82 | "cell_type": "code", 83 | "execution_count": null, 84 | "id": "111e6bb6", 85 | "metadata": {}, 86 | "outputs": [], 87 | "source": [ 88 | "import dask.array as da\n", 89 | "from dask_worker_pools import pool, propagate_pools, visualize_pools\n" 90 | ] 91 | }, 92 | { 93 | "cell_type": "code", 94 | "execution_count": null, 95 | "id": "0e5d843c", 96 | "metadata": {}, 97 | "outputs": [], 98 | "source": [ 99 | "from dask.distributed import Client\n", 100 | "import lz4\n", 101 | "# Client.get_versions('self', check=True)\n", 102 | "client = Client('Dask-Scheduler.local-dask:8786')\n", 103 | "# client = Client('Dask-Scheduler.local-dask:8786',serializers=['dask', 'pickle'],\n", 104 | "# deserializers=['dask', 'pickle']\n", 105 | "# )" 106 | ] 107 | }, 108 | { 109 | "cell_type": "code", 110 | "execution_count": null, 111 | "id": "cdf74138", 112 | "metadata": {}, 113 | "outputs": [], 114 | "source": [ 115 | "client.scheduler_info()['workers']" 116 | ] 117 | }, 118 | { 119 | "cell_type": "code", 120 | "execution_count": null, 121 | "id": "a1ac38b5", 122 | "metadata": {}, 123 | "outputs": [], 124 | "source": [ 125 | "%store -r activity_id\n", 126 | "%store -r variable_id\n", 127 | "%store -r table_id\n", 128 | "variable_ids = variable_id # tas is air temperature at 2m above surface\n", 129 | "table_id = table_id # Monthly data from Atmosphere - would really like this to be daily, but run out of memeory in client ('day' is the id)\n", 130 | "grid = 'gn' #\n", 131 | "\n", 132 | "# Records for Institution, experiment, and source_id are stored in https://github.com/WCRP-CMIP/CMIP6_CVs\n", 133 | "experiment_id = 'ssp245' #['ssp126', 'ssp245', 'ssp370', 'ssp585'] \n", 134 | "activity_ids = activity_id # Search Scenarios & CMIP activities only\n", 135 | "institution_id = 'MOHC' #just looking at our data in this example\n", 136 | "\n", 137 | "print(activity_id)\n", 138 | "print(variable_id)\n", 139 | "print(table_id)" 140 | ] 141 | }, 142 | { 143 | "cell_type": "code", 144 | "execution_count": null, 145 | "id": "688daa32", 146 | "metadata": {}, 147 | "outputs": [], 148 | "source": [ 149 | "session = boto3.session.Session()\n", 150 | "my_region = session.region_name" 151 | ] 152 | }, 153 | { 154 | "cell_type": "markdown", 155 | "id": "2e04920c", 156 | "metadata": {}, 157 | "source": [ 158 | "# Get the historic record" 159 | ] 160 | }, 161 | { 162 | "cell_type": "markdown", 163 | "id": "c4000774", 164 | "metadata": {}, 165 | "source": [ 166 | "### Search ERA metadata from catalog" 167 | ] 168 | }, 169 | { 170 | "cell_type": "markdown", 171 | "id": "0eb31423", 172 | "metadata": {}, 173 | "source": [ 174 | "#### Now we use the store from the ux testing file to actually get the desired attribute" 175 | ] 176 | }, 177 | { 178 | "cell_type": "code", 179 | "execution_count": null, 180 | "id": "71bcd3e8", 181 | "metadata": {}, 182 | "outputs": [], 183 | "source": [ 184 | "%store -r host\n", 185 | "credentials = boto3.Session().get_credentials()\n", 186 | "auth = AWSV4SignerAuth(credentials, my_region)\n", 187 | "index_name = 'era5-pds'\n" 188 | ] 189 | }, 190 | { 191 | "cell_type": "code", 192 | "execution_count": null, 193 | "id": "da623e12", 194 | "metadata": {}, 195 | "outputs": [], 196 | "source": [ 197 | "opensearch_client = OpenSearch(\n", 198 | " hosts = [{'host': host, 'port': 443}],\n", 199 | " http_auth = auth,\n", 200 | " use_ssl = True,\n", 201 | " verify_certs = True,\n", 202 | " connection_class = RequestsHttpConnection\n", 203 | " )" 204 | ] 205 | }, 206 | { 207 | "cell_type": "code", 208 | "execution_count": null, 209 | "id": "3e6ab4ba", 210 | "metadata": {}, 211 | "outputs": [], 212 | "source": [ 213 | "\n", 214 | "\n", 215 | "def query_nc(q, index_name):\n", 216 | " \n", 217 | " queryy = {\n", 218 | " 'size': 1,\n", 219 | " 'query': {\n", 220 | " 'multi_match': {\n", 221 | " 'query': q,\n", 222 | " 'fields': ['fileName']\n", 223 | " }\n", 224 | " }\n", 225 | " }\n", 226 | "\n", 227 | " respons = opensearch_client.search(\n", 228 | " body = queryy,\n", 229 | " index = index_name\n", 230 | " )\n", 231 | " \n", 232 | " res = [i['_source']['fileName'] for i in respons['hits']['hits']][0]\n", 233 | " d_pool = [d['_source']['dask_pool'] for d in respons['hits']['hits']]\n", 234 | " regio = [f['_source']['region'] for f in respons['hits']['hits']]\n", 235 | " regio = list(set(regio))[0]\n", 236 | " d_pool = list(set(d_pool))[0]\n", 237 | " return res, regio, d_pool\n" 238 | ] 239 | }, 240 | { 241 | "cell_type": "code", 242 | "execution_count": null, 243 | "id": "196f4b34", 244 | "metadata": {}, 245 | "outputs": [], 246 | "source": [ 247 | "import urllib\n", 248 | "\n", 249 | "%store -r start_date end_date\n", 250 | "\n", 251 | "start_year = int(start_date.year) #really want this to be 2010\n", 252 | "end_year = int(end_date.year)\n", 253 | "\n", 254 | "lustre_mount_point = \"/fsx\"\n", 255 | "years = list(np.arange(start_year, end_year+1, 1))\n", 256 | "\n", 257 | "# todo: update how we handle this\n", 258 | "months = [\"01\", \"02\", \"03\", \"04\", \"05\", \"06\", \"07\", \"08\", \"09\", \"10\", \"11\", \"12\"] \n" 259 | ] 260 | }, 261 | { 262 | "cell_type": "code", 263 | "execution_count": null, 264 | "id": "545f95f5", 265 | "metadata": {}, 266 | "outputs": [], 267 | "source": [ 268 | "%store -r desired_attribute\n", 269 | "attr1 = desired_attribute\n", 270 | "attr1" 271 | ] 272 | }, 273 | { 274 | "cell_type": "code", 275 | "execution_count": null, 276 | "id": "5d50e9cd", 277 | "metadata": {}, 278 | "outputs": [], 279 | "source": [ 280 | "%store -r start_date\n", 281 | "start_date.month" 282 | ] 283 | }, 284 | { 285 | "cell_type": "code", 286 | "execution_count": null, 287 | "id": "072d504c", 288 | "metadata": {}, 289 | "outputs": [], 290 | "source": [ 291 | "n_list = []\n", 292 | "from dateutil.relativedelta import relativedelta\n", 293 | "current_date = start_date\n", 294 | "\n", 295 | "while current_date <= end_date:\n", 296 | "# print(current_date.month)\n", 297 | "# datem = datetime.datetime.strptime(datetime.datetime.strftime(current_date, \"%Y-%M-%d\"), \"%Y-%m-%d\")\n", 298 | " current_month = str(current_date.month)\n", 299 | " if len(current_month) == 1:\n", 300 | " current_month = \"0\" + current_month\n", 301 | " item = '{}/{}/{}/{}/{}.nc'.format(index_name, current_date.year, current_month, 'data',attr1)\n", 302 | "# print(item)\n", 303 | " n_list.append(item)\n", 304 | " current_date = current_date + relativedelta(months=1)\n", 305 | "# n_list2[0])" 306 | ] 307 | }, 308 | { 309 | "cell_type": "code", 310 | "execution_count": null, 311 | "id": "78b1b63f", 312 | "metadata": {}, 313 | "outputs": [], 314 | "source": [ 315 | "region_dask_pool = []\n", 316 | "region = []\n", 317 | "nc_list = []\n", 318 | "\n", 319 | "for nc in n_list:\n", 320 | " fileName, regn, dask_pool = query_nc(nc, index_name)\n", 321 | " nc_list.append(fileName)\n", 322 | " region_dask_pool.append(dask_pool)\n", 323 | " region.append(regn)\n", 324 | "\n", 325 | "region = list(set(region))[0]\n", 326 | "region_dask_pool = list(set(region_dask_pool))[0]\n", 327 | "print(nc_list)" 328 | ] 329 | }, 330 | { 331 | "cell_type": "code", 332 | "execution_count": null, 333 | "id": "0eafbe22", 334 | "metadata": {}, 335 | "outputs": [], 336 | "source": [ 337 | "historical_pool_region = 'us-east-1'\n", 338 | "%store historical_pool_region" 339 | ] 340 | }, 341 | { 342 | "cell_type": "code", 343 | "execution_count": null, 344 | "id": "116c27ef", 345 | "metadata": {}, 346 | "outputs": [], 347 | "source": [ 348 | "%%time\n", 349 | "with pool(historical_pool_region):\n", 350 | " historical_data = xr.open_mfdataset(nc_list, engine='h5netcdf', concat_dim='time0', combine='nested', coords='minimal', compat='override', parallel=True, chunks={'lon':200,'lat':200,'time0':720})\n", 351 | " %store historical_data" 352 | ] 353 | } 354 | ], 355 | "metadata": { 356 | "kernelspec": { 357 | "display_name": "conda_python3", 358 | "language": "python", 359 | "name": "conda_python3" 360 | }, 361 | "language_info": { 362 | "codemirror_mode": { 363 | "name": "ipython", 364 | "version": 3 365 | }, 366 | "file_extension": ".py", 367 | "mimetype": "text/x-python", 368 | "name": "python", 369 | "nbconvert_exporter": "python", 370 | "pygments_lexer": "ipython3", 371 | "version": "3.8.12" 372 | } 373 | }, 374 | "nbformat": 4, 375 | "nbformat_minor": 5 376 | } 377 | -------------------------------------------------------------------------------- /lib/SagemakerCode/get_predictive_data.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "4d7851e5", 6 | "metadata": {}, 7 | "source": [ 8 | "# Processing Example for AWS illustrating a simple Algorithm to Test Infrastucture\n", 9 | "\n", 10 | "This is based on using an environment simliar to the one that is created from the ASDI CMPI example: https://github.com/awslabs/amazon-asdi/tree/main/examples/cmip6 (this needs to run in us-east-1 , I think as the CMPI references failed from London)\n", 11 | "\n", 12 | "If you want to run this you may need to install some addtional libraries using ```conda install``` from a Terminal" 13 | ] 14 | }, 15 | { 16 | "cell_type": "code", 17 | "execution_count": null, 18 | "id": "5f51b52a", 19 | "metadata": {}, 20 | "outputs": [], 21 | "source": [ 22 | "%matplotlib inline\n", 23 | "import xarray as xr\n", 24 | "import pandas as pd\n", 25 | "import numpy as np\n", 26 | "import matplotlib.pyplot as plt\n", 27 | "import matplotlib\n", 28 | "import intake\n", 29 | "import boto3\n", 30 | "import botocore\n", 31 | "import datetime\n", 32 | "import s3fs\n", 33 | "import fsspec\n", 34 | "import dask\n", 35 | "#import sys\n", 36 | "#import os\n", 37 | "from opensearchpy import OpenSearch, RequestsHttpConnection, AWSV4SignerAuth\n", 38 | "from dask.distributed import performance_report, Client, progress, LocalCluster\n", 39 | "\n", 40 | "font = {'family' : 'sans-serif',\n", 41 | " 'weight' : 'normal',\n", 42 | " 'size' : 18}\n", 43 | "matplotlib.rc('font', **font)" 44 | ] 45 | }, 46 | { 47 | "cell_type": "code", 48 | "execution_count": null, 49 | "id": "cfa1ef27", 50 | "metadata": {}, 51 | "outputs": [], 52 | "source": [ 53 | "import json\n", 54 | "import sys\n", 55 | "import boto3\n", 56 | "import os\n", 57 | "from boto3.dynamodb.conditions import Key, Attr" 58 | ] 59 | }, 60 | { 61 | "cell_type": "code", 62 | "execution_count": null, 63 | "id": "d1ce98b4", 64 | "metadata": {}, 65 | "outputs": [], 66 | "source": [ 67 | "#some paramters for regridding\n", 68 | "regrid_lon=np.arange(0.0,360.0,0.1)\n", 69 | "regrid_lat=np.arange(-90.0,90.0,0.1)\n", 70 | "regrid_method='slinear'" 71 | ] 72 | }, 73 | { 74 | "cell_type": "markdown", 75 | "id": "f827ab0f", 76 | "metadata": {}, 77 | "source": [ 78 | "# Connect to Dask cluster scheduler\n" 79 | ] 80 | }, 81 | { 82 | "cell_type": "code", 83 | "execution_count": null, 84 | "id": "18b9286f", 85 | "metadata": {}, 86 | "outputs": [], 87 | "source": [ 88 | "import dask.array as da\n", 89 | "from dask_worker_pools import pool, propagate_pools, visualize_pools\n" 90 | ] 91 | }, 92 | { 93 | "cell_type": "code", 94 | "execution_count": null, 95 | "id": "d630d81f", 96 | "metadata": {}, 97 | "outputs": [], 98 | "source": [ 99 | "from dask.distributed import Client\n", 100 | "import lz4\n", 101 | "# Client.get_versions('self', check=True)\n", 102 | "client = Client('Dask-Scheduler.local-dask:8786')\n", 103 | "# client = Client('Dask-Scheduler.local-dask:8786',serializers=['dask', 'pickle'],\n", 104 | "# deserializers=['dask', 'pickle']\n", 105 | "# )" 106 | ] 107 | }, 108 | { 109 | "cell_type": "code", 110 | "execution_count": null, 111 | "id": "797c4514", 112 | "metadata": {}, 113 | "outputs": [], 114 | "source": [ 115 | "client.scheduler_info()['workers']" 116 | ] 117 | }, 118 | { 119 | "cell_type": "markdown", 120 | "id": "02991bbc", 121 | "metadata": {}, 122 | "source": [ 123 | "# Get the Climate Prediction" 124 | ] 125 | }, 126 | { 127 | "cell_type": "markdown", 128 | "id": "62779549", 129 | "metadata": {}, 130 | "source": [ 131 | "Using the ASDI CMPI6 data which has a ZARR index we can select some data from a scenario. (this is adapted from examples in https://github.com/awslabs/amazon-asdi/tree/main/examples )" 132 | ] 133 | }, 134 | { 135 | "cell_type": "code", 136 | "execution_count": null, 137 | "id": "d414cde1", 138 | "metadata": {}, 139 | "outputs": [], 140 | "source": [ 141 | "catalog = intake.open_esm_datastore('https://cmip6-pds.s3.amazonaws.com/pangeo-cmip6.json')" 142 | ] 143 | }, 144 | { 145 | "cell_type": "code", 146 | "execution_count": null, 147 | "id": "d231271f", 148 | "metadata": {}, 149 | "outputs": [], 150 | "source": [ 151 | "%store -r activity_id\n", 152 | "%store -r variable_id\n", 153 | "%store -r table_id\n", 154 | "variable_ids = variable_id # tas is air temperature at 2m above surface\n", 155 | "table_id = table_id # Monthly data from Atmosphere - would really like this to be daily, but run out of memeory in client ('day' is the id)\n", 156 | "grid = 'gn' #\n", 157 | "\n", 158 | "# Records for Institution, experiment, and source_id are stored in https://github.com/WCRP-CMIP/CMIP6_CVs\n", 159 | "experiment_id = 'ssp245' #['ssp126', 'ssp245', 'ssp370', 'ssp585'] \n", 160 | "activity_ids = activity_id # Search Scenarios & CMIP activities only\n", 161 | "institution_id = 'MOHC' #just looking at our data in this example\n", 162 | "\n", 163 | "print(activity_id)\n", 164 | "print(variable_id)\n", 165 | "print(table_id)" 166 | ] 167 | }, 168 | { 169 | "cell_type": "code", 170 | "execution_count": null, 171 | "id": "e27ffc6e", 172 | "metadata": {}, 173 | "outputs": [], 174 | "source": [ 175 | "res = catalog.search(activity_id=activity_ids, experiment_id=experiment_id, variable_id=variable_ids, grid_label=grid, table_id=table_id, institution_id=institution_id)\n", 176 | "display(res.df)" 177 | ] 178 | }, 179 | { 180 | "cell_type": "code", 181 | "execution_count": null, 182 | "id": "592568af", 183 | "metadata": {}, 184 | "outputs": [], 185 | "source": [ 186 | "session = boto3.session.Session()\n", 187 | "my_region = session.region_name" 188 | ] 189 | }, 190 | { 191 | "cell_type": "code", 192 | "execution_count": null, 193 | "id": "1b658023", 194 | "metadata": {}, 195 | "outputs": [], 196 | "source": [ 197 | "%store -r host\n", 198 | "credentials = boto3.Session().get_credentials()\n", 199 | "auth = AWSV4SignerAuth(credentials, my_region)\n", 200 | "index_name = 'cmip6-pds' ##Update Index name as needed" 201 | ] 202 | }, 203 | { 204 | "cell_type": "code", 205 | "execution_count": null, 206 | "id": "b5ba834d", 207 | "metadata": {}, 208 | "outputs": [], 209 | "source": [ 210 | "opensearch_client = OpenSearch(\n", 211 | " hosts = [{'host': host, 'port': 443}],\n", 212 | " http_auth = auth,\n", 213 | " use_ssl = True,\n", 214 | " verify_certs = True,\n", 215 | " connection_class = RequestsHttpConnection\n", 216 | " )\n", 217 | "\n", 218 | "def search_cmip_query(q):\n", 219 | " \n", 220 | " queryy = {\n", 221 | " 'size': 5,\n", 222 | " 'query': {\n", 223 | " 'multi_match': {\n", 224 | " 'query': q,\n", 225 | " 'fields': ['fileName']\n", 226 | " }\n", 227 | " }\n", 228 | " }\n", 229 | "\n", 230 | " respons = opensearch_client.search(\n", 231 | " body = queryy,\n", 232 | " index = index_name\n", 233 | " )\n", 234 | " \n", 235 | " res = [i['_source']['fileName'] for i in respons['hits']['hits']]\n", 236 | " d_pool = [d['_source']['dask_pool'] for d in respons['hits']['hits']]\n", 237 | " regio = [f['_source']['region'] for f in respons['hits']['hits']]\n", 238 | " res_fil = list(set([r.split(q.split('/')[-2])[0]+q.split('/')[-2]+\"/\" for r in res]))[0]\n", 239 | " regio = list(set(regio))[0]\n", 240 | " d_pool = list(set(d_pool))[0]\n", 241 | " return res_fil, regio, d_pool" 242 | ] 243 | }, 244 | { 245 | "cell_type": "code", 246 | "execution_count": null, 247 | "id": "696ac6a0", 248 | "metadata": {}, 249 | "outputs": [], 250 | "source": [ 251 | "res.df['zstore']" 252 | ] 253 | }, 254 | { 255 | "cell_type": "code", 256 | "execution_count": null, 257 | "id": "ed9ba27a", 258 | "metadata": {}, 259 | "outputs": [], 260 | "source": [ 261 | "data_region = []\n", 262 | "dask_pool = []\n", 263 | "index_name" 264 | ] 265 | }, 266 | { 267 | "cell_type": "code", 268 | "execution_count": null, 269 | "id": "bec9f741", 270 | "metadata": {}, 271 | "outputs": [], 272 | "source": [ 273 | "for s3_path in res.df['zstore']:\n", 274 | "# print(s3_path)\n", 275 | " query_param = s3_path.split(index_name+'/')[1]\n", 276 | " local_pth, regi, d_pool = search_cmip_query(query_param)\n", 277 | "# print(regi)\n", 278 | " data_region.append(regi)\n", 279 | " dask_pool.append(d_pool)\n", 280 | " res.df['zstore'] = res.df['zstore'].replace([s3_path], local_pth)\n", 281 | "\n", 282 | "data_region = list(set(data_region))[0]\n", 283 | "region_dask_pool = list(set(dask_pool))[0]" 284 | ] 285 | }, 286 | { 287 | "cell_type": "code", 288 | "execution_count": null, 289 | "id": "02c8b757", 290 | "metadata": {}, 291 | "outputs": [], 292 | "source": [ 293 | "\n", 294 | "print(\"Region: {}\".format(data_region))\n", 295 | "print(\"Dask Pool: {}\".format(dask_pool))" 296 | ] 297 | }, 298 | { 299 | "cell_type": "code", 300 | "execution_count": null, 301 | "id": "75c4bc0a", 302 | "metadata": {}, 303 | "outputs": [], 304 | "source": [ 305 | "display(res.df)" 306 | ] 307 | }, 308 | { 309 | "cell_type": "code", 310 | "execution_count": null, 311 | "id": "36a38489", 312 | "metadata": {}, 313 | "outputs": [], 314 | "source": [ 315 | "files_mapper = res.df['zstore'].tolist()\n" 316 | ] 317 | }, 318 | { 319 | "cell_type": "code", 320 | "execution_count": null, 321 | "id": "a9ae3c6b", 322 | "metadata": {}, 323 | "outputs": [], 324 | "source": [ 325 | "files_mapper" 326 | ] 327 | }, 328 | { 329 | "cell_type": "code", 330 | "execution_count": null, 331 | "id": "e89f2125", 332 | "metadata": {}, 333 | "outputs": [], 334 | "source": [ 335 | "prediction_pool_region = 'us-west-2'\n", 336 | "%store prediction_pool_region" 337 | ] 338 | }, 339 | { 340 | "cell_type": "code", 341 | "execution_count": null, 342 | "id": "88ba3021", 343 | "metadata": {}, 344 | "outputs": [], 345 | "source": [ 346 | "\n", 347 | "\n", 348 | "%%time\n", 349 | "with pool(prediction_pool_region):\n", 350 | " datasets = xr.open_mfdataset(files_mapper, engine='zarr', parallel=True, decode_times=True, consolidated=True)\n", 351 | " #datasets = res.to_dataset_dict(zarr_kwargs={'consolidated': True, 'decode_times': True})" 352 | ] 353 | }, 354 | { 355 | "cell_type": "code", 356 | "execution_count": null, 357 | "id": "2713f89f", 358 | "metadata": {}, 359 | "outputs": [], 360 | "source": [ 361 | "predictive_data_set = xr.Dataset()\n", 362 | "\n", 363 | "\n", 364 | "# just select the tail of the date over time (2090-2100)\n", 365 | "for i in datasets:\n", 366 | " ds = datasets[i]\n", 367 | " print(ds)\n", 368 | " total_times = ds['time'].size\n", 369 | " start_index = total_times - (10*12) #this is in months. REally want it to be days\n", 370 | " ds2 = ds.isel(time=np.arange(start_index,total_times)) #last 10 years of data\n", 371 | " predictive_data_set = xr.merge([predictive_data_set, ds2], compat='override')\n", 372 | "\n", 373 | "predictive_data_set\n", 374 | "%store predictive_data_set\n", 375 | "\n" 376 | ] 377 | }, 378 | { 379 | "cell_type": "code", 380 | "execution_count": null, 381 | "id": "7e7ff885", 382 | "metadata": {}, 383 | "outputs": [], 384 | "source": [ 385 | "cost_information = '\\033[1m' + \"The below information is for demonstration only.\" + '\\033[0m' + \" \\nThe total cost for this run was: $X \\nThe amount you have remaining in your account is: $Y \\nTo upgrade, click this link\"\n", 386 | "%store cost_information" 387 | ] 388 | } 389 | ], 390 | "metadata": { 391 | "kernelspec": { 392 | "display_name": "conda_python3", 393 | "language": "python", 394 | "name": "conda_python3" 395 | }, 396 | "language_info": { 397 | "codemirror_mode": { 398 | "name": "ipython", 399 | "version": 3 400 | }, 401 | "file_extension": ".py", 402 | "mimetype": "text/x-python", 403 | "name": "python", 404 | "nbconvert_exporter": "python", 405 | "pygments_lexer": "ipython3", 406 | "version": "3.8.12" 407 | } 408 | }, 409 | "nbformat": 4, 410 | "nbformat_minor": 5 411 | } 412 | -------------------------------------------------------------------------------- /lib/SagemakerCode/get_variables.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "c7c68e4e", 6 | "metadata": {}, 7 | "source": [ 8 | "### Now lets take a look at the available variable types" 9 | ] 10 | }, 11 | { 12 | "cell_type": "code", 13 | "execution_count": null, 14 | "id": "81a28ffd", 15 | "metadata": {}, 16 | "outputs": [], 17 | "source": [ 18 | "from helpers import print_list, create_dropdown\n", 19 | "import ipywidgets as widgets" 20 | ] 21 | }, 22 | { 23 | "cell_type": "code", 24 | "execution_count": null, 25 | "id": "1fe01df5", 26 | "metadata": {}, 27 | "outputs": [], 28 | "source": [ 29 | "%%capture\n", 30 | "data_options = ['predictive','historical', 'both']\n", 31 | "data_type = data_options[0]\n", 32 | "%store data_type" 33 | ] 34 | }, 35 | { 36 | "cell_type": "code", 37 | "execution_count": null, 38 | "id": "d24a308e", 39 | "metadata": {}, 40 | "outputs": [], 41 | "source": [ 42 | "w = widgets.RadioButtons(\n", 43 | " options=data_options,\n", 44 | " description='Data type:',\n", 45 | " disabled=False\n", 46 | ")\n", 47 | "\n", 48 | "\n", 49 | "out = widgets.Output()\n", 50 | "# display(out)\n", 51 | "\n", 52 | "@out.capture()\n", 53 | "def on_change(change):\n", 54 | "# print('method is called when printing this')\n", 55 | " if change['type'] == 'change' and change['name'] == 'value':\n", 56 | " global data_type\n", 57 | " data_type = change.new\n", 58 | " %store data_type\n", 59 | "# print(\"data type changed to: \" + data_type)\n", 60 | "w.observe(on_change)\n", 61 | "\n", 62 | "\n", 63 | "print_list(\"the options for the data type.\", [])\n", 64 | "display(w)" 65 | ] 66 | }, 67 | { 68 | "cell_type": "code", 69 | "execution_count": null, 70 | "id": "997da781", 71 | "metadata": {}, 72 | "outputs": [], 73 | "source": [ 74 | "# print('\\033[1m' + \"Here are the available variable categories:\" '\\033[1m' + )\n", 75 | "variable_types = [\n", 76 | " \"temperature\",\n", 77 | " \"wind\",\n", 78 | " \"wave\",\n", 79 | " \"precipitation\",\n", 80 | " \"snow\",\n", 81 | " \"air_pressure\"\n", 82 | "]\n", 83 | "\n", 84 | "# w = create_dropdown(variable_types, widgets, \"variable type: \")\n" 85 | ] 86 | }, 87 | { 88 | "cell_type": "code", 89 | "execution_count": null, 90 | "id": "9cc850fa", 91 | "metadata": {}, 92 | "outputs": [], 93 | "source": [ 94 | "variable_type = print_list(\"available variable categories\", variable_types)\n", 95 | "variable_type" 96 | ] 97 | }, 98 | { 99 | "cell_type": "code", 100 | "execution_count": null, 101 | "id": "837148f8", 102 | "metadata": {}, 103 | "outputs": [], 104 | "source": [ 105 | "%%capture\n", 106 | "allowed_variables = [\n", 107 | " \"air_temperature_at_2_metres\",\n", 108 | " \"air_pressure_at_mean_sea_level\",\n", 109 | " \"eastward_wind_at_10_metres\", \n", 110 | " \"sea_surface_temperature\",\n", 111 | " \"air_temperature_at_2_metres\", \n", 112 | " \"integral_wrt_time_of_surface_direct_downwelling_shortwave_flux_in_air_1hour_Accumulation\", \n", 113 | " \"sea_surface_wave_from_direction\",\n", 114 | " \"air_temperature_at_2_metres_1hour_Maximum\", \n", 115 | " \"lwe_thickness_of_surface_snow_amount\", \n", 116 | " \"sea_surface_wave_mean_period\",\n", 117 | " \"air_temperature_at_2_metres_1hour_Minimum\", \n", 118 | " \"northward_wind_at_100_metres\", \n", 119 | " \"significant_height_of_wind_and_swell_waves\",\n", 120 | " \"dew_point_temperature_at_2_metres\", \n", 121 | " \"northward_wind_at_10_metres\", \n", 122 | " \"snow_density\",\n", 123 | " \"eastward_wind_at_100_metres\", \n", 124 | " \"precipitation_amount_1hour_Accumulation\", \n", 125 | " \"surface_air_pressure\", \n", 126 | "]\n", 127 | "\n", 128 | "# %store -r variable_type\n", 129 | "\n", 130 | "\n", 131 | "filtered_vars = [x for x in allowed_variables if variable_type in x]\n", 132 | "desired_attribute = filtered_vars[0]\n", 133 | "%store desired_attribute" 134 | ] 135 | }, 136 | { 137 | "cell_type": "code", 138 | "execution_count": null, 139 | "id": "bc009163", 140 | "metadata": {}, 141 | "outputs": [], 142 | "source": [ 143 | "w = widgets.Dropdown(\n", 144 | " options=filtered_vars,\n", 145 | " value=filtered_vars[0],\n", 146 | " description=\"variable: \",\n", 147 | ")\n", 148 | "\n", 149 | "out = widgets.Output()\n", 150 | "# display(out)\n", 151 | "# desired_attribute = filtered_vars[0]\n", 152 | "\n", 153 | "@out.capture()\n", 154 | "def on_change(change):\n", 155 | "# print('method is called when printing this')\n", 156 | " if change['type'] == 'change' and change['name'] == 'value':\n", 157 | " global desired_attribute\n", 158 | " desired_attribute = change.new\n", 159 | " %store desired_attribute\n", 160 | " print(\"desired attribute changed to: \" + desired_attribute)\n", 161 | "w.observe(on_change)\n", 162 | "\n", 163 | "\n", 164 | "print_list(\"the options for the variable.\", [])\n", 165 | "display(w)" 166 | ] 167 | }, 168 | { 169 | "cell_type": "code", 170 | "execution_count": null, 171 | "id": "a838cbfe", 172 | "metadata": {}, 173 | "outputs": [], 174 | "source": [ 175 | "# %store desired_attribute\n", 176 | "lister = []\n", 177 | "if(lister):\n", 178 | " print(\"klist\")\n", 179 | " " 180 | ] 181 | }, 182 | { 183 | "cell_type": "code", 184 | "execution_count": null, 185 | "id": "21c69d19", 186 | "metadata": {}, 187 | "outputs": [], 188 | "source": [ 189 | "# if(data_type != \"predictive\"):\n", 190 | "w = widgets.DatePicker(\n", 191 | " description='Start Date',\n", 192 | " disabled=False\n", 193 | ")\n", 194 | "\n", 195 | "out = widgets.Output()\n", 196 | "# display(out)\n", 197 | "# desired_attribute = filtered_vars[0]\n", 198 | "\n", 199 | "@out.capture()\n", 200 | "def on_change(change):\n", 201 | "# print('method is called when printing this')\n", 202 | " if change['type'] == 'change' and change['name'] == 'value':\n", 203 | " global start_date\n", 204 | " start_date = change.new\n", 205 | " %store start_date\n", 206 | " print(start_date)\n", 207 | "w.observe(on_change)\n", 208 | "print_list(\"the start and end dates.\", [])\n", 209 | "display(w)\n", 210 | "\n", 211 | "w2 = widgets.DatePicker(\n", 212 | " description='End Date',\n", 213 | " disabled=False\n", 214 | ")\n", 215 | "\n", 216 | "out2 = widgets.Output()\n", 217 | "# display(out2)\n", 218 | "# desired_attribute = filtered_vars[0]\n", 219 | "\n", 220 | "@out2.capture()\n", 221 | "def on_change(change):\n", 222 | "# print('method is called when printing this')\n", 223 | " if change['type'] == 'change' and change['name'] == 'value':\n", 224 | " global end_date\n", 225 | " end_date = change.new\n", 226 | " %store end_date\n", 227 | " print(end_date)\n", 228 | "w2.observe(on_change)\n", 229 | "\n", 230 | "\n", 231 | "# print_list(\"the start and end dates.\", [])\n", 232 | "display(w2)" 233 | ] 234 | } 235 | ], 236 | "metadata": { 237 | "kernelspec": { 238 | "display_name": "conda_python3", 239 | "language": "python", 240 | "name": "conda_python3" 241 | }, 242 | "language_info": { 243 | "codemirror_mode": { 244 | "name": "ipython", 245 | "version": 3 246 | }, 247 | "file_extension": ".py", 248 | "mimetype": "text/x-python", 249 | "name": "python", 250 | "nbconvert_exporter": "python", 251 | "pygments_lexer": "ipython3", 252 | "version": "3.8.12" 253 | } 254 | }, 255 | "nbformat": 4, 256 | "nbformat_minor": 5 257 | } 258 | -------------------------------------------------------------------------------- /lib/SagemakerCode/helpers.py: -------------------------------------------------------------------------------- 1 | 2 | def print_list(whats_below, list_to_print): 3 | print("---------------------") 4 | print("") 5 | print('\033[1m' + f'Below are {whats_below}' + '\033[0m') 6 | if(list_to_print): 7 | print("") 8 | print(*list_to_print, sep="\n") 9 | print("") 10 | return input("Which of the above would you like to select? ") 11 | 12 | 13 | def get_desired_value_dict(whats_below, dict_to_print): 14 | print("---------------------") 15 | print("") 16 | print('\033[1m' + f'Below are {whats_below}' + '\033[0m') 17 | print("") 18 | print(*dict_to_print.keys(), sep="\n") 19 | print("") 20 | desired_key = input("Which of the above would you like to select? ") 21 | return dict_to_print[desired_key] 22 | 23 | 24 | 25 | 26 | 27 | def create_dropdown(variable_types, widgets, description, on_change): 28 | w = widgets.Dropdown( 29 | options=variable_types, 30 | value=variable_types[0], 31 | description=description, 32 | ) 33 | 34 | 35 | 36 | return w -------------------------------------------------------------------------------- /lib/SagemakerCode/publish_research.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "id": "78b0ab4d", 7 | "metadata": {}, 8 | "outputs": [], 9 | "source": [ 10 | "import ipywidgets as widgets" 11 | ] 12 | }, 13 | { 14 | "cell_type": "code", 15 | "execution_count": null, 16 | "id": "b53d2957", 17 | "metadata": {}, 18 | "outputs": [], 19 | "source": [ 20 | "button = widgets.Button(\n", 21 | " description='Publish',\n", 22 | " disabled=False,\n", 23 | " button_style='success', # 'success', 'info', 'warning', 'danger' or ''\n", 24 | " tooltip='Publish',\n", 25 | " icon='share' # (FontAwesome names without the `fa-` prefix)\n", 26 | ")\n", 27 | "# print(\"running...\")" 28 | ] 29 | }, 30 | { 31 | "cell_type": "code", 32 | "execution_count": null, 33 | "id": "48505733", 34 | "metadata": {}, 35 | "outputs": [], 36 | "source": [ 37 | "out = widgets.Output()\n", 38 | "display(out)\n", 39 | "\n", 40 | "@out.capture()\n", 41 | "def print_success(e):\n", 42 | " print(\"Congratulations! Published to fake_link.com\")\n", 43 | "\n", 44 | "button.on_click(print_success)" 45 | ] 46 | }, 47 | { 48 | "cell_type": "code", 49 | "execution_count": null, 50 | "id": "1ccc58bc", 51 | "metadata": {}, 52 | "outputs": [], 53 | "source": [ 54 | "display(button)" 55 | ] 56 | } 57 | ], 58 | "metadata": { 59 | "kernelspec": { 60 | "display_name": "conda_python3", 61 | "language": "python", 62 | "name": "conda_python3" 63 | }, 64 | "language_info": { 65 | "codemirror_mode": { 66 | "name": "ipython", 67 | "version": 3 68 | }, 69 | "file_extension": ".py", 70 | "mimetype": "text/x-python", 71 | "name": "python", 72 | "nbconvert_exporter": "python", 73 | "pygments_lexer": "ipython3", 74 | "version": "3.8.12" 75 | } 76 | }, 77 | "nbformat": 4, 78 | "nbformat_minor": 5 79 | } 80 | -------------------------------------------------------------------------------- /lib/SagemakerCode/ux_notebook.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "f8f7f8fb", 6 | "metadata": {}, 7 | "source": [ 8 | "# Disclaimer\n", 9 | "A reminder that this UI is meant to demonstrate the art of the possible. Different components were created in different and possibly disjointed ways to showcase various capabilities. The goal is to help think of what the user experience COULD look like, rather than build a fully functional user interface." 10 | ] 11 | }, 12 | { 13 | "cell_type": "markdown", 14 | "id": "9af95344", 15 | "metadata": {}, 16 | "source": [ 17 | "# Overview\n", 18 | "Welcome to the new way to load data! We have created this notebook so that you can select what type of data you want and then load that data without having to figure out how. You don't need to know where the data is coming from or how to extract it - just pick the specifications you want for the data, run the program, and you will get a dataset returned that you can interact with. \n", 19 | "\n", 20 | "## What to expect in each section\n", 21 | "### Data specification exploration\n", 22 | "In this section, you will be able to see what types of data specifications you might want to select. For example, we'll show you some of the options for the activities, and you can simply select which activities you are interested in seeing. These come from the Amazon Sustanability Data Initiative, or [ASDI](https://sustainability.aboutamazon.com/environment/the-cloud/amazon-sustainability-data-initiative) if you want to reference or learn more about the data specifications.\n", 23 | "\n", 24 | "### Getting the data set\n", 25 | "Now that you have selected the various data sets, you can simply run the cell to get the data set. Behind the scenes, this dynamically determines where the data is stored, how to extract it, and does so in just a few minutes! Feel free to grab a coffee while you wait for it to run. \n", 26 | "\n", 27 | "### Data exploration and analysis\n", 28 | "Now that you have the data, you can use this section of the notebook to run experiments on the data, plot it, create hypotheses, and eventually publish your data. If you decide you want to examine a different data set, you can start again at the beginning and keep all of the code youve written here to analyze the new returned data set. " 29 | ] 30 | }, 31 | { 32 | "cell_type": "markdown", 33 | "id": "c4f46b93", 34 | "metadata": {}, 35 | "source": [ 36 | "# Data specification exploration" 37 | ] 38 | }, 39 | { 40 | "cell_type": "markdown", 41 | "id": "d17b4369", 42 | "metadata": {}, 43 | "source": [ 44 | "## View and select the desired inputs for the data\n", 45 | "The cells below will show you some of the available data, and allow you to specify variables as well as filter down lists to make it easy for you to further specify what data you want. " 46 | ] 47 | }, 48 | { 49 | "cell_type": "code", 50 | "execution_count": null, 51 | "id": "b895f537", 52 | "metadata": {}, 53 | "outputs": [], 54 | "source": [ 55 | "# Display and save variable categories (e.g. temperature)\n", 56 | "%run get_variables.ipynb" 57 | ] 58 | }, 59 | { 60 | "cell_type": "markdown", 61 | "id": "e74049f4", 62 | "metadata": {}, 63 | "source": [ 64 | "### For demo purposes\n", 65 | "For demo purposes, let's take a look at the attribute we've stored" 66 | ] 67 | }, 68 | { 69 | "cell_type": "code", 70 | "execution_count": null, 71 | "id": "f4a7c00d", 72 | "metadata": {}, 73 | "outputs": [], 74 | "source": [ 75 | "%store -r desired_attribute data_type start_date end_date\n", 76 | "print(desired_attribute)\n", 77 | "print(data_type)\n", 78 | "print(start_date)\n", 79 | "print(end_date)" 80 | ] 81 | }, 82 | { 83 | "cell_type": "markdown", 84 | "id": "49072235", 85 | "metadata": {}, 86 | "source": [ 87 | "## Set catalog inputs\n", 88 | "The cells below will show you some of the available catalog inputs, and allow you to specify variables as well as filter down lists to make it easy for you to further specify what data you want. " 89 | ] 90 | }, 91 | { 92 | "cell_type": "code", 93 | "execution_count": null, 94 | "id": "c35518d6", 95 | "metadata": {}, 96 | "outputs": [], 97 | "source": [ 98 | "%run get_catalog_inputs.ipynb" 99 | ] 100 | }, 101 | { 102 | "cell_type": "markdown", 103 | "id": "c5dd3ba4", 104 | "metadata": {}, 105 | "source": [ 106 | "### For demo purposes\n", 107 | "For demo purposes, let's take a look at the attribute we've stored" 108 | ] 109 | }, 110 | { 111 | "cell_type": "code", 112 | "execution_count": null, 113 | "id": "e9bb8587", 114 | "metadata": {}, 115 | "outputs": [], 116 | "source": [ 117 | "%store -r activity_id\n", 118 | "%store -r variable_id\n", 119 | "%store -r table_id\n", 120 | "print(activity_id)\n", 121 | "print(variable_id)\n", 122 | "print(table_id)" 123 | ] 124 | }, 125 | { 126 | "cell_type": "markdown", 127 | "id": "0c38f41b", 128 | "metadata": {}, 129 | "source": [ 130 | "## Getting the data set\n", 131 | "Now that we have selected and stored our variables, we will run the script with (some) of those variables as inputs. We will then get the data as a result" 132 | ] 133 | }, 134 | { 135 | "cell_type": "markdown", 136 | "id": "54a9cc99", 137 | "metadata": {}, 138 | "source": [ 139 | "#### Run processes to generate data set\n", 140 | "Use the inputs provided to run the processes to generate the data set using the filters and specifications provided above" 141 | ] 142 | }, 143 | { 144 | "cell_type": "code", 145 | "execution_count": null, 146 | "id": "7f75424b", 147 | "metadata": {}, 148 | "outputs": [], 149 | "source": [ 150 | "%%time\n", 151 | "%%capture\n", 152 | "%run get_data.ipynb" 153 | ] 154 | }, 155 | { 156 | "cell_type": "markdown", 157 | "id": "7c12a62d", 158 | "metadata": {}, 159 | "source": [ 160 | "### Cost\n", 161 | "Now, let's look at how much this calculation cost to run" 162 | ] 163 | }, 164 | { 165 | "cell_type": "code", 166 | "execution_count": null, 167 | "id": "4e37fd3f", 168 | "metadata": {}, 169 | "outputs": [], 170 | "source": [ 171 | "%store -r cost_information\n", 172 | "print(cost_information)" 173 | ] 174 | }, 175 | { 176 | "cell_type": "markdown", 177 | "id": "a1046aaa", 178 | "metadata": {}, 179 | "source": [ 180 | "## Data exploration and analysis\n", 181 | "We can now do analysis on the data set with the data as an xarray. The cells below calculate the mean and regrided value for the data set" 182 | ] 183 | }, 184 | { 185 | "cell_type": "code", 186 | "execution_count": null, 187 | "id": "199cf9b4", 188 | "metadata": {}, 189 | "outputs": [], 190 | "source": [ 191 | "import numpy as np\n", 192 | "from dask_worker_pools import pool, propagate_pools" 193 | ] 194 | }, 195 | { 196 | "cell_type": "code", 197 | "execution_count": null, 198 | "id": "d1afe8a7", 199 | "metadata": {}, 200 | "outputs": [], 201 | "source": [ 202 | "#some paramters for regridding\n", 203 | "regrid_lon=np.arange(0.0,360.0,0.1)\n", 204 | "regrid_lat=np.arange(-90.0,90.0,0.1)\n", 205 | "regrid_method='slinear'" 206 | ] 207 | }, 208 | { 209 | "cell_type": "markdown", 210 | "id": "5e7fc92a", 211 | "metadata": {}, 212 | "source": [ 213 | "### Predictive" 214 | ] 215 | }, 216 | { 217 | "cell_type": "markdown", 218 | "id": "247379a6", 219 | "metadata": {}, 220 | "source": [ 221 | "#### Retrieve calculated data set" 222 | ] 223 | }, 224 | { 225 | "cell_type": "code", 226 | "execution_count": null, 227 | "id": "5555042b", 228 | "metadata": {}, 229 | "outputs": [], 230 | "source": [ 231 | "%%time\n", 232 | "if (data_type != \"historical\"):\n", 233 | " %store -r prediction_pool_region predictive_data_set\n", 234 | " with pool(prediction_pool_region):\n", 235 | " predicted_tas_mean = (predictive_data_set['tas']-273).mean(dim='time') #mean\n", 236 | " predicted_tas_regridded = predicted_tas_mean.interp(lon=regrid_lon,lat=regrid_lat,method=regrid_method) #dodgy regridding\n", 237 | "\n", 238 | " with pool(prediction_pool_region):\n", 239 | " predicted_tas_regridded.compute() #explict compute so you can see where it happens (could just do .plot() but it would be hidden)\n", 240 | " predicted_tas_regridded\n", 241 | " predicted_tas_regridded.plot(figsize=(14,7))\n", 242 | " plt.title(f'2090-2100 Mean {desired_attribute.replace(\"_\",\" \")}')\n", 243 | "else:\n", 244 | " print(\"Did not run predictive calculations for historical data\")" 245 | ] 246 | }, 247 | { 248 | "cell_type": "markdown", 249 | "id": "3a1dd630", 250 | "metadata": {}, 251 | "source": [ 252 | "### Historical" 253 | ] 254 | }, 255 | { 256 | "cell_type": "markdown", 257 | "id": "8abc73b5", 258 | "metadata": {}, 259 | "source": [ 260 | "#### Perform Calculations" 261 | ] 262 | }, 263 | { 264 | "cell_type": "code", 265 | "execution_count": null, 266 | "id": "1cb13da4", 267 | "metadata": {}, 268 | "outputs": [], 269 | "source": [ 270 | "%%time\n", 271 | "if data_type != \"predictive\":\n", 272 | " %store -r historical_data historical_pool_region\n", 273 | "# %matplotlib widget\n", 274 | " with pool(historical_pool_region):\n", 275 | " historic_temp_in_C = historical_data[desired_attribute] - 273.0\n", 276 | " historic_temp_mean = historic_temp_in_C.mean(dim='time0') #mean\n", 277 | " historic_temp_regridded = historic_temp_mean.interp(lon=regrid_lon,lat=regrid_lat,method=regrid_method) #dodgy regrid\n", 278 | " historic_temp_regridded.compute()\n", 279 | " historic_temp_regridded.plot(figsize=(14,7))\n", 280 | "\n", 281 | " plt.title(f'ERA5 Mean {desired_attribute.replace(\"_\",\" \")}')\n", 282 | "else:\n", 283 | " print(\"Did not run historical calculations for predictive data\")" 284 | ] 285 | }, 286 | { 287 | "cell_type": "markdown", 288 | "id": "30dc0951", 289 | "metadata": {}, 290 | "source": [ 291 | "## Publish analysis\n", 292 | "Once you are ready to publish your analysis, click the button below to publish it as a web page for others to view" 293 | ] 294 | }, 295 | { 296 | "cell_type": "code", 297 | "execution_count": null, 298 | "id": "aedced1f", 299 | "metadata": {}, 300 | "outputs": [], 301 | "source": [ 302 | "%run publish_research.ipynb" 303 | ] 304 | }, 305 | { 306 | "cell_type": "code", 307 | "execution_count": null, 308 | "id": "62ffda82", 309 | "metadata": {}, 310 | "outputs": [], 311 | "source": [] 312 | } 313 | ], 314 | "metadata": { 315 | "kernelspec": { 316 | "display_name": "conda_python3", 317 | "language": "python", 318 | "name": "conda_python3" 319 | }, 320 | "language_info": { 321 | "codemirror_mode": { 322 | "name": "ipython", 323 | "version": 3 324 | }, 325 | "file_extension": ".py", 326 | "mimetype": "text/x-python", 327 | "name": "python", 328 | "nbconvert_exporter": "python", 329 | "pygments_lexer": "ipython3", 330 | "version": "3.8.12" 331 | } 332 | }, 333 | "nbformat": 4, 334 | "nbformat_minor": 5 335 | } 336 | -------------------------------------------------------------------------------- /lib/ScriptsToUpdateOpenSearch/triggerScan.sh: -------------------------------------------------------------------------------- 1 | echo 'Start lfs' > /tmp/triggerScan.log 2 | date +%s >> /tmp/triggerScan.log 3 | lfs find /fsx -type f >> fileToWriteToOpenSearch.txt 4 | echo 'lfs Done, start indexing to OpenSearch' >> /tmp/triggerScan.log 5 | python3 updateOpenSearch.py 6 | date +%s >> /tmp/triggerScan.log 7 | echo 'Done triggerScan.sh' >> /tmp/triggerScan.log 8 | -------------------------------------------------------------------------------- /lib/ScriptsToUpdateOpenSearch/updateOpenSearch.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | 4 | from opensearchpy import OpenSearch, RequestsHttpConnection, AWSV4SignerAuth 5 | from opensearchpy.helpers import bulk 6 | import requests 7 | import boto3 8 | 9 | local_region = requests.get('http://169.254.169.254/latest/meta-data/placement/region').text 10 | 11 | worker_region_ssm = boto3.client('ssm', region_name=local_region) 12 | client_region = worker_region_ssm.get_parameter( 13 | Name='client-region-for-dask-worker-' + local_region 14 | )['Parameter']['Value'] 15 | bucket = worker_region_ssm.get_parameter( 16 | Name='worker-region-data-bucket-for-dask-worker-' + local_region 17 | )['Parameter']['Value'] 18 | 19 | client_region_ssm = boto3.client('ssm', region_name=client_region) 20 | host = client_region_ssm.get_parameter( 21 | Name='client-opensearch-domain-' + client_region 22 | )['Parameter']['Value'] 23 | 24 | credentials = boto3.Session().get_credentials() 25 | auth = AWSV4SignerAuth(credentials, client_region) 26 | 27 | client = OpenSearch( 28 | hosts = [{'host': host, 'port': 443}], 29 | http_auth = auth, 30 | use_ssl = True, 31 | verify_certs = True, 32 | connection_class = RequestsHttpConnection 33 | ) 34 | 35 | file1 = open('fileToWriteToOpenSearch.txt', 'r') 36 | Lines = file1.readlines() 37 | 38 | # Strips the newline character 39 | bulk_data = [] 40 | for line in Lines: 41 | filePath = line.strip() 42 | 43 | bulk_data.append({ 44 | '_index': bucket, 45 | '_id': line.strip(), 46 | '_source': { 47 | 'fileName': filePath, 48 | 'bucket': bucket, 49 | 'region': local_region, 50 | 'dask_pool': local_region, 51 | 'project': bucket 52 | }, 53 | }) 54 | try: 55 | client.indices.delete(index=bucket) 56 | except: 57 | print('Index does not currently exist') 58 | print('Starting Bulk Upload to OpenSearch') 59 | bulk(client, bulk_data) 60 | print('Bulk Done') -------------------------------------------------------------------------------- /lib/SdkConstructs/accept-tgw-request-client.ts: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: MIT-0 3 | 4 | import { Construct } from "constructs"; 5 | import { 6 | AwsCustomResource, 7 | AwsCustomResourcePolicy, 8 | AwsSdkCall, 9 | } from "aws-cdk-lib/custom-resources"; 10 | 11 | interface AcceptTGWRequestClientProps { 12 | attachmentId: string; 13 | region: string; 14 | account: string; 15 | } 16 | 17 | /** 18 | * Make an SDK call from the peer region to accept the peering connection 19 | */ 20 | export class AcceptTGWRequestClient extends AwsCustomResource { 21 | constructor( 22 | scope: Construct, 23 | name: string, 24 | props: AcceptTGWRequestClientProps 25 | ) { 26 | const { attachmentId, region, account } = props; 27 | 28 | const ssmAwsSdkCall: AwsSdkCall = { 29 | service: "EC2", 30 | action: "acceptTransitGatewayPeeringAttachment", 31 | parameters: { 32 | TransitGatewayAttachmentId: attachmentId, 33 | }, 34 | region, 35 | physicalResourceId: { id: "acceptTransitGatewayPeeringAttachment" }, 36 | }; 37 | 38 | super(scope, name, { 39 | onUpdate: ssmAwsSdkCall, 40 | policy: AwsCustomResourcePolicy.fromSdkCalls({ 41 | resources: [ 42 | `arn:aws:ec2:${region}:${account}:transit-gateway-attachment/${attachmentId}`, 43 | ], 44 | }), 45 | }); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /lib/SdkConstructs/create-data-repo-link-lustre.ts: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: MIT-0 3 | 4 | import { 5 | AwsCustomResource, 6 | AwsCustomResourcePolicy, 7 | AwsSdkCall, 8 | } from "aws-cdk-lib/custom-resources"; 9 | import { PolicyStatement } from "aws-cdk-lib/aws-iam"; 10 | import { Construct } from "constructs"; 11 | 12 | interface ParameterProps { 13 | DataRepositoryPath: string; 14 | FileSystemId: string; 15 | FileSystemPath: string; 16 | region: string; 17 | account: string; 18 | } 19 | 20 | /** 21 | * An SDK call to create a data link to a public s3 bucket from lustre 22 | */ 23 | export class CreateDataLinkRepoClient extends AwsCustomResource { 24 | constructor(scope: Construct, name: string, props: ParameterProps) { 25 | const { 26 | DataRepositoryPath, 27 | FileSystemId, 28 | FileSystemPath, 29 | region, 30 | account, 31 | } = props; 32 | 33 | const ssmAwsSdkCall: AwsSdkCall = { 34 | service: "FSx", 35 | action: "createDataRepositoryAssociation", 36 | parameters: { 37 | DataRepositoryPath, 38 | FileSystemId, 39 | FileSystemPath, 40 | }, 41 | physicalResourceId: { id: "createDataRepositoryAssociation" }, 42 | }; 43 | 44 | super(scope, name, { 45 | onUpdate: ssmAwsSdkCall, 46 | policy: AwsCustomResourcePolicy.fromStatements([ 47 | new PolicyStatement({ 48 | actions: ["fsx:CreateDataRepositoryAssociation"], 49 | resources: [ 50 | `arn:aws:fsx:${region}:${account}:association/${FileSystemId}/*`, 51 | `arn:aws:fsx:${region}:${account}:file-system/${FileSystemId}`, 52 | ], 53 | }), 54 | new PolicyStatement({ 55 | actions: [ 56 | "s3:Get*", 57 | "s3:List*", 58 | "s3:PutObject", 59 | "iam:CreateServiceLinkedRole", 60 | "iam:AttachRolePolicy", 61 | "iam:PutRolePolicy", 62 | ], 63 | resources: AwsCustomResourcePolicy.ANY_RESOURCE, 64 | }), 65 | ]), 66 | }); 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /lib/SdkConstructs/default-transit-route-table-id.ts: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: MIT-0 3 | 4 | import { Construct } from "constructs"; 5 | import { 6 | AwsCustomResource, 7 | AwsCustomResourcePolicy, 8 | AwsSdkCall, 9 | } from "aws-cdk-lib/custom-resources"; 10 | 11 | interface TGWRouteTableProps { 12 | parameterName: string; 13 | region: string; 14 | } 15 | 16 | /** 17 | * Make the relevant SDK to pull the default route table ID from the created TGW 18 | */ 19 | export class TransitGatewayRouteTable extends AwsCustomResource { 20 | constructor(scope: Construct, name: string, props: TGWRouteTableProps) { 21 | const { parameterName, region } = props; 22 | 23 | const ssmAwsSdkCall: AwsSdkCall = { 24 | service: "EC2", 25 | action: "describeTransitGateways", 26 | parameters: { 27 | TransitGatewayIds: [parameterName], 28 | }, 29 | region, 30 | physicalResourceId: { 31 | id: "TransitGateways-AssociationDefaultRouteTableId", 32 | }, 33 | }; 34 | 35 | super(scope, name, { 36 | onUpdate: ssmAwsSdkCall, 37 | policy: AwsCustomResourcePolicy.fromSdkCalls({ 38 | resources: AwsCustomResourcePolicy.ANY_RESOURCE, 39 | }), 40 | }); 41 | } 42 | 43 | public getParameterValue(): string { 44 | return this.getResponseField( 45 | "TransitGateways.0.Options.AssociationDefaultRouteTableId" 46 | ).toString(); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /lib/SdkConstructs/ssm-param-reader.ts: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: MIT-0 3 | 4 | import { PolicyStatement } from "aws-cdk-lib/aws-iam"; 5 | import { Construct } from "constructs"; 6 | import { AwsCustomResource, AwsSdkCall } from "aws-cdk-lib/custom-resources"; 7 | 8 | interface SSMParameterReaderProps { 9 | parameterName: string; 10 | region: string; 11 | account: string; 12 | } 13 | 14 | /** 15 | * Generic parameter stack which calls the sdk to retrieve parameters interregionally 16 | */ 17 | export class SSMParameterReader extends AwsCustomResource { 18 | constructor(scope: Construct, name: string, props: SSMParameterReaderProps) { 19 | const { parameterName, region, account } = props; 20 | 21 | const ssmAwsSdkCall: AwsSdkCall = { 22 | service: "SSM", 23 | action: "getParameter", 24 | parameters: { 25 | Name: parameterName, 26 | }, 27 | region, 28 | physicalResourceId: { id: "getParameter" }, // Update physical id to always fetch the latest version 29 | }; 30 | 31 | super(scope, name, { 32 | onUpdate: ssmAwsSdkCall, 33 | policy: { 34 | statements: [ 35 | new PolicyStatement({ 36 | resources: [ 37 | `arn:aws:ssm:${region}:${account}:parameter/${parameterName}`, 38 | ], 39 | actions: ["ssm:GetParameter"], 40 | }), 41 | ], 42 | }, 43 | }); 44 | } 45 | 46 | public getParameterValue(): string { 47 | return this.getResponseField("Parameter.Value").toString(); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /lib/WorkerConstructs/interregion-worker-tgw-route.ts: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: MIT-0 3 | 4 | import { Stack, App, StackProps } from "aws-cdk-lib"; 5 | import { 6 | CfnRoute, 7 | CfnTransitGateway, 8 | CfnTransitGatewayRoute, 9 | Vpc, 10 | } from "aws-cdk-lib/aws-ec2"; 11 | import { NagSuppressions } from "cdk-nag"; 12 | import { IWorker } from "../../bin/interface"; 13 | import { SSMParameterReader } from "../SdkConstructs/ssm-param-reader"; 14 | import { TransitGatewayRouteTable } from "../SdkConstructs/default-transit-route-table-id"; 15 | 16 | export interface ClientRegionProps extends StackProps { 17 | peerWorker: IWorker; 18 | tgw: CfnTransitGateway; 19 | vpc: Vpc; 20 | } 21 | 22 | /** 23 | * Add the relevant route to the route table on a subnet level but also on a tgw level 24 | */ 25 | export class InterRegionTransitGatewayRoute extends Stack { 26 | constructor(scope: App, id: string, props: ClientRegionProps) { 27 | super(scope, id, props); 28 | const { peerWorker, tgw, vpc } = props; 29 | 30 | NagSuppressions.addStackSuppressions(this, [ 31 | { 32 | id: "AwsSolutions-IAM4", 33 | reason: 34 | "Lambda execution policy for custom resources created by higher level CDK constructs", 35 | appliesTo: [ 36 | "Policy::arn::iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", 37 | ], 38 | }, 39 | { 40 | id: "AwsSolutions-L1", 41 | reason: "AWS CDK custom resources uses node 14.x environment", 42 | }, 43 | ]); 44 | 45 | // Pull the default associated route table for the tgw 46 | const transitGatewayConstruct = new TransitGatewayRouteTable( 47 | this, 48 | "Transit Gateway", 49 | { 50 | parameterName: tgw.ref, 51 | region: this.region, 52 | } 53 | ); 54 | const transitGatewayRouteTableId = 55 | transitGatewayConstruct.getParameterValue(); 56 | NagSuppressions.addResourceSuppressions( 57 | transitGatewayConstruct, 58 | [ 59 | { 60 | id: "AwsSolutions-IAM5", 61 | reason: "By default the action specified supports all resource types", 62 | }, 63 | ], 64 | true 65 | ); 66 | 67 | // Pull the attachment ID of the peering connection 68 | const transitGatewayAttachmentId = new SSMParameterReader( 69 | this, 70 | `Transit Attachment ID - ${this.region}`, 71 | { 72 | parameterName: `tgw-attachmentid-${peerWorker.region}-${this.region}`, 73 | region: peerWorker.region, 74 | account: this.account, 75 | } 76 | ).getParameterValue(); 77 | 78 | // Append the route to the transit gateway route table 79 | new CfnTransitGatewayRoute(this, `TGW Route`, { 80 | transitGatewayRouteTableId, 81 | destinationCidrBlock: peerWorker.cidr, 82 | transitGatewayAttachmentId, 83 | }); 84 | 85 | // Loop through and add the routes to the private subnets 86 | for (let i = 0; i < vpc.privateSubnets.length; i++) { 87 | new CfnRoute(this, `Subnet to TGW - ${vpc.privateSubnets[i]}`, { 88 | routeTableId: vpc.privateSubnets[i].routeTable.routeTableId, 89 | destinationCidrBlock: peerWorker.cidr, 90 | transitGatewayId: tgw.ref, 91 | }); 92 | } 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /lib/WorkerConstructs/sync-lustre-to-opensearch.ts: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: MIT-0 3 | 4 | import { App, Stack, StackProps } from "aws-cdk-lib"; 5 | import { AutoScalingGroup } from "aws-cdk-lib/aws-autoscaling"; 6 | import { 7 | AmazonLinuxGeneration, 8 | AmazonLinuxImage, 9 | InstanceType, 10 | Vpc, 11 | } from "aws-cdk-lib/aws-ec2"; 12 | import { LustreFileSystem } from "aws-cdk-lib/aws-fsx"; 13 | import { ManagedPolicy, PolicyStatement } from "aws-cdk-lib/aws-iam"; 14 | import { Key } from "aws-cdk-lib/aws-kms"; 15 | import { Topic } from "aws-cdk-lib/aws-sns"; 16 | import { StringParameter } from "aws-cdk-lib/aws-ssm"; 17 | import { NagSuppressions } from "cdk-nag"; 18 | import { readFileSync } from "fs"; 19 | import { IClient, IWorker } from "../../bin/interface"; 20 | import { SSMParameterReader } from "../SdkConstructs/ssm-param-reader"; 21 | import path = require("path"); 22 | 23 | interface SyncLustreToOpenSearchProps extends StackProps { 24 | client: IClient; 25 | worker: IWorker; 26 | vpc: Vpc; 27 | lustre: LustreFileSystem; 28 | } 29 | 30 | export class SyncLustreToOpenSearch extends Stack { 31 | constructor(scope: App, id: string, props: SyncLustreToOpenSearchProps) { 32 | super(scope, id, props); 33 | const { client, worker, vpc, lustre } = props; 34 | 35 | this.setupOpenSearchUpdates(client, worker, vpc, lustre); 36 | 37 | NagSuppressions.addStackSuppressions(this, [ 38 | { 39 | id: "AwsSolutions-IAM4", 40 | reason: 41 | "Lambda execution policy for custom resources created by higher level CDK constructs", 42 | appliesTo: [ 43 | "Policy::arn::iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", 44 | ], 45 | }, 46 | { 47 | id: "AwsSolutions-L1", 48 | reason: "AWS CDK custom resources uses node 14.x environment", 49 | }, 50 | ]); 51 | } 52 | /** Setup The Open Search for updates 53 | * 54 | * @param client - Object of the client containing pieces such as client region and cidr 55 | * @param worker - Object of the worker containing pieces such as worker region, cidr and data 56 | */ 57 | setupOpenSearchUpdates( 58 | client: IClient, 59 | worker: IWorker, 60 | vpc: Vpc, 61 | lustre: LustreFileSystem 62 | ) { 63 | // Creating a topic for best practise purposes which you could make use of downstream 64 | const snsTopicForUpdates = new Topic(this, "SNS Updates Autoscaling", { 65 | masterKey: new Key(this, "ASG Updates Key", { 66 | enableKeyRotation: true, 67 | }), 68 | }); 69 | 70 | // This autoscaling group creates just one instance which will perform the updates to opensearch 71 | // Can be autoscaled to go up and down in the future 72 | const autoScalingGroup = new AutoScalingGroup(this, "AutoScaling Group", { 73 | vpc, 74 | machineImage: new AmazonLinuxImage({ 75 | generation: AmazonLinuxGeneration.AMAZON_LINUX_2, 76 | }), 77 | instanceType: new InstanceType("m5d.large"), 78 | notifications: [ 79 | { 80 | topic: snsTopicForUpdates, 81 | }, 82 | ], 83 | }); 84 | autoScalingGroup.addToRolePolicy( 85 | new PolicyStatement({ 86 | actions: ["ssm:GetParameter"], 87 | resources: [ 88 | `arn:aws:ssm:${client.region}:${this.account}:parameter/client-opensearch-domain-${client.region}`, 89 | ], 90 | }) 91 | ); 92 | 93 | // The arn is loaded in for the client region opensearch domain 94 | const OpenSearchARN = new SSMParameterReader(this, "OpenSearchARN", { 95 | parameterName: `client-opensearch-arn-${client.region}`, 96 | region: client.region, 97 | account: this.account, 98 | }).getParameterValue(); 99 | autoScalingGroup.addToRolePolicy( 100 | new PolicyStatement({ 101 | resources: [OpenSearchARN, `${OpenSearchARN}/*`], 102 | actions: ["es:ESHttpPut", "es:ESHttpPost", "es:ESHttpDelete"], 103 | }) 104 | ); 105 | 106 | autoScalingGroup.role.addManagedPolicy( 107 | ManagedPolicy.fromAwsManagedPolicyName("AmazonSSMManagedInstanceCore") 108 | ); 109 | NagSuppressions.addResourceSuppressions(autoScalingGroup.role, [ 110 | { 111 | id: "AwsSolutions-IAM4", 112 | reason: "Debug access", 113 | }, 114 | ]); 115 | 116 | NagSuppressions.addResourceSuppressions( 117 | autoScalingGroup, 118 | [ 119 | { 120 | id: "AwsSolutions-IAM5", 121 | reason: "Instance needs access to push to all indicies", 122 | }, 123 | ], 124 | true 125 | ); 126 | 127 | // The two scripts below are loaded onto the ec2 instance and triggered each day to push to opensearch 128 | // Currently it's doing a complete sync each time, but can be optimised into the future 129 | // And does not note deletions 130 | const trigger = readFileSync( 131 | path.join(__dirname, "..", "ScriptsToUpdateOpenSearch/triggerScan.sh"), 132 | { encoding: "utf8", flag: "r" } 133 | ); 134 | const script = readFileSync( 135 | path.join( 136 | __dirname, 137 | "..", 138 | "ScriptsToUpdateOpenSearch/updateOpenSearch.py" 139 | ), 140 | { encoding: "utf8", flag: "r" } 141 | ); 142 | 143 | new StringParameter(this, "ClientRegionForEC2", { 144 | parameterName: `client-region-for-dask-worker-${this.region}`, 145 | stringValue: client.region, 146 | }).grantRead(autoScalingGroup); 147 | new StringParameter(this, "WorkerDataProjectForEC2", { 148 | parameterName: `worker-region-data-bucket-for-dask-worker-${this.region}`, 149 | stringValue: worker.dataset.split("/")[2], 150 | }).grantRead(autoScalingGroup); 151 | 152 | // The userdata for this instance installs libraries, mounts lustre, and setups the sync job to 153 | // opensearch to trigger daily at 1am which should give enough time for the eventrule triggered at 154 | // midnight to finish 155 | autoScalingGroup.addUserData( 156 | "amazon-linux-extras install -y lustre", 157 | "pip3 install opensearch-py boto3", 158 | "mkdir -p /fsx", 159 | `mount -t lustre ${lustre.dnsName}@tcp:/${lustre.mountName} /fsx -o flock`, 160 | `echo ${lustre.dnsName}@tcp:/${lustre.mountName} /fsx lustre defaults,flock,_netdev,x-systemd.automount,x-systemd.requires=network.service 0 0 >> /etc/fstab`, 161 | "echo mountDone", 162 | `echo "${trigger}" > /triggerScan.sh`, 163 | `echo "${script}" > /updateOpenSearch.py`, 164 | 'crontab -l | { cat; echo "0 1 * * * bash /triggerScan.sh"; } | crontab -', 165 | "echo runningSync", 166 | "bash /triggerScan.sh", 167 | "echo syncDone" 168 | ); 169 | } 170 | } 171 | -------------------------------------------------------------------------------- /lib/WorkerConstructs/worker-region-stack.ts: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: MIT-0 3 | 4 | import { Stack, App, StackProps, RemovalPolicy, Duration } from "aws-cdk-lib"; 5 | import { 6 | CfnRoute, 7 | CfnTransitGateway, 8 | CfnTransitGatewayAttachment, 9 | CfnTransitGatewayPeeringAttachment, 10 | FlowLogDestination, 11 | FlowLogTrafficType, 12 | InstanceType, 13 | IpAddresses, 14 | Peer, 15 | Port, 16 | SecurityGroup, 17 | SubnetType, 18 | Vpc, 19 | } from "aws-cdk-lib/aws-ec2"; 20 | import { DockerImageAsset, Platform } from "aws-cdk-lib/aws-ecr-assets"; 21 | import { 22 | AsgCapacityProvider, 23 | Cluster, 24 | ContainerImage, 25 | Ec2Service, 26 | Ec2TaskDefinition, 27 | LogDriver, 28 | NetworkMode, 29 | } from "aws-cdk-lib/aws-ecs"; 30 | import { Rule, Schedule } from "aws-cdk-lib/aws-events"; 31 | import { LambdaFunction } from "aws-cdk-lib/aws-events-targets"; 32 | import { LustreDeploymentType, LustreFileSystem } from "aws-cdk-lib/aws-fsx"; 33 | import { PolicyStatement } from "aws-cdk-lib/aws-iam"; 34 | import { Code, Function, Runtime } from "aws-cdk-lib/aws-lambda"; 35 | import { LogGroup } from "aws-cdk-lib/aws-logs"; 36 | import { Bucket } from "aws-cdk-lib/aws-s3"; 37 | import { StringParameter } from "aws-cdk-lib/aws-ssm"; 38 | import { 39 | AwsCustomResource, 40 | AwsCustomResourcePolicy, 41 | } from "aws-cdk-lib/custom-resources"; 42 | import { NagSuppressions } from "cdk-nag"; 43 | import { IClient, IWorker } from "../../bin/interface"; 44 | import { AcceptTGWRequestClient } from "../SdkConstructs/accept-tgw-request-client"; 45 | import { CreateDataLinkRepoClient } from "../SdkConstructs/create-data-repo-link-lustre"; 46 | import { SSMParameterReader } from "../SdkConstructs/ssm-param-reader"; 47 | import path = require("path"); 48 | 49 | export interface WorkerRegionProps extends StackProps { 50 | client: IClient; 51 | worker: IWorker; 52 | } 53 | 54 | /** 55 | * The worker region stack creates all the relevant infrastructure needs to launch a worker pool that 56 | * connect to the client region scheduler 57 | */ 58 | export class WorkerRegion extends Stack { 59 | public vpc: Vpc; 60 | public tgw: CfnTransitGateway; 61 | public attachmentID: CfnTransitGatewayPeeringAttachment; 62 | public lustre: LustreFileSystem; 63 | lustreBucket: Bucket; 64 | RepoFn: Function; 65 | dataLink: CreateDataLinkRepoClient; 66 | 67 | constructor(scope: App, id: string, props: WorkerRegionProps) { 68 | super(scope, id, props); 69 | const { client, worker } = props; 70 | 71 | this.setupEnvironment(client, worker); 72 | this.setupRegionalLustre(worker); 73 | this.setupDaskWorkers(client); 74 | 75 | NagSuppressions.addStackSuppressions(this, [ 76 | { 77 | id: "AwsSolutions-IAM4", 78 | reason: 79 | "Lambda execution policy for custom resources created by higher level CDK constructs", 80 | appliesTo: [ 81 | "Policy::arn::iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", 82 | ], 83 | }, 84 | { 85 | id: "AwsSolutions-L1", 86 | reason: "AWS CDK custom resources uses node 14.x environment", 87 | }, 88 | ]); 89 | } 90 | 91 | /** Setup Environment 92 | * 93 | * Setup for the worker a tgw, vpc, peering of the local vpc, accept the peer, add a route and 94 | * associate the vpc to the hosted zone 95 | * 96 | * @param client - Object of the client containing pieces such as client region and cidr 97 | * @param worker - Object of the worker containing pieces such as worker region, cidr and data 98 | */ 99 | setupEnvironment(client: IClient, worker: IWorker) { 100 | // Create the tgw and save param 101 | this.tgw = new CfnTransitGateway(this, "TGW", {}); 102 | new StringParameter(this, `TGW Param - ${this.region}`, { 103 | parameterName: `tgw-param-${this.region}`, 104 | stringValue: this.tgw.ref, 105 | }); 106 | 107 | // Pull the id for peering 108 | const peerTransitGatewayId = new SSMParameterReader(this, "TGW Param", { 109 | parameterName: `tgw-param-${client.region}`, 110 | region: client.region, 111 | account: this.account, 112 | }).getParameterValue(); 113 | // Establish a peering connection 114 | this.attachmentID = new CfnTransitGatewayPeeringAttachment( 115 | this, 116 | "Peering Connection", 117 | { 118 | peerAccountId: this.account, 119 | peerRegion: client.region, 120 | peerTransitGatewayId, 121 | transitGatewayId: this.tgw.ref, 122 | } 123 | ); 124 | this.attachmentID.addDependsOn(this.tgw); 125 | // Accept once established the peering 126 | this.acceptRequest(client); 127 | 128 | // Create the Worker VPC 129 | this.vpc = new Vpc(this, "Worker VPC", { 130 | ipAddresses: IpAddresses.cidr(worker.cidr), 131 | flowLogs: { 132 | cloudwatch: { 133 | destination: FlowLogDestination.toCloudWatchLogs( 134 | new LogGroup(this, "WorkerVpcFlowLogs") 135 | ), 136 | trafficType: FlowLogTrafficType.ALL, 137 | }, 138 | }, 139 | }); 140 | 141 | // Create an attachment of the local VPC to the TGW 142 | new CfnTransitGatewayAttachment(this, "tgw-attachment", { 143 | subnetIds: this.vpc.privateSubnets.map(({ subnetId }) => subnetId), 144 | transitGatewayId: this.tgw.ref, 145 | vpcId: this.vpc.vpcId, 146 | }).addDependsOn(this.tgw); 147 | 148 | // Save this param as other regions will need it 149 | new StringParameter(this, "TGW Attach Param", { 150 | parameterName: `tgw-attachmentid-${this.region}`, 151 | stringValue: this.attachmentID.attrTransitGatewayAttachmentId, 152 | }); 153 | 154 | // And add to each private subnet the routing of client traffic to the client region via tgw 155 | for (let i = 0; i < this.vpc.privateSubnets.length; i++) { 156 | new CfnRoute(this, `Subnet to TGW - ${this.vpc.privateSubnets[i]}`, { 157 | routeTableId: this.vpc.privateSubnets[i].routeTable.routeTableId, 158 | destinationCidrBlock: client.cidr, 159 | transitGatewayId: this.tgw.ref, 160 | }).addDependsOn(this.attachmentID); 161 | } 162 | 163 | // Pull the namespace created in the client region 164 | const HostedZoneId = new SSMParameterReader(this, "PrivateNP Param", { 165 | parameterName: `privatenp-hostedid-param-${client.region}`, 166 | region: client.region, 167 | account: this.account, 168 | }).getParameterValue(); 169 | // and make an sdk call to gain access to resolve that DNS in this space 170 | const associateVPC = new AwsCustomResource( 171 | this, 172 | "AssociateVPCWithHostedZone", 173 | { 174 | onCreate: { 175 | service: "Route53", 176 | action: "associateVPCWithHostedZone", 177 | parameters: { 178 | HostedZoneId, 179 | VPC: { VPCId: this.vpc.vpcId, VPCRegion: this.region }, 180 | }, 181 | physicalResourceId: { id: "associateVPCWithHostedZone" }, 182 | }, 183 | policy: AwsCustomResourcePolicy.fromStatements([ 184 | new PolicyStatement({ 185 | actions: ["route53:AssociateVPCWithHostedZone"], 186 | resources: [`arn:aws:route53:::hostedzone/${HostedZoneId}`], 187 | }), 188 | new PolicyStatement({ 189 | actions: ["ec2:DescribeVpcs"], 190 | resources: AwsCustomResourcePolicy.ANY_RESOURCE, 191 | }), 192 | ]), 193 | } 194 | ); 195 | NagSuppressions.addResourceSuppressions( 196 | associateVPC, 197 | [ 198 | { 199 | id: "AwsSolutions-IAM5", 200 | reason: "Call needs access to all resources", 201 | }, 202 | ], 203 | true 204 | ); 205 | } 206 | 207 | /** Accept Request 208 | * 209 | * Make an sdk call that will accept the peering connection 210 | * 211 | * @param client - Object of the client containing pieces such as client region and cidr 212 | */ 213 | acceptRequest(client: IClient) { 214 | new AcceptTGWRequestClient(this, "Accept Request", { 215 | attachmentId: this.attachmentID.attrTransitGatewayAttachmentId, 216 | region: client.region, 217 | account: this.account, 218 | }); 219 | } 220 | 221 | /** Setup the Regional Lustre 222 | * 223 | * Lustre is the middleware we use for rapid access to public s3 data. Lustre connects to the public 224 | * s3 data, loading it into the region so that the workers can work with the data as if it were a 225 | * local filesystem 226 | * 227 | * @param worker - Object of the worker containing pieces such as worker region, cidr and data 228 | */ 229 | setupRegionalLustre(worker: IWorker) { 230 | // The lustre security group allows certain ports to mount to the ec2 instance 231 | const secGroup = new SecurityGroup(this, "Lustre Security Group", { 232 | vpc: this.vpc, 233 | }); 234 | secGroup.addIngressRule(Peer.ipv4(this.vpc.vpcCidrBlock), Port.tcp(988)); 235 | secGroup.addIngressRule( 236 | Peer.ipv4(this.vpc.vpcCidrBlock), 237 | Port.tcpRange(1021, 1023) 238 | ); 239 | 240 | // Using persistent 2 we must create the data link to the repo after it's created 241 | this.lustre = new LustreFileSystem(this, "Lustre File System", { 242 | lustreConfiguration: { 243 | deploymentType: LustreDeploymentType.PERSISTENT_2, 244 | perUnitStorageThroughput: 1000, 245 | }, 246 | storageCapacityGiB: 1200, 247 | vpc: this.vpc, 248 | vpcSubnet: this.vpc.privateSubnets[0], 249 | securityGroup: secGroup, 250 | removalPolicy: RemovalPolicy.DESTROY, 251 | }); 252 | // An SDK to have the data link create after the lustre filesystem has been called to be created 253 | this.dataLink = new CreateDataLinkRepoClient(this, "DataRepoLustre", { 254 | DataRepositoryPath: worker.dataset, 255 | FileSystemId: this.lustre.fileSystemId, 256 | FileSystemPath: `/${this.region}/${worker.lustreFileSystemPath}`, 257 | region: this.region, 258 | account: this.account, 259 | }); 260 | 261 | NagSuppressions.addResourceSuppressions( 262 | this.dataLink, 263 | [ 264 | { 265 | id: "AwsSolutions-IAM5", 266 | reason: "Needs a * on association Id", 267 | }, 268 | ], 269 | true 270 | ); 271 | // We then create a function which when triggered on a scheduled basis will sync lustre to s3 272 | this.RepoFn = new Function(this, "Scheduled Lustre Repo Refresh", { 273 | runtime: Runtime.NODEJS_18_X, 274 | handler: "index.handler", 275 | code: Code.fromAsset(path.join(__dirname, "..", "LustreRepoTrigger")), 276 | environment: { 277 | FileSystemId: this.lustre.fileSystemId, 278 | }, 279 | initialPolicy: [ 280 | new PolicyStatement({ 281 | resources: [ 282 | `arn:aws:fsx:${this.region}:${this.account}:file-system/${this.lustre.fileSystemId}`, 283 | `arn:aws:fsx:${this.region}:${this.account}:task/*`, 284 | ], 285 | actions: ["fsx:CreateDataRepositoryTask"], 286 | }), 287 | ], 288 | }); 289 | NagSuppressions.addResourceSuppressions( 290 | this.RepoFn, 291 | [ 292 | { 293 | id: "AwsSolutions-IAM5", 294 | reason: 295 | "Function needs access to all task ids as they are dynamically created with uuid", 296 | }, 297 | ], 298 | true 299 | ); 300 | // After launch the rule below will prompt lustre to sync with the s3 bucket every day at midnight 301 | new Rule(this, "Schedule Rule", { 302 | schedule: Schedule.cron({ minute: "0", hour: "0" }), 303 | targets: [new LambdaFunction(this.RepoFn)], 304 | }); 305 | } 306 | 307 | /** Setup the Dask Workers 308 | * 309 | * Lastly we setup and run the workers which should connect in to the scheduler. Note that if you 310 | * deploy all the workers, it will not be able to connect until the later stacks that connect the tgw 311 | * have completed. 312 | * Also if you see unexpected errors in workers disconnecting to the to the client, first check the logs 313 | * and then check versioning between the notebook, scheduler and workers by running 314 | * 315 | * Run the below in the jupyter notebook 316 | * client.get_versions(check=True) 317 | * Sometimes version mismatch can cause unexpected issues 318 | * 319 | * @param client - Object of the client containing pieces such as client region and cidr 320 | */ 321 | setupDaskWorkers(client: IClient) { 322 | // Spin up the worker cluster. May need to increase your accounts quota for instances 323 | // beyond the account max 324 | const cluster = new Cluster(this, "Worker Cluster", { 325 | clusterName: "Dask-Workers", 326 | containerInsights: true, 327 | vpc: this.vpc, 328 | capacity: { 329 | instanceType: new InstanceType("m5d.4xlarge"), 330 | minCapacity: 0, 331 | maxCapacity: 12, 332 | vpcSubnets: { 333 | subnetType: SubnetType.PRIVATE_WITH_EGRESS, 334 | }, 335 | }, 336 | }); 337 | NagSuppressions.addResourceSuppressions( 338 | cluster.autoscalingGroup!, 339 | [ 340 | { 341 | id: "AwsSolutions-SNS2", 342 | reason: 343 | "SNS is a default asset created to which is not exposed publicly", 344 | }, 345 | { 346 | id: "AwsSolutions-SNS3", 347 | reason: 348 | "SNS is a default asset created to which is not exposed publicly", 349 | }, 350 | { 351 | id: "AwsSolutions-AS3", 352 | reason: 353 | "SNS is a default asset created to which is not exposed publicly", 354 | }, 355 | { 356 | id: "AwsSolutions-IAM5", 357 | reason: 358 | "Internally created service role generate by the cluster construct to drain instances of this cluster", 359 | }, 360 | ], 361 | true 362 | ); 363 | 364 | // User data will install lustre and mount it 365 | cluster.autoscalingGroup!.addUserData( 366 | "amazon-linux-extras install -y lustre", 367 | "mkdir -p /fsx", 368 | `mount -t lustre ${this.lustre.dnsName}@tcp:/${this.lustre.mountName} /fsx -o noatime,flock`, 369 | `echo ${this.lustre.dnsName}@tcp:/${this.lustre.mountName} /fsx lustre defaults,flock,_netdev,x-systemd.automount,x-systemd.requires=network.service 0 0 >> /etc/fstab`, 370 | "echo mountDone" 371 | ); 372 | 373 | // We are using an autoscaling capacity provider which will manage the scaling of instances. We 374 | // just need to worry about task scaling 375 | const capacityProvider = new AsgCapacityProvider( 376 | this, 377 | "AsgCapacityProvider", 378 | { 379 | autoScalingGroup: cluster.autoscalingGroup!, 380 | targetCapacityPercent: 80, 381 | } 382 | ); 383 | cluster.addAsgCapacityProvider(capacityProvider); 384 | 385 | // Definition created for the workers 386 | const taskDefinition = new Ec2TaskDefinition(this, "Worker Definition", { 387 | family: "Dask-Worker", 388 | networkMode: NetworkMode.AWS_VPC, 389 | volumes: [ 390 | { 391 | name: "Lustre", 392 | host: { 393 | sourcePath: "/fsx", 394 | }, 395 | }, 396 | ], 397 | }); 398 | 399 | // Setup the worker to run multiple workers with multiple threads 400 | // Feel free to adjust these figures to optimise on your workload 401 | const NWORKERS = 10; 402 | const THREADS = 3; 403 | const container = taskDefinition.addContainer("Container", { 404 | containerName: "Dask", 405 | memoryReservationMiB: 25000, 406 | image: ContainerImage.fromDockerImageAsset( 407 | new DockerImageAsset(this, "Worker Image Repo", { 408 | directory: path.join(__dirname, "..", "DaskImage"), 409 | platform: Platform.LINUX_AMD64, 410 | }) 411 | ), 412 | command: [ 413 | "bin/sh", 414 | "-c", 415 | `pip3 install --upgrade xarray[complete] intake_esm s3fs eccodes git+https://github.com/gjoseph92/dask-worker-pools.git@main && dask worker Dask-Scheduler.local-dask:8786 --worker-port 9000:${ 416 | 9000 + NWORKERS - 1 417 | } --nanny-port ${9000 + NWORKERS}:${ 418 | 9000 + NWORKERS * 2 - 1 419 | } --resources pool-${ 420 | this.region 421 | }=1 --nworkers ${NWORKERS} --nthreads ${THREADS} --no-dashboard`, 422 | ], 423 | essential: true, 424 | logging: LogDriver.awsLogs({ 425 | streamPrefix: "ecs", 426 | logGroup: new LogGroup(this, "Dask Worker Log Group"), 427 | }), 428 | portMappings: [...Array(NWORKERS * 2).keys()].map((x) => { 429 | return { containerPort: 9000 + x }; 430 | }), 431 | }); 432 | container.addMountPoints({ 433 | sourceVolume: "Lustre", 434 | containerPath: "/fsx", 435 | readOnly: false, 436 | }); 437 | NagSuppressions.addResourceSuppressions( 438 | new AwsCustomResource(this, "Enable Scanning on Repo", { 439 | onCreate: { 440 | service: "ECR", 441 | action: "putRegistryScanningConfiguration", 442 | physicalResourceId: { id: "putRegistryScanningConfiguration" }, 443 | parameters: { 444 | scanType: "ENHANCED", 445 | }, 446 | }, 447 | policy: AwsCustomResourcePolicy.fromStatements([ 448 | new PolicyStatement({ 449 | actions: [ 450 | "iam:CreateServiceLinkedRole", 451 | "inspector2:Enable", 452 | "ecr:PutRegistryScanningConfiguration", 453 | ], 454 | resources: AwsCustomResourcePolicy.ANY_RESOURCE, 455 | }), 456 | ]), 457 | }), 458 | [ 459 | { 460 | id: "AwsSolutions-IAM5", 461 | reason: "Call needs access to all resources", 462 | }, 463 | ], 464 | true 465 | ); 466 | 467 | // Security group will open up to multiple ports based on how many workers you have set 468 | const WorkerSecurityGroup = new SecurityGroup( 469 | this, 470 | "Worker Security Group", 471 | { vpc: this.vpc } 472 | ); 473 | WorkerSecurityGroup.addIngressRule( 474 | Peer.ipv4(client.cidr), 475 | Port.tcpRange(9000, 9000 + NWORKERS * 2 - 1), 476 | "Allow Scheduler connect to Workers" 477 | ); 478 | WorkerSecurityGroup.addIngressRule( 479 | Peer.ipv4(this.vpc.vpcCidrBlock), 480 | Port.tcpRange(9000, 9000 + NWORKERS * 2 - 1), 481 | "Allow Workers in this region to talk to themselves" 482 | ); 483 | 484 | // Spin up the below service on ECS 485 | const ec2s = new Ec2Service(this, "Workers", { 486 | serviceName: "Dask-Workers-ecs", 487 | enableExecuteCommand: true, 488 | taskDefinition, 489 | cluster, 490 | securityGroups: [WorkerSecurityGroup], 491 | }); 492 | // We configure the autoscaling activies below. Note that test two tasks can work on a single instance 493 | const autoScalingGroup = ec2s.autoScaleTaskCount({ 494 | minCapacity: 1, 495 | maxCapacity: 16, 496 | }); 497 | // Scaling on CPU when it's above 40%, you can vary cooling periods as you see fit 498 | // The below will scale up when 3 consecutive data points is above 40% 499 | // And scale down when 15 datapoints are below ~37% 500 | autoScalingGroup.scaleOnCpuUtilization("CPUScaling", { 501 | targetUtilizationPercent: 40, 502 | scaleOutCooldown: Duration.minutes(5), 503 | scaleInCooldown: Duration.minutes(5), 504 | }); 505 | 506 | NagSuppressions.addResourceSuppressions( 507 | taskDefinition, 508 | [ 509 | { 510 | id: "AwsSolutions-IAM5", 511 | reason: 512 | "Basic role created to which allows ECS to publish dynamic logs and ssm messages", 513 | }, 514 | ], 515 | true 516 | ); 517 | 518 | // On the first launch of this CDK we would like to trigger the job immediately to sync. 519 | // It's positioning at the bottom is to give the data association enough time to link before triggering 520 | new AwsCustomResource(this, "Trigger Sync Job Now", { 521 | onCreate: { 522 | service: "Lambda", 523 | action: "invoke", 524 | physicalResourceId: { id: "invokeLustreDataRepoSyncTask" }, 525 | parameters: { 526 | FunctionName: this.RepoFn.functionName, 527 | }, 528 | }, 529 | policy: AwsCustomResourcePolicy.fromStatements([ 530 | new PolicyStatement({ 531 | actions: ["lambda:InvokeFunction"], 532 | resources: [ 533 | `arn:aws:lambda:${this.region}:${this.account}:function:${this.RepoFn.functionName}`, 534 | ], 535 | }), 536 | ]), 537 | }).node.addDependency(this.dataLink, this.RepoFn, this.lustre, ec2s); 538 | } 539 | } 540 | -------------------------------------------------------------------------------- /lib/WorkerConstructs/worker-region-tgw-route.ts: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: MIT-0 3 | 4 | import { Stack, App, StackProps } from "aws-cdk-lib"; 5 | import { 6 | CfnTransitGateway, 7 | CfnTransitGatewayPeeringAttachment, 8 | CfnTransitGatewayRoute, 9 | Vpc, 10 | } from "aws-cdk-lib/aws-ec2"; 11 | import { IClient } from "../../bin/interface"; 12 | import { TransitGatewayRouteTable } from "../SdkConstructs/default-transit-route-table-id"; 13 | import { NagSuppressions } from "cdk-nag"; 14 | 15 | export interface WorkerRegionTGWRouteProps extends StackProps { 16 | client: IClient; 17 | tgw: CfnTransitGateway; 18 | attachmentId: CfnTransitGatewayPeeringAttachment; 19 | } 20 | 21 | /** 22 | * This stack is purposefully seperate to the worker in that it must wait until the peer between the client 23 | * and worker regions has been established in order to the add a route from the region to the tgw attachment, 24 | * which points to the client 25 | */ 26 | export class WorkerRegionTransitGatewayRoute extends Stack { 27 | vpc: Vpc; 28 | 29 | constructor(scope: App, id: string, props: WorkerRegionTGWRouteProps) { 30 | super(scope, id, props); 31 | const { client, tgw, attachmentId } = props; 32 | 33 | NagSuppressions.addStackSuppressions(this, [ 34 | { 35 | id: "AwsSolutions-IAM4", 36 | reason: 37 | "Lambda execution policy for custom resources created by higher level CDK constructs", 38 | appliesTo: [ 39 | "Policy::arn::iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", 40 | ], 41 | }, 42 | { 43 | id: "AwsSolutions-L1", 44 | reason: "AWS CDK custom resources uses node 14.x environment", 45 | }, 46 | ]); 47 | 48 | // Load in the default route table associated to the tgw we wish to modify 49 | const transitConstruct = new TransitGatewayRouteTable( 50 | this, 51 | "Transit Gateway", 52 | { 53 | parameterName: tgw.ref, 54 | region: this.region, 55 | } 56 | ); 57 | const transitGatewayRouteTableId = transitConstruct.getParameterValue(); 58 | 59 | NagSuppressions.addResourceSuppressions( 60 | transitConstruct, 61 | [ 62 | { 63 | id: "AwsSolutions-L1", 64 | reason: 65 | "This will be updated when the Construct for this lambda is updated https://github.com/aws/aws-cdk/blob/62d7bf83b4bfe6358e86ecf1c332e51a3909bd8a/packages/%40aws-cdk/custom-resources/lib/aws-custom-resource/aws-custom-resource.ts#L397", 66 | }, 67 | { 68 | id: "AwsSolutions-IAM5", 69 | reason: "By default the action specified supports all resource types", 70 | }, 71 | ], 72 | true 73 | ); 74 | 75 | // Append the route 76 | new CfnTransitGatewayRoute(this, "TGW Route", { 77 | transitGatewayRouteTableId, 78 | destinationCidrBlock: client.cidr, 79 | transitGatewayAttachmentId: attachmentId.attrTransitGatewayAttachmentId, 80 | }); 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /lib/WorkerConstructs/worker-to-worker-tgw.ts: -------------------------------------------------------------------------------- 1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | // SPDX-License-Identifier: MIT-0 3 | 4 | import { Stack, App, StackProps } from "aws-cdk-lib"; 5 | import { 6 | CfnRoute, 7 | CfnTransitGatewayPeeringAttachment, 8 | Vpc, 9 | } from "aws-cdk-lib/aws-ec2"; 10 | import { StringParameter } from "aws-cdk-lib/aws-ssm"; 11 | import { NagSuppressions } from "cdk-nag"; 12 | import { IWorker } from "../../bin/interface"; 13 | import { SSMParameterReader } from "../SdkConstructs/ssm-param-reader"; 14 | import { AcceptTGWRequestClient } from "../SdkConstructs/accept-tgw-request-client"; 15 | 16 | export interface WorkerToWorkerTGWProps extends StackProps { 17 | peerWorker: IWorker; 18 | vpc: Vpc; 19 | } 20 | 21 | /** 22 | * This class creates the peer connection between the regions, accepts the request 23 | * and adds the relevant routes to the private subnets 24 | */ 25 | export class WorkerToWorkerTGW extends Stack { 26 | public attachmentID: CfnTransitGatewayPeeringAttachment; 27 | public transitGatewayId: string; 28 | 29 | constructor(scope: App, id: string, props: WorkerToWorkerTGWProps) { 30 | super(scope, id, props); 31 | const { peerWorker, vpc } = props; 32 | 33 | NagSuppressions.addStackSuppressions(this, [ 34 | { 35 | id: "AwsSolutions-IAM4", 36 | reason: 37 | "Lambda execution policy for custom resources created by higher level CDK constructs", 38 | appliesTo: [ 39 | "Policy::arn::iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", 40 | ], 41 | }, 42 | { 43 | id: "AwsSolutions-L1", 44 | reason: "AWS CDK custom resources uses node 14.x environment", 45 | }, 46 | ]); 47 | 48 | // Pull the exisitng TGW from the peer region we wish to connect to 49 | const peerTransitGatewayId = new SSMParameterReader( 50 | this, 51 | `TGW Param - ${peerWorker.region}`, 52 | { 53 | parameterName: `tgw-param-${peerWorker.region}`, 54 | region: peerWorker.region, 55 | account: this.account, 56 | } 57 | ).getParameterValue(); 58 | // Pull the local TGW 59 | const transitGatewayId = new SSMParameterReader( 60 | this, 61 | `TGW Param - ${this.region}`, 62 | { 63 | parameterName: `tgw-param-${this.region}`, 64 | region: this.region, 65 | account: this.account, 66 | } 67 | ).getParameterValue(); 68 | // Create the peering attachment 69 | this.attachmentID = new CfnTransitGatewayPeeringAttachment( 70 | this, 71 | "Peering Connection", 72 | { 73 | peerAccountId: this.account, 74 | peerRegion: peerWorker.region, 75 | peerTransitGatewayId, 76 | transitGatewayId, 77 | } 78 | ); 79 | // This attachment ID will be required by the peer region for the TGW route table 80 | new StringParameter(this, `Peering ID - ${peerWorker.region}`, { 81 | parameterName: `tgw-attachmentid-${this.region}-${peerWorker.region}`, 82 | stringValue: this.attachmentID.attrTransitGatewayAttachmentId, 83 | }); 84 | // Accept the request to peer from the peer region 85 | this.acceptRequest(peerWorker.region); 86 | 87 | // Loop the private subnets adding the route for worker region cidrs to the TGW 88 | for (let i = 0; i < vpc.privateSubnets.length; i++) { 89 | new CfnRoute(this, `Subnet to TGW - ${vpc.privateSubnets[i]}`, { 90 | routeTableId: vpc.privateSubnets[i].routeTable.routeTableId, 91 | destinationCidrBlock: peerWorker.cidr, 92 | transitGatewayId, 93 | }).addDependsOn(this.attachmentID); 94 | } 95 | } 96 | 97 | /** 98 | * Accept from the peer region the request to peer 99 | * 100 | * @param peerRegion - The peering region. e.g. us-west-2 101 | */ 102 | acceptRequest(peerRegion: string) { 103 | new AcceptTGWRequestClient(this, "Accept Request", { 104 | attachmentId: this.attachmentID.attrTransitGatewayAttachmentId, 105 | region: peerRegion, 106 | account: this.account, 107 | }); 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "cross-region-dask-aws", 3 | "version": "0.1.0", 4 | "lockfileVersion": 3, 5 | "requires": true, 6 | "packages": { 7 | "": { 8 | "name": "cross-region-dask-aws", 9 | "version": "0.1.0", 10 | "dependencies": { 11 | "aws-cdk-lib": "^2.53.0", 12 | "cdk-nag": "^2.21.17", 13 | "constructs": "^10.0.0", 14 | "source-map-support": "^0.5.21" 15 | }, 16 | "bin": { 17 | "cross-region-dask-aws": "bin/cross-region-dask-aws.js" 18 | }, 19 | "devDependencies": { 20 | "@types/node": "10.17.27", 21 | "@types/prettier": "2.6.0", 22 | "aws-cdk": "2.50.0", 23 | "ts-node": "^10.9.1", 24 | "typescript": "~3.9.7" 25 | } 26 | }, 27 | "node_modules/@aws-cdk/asset-awscli-v1": { 28 | "version": "2.2.200", 29 | "resolved": "https://registry.npmjs.org/@aws-cdk/asset-awscli-v1/-/asset-awscli-v1-2.2.200.tgz", 30 | "integrity": "sha512-Kf5J8DfJK4wZFWT2Myca0lhwke7LwHcHBo+4TvWOGJrFVVKVuuiLCkzPPRBQQVDj0Vtn2NBokZAz8pfMpAqAKg==" 31 | }, 32 | "node_modules/@aws-cdk/asset-kubectl-v20": { 33 | "version": "2.1.2", 34 | "resolved": "https://registry.npmjs.org/@aws-cdk/asset-kubectl-v20/-/asset-kubectl-v20-2.1.2.tgz", 35 | "integrity": "sha512-3M2tELJOxQv0apCIiuKQ4pAbncz9GuLwnKFqxifWfe77wuMxyTRPmxssYHs42ePqzap1LT6GDcPygGs+hHstLg==" 36 | }, 37 | "node_modules/@aws-cdk/asset-node-proxy-agent-v6": { 38 | "version": "2.0.1", 39 | "resolved": "https://registry.npmjs.org/@aws-cdk/asset-node-proxy-agent-v6/-/asset-node-proxy-agent-v6-2.0.1.tgz", 40 | "integrity": "sha512-DDt4SLdLOwWCjGtltH4VCST7hpOI5DzieuhGZsBpZ+AgJdSI2GCjklCXm0GCTwJG/SolkL5dtQXyUKgg9luBDg==" 41 | }, 42 | "node_modules/@cspotcode/source-map-support": { 43 | "version": "0.8.1", 44 | "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", 45 | "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", 46 | "dev": true, 47 | "dependencies": { 48 | "@jridgewell/trace-mapping": "0.3.9" 49 | }, 50 | "engines": { 51 | "node": ">=12" 52 | } 53 | }, 54 | "node_modules/@jridgewell/resolve-uri": { 55 | "version": "3.1.1", 56 | "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.1.tgz", 57 | "integrity": "sha512-dSYZh7HhCDtCKm4QakX0xFpsRDqjjtZf/kjI/v3T3Nwt5r8/qz/M19F9ySyOqU94SXBmeG9ttTul+YnR4LOxFA==", 58 | "dev": true, 59 | "engines": { 60 | "node": ">=6.0.0" 61 | } 62 | }, 63 | "node_modules/@jridgewell/sourcemap-codec": { 64 | "version": "1.4.15", 65 | "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz", 66 | "integrity": "sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==", 67 | "dev": true 68 | }, 69 | "node_modules/@jridgewell/trace-mapping": { 70 | "version": "0.3.9", 71 | "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz", 72 | "integrity": "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==", 73 | "dev": true, 74 | "dependencies": { 75 | "@jridgewell/resolve-uri": "^3.0.3", 76 | "@jridgewell/sourcemap-codec": "^1.4.10" 77 | } 78 | }, 79 | "node_modules/@tsconfig/node10": { 80 | "version": "1.0.9", 81 | "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.9.tgz", 82 | "integrity": "sha512-jNsYVVxU8v5g43Erja32laIDHXeoNvFEpX33OK4d6hljo3jDhCBDhx5dhCCTMWUojscpAagGiRkBKxpdl9fxqA==", 83 | "dev": true 84 | }, 85 | "node_modules/@tsconfig/node12": { 86 | "version": "1.0.11", 87 | "resolved": "https://registry.npmjs.org/@tsconfig/node12/-/node12-1.0.11.tgz", 88 | "integrity": "sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==", 89 | "dev": true 90 | }, 91 | "node_modules/@tsconfig/node14": { 92 | "version": "1.0.3", 93 | "resolved": "https://registry.npmjs.org/@tsconfig/node14/-/node14-1.0.3.tgz", 94 | "integrity": "sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==", 95 | "dev": true 96 | }, 97 | "node_modules/@tsconfig/node16": { 98 | "version": "1.0.4", 99 | "resolved": "https://registry.npmjs.org/@tsconfig/node16/-/node16-1.0.4.tgz", 100 | "integrity": "sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==", 101 | "dev": true 102 | }, 103 | "node_modules/@types/node": { 104 | "version": "10.17.27", 105 | "resolved": "https://registry.npmjs.org/@types/node/-/node-10.17.27.tgz", 106 | "integrity": "sha512-J0oqm9ZfAXaPdwNXMMgAhylw5fhmXkToJd06vuDUSAgEDZ/n/69/69UmyBZbc+zT34UnShuDSBqvim3SPnozJg==", 107 | "dev": true 108 | }, 109 | "node_modules/@types/prettier": { 110 | "version": "2.6.0", 111 | "resolved": "https://registry.npmjs.org/@types/prettier/-/prettier-2.6.0.tgz", 112 | "integrity": "sha512-G/AdOadiZhnJp0jXCaBQU449W2h716OW/EoXeYkCytxKL06X1WCXB4DZpp8TpZ8eyIJVS1cw4lrlkkSYU21cDw==", 113 | "dev": true 114 | }, 115 | "node_modules/acorn": { 116 | "version": "8.10.0", 117 | "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.10.0.tgz", 118 | "integrity": "sha512-F0SAmZ8iUtS//m8DmCTA0jlh6TDKkHQyK6xc6V4KDTyZKA9dnvX9/3sRTVQrWm79glUAZbnmmNcdYwUIHWVybw==", 119 | "dev": true, 120 | "bin": { 121 | "acorn": "bin/acorn" 122 | }, 123 | "engines": { 124 | "node": ">=0.4.0" 125 | } 126 | }, 127 | "node_modules/acorn-walk": { 128 | "version": "8.2.0", 129 | "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.2.0.tgz", 130 | "integrity": "sha512-k+iyHEuPgSw6SbuDpGQM+06HQUa04DZ3o+F6CSzXMvvI5KMvnaEqXe+YVe555R9nn6GPt404fos4wcgpw12SDA==", 131 | "dev": true, 132 | "engines": { 133 | "node": ">=0.4.0" 134 | } 135 | }, 136 | "node_modules/arg": { 137 | "version": "4.1.3", 138 | "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz", 139 | "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==", 140 | "dev": true 141 | }, 142 | "node_modules/aws-cdk": { 143 | "version": "2.50.0", 144 | "resolved": "https://registry.npmjs.org/aws-cdk/-/aws-cdk-2.50.0.tgz", 145 | "integrity": "sha512-55vmKTf2DZRqioumVfXn+S0H9oAbpRK3HFHY8EjZ5ykR5tq2+XiMWEZkYduX2HJhVAeHJJIS6h+Okk3smZjeqw==", 146 | "dev": true, 147 | "bin": { 148 | "cdk": "bin/cdk" 149 | }, 150 | "engines": { 151 | "node": ">= 14.15.0" 152 | }, 153 | "optionalDependencies": { 154 | "fsevents": "2.3.2" 155 | } 156 | }, 157 | "node_modules/aws-cdk-lib": { 158 | "version": "2.102.0", 159 | "resolved": "https://registry.npmjs.org/aws-cdk-lib/-/aws-cdk-lib-2.102.0.tgz", 160 | "integrity": "sha512-pYcKGlshU2j7n3f8TbJ1CCrwNnLsgGd17G7p/s9njIU8xakU4tIwuNyo4Q9HHQA7aUb3enPI/afAn1A6gp7TrA==", 161 | "bundleDependencies": [ 162 | "@balena/dockerignore", 163 | "case", 164 | "fs-extra", 165 | "ignore", 166 | "jsonschema", 167 | "minimatch", 168 | "punycode", 169 | "semver", 170 | "table", 171 | "yaml" 172 | ], 173 | "dependencies": { 174 | "@aws-cdk/asset-awscli-v1": "^2.2.200", 175 | "@aws-cdk/asset-kubectl-v20": "^2.1.2", 176 | "@aws-cdk/asset-node-proxy-agent-v6": "^2.0.1", 177 | "@balena/dockerignore": "^1.0.2", 178 | "case": "1.6.3", 179 | "fs-extra": "^11.1.1", 180 | "ignore": "^5.2.4", 181 | "jsonschema": "^1.4.1", 182 | "minimatch": "^3.1.2", 183 | "punycode": "^2.3.0", 184 | "semver": "^7.5.4", 185 | "table": "^6.8.1", 186 | "yaml": "1.10.2" 187 | }, 188 | "engines": { 189 | "node": ">= 14.15.0" 190 | }, 191 | "peerDependencies": { 192 | "constructs": "^10.0.0" 193 | } 194 | }, 195 | "node_modules/aws-cdk-lib/node_modules/@balena/dockerignore": { 196 | "version": "1.0.2", 197 | "inBundle": true, 198 | "license": "Apache-2.0" 199 | }, 200 | "node_modules/aws-cdk-lib/node_modules/ajv": { 201 | "version": "8.12.0", 202 | "inBundle": true, 203 | "license": "MIT", 204 | "dependencies": { 205 | "fast-deep-equal": "^3.1.1", 206 | "json-schema-traverse": "^1.0.0", 207 | "require-from-string": "^2.0.2", 208 | "uri-js": "^4.2.2" 209 | }, 210 | "funding": { 211 | "type": "github", 212 | "url": "https://github.com/sponsors/epoberezkin" 213 | } 214 | }, 215 | "node_modules/aws-cdk-lib/node_modules/ansi-regex": { 216 | "version": "5.0.1", 217 | "inBundle": true, 218 | "license": "MIT", 219 | "engines": { 220 | "node": ">=8" 221 | } 222 | }, 223 | "node_modules/aws-cdk-lib/node_modules/ansi-styles": { 224 | "version": "4.3.0", 225 | "inBundle": true, 226 | "license": "MIT", 227 | "dependencies": { 228 | "color-convert": "^2.0.1" 229 | }, 230 | "engines": { 231 | "node": ">=8" 232 | }, 233 | "funding": { 234 | "url": "https://github.com/chalk/ansi-styles?sponsor=1" 235 | } 236 | }, 237 | "node_modules/aws-cdk-lib/node_modules/astral-regex": { 238 | "version": "2.0.0", 239 | "inBundle": true, 240 | "license": "MIT", 241 | "engines": { 242 | "node": ">=8" 243 | } 244 | }, 245 | "node_modules/aws-cdk-lib/node_modules/balanced-match": { 246 | "version": "1.0.2", 247 | "inBundle": true, 248 | "license": "MIT" 249 | }, 250 | "node_modules/aws-cdk-lib/node_modules/brace-expansion": { 251 | "version": "1.1.11", 252 | "inBundle": true, 253 | "license": "MIT", 254 | "dependencies": { 255 | "balanced-match": "^1.0.0", 256 | "concat-map": "0.0.1" 257 | } 258 | }, 259 | "node_modules/aws-cdk-lib/node_modules/case": { 260 | "version": "1.6.3", 261 | "inBundle": true, 262 | "license": "(MIT OR GPL-3.0-or-later)", 263 | "engines": { 264 | "node": ">= 0.8.0" 265 | } 266 | }, 267 | "node_modules/aws-cdk-lib/node_modules/color-convert": { 268 | "version": "2.0.1", 269 | "inBundle": true, 270 | "license": "MIT", 271 | "dependencies": { 272 | "color-name": "~1.1.4" 273 | }, 274 | "engines": { 275 | "node": ">=7.0.0" 276 | } 277 | }, 278 | "node_modules/aws-cdk-lib/node_modules/color-name": { 279 | "version": "1.1.4", 280 | "inBundle": true, 281 | "license": "MIT" 282 | }, 283 | "node_modules/aws-cdk-lib/node_modules/concat-map": { 284 | "version": "0.0.1", 285 | "inBundle": true, 286 | "license": "MIT" 287 | }, 288 | "node_modules/aws-cdk-lib/node_modules/emoji-regex": { 289 | "version": "8.0.0", 290 | "inBundle": true, 291 | "license": "MIT" 292 | }, 293 | "node_modules/aws-cdk-lib/node_modules/fast-deep-equal": { 294 | "version": "3.1.3", 295 | "inBundle": true, 296 | "license": "MIT" 297 | }, 298 | "node_modules/aws-cdk-lib/node_modules/fs-extra": { 299 | "version": "11.1.1", 300 | "inBundle": true, 301 | "license": "MIT", 302 | "dependencies": { 303 | "graceful-fs": "^4.2.0", 304 | "jsonfile": "^6.0.1", 305 | "universalify": "^2.0.0" 306 | }, 307 | "engines": { 308 | "node": ">=14.14" 309 | } 310 | }, 311 | "node_modules/aws-cdk-lib/node_modules/graceful-fs": { 312 | "version": "4.2.11", 313 | "inBundle": true, 314 | "license": "ISC" 315 | }, 316 | "node_modules/aws-cdk-lib/node_modules/ignore": { 317 | "version": "5.2.4", 318 | "inBundle": true, 319 | "license": "MIT", 320 | "engines": { 321 | "node": ">= 4" 322 | } 323 | }, 324 | "node_modules/aws-cdk-lib/node_modules/is-fullwidth-code-point": { 325 | "version": "3.0.0", 326 | "inBundle": true, 327 | "license": "MIT", 328 | "engines": { 329 | "node": ">=8" 330 | } 331 | }, 332 | "node_modules/aws-cdk-lib/node_modules/json-schema-traverse": { 333 | "version": "1.0.0", 334 | "inBundle": true, 335 | "license": "MIT" 336 | }, 337 | "node_modules/aws-cdk-lib/node_modules/jsonfile": { 338 | "version": "6.1.0", 339 | "inBundle": true, 340 | "license": "MIT", 341 | "dependencies": { 342 | "universalify": "^2.0.0" 343 | }, 344 | "optionalDependencies": { 345 | "graceful-fs": "^4.1.6" 346 | } 347 | }, 348 | "node_modules/aws-cdk-lib/node_modules/jsonschema": { 349 | "version": "1.4.1", 350 | "inBundle": true, 351 | "license": "MIT", 352 | "engines": { 353 | "node": "*" 354 | } 355 | }, 356 | "node_modules/aws-cdk-lib/node_modules/lodash.truncate": { 357 | "version": "4.4.2", 358 | "inBundle": true, 359 | "license": "MIT" 360 | }, 361 | "node_modules/aws-cdk-lib/node_modules/lru-cache": { 362 | "version": "6.0.0", 363 | "inBundle": true, 364 | "license": "ISC", 365 | "dependencies": { 366 | "yallist": "^4.0.0" 367 | }, 368 | "engines": { 369 | "node": ">=10" 370 | } 371 | }, 372 | "node_modules/aws-cdk-lib/node_modules/minimatch": { 373 | "version": "3.1.2", 374 | "inBundle": true, 375 | "license": "ISC", 376 | "dependencies": { 377 | "brace-expansion": "^1.1.7" 378 | }, 379 | "engines": { 380 | "node": "*" 381 | } 382 | }, 383 | "node_modules/aws-cdk-lib/node_modules/punycode": { 384 | "version": "2.3.0", 385 | "inBundle": true, 386 | "license": "MIT", 387 | "engines": { 388 | "node": ">=6" 389 | } 390 | }, 391 | "node_modules/aws-cdk-lib/node_modules/require-from-string": { 392 | "version": "2.0.2", 393 | "inBundle": true, 394 | "license": "MIT", 395 | "engines": { 396 | "node": ">=0.10.0" 397 | } 398 | }, 399 | "node_modules/aws-cdk-lib/node_modules/semver": { 400 | "version": "7.5.4", 401 | "inBundle": true, 402 | "license": "ISC", 403 | "dependencies": { 404 | "lru-cache": "^6.0.0" 405 | }, 406 | "bin": { 407 | "semver": "bin/semver.js" 408 | }, 409 | "engines": { 410 | "node": ">=10" 411 | } 412 | }, 413 | "node_modules/aws-cdk-lib/node_modules/slice-ansi": { 414 | "version": "4.0.0", 415 | "inBundle": true, 416 | "license": "MIT", 417 | "dependencies": { 418 | "ansi-styles": "^4.0.0", 419 | "astral-regex": "^2.0.0", 420 | "is-fullwidth-code-point": "^3.0.0" 421 | }, 422 | "engines": { 423 | "node": ">=10" 424 | }, 425 | "funding": { 426 | "url": "https://github.com/chalk/slice-ansi?sponsor=1" 427 | } 428 | }, 429 | "node_modules/aws-cdk-lib/node_modules/string-width": { 430 | "version": "4.2.3", 431 | "inBundle": true, 432 | "license": "MIT", 433 | "dependencies": { 434 | "emoji-regex": "^8.0.0", 435 | "is-fullwidth-code-point": "^3.0.0", 436 | "strip-ansi": "^6.0.1" 437 | }, 438 | "engines": { 439 | "node": ">=8" 440 | } 441 | }, 442 | "node_modules/aws-cdk-lib/node_modules/strip-ansi": { 443 | "version": "6.0.1", 444 | "inBundle": true, 445 | "license": "MIT", 446 | "dependencies": { 447 | "ansi-regex": "^5.0.1" 448 | }, 449 | "engines": { 450 | "node": ">=8" 451 | } 452 | }, 453 | "node_modules/aws-cdk-lib/node_modules/table": { 454 | "version": "6.8.1", 455 | "inBundle": true, 456 | "license": "BSD-3-Clause", 457 | "dependencies": { 458 | "ajv": "^8.0.1", 459 | "lodash.truncate": "^4.4.2", 460 | "slice-ansi": "^4.0.0", 461 | "string-width": "^4.2.3", 462 | "strip-ansi": "^6.0.1" 463 | }, 464 | "engines": { 465 | "node": ">=10.0.0" 466 | } 467 | }, 468 | "node_modules/aws-cdk-lib/node_modules/universalify": { 469 | "version": "2.0.0", 470 | "inBundle": true, 471 | "license": "MIT", 472 | "engines": { 473 | "node": ">= 10.0.0" 474 | } 475 | }, 476 | "node_modules/aws-cdk-lib/node_modules/uri-js": { 477 | "version": "4.4.1", 478 | "inBundle": true, 479 | "license": "BSD-2-Clause", 480 | "dependencies": { 481 | "punycode": "^2.1.0" 482 | } 483 | }, 484 | "node_modules/aws-cdk-lib/node_modules/yallist": { 485 | "version": "4.0.0", 486 | "inBundle": true, 487 | "license": "ISC" 488 | }, 489 | "node_modules/aws-cdk-lib/node_modules/yaml": { 490 | "version": "1.10.2", 491 | "inBundle": true, 492 | "license": "ISC", 493 | "engines": { 494 | "node": ">= 6" 495 | } 496 | }, 497 | "node_modules/buffer-from": { 498 | "version": "1.1.2", 499 | "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", 500 | "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==" 501 | }, 502 | "node_modules/cdk-nag": { 503 | "version": "2.27.170", 504 | "resolved": "https://registry.npmjs.org/cdk-nag/-/cdk-nag-2.27.170.tgz", 505 | "integrity": "sha512-vk4PsB6R+RKudEG2DO7ylr4aGCPeJabFMsVo3+iNv0mRyaIo+MlVdGCqHlfjaYq94L9MknnwSeEPpY7Q7jNT+g==", 506 | "peerDependencies": { 507 | "aws-cdk-lib": "^2.78.0", 508 | "constructs": "^10.0.5" 509 | } 510 | }, 511 | "node_modules/constructs": { 512 | "version": "10.3.0", 513 | "resolved": "https://registry.npmjs.org/constructs/-/constructs-10.3.0.tgz", 514 | "integrity": "sha512-vbK8i3rIb/xwZxSpTjz3SagHn1qq9BChLEfy5Hf6fB3/2eFbrwt2n9kHwQcS0CPTRBesreeAcsJfMq2229FnbQ==", 515 | "engines": { 516 | "node": ">= 16.14.0" 517 | } 518 | }, 519 | "node_modules/create-require": { 520 | "version": "1.1.1", 521 | "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", 522 | "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", 523 | "dev": true 524 | }, 525 | "node_modules/diff": { 526 | "version": "4.0.2", 527 | "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", 528 | "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==", 529 | "dev": true, 530 | "engines": { 531 | "node": ">=0.3.1" 532 | } 533 | }, 534 | "node_modules/fsevents": { 535 | "version": "2.3.2", 536 | "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", 537 | "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", 538 | "dev": true, 539 | "hasInstallScript": true, 540 | "optional": true, 541 | "os": [ 542 | "darwin" 543 | ], 544 | "engines": { 545 | "node": "^8.16.0 || ^10.6.0 || >=11.0.0" 546 | } 547 | }, 548 | "node_modules/make-error": { 549 | "version": "1.3.6", 550 | "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", 551 | "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==", 552 | "dev": true 553 | }, 554 | "node_modules/source-map": { 555 | "version": "0.6.1", 556 | "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", 557 | "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", 558 | "engines": { 559 | "node": ">=0.10.0" 560 | } 561 | }, 562 | "node_modules/source-map-support": { 563 | "version": "0.5.21", 564 | "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.21.tgz", 565 | "integrity": "sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==", 566 | "dependencies": { 567 | "buffer-from": "^1.0.0", 568 | "source-map": "^0.6.0" 569 | } 570 | }, 571 | "node_modules/ts-node": { 572 | "version": "10.9.1", 573 | "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.1.tgz", 574 | "integrity": "sha512-NtVysVPkxxrwFGUUxGYhfux8k78pQB3JqYBXlLRZgdGUqTO5wU/UyHop5p70iEbGhB7q5KmiZiU0Y3KlJrScEw==", 575 | "dev": true, 576 | "dependencies": { 577 | "@cspotcode/source-map-support": "^0.8.0", 578 | "@tsconfig/node10": "^1.0.7", 579 | "@tsconfig/node12": "^1.0.7", 580 | "@tsconfig/node14": "^1.0.0", 581 | "@tsconfig/node16": "^1.0.2", 582 | "acorn": "^8.4.1", 583 | "acorn-walk": "^8.1.1", 584 | "arg": "^4.1.0", 585 | "create-require": "^1.1.0", 586 | "diff": "^4.0.1", 587 | "make-error": "^1.1.1", 588 | "v8-compile-cache-lib": "^3.0.1", 589 | "yn": "3.1.1" 590 | }, 591 | "bin": { 592 | "ts-node": "dist/bin.js", 593 | "ts-node-cwd": "dist/bin-cwd.js", 594 | "ts-node-esm": "dist/bin-esm.js", 595 | "ts-node-script": "dist/bin-script.js", 596 | "ts-node-transpile-only": "dist/bin-transpile.js", 597 | "ts-script": "dist/bin-script-deprecated.js" 598 | }, 599 | "peerDependencies": { 600 | "@swc/core": ">=1.2.50", 601 | "@swc/wasm": ">=1.2.50", 602 | "@types/node": "*", 603 | "typescript": ">=2.7" 604 | }, 605 | "peerDependenciesMeta": { 606 | "@swc/core": { 607 | "optional": true 608 | }, 609 | "@swc/wasm": { 610 | "optional": true 611 | } 612 | } 613 | }, 614 | "node_modules/typescript": { 615 | "version": "3.9.10", 616 | "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.9.10.tgz", 617 | "integrity": "sha512-w6fIxVE/H1PkLKcCPsFqKE7Kv7QUwhU8qQY2MueZXWx5cPZdwFupLgKK3vntcK98BtNHZtAF4LA/yl2a7k8R6Q==", 618 | "dev": true, 619 | "bin": { 620 | "tsc": "bin/tsc", 621 | "tsserver": "bin/tsserver" 622 | }, 623 | "engines": { 624 | "node": ">=4.2.0" 625 | } 626 | }, 627 | "node_modules/v8-compile-cache-lib": { 628 | "version": "3.0.1", 629 | "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", 630 | "integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==", 631 | "dev": true 632 | }, 633 | "node_modules/yn": { 634 | "version": "3.1.1", 635 | "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", 636 | "integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==", 637 | "dev": true, 638 | "engines": { 639 | "node": ">=6" 640 | } 641 | } 642 | } 643 | } 644 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "cross-region-dask-aws", 3 | "version": "0.1.0", 4 | "bin": { 5 | "cross-region-dask-aws": "bin/cross-region-dask-aws.js" 6 | }, 7 | "scripts": { 8 | "build": "tsc", 9 | "watch": "tsc -w", 10 | "cdk": "cdk" 11 | }, 12 | "devDependencies": { 13 | "@types/node": "10.17.27", 14 | "@types/prettier": "2.6.0", 15 | "aws-cdk": "2.50.0", 16 | "ts-node": "^10.9.1", 17 | "typescript": "~3.9.7" 18 | }, 19 | "dependencies": { 20 | "aws-cdk-lib": "^2.53.0", 21 | "cdk-nag": "^2.21.17", 22 | "constructs": "^10.0.0", 23 | "source-map-support": "^0.5.21" 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES2018", 4 | "module": "commonjs", 5 | "lib": ["es2018"], 6 | "declaration": true, 7 | "strict": true, 8 | "noImplicitAny": true, 9 | "strictNullChecks": true, 10 | "noImplicitThis": true, 11 | "alwaysStrict": true, 12 | "noUnusedLocals": false, 13 | "noUnusedParameters": false, 14 | "noImplicitReturns": true, 15 | "noFallthroughCasesInSwitch": false, 16 | "inlineSourceMap": true, 17 | "inlineSources": true, 18 | "experimentalDecorators": true, 19 | "strictPropertyInitialization": false, 20 | "typeRoots": ["./node_modules/@types"] 21 | }, 22 | "exclude": ["node_modules", "cdk.out"] 23 | } 24 | --------------------------------------------------------------------------------