├── .bumpversion.cfg ├── .custom.mk.example ├── .github ├── PULL_REQUEST_TEMPLATE.md └── workflows │ ├── publish.yaml │ └── release.yaml ├── .gitignore ├── .pre-commit-config.yaml ├── .taskcat.yml ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── LOCAL_DEVELOPMENT.md ├── Makefile ├── README.md ├── cfn-publish.config ├── cfn ├── client-vpn.template ├── efs.template ├── iam.template ├── license-server.template ├── main.template ├── render-node.template ├── repository.template ├── security-groups-helper.template ├── vpc.template └── workstation.template ├── ci └── include.lst ├── docs ├── blender │ ├── addon-checkbox.png │ ├── blender-client.png │ ├── blender-file.png │ ├── blender-job.png │ ├── blender-output.png │ ├── blender-scene.png │ ├── blender.md │ ├── bmw-demo.png │ ├── bmw-job-submit.png │ ├── bmw-job.png │ ├── deadline-job.png │ └── install-addon.png ├── deadline │ ├── add-nodes.png │ ├── enable-super-mode.png │ ├── manage-groups.png │ ├── monitor-error.png │ ├── start-deadline.png │ ├── workers-group.png │ └── workers-setup.md └── develop │ ├── LOCAL_DEVELOPMENT.md │ ├── aws-dcc-vpn.png │ ├── aws-dcc.png │ ├── cloudformation-launch-stack.png │ ├── dcv-conn.png │ ├── dcv-subscription.png │ ├── teradici-conn.png │ └── teradici-subscription.png ├── requirements.txt └── scripts ├── 70-ec2-nvme-devices.rules ├── ebsnvme-id └── ec2-mount-ebs-volume.sh /.bumpversion.cfg: -------------------------------------------------------------------------------- 1 | [bumpversion] 2 | current_version = 1.6.9 3 | commit = False 4 | tag = False 5 | 6 | [bumpversion:file:cfn/main.template] 7 | -------------------------------------------------------------------------------- /.custom.mk.example: -------------------------------------------------------------------------------- 1 | AWS_REGION="" 2 | BUCKET_NAME="" 3 | STACK_NAME="" 4 | CREATE_VPN_ENDPOINT="" 5 | SERVER_CERT_ARN="" 6 | CLIENT_CERT_ARN="" 7 | WORKSTATION_SUBNET_PLACEMENT="" 8 | EC2_USER_PASSWORD="" 9 | RENDER_NODE_TARGET_CAPACITY="" 10 | RENDER_NODE_INSTANCE_TYPE="" 11 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | *Issue #, if available:* 2 | 3 | *Description of changes:* 4 | 5 | 6 | By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice. 7 | -------------------------------------------------------------------------------- /.github/workflows/publish.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Publish Version 3 | on: 4 | release: 5 | types: 6 | - created 7 | - edited 8 | jobs: 9 | publish: 10 | name: Publish Version 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@v2 14 | - name: Fetch Tags 15 | run: git fetch --depth=1 origin +refs/tags/*:refs/tags/* || true 16 | - name: Configure AWS credentials 17 | uses: aws-actions/configure-aws-credentials@v1 18 | with: 19 | aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} 20 | aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} 21 | aws-session-token: ${{ secrets.AWS_SESSION_TOKEN }} 22 | aws-region: ${{ secrets.REGION }} 23 | - name: Set Version 24 | id: version 25 | run: echo "VERSION=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_ENV 26 | # Cache 27 | - uses: actions/cache@v1 28 | with: 29 | path: ~/.cache/pip 30 | key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} 31 | restore-keys: | 32 | ${{ runner.os }}-pip- 33 | # Setup 34 | - name: Set up Python 3.7 35 | uses: actions/setup-python@v1 36 | with: 37 | python-version: 3.7 38 | - name: Install dependencies 39 | run: pip install -r requirements.txt 40 | # Package and Upload Archive 41 | - name: Build Release 42 | run: make cfn-publish-package 43 | - name: Upload artefact 44 | run: aws s3 cp packaged.zip s3://$CFN_BUCKET/aws-digital-content-creation-render-environment/$VERSION/aws-digital-content-creation-render-environment.zip 45 | env: 46 | CFN_BUCKET: ${{ secrets.CFN_BUCKET }} 47 | -------------------------------------------------------------------------------- /.github/workflows/release.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Release Version 3 | on: 4 | push: 5 | branches: 6 | - master 7 | jobs: 8 | release: 9 | name: Release Version 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v2 13 | - run: git fetch --depth=1 origin +refs/tags/*:refs/tags/* || true 14 | # Cache 15 | - uses: actions/cache@v1 16 | with: 17 | path: ~/.cache/pip 18 | key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} 19 | restore-keys: | 20 | ${{ runner.os }}-pip- 21 | # Setup 22 | - name: Set up Python 3.7 23 | uses: actions/setup-python@v1 24 | with: 25 | python-version: 3.7 26 | - name: Install Dependencies 27 | run: pip install -r requirements.txt 28 | # Release if required 29 | - name: Set Version 30 | id: version 31 | run: | 32 | function version { echo "$@" | awk -F. '{ printf("%d%03d%03d%03d\n", $1,$2,$3,$4); }'; } 33 | echo "THIS_VERSION=$(make version | sed s/^v//)" >> $GITHUB_ENV 34 | echo "THIS_VERSION_COMPARABLE=$(version $(make version | sed s/^v//))" >> $GITHUB_ENV 35 | echo "LATEST_VERSION_COMPARABLE=$(version $(git describe --tags $(git rev-list --tags --max-count=1) | sed s/^v// 2> /dev/null || echo '0'))" >> $GITHUB_ENV 36 | - name: Create Release 37 | id: create_release 38 | uses: actions/create-release@latest 39 | if: env.THIS_VERSION_COMPARABLE > env.LATEST_VERSION_COMPARABLE 40 | env: 41 | GITHUB_TOKEN: ${{ secrets.RELEASE_TOKEN }} 42 | with: 43 | tag_name: v${{ env.THIS_VERSION }} 44 | release_name: Release v${{ env.THIS_VERSION }} 45 | body: |- 46 | See the commits for a list of features included in this release 47 | draft: false 48 | prerelease: false 49 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Local development 2 | /artefacts/ 3 | /.idea* 4 | /.taskcat/ 5 | /taskcat_outputs/ 6 | /.custom.mk 7 | /cfn/packaged.template 8 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | # See https://pre-commit.com for more information 2 | # See https://pre-commit.com/hooks.html for more hooks 3 | 4 | repos: 5 | # General 6 | - repo: https://github.com/pre-commit/pre-commit-hooks 7 | rev: v3.2.0 8 | hooks: 9 | - id: check-case-conflict 10 | - id: detect-private-key 11 | - id: trailing-whitespace 12 | - id: end-of-file-fixer 13 | - id: mixed-line-ending 14 | args: 15 | - --fix=lf 16 | - id: check-yaml 17 | - id: pretty-format-json 18 | args: 19 | - --autofix 20 | - --indent=2 21 | - --no-sort-keys 22 | exclude: /package-lock\.json$ 23 | - id: check-added-large-files 24 | args: 25 | - --maxkb=1000 26 | - id: check-merge-conflict 27 | 28 | # Secrets 29 | - repo: https://github.com/awslabs/git-secrets 30 | rev: 80230afa8c8bdeac766a0fece36f95ffaa0be778 31 | hooks: 32 | - id: git-secrets 33 | entry: git-secrets --scan 34 | files: "." 35 | 36 | # CloudFormation 37 | - repo: https://github.com/aws-cloudformation/cfn-python-lint 38 | rev: v0.37.1 39 | hooks: 40 | - id: cfn-python-lint 41 | name: AWS CloudFormation Linter 42 | files: \.(template)$ 43 | args: [--ignore-checks=E3002, --ignore-checks=W4002] 44 | 45 | - repo: https://github.com/aws-cloudformation/rain 46 | rev: 46a8f63541d848c8b57c129babdd124f324fdd78 47 | hooks: 48 | - id: cfn-format 49 | files: \.template$ 50 | 51 | # Python 52 | - repo: https://github.com/pycqa/pylint 53 | rev: pylint-2.6.0 54 | hooks: 55 | - id: pylint 56 | args: 57 | - --errors-only 58 | 59 | - repo: https://github.com/PyCQA/isort 60 | rev: 5.5.4 61 | hooks: 62 | - id: isort 63 | 64 | - repo: https://github.com/psf/black 65 | rev: stable 66 | hooks: 67 | - id: black 68 | -------------------------------------------------------------------------------- /.taskcat.yml: -------------------------------------------------------------------------------- 1 | project: 2 | name: aws-digital-content-creation-render-environment 3 | owner: rezabekf@amazon.co.uk 4 | package_lambda: false 5 | regions: 6 | - us-east-1 7 | - us-east-2 8 | - us-west-2 9 | - ap-northeast-1 10 | - ap-southeast-1 11 | - eu-central-1 12 | - eu-west-1 13 | tests: 14 | test-g4dn.xlarge: 15 | template: ./cfn/packaged.template 16 | parameters: 17 | AvailabilityZones: $[taskcat_genaz_2] 18 | StackName: TaskcatStack 19 | EC2UserPassword: $[taskcat_genpass_8A] 20 | WorkstationInstanceType: g4dn.xlarge 21 | regions: 22 | - us-east-1 23 | - eu-west-1 24 | 25 | test-m5dn.xlarge: 26 | auth: 27 | default: builders 28 | template: ./cfn/packaged.template 29 | parameters: 30 | AvailabilityZones: $[taskcat_genaz_2] 31 | StackName: TaskcatStack 32 | EC2UserPassword: $[taskcat_genpass_8A] 33 | WorkstationInstanceType: m5dn.xlarge 34 | regions: 35 | - us-east-1 36 | - eu-west-1 37 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | ## Code of Conduct 2 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 3 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 4 | opensource-codeofconduct@amazon.com with any additional questions or comments. 5 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing Guidelines 2 | 3 | Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional 4 | documentation, we greatly value feedback and contributions from our community. 5 | 6 | Please read through this document before submitting any issues or pull requests to ensure we have all the necessary 7 | information to effectively respond to your bug report or contribution. 8 | 9 | 10 | ## Reporting Bugs/Feature Requests 11 | 12 | We welcome you to use the GitHub issue tracker to report bugs or suggest features. 13 | 14 | When filing an issue, please check existing open, or recently closed, issues to make sure somebody else hasn't already 15 | reported the issue. Please try to include as much information as you can. Details like these are incredibly useful: 16 | 17 | * A reproducible test case or series of steps 18 | * The version of our code being used 19 | * Any modifications you've made relevant to the bug 20 | * Anything unusual about your environment or deployment 21 | 22 | 23 | ## Contributing via Pull Requests 24 | Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that: 25 | 26 | 1. You are working against the latest source on the *master* branch. 27 | 2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already. 28 | 3. You open an issue to discuss any significant work - we would hate for your time to be wasted. 29 | 30 | To send us a pull request, please: 31 | 32 | 1. Fork the repository. 33 | 2. Modify the source; please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change. 34 | 3. Ensure local tests pass. 35 | 4. Commit to your fork using clear commit messages. 36 | 5. Send us a pull request, answering any default questions in the pull request interface. 37 | 6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation. 38 | 39 | GitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and 40 | [creating a pull request](https://help.github.com/articles/creating-a-pull-request/). 41 | 42 | 43 | ## Finding contributions to work on 44 | Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels (enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any 'help wanted' issues is a great place to start. 45 | 46 | 47 | ## Code of Conduct 48 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 49 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 50 | opensource-codeofconduct@amazon.com with any additional questions or comments. 51 | 52 | 53 | ## Security issue notifications 54 | If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public github issue. 55 | 56 | 57 | ## Licensing 58 | 59 | See the [LICENSE](LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution. 60 | 61 | We may ask you to sign a [Contributor License Agreement (CLA)](http://en.wikipedia.org/wiki/Contributor_License_Agreement) for larger changes. 62 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of 4 | this software and associated documentation files (the "Software"), to deal in 5 | the Software without restriction, including without limitation the rights to 6 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 7 | the Software, and to permit persons to whom the Software is furnished to do so. 8 | 9 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 10 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 11 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 12 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 13 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 14 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 15 | -------------------------------------------------------------------------------- /LOCAL_DEVELOPMENT.md: -------------------------------------------------------------------------------- 1 | # Local Development 2 | 3 | This section details how to run the solution locally and deploy your code changes from the command line. 4 | 5 | ### Pre-Requisites 6 | 7 | The following dependencies must be installed: 8 | 9 | - AWS CLI 10 | - Python >=3.8 and pip 11 | - virtualenv 12 | 13 | ### Build and Deploy from Source 14 | To deploy the solution manually from the source to your AWS account, run the following: 15 | 16 | 1. Set the env variables 17 | ```shell script 18 | BUCKET_NAME="" 19 | AWS_REGION="" 20 | ``` 21 | 1. Create S3 bucket 22 | ```shell script 23 | aws s3 mb s3://${BUCKET_NAME} --region ${AWS_REGION} 24 | ``` 25 | 1. Create a `.custom.mk` file and populate it with your own values 26 | ```shell script 27 | cp .custom.mk.example .custom.mk 28 | ``` 29 | |Variable Label|Example|Description| 30 | |--------------|-------|-----------| 31 | |AWS_REGION|eu-west-1|The AWS region to deploy the solution to| 32 | |BUCKET_NAME|my-unique-bucket-name|Use the same value from step above| 33 | |STACK_NAME|aws-dcc|The name of the stack| 34 | |CREATE_VPN_ENDPOINT|true|Determinate whatever Client VPN will be created| 35 | |SERVER_CERT_ARN|arn:aws:acm:xxxxx|Server ACM certificate ARN| 36 | |CLIENT_CERT_ARN|arn:aws:acm:xxxxx|Client ACM certificate ARN| 37 | |WORKSTATION_SUBNET_PLACEMENT|Private|Set to private if Client VPN is set to true| 38 | |EC2_USER_PASSWORD|MyStrongPassword|Password to access workstation| 39 | |RENDER_NODE_TARGET_CAPACITY|2|How many nodes to deploy?| 40 | |RENDER_NODE_INSTANCE_TYPE|c5.4xlarge,c4.4xlarge|Type of render node instances| 41 | 1. Initialize the local environment 42 | ```shell script 43 | make init 44 | ``` 45 | 1. Activate `virtualenv` environment. 46 | ```shell script 47 | source venv/bin/activate 48 | ``` 49 | 50 | ### Deploy the stack 51 | This will deploy the nested stack using the AWS CLI profile of the current shell. By default this will be the profile `default`. 52 | ```shell script 53 | make deploy 54 | ``` 55 | 56 | ### Test changes 57 | The following command will run `pre-commit` tests. This should be run before every new commit. 58 | ```shell script 59 | make test 60 | ``` 61 | 62 | ### Clean the virtual environment 63 | This command will delete the virtual environment and all installed packages install via `make init` 64 | ```shell script 65 | make clean 66 | ``` 67 | 68 | ### Delete the resources created via CloudFormation 69 | Below command will delete deployed stack 70 | ```shell script 71 | make delete 72 | ``` 73 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | SHELL := /bin/bash 2 | 3 | .PHONY : help init deploy test clean delete 4 | .DEFAULT: help 5 | 6 | # Check for .custom.mk file if exists 7 | CUSTOM_FILE ?= .custom.mk 8 | CUSTOM_EXAMPLE ?= .custom.mk.example 9 | ifneq ("$(wildcard $(CUSTOM_FILE))","") 10 | include $(CUSTOM_FILE) 11 | else ifneq ("$(wildcard $(CUSTOM_EXAMPLE))","") 12 | include $(CUSTOM_EXAMPLE) 13 | else 14 | $(error File `.custom.mk` doesnt exist, please create one.) 15 | endif 16 | 17 | help: 18 | @echo "init generate project for local development" 19 | @echo "deploy deploy solution from source" 20 | @echo "test run pre-commit checks" 21 | @echo "clean delete virtualenv and installed libraries" 22 | @echo "delete delete deployed stacks" 23 | 24 | # Install local dependencies and git hooks 25 | init: venv 26 | venv/bin/pre-commit install 27 | 28 | deploy: package 29 | @printf "\n--> Deploying %s template...\n" $(STACK_NAME) 30 | @aws cloudformation deploy \ 31 | --template-file ./cfn/packaged.template \ 32 | --stack-name $(STACK_NAME) \ 33 | --region $(AWS_REGION) \ 34 | --capabilities CAPABILITY_NAMED_IAM CAPABILITY_AUTO_EXPAND \ 35 | --parameter-overrides \ 36 | AvailabilityZones=$(AWS_REGION)a,$(AWS_REGION)b \ 37 | EC2UserPassword=$(EC2_USER_PASSWORD) \ 38 | RenderNodeTargetCapacity=$(RENDER_NODE_TARGET_CAPACITY) \ 39 | WorkstationAccessCIDR=$(WORKSTATION_ACCESS_CIDR) \ 40 | WorkstationConnectionManager=$(WORKSTATION_CONNECTION_MANAGER) \ 41 | CreateVPNEndpoint=$(CREATE_VPN_ENDPOINT) \ 42 | ServerCertificateArn=$(SERVER_CERT_ARN) \ 43 | ClientCertificateArn=$(CLIENT_CERT_ARN) 44 | 45 | package: 46 | @printf "\n--> Packaging and uploading templates to the %s S3 bucket ...\n" $(BUCKET_NAME) 47 | @aws cloudformation package \ 48 | --template-file ./cfn/main.template \ 49 | --s3-bucket $(BUCKET_NAME) \ 50 | --s3-prefix $(STACK_NAME) \ 51 | --output-template-file ./cfn/packaged.template \ 52 | --region $(AWS_REGION) 53 | 54 | # Package for cfn-publish CI 55 | cfn-publish-package: 56 | zip -r packaged.zip -@ < ci/include.lst 57 | 58 | # virtualenv setup 59 | venv: venv/bin/activate 60 | 61 | venv/bin/activate: requirements.txt 62 | test -d venv || virtualenv venv 63 | . venv/bin/activate; pip install -Ur requirements.txt 64 | touch venv/bin/activate 65 | 66 | test: 67 | pre-commit run --all-files 68 | 69 | version: 70 | @bumpversion --dry-run --list cfn/main.template | grep current_version | sed s/'^.*='// 71 | 72 | # Cleanup local build 73 | clean: 74 | rm -rf venv 75 | find . -iname "*.pyc" -delete 76 | 77 | delete: 78 | @printf "\n--> Deleting %s stack...\n" $(STACK_NAME) 79 | @aws cloudformation delete-stack \ 80 | --stack-name $(STACK_NAME) 81 | @printf "\n--> $(STACK_NAME) deletion has been submitted, check AWS CloudFormation Console for an update..." 82 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # AWS Digital Content Creation Render Environment (Linux Workstation) 2 | 3 | [![Build Status](https://github.com/aws-samples/aws-digital-content-creation-render-environment/workflows/Publish%20Version/badge.svg)](https://github.com/aws-samples/aws-digital-content-creation-render-environment/actions) 4 | 5 | This solution deploys an AWS Cloud environment that helps media and entertainment organizations with burst rendering workloads. This CloudFormation template will deploy and setup [AWS ThinkBox Deadline](https://www.awsthinkbox.com/deadline) Database and Repository, Workstation, License Server, and Render nodes. 6 | 7 | The solution can deploy either Teradici PCoIP or NiceDCV Linux AMI workstation. Also, there is an option to place workstation in a Private subnet and connect to it via AWS ClientVpn. 8 | 9 | Furthermore, the solution comes with [Blender](https://www.blender.org/) installation scripts, which is a popular open-source software for 3D modeling, animation, rendering and more... 10 | 11 | > Note: For Windows Render environment refer to this solution [AWS Content Creation Render Environment for Windows](https://github.com/aws-samples/aws-content-creation-render-environment-for-windows) 12 | 13 | ## Index 14 | 15 | - [Architecture](#architecture) 16 | - [Getting Started](#getting-started) 17 | - [Prerequisites](#prerequisites) 18 | - [Deployment](#deployment) 19 | - [Login to Workstation and Start Deadline application](#login-to-workstation-and-start-deadline-application) 20 | - [Connect to Workstation deployed in Private Subnet](#connect-to-workstation-deployed-in-private-subnet) 21 | - [Connect to Workstation deployed in Public Subnet](#connect-to-workstation-deployed-in-public-subnet) 22 | 23 | - [Render Something to Test the Setup](#render-something-to-test-the-setup) 24 | - [Local Development](#local-development) 25 | - [Running the tests](#running-the-tests) 26 | - [Clean up](#clean-up) 27 | - [Contributing](#contributing) 28 | 29 | ### Architecture 30 | 31 | **Workstation in Private Subnet** 32 | 33 | ![aws-dcc-vpn.png](docs/develop/aws-dcc-vpn.png) 34 | 35 | 36 | **Workstation in Public Subnet** 37 | ![architecture-png](docs/develop/aws-dcc.png) 38 | 39 | ## Getting Started 40 | 41 | ### Prerequisites 42 | 1. To deploy the application you will require an AWS account. If you don’t already have an AWS account, create one at by following the on-screen instructions. Your access to the AWS account must have IAM permissions to launch AWS CloudFormation templates that create IAM roles. 43 | 1. If deploying an [AWS Client VPN](https://docs.aws.amazon.com/vpn/latest/clientvpn-user/client-vpn-user-what-is.html) endpoint. You must generate a server/client certificate and upload those certificates to [AWS Certificate Manager](https://docs.aws.amazon.com/acm/latest/userguide/acm-overview.html). Click [here](https://docs.aws.amazon.com/vpn/latest/clientvpn-admin/client-authentication.html#mutual) and follow the instructions in Mutual authentication to create a client and server certificates. 44 | 1. Configuration to AWS Client VPN is done via [AWS CLI](https://aws.amazon.com/cli/). You need to [install](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-install.html) and [configure](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html) AWS CLI if you are planning to use Client VPN. 45 | 1. The solution is using instance AMI from AWS Marketplace. To be able to deploy the infrastructure, you need to subscribe and confirm T&C for particular AMI on the marketplace. 46 | 1. To subscribe for Teradici Cloud Access Software for CentOS 7 click [here](https://aws.amazon.com/marketplace/pp/B07CT11PCQ) 47 | - In the top right corner click on **Continue to Subscribe** 48 | ![teradici-subscription](docs/develop/teradici-subscription.png) 49 | - Log in to your AWS account 50 | - Accept Terms and wait for subscription to activate 51 | 1. Subscription for Nice DCV are not required and licensing is free of charge if running DCV server on EC2 instance. 52 | - [Link to marketplace](https://aws.amazon.com/marketplace/pp/B07XLPJWM6) 53 | ![dcv-subscription](docs/develop/dcv-subscription.png) 54 | ### Deployment 55 | 56 | The environment is deployed as an [AWS CloudFormation](https://aws.amazon.com/cloudformation) template. 57 | 58 | > **Note** 59 | You are responsible for the cost of the AWS services used while running this sample deployment. There is no additional cost for using this sample. For full details, see the pricing pages for each AWS service you will be using in this sample. Prices are subject to change. 60 | 61 | 1. Deploy the latest CloudFormation template by following the link below for your preferred AWS region: 62 | 63 | > It will take about 30-45 minutes for the deployment to finish. 64 | 65 | |Region|Launch Template| 66 | |------|---------------| 67 | |**US East (N. Virginia)** us-east-1|[![Launch Stack](docs/develop/cloudformation-launch-stack.png)](https://console.aws.amazon.com/cloudformation/home?region=us-east-1#/stacks/new?stackName=aws-digital-content-creation-render-environment&templateURL=https://s3.amazonaws.com/solution-builders-us-east-1/aws-digital-content-creation-render-environment/latest/main.template)| 68 | |**US East (Ohio)** us-east-2|[![Launch Stack](docs/develop/cloudformation-launch-stack.png)](https://console.aws.amazon.com/cloudformation/home?region=us-east-2#/stacks/new?stackName=aws-digital-content-creation-render-environment&templateURL=https://s3.amazonaws.com/solution-builders-us-east-2/aws-digital-content-creation-render-environment/latest/main.template)| 69 | |**US West (N. California)** us-west-1|[![Launch Stack](docs/develop/cloudformation-launch-stack.png)](https://console.aws.amazon.com/cloudformation/home?region=us-west-1#/stacks/new?stackName=aws-digital-content-creation-render-environment&templateURL=https://s3.amazonaws.com/solution-builders-us-west-1/aws-digital-content-creation-render-environment/latest/main.template)| 70 | |**US West (Oregon)** us-west-2|[![Launch Stack](docs/develop/cloudformation-launch-stack.png)](https://console.aws.amazon.com/cloudformation/home?region=us-west-2#/stacks/new?stackName=aws-digital-content-creation-render-environment&templateURL=https://s3.amazonaws.com/solution-builders-us-west-2/aws-digital-content-creation-render-environment/latest/main.template)| 71 | |**Europe (Ireland)** eu-west-1|[![Launch Stack](docs/develop/cloudformation-launch-stack.png)](https://console.aws.amazon.com/cloudformation/home?region=eu-west-1#/stacks/new?stackName=aws-digital-content-creation-render-environment&templateURL=https://s3.amazonaws.com/solution-builders-eu-west-1/aws-digital-content-creation-render-environment/latest/main.template)| 72 | |**Europe (London)** eu-west-2|[![Launch Stack](docs/develop/cloudformation-launch-stack.png)](https://console.aws.amazon.com/cloudformation/home?region=eu-west-2#/stacks/new?stackName=aws-digital-content-creation-render-environment&templateURL=https://s3.amazonaws.com/solution-builders-eu-west-2/aws-digital-content-creation-render-environment/latest/main.template)| 73 | |**Europe (Paris)** eu-west-3|[![Launch Stack](docs/develop/cloudformation-launch-stack.png)](https://console.aws.amazon.com/cloudformation/home?region=eu-west-3#/stacks/new?stackName=aws-digital-content-creation-render-environment&templateURL=https://s3.amazonaws.com/solution-builders-eu-west-3/aws-digital-content-creation-render-environment/latest/main.template)| 74 | |**Europe (Frankfurt)** eu-central-1|[![Launch Stack](docs/develop/cloudformation-launch-stack.png)](https://console.aws.amazon.com/cloudformation/home?region=eu-central-1#/stacks/new?stackName=aws-digital-content-creation-render-environment&templateURL=https://s3.amazonaws.com/solution-builders-eu-central-1/aws-digital-content-creation-render-environment/latest/main.template)| 75 | |**Europe (Stockholm)** eu-north-1|[![Launch Stack](docs/develop/cloudformation-launch-stack.png)](https://console.aws.amazon.com/cloudformation/home?region=eu-north-1#/stacks/new?stackName=aws-digital-content-creation-render-environment&templateURL=https://s3.amazonaws.com/solution-builders-eu-north-1/aws-digital-content-creation-render-environment/latest/main.template)| 76 | |**Asia Pacific (Mumbai)** ap-south-1|[![Launch Stack](docs/develop/cloudformation-launch-stack.png)](https://console.aws.amazon.com/cloudformation/home?region=ap-south-1#/stacks/new?stackName=aws-digital-content-creation-render-environment&templateURL=https://s3.amazonaws.com/solution-builders-ap-south-1/aws-digital-content-creation-render-environment/latest/main.template)| 77 | |**Asia Pacific (Tokyo)** ap-northeast-1|[![Launch Stack](docs/develop/cloudformation-launch-stack.png)](https://console.aws.amazon.com/cloudformation/home?region=ap-northeast-1#/stacks/new?stackName=aws-digital-content-creation-render-environment&templateURL=https://s3.amazonaws.com/solution-builders-ap-northeast-1/aws-digital-content-creation-render-environment/latest/main.template)| 78 | |**Asia Pacific (Sydney)** ap-southeast-2|[![Launch Stack](docs/develop/cloudformation-launch-stack.png)](https://console.aws.amazon.com/cloudformation/home?region=ap-southeast-2#/stacks/new?stackName=aws-digital-content-creation-render-environment&templateURL=https://s3.amazonaws.com/solution-builders-ap-southeast-2/aws-digital-content-creation-render-environment/latest/main.template)| 79 | |**Asia Pacific (Singapore)** ap-southeast-1|[![Launch Stack](docs/develop/cloudformation-launch-stack.png)](https://console.aws.amazon.com/cloudformation/home?region=ap-southeast-1#/stacks/new?stackName=aws-digital-content-creation-render-environment&templateURL=https://s3.amazonaws.com/solution-builders-ap-southeast-1/aws-digital-content-creation-render-environment/latest/main.template)| 80 | |**Asia Pacific (Seoul)** ap-northeast-2|[![Launch Stack](docs/develop/cloudformation-launch-stack.png)](https://console.aws.amazon.com/cloudformation/home?region=ap-northeast-2#/stacks/new?stackName=aws-digital-content-creation-render-environment&templateURL=https://s3.amazonaws.com/solution-builders-ap-northeast-2/aws-digital-content-creation-render-environment/latest/main.template)| 81 | |**South America (São Paulo)** sa-east-1|[![Launch Stack](docs/develop/cloudformation-launch-stack.png)](https://console.aws.amazon.com/cloudformation/home?region=sa-east-1#/stacks/new?stackName=aws-digital-content-creation-render-environment&templateURL=https://s3.amazonaws.com/solution-builders-sa-east-1/aws-digital-content-creation-render-environment/latest/main.template)| 82 | 83 | 1. If prompted, login using your AWS account credentials. 84 | 1. You should see a screen titled **Create Stack** at the **Specify template** step. The fields specifying the CloudFormation template are pre-populated. Click the **Next** button at the bottom of the page. 85 | 1. On the **Specify stack details** screen you may customize the following parameters of the CloudFormation stack: 86 | 1. Stack name has been pre-populated with `aws-digital-content-creation-render-environment` name. You can change this and provide own stack name. 87 | 88 | |Parameter label|Default|Description| 89 | |---------------|-------|-----------| 90 | |Availability Zones | `Requires input` | The list of Availability Zones to use for the subnets in the VPC. Select **two** Availability Zones from your list. | 91 | |VPC name | dcc-vpc | The name of the VPC. | 92 | |VPC CIDR | 10.0.0.0/16 | The CIDR block for the VPC. | 93 | |Public Subnet 1 CIDR | 10.0.0.0/24 | The CIDR block for the Public Subnet located in Availability Zone 1 of the VPC. | 94 | |Public Subnet 2 CIDR | 10.0.1.0/24 | The CIDR block for the Public Subnet located in Availability Zone 2 of the VPC. | 95 | |Private Subnet 1 CIDR | 10.0.2.0/24 | The CIDR block for the Private Subnet located in Availability Zone 1 of the VPC. | 96 | |Private Subnet 2 CIDR | 10.0.3.0/24 | The CIDR block for the Private Subnet located in Availability Zone 2 of the VPC. | 97 | |Create VPN Endpoint | false | Should the CloudFormation create a Client VPN Endpoint. If set to `true` ,workstation will be placed in Private Subnet and you have to set up server and client certificates in AWS Certificate Manager, please follow instructions how to set this up [here](https://docs.aws.amazon.com/vpn/latest/clientvpn-admin/client-authentication.html#mutual). | 98 | |Client CIDR for VPN Endpoint | 10.50.0.0/20 | If creating Client VPN endpoint in the solution, specify the IPv4 address range. It should be in CIDR notation from which to assign client IP addresses. The address range cannot overlap with the local CIDR of the VPC in which the associated subnet is located, or the routes that you add manually. | 99 | |ACM Server Certificate ARN | N/A | If Client VPN is set to `true`, please provide your server certificate ARN, e.g. `arn:aws:acm:xxxxx` | 100 | |ACM Client Certificate ARN | N/A | If Client VPN is set to `true`, please provide your client certificate ARN, e.g. `arn:aws:acm:xxxxx` | 101 | |Target Network CIDR for VPN Endpoint | 10.0.0.0/16 | If creating Client VPN endpoint in the solution, specify the IPv4 address range, in CIDR notation, of the network for which access is being authorized. | 102 | |Workstation instance type | g4dn.xlarge | The EC2 instance type for the Deadline workstation. | 103 | |Workstation connection manager | teradici | Specify whether you want to run Teradici PCoIP or NiceDcv server to connect to the workstation. | 104 | |Workstation EBS volume size| 100 | Volume size of workstation instance, in GiB. | 105 | |Workstation access CIDR| 10.0.0.0/16 | CIDR block of an on-premise IP address. Input your network's current public or private IP depending if the Workstation is being placed in a public or private subnet. | 106 | |EC2 user password| `Requires input` | The **ec2-user** password for remote access to NICEDCV workstation server and to access Deadline Repository samba share. | 107 | |Render scheduler instance type | m5.2xlarge | The EC2 instance type for the Deadline repository. | 108 | |Deadline application version | 10.1.3.6 | The version of Deadline application. | 109 | |License server instance type| m5.large | The EC2 instance type for the Deadline License server. | 110 | |License server version | 1.1 | The version of License server. | 111 | |Deadline license key | 123456789012 | If you have license key, add it here. | 112 | |Render node instance type | c5.4xlarge,c4.4xlarge | The EC2 instance type for the Deadline Render nodes | 113 | |Render node capacity | 2 | The number of instances in the Spot Fleet. | 114 | |ArtefactBucketName | aws-digital-content-creation-render-environment | S3 bucket name for the application assets. Contains Deadline installation software.| 115 | |Environment | DEV | The type of environment to tag your infrastructure with. You can specify DEV (development), TEST (test), or PROD (production). | 116 | 117 | 1. When completed, click **Next**. 118 | 1. [Configure stack options](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/cfn-console-add-tags.html) if desired, then click **Next**. 119 | 1. On the **Review** screen, you must check the boxes for: 120 | * "*I acknowledge that AWS CloudFormation might create IAM resources*" 121 | * "*I acknowledge that AWS CloudFormation might create IAM resources with custom names*" 122 | * "*I acknowledge that AWS CloudFormation might require the following capability: CAPABILITY_AUTO_EXPAND*" 123 | 124 | These are required to allow CloudFormation to create a Role to allow access to resources needed by the stack and name the resources in a dynamic way. 125 | 1. Click **Create Change Set**. 126 | 1. On the **Change Set** screen, click **Execute** to launch your stack. 127 | > You may need to wait for the *Execution status* of the change set to become **AVAILABLE** before the **Execute** button becomes available. 128 | 1. Wait for the CloudFormation stack to launch. Completion is indicated when the **Stack status** is **CREATE_COMPLETE**. 129 | > You can monitor the stack creation progress in the "Events" tab. 130 | 131 | ### Login to Workstation and Start Deadline application 132 | 133 | #### Connect to Workstation deployed in Public Subnet 134 | To connect to workstation in Public Subnet follow the steps below: 135 | 136 | 1. Note the **WorkstationIP** displayed in the **Outputs** tab of the **root** stack (it is the one without word **NESTED**). 137 | 1. Based on Connection manager selected: 138 | 1. *Teradici log in*: 139 | 1. Install the Teradici PCoIP client from [Teradici Software and Mobile Clients](https://docs.teradici.com/find/product/software-and-mobile-clients) 140 | 1. In PCoIP client for **Host Address or Code**, paste the **WorkstationIP** 141 | 1. Click **Next**, and in pop up window confirm **Connect Insecurely** 142 | 1. **UserName**: ec2-user 143 | 1. **Password**: provide same password as used in **EC2UserPassword** parameter 144 | 1. Login 145 | ![teradici-conn](docs/develop/teradici-conn.png) 146 | 1. *NiceDcv log in*: 147 | 1. On the web browser or in the [NICE DCV Client](https://download.nice-dcv.com/), paste the **WorkstationIP** address 148 | 1. **UserName**: ec2-user 149 | 1. **Password**: provide same password as used in **EC2UserPassword** parameter. 150 | 1. Login 151 | ![dcv-conn](docs/develop/dcv-conn.png) 152 | 153 | #### Connect to Workstation deployed in Private Subnet 154 | To connect to workstation in Private Subnet you will need to set up VPN client. 155 | 156 | ##### Step 1: Download the Client VPN Endpoint Configuration File 157 | The configuration file includes the Client VPN endpoint and certificate information required to establish a VPN connection. You must provide this file to the clients who need to connect to the Client VPN endpoint to establish a VPN connection. The client uploads this file into their VPN client application. 158 | 159 | **To download and prepare the Client VPN endpoint configuration file (AWS CLI)** 160 | 161 | 1. Note the **ClientVpnEndpointID** displayed in the **Outputs** tab of the **root** stack (it is the one without word **NESTED**). 162 | 1. Navigate to the folder with server and client certificates you have created as part of the [Prerequisites](#prerequisites) in step 2. 163 | ``` 164 | cd ~/custom_folder/ 165 | ``` 166 | 1. Download the Client VPN endpoint configuration file. 167 | ``` 168 | AWS_REGION="the region you deployed CloudFromtaion stack into, e.g. us-east-1" 169 | ENDPOINT_ID="paste the id from step one" 170 | 171 | aws ec2 export-client-vpn-client-configuration --client-vpn-endpoint-id ${ENDPOINT_ID} --output text > client-config.ovpn --region ${AWS_REGION} 172 | ``` 173 | 1. Append the client certificate and key to the Client VPN endpoint configuration file. 174 | ```bash 175 | cat >> client-config.ovpn 176 | cert ~/custom_folder/client1.domain.tld.crt 177 | key ~/custom_folder/client1.domain.tld.key 178 | 179 | CTRL+C 180 | ``` 181 | 1. Verify that you have all the files in `~/custom_folder/` directory. 182 | ``` 183 | ls -l 184 | 185 | ca.crt 186 | client-config.ovpn 187 | client1.domain.tld.crt 188 | client1.domain.tld.key 189 | server.crt 190 | server.key 191 | ``` 192 | ##### Step 2: Configure OpenVPN Client 193 | Now that we have VPN configuration file ready, the last step is to install VPN client on your machine. The following procedure shows how to establish a VPN connection using the `Tunnelblick` client application on a macOS computer. 194 | 195 | ```bash 196 | brew search tunnelblick 197 | brew cask install tunnelblick 198 | ``` 199 | 200 | Once the client is installed, please follow the 201 | [AWS Connect to VPN documentation](https://docs.aws.amazon.com/vpn/latest/clientvpn-user/macos.html). 202 | 203 | ##### Step 3: Connect to Deadline workstation instance 204 | Follow the same steps as in Connect to [Workstation deployed in Public Subnet](#connect-to-workstation-deployed-in-public-subnet) 205 | 206 | ## Render Something to Test the Setup 207 | 208 | Now you’ve got most of the pieces together, it’s a good idea to test the setup to make sure you can submit a render from your workstation, have the Render Scheduler pick it up, and run the render on your Worker. 209 | 210 | For this part, you will use [Blender](https://www.blender.org/) to render something. 211 | 212 | Follow the instructions to: 213 | 214 | * set up [Deadline Monitor](docs/deadline/workers-setup.md) 215 | * set up [Blender and submit a job](docs/blender/blender.md) 216 | 217 | ## Local Development 218 | See [Local Development](docs/develop/LOCAL_DEVELOPMENT.md) guide to get a copy of the project up and running on your local machine for development and testing purposes. 219 | 220 | ## Running the tests 221 | 222 | The solution has been tested using [taskcat](https://github.com/aws-quickstart/taskcat) the tool that tests AWS CloudFormation templates. 223 | 224 | Install taskcat 225 | ``` 226 | pip install taskcat 227 | ``` 228 | 229 | > You need to package the templates to S3 before you can run the test below. See [Getting Started](#getting-started) instructions. 230 | 231 | From within project root directory run: 232 | ``` 233 | taskat test run 234 | ``` 235 | 236 | You can modify the `taskcat` configuration file with your own tests here [.taskcat.yml](.taskcat.yml) 237 | 238 | ## Clean up 239 | 240 | Follow these steps to clean up created resources: 241 | 242 | 1. In the **[CloudFormation console](https://console.aws.amazon.com/cloudformation)**, select the **root** stack you have created. For example `aws-digital-content-creation-render-environment `. 243 | 1. The **root** stack will handle the deletion of all the **children** stacks for you. 244 | 1. In the top right corner, click on **Delete**. 245 | 1. In the pop up window click on **Delete stack**. 246 | 1. You can click the **refresh** button a few times until you see in the status **DELETE_COMPLETE**. 247 | 248 | ## Contributing 249 | 250 | Contributions are more than welcome. Please read the [code of conduct](CODE_OF_CONDUCT.md) and the [contributing guidelines](CONTRIBUTING.md). 251 | 252 | ## Authors 253 | 254 | See the list of [contributors](https://github.com/aws-samples/aws-digital-content-creation-render-environment/contributors) who participated in this project. 255 | 256 | ## License 257 | 258 | This library is licensed under the MIT-0 License. See the LICENSE file. - see the [LICENSE](LICENSE) file for details. 259 | -------------------------------------------------------------------------------- /cfn-publish.config: -------------------------------------------------------------------------------- 1 | bucket_name_prefix="solution-builders" 2 | acl="public-read" 3 | templates="cfn/main.template" 4 | regions="eu-north-1 ap-south-1 eu-west-3 eu-west-2 eu-west-1 ap-northeast-2 ap-northeast-1 sa-east-1 ap-southeast-1 ap-southeast-2 eu-central-1 us-east-1 us-east-2 us-west-1 us-west-2" 5 | -------------------------------------------------------------------------------- /cfn/client-vpn.template: -------------------------------------------------------------------------------- 1 | AWSTemplateFormatVersion: "2010-09-09" 2 | 3 | Description: This template provisions Client VPN Resources 4 | 5 | Parameters: 6 | ClientCidrBlock: 7 | Type: String 8 | AllowedPattern: ^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/([0-9]|[1-2][0-9]|3[0-2]))$ 9 | ConstraintDescription: CIDR block parameter must be in the form x.x.x.x/16-28 10 | 11 | ServerCertificateArn: 12 | Description: Specify Server Cert Arn for VPN endpoint. 13 | Type: String 14 | 15 | ClientCertificateArn: 16 | Description: Specify Client Cert Arn for VPN endpoint. 17 | Type: String 18 | 19 | SubnetID: 20 | Description: The ID of the subnet to associate with the Client VPN endpoint. 21 | Type: String 22 | 23 | TargetNetworkCidr: 24 | Description: The IPv4 address range, in CIDR notation, of the network for which access is being authorized. 25 | Type: String 26 | AllowedPattern: ^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/([0-9]|[1-2][0-9]|3[0-2]))$ 27 | ConstraintDescription: CIDR block parameter must be in the form x.x.x.x/16-28 28 | 29 | VPC: 30 | Description: The ID of the VPC to associate with the Client VPN endpoint. 31 | Type: String 32 | 33 | Resources: 34 | ClientVpnEndpoint: 35 | Type: AWS::EC2::ClientVpnEndpoint 36 | Properties: 37 | AuthenticationOptions: 38 | - MutualAuthentication: 39 | ClientRootCertificateChainArn: !Ref ClientCertificateArn 40 | Type: certificate-authentication 41 | ClientCidrBlock: !Ref ClientCidrBlock 42 | ConnectionLogOptions: 43 | Enabled: false 44 | Description: Client VPN to connect to instances in Private Subnet 1 45 | ServerCertificateArn: !Ref ServerCertificateArn 46 | VpcId: !Ref VPC 47 | SecurityGroupIds: 48 | - !Ref VPNSecurityGroup 49 | TagSpecifications: 50 | - ResourceType: client-vpn-endpoint 51 | Tags: 52 | - Key: Name 53 | Value: VFX-ClientVPN 54 | 55 | VpnEndpointTargetNetworkAssociation: 56 | Type: AWS::EC2::ClientVpnTargetNetworkAssociation 57 | Properties: 58 | ClientVpnEndpointId: !Ref ClientVpnEndpoint 59 | SubnetId: !Ref SubnetID 60 | 61 | VpnEndpointAuthorizationRule: 62 | Type: AWS::EC2::ClientVpnAuthorizationRule 63 | Properties: 64 | ClientVpnEndpointId: !Ref ClientVpnEndpoint 65 | AuthorizeAllGroups: true 66 | Description: Allow access to VFX Host subnet. 67 | TargetNetworkCidr: !Ref TargetNetworkCidr 68 | 69 | InternetAuthRule: 70 | Type: AWS::EC2::ClientVpnAuthorizationRule 71 | Properties: 72 | ClientVpnEndpointId: !Ref ClientVpnEndpoint 73 | AuthorizeAllGroups: true 74 | TargetNetworkCidr: 0.0.0.0/0 75 | Description: Allow access to internet 76 | 77 | InternetRoute: 78 | Type: AWS::EC2::ClientVpnRoute 79 | DependsOn: VpnEndpointTargetNetworkAssociation 80 | Properties: 81 | ClientVpnEndpointId: !Ref ClientVpnEndpoint 82 | TargetVpcSubnetId: !Ref SubnetID 83 | DestinationCidrBlock: 0.0.0.0/0 84 | Description: Route to the internet 85 | 86 | VPNSecurityGroup: 87 | Type: AWS::EC2::SecurityGroup 88 | Properties: 89 | GroupName: Security Group for VPN Clients 90 | GroupDescription: This security group is attached to the Client VPN Endpoint. 91 | VpcId: !Ref VPC 92 | 93 | VPNHostSecurityGroupIngress1: 94 | Type: AWS::EC2::SecurityGroupIngress 95 | Properties: 96 | GroupId: !Ref VPNSecurityGroup 97 | IpProtocol: "-1" 98 | SourceSecurityGroupId: !Ref VPNSecurityGroup 99 | 100 | Outputs: 101 | VPNSecurityGroupID: 102 | Description: VPN Security Group ID 103 | Value: !GetAtt VPNSecurityGroup.GroupId 104 | 105 | ClientVpnEndpointID: 106 | Value: !Ref ClientVpnEndpoint 107 | -------------------------------------------------------------------------------- /cfn/efs.template: -------------------------------------------------------------------------------- 1 | AWSTemplateFormatVersion: "2010-09-09" 2 | 3 | Description: Provides shared Elastic File System for Repo, Workstation and Render nodes. 4 | 5 | Parameters: 6 | VpcID: 7 | Type: String 8 | 9 | PrivateSubnet1: 10 | Type: String 11 | 12 | PrivateSubnet2: 13 | Type: String 14 | 15 | RepositorySecurityGroup: 16 | Type: AWS::EC2::SecurityGroup::Id 17 | 18 | RenderNodeSecurityGroup: 19 | Type: AWS::EC2::SecurityGroup::Id 20 | 21 | WorkstationSecurityGroup: 22 | Type: AWS::EC2::SecurityGroup::Id 23 | 24 | Environment: 25 | Type: String 26 | 27 | Resources: 28 | EfsSecurityGroup: 29 | Type: AWS::EC2::SecurityGroup 30 | Properties: 31 | VpcId: !Ref VpcID 32 | GroupDescription: EFS Security group for mount targets 33 | SecurityGroupIngress: 34 | - IpProtocol: tcp 35 | FromPort: 2049 36 | ToPort: 2049 37 | SourceSecurityGroupId: !Ref RepositorySecurityGroup 38 | - IpProtocol: tcp 39 | FromPort: 2049 40 | ToPort: 2049 41 | SourceSecurityGroupId: !Ref RenderNodeSecurityGroup 42 | - IpProtocol: tcp 43 | FromPort: 2049 44 | ToPort: 2049 45 | SourceSecurityGroupId: !Ref WorkstationSecurityGroup 46 | Tags: 47 | - Key: Name 48 | Value: deadline-efs-sg 49 | - Key: Environment 50 | Value: !Ref Environment 51 | 52 | EfsFileSystem: 53 | Type: AWS::EFS::FileSystem 54 | Properties: 55 | Encrypted: true 56 | PerformanceMode: generalPurpose 57 | FileSystemTags: 58 | - Key: Name 59 | Value: deadline-efs 60 | 61 | EfsMountTarget0: 62 | Type: AWS::EFS::MountTarget 63 | Properties: 64 | FileSystemId: !Ref EfsFileSystem 65 | SubnetId: !Ref PrivateSubnet1 66 | SecurityGroups: 67 | - !Ref EfsSecurityGroup 68 | - !Ref RepositorySecurityGroup 69 | - !Ref RenderNodeSecurityGroup 70 | - !Ref WorkstationSecurityGroup 71 | 72 | EfsMountTarget1: 73 | Type: AWS::EFS::MountTarget 74 | Properties: 75 | FileSystemId: !Ref EfsFileSystem 76 | SubnetId: !Ref PrivateSubnet2 77 | SecurityGroups: 78 | - !Ref EfsSecurityGroup 79 | - !Ref RepositorySecurityGroup 80 | - !Ref RenderNodeSecurityGroup 81 | - !Ref WorkstationSecurityGroup 82 | 83 | Outputs: 84 | EfsFileSystem: 85 | Value: !Ref EfsFileSystem 86 | -------------------------------------------------------------------------------- /cfn/iam.template: -------------------------------------------------------------------------------- 1 | AWSTemplateFormatVersion: "2010-09-09" 2 | 3 | Description: Provides the IAM policies and access configuration for the AWS Digital Content Creation Architecture. 4 | 5 | Resources: 6 | EC2SessionManagerRole: 7 | Type: AWS::IAM::Role 8 | Properties: 9 | AssumeRolePolicyDocument: 10 | Statement: 11 | - Effect: Allow 12 | Principal: 13 | Service: 14 | - ec2.amazonaws.com 15 | Action: 16 | - sts:AssumeRole 17 | Path: / 18 | ManagedPolicyArns: 19 | - arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore 20 | 21 | EC2InstanceProfile: 22 | Type: AWS::IAM::InstanceProfile 23 | Properties: 24 | Path: / 25 | Roles: 26 | - !Ref EC2SessionManagerRole 27 | 28 | #-- The managed policy for Amazon EC2 Role to enable AWS Systems Manager service core functionality. --# 29 | 30 | WorkstationSessionManagerRole: 31 | Type: AWS::IAM::Role 32 | Properties: 33 | AssumeRolePolicyDocument: 34 | Statement: 35 | - Effect: Allow 36 | Principal: 37 | Service: 38 | - ec2.amazonaws.com 39 | Action: 40 | - sts:AssumeRole 41 | Path: / 42 | ManagedPolicyArns: 43 | - arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore 44 | 45 | #-- Amazon EC2 instance periodically connects to an Amazon S3 bucket to determine whether a valid license is available. --# 46 | 47 | NiceDCVLicensePolicy: 48 | Type: AWS::IAM::Policy 49 | Properties: 50 | PolicyName: DcvInstallAccess 51 | PolicyDocument: 52 | Version: "2012-10-17" 53 | Statement: 54 | - Effect: Allow 55 | Action: 56 | - s3:GetObject 57 | Resource: !Sub arn:aws:s3:::dcv-license.${AWS::Region}/* 58 | Roles: 59 | - !Ref WorkstationSessionManagerRole 60 | 61 | WorkstationInstanceProfile: 62 | Type: AWS::IAM::InstanceProfile 63 | Properties: 64 | Path: / 65 | Roles: 66 | - !Ref WorkstationSessionManagerRole 67 | 68 | #-- Allows EC2 Spot Fleet to request, terminate and tag Spot Instances on your behalf. --# 69 | 70 | SpotFleetRole: 71 | Type: AWS::IAM::Role 72 | Properties: 73 | AssumeRolePolicyDocument: 74 | Statement: 75 | - Effect: Allow 76 | Principal: 77 | Service: 78 | - spotfleet.amazonaws.com 79 | Action: 80 | - sts:AssumeRole 81 | ManagedPolicyArns: 82 | - arn:aws:iam::aws:policy/service-role/AmazonEC2SpotFleetTaggingRole 83 | 84 | Outputs: 85 | EC2InstanceProfile: 86 | Value: !Ref EC2InstanceProfile 87 | 88 | WorkstationInstanceProfile: 89 | Value: !Ref WorkstationInstanceProfile 90 | 91 | SpotFleetARN: 92 | Value: !GetAtt SpotFleetRole.Arn 93 | -------------------------------------------------------------------------------- /cfn/license-server.template: -------------------------------------------------------------------------------- 1 | AWSTemplateFormatVersion: "2010-09-09" 2 | 3 | Description: Provides configuration for a Deadline license server instance. 4 | 5 | Parameters: 6 | PrivateSubnet1: 7 | Type: AWS::EC2::Subnet::Id 8 | 9 | LicenseServerSecurityGroup: 10 | Type: AWS::EC2::SecurityGroup::Id 11 | 12 | InstanceType: 13 | Type: String 14 | 15 | InstanceAMI: 16 | Type: AWS::EC2::Image::Id 17 | 18 | Environment: 19 | Type: String 20 | 21 | EC2InstanceProfile: 22 | Type: String 23 | 24 | LicenseServerVersion: 25 | Type: String 26 | 27 | DeadlineLicense: 28 | Type: String 29 | NoEcho: true 30 | 31 | ArtefactBucketName: 32 | Type: String 33 | 34 | Resources: 35 | LicenseServerInstance: 36 | CreationPolicy: 37 | ResourceSignal: 38 | Count: 1 39 | Timeout: PT10M 40 | Type: AWS::EC2::Instance 41 | Metadata: 42 | AWS::CloudFormation::Init: 43 | configSets: 44 | dcc_install: 45 | - instance_prep 46 | - artefacts_download 47 | - app_install 48 | instance_prep: 49 | files: 50 | /opt/scripts/ec2-mount-ebs-volume.sh: 51 | content: | 52 | #!/usr/bin/env bash 53 | mkdir -p /data 54 | while [ ! -b $(readlink -f /dev/xvdh) ]; do echo "waiting for device /dev/xvdh"; sleep 5 ; done 55 | blkid $(readlink -f /dev/xvdh) || mkfs -t ext4 $(readlink -f /dev/xvdh) 56 | e2label $(readlink -f /dev/xvdh) dcc-data 57 | grep -q ^LABEL=dcc-data /etc/fstab || echo 'LABEL=dcc-data /data ext4 defaults' >> /etc/fstab 58 | grep -q "^$(readlink -f /dev/xvdh) /data " /proc/mounts || mount /data 59 | mode: "000755" 60 | owner: root 61 | group: root 62 | /etc/cfn/cfn-hup.conf: 63 | content: !Sub | 64 | [main] 65 | stack=${AWS::StackId} 66 | region=${AWS::Region} 67 | interval=1 68 | mode: "000400" 69 | owner: root 70 | group: root 71 | /etc/cfn/hooks.d/cfn-auto-reloader.conf: 72 | content: !Sub | 73 | [cfn-auto-reloader-hook] 74 | triggers=post.update 75 | path=Resources.LicenseServerInstance.Metadata.AWS::CloudFormation::Init 76 | action=/opt/aws/bin/cfn-init -v --stack ${AWS::StackName} --resource LicenseServerInstance --configsets dcc_install --region ${AWS::Region} 77 | runas=root 78 | mode: "000400" 79 | owner: root 80 | group: root 81 | commands: 82 | 01-mount-ebs-volume: 83 | command: sh /opt/scripts/ec2-mount-ebs-volume.sh > /opt/scripts/ec2-mount-ebs-volume.log 2>&1 84 | test: test -x /opt/scripts/ec2-mount-ebs-volume.sh # check if filename exists and is executable 85 | ignoreErrors: false 86 | services: 87 | sysvinit: 88 | cfn-hup: 89 | enabled: true 90 | ensureRunning: true 91 | files: 92 | - /etc/cfn/cfn-hup.conf 93 | - /etc/cfn/hooks.d/cfn-auto-reloader.conf 94 | amazon-ssm-agent: 95 | enabled: true 96 | ensureRunning: true 97 | artefacts_download: 98 | sources: 99 | /data/thinkbox: !Sub https://${ArtefactBucketName}.s3.amazonaws.com/ThinkboxLicenseServer-${LicenseServerVersion}-linux-x64-installer.zip 100 | app_install: 101 | files: 102 | /data/thinkbox/license.lic: 103 | content: !Sub | 104 | ${DeadlineLicense} 105 | commands: 106 | 01-set_permissions: 107 | command: chmod 500 /data/thinkbox/ThinkboxLicenseServer* 108 | 02-install_license_server: 109 | command: | 110 | ls /data/thinkbox | grep ThinkboxLicenseServer | \ 111 | (read data; /data/thinkbox/$data \ 112 | --mode text \ 113 | --licenseFile /data/thinkbox/license.lic) 114 | ignoreErrors: true 115 | Properties: 116 | BlockDeviceMappings: 117 | - DeviceName: /dev/xvdh 118 | Ebs: 119 | VolumeSize: 100 120 | VolumeType: gp2 121 | Encrypted: true 122 | DeleteOnTermination: true 123 | IamInstanceProfile: !Ref EC2InstanceProfile 124 | ImageId: !Ref InstanceAMI 125 | InstanceType: !Ref InstanceType 126 | SecurityGroupIds: 127 | - !Ref LicenseServerSecurityGroup 128 | SubnetId: !Ref PrivateSubnet1 129 | Tags: 130 | - Key: Name 131 | Value: deadline-license-server 132 | - Key: Environment 133 | Value: !Ref Environment 134 | UserData: !Base64 135 | Fn::Sub: | 136 | #!/bin/bash -xe 137 | # Update aws-cfn-bootstrap to the latest 138 | yum install -y aws-cfn-bootstrap 139 | # Call cfn-init script to install files and packages 140 | /opt/aws/bin/cfn-init -v --stack ${AWS::StackName} --resource LicenseServerInstance --configsets dcc_install --region ${AWS::Region} 141 | # Call cfn-signal script to send a signal with exit code 142 | /opt/aws/bin/cfn-signal --exit-code $? --stack ${AWS::StackName} --resource LicenseServerInstance --region ${AWS::Region} 143 | 144 | Outputs: 145 | LicenseServerInstancePrivateIp: 146 | Value: !GetAtt LicenseServerInstance.PrivateIp 147 | 148 | DeadlineLicense: 149 | Value: !Ref DeadlineLicense 150 | -------------------------------------------------------------------------------- /cfn/main.template: -------------------------------------------------------------------------------- 1 | AWSTemplateFormatVersion: "2010-09-09" 2 | 3 | Description: AWS Digital Content Creation Architecture - AWS ThinkBox Deadline. (uksb-1qgdqu75l) 4 | 5 | Metadata: 6 | Version: 1.6.9 7 | 8 | AWS::CloudFormation::Interface: 9 | ParameterGroups: 10 | - Label: 11 | default: Region configuration 12 | Parameters: 13 | - AvailabilityZones 14 | - Label: 15 | default: VPC network configuration 16 | Parameters: 17 | - VPCName 18 | - VPCCidr 19 | - VPCPublicSubnet1Cidr 20 | - VPCPublicSubnet2Cidr 21 | - VPCPrivateSubnet1Cidr 22 | - VPCPrivateSubnet2Cidr 23 | - Label: 24 | default: Client VPN Configuration 25 | Parameters: 26 | - CreateVPNEndpoint 27 | - ClientCidrBlock 28 | - ServerCertificateArn 29 | - ClientCertificateArn 30 | - TargetNetworkCidr 31 | - Label: 32 | default: License server configuration 33 | Parameters: 34 | - LicenseServerAmiId 35 | - LicenseServerInstanceType 36 | - LicenseServerVersion 37 | - DeadlineLicense 38 | - Label: 39 | default: Repository and database configuration 40 | Parameters: 41 | - RepositoryAmiId 42 | - RepositoryInstanceType 43 | - AppVersion 44 | - Label: 45 | default: Workstation configuration 46 | Parameters: 47 | - WorkstationInstanceType 48 | - WorkstationConnectionManager 49 | - WorkstationEBSVolumeSize 50 | - WorkstationAccessCIDR 51 | - EC2UserPassword 52 | - Label: 53 | default: Render node configuration 54 | Parameters: 55 | - RenderNodeAmiId 56 | - RenderNodeInstanceType 57 | - RenderNodeTargetCapacity 58 | ParameterLabels: 59 | AvailabilityZones: 60 | default: Availability Zones 61 | VPCName: 62 | default: VPC name 63 | VPCCidr: 64 | default: VPC CIDR 65 | VPCPublicSubnet1Cidr: 66 | default: Public Subnet 1 CIDR 67 | VPCPublicSubnet2Cidr: 68 | default: Public Subnet 2 CIDR 69 | VPCPrivateSubnet1Cidr: 70 | default: Private Subnet 1 CIDR 71 | VPCPrivateSubnet2Cidr: 72 | default: Private Subnet 2 CIDR 73 | CreateVPNEndpoint: 74 | default: Create VPN Endpoint. 75 | ClientCidrBlock: 76 | default: Client CIDR for VPN Endpoint 77 | ServerCertificateArn: 78 | default: ACM Server Certificate ARN 79 | ClientCertificateArn: 80 | default: ACM Client Certificate ARN 81 | TargetNetworkCidr: 82 | default: Target Network CIDR for VPN Endpoint 83 | LicenseServerAmiId: 84 | default: License server AMI ID 85 | LicenseServerInstanceType: 86 | default: License server instance type 87 | LicenseServerVersion: 88 | default: License server version 89 | DeadlineLicense: 90 | default: Deadline license key 91 | RepositoryAmiId: 92 | default: Render Scheduler AMI ID 93 | RepositoryInstanceType: 94 | default: Render scheduler instance type 95 | AppVersion: 96 | default: Deadline application version 97 | WorkstationInstanceType: 98 | default: Workstation instance type 99 | WorkstationConnectionManager: 100 | deafult: Workstation connection manager 101 | WorkstationEBSVolumeSize: 102 | default: Workstation EBS volume size 103 | EC2UserPassword: 104 | default: EC2 user password 105 | WorkstationAccessCIDR: 106 | default: Workstation access CIDR 107 | RenderNodeAmiId: 108 | default: Render Node AMI ID 109 | RenderNodeInstanceType: 110 | default: Render node instance type 111 | RenderNodeTargetCapacity: 112 | default: Render node capacity 113 | 114 | Parameters: 115 | AvailabilityZones: 116 | Description: The list of Availability Zones to use for the subnets in the VPC. Select **two** Availability Zones from your list. 117 | Type: List 118 | 119 | VPCName: 120 | Description: The name of the VPC. 121 | Type: String 122 | Default: dcc-vpc 123 | 124 | VPCCidr: 125 | Description: The CIDR block for the VPC. 126 | Type: String 127 | Default: 10.0.0.0/16 128 | AllowedPattern: ^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/([0-9]|[1-2][0-9]|3[0-2]))$ 129 | ConstraintDescription: CIDR block parameter must be in the form x.x.x.x/16-28 130 | 131 | VPCPublicSubnet1Cidr: 132 | Description: The CIDR block for the Public Subnet located in Availability Zone 1 of the VPC. 133 | Type: String 134 | Default: 10.0.0.0/24 135 | AllowedPattern: ^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/([0-9]|[1-2][0-9]|3[0-2]))$ 136 | ConstraintDescription: CIDR block parameter must be in the form x.x.x.x/16-28 137 | 138 | VPCPublicSubnet2Cidr: 139 | Description: The CIDR block for the Public Subnet located in Availability Zone 2 of the VPC. 140 | Type: String 141 | Default: 10.0.1.0/24 142 | AllowedPattern: ^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/([0-9]|[1-2][0-9]|3[0-2]))$ 143 | ConstraintDescription: CIDR block parameter must be in the form x.x.x.x/16-28 144 | 145 | VPCPrivateSubnet1Cidr: 146 | Description: The CIDR block for the Private Subnet located in Availability Zone 1 of the VPC. 147 | Type: String 148 | Default: 10.0.2.0/24 149 | AllowedPattern: ^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/([0-9]|[1-2][0-9]|3[0-2]))$ 150 | ConstraintDescription: CIDR block parameter must be in the form x.x.x.x/16-28 151 | 152 | VPCPrivateSubnet2Cidr: 153 | Description: The CIDR block for the Private Subnet located in Availability Zone 2 of the VPC. 154 | Type: String 155 | Default: 10.0.3.0/24 156 | AllowedPattern: ^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/([0-9]|[1-2][0-9]|3[0-2]))$ 157 | ConstraintDescription: CIDR block parameter must be in the form x.x.x.x/16-28 158 | 159 | CreateVPNEndpoint: 160 | Description: Should the CloudFormation create a Client VPN Endpoint. Workstation will be placed in Private subnet, if set to 'true'. (Specify 'true' or 'false') 161 | Type: String 162 | AllowedValues: 163 | - false 164 | - true 165 | Default: false 166 | ConstraintDescription: Must specify 'true' or 'false' 167 | 168 | ClientCidrBlock: 169 | Description: If creating Client VPN endpoint in the solution, specify the IPv4 address range. It should be in CIDR notation from which to assign client IP addresses. The address range cannot overlap with the local CIDR of the VPC in which the associated subnet. 170 | Type: String 171 | Default: 10.50.0.0/20 172 | AllowedPattern: ^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/(1[6-9]|2[0-9]|3[0-2]))$ 173 | ConstraintDescription: CIDR block parameter must be in the form x.x.x.x/16-32 174 | 175 | ServerCertificateArn: 176 | Description: Required if Create Client VPN endpoint is set to `true`. 177 | Type: String 178 | Default: N/A 179 | 180 | ClientCertificateArn: 181 | Description: Required if Create Client VPN endpoint is set to `true`. 182 | Type: String 183 | Default: N/A 184 | 185 | TargetNetworkCidr: 186 | Description: If creating Client VPN endpoint in the solution, specify the IPv4 address range, in CIDR notation, of the network for which access is being authorized. 187 | Type: String 188 | Default: 10.0.0.0/16 189 | AllowedPattern: ^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/(0[0-9]|1[0-9]|2[0-9]|3[0-2]))$ 190 | ConstraintDescription: CIDR block parameter must be in the form x.x.x.x/0-32 191 | 192 | LicenseServerAmiId: 193 | Description: Latest Amazon Linux AMI Using Systems Manager Parameter Store. 194 | Type: AWS::SSM::Parameter::Value 195 | Default: /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-gp2 196 | 197 | LicenseServerInstanceType: 198 | Description: The EC2 instance type for the Deadline License server. 199 | Type: String 200 | Default: m5.xlarge 201 | 202 | LicenseServerVersion: 203 | Description: The version of License server. 204 | Type: String 205 | AllowedValues: 206 | - 1.1 207 | - 11.13.1.2.0 208 | Default: 1.1 209 | 210 | DeadlineLicense: 211 | Description: If you have license key, add it here. 212 | Type: String 213 | Default: 123456789012 214 | NoEcho: true 215 | 216 | RepositoryAmiId: 217 | Description: Latest Amazon Linux AMI Using Systems Manager Parameter Store. 218 | Type: AWS::SSM::Parameter::Value 219 | Default: /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-gp2 220 | 221 | RepositoryInstanceType: 222 | Description: The EC2 instance type for the Deadline repository. 223 | Type: String 224 | Default: m5.2xlarge 225 | 226 | AppVersion: 227 | Description: The version of Deadline application. 228 | Type: String 229 | AllowedValues: 230 | - 10.1.3.6 231 | - 10.0.24.4 232 | Default: 10.1.3.6 233 | 234 | WorkstationInstanceType: 235 | Description: Artist workstation instance type. 236 | Type: String 237 | AllowedValues: 238 | - g4dn.xlarge 239 | - g4dn.2xlarge 240 | - g4dn.4xlarge 241 | - g4dn.8xlarge 242 | - g4dn.12xlarge 243 | - g4dn.16xlarge 244 | Default: g4dn.xlarge 245 | 246 | WorkstationConnectionManager: 247 | Description: Specify whether you want to run Teradici PCoIP or NiceDcv server to connect to the workstation. 248 | Type: String 249 | AllowedValues: 250 | - teradici 251 | - nicedcv 252 | Default: teradici 253 | 254 | WorkstationEBSVolumeSize: 255 | Description: Volume size of the Workstation instance in GiB. 256 | Type: Number 257 | Default: 100 258 | MaxValue: 16000 259 | MinValue: 100 260 | 261 | WorkstationAccessCIDR: 262 | Description: CIDR block of an on-premise IP address. Input your network's current public or private IP depending if the Workstation is being placed in a public or private subnet. 263 | Type: String 264 | Default: 0.0.0.0/0 265 | AllowedPattern: ^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/([0-9]|[1-2][0-9]|3[0-2]))$ 266 | 267 | EC2UserPassword: 268 | Description: Set password for ec2-user. 269 | Type: String 270 | NoEcho: true 271 | 272 | RenderNodeAmiId: 273 | Description: Latest Amazon Linux AMI Using Systems Manager Parameter Store. 274 | Type: AWS::SSM::Parameter::Value 275 | Default: /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-gp2 276 | 277 | RenderNodeInstanceType: 278 | Description: The EC2 instance type for the Deadline Render nodes. 279 | Type: String 280 | Default: c5.4xlarge 281 | 282 | RenderNodeTargetCapacity: 283 | Description: The number of instances in the spot fleet. 284 | Type: String 285 | Default: 2 286 | 287 | Environment: 288 | Description: The type of environment to tag your infrastructure with. You can specify DEV (development), TEST (test), or PROD (production). 289 | Type: String 290 | AllowedValues: 291 | - DEV 292 | - TEST 293 | - PROD 294 | Default: DEV 295 | 296 | ArtefactBucketName: 297 | Description: S3 bucket name for the application assets. 298 | Type: String 299 | Default: aws-digital-content-creation-render-environment 300 | AllowedPattern: ^[0-9a-zA-Z]+([0-9a-zA-Z-.]*[0-9a-zA-Z])*$ 301 | ConstraintDescription: S3 bucket name can include numbers, lowercase letters, uppercase letters, periods (.), and hyphens (-). It cannot start or end with a hyphen (-). 302 | 303 | Mappings: 304 | AMIRegionMap: 305 | eu-north-1: 306 | NICEDCV: ami-0bdfd5a2e10328d8d 307 | TERADICI: ami-0ab314a4778b8f334 308 | ap-south-1: 309 | NICEDCV: ami-0c2a5e20ed035feca 310 | TERADICI: ami-0a087b5315aec2a38 311 | eu-west-3: 312 | NICEDCV: ami-04e9155e27944d4e0 313 | TERADICI: ami-0575fa8f425ca91b4 314 | eu-west-2: 315 | NICEDCV: ami-045be6886a93c8130 316 | TERADICI: ami-09b559de08f3663c6 317 | eu-west-1: 318 | NICEDCV: ami-0547e6987ff6a09e6 319 | TERADICI: ami-0cc05ee0e4c836c97 320 | ap-northeast-2: 321 | NICEDCV: ami-0a8e306e2265fbd90 322 | TERADICI: ami-0b63c000dc6b2b9a3 323 | ap-northeast-1: 324 | NICEDCV: ami-0a651db9a351a96a6 325 | TERADICI: ami-0238d8c867840b4a3 326 | sa-east-1: 327 | NICEDCV: ami-0bebd725a3c8b578f 328 | TERADICI: ami-03ee2f9fce0ea4c80 329 | ap-southeast-1: 330 | NICEDCV: ami-05a42cd504e9d2f85 331 | TERADICI: ami-0377ce14385dde95a 332 | ap-southeast-2: 333 | NICEDCV: ami-0b0154d9daba62781 334 | TERADICI: ami-0a535b9dbf5527d00 335 | eu-central-1: 336 | NICEDCV: ami-0aa4f904d736d1114 337 | TERADICI: ami-0d7832ef57eb4f50b 338 | us-east-1: 339 | NICEDCV: ami-05f5825affeec78e9 340 | TERADICI: ami-0e895052ddc54bb63 341 | us-east-2: 342 | NICEDCV: ami-05c85de49cdb0ce40 343 | TERADICI: ami-0bd8ecd5704337970 344 | us-west-1: 345 | NICEDCV: ami-0af3c3830a0d0d567 346 | TERADICI: ami-0c118e5bf07d4c8cc 347 | us-west-2: 348 | NICEDCV: ami-019ff0a55b514e755 349 | TERADICI: ami-0b4e0e6cb3e9e82e5 350 | 351 | Conditions: 352 | CreateClientVPN: !Equals [!Ref CreateVPNEndpoint, true] 353 | 354 | IsTeradici: !Equals [!Ref WorkstationConnectionManager, teradici] 355 | 356 | Transform: AWS::Serverless-2016-10-31 357 | 358 | Resources: 359 | IAMStack: 360 | Type: AWS::CloudFormation::Stack 361 | Properties: 362 | TemplateURL: iam.template 363 | 364 | VPCStack: 365 | Type: AWS::CloudFormation::Stack 366 | Properties: 367 | TemplateURL: vpc.template 368 | Parameters: 369 | AvailabilityZones: !Join 370 | - ',' 371 | - !Ref AvailabilityZones 372 | VPCName: !Ref VPCName 373 | VPCCidr: !Ref VPCCidr 374 | VPCPublicSubnet1Cidr: !Ref VPCPublicSubnet1Cidr 375 | VPCPublicSubnet2Cidr: !Ref VPCPublicSubnet2Cidr 376 | VPCPrivateSubnet1Cidr: !Ref VPCPrivateSubnet1Cidr 377 | VPCPrivateSubnet2Cidr: !Ref VPCPrivateSubnet2Cidr 378 | 379 | ClientVPNStack: 380 | Type: AWS::CloudFormation::Stack 381 | Properties: 382 | TemplateURL: client-vpn.template 383 | Parameters: 384 | ClientCidrBlock: !Ref ClientCidrBlock 385 | ServerCertificateArn: !Ref ServerCertificateArn 386 | ClientCertificateArn: !Ref ClientCertificateArn 387 | SubnetID: !GetAtt VPCStack.Outputs.PrivateSubnet1 388 | TargetNetworkCidr: !Ref TargetNetworkCidr 389 | VPC: !GetAtt VPCStack.Outputs.VpcID 390 | Condition: CreateClientVPN 391 | 392 | SecurityGroupsHelperStack: 393 | Type: AWS::CloudFormation::Stack 394 | Properties: 395 | TemplateURL: security-groups-helper.template 396 | Parameters: 397 | VpcID: !GetAtt VPCStack.Outputs.VpcID 398 | WorkstationAccessCIDR: !Ref WorkstationAccessCIDR 399 | Environment: !Ref Environment 400 | WorkstationConnectionManager: !Ref WorkstationConnectionManager 401 | 402 | EfsStack: 403 | Type: AWS::CloudFormation::Stack 404 | Properties: 405 | TemplateURL: efs.template 406 | Parameters: 407 | VpcID: !GetAtt VPCStack.Outputs.VpcID 408 | PrivateSubnet1: !GetAtt VPCStack.Outputs.PrivateSubnet1 409 | PrivateSubnet2: !GetAtt VPCStack.Outputs.PrivateSubnet2 410 | RepositorySecurityGroup: !GetAtt SecurityGroupsHelperStack.Outputs.RepositorySecurityGroup 411 | RenderNodeSecurityGroup: !GetAtt SecurityGroupsHelperStack.Outputs.RenderNodeSecurityGroup 412 | WorkstationSecurityGroup: !GetAtt SecurityGroupsHelperStack.Outputs.WorkstationSecurityGroup 413 | Environment: !Ref Environment 414 | 415 | LicenseServerStack: 416 | Type: AWS::CloudFormation::Stack 417 | Properties: 418 | TemplateURL: license-server.template 419 | Parameters: 420 | PrivateSubnet1: !GetAtt VPCStack.Outputs.PrivateSubnet1 421 | LicenseServerSecurityGroup: !GetAtt SecurityGroupsHelperStack.Outputs.LicenseServerSecurityGroup 422 | InstanceType: !Ref LicenseServerInstanceType 423 | InstanceAMI: !Ref LicenseServerAmiId 424 | EC2InstanceProfile: !GetAtt IAMStack.Outputs.EC2InstanceProfile 425 | LicenseServerVersion: !Ref LicenseServerVersion 426 | DeadlineLicense: !Ref DeadlineLicense 427 | ArtefactBucketName: !Ref ArtefactBucketName 428 | Environment: !Ref Environment 429 | 430 | RepositoryStack: 431 | Type: AWS::CloudFormation::Stack 432 | Properties: 433 | TemplateURL: repository.template 434 | Parameters: 435 | PrivateSubnet1: !GetAtt VPCStack.Outputs.PrivateSubnet1 436 | RepositorySecurityGroup: !GetAtt SecurityGroupsHelperStack.Outputs.RepositorySecurityGroup 437 | EfsFileSystem: !GetAtt EfsStack.Outputs.EfsFileSystem 438 | InstanceType: !Ref RepositoryInstanceType 439 | InstanceAMI: !Ref RepositoryAmiId 440 | EC2InstanceProfile: !GetAtt IAMStack.Outputs.EC2InstanceProfile 441 | EC2UserPassword: !Ref EC2UserPassword #pragma: allowlist secret 442 | AppVersion: !Ref AppVersion 443 | Environment: !Ref Environment 444 | ArtefactBucketName: !Ref ArtefactBucketName 445 | 446 | WorkstationStack: 447 | Type: AWS::CloudFormation::Stack 448 | DependsOn: 449 | - RepositoryStack 450 | Properties: 451 | TemplateURL: workstation.template 452 | Parameters: 453 | SubnetID: !If 454 | - CreateClientVPN 455 | - !GetAtt VPCStack.Outputs.PrivateSubnet1 456 | - !GetAtt VPCStack.Outputs.PublicSubnet1 457 | CreateVPNEndpoint: !Ref CreateVPNEndpoint 458 | WorkstationSecurityGroup: !GetAtt SecurityGroupsHelperStack.Outputs.WorkstationSecurityGroup 459 | EfsFileSystem: !GetAtt EfsStack.Outputs.EfsFileSystem 460 | InstanceType: !Ref WorkstationInstanceType 461 | WorkstationEBSVolumeSize: !Ref WorkstationEBSVolumeSize 462 | InstanceAMI: !If 463 | - IsTeradici 464 | - !FindInMap [AMIRegionMap, !Ref 'AWS::Region', TERADICI] 465 | - !FindInMap [AMIRegionMap, !Ref 'AWS::Region', NICEDCV] 466 | WorkstationInstanceProfile: !GetAtt IAMStack.Outputs.WorkstationInstanceProfile 467 | WorkstationConnectionManager: !Ref WorkstationConnectionManager 468 | AppVersion: !Ref AppVersion 469 | Environment: !Ref Environment 470 | ArtefactBucketName: !Ref ArtefactBucketName 471 | LicenseServerInstancePrivateIp: !GetAtt LicenseServerStack.Outputs.LicenseServerInstancePrivateIp 472 | EC2UserPassword: !Ref EC2UserPassword #pragma: allowlist secret 473 | 474 | RenderNodeStack: 475 | Type: AWS::CloudFormation::Stack 476 | DependsOn: RepositoryStack 477 | Properties: 478 | TemplateURL: render-node.template 479 | Parameters: 480 | PrivateSubnet1: !GetAtt VPCStack.Outputs.PrivateSubnet1 481 | RenderNodeSecurityGroup: !GetAtt SecurityGroupsHelperStack.Outputs.RenderNodeSecurityGroup 482 | EfsFileSystem: !GetAtt EfsStack.Outputs.EfsFileSystem 483 | InstanceType: !Ref RenderNodeInstanceType 484 | InstanceAMI: !Ref RenderNodeAmiId 485 | Environment: !Ref Environment 486 | EC2InstanceProfile: !GetAtt IAMStack.Outputs.EC2InstanceProfile 487 | SpotFleetARN: !GetAtt IAMStack.Outputs.SpotFleetARN 488 | AppVersion: !Ref AppVersion 489 | LicenseServerInstancePrivateIp: !GetAtt LicenseServerStack.Outputs.LicenseServerInstancePrivateIp 490 | EC2UserPassword: !Ref EC2UserPassword #pragma: allowlist secret 491 | ArtefactBucketName: !Ref ArtefactBucketName 492 | RenderNodeTargetCapacity: !Ref RenderNodeTargetCapacity 493 | 494 | Outputs: 495 | ClientVpnEndpointID: 496 | Value: !GetAtt ClientVPNStack.Outputs.ClientVpnEndpointID 497 | Condition: CreateClientVPN 498 | 499 | LicenseServerIPAddress: 500 | Value: !GetAtt LicenseServerStack.Outputs.LicenseServerInstancePrivateIp 501 | 502 | RenderSchedulerIPAddress: 503 | Value: !GetAtt RepositoryStack.Outputs.RepositoryInstancePrivateIp 504 | 505 | WorkstationIP: 506 | Value: !If 507 | - IsTeradici 508 | - !GetAtt WorkstationStack.Outputs.WorkstationIP 509 | - !Join ["", ['https://', !GetAtt WorkstationStack.Outputs.WorkstationIP, ':8443']] 510 | -------------------------------------------------------------------------------- /cfn/render-node.template: -------------------------------------------------------------------------------- 1 | AWSTemplateFormatVersion: "2010-09-09" 2 | 3 | Description: Provides configuration for a Deadline workstation instance. 4 | 5 | Parameters: 6 | PrivateSubnet1: 7 | Type: AWS::EC2::Subnet::Id 8 | 9 | RenderNodeSecurityGroup: 10 | Type: AWS::EC2::SecurityGroup::Id 11 | 12 | InstanceType: 13 | Type: String 14 | 15 | InstanceAMI: 16 | Type: AWS::EC2::Image::Id 17 | 18 | Environment: 19 | Type: String 20 | 21 | EC2InstanceProfile: 22 | Type: String 23 | 24 | SpotFleetARN: 25 | Type: String 26 | 27 | AppVersion: 28 | Type: String 29 | 30 | LicenseServerInstancePrivateIp: 31 | Type: String 32 | 33 | EC2UserPassword: 34 | Type: String 35 | NoEcho: true 36 | 37 | ArtefactBucketName: 38 | Type: String 39 | 40 | RenderNodeTargetCapacity: 41 | Type: String 42 | 43 | EfsFileSystem: 44 | Type: String 45 | 46 | Resources: 47 | RenderNodeLaunchTemplate: 48 | Type: AWS::EC2::LaunchTemplate 49 | Metadata: 50 | AWS::CloudFormation::Init: 51 | configSets: 52 | dcc_install: 53 | - instance_prep 54 | - artefacts_download 55 | - blender_download 56 | - app_install 57 | instance_prep: 58 | packages: 59 | yum: 60 | system-lsb: [] 61 | cifs-utils: [] 62 | files: 63 | /opt/scripts/ec2-mount-ebs-volume.sh: 64 | content: | 65 | #!/usr/bin/env bash 66 | mkdir -p /data 67 | while [ ! -b $(readlink -f /dev/xvdh) ]; do echo "waiting for device /dev/xvdh"; sleep 5 ; done 68 | blkid $(readlink -f /dev/xvdh) || mkfs -t ext4 $(readlink -f /dev/xvdh) 69 | e2label $(readlink -f /dev/xvdh) dcc-data 70 | grep -q ^LABEL=dcc-data /etc/fstab || echo 'LABEL=dcc-data /data ext4 defaults' >> /etc/fstab 71 | grep -q "^$(readlink -f /dev/xvdh) /data " /proc/mounts || mount /data 72 | mode: "000755" 73 | owner: root 74 | group: root 75 | /opt/scripts/dns.sh: 76 | content: !Sub | 77 | #!/usr/bin/env bash 78 | results=1 79 | while [[ $results != 0 ]]; do 80 | nslookup ${EfsFileSystem}.efs.${AWS::Region}.amazonaws.com 81 | results=$? 82 | if [[ $results = 1 ]]; then 83 | sleep 30s 84 | fi 85 | done 86 | mode: "000755" 87 | owner: root 88 | group: root 89 | /etc/cfn/cfn-hup.conf: 90 | content: !Sub | 91 | [main] 92 | stack=${AWS::StackId} 93 | region=${AWS::Region} 94 | interval=1 95 | mode: "000400" 96 | owner: root 97 | group: root 98 | /etc/cfn/hooks.d/cfn-auto-reloader.conf: 99 | content: !Sub | 100 | [cfn-auto-reloader-hook] 101 | triggers=post.update 102 | path=Resources.RenderNodeLaunchTemplate.Metadata.AWS::CloudFormation::Init 103 | action=/opt/aws/bin/cfn-init -v --stack ${AWS::StackName} --resource RenderNodeLaunchTemplate --configsets dcc_install --region ${AWS::Region} 104 | runas=root 105 | mode: "000400" 106 | owner: root 107 | group: root 108 | commands: 109 | 01-mount-ebs-volume: 110 | command: sh /opt/scripts/ec2-mount-ebs-volume.sh > /opt/scripts/ec2-mount-ebs-volume.log 2>&1 111 | test: test -x /opt/scripts/ec2-mount-ebs-volume.sh # check if filename exists and is executable 112 | ignoreErrors: false 113 | 02-wait_for_dns_propogation: 114 | command: sh /opt/scripts/dns.sh 115 | 03-mount_efs: 116 | command: !Sub | 117 | #!/bin/bash -xe 118 | EFS_DIRECTORY=/mnt/efs 119 | mkdir $EFS_DIRECTORY 120 | echo "${EfsFileSystem}:/ $EFS_DIRECTORY efs _netdev" >> /etc/fstab 121 | mount -a -t efs defaults 122 | 04-set_ec2_user_password: 123 | command: !Sub echo ${EC2UserPassword} | tee - | passwd ec2-user 124 | services: 125 | sysvinit: 126 | cfn-hup: 127 | enabled: true 128 | ensureRunning: true 129 | files: 130 | - /etc/cfn/cfn-hup.conf 131 | - /etc/cfn/hooks.d/cfn-auto-reloader.conf 132 | amazon-ssm-agent: 133 | enabled: true 134 | ensureRunning: true 135 | artefacts_download: 136 | sources: 137 | /data/thinkbox: !Sub https://${ArtefactBucketName}.s3.amazonaws.com/Deadline-${AppVersion}-linux-installers.tar 138 | blender_download: 139 | sources: 140 | /usr/local: !Sub https://${ArtefactBucketName}.s3.amazonaws.com/Blender.zip 141 | app_install: 142 | commands: 143 | 05-install-client: 144 | command: !Sub | 145 | ls /data/thinkbox | grep DeadlineClient | \ 146 | (read data; /data/thinkbox/$data \ 147 | --mode unattended \ 148 | --prefix "/data/thinkbox/Deadline10" \ 149 | --connectiontype Repository \ 150 | --repositorydir "/mnt/efs/DeadlineRepository10" \ 151 | --licensemode Standard \ 152 | --licenseserver @${LicenseServerInstancePrivateIp} \ 153 | --launcherstartup true \ 154 | --slavestartup true \ 155 | --daemonuser "" \ 156 | --launcherdaemon true) 157 | ignoreErrors: true 158 | Properties: 159 | LaunchTemplateData: 160 | BlockDeviceMappings: 161 | - DeviceName: /dev/xvdh 162 | Ebs: 163 | VolumeSize: 100 164 | VolumeType: gp2 165 | Encrypted: true 166 | ImageId: !Ref InstanceAMI 167 | IamInstanceProfile: 168 | Name: !Ref EC2InstanceProfile 169 | SecurityGroupIds: 170 | - !Ref RenderNodeSecurityGroup 171 | TagSpecifications: 172 | - ResourceType: instance 173 | Tags: 174 | - Key: Name 175 | Value: deadline-rendernode 176 | - Key: Environment 177 | Value: !Ref Environment 178 | UserData: !Base64 179 | Fn::Sub: | 180 | #!/bin/bash -xe 181 | # Update aws-cfn-bootstrap to the latest 182 | yum install -y aws-cfn-bootstrap 183 | # Install efs-utils 184 | yum install -y amazon-efs-utils 185 | # Call cfn-init script to install files and packages 186 | /opt/aws/bin/cfn-init -v --stack ${AWS::StackName} --resource RenderNodeLaunchTemplate --configsets dcc_install --region ${AWS::Region} 187 | 188 | RenderNodeSpotFleet: 189 | Type: AWS::EC2::SpotFleet 190 | Properties: 191 | SpotFleetRequestConfigData: 192 | AllocationStrategy: diversified 193 | IamFleetRole: !Ref SpotFleetARN 194 | TargetCapacity: !Ref RenderNodeTargetCapacity 195 | LaunchTemplateConfigs: 196 | - LaunchTemplateSpecification: 197 | LaunchTemplateId: !Ref RenderNodeLaunchTemplate 198 | Version: "1" 199 | Overrides: 200 | - InstanceType: !Ref InstanceType 201 | SubnetId: !Ref PrivateSubnet1 202 | 203 | Outputs: 204 | DeadlineVersion: 205 | Value: !Ref AppVersion 206 | -------------------------------------------------------------------------------- /cfn/repository.template: -------------------------------------------------------------------------------- 1 | AWSTemplateFormatVersion: "2010-09-09" 2 | 3 | Description: Provides configuration for a Deadline database and repository instance. 4 | 5 | Parameters: 6 | PrivateSubnet1: 7 | Type: AWS::EC2::Subnet::Id 8 | 9 | RepositorySecurityGroup: 10 | Type: AWS::EC2::SecurityGroup::Id 11 | 12 | InstanceType: 13 | Type: String 14 | 15 | InstanceAMI: 16 | Type: AWS::EC2::Image::Id 17 | 18 | EC2InstanceProfile: 19 | Type: String 20 | 21 | AppVersion: 22 | Type: String 23 | 24 | EC2UserPassword: 25 | Type: String 26 | NoEcho: true 27 | 28 | ArtefactBucketName: 29 | Type: String 30 | 31 | Environment: 32 | Type: String 33 | 34 | EfsFileSystem: 35 | Type: String 36 | 37 | Resources: 38 | RepositoryInstance: 39 | CreationPolicy: 40 | ResourceSignal: 41 | Count: 1 42 | Timeout: PT30M 43 | Type: AWS::EC2::Instance 44 | Metadata: 45 | AWS::CloudFormation::Init: 46 | configSets: 47 | dcc_install: 48 | - instance_prep 49 | - artefacts_download 50 | - app_install 51 | instance_prep: 52 | packages: 53 | yum: 54 | samba: [] 55 | files: 56 | /opt/scripts/ec2-mount-ebs-volume.sh: 57 | content: | 58 | #!/usr/bin/env bash 59 | mkdir -p /data 60 | while [ ! -b $(readlink -f /dev/xvdh) ]; do echo "waiting for device /dev/xvdh"; sleep 5 ; done 61 | blkid $(readlink -f /dev/xvdh) || mkfs -t ext4 $(readlink -f /dev/xvdh) 62 | e2label $(readlink -f /dev/xvdh) dcc-data 63 | grep -q ^LABEL=dcc-data /etc/fstab || echo 'LABEL=dcc-data /data ext4 defaults' >> /etc/fstab 64 | grep -q "^$(readlink -f /dev/xvdh) /data " /proc/mounts || mount /data 65 | mode: "000755" 66 | owner: root 67 | group: root 68 | /opt/scripts/dns.sh: 69 | content: !Sub | 70 | #!/usr/bin/env bash 71 | results=1 72 | while [[ $results != 0 ]]; do 73 | nslookup ${EfsFileSystem}.efs.${AWS::Region}.amazonaws.com 74 | results=$? 75 | if [[ $results = 1 ]]; then 76 | sleep 30s 77 | fi 78 | done 79 | mode: "000755" 80 | owner: root 81 | group: root 82 | /etc/cfn/cfn-hup.conf: 83 | content: !Sub | 84 | [main] 85 | stack=${AWS::StackId} 86 | region=${AWS::Region} 87 | interval=1 88 | mode: "000400" 89 | owner: root 90 | group: root 91 | /etc/cfn/hooks.d/cfn-auto-reloader.conf: 92 | content: !Sub | 93 | [cfn-auto-reloader-hook] 94 | triggers=post.update 95 | path=Resources.RepositoryInstance.Metadata.AWS::CloudFormation::Init 96 | action=/opt/aws/bin/cfn-init -v --stack ${AWS::StackName} --resource RepositoryInstance --configsets dcc_install --region ${AWS::Region} 97 | runas=root 98 | mode: "000400" 99 | owner: root 100 | group: root 101 | commands: 102 | 01-mount-ebs-volume: 103 | command: sh /opt/scripts/ec2-mount-ebs-volume.sh > /opt/scripts/ec2-mount-ebs-volume.log 2>&1 104 | test: test -x /opt/scripts/ec2-mount-ebs-volume.sh # check if filename exists and is executable 105 | ignoreErrors: false 106 | 02-wait_for_dns_propogation: 107 | command: sh /opt/scripts/dns.sh 108 | 03-mount_efs: 109 | command: !Sub | 110 | #!/bin/bash -xe 111 | EFS_DIRECTORY=/mnt/efs 112 | mkdir $EFS_DIRECTORY 113 | echo "${EfsFileSystem}:/ $EFS_DIRECTORY efs _netdev" >> /etc/fstab 114 | mount -a -t efs defaults 115 | 04-set-ec2-user-password: 116 | command: !Sub echo ${EC2UserPassword} | tee - | passwd ec2-user 117 | services: 118 | sysvinit: 119 | cfn-hup: 120 | enabled: true 121 | ensureRunning: true 122 | files: 123 | - /etc/cfn/cfn-hup.conf 124 | - /etc/cfn/hooks.d/cfn-auto-reloader.conf 125 | amazon-ssm-agent: 126 | enabled: true 127 | ensureRunning: true 128 | artefacts_download: 129 | sources: 130 | /data/thinkbox: !Sub https://${ArtefactBucketName}.s3.amazonaws.com/Deadline-${AppVersion}-linux-installers.tar 131 | app_install: 132 | commands: 133 | 05-install_repository: 134 | command: | 135 | ls /data/thinkbox | grep DeadlineRepository | \ 136 | (read data; /data/thinkbox/$data \ 137 | --mode unattended \ 138 | --prefix "/mnt/efs/DeadlineRepository10" \ 139 | --installmongodb true \ 140 | --mongodir "/opt/Thinkbox/DeadlineDatabase10" \ 141 | --requireSSL false) 142 | ignoreErrors: true 143 | 07-set_repo_permissions: 144 | command: | 145 | chown -R ec2-user:nobody /mnt/efs/DeadlineRepository10 146 | chmod -R 505 /mnt/efs/DeadlineRepository10 147 | cd /mnt/efs/DeadlineRepository10 && chmod -R 707 jobs/ jobsArchived/ reports/ 148 | ignoreErrors: false 149 | Properties: 150 | BlockDeviceMappings: 151 | - DeviceName: /dev/xvdh 152 | Ebs: 153 | VolumeSize: 100 154 | VolumeType: gp2 155 | Encrypted: true 156 | DeleteOnTermination: true 157 | IamInstanceProfile: !Ref EC2InstanceProfile 158 | ImageId: !Ref InstanceAMI 159 | InstanceType: !Ref InstanceType 160 | SecurityGroupIds: 161 | - !Ref RepositorySecurityGroup 162 | SubnetId: !Ref PrivateSubnet1 163 | Tags: 164 | - Key: Name 165 | Value: deadline-repository 166 | - Key: Environment 167 | Value: !Ref Environment 168 | UserData: !Base64 169 | Fn::Sub: | 170 | #!/bin/bash -xe 171 | # Update aws-cfn-bootstrap to the latest 172 | yum install -y aws-cfn-bootstrap 173 | # Install efs-utils 174 | yum install -y amazon-efs-utils 175 | # !!!Warning: issue with en/disabling SELinux in enforcing mode on the latest Amazon Linux 2 AMIs with version numbers 20191014 and 20191024 176 | # sed -i 's/enforcing/disabled/g' /etc/selinux/config /etc/selinux/config 177 | # sudo setenforce 0 178 | # Call cfn-init script to install files and packages 179 | /opt/aws/bin/cfn-init -v --stack ${AWS::StackName} --resource RepositoryInstance --configsets dcc_install --region ${AWS::Region} 180 | # Call cfn-signal script to send a signal with exit code 181 | /opt/aws/bin/cfn-signal --exit-code $? --stack ${AWS::StackName} --resource RepositoryInstance --region ${AWS::Region} 182 | 183 | Outputs: 184 | RepositoryInstancePrivateIp: 185 | Value: !GetAtt RepositoryInstance.PrivateIp 186 | 187 | DeadlineVersion: 188 | Value: !Ref AppVersion 189 | -------------------------------------------------------------------------------- /cfn/security-groups-helper.template: -------------------------------------------------------------------------------- 1 | AWSTemplateFormatVersion: "2010-09-09" 2 | 3 | Description: Provides cross-reference for Security Groups in the ingress and egress rules. 4 | 5 | Parameters: 6 | VpcID: 7 | Type: String 8 | 9 | WorkstationAccessCIDR: 10 | Type: String 11 | AllowedPattern: ^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/([0-9]|[1-2][0-9]|3[0-2]))$ 12 | 13 | WorkstationConnectionManager: 14 | Type: String 15 | 16 | Environment: 17 | Type: String 18 | 19 | Conditions: 20 | IsTeradici: !Equals [!Ref WorkstationConnectionManager, teradici] 21 | 22 | Resources: 23 | LicenseServerSecurityGroup: 24 | Type: AWS::EC2::SecurityGroup 25 | Properties: 26 | GroupDescription: Security group for DL License Server instance 27 | VpcId: !Ref VpcID 28 | Tags: 29 | - Key: Name 30 | Value: license-server-sg 31 | - Key: Environment 32 | Value: !Ref Environment 33 | 34 | RepositorySecurityGroup: 35 | Type: AWS::EC2::SecurityGroup 36 | Properties: 37 | GroupDescription: Security group for DL Repository instance 38 | VpcId: !Ref VpcID 39 | Tags: 40 | - Key: Name 41 | Value: deadline-repository-sg 42 | - Key: Environment 43 | Value: !Ref Environment 44 | 45 | RenderNodeSecurityGroup: 46 | Type: AWS::EC2::SecurityGroup 47 | Properties: 48 | GroupDescription: Security group for DL Render Node instance 49 | VpcId: !Ref VpcID 50 | Tags: 51 | - Key: Name 52 | Value: deadline-rendernode-sg 53 | - Key: Environment 54 | Value: !Ref Environment 55 | 56 | WorkstationSecurityGroup: 57 | Type: AWS::EC2::SecurityGroup 58 | Properties: 59 | GroupDescription: Security group for Workstation instance 60 | VpcId: !Ref VpcID 61 | SecurityGroupIngress: !If 62 | - IsTeradici 63 | - - IpProtocol: tcp 64 | FromPort: 443 65 | ToPort: 443 66 | CidrIp: !Ref WorkstationAccessCIDR 67 | - IpProtocol: tcp 68 | FromPort: 4172 69 | ToPort: 4172 70 | CidrIp: !Ref WorkstationAccessCIDR 71 | - IpProtocol: udp 72 | FromPort: 4172 73 | ToPort: 4172 74 | CidrIp: !Ref WorkstationAccessCIDR 75 | - - IpProtocol: tcp 76 | FromPort: 8443 77 | ToPort: 8443 78 | CidrIp: !Ref WorkstationAccessCIDR 79 | Tags: 80 | - Key: Name 81 | Value: deadline-workstation-sg 82 | - Key: Environment 83 | Value: !Ref Environment 84 | 85 | # <-- Grant access for Deadline Repository --> 86 | 87 | RepositoryToLicenseServerIngress: 88 | Type: AWS::EC2::SecurityGroupIngress 89 | Properties: 90 | GroupId: !Ref LicenseServerSecurityGroup 91 | Description: deadline-repository-sg 92 | IpProtocol: "-1" 93 | SourceSecurityGroupId: !Ref RepositorySecurityGroup 94 | 95 | RepositoryToRenderNodeIngress: 96 | Type: AWS::EC2::SecurityGroupIngress 97 | Properties: 98 | GroupId: !Ref RenderNodeSecurityGroup 99 | Description: deadline-repository-sg 100 | IpProtocol: "-1" 101 | SourceSecurityGroupId: !Ref RepositorySecurityGroup 102 | 103 | RepositoryToWorkstationIngress: 104 | Type: AWS::EC2::SecurityGroupIngress 105 | Properties: 106 | GroupId: !Ref WorkstationSecurityGroup 107 | Description: deadline-repository-sg 108 | IpProtocol: "-1" 109 | SourceSecurityGroupId: !Ref RepositorySecurityGroup 110 | 111 | # <-- Grant access for Deadline Render Node --> 112 | 113 | RenderNodeToRepositoryIngress: 114 | Type: AWS::EC2::SecurityGroupIngress 115 | Properties: 116 | GroupId: !Ref RepositorySecurityGroup 117 | Description: deadline-rendernode-sg 118 | IpProtocol: "-1" 119 | SourceSecurityGroupId: !Ref RenderNodeSecurityGroup 120 | 121 | RenderNodeToWorkstationIngress: 122 | Type: AWS::EC2::SecurityGroupIngress 123 | Properties: 124 | GroupId: !Ref WorkstationSecurityGroup 125 | Description: deadline-rendernode-sg 126 | IpProtocol: "-1" 127 | SourceSecurityGroupId: !Ref RenderNodeSecurityGroup 128 | 129 | RenderNodeToLicenseServerIngress: 130 | Type: AWS::EC2::SecurityGroupIngress 131 | Properties: 132 | GroupId: !Ref LicenseServerSecurityGroup 133 | Description: deadline-rendernode-sg 134 | IpProtocol: "-1" 135 | SourceSecurityGroupId: !Ref RenderNodeSecurityGroup 136 | 137 | # <-- Grant access for Deadline Workstation --> 138 | 139 | WorkstationToRenderNodeIngress: 140 | Type: AWS::EC2::SecurityGroupIngress 141 | Properties: 142 | GroupId: !Ref RenderNodeSecurityGroup 143 | Description: deadline-workstation-sg 144 | IpProtocol: "-1" 145 | SourceSecurityGroupId: !Ref WorkstationSecurityGroup 146 | 147 | WorkstationToRepositoryIngress: 148 | Type: AWS::EC2::SecurityGroupIngress 149 | Properties: 150 | GroupId: !Ref RepositorySecurityGroup 151 | Description: deadline-workstation-sg 152 | IpProtocol: "-1" 153 | SourceSecurityGroupId: !Ref WorkstationSecurityGroup 154 | 155 | WorkstationToLicenseServerIngress: 156 | Type: AWS::EC2::SecurityGroupIngress 157 | Properties: 158 | GroupId: !Ref LicenseServerSecurityGroup 159 | Description: deadline-workstation-sg 160 | IpProtocol: "-1" 161 | SourceSecurityGroupId: !Ref WorkstationSecurityGroup 162 | 163 | Outputs: 164 | LicenseServerSecurityGroup: 165 | Value: !Ref LicenseServerSecurityGroup 166 | 167 | RepositorySecurityGroup: 168 | Value: !Ref RepositorySecurityGroup 169 | 170 | RenderNodeSecurityGroup: 171 | Value: !Ref RenderNodeSecurityGroup 172 | 173 | WorkstationSecurityGroup: 174 | Value: !Ref WorkstationSecurityGroup 175 | -------------------------------------------------------------------------------- /cfn/vpc.template: -------------------------------------------------------------------------------- 1 | AWSTemplateFormatVersion: "2010-09-09" 2 | 3 | Description: Provides VPC networking configuration 4 | 5 | Parameters: 6 | AvailabilityZones: 7 | Description: The list of Availability Zones to use for the subnets in the VPC. 8 | Type: List 9 | 10 | VPCName: 11 | Description: The VPC Name 12 | Type: String 13 | Default: My VPC 14 | 15 | VPCCidr: 16 | Type: String 17 | Default: 10.0.0.0/16 18 | AllowedPattern: ^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/([0-9]|[1-2][0-9]|3[0-2]))$ 19 | ConstraintDescription: CIDR block parameter must be in the form x.x.x.x/16-28 20 | 21 | VPCPublicSubnet1Cidr: 22 | Description: CIDR block of Public Subnet 1 23 | Type: String 24 | Default: 10.0.0.0/24 25 | AllowedPattern: ^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/([0-9]|[1-2][0-9]|3[0-2]))$ 26 | ConstraintDescription: CIDR block parameter must be in the form x.x.x.x/16-28 27 | 28 | VPCPublicSubnet2Cidr: 29 | Description: CIDR block of Public Subnet 2 30 | Type: String 31 | Default: 10.0.1.0/24 32 | AllowedPattern: ^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/([0-9]|[1-2][0-9]|3[0-2]))$ 33 | ConstraintDescription: CIDR block parameter must be in the form x.x.x.x/16-28 34 | 35 | VPCPrivateSubnet1Cidr: 36 | Description: CIDR block of Private Subnet 1 37 | Type: String 38 | Default: 10.0.2.0/24 39 | AllowedPattern: ^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/([0-9]|[1-2][0-9]|3[0-2]))$ 40 | ConstraintDescription: CIDR block parameter must be in the form x.x.x.x/16-28 41 | 42 | VPCPrivateSubnet2Cidr: 43 | Description: CIDR block of Private Subnet 2 44 | Type: String 45 | Default: 10.0.3.0/24 46 | AllowedPattern: ^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/([0-9]|[1-2][0-9]|3[0-2]))$ 47 | ConstraintDescription: CIDR block parameter must be in the form x.x.x.x/16-28 48 | 49 | Resources: 50 | VPC: 51 | Type: AWS::EC2::VPC 52 | Properties: 53 | CidrBlock: !Ref VPCCidr 54 | EnableDnsHostnames: true 55 | EnableDnsSupport: true 56 | InstanceTenancy: default 57 | Tags: 58 | - Key: VpcId 59 | Value: !Ref VPCName 60 | 61 | VPCPublicSubnet1Subnet: 62 | Type: AWS::EC2::Subnet 63 | Properties: 64 | CidrBlock: !Ref VPCPublicSubnet1Cidr 65 | VpcId: !Ref VPC 66 | AvailabilityZone: !Select [0, !Ref AvailabilityZones] 67 | MapPublicIpOnLaunch: true 68 | Tags: 69 | - Key: Name 70 | Value: !Join [/, [!Ref VPCName, PublicSubnet1]] 71 | 72 | VPCPublicSubnet1RouteTable: 73 | Type: AWS::EC2::RouteTable 74 | Properties: 75 | VpcId: !Ref VPC 76 | Tags: 77 | - Key: Name 78 | Value: !Join [/, [!Ref VPCName, PublicSubnet1]] 79 | 80 | VPCPublicSubnet1RouteTableAssociation: 81 | Type: AWS::EC2::SubnetRouteTableAssociation 82 | Properties: 83 | RouteTableId: !Ref VPCPublicSubnet1RouteTable 84 | SubnetId: !Ref VPCPublicSubnet1Subnet 85 | 86 | VPCPublicSubnet1DefaultRoute: 87 | Type: AWS::EC2::Route 88 | DependsOn: 89 | - VPCGW 90 | Properties: 91 | RouteTableId: !Ref VPCPublicSubnet1RouteTable 92 | DestinationCidrBlock: 0.0.0.0/0 93 | GatewayId: !Ref VPCIGW 94 | 95 | VPCPublicSubnet1EIP: 96 | Type: AWS::EC2::EIP 97 | Properties: 98 | Domain: vpc 99 | 100 | VPCPublicSubnet1NATGateway: 101 | Type: AWS::EC2::NatGateway 102 | Properties: 103 | AllocationId: !GetAtt VPCPublicSubnet1EIP.AllocationId 104 | SubnetId: !Ref VPCPublicSubnet1Subnet 105 | Tags: 106 | - Key: Name 107 | Value: !Join [/, [!Ref VPCName, PublicSubnet1]] 108 | 109 | VPCPublicSubnet2Subnet: 110 | Type: AWS::EC2::Subnet 111 | Properties: 112 | CidrBlock: !Ref VPCPublicSubnet2Cidr 113 | VpcId: !Ref VPC 114 | AvailabilityZone: !Select [1, !Ref AvailabilityZones] 115 | MapPublicIpOnLaunch: true 116 | Tags: 117 | - Key: Name 118 | Value: !Join [/, [!Ref VPCName, PublicSubnet2]] 119 | 120 | VPCPublicSubnet2RouteTable: 121 | Type: AWS::EC2::RouteTable 122 | Properties: 123 | VpcId: !Ref VPC 124 | Tags: 125 | - Key: Name 126 | Value: !Join [/, [!Ref VPCName, PublicSubnet2]] 127 | 128 | VPCPublicSubnet2RouteTableAssociation: 129 | Type: AWS::EC2::SubnetRouteTableAssociation 130 | Properties: 131 | RouteTableId: !Ref VPCPublicSubnet2RouteTable 132 | SubnetId: !Ref VPCPublicSubnet2Subnet 133 | 134 | VPCPublicSubnet2DefaultRoute: 135 | Type: AWS::EC2::Route 136 | DependsOn: 137 | - VPCGW 138 | Properties: 139 | RouteTableId: !Ref VPCPublicSubnet2RouteTable 140 | DestinationCidrBlock: 0.0.0.0/0 141 | GatewayId: !Ref VPCIGW 142 | 143 | VPCPrivateSubnet1Subnet: 144 | Type: AWS::EC2::Subnet 145 | Properties: 146 | CidrBlock: !Ref VPCPrivateSubnet1Cidr 147 | VpcId: !Ref VPC 148 | AvailabilityZone: !Select [0, !Ref AvailabilityZones] 149 | MapPublicIpOnLaunch: false 150 | Tags: 151 | - Key: Name 152 | Value: !Join [/, [!Ref VPCName, PrivateSubnet1]] 153 | 154 | VPCPrivateSubnet1RouteTable: 155 | Type: AWS::EC2::RouteTable 156 | Properties: 157 | VpcId: !Ref VPC 158 | Tags: 159 | - Key: Name 160 | Value: !Join [/, [!Ref VPCName, PrivateSubnet1]] 161 | 162 | VPCPrivateSubnet1RouteTableAssociation: 163 | Type: AWS::EC2::SubnetRouteTableAssociation 164 | Properties: 165 | RouteTableId: !Ref VPCPrivateSubnet1RouteTable 166 | SubnetId: !Ref VPCPrivateSubnet1Subnet 167 | 168 | VPCPrivateSubnet1DefaultRoute: 169 | Type: AWS::EC2::Route 170 | Properties: 171 | RouteTableId: !Ref VPCPrivateSubnet1RouteTable 172 | DestinationCidrBlock: 0.0.0.0/0 173 | NatGatewayId: !Ref VPCPublicSubnet1NATGateway 174 | 175 | VPCPrivateSubnet2Subnet: 176 | Type: AWS::EC2::Subnet 177 | Properties: 178 | CidrBlock: !Ref VPCPrivateSubnet2Cidr 179 | VpcId: !Ref VPC 180 | AvailabilityZone: !Select [1, !Ref AvailabilityZones] 181 | MapPublicIpOnLaunch: false 182 | Tags: 183 | - Key: Name 184 | Value: !Join [/, [!Ref VPCName, PrivateSubnet2]] 185 | 186 | VPCPrivateSubnet2RouteTable: 187 | Type: AWS::EC2::RouteTable 188 | Properties: 189 | VpcId: !Ref VPC 190 | Tags: 191 | - Key: Name 192 | Value: !Join [/, [!Ref VPCName, PrivateSubnet2]] 193 | 194 | VPCPrivateSubnet2RouteTableAssociation: 195 | Type: AWS::EC2::SubnetRouteTableAssociation 196 | Properties: 197 | RouteTableId: !Ref VPCPrivateSubnet2RouteTable 198 | SubnetId: !Ref VPCPrivateSubnet2Subnet 199 | 200 | VPCPrivateSubnet2DefaultRoute: 201 | Type: AWS::EC2::Route 202 | Properties: 203 | RouteTableId: !Ref VPCPrivateSubnet2RouteTable 204 | DestinationCidrBlock: 0.0.0.0/0 205 | NatGatewayId: !Ref VPCPublicSubnet1NATGateway 206 | 207 | VPCIGW: 208 | Type: AWS::EC2::InternetGateway 209 | Properties: 210 | Tags: 211 | - Key: Name 212 | Value: VPC/VPC 213 | 214 | VPCGW: 215 | Type: AWS::EC2::VPCGatewayAttachment 216 | Properties: 217 | VpcId: !Ref VPC 218 | InternetGatewayId: !Ref VPCIGW 219 | 220 | Outputs: 221 | VpcID: 222 | Value: !Ref VPC 223 | 224 | PublicSubnet1: 225 | Value: !Ref VPCPublicSubnet1Subnet 226 | 227 | PublicSubnet2: 228 | Value: !Ref VPCPublicSubnet2Subnet 229 | 230 | PrivateSubnet1: 231 | Value: !Ref VPCPrivateSubnet1Subnet 232 | 233 | PrivateSubnet2: 234 | Value: !Ref VPCPrivateSubnet2Subnet 235 | -------------------------------------------------------------------------------- /cfn/workstation.template: -------------------------------------------------------------------------------- 1 | AWSTemplateFormatVersion: "2010-09-09" 2 | 3 | Description: Provides configuration for a Deadline workstation instance. 4 | 5 | Parameters: 6 | SubnetID: 7 | Type: AWS::EC2::Subnet::Id 8 | 9 | CreateVPNEndpoint: 10 | Type: String 11 | 12 | WorkstationSecurityGroup: 13 | Type: AWS::EC2::SecurityGroup::Id 14 | 15 | InstanceType: 16 | Type: String 17 | 18 | InstanceAMI: 19 | Type: AWS::EC2::Image::Id 20 | 21 | WorkstationEBSVolumeSize: 22 | Type: Number 23 | 24 | Environment: 25 | Type: String 26 | 27 | WorkstationInstanceProfile: 28 | Type: String 29 | 30 | AppVersion: 31 | Type: String 32 | 33 | LicenseServerInstancePrivateIp: 34 | Type: String 35 | 36 | EC2UserPassword: 37 | Type: String 38 | NoEcho: true 39 | 40 | ArtefactBucketName: 41 | Type: String 42 | 43 | EfsFileSystem: 44 | Type: String 45 | 46 | WorkstationConnectionManager: 47 | Type: String 48 | 49 | Conditions: 50 | CreateClientVPN: !Equals [!Ref CreateVPNEndpoint, true] 51 | 52 | IsTeradici: !Equals [!Ref WorkstationConnectionManager, teradici] 53 | 54 | Resources: 55 | WorkstationInstance: 56 | CreationPolicy: 57 | ResourceSignal: 58 | Count: 1 59 | Timeout: PT30M 60 | Type: AWS::EC2::Instance 61 | Metadata: 62 | AWS::CloudFormation::Init: 63 | configSets: 64 | dcc_install: 65 | - instance_prep 66 | - artefacts_download 67 | - blender_download 68 | - app_install 69 | - post_install 70 | instance_prep: 71 | packages: !If 72 | - IsTeradici 73 | - yum: 74 | redhat-lsb: [] 75 | cifs-utils: [] 76 | - yum: 77 | system-lsb: [] 78 | cifs-utils: [] 79 | files: 80 | /sbin/ebsnvme-id: !If 81 | - IsTeradici 82 | - source: https://raw.githubusercontent.com/aws-samples/aws-digital-content-creation-render-environment/master/scripts/ebsnvme-id 83 | mode: "000755" 84 | owner: root 85 | group: root 86 | - !Ref AWS::NoValue 87 | /etc/udev/rules.d/70-ec2-nvme-devices.rules: !If 88 | - IsTeradici 89 | - source: https://raw.githubusercontent.com/aws-samples/aws-digital-content-creation-render-environment/master/scripts/70-ec2-nvme-devices.rules 90 | mode: "000755" 91 | owner: root 92 | group: root 93 | - !Ref AWS::NoValue 94 | /opt/scripts/ec2-mount-ebs-volume.sh: 95 | content: | 96 | #!/usr/bin/env bash 97 | mkdir -p /data 98 | while [ ! -b $(readlink -f /dev/xvdh) ]; do echo "waiting for device /dev/xvdh"; sleep 5 ; done 99 | blkid $(readlink -f /dev/xvdh) || mkfs -t ext4 $(readlink -f /dev/xvdh) 100 | e2label $(readlink -f /dev/xvdh) dcc-data 101 | grep -q ^LABEL=dcc-data /etc/fstab || echo 'LABEL=dcc-data /data ext4 defaults' >> /etc/fstab 102 | grep -q "^$(readlink -f /dev/xvdh) /data " /proc/mounts || mount /data 103 | mode: "000755" 104 | owner: root 105 | group: root 106 | /opt/scripts/dns.sh: 107 | content: !Sub | 108 | #!/usr/bin/env bash 109 | results=1 110 | while [[ $results != 0 ]]; do 111 | nslookup ${EfsFileSystem}.efs.${AWS::Region}.amazonaws.com 112 | results=$? 113 | if [[ $results = 1 ]]; then 114 | sleep 30s 115 | fi 116 | done 117 | mode: "000755" 118 | owner: root 119 | group: root 120 | /etc/dcv/dcv.conf: !If 121 | - IsTeradici 122 | - !Ref AWS::NoValue 123 | - content: | 124 | [session-management] 125 | create-session = true 126 | [session-management/automatic-console-session] 127 | owner="ec2-user" 128 | [display] 129 | cuda-devices=['0'] 130 | [display/linux] 131 | gl-displays=[':0.0'] 132 | /etc/cfn/cfn-hup.conf: 133 | content: !Sub | 134 | [main] 135 | stack=${AWS::StackId} 136 | region=${AWS::Region} 137 | interval=1 138 | mode: "000400" 139 | owner: root 140 | group: root 141 | /etc/cfn/hooks.d/cfn-auto-reloader.conf: 142 | content: !Sub | 143 | [cfn-auto-reloader-hook] 144 | triggers=post.update 145 | path=Resources.WorkstationInstance.Metadata.AWS::CloudFormation::Init 146 | action=/opt/aws/bin/cfn-init -v --stack ${AWS::StackName} --resource WorkstationInstance --configsets dcc_install --region ${AWS::Region} 147 | runas=root 148 | mode: "000400" 149 | owner: root 150 | group: root 151 | commands: 152 | 00-reload-udev-rules: 153 | command: udevadm control --reload-rules && udevadm trigger 154 | 01-mount-ebs-volume: 155 | command: sh /opt/scripts/ec2-mount-ebs-volume.sh > /opt/scripts/ec2-mount-ebs-volume.log 2>&1 156 | test: test -x /opt/scripts/ec2-mount-ebs-volume.sh # check if filename exists and is executable 157 | ignoreErrors: false 158 | 02-wait_for_dns_propogation: 159 | command: sh /opt/scripts/dns.sh 160 | test: test -x /opt/scripts/dns.sh 161 | 03-mount_efs: 162 | command: !Sub | 163 | #!/bin/bash -xe 164 | EFS_DIRECTORY=/mnt/efs 165 | mkdir $EFS_DIRECTORY 166 | echo "${EfsFileSystem}:/ $EFS_DIRECTORY efs _netdev" >> /etc/fstab 167 | mount -a -t efs defaults 168 | ignoreErrors: false 169 | 04-add-ec2-user: !If 170 | - IsTeradici 171 | - command: !Sub | 172 | userdel centos 173 | useradd ec2-user 174 | echo ${EC2UserPassword} | tee - | passwd ec2-user 175 | usermod -a -G adm,wheel,systemd-journal ec2-user 176 | systemctl restart pcoip 177 | - command: !Sub | 178 | echo ${EC2UserPassword} | tee - | passwd ec2-user 179 | services: 180 | sysvinit: 181 | dcvserver: !If 182 | - IsTeradici 183 | - !Ref AWS::NoValue 184 | - enabled: true 185 | ensureRunning: true 186 | files: 187 | - /etc/dcv/dcv.conf 188 | cfn-hup: 189 | enabled: true 190 | ensureRunning: true 191 | files: 192 | - /etc/cfn/cfn-hup.conf 193 | - /etc/cfn/hooks.d/cfn-auto-reloader.conf 194 | amazon-ssm-agent: 195 | enabled: true 196 | ensureRunning: true 197 | artefacts_download: 198 | sources: 199 | /data/thinkbox: !Sub https://${ArtefactBucketName}.s3.amazonaws.com/Deadline-${AppVersion}-linux-installers.tar 200 | blender_download: 201 | sources: 202 | /usr/local: !Sub https://${ArtefactBucketName}.s3.amazonaws.com/Blender.zip 203 | app_install: 204 | commands: 205 | 05-install-client: 206 | command: !Sub | 207 | ls /data/thinkbox | grep DeadlineClient | \ 208 | (read data; /data/thinkbox/$data \ 209 | --mode unattended \ 210 | --prefix "/data/thinkbox/Deadline10" \ 211 | --connectiontype Repository \ 212 | --repositorydir "/mnt/efs/DeadlineRepository10" \ 213 | --licensemode Standard \ 214 | --licenseserver @${LicenseServerInstancePrivateIp} \ 215 | --launcherstartup true \ 216 | --slavestartup false) 217 | ignoreErrors: true 218 | post_install: 219 | packages: !If 220 | - IsTeradici 221 | - yum: 222 | gimp: [] 223 | - amazon-linux-extras: 224 | gimp: [] 225 | sources: 226 | /mnt/efs/assets: !Sub https://${ArtefactBucketName}.s3.amazonaws.com/BMW27_2.blend.zip 227 | commands: 228 | 06-set-assets-folder-permissions: 229 | command: chown -R ec2-user:root /mnt/efs/assets 230 | 07-install-google-chrome: 231 | command: | 232 | cd ~/ 233 | wget https://dl.google.com/linux/direct/google-chrome-stable_current_x86_64.rpm 234 | yum localinstall -y google-chrome-stable_current_x86_64.rpm 235 | ignoreErrors: true 236 | 08-set-permission-on-efs: 237 | command: chown -R ec2-user:root /mnt/efs 238 | Properties: 239 | BlockDeviceMappings: 240 | - DeviceName: /dev/xvdh 241 | Ebs: 242 | VolumeSize: !Ref WorkstationEBSVolumeSize 243 | VolumeType: gp2 244 | Encrypted: true 245 | DeleteOnTermination: true 246 | IamInstanceProfile: !Ref WorkstationInstanceProfile 247 | ImageId: !Ref InstanceAMI 248 | InstanceType: !Ref InstanceType 249 | SecurityGroupIds: 250 | - !Ref WorkstationSecurityGroup 251 | SubnetId: !Ref SubnetID 252 | Tags: 253 | - Key: Name 254 | Value: deadline-workstation 255 | - Key: Environment 256 | Value: !Ref Environment 257 | UserData: !If 258 | - IsTeradici 259 | - !Base64 260 | Fn::Sub: | 261 | #!/bin/bash -xe 262 | # add extras for yum repo 263 | yum -y --enablerepo=extras install epel-release 264 | # grab the latest cfn bootstrap and install it 265 | sudo yum install -y https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.amzn1.noarch.rpm 266 | sudo ln -s /usr/local/lib/python2.7/site-packages/cfnbootstrap /usr/lib/python2.7/site-packages/cfnbootstrap 267 | # install efs-utils 268 | git clone https://github.com/aws/efs-utils 269 | make rpm --directory ./efs-utils/ 270 | yum -y install ./efs-utils/build/amazon-efs-utils*.rpm 271 | # install amazon-ssm-agent 272 | yum install -y https://s3.${AWS::Region}.amazonaws.com/amazon-ssm-${AWS::Region}/latest/linux_amd64/amazon-ssm-agent.rpm 273 | # Call cfn-init script to install files and packages 274 | /opt/aws/bin/cfn-init -v --stack ${AWS::StackName} --resource WorkstationInstance --configsets dcc_install --region ${AWS::Region} 275 | # Call cfn-signal script to send a signal with exit code 276 | /opt/aws/bin/cfn-signal --exit-code $? --stack ${AWS::StackName} --resource WorkstationInstance --region ${AWS::Region} 277 | - !Base64 278 | Fn::Sub: | 279 | #!/bin/bash -xe 280 | # Update aws-cfn-bootstrap to the latest 281 | yum install -y aws-cfn-bootstrap 282 | # Install efs-utils 283 | yum install -y amazon-efs-utils 284 | # Call cfn-init script to install files and packages 285 | /opt/aws/bin/cfn-init -v --stack ${AWS::StackName} --resource WorkstationInstance --configsets dcc_install --region ${AWS::Region} 286 | # Call cfn-signal script to send a signal with exit code 287 | /opt/aws/bin/cfn-signal --exit-code $? --stack ${AWS::StackName} --resource WorkstationInstance --region ${AWS::Region} 288 | 289 | Outputs: 290 | WorkstationInstancePrivateIp: 291 | Value: !GetAtt WorkstationInstance.PrivateIp 292 | 293 | DeadlineVersion: 294 | Value: !Ref AppVersion 295 | 296 | WorkstationIP: 297 | Value: !If [CreateClientVPN, !GetAtt WorkstationInstance.PrivateIp, !GetAtt WorkstationInstance.PublicIp] 298 | -------------------------------------------------------------------------------- /ci/include.lst: -------------------------------------------------------------------------------- 1 | cfn 2 | scripts 3 | cfn-publish.config 4 | -------------------------------------------------------------------------------- /docs/blender/addon-checkbox.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-digital-content-creation-render-environment/66c843ad1790d2000f95b5cdb1c8d4d81f3a1b99/docs/blender/addon-checkbox.png -------------------------------------------------------------------------------- /docs/blender/blender-client.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-digital-content-creation-render-environment/66c843ad1790d2000f95b5cdb1c8d4d81f3a1b99/docs/blender/blender-client.png -------------------------------------------------------------------------------- /docs/blender/blender-file.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-digital-content-creation-render-environment/66c843ad1790d2000f95b5cdb1c8d4d81f3a1b99/docs/blender/blender-file.png -------------------------------------------------------------------------------- /docs/blender/blender-job.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-digital-content-creation-render-environment/66c843ad1790d2000f95b5cdb1c8d4d81f3a1b99/docs/blender/blender-job.png -------------------------------------------------------------------------------- /docs/blender/blender-output.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-digital-content-creation-render-environment/66c843ad1790d2000f95b5cdb1c8d4d81f3a1b99/docs/blender/blender-output.png -------------------------------------------------------------------------------- /docs/blender/blender-scene.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-digital-content-creation-render-environment/66c843ad1790d2000f95b5cdb1c8d4d81f3a1b99/docs/blender/blender-scene.png -------------------------------------------------------------------------------- /docs/blender/blender.md: -------------------------------------------------------------------------------- 1 | ## Set up Blender 2 | 3 | This will involve using the workstation to set up a shot to do a test render. 4 | 5 | 1. Log into your **Workstation** using **ec2-user** user account. 6 | 1. Open the terminal and create a `test_project` folder in **projects** directory `/mnt/efs/test_project` 7 | 8 | ``` 9 | mkdir -p /mnt/efs/test_project 10 | ``` 11 | 12 | 1. Launch **Blender** by running the blender shortcut located in */usr/local/Blender* 13 | 14 | ``` 15 | /usr/local/Blender/blender 16 | ``` 17 | 18 | 1. The Deadline Blender submitter needs to be re-installed for each user, so you should do that now: 19 | * Go to **Edit → Preferences...** 20 | * Click on **Add-ons** in the left panel 21 | * Click **Install...** 22 | 23 | ![install-addon.png](install-addon.png) 24 | 25 | * Navigate to **/mnt/efs/DeadlineRepository10/submission/Blender/Client/** 26 | * Choose **DeadlineBlenderClient.py** 27 | * Click **Install Add-on** 28 | 29 | ![blender-client.png](blender-client.png) 30 | 31 | * Click the checkbox next to **Render: Submit Blender to Deadline** add-on and close the Preferences window 32 | 33 | ![addon-checkbox.png](addon-checkbox.png) 34 | 35 | ## Test the Blender set up 36 | 37 | 1. For your test render, you can just use the default blender file with a cube and camera (this is just a test to make sure everything is working correctly. Later on you will make a super fancy file, using Blender Demo library.) 38 | 1. Set your output settings: 39 | * In the **Properties** Panel on the right, click on **Render Properties** 40 | * For **Render Engine** choose **Cycles** 41 | 42 | ![blender-scene.png](blender-scene.png) 43 | 44 | * Click on **Output Properties** (the icon looks like an ink-jet printer) 45 | * Under **Output** change the value to **//####.png** 46 | 47 | ![blender-output.png](blender-output.png) 48 | 49 | * Now save the file with `Shift Ctrl S` as `test.blend` in `/mnt/efs/test_project/` 50 | 51 | ![blender-file.png](blender-file.png) 52 | 53 | 1. Submit your render 54 | * Choose **Render → Submit to Deadline** 55 | * Set the **Group** to **linux_workers** 56 | * Set the **Frame List** to **1-10** 57 | * Make sure the **Blender File** and **Output File** is set properly 58 | 59 | ![blender-job.png](blender-job.png) 60 | 61 | * Click **Submit** and **Close** the window 62 | 63 | 1. Go back to Deadline monitor application. You should see your render going in the **Deadline Monitor** 64 | 65 | ![deadline-job.png](deadline-job.png) 66 | 67 | ## Render something fancy 68 | 69 | Now that our rendering farm is set up, let's render something cool. Blender provides ready to render [Demo Files](https://www.blender.org/download/demo-files/), so let's go ahead and use one. 70 | 71 | 1. The sample file can be found in `/mnt/efs/assets` folder 72 | 1. In Blender, go to `File -> Open` and inside **bmw272** folder open the `bmw272.cpu.blend` file 73 | 1. Click on **Output Properties** and change the value to `//####.png` 74 | 1. Choose **Render -> Submit to Deadline** 75 | 1. Set the **Group** to **linux_workers** 76 | 1. Change the **Frame List**to **1-2** 77 | 1. Make sure the **Blender File** and **Output File** are set properly 78 | 79 | ![bmw-job-submit.png](bmw-job-submit.png) 80 | 81 | 1. Click **Submit** and **Close** the window 82 | 1. Go back to Deadline monitor application. You should see your render going in the **Deadline Monitor** 83 | 1. Once the job is finished, **right click** on `Task ID 01 | Frame 2` to view the finished outcome of the render job 84 | 85 | ![bmw-job.png](bmw-job.png) 86 | 87 | 1. It will take about 4 minutes for the job to finish. Below is the outcome of the popular BMW demo by Mike Pan rendered. 88 | 89 | ![bmw-demo.png](bmw-demo.png) 90 | 91 | > Congratulations, you have successfully sent a job to render nodes. You can now go back to the main [README](../../README.md). 92 | -------------------------------------------------------------------------------- /docs/blender/bmw-demo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-digital-content-creation-render-environment/66c843ad1790d2000f95b5cdb1c8d4d81f3a1b99/docs/blender/bmw-demo.png -------------------------------------------------------------------------------- /docs/blender/bmw-job-submit.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-digital-content-creation-render-environment/66c843ad1790d2000f95b5cdb1c8d4d81f3a1b99/docs/blender/bmw-job-submit.png -------------------------------------------------------------------------------- /docs/blender/bmw-job.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-digital-content-creation-render-environment/66c843ad1790d2000f95b5cdb1c8d4d81f3a1b99/docs/blender/bmw-job.png -------------------------------------------------------------------------------- /docs/blender/deadline-job.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-digital-content-creation-render-environment/66c843ad1790d2000f95b5cdb1c8d4d81f3a1b99/docs/blender/deadline-job.png -------------------------------------------------------------------------------- /docs/blender/install-addon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-digital-content-creation-render-environment/66c843ad1790d2000f95b5cdb1c8d4d81f3a1b99/docs/blender/install-addon.png -------------------------------------------------------------------------------- /docs/deadline/add-nodes.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-digital-content-creation-render-environment/66c843ad1790d2000f95b5cdb1c8d4d81f3a1b99/docs/deadline/add-nodes.png -------------------------------------------------------------------------------- /docs/deadline/enable-super-mode.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-digital-content-creation-render-environment/66c843ad1790d2000f95b5cdb1c8d4d81f3a1b99/docs/deadline/enable-super-mode.png -------------------------------------------------------------------------------- /docs/deadline/manage-groups.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-digital-content-creation-render-environment/66c843ad1790d2000f95b5cdb1c8d4d81f3a1b99/docs/deadline/manage-groups.png -------------------------------------------------------------------------------- /docs/deadline/monitor-error.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-digital-content-creation-render-environment/66c843ad1790d2000f95b5cdb1c8d4d81f3a1b99/docs/deadline/monitor-error.png -------------------------------------------------------------------------------- /docs/deadline/start-deadline.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-digital-content-creation-render-environment/66c843ad1790d2000f95b5cdb1c8d4d81f3a1b99/docs/deadline/start-deadline.png -------------------------------------------------------------------------------- /docs/deadline/workers-group.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-digital-content-creation-render-environment/66c843ad1790d2000f95b5cdb1c8d4d81f3a1b99/docs/deadline/workers-group.png -------------------------------------------------------------------------------- /docs/deadline/workers-setup.md: -------------------------------------------------------------------------------- 1 | ## Setup Deadline Monitor 2 | The [Deadline Monitor](https://www.awsthinkbox.com/blog/deadline-monitor-interface-customization) is your main entry point for overseeing and managing your render farm. It's a powerful application that allows you to view, configure, and interact with the many of the moving parts that make up Deadline. 3 | 4 | #### Launch the Monitor 5 | 6 | `Applications -> Other -> Deadline Monitor 10` 7 | 8 | ![start-deadline.png](start-deadline.png) 9 | 10 | #### Enable Super User Mode 11 | 12 | `Tools -> Super User Mode` 13 | 14 | ![enable-super-mode.png](enable-super-mode.png) 15 | 16 | #### Set up the workers group 17 | 18 | `Tools -> Manage Groups` 19 | 20 | ![manage-groups.png](manage-groups.png) 21 | 22 | 1. Click on **New** to create a group. 23 | 24 | 1. Give the group name **linux_workers** 25 | 26 | ![workers-group.png](workers-group.png) 27 | 28 | 1. Select all the nodes desired with **CTR+A** and click on **Add** 29 | 30 | ![add-nodes.png](add-nodes.png) 31 | 32 | 1. Click **Ok**. 33 | 34 | Next step is to set up [Blender plug-in](../blender/blender.md). 35 | 36 | #### Troubleshooting 37 | If you encounter an issue with Deadline monitor to be unable to connect to Repository, run the command below to set correct ownership on the `/mnt/efs` directory. 38 | 39 | ![monitor-error](monitor-error.png) 40 | 41 | ``` 42 | sudo chown -R ec2-user:root /mnt/efs 43 | ``` 44 | -------------------------------------------------------------------------------- /docs/develop/LOCAL_DEVELOPMENT.md: -------------------------------------------------------------------------------- 1 | # Local Development 2 | 3 | This section details how to run the solution locally and deploy your code changes from the command line. 4 | 5 | ### Pre-Requisites 6 | 7 | The following dependencies must be installed: 8 | 9 | - AWS CLI 10 | - Python >=3.8 and pip 11 | - virtualenv 12 | 13 | Once you have installed pre-requisites, clone the repository to your local machine and run the following command to create a `virtualenv` and install all dependencies. 14 | ``` 15 | make init 16 | ``` 17 | 18 | Activate `virtualenv` environment. 19 | ``` 20 | source venv/bin/activate 21 | ``` 22 | 23 | ### Build and Deploy from Source 24 | 25 | To deploy the solution manually from the source to your AWS account, run the 26 | following: 27 | 28 | 1. Create a S3 bucket 29 | ``` 30 | BUCKET_NAME="your-s3-bucket-name" 31 | AWS_REGION="aws-region(e.g. us-east-1)" 32 | aws s3 mb s3://${BUCKET_NAME} --region $AWS_REGION 33 | ``` 34 | 35 | 1. Create an `.custom.mk` file and populate it with your own values 36 | ``` 37 | cp .custom.mk.example .custom.mk 38 | ``` 39 | 1. Deploy the stack 40 | ``` 41 | make deploy 42 | ``` 43 | 44 | This will deploy the nested stack using the AWS CLI profile of the current shell. By default this will be the profile `default`. 45 | -------------------------------------------------------------------------------- /docs/develop/aws-dcc-vpn.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-digital-content-creation-render-environment/66c843ad1790d2000f95b5cdb1c8d4d81f3a1b99/docs/develop/aws-dcc-vpn.png -------------------------------------------------------------------------------- /docs/develop/aws-dcc.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-digital-content-creation-render-environment/66c843ad1790d2000f95b5cdb1c8d4d81f3a1b99/docs/develop/aws-dcc.png -------------------------------------------------------------------------------- /docs/develop/cloudformation-launch-stack.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-digital-content-creation-render-environment/66c843ad1790d2000f95b5cdb1c8d4d81f3a1b99/docs/develop/cloudformation-launch-stack.png -------------------------------------------------------------------------------- /docs/develop/dcv-conn.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-digital-content-creation-render-environment/66c843ad1790d2000f95b5cdb1c8d4d81f3a1b99/docs/develop/dcv-conn.png -------------------------------------------------------------------------------- /docs/develop/dcv-subscription.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-digital-content-creation-render-environment/66c843ad1790d2000f95b5cdb1c8d4d81f3a1b99/docs/develop/dcv-subscription.png -------------------------------------------------------------------------------- /docs/develop/teradici-conn.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-digital-content-creation-render-environment/66c843ad1790d2000f95b5cdb1c8d4d81f3a1b99/docs/develop/teradici-conn.png -------------------------------------------------------------------------------- /docs/develop/teradici-subscription.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-digital-content-creation-render-environment/66c843ad1790d2000f95b5cdb1c8d4d81f3a1b99/docs/develop/teradici-subscription.png -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | awscli 2 | bump2version 3 | pre-commit 4 | taskcat 5 | -------------------------------------------------------------------------------- /scripts/70-ec2-nvme-devices.rules: -------------------------------------------------------------------------------- 1 | # Copyright 2006 Amazon.com, Inc. and its affiliates. All Rights Reserved. 2 | # 3 | # Licensed under the MIT License. See the LICENSE accompanying this file 4 | # for the specific language governing permissions and limitations under 5 | # the License. 6 | 7 | #nvme-ns-* devices 8 | KERNEL=="nvme[0-9]*n[0-9]*", ENV{DEVTYPE}=="disk", ATTRS{serial}=="?*", ATTRS{model}=="?*", SYMLINK+="disk/by-id/nvme-$attr{model}_$attr{serial}-ns-%n", OPTIONS+="string_escape=replace" 9 | 10 | #nvme partitions 11 | KERNEL=="nvme[0-9]*n[0-9]*p[0-9]*", ENV{DEVTYPE}=="partition", ATTRS{serial}=="?*", ATTRS{model}=="?*", IMPORT{program}="ec2nvme-nsid %k" 12 | KERNEL=="nvme[0-9]*n[0-9]*p[0-9]*", ENV{DEVTYPE}=="partition", ATTRS{serial}=="?*", ATTRS{model}=="?*", ENV{_NS_ID}=="?*", SYMLINK+="disk/by-id/nvme-$attr{model}_$attr{serial}-ns-$env{_NS_ID}-part%n", OPTIONS+="string_escape=replace" 13 | 14 | # ebs nvme devices 15 | KERNEL=="nvme[0-9]*n[0-9]*", ENV{DEVTYPE}=="disk", ATTRS{model}=="Amazon Elastic Block Store", PROGRAM="/sbin/ebsnvme-id -u /dev/%k", SYMLINK+="%c" 16 | KERNEL=="nvme[0-9]*n[0-9]*p[0-9]*", ENV{DEVTYPE}=="partition", ATTRS{model}=="Amazon Elastic Block Store", PROGRAM="/sbin/ebsnvme-id -u /dev/%k", SYMLINK+="%c%n" 17 | # instance store nvme devices 18 | KERNEL=="nvme[0-9]*n[0-9]*", ENV{DEVTYPE}=="disk", ATTRS{model}=="Amazon EC2 NVMe Instance Storage", ATTR{queue/io_timeout}="90000" 19 | -------------------------------------------------------------------------------- /scripts/ebsnvme-id: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2.7 2 | 3 | # Copyright 2017 Amazon.com, Inc. and its affiliates. All Rights Reserved. 4 | # 5 | # Licensed under the MIT License. See the LICENSE accompanying this file 6 | # for the specific language governing permissions and limitations under 7 | # the License. 8 | 9 | """ 10 | Usage: 11 | Read EBS device information and provide information about 12 | the volume. 13 | """ 14 | 15 | import argparse 16 | from ctypes import * 17 | from fcntl import ioctl 18 | import sys 19 | 20 | NVME_ADMIN_IDENTIFY = 0x06 21 | NVME_IOCTL_ADMIN_CMD = 0xC0484E41 22 | AMZN_NVME_VID = 0x1D0F 23 | AMZN_NVME_EBS_MN = "Amazon Elastic Block Store" 24 | 25 | class nvme_admin_command(Structure): 26 | _pack_ = 1 27 | _fields_ = [("opcode", c_uint8), # op code 28 | ("flags", c_uint8), # fused operation 29 | ("cid", c_uint16), # command id 30 | ("nsid", c_uint32), # namespace id 31 | ("reserved0", c_uint64), 32 | ("mptr", c_uint64), # metadata pointer 33 | ("addr", c_uint64), # data pointer 34 | ("mlen", c_uint32), # metadata length 35 | ("alen", c_uint32), # data length 36 | ("cdw10", c_uint32), 37 | ("cdw11", c_uint32), 38 | ("cdw12", c_uint32), 39 | ("cdw13", c_uint32), 40 | ("cdw14", c_uint32), 41 | ("cdw15", c_uint32), 42 | ("reserved1", c_uint64)] 43 | 44 | class nvme_identify_controller_amzn_vs(Structure): 45 | _pack_ = 1 46 | _fields_ = [("bdev", c_char * 32), # block device name 47 | ("reserved0", c_char * (1024 - 32))] 48 | 49 | class nvme_identify_controller_psd(Structure): 50 | _pack_ = 1 51 | _fields_ = [("mp", c_uint16), # maximum power 52 | ("reserved0", c_uint16), 53 | ("enlat", c_uint32), # entry latency 54 | ("exlat", c_uint32), # exit latency 55 | ("rrt", c_uint8), # relative read throughput 56 | ("rrl", c_uint8), # relative read latency 57 | ("rwt", c_uint8), # relative write throughput 58 | ("rwl", c_uint8), # relative write latency 59 | ("reserved1", c_char * 16)] 60 | 61 | class nvme_identify_controller(Structure): 62 | _pack_ = 1 63 | _fields_ = [("vid", c_uint16), # PCI Vendor ID 64 | ("ssvid", c_uint16), # PCI Subsystem Vendor ID 65 | ("sn", c_char * 20), # Serial Number 66 | ("mn", c_char * 40), # Module Number 67 | ("fr", c_char * 8), # Firmware Revision 68 | ("rab", c_uint8), # Recommend Arbitration Burst 69 | ("ieee", c_uint8 * 3), # IEEE OUI Identifier 70 | ("mic", c_uint8), # Multi-Interface Capabilities 71 | ("mdts", c_uint8), # Maximum Data Transfer Size 72 | ("reserved0", c_uint8 * (256 - 78)), 73 | ("oacs", c_uint16), # Optional Admin Command Support 74 | ("acl", c_uint8), # Abort Command Limit 75 | ("aerl", c_uint8), # Asynchronous Event Request Limit 76 | ("frmw", c_uint8), # Firmware Updates 77 | ("lpa", c_uint8), # Log Page Attributes 78 | ("elpe", c_uint8), # Error Log Page Entries 79 | ("npss", c_uint8), # Number of Power States Support 80 | ("avscc", c_uint8), # Admin Vendor Specific Command Configuration 81 | ("reserved1", c_uint8 * (512 - 265)), 82 | ("sqes", c_uint8), # Submission Queue Entry Size 83 | ("cqes", c_uint8), # Completion Queue Entry Size 84 | ("reserved2", c_uint16), 85 | ("nn", c_uint32), # Number of Namespaces 86 | ("oncs", c_uint16), # Optional NVM Command Support 87 | ("fuses", c_uint16), # Fused Operation Support 88 | ("fna", c_uint8), # Format NVM Attributes 89 | ("vwc", c_uint8), # Volatile Write Cache 90 | ("awun", c_uint16), # Atomic Write Unit Normal 91 | ("awupf", c_uint16), # Atomic Write Unit Power Fail 92 | ("nvscc", c_uint8), # NVM Vendor Specific Command Configuration 93 | ("reserved3", c_uint8 * (704 - 531)), 94 | ("reserved4", c_uint8 * (2048 - 704)), 95 | ("psd", nvme_identify_controller_psd * 32), # Power State Descriptor 96 | ("vs", nvme_identify_controller_amzn_vs)] # Vendor Specific 97 | 98 | class ebs_nvme_device: 99 | def __init__(self, device): 100 | self.device = device 101 | self.ctrl_identify() 102 | 103 | def _nvme_ioctl(self, id_response, id_len): 104 | admin_cmd = nvme_admin_command(opcode = NVME_ADMIN_IDENTIFY, 105 | addr = id_response, 106 | alen = id_len, 107 | cdw10 = 1) 108 | 109 | with open(self.device, "rw") as nvme: 110 | ioctl(nvme, NVME_IOCTL_ADMIN_CMD, admin_cmd) 111 | 112 | def ctrl_identify(self): 113 | self.id_ctrl = nvme_identify_controller() 114 | self._nvme_ioctl(addressof(self.id_ctrl), sizeof(self.id_ctrl)) 115 | 116 | if self.id_ctrl.vid != AMZN_NVME_VID or self.id_ctrl.mn.strip() != AMZN_NVME_EBS_MN: 117 | raise TypeError("[ERROR] Not an EBS device: '{0}'".format(self.device)) 118 | 119 | def get_volume_id(self): 120 | vol = self.id_ctrl.sn 121 | 122 | if vol.startswith("vol") and vol[3] != "-": 123 | vol = "vol-" + vol[3:] 124 | 125 | return vol 126 | 127 | def get_block_device(self, stripped=False): 128 | dev = self.id_ctrl.vs.bdev.strip() 129 | 130 | if stripped and dev.startswith("/dev/"): 131 | dev = dev[5:] 132 | 133 | return dev 134 | 135 | if __name__ == "__main__": 136 | parser = argparse.ArgumentParser(description="Reads EBS information from NVMe devices.") 137 | parser.add_argument("device", nargs=1, help="Device to query") 138 | 139 | display = parser.add_argument_group("Display Options") 140 | display.add_argument("-v", "--volume", action="store_true", 141 | help="Return volume-id") 142 | display.add_argument("-b", "--block-dev", action="store_true", 143 | help="Return block device mapping") 144 | display.add_argument("-u", "--udev", action="store_true", 145 | help="Output data in format suitable for udev rules") 146 | 147 | if len(sys.argv) < 2: 148 | parser.print_help() 149 | sys.exit(1) 150 | 151 | args = parser.parse_args() 152 | 153 | get_all = not (args.udev or args.volume or args.block_dev) 154 | 155 | try: 156 | dev = ebs_nvme_device(args.device[0]) 157 | except (IOError, TypeError) as err: 158 | print >> sys.stderr, err 159 | sys.exit(1) 160 | 161 | if get_all or args.volume: 162 | print "Volume ID: {0}".format(dev.get_volume_id()) 163 | if get_all or args.block_dev or args.udev: 164 | print dev.get_block_device(args.udev) 165 | -------------------------------------------------------------------------------- /scripts/ec2-mount-ebs-volume.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | mkdir -p /data 3 | while [[ ! -b $(readlink -f /dev/xvdh) ]]; do echo "waiting for device /dev/xvdh"; sleep 5 ; done 4 | blkid $(readlink -f /dev/xvdh) || mkfs -t ext4 $(readlink -f /dev/xvdh) 5 | e2label $(readlink -f /dev/xvdh) dcc-data 6 | grep -q ^LABEL=dcc-data /etc/fstab || echo 'LABEL=dcc-data /data ext4 defaults' >> /etc/fstab 7 | grep -q "^$(readlink -f /dev/xvdh) /data " /proc/mounts || mount /data 8 | --------------------------------------------------------------------------------