├── tests ├── unit │ ├── __init__.py │ ├── test_describe.py │ ├── test_docs_examples_links.py │ ├── test_aws.py │ ├── test_cancel_wait.py │ ├── test_helper.py │ ├── test_resources.py │ ├── test_stacks.py │ ├── test_remove.py │ ├── test_rollback.py │ ├── conftest.py │ ├── test_yaml_tags.py │ ├── test_cli.py │ ├── test_s3.py │ ├── test_stack_waiter.py │ ├── test_deploy.py │ ├── test_new.py │ ├── test_config_file.py │ ├── test_diff.py │ └── test_template.py ├── integration │ ├── __init__.py │ ├── stack.config.yaml │ ├── integration-user.template.yaml │ └── test_basic.py └── __init__.py ├── docs ├── examples │ ├── nested_relative_module │ │ ├── modules │ │ │ ├── nestedModule │ │ │ │ └── testmodule.template.yaml │ │ │ ├── submodule1.template.yaml │ │ │ └── bucket2.template.yaml │ │ ├── module1.template.yml │ │ └── bucket1.template.yaml │ ├── s3-bucket │ │ ├── stack.config.yaml │ │ └── bucket.template.yaml │ ├── s3-lambda │ │ ├── Makefile │ │ ├── code.py │ │ ├── stack.config.yaml │ │ ├── role.template.yaml │ │ ├── bucket.template.yml │ │ └── lambda.template.yaml │ ├── lambda-step-function │ │ ├── stack.config.yaml │ │ ├── state-machine.json │ │ ├── step-function.template.yaml │ │ ├── code.py │ │ ├── lambda.template.yaml │ │ ├── lambda-role.template.yaml │ │ └── step-function-role.template.yaml │ ├── nested-stack │ │ ├── stack.config.yaml │ │ ├── nested.template.yaml │ │ └── nested.yaml │ ├── config-file │ │ ├── test.template.yml │ │ └── test.config.yaml │ ├── commit-build-pipeline │ │ ├── stack.config.yaml │ │ ├── README.md │ │ ├── commit.template.yml │ │ ├── pipeline.template.yml │ │ ├── role.template.yml │ │ └── build.template.yml │ ├── commit-build │ │ ├── README.md │ │ ├── commit.template.yml │ │ ├── hook.py │ │ ├── build.template.yml │ │ └── post-push-hook.template.yml │ └── custom-resource │ │ ├── resource.py │ │ ├── README.md │ │ └── custom.template.yaml ├── s3-deployment.md ├── commands │ ├── stack-set_remove.md │ ├── _index.md │ ├── resources.md │ ├── stacks.md │ ├── template.md │ ├── wait.md │ ├── cancel.md │ ├── remove.md │ ├── deploy.md │ ├── stack-set_create.md │ ├── stack-set_add-instances.md │ ├── describe.md │ ├── stack-set_diff.md │ ├── stack-set_remove-instances.md │ ├── diff.md │ ├── new.md │ ├── change.md │ └── stack-set_update.md ├── examples.md ├── config-file.md ├── artifacts.md ├── stack-sets.md └── modules.md ├── formica ├── exceptions.py ├── aws_base.py ├── __init__.py ├── aws.py ├── yaml_tags.py ├── helper.py ├── stack_waiter.py ├── diff.py ├── s3.py ├── change_set.py └── loader.py ├── .dockerignore ├── stacks ├── default.config.yaml ├── modules │ ├── submodule │ │ └── submodule.template.yaml │ └── module.template.yaml ├── README.md ├── test.config.yaml └── test.template.yaml ├── pyproject.toml ├── setup.cfg ├── Manifest.in ├── Release.Dockerfile ├── scripts └── replace-usage.bash ├── Whalebrew.Dockerfile ├── .github └── workflows │ ├── integration.yml │ └── unit-test.yml ├── docker-compose.yml ├── Dockerfile ├── LICENSE ├── CHANGELOG.md ├── Makefile ├── setup.py ├── README.md ├── .gitignore └── CODE_OF_CONDUCT.md /tests/unit/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/integration/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/examples/nested_relative_module/modules/nestedModule/testmodule.template.yaml: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/examples/s3-bucket/stack.config.yaml: -------------------------------------------------------------------------------- 1 | stack: formica-s3 2 | resource-types: true -------------------------------------------------------------------------------- /formica/exceptions.py: -------------------------------------------------------------------------------- 1 | class FormicaArgumentException(Exception): 2 | pass 3 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | logging.basicConfig(level=logging.WARNING) 4 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | tests 2 | .cache 3 | .git 4 | .formica_cli.egg-info 5 | scripts 6 | tmp 7 | docs -------------------------------------------------------------------------------- /stacks/default.config.yaml: -------------------------------------------------------------------------------- 1 | vars: 2 | PolicyName: OtherPolicyName 3 | stack: some-other-stack -------------------------------------------------------------------------------- /docs/examples/s3-lambda/Makefile: -------------------------------------------------------------------------------- 1 | zip-code: 2 | mkdir -p build 3 | zip build/code.py.zip code.py 4 | -------------------------------------------------------------------------------- /docs/examples/s3-lambda/code.py: -------------------------------------------------------------------------------- 1 | def handler(event, context): 2 | print(event) 3 | return event 4 | -------------------------------------------------------------------------------- /docs/examples/nested_relative_module/module1.template.yml: -------------------------------------------------------------------------------- 1 | Resources: 2 | Modules: 3 | From: Modules::Submodule1 -------------------------------------------------------------------------------- /docs/examples/nested_relative_module/modules/submodule1.template.yaml: -------------------------------------------------------------------------------- 1 | Resources: 2 | Modules: 3 | From: .::bucket2 -------------------------------------------------------------------------------- /docs/examples/lambda-step-function/stack.config.yaml: -------------------------------------------------------------------------------- 1 | stack: formica-lambda-step-function 2 | capabilities: 3 | - CAPABILITY_IAM -------------------------------------------------------------------------------- /docs/examples/nested_relative_module/bucket1.template.yaml: -------------------------------------------------------------------------------- 1 | Resources: 2 | DeploymentBucket: 3 | Type: "AWS::S3::Bucket" -------------------------------------------------------------------------------- /docs/examples/nested-stack/stack.config.yaml: -------------------------------------------------------------------------------- 1 | stack: formica-nested-stack 2 | artifacts: 3 | - nested.yaml 4 | upload_artifacts: true -------------------------------------------------------------------------------- /docs/examples/nested_relative_module/modules/bucket2.template.yaml: -------------------------------------------------------------------------------- 1 | Resources: 2 | DeploymentBucket2: 3 | Type: "AWS::S3::Bucket" 4 | -------------------------------------------------------------------------------- /tests/integration/stack.config.yaml: -------------------------------------------------------------------------------- 1 | stack: formica-integration-test-user 2 | capabilities: 3 | - CAPABILITY_IAM 4 | region: eu-central-1 -------------------------------------------------------------------------------- /docs/examples/s3-lambda/stack.config.yaml: -------------------------------------------------------------------------------- 1 | stack: s3-lambda 2 | s3: true 3 | capabilities: 4 | - CAPABILITY_IAM 5 | artifacts: 6 | - build/code.py.zip 7 | - code.py -------------------------------------------------------------------------------- /stacks/modules/submodule/submodule.template.yaml: -------------------------------------------------------------------------------- 1 | Resources: 2 | {{ module_name }}SubModuleBucket: 3 | Type: AWS::S3::Bucket 4 | OtherBucket: 5 | From: .. -------------------------------------------------------------------------------- /stacks/modules/module.template.yaml: -------------------------------------------------------------------------------- 1 | Resources: 2 | {{ module_name }}ModuleBucket: 3 | Type: AWS::S3::Bucket 4 | Properties: 5 | BucketName: {{ BucketName | novalue }} -------------------------------------------------------------------------------- /docs/examples/config-file/test.template.yml: -------------------------------------------------------------------------------- 1 | {% set bucket = "DeploymentBucket" %} 2 | Resources: 3 | {{ bucket }}: 4 | Type: "AWS::S3::Bucket" 5 | {{ bucket }}2: 6 | Type: "AWS::S3::Bucket" 7 | -------------------------------------------------------------------------------- /stacks/README.md: -------------------------------------------------------------------------------- 1 | # Stacks 2 | 3 | This folder has Formica templates set up to test different template loading mechanisms during development. 4 | 5 | Its mostly to play around with some simple stack setups. -------------------------------------------------------------------------------- /formica/aws_base.py: -------------------------------------------------------------------------------- 1 | class AWSBase(object): 2 | def __init__(self, session): 3 | self.session = session 4 | 5 | def cf_client(self): 6 | return self.session.client("cloudformation") 7 | -------------------------------------------------------------------------------- /docs/examples/config-file/test.config.yaml: -------------------------------------------------------------------------------- 1 | stack: teststack 2 | # parameters: 3 | # BucketName: formica-test-bucket-5453245 4 | # SomeParameter: now-for-something-different 5 | tags: 6 | StackTag: formica 7 | region: us-east-1 -------------------------------------------------------------------------------- /docs/examples/commit-build-pipeline/stack.config.yaml: -------------------------------------------------------------------------------- 1 | stack: commit-build-pipeline 2 | stack-set: commit-build-pipeline 3 | capabilities: 4 | - CAPABILITY_IAM 5 | parameters: 6 | TimeoutInMinutes: '25' 7 | tags: 8 | Key: Value1 9 | Some: '2' -------------------------------------------------------------------------------- /docs/examples/nested-stack/nested.template.yaml: -------------------------------------------------------------------------------- 1 | Resources: 2 | NestedStack: 3 | Type: AWS::CloudFormation::Stack 4 | Properties: 5 | TemplateURL: "https://s3.amazonaws.com/{{artifacts['nested.yaml'].bucket}}/{{artifacts['nested.yaml'].key}}" -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | # See PEP 518 for the spec of this file 2 | # https://www.python.org/dev/peps/pep-0518/ 3 | 4 | [tool.black] 5 | # See https://github.com/ambv/black/blob/master/pyproject.toml 6 | target_version = ['py310', 'py311', 'py312'] 7 | line_length=119 -------------------------------------------------------------------------------- /stacks/test.config.yaml: -------------------------------------------------------------------------------- 1 | stack: formica-test-stack 2 | # role-arn: arn:aws:iam::080551076419:role/test/1/formica-test-stack-2-TestRole1-1PGVZJ88QDVBD 3 | capabilities: 4 | - CAPABILITY_IAM 5 | vars: 6 | buckets: 7 | - 1 8 | - 2 9 | PolicyName: TestRolePolicy -------------------------------------------------------------------------------- /docs/examples/nested-stack/nested.yaml: -------------------------------------------------------------------------------- 1 | Resources: 2 | NestedBucket: 3 | Type: AWS::S3::Bucket 4 | NestedBucket2: 5 | Type: AWS::S3::Bucket 6 | Properties: 7 | BucketName: formica-nested-stack-bucket-name 8 | # NestedBucket3: 9 | # Type: AWS::S3::Bucket -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [pycodestyle] 2 | max-line-length = 120 3 | ignore = E402 4 | 5 | [flake8] 6 | max-line-length = 120 7 | ignore = E402 8 | 9 | [mutmut] 10 | paths_to_mutate=formica 11 | backup=False 12 | runner=pytest -x tests/unit 13 | tests_dir=tests/unit/ 14 | dict_synonyms=Struct, NamedStruct -------------------------------------------------------------------------------- /docs/examples/commit-build/README.md: -------------------------------------------------------------------------------- 1 | # Code Commit -> Code Build 2 | 3 | This example will set up a CodeCommit repository. When you push into the repository it will 4 | run a Lambda function that triggers a CodeBuild build. 5 | 6 | The repository URL is added to the Output of the stack so you can push into the repository. -------------------------------------------------------------------------------- /Manifest.in: -------------------------------------------------------------------------------- 1 | # MANIFEST.in 2 | exclude .gitignore 3 | exclude .coverage 4 | exclude .travis.yml 5 | exclude docs 6 | exclude README.md 7 | include README.rst 8 | include LICENSE 9 | include setup.cfg 10 | prune .cache 11 | prune .git 12 | prune build 13 | prune dist 14 | recursive-exclude *.egg-info * 15 | recursive-exclude tests * 16 | -------------------------------------------------------------------------------- /docs/examples/commit-build-pipeline/README.md: -------------------------------------------------------------------------------- 1 | # Code Commit -> Code Pipeline -> Code Build 2 | 3 | This example will set up a CodeCommit repository. When you push into the master branch 4 | it will trigger CodePipeline and run CodeBuild build. 5 | 6 | The repository URL is added to the Output of the stack, make sure to push to the master branch of that repo. -------------------------------------------------------------------------------- /docs/examples/commit-build-pipeline/commit.template.yml: -------------------------------------------------------------------------------- 1 | Resources: 2 | CodeCommitRepository: 3 | Type: AWS::CodeCommit::Repository 4 | Properties: 5 | RepositoryName: formica-test-repo 6 | RepositoryDescription: A Test Repository for Formica 7 | Outputs: 8 | SSHURLCodeCommitRepository: 9 | Value: !GetAtt CodeCommitRepository.CloneUrlSsh -------------------------------------------------------------------------------- /docs/examples/custom-resource/resource.py: -------------------------------------------------------------------------------- 1 | import cfnresponse 2 | 3 | 4 | def handler(event, context): 5 | print(event) 6 | response_data = {} 7 | response_data['Data'] = 'DataResponse' 8 | response_data['Reason'] = 'SomeTestReason' 9 | cfnresponse.send(event, context, cfnresponse.SUCCESS, response_data, "CustomResourcePhysicalID") 10 | -------------------------------------------------------------------------------- /tests/unit/test_describe.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from mock import Mock 3 | 4 | from formica import cli 5 | from tests.unit.constants import STACK 6 | 7 | 8 | def test_describes_change_set(boto_client, change_set): 9 | cli.main(['describe', '--stack', STACK]) 10 | change_set.assert_called_with(stack=STACK) 11 | change_set.return_value.describe.assert_called_once() 12 | -------------------------------------------------------------------------------- /docs/examples/custom-resource/README.md: -------------------------------------------------------------------------------- 1 | # Custom Resources 2 | 3 | This example implements a CloudFormation custom resource through Lambda. 4 | 5 | The code for the custom resource is read from `resource.py` and included directly into the lambda functions CF Code attribute. 6 | 7 | The `cfnresponse` module provided by AWS is used to send the result of the lambda call back to CloudFormation. -------------------------------------------------------------------------------- /formica/__init__.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import sys 3 | 4 | __version__ = "0.15.0" 5 | 6 | CHANGE_SET_FORMAT = "{stack}-change-set" 7 | 8 | logger = logging.getLogger("formica") 9 | handler = logging.StreamHandler(sys.stdout) 10 | formatter = logging.Formatter("%(message)s") 11 | handler.setFormatter(formatter) 12 | logger.addHandler(handler) 13 | logger.setLevel(logging.INFO) 14 | -------------------------------------------------------------------------------- /Release.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:alpine 2 | 3 | LABEL io.whalebrew.name 'formica' 4 | LABEL io.whalebrew.config.environment '["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", "AWS_DEFAULT_REGION", "AWS_DEFAULT_PROFILE", "AWS_PROFILE", "AWS_CONFIG_FILE"]' 5 | LABEL io.whalebrew.config.volumes '["~/.aws:/.aws"]' 6 | 7 | RUN pip install --upgrade pip 8 | RUN pip install formica-cli 9 | 10 | ENTRYPOINT ["formica"] 11 | -------------------------------------------------------------------------------- /docs/examples/lambda-step-function/state-machine.json: -------------------------------------------------------------------------------- 1 | {% set num = 10%} 2 | { 3 | "StartAt": "Hello0", 4 | "States": { 5 | {% for i in range(num) %} 6 | "Hello{{ i }}": { 7 | "Type": "Task", 8 | "Resource": "${lambda}", 9 | "Next": "Hello{{i+1}}" 10 | }, 11 | {% endfor %} 12 | "Hello{{num}}": { 13 | "Type": "Task", 14 | "Resource": "${lambda}", 15 | "End": true 16 | } 17 | } 18 | } -------------------------------------------------------------------------------- /scripts/replace-usage.bash: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | command_files=$(ls docs/commands/*.md | grep -v 'index') 6 | 7 | for file in $command_files 8 | do 9 | echo "Updating Usage in $file" 10 | command=$(echo $file | sed 's/.*\/.*\/\(.*\).md/\1/g' | sed 's/_/ /g') 11 | sed -i '/^## Usage/Q' $file 12 | echo -e '## Usage\n\n```' >> ./$file 13 | formica $command --help >> ./$file 14 | echo '```' >> ./$file 15 | done -------------------------------------------------------------------------------- /docs/examples/commit-build/commit.template.yml: -------------------------------------------------------------------------------- 1 | Resources: 2 | CodeCommitRepository: 3 | Type: AWS::CodeCommit::Repository 4 | Properties: 5 | RepositoryDescription: CPR Test Repository 6 | Triggers: 7 | - DestinationArn: !GetAtt PostPushHookLambdaFunction.Arn 8 | Name: ChangeLambda 9 | Events: ['all'] 10 | Outputs: 11 | SSHURLCodeCommitRepository: 12 | Value: !GetAtt CodeCommitRepository.CloneUrlSsh -------------------------------------------------------------------------------- /Whalebrew.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:alpine 2 | 3 | RUN apk add --no-cache groff less mailcap 4 | 5 | LABEL io.whalebrew.name 'formica' 6 | LABEL io.whalebrew.config.environment '["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", "AWS_DEFAULT_REGION", "AWS_PROFILE", "AWS_CONFIG_FILE"]' 7 | LABEL io.whalebrew.config.volumes '["~/.aws:/.aws"]' 8 | 9 | WORKDIR /formica 10 | 11 | COPY ./ ./ 12 | 13 | RUN pip install . 14 | 15 | ENTRYPOINT ["formica"] -------------------------------------------------------------------------------- /tests/unit/test_docs_examples_links.py: -------------------------------------------------------------------------------- 1 | import os 2 | import re 3 | 4 | 5 | def test_validate_examples_are_linked(): 6 | examples_directory = 'docs/examples' 7 | 8 | with open('docs/examples.md') as file: 9 | documentation = file.read() 10 | for dir in os.listdir(examples_directory): 11 | to_find = '(https://github.com/theserverlessway/formica/tree/master/docs/examples/{})'.format(dir) 12 | assert to_find in documentation 13 | -------------------------------------------------------------------------------- /docs/examples/lambda-step-function/step-function.template.yaml: -------------------------------------------------------------------------------- 1 | Resources: 2 | MyStateMachine: 3 | Type: AWS::StepFunctions::StateMachine 4 | Properties: 5 | DefinitionString: 6 | Fn::Sub: 7 | - "{{ code('state-machine.json') }}" 8 | - lambda: 9 | Fn::GetAtt: 10 | - StepFunctionLambdaFunction 11 | - Arn 12 | RoleArn: 13 | Fn::GetAtt: 14 | - StepFunctionExecutionRole 15 | - Arn -------------------------------------------------------------------------------- /docs/examples/lambda-step-function/code.py: -------------------------------------------------------------------------------- 1 | import urllib.request 2 | 3 | def handler(event, context): 4 | result = {} 5 | print(event) 6 | contents = urllib.request.urlopen("http://theserverlessway.com").read() 7 | if event.get('num') is not None: 8 | print('Adding to Result') 9 | result['num'] = int(event['num']) + 1 10 | else: 11 | print('Setting 0') 12 | result['num'] = 0 13 | print('Result is: {}'.format(result)) 14 | return result 15 | -------------------------------------------------------------------------------- /docs/examples/commit-build/hook.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | 4 | import boto3 5 | 6 | logger = logging.getLogger() 7 | logger.setLevel(logging.INFO) 8 | 9 | 10 | def handler(event, context): 11 | client = boto3.client('codebuild') 12 | project = os.environ['CodeBuildProject'] 13 | commit_id = event['Records'][0]['codecommit']['references'][0]['commit'] 14 | logging.info('START_BUILD {}'.format({'project': project, 'commit_id': commit_id})) 15 | client.start_build(projectName=project, sourceVersion=commit_id) 16 | -------------------------------------------------------------------------------- /formica/aws.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | import botocore 3 | from botocore import credentials 4 | import os 5 | 6 | 7 | def initialize(region, profile): 8 | cli_cache = os.path.join(os.path.expanduser("~"), ".aws/cli/cache") 9 | 10 | session = botocore.session.Session(profile=profile) 11 | session.get_component("credential_provider").get_provider("assume-role").cache = credentials.JSONFileCache( 12 | cli_cache 13 | ) 14 | boto3.setup_default_session(botocore_session=session, region_name=region, profile_name=profile) 15 | -------------------------------------------------------------------------------- /docs/examples/s3-bucket/bucket.template.yaml: -------------------------------------------------------------------------------- 1 | Parameters: 2 | BucketName: 3 | Default: testbucket 4 | Type: String 5 | BucketName2: 6 | Default: testbucket2 7 | Type: String 8 | 9 | {% set bucket = "DeploymentBucket" %} 10 | Resources: 11 | {{ bucket }}: 12 | Type: "AWS::S3::Bucket" 13 | Properties: 14 | BucketName: !Sub ${AWS::StackName}-${BucketName}-${AWS::AccountId} 15 | {{ bucket }}2: 16 | Type: "AWS::S3::Bucket" 17 | Properties: 18 | BucketName: !Sub ${AWS::StackName}-${BucketName2}-${AWS::AccountId} 19 | -------------------------------------------------------------------------------- /.github/workflows/integration.yml: -------------------------------------------------------------------------------- 1 | name: integration-test 2 | on: [push] 3 | jobs: 4 | integration-test: 5 | runs-on: ubuntu-latest 6 | steps: 7 | - uses: actions/checkout@v4 8 | - uses: actions/setup-python@v5 9 | with: 10 | python-version: '3.12' 11 | - run: make dependencies 12 | - run: make integration-test 13 | env: 14 | AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} 15 | AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} 16 | AWS_DEFAULT_REGION: eu-central-1 -------------------------------------------------------------------------------- /docs/examples/lambda-step-function/lambda.template.yaml: -------------------------------------------------------------------------------- 1 | Resources: 2 | LambdaLogGroup: 3 | Type: AWS::Logs::LogGroup 4 | Properties: 5 | LogGroupName: 6 | Fn::Join: 7 | - '' 8 | - - "/aws/lambda/" 9 | - Ref: StepFunctionLambdaFunction 10 | StepFunctionLambdaFunction: 11 | Type: AWS::Lambda::Function 12 | Properties: 13 | Code: 14 | ZipFile: "{{ code('code.py') }}" 15 | Handler: index.handler 16 | Role: 17 | Fn::GetAtt: 18 | - LambdaExecutionRole 19 | - Arn 20 | Runtime: python2.7 21 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2' 2 | services: 3 | formica: 4 | build: 5 | dockerfile: Dockerfile 6 | context: . 7 | volumes: 8 | - .:/app 9 | - ~/.aws/:/root/.aws 10 | - /app/formica_cli.egg-info 11 | - /app/build 12 | - /app/dist 13 | - /app/.pytest_cache 14 | - /app/.idea 15 | - /app/.cache 16 | - /app/htmlcov 17 | - ./.bash_history:/root/.bash_history 18 | environment: 19 | - AWS_ACCESS_KEY_ID 20 | - AWS_SECRET_ACCESS_KEY 21 | - AWS_PROFILE 22 | - AWS_SESSION_TOKEN 23 | - AWS_SECURITY_TOKEN 24 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:latest 2 | 3 | WORKDIR /app 4 | 5 | RUN apt-get update -y 6 | RUN apt-get install -y groff pandoc gcc libffi-dev libssl-dev openssl musl-dev python-dev bash 7 | 8 | # Copy Setup files early so they bust caches on dependencies 9 | COPY setup.py setup.py 10 | COPY setup.cfg setup.cfg 11 | 12 | RUN pip install -U wheel pygments twine 13 | RUN pip install -U awslogs awscli 14 | COPY build-requirements.txt build-requirements.txt 15 | RUN pip install -U -r build-requirements.txt 16 | 17 | COPY formica/__init__.py formica/__init__.py 18 | COPY README.md README.md 19 | 20 | RUN pandoc --from=markdown --to=rst --output=README.rst README.md 21 | 22 | RUN python setup.py develop 23 | -------------------------------------------------------------------------------- /tests/unit/test_aws.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from formica import aws 4 | from tests.unit.constants import REGION, PROFILE 5 | 6 | 7 | @pytest.fixture 8 | def boto(mocker): 9 | return mocker.patch('formica.aws.boto3') 10 | 11 | 12 | def test_init_without_parameters(boto, session, botocore_session, mocker): 13 | region = 'region' 14 | profile = 'profile' 15 | session_mock = mocker.Mock() 16 | botocore_session.return_value = session_mock 17 | 18 | aws.initialize(region, profile) 19 | botocore_session.assert_called_with(profile=profile) 20 | 21 | boto.setup_default_session.assert_called_with(botocore_session=session_mock, region_name=region, profile_name=profile) 22 | -------------------------------------------------------------------------------- /docs/examples/lambda-step-function/lambda-role.template.yaml: -------------------------------------------------------------------------------- 1 | Resources: 2 | LambdaExecutionRole: 3 | Properties: 4 | AssumeRolePolicyDocument: 5 | Statement: 6 | - Action: 7 | - sts:AssumeRole 8 | Effect: Allow 9 | Principal: 10 | Service: 11 | - lambda.amazonaws.com 12 | Version: '2012-10-17' 13 | Path: "/" 14 | Policies: 15 | - PolicyDocument: 16 | Statement: 17 | - Action: 18 | - "logs:CreateLogStream" 19 | - "logs:PutLogEvents" 20 | Effect: Allow 21 | Resource: "arn:aws:logs:*:*:*" 22 | Version: '2012-10-17' 23 | PolicyName: root 24 | Type: AWS::IAM::Role -------------------------------------------------------------------------------- /docs/examples/s3-lambda/role.template.yaml: -------------------------------------------------------------------------------- 1 | Resources: 2 | LambdaExecutionRole: 3 | Properties: 4 | PermissionsBoundary: !Sub arn:aws:iam::${AWS::AccountId}:policy/CreatedIdentitiesPermissionsBoundary 5 | AssumeRolePolicyDocument: 6 | Statement: 7 | - Action: 8 | - sts:AssumeRole 9 | Effect: Allow 10 | Principal: 11 | Service: 12 | - lambda.amazonaws.com 13 | Version: '2012-10-17' 14 | Path: "/" 15 | Policies: 16 | - PolicyDocument: 17 | Statement: 18 | - Action: 19 | - "logs:CreateLogStream" 20 | - "logs:PutLogEvents" 21 | Effect: Allow 22 | Resource: "arn:aws:logs:*:*:*" 23 | Version: '2012-10-17' 24 | PolicyName: root 25 | Type: AWS::IAM::Role -------------------------------------------------------------------------------- /docs/s3-deployment.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: S3 Deployment 3 | weight: 500 4 | --- 5 | 6 | Sometimes your template is too large to be deployed directly to the CloudFormation API and has to be pushed to S3 first. 7 | 8 | The `--s3` option will create a new bucket, push the template into the bucket and deploy the template to CloudFormation. 9 | After the deployment the template and bucket will be removed. 10 | 11 | S3 is only used as transient storage during the deployment and is not considered for longer time storage at the moment. 12 | 13 | If you want to limit the buckets formica has access to you can use the following IAM statement to give a user or role 14 | only access to buckets that start with `formica-deploy-`. 15 | 16 | ``` 17 | Statement: 18 | - Action: 19 | - 's3:*' 20 | Effect: Allow 21 | Resource: arn:aws:s3:::formica-deploy-* 22 | ``` -------------------------------------------------------------------------------- /tests/unit/test_cancel_wait.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from mock import Mock 3 | 4 | from formica import cli 5 | from tests.unit.constants import STACK, STACK_ID, EVENT_ID 6 | 7 | 8 | def test_cancel_stack_update(boto_client, aws_client, stack_waiter): 9 | aws_client.describe_stacks.return_value = {'Stacks': [{'StackId': STACK_ID}]} 10 | aws_client.describe_stack_events.return_value = {'StackEvents': [{'EventId': EVENT_ID}]} 11 | cli.main(['cancel', '--stack', STACK]) 12 | boto_client.assert_called_with('cloudformation') 13 | aws_client.cancel_update_stack.assert_called_with(StackName=STACK) 14 | 15 | 16 | def test_wait(aws_client, stack_waiter): 17 | aws_client.describe_stacks.return_value = {'Stacks': [{'StackId': STACK_ID}]} 18 | aws_client.describe_stack_events.return_value = {'StackEvents': [{'EventId': EVENT_ID}]} 19 | cli.main(['wait', '--stack', STACK]) 20 | -------------------------------------------------------------------------------- /docs/examples/lambda-step-function/step-function-role.template.yaml: -------------------------------------------------------------------------------- 1 | Resources: 2 | StepFunctionExecutionRole: 3 | Properties: 4 | AssumeRolePolicyDocument: 5 | Statement: 6 | - Action: 7 | - sts:AssumeRole 8 | Effect: Allow 9 | Principal: 10 | Service: 11 | - Fn::Join: 12 | - '' 13 | - - states. 14 | - Ref: AWS::Region 15 | - .amazonaws.com 16 | Version: '2012-10-17' 17 | Path: "/" 18 | Policies: 19 | - PolicyDocument: 20 | Statement: 21 | - Action: 22 | - "lambda:InvokeFunction" 23 | Effect: Allow 24 | Resource: 25 | Fn::GetAtt: 26 | - StepFunctionLambdaFunction 27 | - Arn 28 | Version: '2012-10-17' 29 | PolicyName: root 30 | Type: AWS::IAM::Role -------------------------------------------------------------------------------- /docs/commands/stack-set_remove.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: StackSet Remove 3 | weight: 100 4 | --- 5 | 6 | # `formica stack-set remove` 7 | 8 | The `formica stack-set remove` command allows you to remove a StackSet after you 9 | removed all the instances of that StackSet. 10 | 11 | ## Usage 12 | 13 | ``` 14 | usage: formica stack-set remove [-h] [--region REGION] [--profile PROFILE] 15 | [--stack-set STACK-Set] 16 | [--config-file CONFIG_FILE [CONFIG_FILE ...]] 17 | 18 | Remove a Stack Set 19 | 20 | options: 21 | -h, --help show this help message and exit 22 | --region REGION The AWS region to use 23 | --profile PROFILE The AWS profile to use 24 | --stack-set STACK-Set, -s STACK-Set 25 | The Stack Set to use 26 | --config-file CONFIG_FILE [CONFIG_FILE ...], -c CONFIG_FILE [CONFIG_FILE ...] 27 | Set the config files to use 28 | ``` 29 | -------------------------------------------------------------------------------- /docs/examples/s3-lambda/bucket.template.yml: -------------------------------------------------------------------------------- 1 | Resources: 2 | TestBucket: 3 | Type: AWS::S3::Bucket 4 | Properties: 5 | NotificationConfiguration: 6 | LambdaConfigurations: 7 | - Event: s3:ObjectCreated:* 8 | Function: 9 | Fn::GetAtt: 10 | - TestFunction 11 | - Arn 12 | - Event: s3:ObjectRemoved:* 13 | Function: 14 | Fn::GetAtt: 15 | - TestFunction 16 | - Arn 17 | 18 | 19 | TestBucketS3: 20 | Type: AWS::S3::Bucket 21 | Properties: 22 | NotificationConfiguration: 23 | LambdaConfigurations: 24 | - Event: s3:ObjectCreated:* 25 | Function: 26 | Fn::GetAtt: 27 | - TestFunctionS3 28 | - Arn 29 | - Event: s3:ObjectRemoved:* 30 | Function: 31 | Fn::GetAtt: 32 | - TestFunctionS3 33 | - Arn 34 | 35 | -------------------------------------------------------------------------------- /tests/unit/test_helper.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from formica import helper 4 | 5 | 6 | def test_with_artifacts(mocker, temp_bucket): 7 | class Namespace: 8 | def __init__(self): 9 | self.artifacts = ['file1'] 10 | 11 | print(temp_bucket) 12 | function = mocker.Mock() 13 | func = helper.with_artifacts(function) 14 | n = Namespace() 15 | print(n.artifacts) 16 | func(n) 17 | temp_bucket.add_file.assert_called_with('file1') 18 | temp_bucket.upload.assert_called() 19 | function.assert_called_with(n) 20 | 21 | 22 | def test_with_artifacts_with_empty_artifacts(mocker): 23 | class Namespace: 24 | def __init__(self): 25 | self.artifacts = [] 26 | 27 | t = mocker.patch('formica.helper.temporary_bucket') 28 | function = mocker.Mock() 29 | func = helper.with_artifacts(function) 30 | n = Namespace() 31 | func(n) 32 | t.assert_not_called() 33 | function.assert_called_with(n) 34 | -------------------------------------------------------------------------------- /stacks/test.template.yaml: -------------------------------------------------------------------------------- 1 | Resources: 2 | {% for bucket in buckets %} 3 | S3TestBucket{{ bucket }}: 4 | Type: AWS::S3::Bucket 5 | {% endfor %} 6 | 7 | TestRole1: 8 | Properties: 9 | AssumeRolePolicyDocument: 10 | Version: "2012-10-17" 11 | Statement: 12 | - 13 | Effect: "Allow" 14 | Principal: 15 | Service: 16 | - "cloudformation.amazonaws.com" 17 | Action: 18 | - "sts:AssumeRole" 19 | Path: /test/1/ 20 | Policies: 21 | - PolicyDocument: 22 | Version: "2012-10-17" 23 | Statement: 24 | - 25 | Effect: "Allow" 26 | Action: "*" 27 | Resource: "*" 28 | PolicyName: {{ PolicyName }} 29 | Type: AWS::IAM::Role 30 | 31 | Bucket1: 32 | From: Modules 33 | Properties: 34 | BucketName: formica-test-bucket-name-something-here 35 | Bucket2: 36 | From: Modules::Submodule::Submodule 37 | -------------------------------------------------------------------------------- /tests/unit/test_resources.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import mock 3 | from mock import Mock 4 | 5 | from formica import cli 6 | from formica.cli import RESOURCE_HEADERS 7 | from tests.unit.constants import STACK, LIST_STACK_RESOURCES 8 | 9 | 10 | def test_print_stacks(client, session, logger): 11 | client.get_paginator.return_value.paginate.return_value = [LIST_STACK_RESOURCES] 12 | cli.main(['resources', '--stack', STACK]) 13 | 14 | client.get_paginator.assert_called_with('list_stack_resources') 15 | client.get_paginator.return_value.paginate.assert_called_with(StackName=STACK) 16 | 17 | logger.info.assert_called_with(mock.ANY) 18 | args = logger.info.call_args[0] 19 | 20 | to_search = [] 21 | to_search.extend(RESOURCE_HEADERS) 22 | to_search.extend(['AWS::Route53::HostedZone']) 23 | to_search.extend(['FlomotlikMe']) 24 | to_search.extend(['CREATE_COMPLETE']) 25 | to_search.extend(['ZAYGDOKFPYFK6']) 26 | change_set_output = args[0] 27 | for term in to_search: 28 | assert term in change_set_output 29 | assert 'None' not in change_set_output 30 | -------------------------------------------------------------------------------- /tests/integration/integration-user.template.yaml: -------------------------------------------------------------------------------- 1 | Resources: 2 | FormicaIntegrationTestUser: 3 | Properties: 4 | PermissionsBoundary: !Sub arn:aws:iam::${AWS::AccountId}:policy/CreatedIdentitiesPermissionsBoundary 5 | Policies: 6 | - PolicyDocument: 7 | Statement: 8 | - Sid: S3Access 9 | Effect: Allow 10 | Action: 11 | - 's3:*' 12 | Resource: 13 | - 'arn:aws:s3:::formica-it-*' 14 | - 'arn:aws:s3:::formica-deploy-*' 15 | - Sid: CloudFormationAccess 16 | Effect: Allow 17 | Action: 18 | - 'cloudformation:*' 19 | Resource: 20 | - !Sub arn:aws:cloudformation:${AWS::Region}:${AWS::AccountId}:stack/formica-it-* 21 | - Sid: CloudFormationDescribeStacks 22 | Effect: Allow 23 | Action: 24 | - 'cloudformation:DescribeStacks' 25 | Resource: 26 | - '*' 27 | PolicyName: formicadeployment 28 | Type: AWS::IAM::User -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Florian Motlik 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /tests/unit/test_stacks.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import mock 3 | from mock import Mock 4 | 5 | from formica import cli 6 | from formica.cli import STACK_HEADERS 7 | from tests.unit.constants import DESCRIBE_STACKS 8 | 9 | 10 | def test_print_stacks(client, logger): 11 | client.describe_stacks.return_value = DESCRIBE_STACKS 12 | cli.main(['stacks']) 13 | 14 | logger.info.assert_called_with(mock.ANY) 15 | args = logger.info.call_args[0] 16 | 17 | to_search = [] 18 | to_search.extend(STACK_HEADERS) 19 | to_search.extend(['teststack']) 20 | to_search.extend(['2017-01-31 11:51:43.596000']) 21 | to_search.extend(['2017-01-31 13:55:20.357000']) 22 | to_search.extend(['UPDATE_COMPLETE']) 23 | change_set_output = args[0] 24 | for term in to_search: 25 | assert term in change_set_output 26 | assert 'None' not in change_set_output 27 | 28 | 29 | def test_does_not_fail_without_update_date(client, logger): 30 | client.describe_stacks.return_value = { 31 | "Stacks": [{'StackName': 'abc', 'CreationTime': '12345', 'StackStatus': 'status'}] 32 | } 33 | 34 | cli.main(['stacks']) 35 | logger.info.assert_called() 36 | -------------------------------------------------------------------------------- /docs/examples/commit-build-pipeline/pipeline.template.yml: -------------------------------------------------------------------------------- 1 | Resources: 2 | CodeCommitCodePipeline: 3 | Type: AWS::CodePipeline::Pipeline 4 | Properties: 5 | RoleArn: !GetAtt CodeCommitAccessRole.Arn 6 | ArtifactStore: 7 | Type: S3 8 | Location: !Ref PipelineBucket 9 | Stages: 10 | - Name: Source 11 | Actions: 12 | - Name: FetchSource 13 | ActionTypeId: 14 | Category: Source 15 | Owner: AWS 16 | Provider: CodeCommit 17 | Version: 1 18 | Configuration: 19 | RepositoryName: !GetAtt CodeCommitRepository.Name 20 | BranchName: master 21 | OutputArtifacts: 22 | - Name: Source 23 | - Name: Build 24 | Actions: 25 | - Name: BuildAndTestApp 26 | ActionTypeId: 27 | Category: Build 28 | Owner: AWS 29 | Provider: CodeBuild 30 | Version: 1 31 | Configuration: 32 | ProjectName: !Ref CodeBuildProject 33 | InputArtifacts: 34 | - Name: Source 35 | PipelineBucket: 36 | Type: AWS::S3::Bucket -------------------------------------------------------------------------------- /.github/workflows/unit-test.yml: -------------------------------------------------------------------------------- 1 | name: unit-test 2 | on: [push, pull_request] 3 | jobs: 4 | unit-test: 5 | runs-on: ubuntu-latest 6 | strategy: 7 | matrix: 8 | python: ['3.10', '3.11', '3.12'] 9 | steps: 10 | - uses: actions/checkout@v4 11 | - uses: actions/setup-python@v5 12 | with: 13 | python-version: ${{ matrix.python }} 14 | - run: make dependencies 15 | - run: make check-code 16 | - run: make test 17 | env: 18 | AWS_DEFAULT_REGION: eu-central-1 19 | - name: Coveralls Reporting 20 | run: | 21 | echo "$GITHUB_REF" | sed "s/refs\/heads\///g" 22 | GIT_BRANCH=$(echo "$GITHUB_REF" | sed "s/refs\/heads\///g") coveralls 23 | env: 24 | COVERALLS_REPO_TOKEN: iKd3pRRJvcQpdTiX3yKgAagmIwlob4DI2 25 | COVERALLS_PARALLEL: true 26 | AWS_DEFAULT_REGION: eu-central-1 27 | coveralls_merge: 28 | needs: unit-test 29 | runs-on: ubuntu-latest 30 | steps: 31 | - name: Coveralls Parallel finish 32 | run: curl -k https://coveralls.io/webhook?repo_token=$COVERALLS_REPO_TOKEN -d "payload[build_num]=$GITHUB_SHA&payload[status]=done" 33 | env: 34 | COVERALLS_REPO_TOKEN: iKd3pRRJvcQpdTiX3yKgAagmIwlob4DI2 -------------------------------------------------------------------------------- /docs/commands/_index.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Commands Reference 3 | disable_section_links: true 4 | weight: 100 5 | --- 6 | 7 | ## Stacks 8 | 9 | * [cancel:](cancel) Cancel a deployment 10 | * [change:](change) Create a change set for an existing stack 11 | * [deploy:](deploy) Deploy the latest change set for a stack 12 | * [describe:](describe) Describe the latest change set 13 | * [diff](diff) Print a diff between local and deployed stack 14 | * [new:](new) Create a change set for a new stack 15 | * [remove:](remove) Remove the configured stack 16 | * [resources:](resources) List all resources of a stack 17 | * [stacks:](stacks) List all stacks 18 | * [template:](template) Print the current template 19 | * [wait:](wait) Wait for a deployment to finish 20 | 21 | ## Stack Sets 22 | 23 | * [stack-set add-instances:](stack-set_add-instances) Add an Instance to a StackSet 24 | * [stack-set create:](stack-set_create) Create a StackSet 25 | * [stack-set diff:](stack-set_diff) Print a diff between local and deployed StackSet template 26 | * [stack-set remove-instances:](stack-set_remove-instances) Remove an Instance from a StackSet 27 | * [stack-set remove:](stack-set_remove) Remove a StackSet 28 | * [stack-set update:](stack-set_update) Update a StackSet 29 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # 0.7.0 2 | 3 | * [New Module Syntax](https://github.com/flomotlik/formica/pull/78) - [Documentation](https://theserverlessway.com/tools/formica/modules/) 4 | * [Multi config file and overwrite support](https://github.com/flomotlik/formica/pull/80) - [Documentation](https://theserverlessway.com/tools/formica/config-file/) 5 | * [Support deployment through S3 for larger templates](https://github.com/flomotlik/formica/pull/82) - [Documentation](https://theserverlessway.com/tools/formica/s3-deployment/) 6 | * [Move documentation](https://github.com/flomotlik/formica/pull/79) - [New Documentation](https://theserverlessway.com/tools/formica/) 7 | 8 | # 0.6.2 9 | 10 | * [Support for role-arn CloudFormation parameter](https://github.com/flomotlik/formica/pull/78) - [Documentation]() 11 | 12 | # 0.6.0 13 | 14 | * [Support for Jinja2 vars in config files](https://github.com/flomotlik/formica/pull/64): [docs](https://github.com/flomotlik/formica/blob/master/docs/config-file.md) 15 | 16 | [Comparison since last Version](https://github.com/flomotlik/formica/compare/0.5.2...0.6.0) 17 | 18 | # 0.5.0 19 | 20 | * [Config file support](https://github.com/flomotlik/formica/issues/55): [docs](https://github.com/flomotlik/formica/blob/master/docs/config-file.md) -------------------------------------------------------------------------------- /docs/examples/custom-resource/custom.template.yaml: -------------------------------------------------------------------------------- 1 | Resources: 2 | LambdaExecutionRole: 3 | Type: AWS::IAM::Role 4 | Properties: 5 | AssumeRolePolicyDocument: 6 | Version: '2012-10-17' 7 | Statement: 8 | - Effect: Allow 9 | Principal: 10 | Service: 11 | - lambda.amazonaws.com 12 | Action: 13 | - sts:AssumeRole 14 | Path: "/" 15 | Policies: 16 | - PolicyName: root 17 | PolicyDocument: 18 | Version: '2012-10-17' 19 | Statement: 20 | - Effect: Allow 21 | Action: 22 | - logs:CreateLogStream 23 | - logs:CreateLogGroup 24 | - logs:PutLogEvents 25 | Resource: arn:aws:logs:*:*:* 26 | CustomFunction: 27 | Type: AWS::Lambda::Function 28 | Properties: 29 | Code: 30 | ZipFile: {{ code('resource.py') }} 31 | Handler: index.handler 32 | Role: 33 | Fn::GetAtt: 34 | - LambdaExecutionRole 35 | - Arn 36 | Runtime: python2.7 37 | CustomResource: 38 | Type: Custom::TestResource 39 | Properties: 40 | ServiceToken: 41 | Fn::GetAtt: 42 | - CustomFunction 43 | - Arn 44 | FunctionName: 45 | Ref: CustomFunction -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: build 2 | 3 | dependencies: 4 | python --version 5 | pip install -U -r build-requirements.txt 6 | python setup.py develop 7 | 8 | test: 9 | py.test --cov-branch --cov-report html --cov-report term-missing --cov=formica tests/unit 10 | 11 | check-code: 12 | black --check --verbose ./formica 13 | pyflakes ./formica 14 | grep -r 'print(' formica; [ "$$?" -gt 0 ] 15 | 16 | mutation: 17 | mutmut run 18 | 19 | integration-test: 20 | py.test -s tests/integration 21 | 22 | build-dev: 23 | docker-compose build formica 24 | 25 | shell: build-dev 26 | touch .bash_history 27 | docker-compose run formica bash 28 | 29 | clean: 30 | rm -fr dist/* build/* formica_cli.egg-info/* htmlcov/* .pytest-cache/* 31 | 32 | release-pypi: 33 | docker-compose run formica bash -c "make clean && python setup.py sdist bdist_wheel && pandoc --from=markdown --to=rst --output=build/README.rst README.md && twine upload dist/*" 34 | 35 | release-docker: 36 | docker build --no-cache -t flomotlik/formica -f Release.Dockerfile . 37 | docker push flomotlik/formica 38 | 39 | release: release-pypi release-docker 40 | 41 | install: 42 | docker build -t flomotlik/formica:whalebrew -f Whalebrew.Dockerfile . 43 | whalebrew install -f flomotlik/formica:whalebrew 44 | 45 | update-usage: 46 | bash scripts/replace-usage.bash -------------------------------------------------------------------------------- /docs/examples/commit-build-pipeline/role.template.yml: -------------------------------------------------------------------------------- 1 | Resources: 2 | CodeCommitAccessRole: 3 | Properties: 4 | PermissionsBoundary: !Sub arn:aws:iam::${AWS::AccountId}:policy/CreatedIdentitiesPermissionsBoundary 5 | AssumeRolePolicyDocument: 6 | Version: '2012-10-17' 7 | Statement: 8 | - Effect: Allow 9 | Principal: 10 | Service: 11 | - 'codepipeline.amazonaws.com' 12 | Action: 13 | - 'sts:AssumeRole' 14 | Path: "/" 15 | Policies: 16 | - PolicyDocument: 17 | Statement: 18 | - Action: 19 | - "codecommit:GetBranch" 20 | - "codecommit:GetCommit" 21 | - "codecommit:UploadArchive" 22 | - "codecommit:GetUploadArchiveStatus" 23 | - "codecommit:CancelUploadArchive" 24 | Effect: Allow 25 | Resource: !GetAtt CodeCommitRepository.Arn 26 | - Action: 27 | - "s3:*" 28 | Effect: Allow 29 | Resource: !Join ['', ['arn:aws:s3:::', !Ref PipelineBucket, '/*']] 30 | - Action: 31 | - "codebuild:*" 32 | Effect: Allow 33 | Resource: !GetAtt CodeBuildProject.Arn 34 | Version: '2012-10-17' 35 | PolicyName: root 36 | Type: AWS::IAM::Role 37 | -------------------------------------------------------------------------------- /docs/examples.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Examples 3 | weight: 700 4 | --- 5 | 6 | ## Module System 7 | * [nested and relative modules](https://github.com/theserverlessway/formica/tree/master/docs/examples/nested_relative_module) 8 | 9 | ## S3 10 | * [Simple S3 Bucket](https://github.com/theserverlessway/formica/tree/master/docs/examples/s3-bucket) 11 | * [S3 Bucket that runs Lambda on File upload/remove](https://github.com/theserverlessway/formica/tree/master/docs/examples/s3-lambda) 12 | 13 | ## Step Functions 14 | * [Lambda Step functions with dynamic steps](https://github.com/theserverlessway/formica/tree/master/docs/examples/lambda-step-function) 15 | 16 | ## Custom Resources 17 | * [Custom Resource](https://github.com/theserverlessway/formica/tree/master/docs/examples/custom-resource) 18 | 19 | ## CodeCommit, CodePipeline, CodeBuild 20 | 21 | * [CodeCommit and CodeBuild for every push](https://github.com/theserverlessway/formica/tree/master/docs/examples/commit-build) 22 | * [CodeCommit -> CodePipeline -> CodeBuild for the master branch](https://github.com/theserverlessway/formica/tree/master/docs/examples/commit-build-pipeline) 23 | 24 | ## Configuration File 25 | 26 | * [Configuration File](https://github.com/theserverlessway/formica/tree/master/docs/examples/config-file) 27 | 28 | ## Nested Stack with template Artifacts 29 | 30 | * [Nested Stack with template Artifacts](https://github.com/theserverlessway/formica/tree/master/docs/examples/nested-stack) -------------------------------------------------------------------------------- /docs/config-file.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Configuration File Reference 3 | weight: 300 4 | --- 5 | 6 | So you don't have to repeat command line options constantly Formica supports config files. Those files allow you to set anything, up to the AWS profile or region to use. 7 | Command line arguments will have precedence over the config files, so you can use a config file, but still change values. Values will be merged, so if you for example specify tags in the config file and on the command line they will be merged or overwritten with the command line taking precedence. 8 | 9 | To make it easy to have centralise common options and then have different config files for different environments (e.g. dev/staging/production) you can set multiple config files. They will be loaded in order and will overwrite earlier values. So for example if you set a default Parameter for a stack but then overwrite it in another config file the latter will be used. 10 | 11 | Config files can either be json or yaml. The `--config-file` (or `-c` as a shorthand) option can be used to set the config files. 12 | 13 | ### Example 14 | 15 | The following example contains all available options you can set: 16 | 17 | ```yaml 18 | stack: teststack 19 | parameters: 20 | BucketName: s3-bucket-name 21 | SomeParameter: some-value 22 | tags: 23 | StackTag: formica-test-tag 24 | capabilities: 25 | - CAPABILITY_IAM 26 | - CAPABILITY_NAMED_IAM 27 | region: us-east-1 28 | profile: production 29 | vars: 30 | domain: flomotlik.me 31 | ``` 32 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | """Packaging settings.""" 2 | 3 | from os.path import abspath, dirname, join, isfile 4 | 5 | from setuptools import setup 6 | 7 | from formica import __version__ 8 | 9 | this_dir = abspath(dirname(__file__)) 10 | path = join(this_dir, 'build/README.rst') 11 | long_description = '' 12 | if isfile(path): 13 | with open(path) as file: 14 | long_description = file.read() 15 | 16 | setup( 17 | name='formica-cli', 18 | python_requires=">=3.10", 19 | version=__version__, 20 | description='Simple AWS CloudFormation stack management tooling.', 21 | long_description=long_description, 22 | url='https://github.com/flomotlik/formica', 23 | author='Florian Motlik', 24 | author_email='flo@flomotlik.me', 25 | license='MIT', 26 | classifiers=[ 27 | 'Intended Audience :: Developers', 28 | 'Topic :: Utilities', 29 | 'License :: Public Domain', 30 | 'Natural Language :: English', 31 | 'Operating System :: OS Independent', 32 | 'Programming Language :: Python :: 3', 33 | 'Programming Language :: Python :: 3.10', 34 | 'Programming Language :: Python :: 3.11', 35 | 'Programming Language :: Python :: 3.12', 36 | ], 37 | keywords='cloudformation, aws, cloud', 38 | packages=['formica'], 39 | install_requires=['boto3>=1.18.35,<2.0.0', 'texttable>=1.2.0', 'jinja2>=3.0', 'pyyaml>=4.2b1', 40 | 'deepdiff>=5.0.0', 'arrow>=1.0.0', 'argcomplete>=1.9.4'], 41 | entry_points={ 42 | 'console_scripts': [ 43 | 'formica=formica.cli:formica', 44 | ], 45 | } 46 | ) 47 | -------------------------------------------------------------------------------- /docs/examples/commit-build/build.template.yml: -------------------------------------------------------------------------------- 1 | Resources: 2 | CodeBuildProject: 3 | Type: 'AWS::CodeBuild::Project' 4 | Properties: 5 | Artifacts: 6 | Type: NO_ARTIFACTS 7 | Environment: 8 | ComputeType: 'BUILD_GENERAL1_SMALL' 9 | Image: 'python:3.6' 10 | Type: 'LINUX_CONTAINER' 11 | Name: !Sub '${AWS::StackName}-build' 12 | ServiceRole: !GetAtt 'CodeBuildRole.Arn' 13 | Source: 14 | Type: CODECOMMIT 15 | Location: !GetAtt CodeCommitRepository.CloneUrlHttp 16 | BuildSpec: | 17 | version: 0.1 18 | phases: 19 | build: 20 | commands: 21 | - 'ls' 22 | TimeoutInMinutes: 10 23 | CodeBuildRole: 24 | Type: 'AWS::IAM::Role' 25 | Properties: 26 | AssumeRolePolicyDocument: 27 | Version: '2012-10-17' 28 | Statement: 29 | - Effect: Allow 30 | Principal: 31 | Service: 32 | - 'codebuild.amazonaws.com' 33 | Action: 34 | - 'sts:AssumeRole' 35 | Policies: 36 | - PolicyName: ServiceRole 37 | PolicyDocument: 38 | Version: '2012-10-17' 39 | Statement: 40 | - Sid: CloudWatchLogsPolicy 41 | Effect: Allow 42 | Action: 43 | - 'logs:CreateLogGroup' 44 | - 'logs:CreateLogStream' 45 | - 'logs:PutLogEvents' 46 | Resource: '*' 47 | - Sid: CodeCommitPolicy 48 | Effect: Allow 49 | Action: 'codecommit:GitPull' 50 | Resource: !GetAtt CodeCommitRepository.Arn -------------------------------------------------------------------------------- /tests/unit/test_remove.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from mock import patch, Mock 3 | 4 | from formica import cli 5 | from tests.unit.constants import REGION, PROFILE, STACK, EVENT_ID, STACK_ID, ROLE_ARN 6 | 7 | 8 | def test_removes_stack(change_set, session, loader, stack_waiter, client): 9 | client.describe_stacks.return_value = {'Stacks': [{'StackId': STACK_ID}]} 10 | session.return_value.client.return_value = client 11 | client.describe_stack_events.return_value = {'StackEvents': [{'EventId': EVENT_ID}]} 12 | 13 | cli.main(['remove', '--stack', STACK, '--profile', PROFILE, '--region', REGION]) 14 | client.describe_stacks.assert_called_with(StackName=STACK) 15 | client.describe_stack_events.assert_called_with(StackName=STACK) 16 | client.delete_stack.assert_called_with(StackName=STACK) 17 | stack_waiter.assert_called_with(STACK_ID) 18 | stack_waiter.return_value.wait.assert_called_with(EVENT_ID) 19 | 20 | 21 | def test_removes_stack_with_role(change_set, session, loader, stack_waiter, client): 22 | client.describe_stacks.return_value = {'Stacks': [{'StackId': STACK_ID}]} 23 | client.describe_stack_events.return_value = {'StackEvents': [{'EventId': EVENT_ID}]} 24 | 25 | cli.main(['remove', '--stack', STACK, '--profile', PROFILE, '--region', REGION, '--role-arn', ROLE_ARN]) 26 | client.describe_stacks.assert_called_with(StackName=STACK) 27 | client.describe_stack_events.assert_called_with(StackName=STACK) 28 | client.delete_stack.assert_called_with(StackName=STACK, RoleARN=ROLE_ARN) 29 | stack_waiter.assert_called_with(STACK_ID) 30 | stack_waiter.return_value.wait.assert_called_with(EVENT_ID) 31 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Formica 2 | [![Build Status](https://travis-ci.org/theserverlessway/formica.svg?branch=master)](https://travis-ci.org/theserverlessway/formica) 3 | [![PyPI version](https://badge.fury.io/py/formica-cli.svg)](https://pypi.python.org/pypi/formica-cli) 4 | [![license](https://img.shields.io/github/license/theserverlessway/formica.svg)](LICENSE) 5 | [![Coverage Status](https://coveralls.io/repos/github/theserverlessway/formica/badge.svg?branch=master)](https://coveralls.io/github/theserverlessway/formica?branch=master) 6 | 7 | >Dropped Python 2 compatibility, please upgrade to Python 3. The tool might work or it might not, its not tested against Python 2 8 | 9 | Formica makes it easy to create and deploy CloudFormation stacks. It uses CloudFormation syntax with yaml and json support to define your templates. Any existing stack can be used directly, but formica also has built-in modularity so you can reuse and share CloudFormation stack components easily. This allows you to start from an existing stack but split it up into separate files easily. 10 | 11 | For dynamic elements in your templates Formica supports [jinja2](http://jinja.pocoo.org/docs/2.9/templates/) as a templating 12 | engine. Jinja2 is widely used, for example in ansible configuration files. 13 | 14 | ## Installation 15 | 16 | Formica can be installed through pip: 17 | 18 | ``` 19 | pip install formica-cli 20 | ``` 21 | 22 | Alternatively you can clone this repository and run 23 | 24 | ``` 25 | python setup.py install 26 | ``` 27 | 28 | 29 | ## Documentation 30 | 31 | Check out the full [Documentation and Quickstart on TheServerlessWay.com](https://theserverlessway.com/tools/formica/) 32 | -------------------------------------------------------------------------------- /tests/unit/test_rollback.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from mock import patch, Mock 3 | 4 | from formica import cli 5 | from tests.unit.constants import REGION, PROFILE, STACK, EVENT_ID, STACK_ID, ROLE_ARN 6 | 7 | 8 | def test_rollback_stack(change_set, session, loader, stack_waiter, client): 9 | client.describe_stacks.return_value = {'Stacks': [{'StackId': STACK_ID}]} 10 | session.return_value.client.return_value = client 11 | client.describe_stack_events.return_value = {'StackEvents': [{'EventId': EVENT_ID}]} 12 | 13 | cli.main(['rollback', '--stack', STACK, '--profile', PROFILE, '--region', REGION]) 14 | client.describe_stacks.assert_called_with(StackName=STACK) 15 | client.describe_stack_events.assert_called_with(StackName=STACK) 16 | client.rollback_stack.assert_called_with(StackName=STACK) 17 | stack_waiter.assert_called_with(STACK_ID) 18 | stack_waiter.return_value.wait.assert_called_with(EVENT_ID) 19 | 20 | 21 | def test_rollback_stack_with_role(change_set, session, loader, stack_waiter, client): 22 | client.describe_stacks.return_value = {'Stacks': [{'StackId': STACK_ID}]} 23 | client.describe_stack_events.return_value = {'StackEvents': [{'EventId': EVENT_ID}]} 24 | 25 | cli.main(['rollback', '--stack', STACK, '--profile', PROFILE, '--region', REGION, '--role-arn', ROLE_ARN]) 26 | client.describe_stacks.assert_called_with(StackName=STACK) 27 | client.describe_stack_events.assert_called_with(StackName=STACK) 28 | client.rollback_stack.assert_called_with(StackName=STACK, RoleARN=ROLE_ARN) 29 | stack_waiter.assert_called_with(STACK_ID) 30 | stack_waiter.return_value.wait.assert_called_with(EVENT_ID) 31 | -------------------------------------------------------------------------------- /docs/commands/resources.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Resources 3 | weight: 100 4 | --- 5 | 6 | # `formica resources` 7 | 8 | Through the resources command you can list all resources for a deployed stack so its easy for you to get the physical id of a resource you deployed. 9 | 10 | The command will print the logical id, physical id, type and status. 11 | 12 | ## Example 13 | 14 | ``` 15 | root@67c57a89511a:/app/docs/examples/s3-bucket# formica resources --stack formica-example-stack 16 | +------------------+------------------------------------------------------+-----------------+-----------------+ 17 | | Logical ID | Physical ID | Type | Status | 18 | +==================+======================================================+=================+=================+ 19 | | DeploymentBucket | formica-example-stack-deploymentbucket-1tzvltuaftxso | AWS::S3::Bucket | CREATE_COMPLETE | 20 | +------------------+------------------------------------------------------+-----------------+-----------------+ 21 | ``` 22 | 23 | ## Usage 24 | 25 | ``` 26 | usage: formica resources [-h] [--region REGION] [--profile PROFILE] 27 | [--stack STACK] 28 | [--config-file CONFIG_FILE [CONFIG_FILE ...]] 29 | 30 | List all resources of a stack 31 | 32 | options: 33 | -h, --help show this help message and exit 34 | --region REGION The AWS region to use 35 | --profile PROFILE The AWS profile to use 36 | --stack STACK, -s STACK 37 | The Stack to use 38 | --config-file CONFIG_FILE [CONFIG_FILE ...], -c CONFIG_FILE [CONFIG_FILE ...] 39 | Set the config files to use 40 | ``` 41 | -------------------------------------------------------------------------------- /docs/examples/s3-lambda/lambda.template.yaml: -------------------------------------------------------------------------------- 1 | Resources: 2 | LambdaInvokePermission: 3 | Properties: 4 | Action: lambda:InvokeFunction 5 | FunctionName: 6 | Ref: TestFunction 7 | Principal: s3.amazonaws.com 8 | SourceAccount: 9 | Ref: AWS::AccountId 10 | Type: AWS::Lambda::Permission 11 | 12 | LambdaLogGroup: 13 | Type: AWS::Logs::LogGroup 14 | Properties: 15 | LogGroupName: 16 | Fn::Join: 17 | - '' 18 | - - "/aws/lambda/" 19 | - Ref: TestFunction 20 | 21 | TestFunction: 22 | Type: AWS::Lambda::Function 23 | Properties: 24 | Code: 25 | ZipFile: "{{ code('code.py') }}" 26 | Handler: index.handler 27 | Role: 28 | Fn::GetAtt: 29 | - LambdaExecutionRole 30 | - Arn 31 | Runtime: python2.7 32 | 33 | LambdaInvokePermissionS3: 34 | Properties: 35 | Action: lambda:InvokeFunction 36 | FunctionName: 37 | Ref: TestFunctionS3 38 | Principal: s3.amazonaws.com 39 | SourceAccount: 40 | Ref: AWS::AccountId 41 | Type: AWS::Lambda::Permission 42 | 43 | LambdaLogGroupS3: 44 | Type: AWS::Logs::LogGroup 45 | Properties: 46 | LogGroupName: 47 | Fn::Join: 48 | - '' 49 | - - "/aws/lambda/" 50 | - Ref: TestFunctionS3 51 | 52 | TestFunctionS3: 53 | Type: AWS::Lambda::Function 54 | Properties: 55 | Code: 56 | S3Bucket: {{artifacts['build/code.py.zip'].bucket}} 57 | S3Key: {{artifacts['build/code.py.zip'].key}} 58 | Handler: code.handler 59 | Role: 60 | Fn::GetAtt: 61 | - LambdaExecutionRole 62 | - Arn 63 | Runtime: python3.8 64 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Temporary files 2 | *~ 3 | 4 | # Byte-compiled / optimized / DLL files 5 | __pycache__/ 6 | *.py[cod] 7 | *$py.class 8 | 9 | # C extensions 10 | *.so 11 | 12 | # Distribution / packaging 13 | .Python 14 | env/ 15 | build/ 16 | develop-eggs/ 17 | dist/ 18 | downloads/ 19 | eggs/ 20 | .eggs/ 21 | lib/ 22 | lib64/ 23 | parts/ 24 | sdist/ 25 | var/ 26 | *.egg-info/ 27 | .installed.cfg 28 | *.egg 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *,cover 49 | .hypothesis/ 50 | 51 | # Translations 52 | *.mo 53 | *.pot 54 | 55 | # Django stuff: 56 | *.log 57 | local_settings.py 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # IPython Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # dotenv 82 | .env 83 | 84 | # virtualenv 85 | venv/ 86 | ENV/ 87 | 88 | # Spyder project settings 89 | .spyderproject 90 | 91 | # Rope project settings 92 | .ropeproject 93 | 94 | # IDE 95 | .idea 96 | 97 | # Pypi 98 | README.rst 99 | 100 | .pytest_cache 101 | .bash_history 102 | .mutmut-cache 103 | 104 | 105 | code.py.zip -------------------------------------------------------------------------------- /docs/examples/commit-build/post-push-hook.template.yml: -------------------------------------------------------------------------------- 1 | Resources: 2 | LambdaLogGroup: 3 | Type: AWS::Logs::LogGroup 4 | Properties: 5 | LogGroupName: !Sub "/aws/lambda/${PostPushHookLambdaFunction}" 6 | PostPushHookLambdaFunction: 7 | Type: AWS::Lambda::Function 8 | Properties: 9 | Code: 10 | ZipFile: "{{ code('hook.py') }}" 11 | Handler: index.handler 12 | Role: !GetAtt LambdaExecutionRole.Arn 13 | Runtime: python2.7 14 | Environment: 15 | Variables: 16 | CodeBuildProject: !Sub '${AWS::StackName}-build' 17 | LambdaExecutionRole: 18 | Properties: 19 | AssumeRolePolicyDocument: 20 | Statement: 21 | - Action: 22 | - sts:AssumeRole 23 | Effect: Allow 24 | Principal: 25 | Service: 26 | - lambda.amazonaws.com 27 | Version: '2012-10-17' 28 | Path: "/" 29 | Policies: 30 | - PolicyDocument: 31 | Statement: 32 | - Action: 33 | - "logs:CreateLogStream" 34 | - "logs:PutLogEvents" 35 | Effect: Allow 36 | Resource: "arn:aws:logs:*:*:*" 37 | - Action: 38 | - "codebuild:StartBuild" 39 | Effect: Allow 40 | # We can't use a ref here as it would lead to a 41 | # cyclic dependency between CodeBuild, CodeCommit and the Lambda 42 | Resource: !Sub 'arn:aws:codebuild:${AWS::Region}:${AWS::AccountId}:project/${AWS::StackName}-build' 43 | Version: '2012-10-17' 44 | PolicyName: root 45 | Type: AWS::IAM::Role 46 | LambdaInvokePermission: 47 | Properties: 48 | Action: lambda:InvokeFunction 49 | FunctionName: !Ref PostPushHookLambdaFunction 50 | Principal: codecommit.amazonaws.com 51 | SourceAccount: !Ref AWS::AccountId 52 | SourceArn: !GetAtt CodeCommitRepository.Arn 53 | Type: AWS::Lambda::Permission -------------------------------------------------------------------------------- /docs/examples/commit-build-pipeline/build.template.yml: -------------------------------------------------------------------------------- 1 | Parameters: 2 | TimeoutInMinutes: 3 | Type: Number 4 | Default: 10 5 | 6 | 7 | Resources: 8 | CodeBuildProject: 9 | Type: 'AWS::CodeBuild::Project' 10 | Properties: 11 | Artifacts: 12 | Type: CODEPIPELINE 13 | Environment: 14 | ComputeType: 'BUILD_GENERAL1_SMALL' 15 | Image: 'python:3.6' 16 | Type: 'LINUX_CONTAINER' 17 | Name: !Sub '${AWS::StackName}-app' 18 | ServiceRole: !GetAtt 'CodeBuildRole.Arn' 19 | Source: 20 | Type: CODEPIPELINE 21 | BuildSpec: | 22 | version: 0.1 23 | phases: 24 | build: 25 | commands: 26 | - 'ls' 27 | TimeoutInMinutes: !Ref TimeoutInMinutes 28 | CodeBuildRole: 29 | Type: 'AWS::IAM::Role' 30 | Properties: 31 | PermissionsBoundary: !Sub arn:aws:iam::${AWS::AccountId}:policy/CreatedIdentitiesPermissionsBoundary 32 | AssumeRolePolicyDocument: 33 | Version: '2012-10-17' 34 | Statement: 35 | - Effect: Allow 36 | Principal: 37 | Service: 38 | - 'codebuild.amazonaws.com' 39 | Action: 40 | - 'sts:AssumeRole' 41 | Policies: 42 | - PolicyName: ServiceRole 43 | PolicyDocument: 44 | Version: '2012-10-17' 45 | Statement: 46 | - Sid: CloudWatchLogsPolicy 47 | Effect: Allow 48 | Action: 49 | - 'logs:CreateLogGroup' 50 | - 'logs:CreateLogStream' 51 | - 'logs:PutLogEvents' 52 | Resource: '*' 53 | - Sid: CodeCommitPolicy 54 | Effect: Allow 55 | Action: 'codecommit:GitPull' 56 | Resource: '*' 57 | - Sid: S3GetObjectPolicy 58 | Effect: Allow 59 | Action: 60 | - 's3:GetObject' 61 | - 's3:GetObjectVersion' 62 | Resource: '*' 63 | - Sid: S3PutObjectPolicy 64 | Effect: 'Allow' 65 | Action: 's3:PutObject' 66 | Resource: '*' -------------------------------------------------------------------------------- /tests/unit/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | 4 | @pytest.fixture 5 | def botocore_session(mocker): 6 | return mocker.patch('botocore.session.Session') 7 | 8 | 9 | @pytest.fixture 10 | def session(botocore_session): 11 | return botocore_session 12 | 13 | 14 | @pytest.fixture 15 | def boto_client(mocker): 16 | mocker.patch('formica.aws.boto3') 17 | mocker.patch('formica.aws.botocore') 18 | return mocker.patch('boto3.client') 19 | 20 | @pytest.fixture 21 | def boto_resource(mocker): 22 | return mocker.patch('boto3.resource') 23 | 24 | @pytest.fixture 25 | def aws_client(boto_client, mocker): 26 | client_mock = mocker.Mock() 27 | boto_client.return_value = client_mock 28 | return client_mock 29 | 30 | 31 | @pytest.fixture 32 | def client(aws_client): 33 | return aws_client 34 | 35 | 36 | @pytest.fixture 37 | def logger(mocker): 38 | return mocker.patch('formica.cli.logger') 39 | 40 | 41 | @pytest.fixture 42 | def loader(mocker): 43 | return mocker.patch('formica.loader.Loader') 44 | 45 | 46 | @pytest.fixture 47 | def change_set(mocker): 48 | return mocker.patch('formica.change_set.ChangeSet') 49 | 50 | 51 | @pytest.fixture 52 | def stack_waiter(mocker): 53 | return mocker.patch('formica.stack_waiter.StackWaiter') 54 | 55 | @pytest.fixture 56 | def paginators(mocker): 57 | def mock_paginate(**kwargs): 58 | def sideeffect(paginator): 59 | m = mocker.Mock() 60 | m.paginate.return_value = kwargs.get(paginator) 61 | return m 62 | return sideeffect 63 | 64 | return mock_paginate 65 | 66 | 67 | @pytest.fixture 68 | def uuid4(mocker): 69 | return mocker.patch('uuid.uuid4') 70 | 71 | @pytest.fixture 72 | def temp_bucket(mocker): 73 | t = mocker.patch('formica.helper.temporary_bucket') 74 | tempbucket_mock = mocker.Mock() 75 | t.return_value.__enter__.return_value = tempbucket_mock 76 | return tempbucket_mock 77 | 78 | @pytest.fixture 79 | def temp_bucket_cli(mocker): 80 | t = mocker.patch('formica.cli.temporary_bucket') 81 | tempbucket_mock = mocker.Mock() 82 | t.return_value.__enter__.return_value = tempbucket_mock 83 | return tempbucket_mock -------------------------------------------------------------------------------- /formica/yaml_tags.py: -------------------------------------------------------------------------------- 1 | import yaml 2 | from yaml.nodes import SequenceNode, ScalarNode, MappingNode, CollectionNode 3 | from yaml.resolver import BaseResolver 4 | 5 | 6 | class BaseFunction(yaml.YAMLObject): 7 | @classmethod 8 | def tag(self, node): 9 | return node.lstrip("!") 10 | 11 | @classmethod 12 | def fn_tag(self, node): 13 | return "Fn::" + self.tag(node) 14 | 15 | @classmethod 16 | def from_yaml(cls, loader, node): 17 | result = node.value 18 | kind = type(node) 19 | if isinstance(node, CollectionNode): 20 | if kind is SequenceNode: 21 | tag = BaseResolver.DEFAULT_SEQUENCE_TAG 22 | elif kind is MappingNode: 23 | tag = BaseResolver.DEFAULT_MAPPING_TAG 24 | new_node = type(node)(tag, node.value) 25 | result = loader.construct_object(new_node, deep=True) 26 | return {cls.fn_tag(node.tag): result} 27 | 28 | 29 | def create_classes(functions, base_type=BaseFunction): 30 | for function in functions: 31 | type(function, (base_type,), {"yaml_tag": "!" + function}) 32 | 33 | 34 | fn_functions = [ 35 | "Base64", 36 | "And", 37 | "Equals", 38 | "If", 39 | "Not", 40 | "Or", 41 | "FindInMap", 42 | "GetAZs", 43 | "ImportValue", 44 | "Join", 45 | "Select", 46 | "Split", 47 | "Sub", 48 | "Cidr", 49 | ] 50 | 51 | create_classes(fn_functions) 52 | 53 | 54 | class SplitFunction(BaseFunction): 55 | @classmethod 56 | def from_yaml(cls, loader, node): 57 | if type(node) is ScalarNode: 58 | result = node.value.split(".", 1) 59 | else: 60 | result = [loader.construct_object(child, deep=True) for child in node.value] 61 | return {cls.fn_tag(node.tag): result} 62 | 63 | 64 | split_functions = ["GetAtt"] 65 | 66 | create_classes(split_functions, SplitFunction) 67 | 68 | non_fn_functions = ["Ref", "Condition"] 69 | 70 | 71 | class NonFnFunction(BaseFunction): 72 | @classmethod 73 | def from_yaml(cls, loader, node): 74 | return {cls.tag(node.tag): node.value} 75 | 76 | 77 | create_classes(non_fn_functions, NonFnFunction) 78 | -------------------------------------------------------------------------------- /docs/commands/stacks.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Stacks 3 | weight: 100 4 | --- 5 | 6 | # `formica stacks` 7 | 8 | Through the stack command you can get an overview over all the CloudFormation stacks that are in the region you specified. 9 | 10 | ## Example 11 | 12 | ``` 13 | root@07e549506145:/app# formica stacks 14 | Current Stacks: 15 | +-------------------------------+----------------------------------+----------------------------------+--------------------+ 16 | | Name | Created At | Updated At | Status | 17 | +===============================+==================================+==================================+====================+ 18 | | testuniquestack | 2017-02-16 17:10:05.424000+00:00 | | REVIEW_IN_PROGRESS | 19 | +-------------------------------+----------------------------------+----------------------------------+--------------------+ 20 | | formica-example-stack | 2017-02-15 11:18:36.430000+00:00 | 2017-02-15 11:40:19.656000+00:00 | UPDATE_COMPLETE | 21 | +-------------------------------+----------------------------------+----------------------------------+--------------------+ 22 | | formica-integration-test-user | 2017-02-10 13:13:24.004000+00:00 | 2017-02-10 15:54:39.394000+00:00 | UPDATE_COMPLETE | 23 | +-------------------------------+----------------------------------+----------------------------------+--------------------+ 24 | | flomotlikme | 2017-01-13 16:12:18.300000+00:00 | 2017-02-15 23:03:01.626000+00:00 | UPDATE_COMPLETE | 25 | +-------------------------------+----------------------------------+----------------------------------+--------------------+ 26 | ``` 27 | 28 | ## Usage 29 | 30 | ``` 31 | usage: formica stacks [-h] [--region REGION] [--profile PROFILE] 32 | [--config-file CONFIG_FILE [CONFIG_FILE ...]] 33 | 34 | List all stacks 35 | 36 | options: 37 | -h, --help show this help message and exit 38 | --region REGION The AWS region to use 39 | --profile PROFILE The AWS profile to use 40 | --config-file CONFIG_FILE [CONFIG_FILE ...], -c CONFIG_FILE [CONFIG_FILE ...] 41 | Set the config files to use 42 | ``` 43 | -------------------------------------------------------------------------------- /docs/commands/template.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Template 3 | weight: 100 4 | --- 5 | 6 | # `formica template` 7 | 8 | Load the CloudFormation template from the `*.template.(yml|yaml|json)` files in the current folder and print it. 9 | 10 | ## Example 11 | 12 | ``` 13 | root@07e549506145:/app/docs/examples/s3-bucket# formica template 14 | { 15 | "Resources": { 16 | "DeploymentBucket": { 17 | "Type": "AWS::S3::Bucket" 18 | }, 19 | "DeploymentBucket2": { 20 | "Type": "AWS::S3::Bucket" 21 | } 22 | } 23 | } 24 | ``` 25 | 26 | ## Usage 27 | 28 | ``` 29 | usage: formica template [-h] [--config-file CONFIG_FILE [CONFIG_FILE ...]] 30 | [--vars KEY=Value [KEY=Value ...]] [-y] 31 | [--artifacts ARTIFACTS [ARTIFACTS ...]] 32 | [--organization-variables] 33 | [--organization-region-variables] 34 | [--organization-account-variables] [--region REGION] 35 | [--profile PROFILE] 36 | 37 | Print the current template 38 | 39 | options: 40 | -h, --help show this help message and exit 41 | --config-file CONFIG_FILE [CONFIG_FILE ...], -c CONFIG_FILE [CONFIG_FILE ...] 42 | Set the config files to use 43 | --vars KEY=Value [KEY=Value ...] 44 | Add one or multiple Jinja2 variables 45 | -y, --yaml print output as yaml 46 | --artifacts ARTIFACTS [ARTIFACTS ...] 47 | Add one or more artifacts to push to S3 before 48 | deployment 49 | --organization-variables 50 | Add AWSAccounts, AWSSubAccounts, AWSMainAccount and 51 | AWSRegions as Jinja variables with an Email, Id and 52 | Name field for each account 53 | --organization-region-variables 54 | Add AWSRegions as Jinja variables 55 | --organization-account-variables 56 | Add AWSAccounts, AWSSubAccounts, and AWSMainAccount as 57 | Jinja variables with an Email, Id, and Name field for 58 | each account 59 | --region REGION The AWS region to use 60 | --profile PROFILE The AWS profile to use 61 | ``` 62 | -------------------------------------------------------------------------------- /docs/commands/wait.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Wait 3 | weight: 100 4 | --- 5 | 6 | # `formica wait` 7 | 8 | The Wait command allows you to wait for any stack to finish an update or removal. It will start following 9 | the stack from the last event and list all events until the stack has finished the current operation. 10 | 11 | ## Example 12 | 13 | ``` 14 | root@07e549506145:/app/docs/examples/s3-bucket# formica wait --stack formica-examples-stack 15 | +------------------------------+--------------------------+--------------------------------+--------------------------------+----------------------------------------------------+ 16 | | Timestamp | Status | Type | Logical ID | Status reason | 17 | +------------------------------+--------------------------+--------------------------------+--------------------------------+----------------------------------------------------+ 18 | 2017-02-16 19:14:59 UTC+0000 DELETE_IN_PROGRESS AWS::CloudFormation::Stack formica-examples-stack User Initiated 19 | 2017-02-16 19:15:30 UTC+0000 DELETE_IN_PROGRESS AWS::S3::Bucket DeploymentBucket 20 | 2017-02-16 19:15:30 UTC+0000 DELETE_IN_PROGRESS AWS::S3::Bucket DeploymentBucket2 21 | 2017-02-16 19:15:51 UTC+0000 DELETE_COMPLETE AWS::S3::Bucket DeploymentBucket2 22 | 2017-02-16 19:15:51 UTC+0000 DELETE_COMPLETE AWS::S3::Bucket DeploymentBucket 23 | 2017-02-16 19:15:52 UTC+0000 DELETE_COMPLETE AWS::CloudFormation::Stack formica-examples-stack 24 | ``` 25 | 26 | ## Usage 27 | 28 | ``` 29 | usage: formica wait [-h] [--region REGION] [--profile PROFILE] [--stack STACK] 30 | [--config-file CONFIG_FILE [CONFIG_FILE ...]] 31 | 32 | Wait for a Stack to be deployed or removed 33 | 34 | options: 35 | -h, --help show this help message and exit 36 | --region REGION The AWS region to use 37 | --profile PROFILE The AWS profile to use 38 | --stack STACK, -s STACK 39 | The Stack to use 40 | --config-file CONFIG_FILE [CONFIG_FILE ...], -c CONFIG_FILE [CONFIG_FILE ...] 41 | Set the config files to use 42 | ``` 43 | -------------------------------------------------------------------------------- /docs/commands/cancel.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Cancel 3 | weight: 100 4 | --- 5 | 6 | # `formica cancel` 7 | 8 | Through the cancel command you can cancel a stack update. 9 | 10 | After the stack update was started you can cancel it. Formica will wait for the stack to clean up. 11 | If you are creating a stack or all update operations have run and your stack is currently in cleanup mode 12 | the cancel operation will fail. 13 | 14 | ## Example 15 | 16 | ``` 17 | root@07e549506145:/app/docs/examples/s3-bucket# formica cancel --stack formica-examples-stack 18 | +------------------------------+--------------------------+--------------------------------+--------------------------------+----------------------------------------------------+ 19 | | Timestamp | Status | Type | Logical ID | Status reason | 20 | +------------------------------+--------------------------+--------------------------------+--------------------------------+----------------------------------------------------+ 21 | 2017-02-16 19:14:59 UTC+0000 DELETE_IN_PROGRESS AWS::CloudFormation::Stack formica-examples-stack User Initiated 22 | 2017-02-16 19:15:30 UTC+0000 DELETE_IN_PROGRESS AWS::S3::Bucket DeploymentBucket 23 | 2017-02-16 19:15:30 UTC+0000 DELETE_IN_PROGRESS AWS::S3::Bucket DeploymentBucket2 24 | 2017-02-16 19:15:51 UTC+0000 DELETE_COMPLETE AWS::S3::Bucket DeploymentBucket2 25 | 2017-02-16 19:15:51 UTC+0000 DELETE_COMPLETE AWS::S3::Bucket DeploymentBucket 26 | 2017-02-16 19:15:52 UTC+0000 DELETE_COMPLETE AWS::CloudFormation::Stack formica-examples-stack 27 | ``` 28 | 29 | ## Usage 30 | 31 | ``` 32 | usage: formica cancel [-h] [--region REGION] [--profile PROFILE] 33 | [--stack STACK] 34 | [--config-file CONFIG_FILE [CONFIG_FILE ...]] 35 | 36 | Cancel a Stack Update 37 | 38 | options: 39 | -h, --help show this help message and exit 40 | --region REGION The AWS region to use 41 | --profile PROFILE The AWS profile to use 42 | --stack STACK, -s STACK 43 | The Stack to use 44 | --config-file CONFIG_FILE [CONFIG_FILE ...], -c CONFIG_FILE [CONFIG_FILE ...] 45 | Set the config files to use 46 | ``` 47 | -------------------------------------------------------------------------------- /docs/artifacts.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Artifacts 3 | weight: 700 4 | --- 5 | 6 | Sometimes you might want to upload a zipfile for a Lambda function or scripts to use with EC2 Auto Scaling groups to S3 or anything else where CloudFormation or Resources you're creating should reference artifacts in S3. To set this up you can use the `--artifacts` argument or config file option. 7 | 8 | Artifacts takes a list of file names, reads them and hashes them. These hashes are then used to create a temporary S3 Bucket which will be created at deployment. When creating a changeset, diffing your template or just taking a look at the current template, the same calculations are performed and the Bucket as well as S3 Object name will be set in Jinja variables so you can access them in your template.. 9 | 10 | The bucket will always start with `formica-deploy-` followed by the bucket hash so you can set up IAM Roles for deployment to have access to resources starting with this prefix. 11 | 12 | Because the files are hashed when the files don't change neither do your templates and CloudFormation will not redeploy any resources. 13 | 14 | A simple example would be the following artifact for a Lambda function. The example can also be seen in the [S3-Lambda Example](https://github.com/theserverlessway/formica/tree/master/docs/examples/s3-lambda) At first we're zipping `code.py` into `build/code.py.zip`.: 15 | 16 | ``` 17 | mkdir -p build 18 | zip build/code.py.zip code.py 19 | ``` 20 | 21 | Now we add the artifacts to the configuration file: 22 | 23 | ``` 24 | stack: s3-lambda 25 | capabilities: 26 | - CAPABILITY_IAM 27 | artifacts: 28 | - build/code.py.zip 29 | ``` 30 | 31 | In our lambda function config we can then use the artifact file name to get to the bucket and key: 32 | 33 | ``` 34 | TestFunctionS3: 35 | Type: AWS::Lambda::Function 36 | Properties: 37 | Code: 38 | S3Bucket: {{artifacts['build/code.py.zip'].bucket}} 39 | S3Key: {{artifacts['build/code.py.zip'].key}} 40 | Handler: code.handler 41 | Role: 42 | Fn::GetAtt: 43 | - LambdaExecutionRole 44 | - Arn 45 | Runtime: python3.8 46 | ``` 47 | 48 | The full path is used for referencing the specific file. Based on the file content we'll create hashes and all the hashes of the files will be used to create a hash used for the bucket name. In addition the AccountID and Region will be used for the bucket hash to make sure its unique and scoped to the account and region. 49 | 50 | After deployment the bucket and all objects in it will be removed. -------------------------------------------------------------------------------- /tests/unit/test_yaml_tags.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from path import Path 3 | 4 | from formica.loader import Loader 5 | 6 | 7 | @pytest.fixture() 8 | def loader(): 9 | return Loader() 10 | 11 | 12 | @pytest.fixture() 13 | def runner(loader, tmpdir): 14 | def validate(yaml, expect): 15 | with Path(tmpdir): 16 | with open('test.template.yaml', 'w') as f: 17 | f.write(yaml) 18 | loader.load() 19 | assert loader.template_dictionary() == expect 20 | 21 | return validate 22 | 23 | 24 | @pytest.mark.parametrize('input,expected', [ 25 | ('Resources: !Base64 something', {'Resources': {"Fn::Base64": 'something'}}), 26 | ('Resources: !And [ !Equals ["a", "b"], !Equals ["c", "d"], ] ', 27 | {'Resources': {"Fn::And": [{"Fn::Equals": ['a', 'b']}, {"Fn::Equals": ['c', 'd']}]}}), 28 | ('Resources: !If [A, !Ref B, !Ref C]', {'Resources': {"Fn::If": ['A', {"Ref": "B"}, {"Ref": "C"}]}}), 29 | ('Resources: !Not [!Equals [!Ref A, prod]]', {'Resources': {"Fn::Not": [{'Fn::Equals': [{"Ref": "A"}, "prod"]}]}}), 30 | ('Resources: !Or [ !Equals ["a", "b"], !Equals ["c", "d"], ] ', 31 | {'Resources': {"Fn::Or": [{"Fn::Equals": ['a', 'b']}, {"Fn::Equals": ['c', 'd']}]}}), 32 | ('Resources: !FindInMap [ RegionMap, !Ref "A", 32 ]', 33 | {'Resources': {"Fn::FindInMap": ['RegionMap', {"Ref": "A"}, 32]}}), 34 | ('Resources: !GetAtt A.B', {'Resources': {"Fn::GetAtt": ['A', 'B']}}), 35 | ('Resources: !GetAtt A.B.C', {'Resources': {"Fn::GetAtt": ['A', 'B.C']}}), 36 | ('Resources: !GetAZs us-east-1', {'Resources': {"Fn::GetAZs": 'us-east-1'}}), 37 | ('Resources: !ImportValue ABC', {'Resources': {"Fn::ImportValue": 'ABC'}}), 38 | ('Resources: !Join ["", "A", "B"]', {'Resources': {"Fn::Join": ['', 'A', 'B']}}), 39 | ('Resources: !Select [1, ["A", "B"]]', {'Resources': {"Fn::Select": [1, ['A', 'B']]}}), 40 | ('Resources: !Split ["." , "A.B" ]', {'Resources': {"Fn::Split": ['.', 'A.B']}}), 41 | ('Resources: !Sub ["${String}", {A: B} ]', {'Resources': {"Fn::Sub": ['${String}', {"A": "B"}]}}), 42 | ('Resources: !Ref ABC', {'Resources': {"Ref": 'ABC'}}), 43 | ('Resources: !GetAZs\n Ref: AWS::Region', {'Resources': {'Fn::GetAZs': {"Ref": 'AWS::Region'}}}), 44 | ('Resources: !GetAtt ["abc", "def"]', {'Resources': {'Fn::GetAtt': ['abc', 'def']}}), 45 | ('Resources: !Condition TestCondition', {'Resources': {'Condition': 'TestCondition'}}), 46 | ('Resources: !Cidr [ "A", "B", "C" ]', {'Resources': {'Fn::Cidr': ["A", "B", "C"]}}), 47 | 48 | ]) 49 | def test_yaml_tag(runner, input, expected): 50 | runner(input, expected) 51 | -------------------------------------------------------------------------------- /docs/commands/remove.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Remove 3 | weight: 100 4 | --- 5 | 6 | # `formica remove` 7 | 8 | Through the remove command you can remove an existing stack. 9 | 10 | After starting to remove the stack it will follow the stack events until the removal is finshed. In case the removal failed it will exit with a non-zero exit status. 11 | 12 | ## Example 13 | 14 | ``` 15 | root@07e549506145:/app/docs/examples/s3-bucket# formica remove --stack formica-examples-stack 16 | Removing Stack and waiting for it to be removed, ... 17 | +------------------------------+--------------------------+--------------------------------+--------------------------------+----------------------------------------------------+ 18 | | Timestamp | Status | Type | Logical ID | Status reason | 19 | +------------------------------+--------------------------+--------------------------------+--------------------------------+----------------------------------------------------+ 20 | 2017-02-16 19:14:59 UTC+0000 DELETE_IN_PROGRESS AWS::CloudFormation::Stack formica-examples-stack User Initiated 21 | 2017-02-16 19:15:30 UTC+0000 DELETE_IN_PROGRESS AWS::S3::Bucket DeploymentBucket 22 | 2017-02-16 19:15:30 UTC+0000 DELETE_IN_PROGRESS AWS::S3::Bucket DeploymentBucket2 23 | 2017-02-16 19:15:51 UTC+0000 DELETE_COMPLETE AWS::S3::Bucket DeploymentBucket2 24 | 2017-02-16 19:15:51 UTC+0000 DELETE_COMPLETE AWS::S3::Bucket DeploymentBucket 25 | 2017-02-16 19:15:52 UTC+0000 DELETE_COMPLETE AWS::CloudFormation::Stack formica-examples-stack 26 | ``` 27 | 28 | ## Usage 29 | 30 | ``` 31 | usage: formica remove [-h] [--region REGION] [--profile PROFILE] 32 | [--stack STACK] [--role-arn ROLE_ARN] 33 | [--role-name ROLE_NAME] 34 | [--config-file CONFIG_FILE [CONFIG_FILE ...]] 35 | 36 | Remove the configured stack 37 | 38 | options: 39 | -h, --help show this help message and exit 40 | --region REGION The AWS region to use 41 | --profile PROFILE The AWS profile to use 42 | --stack STACK, -s STACK 43 | The Stack to use 44 | --role-arn ROLE_ARN Set a separate role ARN to pass to the stack 45 | --role-name ROLE_NAME 46 | Set a role name that will be translated to the ARN 47 | --config-file CONFIG_FILE [CONFIG_FILE ...], -c CONFIG_FILE [CONFIG_FILE ...] 48 | Set the config files to use 49 | ``` 50 | -------------------------------------------------------------------------------- /formica/helper.py: -------------------------------------------------------------------------------- 1 | from .s3 import temporary_bucket 2 | 3 | 4 | def name(*names): 5 | name = "".join(map(lambda name: name.title(), names)) 6 | name = "".join(e for e in name if e.isalnum()) 7 | return name 8 | 9 | 10 | def collect_stack_set_vars(args): 11 | variables = args.vars or {} 12 | if args.organization_variables: 13 | variables.update(accounts_and_regions()) 14 | else: 15 | if args.organization_region_variables: 16 | variables.update(aws_regions()) 17 | if args.organization_account_variables: 18 | variables.update(aws_accounts()) 19 | 20 | return variables 21 | 22 | 23 | def collect_vars(args): 24 | variables = collect_stack_set_vars(args) 25 | if args.artifacts: 26 | variables.update(artifact_variables(args.artifacts, vars(args).get("stack", ""))) 27 | 28 | return variables 29 | 30 | 31 | def accounts_and_regions(): 32 | acc = aws_accounts() 33 | acc.update(aws_regions()) 34 | return acc 35 | 36 | 37 | def aws_regions(): 38 | import boto3 39 | 40 | ec2 = boto3.client("ec2") 41 | regions = ec2.describe_regions() 42 | regions = [r["RegionName"] for r in regions["Regions"]] 43 | return {"AWSRegions": regions} 44 | 45 | 46 | def aws_accounts(): 47 | import boto3 48 | 49 | organizations = boto3.client("organizations") 50 | sts = boto3.client("sts") 51 | 52 | paginator = organizations.get_paginator("list_accounts") 53 | 54 | pages = paginator.paginate() 55 | accounts = [ 56 | {"Id": a["Id"], "Name": a["Name"], "Email": a["Email"]} 57 | for page in pages 58 | for a in page["Accounts"] 59 | if a["Status"] == "ACTIVE" 60 | ] 61 | account_id = sts.get_caller_identity()["Account"] 62 | return { 63 | "AWSMainAccount": [a for a in accounts if a["Id"] == account_id][0], 64 | "AWSAccounts": accounts, 65 | "AWSSubAccounts": [a for a in accounts if a["Id"] != account_id], 66 | } 67 | 68 | 69 | def main_account_id(): 70 | import boto3 71 | 72 | sts = boto3.client("sts") 73 | identity = sts.get_caller_identity() 74 | return identity["Account"] 75 | 76 | 77 | def artifact_variables(artifacts, seed): 78 | class Artifact: 79 | def __init__(self, key, bucket): 80 | self.key = key 81 | self.bucket = bucket 82 | 83 | artifact_keys = {} 84 | with temporary_bucket(seed=seed) as t: 85 | for a in artifacts: 86 | artifact_keys[a] = t.add_file(a) 87 | finished_vars = {key: Artifact(value, t.name) for key, value in artifact_keys.items()} 88 | return {"artifacts": finished_vars} 89 | 90 | 91 | def with_artifacts(function): 92 | def handle_artifacts(args): 93 | if args.artifacts: 94 | seed = vars(args).get("stack", "") 95 | with temporary_bucket(seed=seed) as t: 96 | for a in args.artifacts: 97 | t.add_file(a) 98 | t.upload() 99 | function(args) 100 | else: 101 | function(args) 102 | 103 | return handle_artifacts 104 | -------------------------------------------------------------------------------- /docs/commands/deploy.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Deploy 3 | weight: 100 4 | --- 5 | 6 | # `formica deploy` 7 | 8 | Through the deploy command you can execute a previously created ChangeSet. This works for both the [`formica new`]({{< relref "new.md" >}}) and [`formica change`]({{< relref "change.md" >}}) ChangeSets. 9 | 10 | After starting the update to the stack it will follow the stack events until the deployment is finshed. In case the deployment failed it will exit with a non-zero exit status. 11 | 12 | ## Example 13 | 14 | ``` 15 | root@07e549506145:/app/docs/examples/s3-bucket# formica deploy --stack formica-examples-stack 16 | +------------------------------+--------------------------+--------------------------------+--------------------------------+----------------------------------------------------+ 17 | | Timestamp | Status | Type | Logical ID | Status reason | 18 | +------------------------------+--------------------------+--------------------------------+--------------------------------+----------------------------------------------------+ 19 | 2017-02-16 19:12:51 UTC+0000 CREATE_IN_PROGRESS AWS::CloudFormation::Stack formica-examples-stack User Initiated 20 | 2017-02-16 19:13:01 UTC+0000 CREATE_IN_PROGRESS AWS::S3::Bucket DeploymentBucket 21 | 2017-02-16 19:13:01 UTC+0000 CREATE_IN_PROGRESS AWS::S3::Bucket DeploymentBucket2 22 | 2017-02-16 19:13:02 UTC+0000 CREATE_IN_PROGRESS AWS::S3::Bucket DeploymentBucket Resource creation Initiated 23 | 2017-02-16 19:13:02 UTC+0000 CREATE_IN_PROGRESS AWS::S3::Bucket DeploymentBucket2 Resource creation Initiated 24 | 2017-02-16 19:13:23 UTC+0000 CREATE_COMPLETE AWS::S3::Bucket DeploymentBucket 25 | 2017-02-16 19:13:23 UTC+0000 CREATE_COMPLETE AWS::S3::Bucket DeploymentBucket2 26 | 2017-02-16 19:13:25 UTC+0000 CREATE_COMPLETE AWS::CloudFormation::Stack formica-examples-stack 27 | ``` 28 | 29 | ## Usage 30 | 31 | ``` 32 | usage: formica deploy [-h] [--region REGION] [--profile PROFILE] 33 | [--artifacts ARTIFACTS [ARTIFACTS ...]] [--stack STACK] 34 | [--config-file CONFIG_FILE [CONFIG_FILE ...]] 35 | [--timeout TIMEOUT] [--disable-rollback] 36 | 37 | Deploy the latest change set for a stack 38 | 39 | options: 40 | -h, --help show this help message and exit 41 | --region REGION The AWS region to use 42 | --profile PROFILE The AWS profile to use 43 | --artifacts ARTIFACTS [ARTIFACTS ...] 44 | Add one or more artifacts to push to S3 before 45 | deployment 46 | --stack STACK, -s STACK 47 | The Stack to use 48 | --config-file CONFIG_FILE [CONFIG_FILE ...], -c CONFIG_FILE [CONFIG_FILE ...] 49 | Set the config files to use 50 | --timeout TIMEOUT Set the Timeout in minutes before the Update is 51 | canceled 52 | --disable-rollback Do not roll back in case of a failed deployment 53 | ``` 54 | -------------------------------------------------------------------------------- /tests/unit/test_cli.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from formica import cli, __version__ 3 | from botocore.exceptions import ProfileNotFound, NoCredentialsError, NoRegionError, ClientError 4 | 5 | from tests.unit.constants import STACK, MESSAGE 6 | 7 | METHODS = ['change', 'deploy', 'new', 'remove', 'resources'] 8 | NO_STACK_METHODS = ['stacks'] 9 | 10 | Exceptions = [ProfileNotFound, NoCredentialsError, NoRegionError, ClientError] 11 | 12 | 13 | @pytest.fixture 14 | def logger(mocker): 15 | return mocker.patch('formica.cli.logger') 16 | 17 | def test_fails_for_no_arguments(capsys): 18 | with pytest.raises(SystemExit): 19 | cli.main([]) 20 | out, err = capsys.readouterr() 21 | assert 'usage:' in err 22 | 23 | 24 | def test_prints_version(capsys): 25 | with pytest.raises(SystemExit): 26 | cli.main(['--version']) 27 | out = "\n".join(capsys.readouterr()) 28 | assert __version__.strip() in out 29 | 30 | 31 | def test_commands_use_exception_handling(session, logger): 32 | session.side_effect = NoCredentialsError() 33 | for method in METHODS: 34 | with pytest.raises(SystemExit) as pytest_wrapped_e: 35 | cli.main([method, '--stack', STACK]) 36 | assert pytest_wrapped_e.value.code == 1 37 | 38 | for method in NO_STACK_METHODS: 39 | with pytest.raises(SystemExit) as pytest_wrapped_e: 40 | cli.main([method]) 41 | assert pytest_wrapped_e.value.code == 1 42 | 43 | 44 | def test_catches_client_errors(session, logger): 45 | session.side_effect = ClientError({'Error': {'Code': 'ValidationError', 'Message': MESSAGE}}, 'ERROR_TEST') 46 | for method in METHODS: 47 | with pytest.raises(SystemExit) as pytest_wrapped_e: 48 | cli.main([method, '--stack', STACK]) 49 | logger.info.assert_called_with(MESSAGE) 50 | assert pytest_wrapped_e.value.code == 1 51 | 52 | for method in NO_STACK_METHODS: 53 | with pytest.raises(SystemExit) as pytest_wrapped_e: 54 | cli.main([method]) 55 | assert pytest_wrapped_e.value.code == 1 56 | 57 | 58 | def test_catches_arbitrary_client_error(session, logger): 59 | error = ClientError({'Error': {'Code': 'SOMEOTHER', 'Message': MESSAGE}}, 'ERROR_TEST') 60 | session.side_effect = error 61 | for method in METHODS: 62 | with pytest.raises(SystemExit) as pytest_wrapped_e: 63 | cli.main([method, '--stack', STACK]) 64 | assert pytest_wrapped_e.value.code == 2 65 | logger.info.assert_called_with(error) 66 | 67 | for method in NO_STACK_METHODS: 68 | with pytest.raises(SystemExit) as pytest_wrapped_e: 69 | cli.main([method]) 70 | assert pytest_wrapped_e.value.code == 2 71 | 72 | 73 | def test_fails_with_wrong_parameter_format(capsys): 74 | with pytest.raises(SystemExit) as pytest_wrapped_e: 75 | cli.main(['new', '--stack', STACK, '--parameters', 'Test:Test']) 76 | out, err = capsys.readouterr() 77 | assert "argument --parameters: Test:Test needs to be in format KEY=VALUE" in err 78 | assert pytest_wrapped_e.value.code == 2 79 | 80 | 81 | def test_can_provide_profile_for_template_comand(capsys): 82 | with pytest.raises(SystemExit) as pytest_wrapped_e: 83 | cli.main(['template', '--profile', 'testAwsCliProfile']) 84 | out, err = capsys.readouterr() 85 | assert "" == out 86 | 87 | 88 | -------------------------------------------------------------------------------- /docs/commands/stack-set_create.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: StackSet Create 3 | weight: 100 4 | --- 5 | 6 | # `formica stack-set create` 7 | 8 | The `formica stack-set create` command allows you to create a new StackSet in your current AWS account. 9 | In a later step you can add/remove instances to the StackSet or update it with a new template. 10 | 11 | ## Usage 12 | 13 | ``` 14 | usage: formica stack-set create [-h] [--region REGION] [--profile PROFILE] 15 | [--stack-set STACK-Set] 16 | [--parameters KEY=Value [KEY=Value ...]] 17 | [--main-account-parameter] 18 | [--tags KEY=Value [KEY=Value ...]] 19 | [--capabilities Cap1 Cap2 [Cap1 Cap2 ...]] 20 | [--config-file CONFIG_FILE [CONFIG_FILE ...]] 21 | [--vars KEY=Value [KEY=Value ...]] 22 | [--administration-role-arn ADMINISTRATION_ROLE_ARN] 23 | [--administration-role-name ADMINISTRATION_ROLE_NAME] 24 | [--execution-role-name EXECUTION_ROLE_NAME] 25 | [--organization-variables] 26 | [--organization-region-variables] 27 | [--organization-account-variables] 28 | 29 | Create a Stack Set 30 | 31 | options: 32 | -h, --help show this help message and exit 33 | --region REGION The AWS region to use 34 | --profile PROFILE The AWS profile to use 35 | --stack-set STACK-Set, -s STACK-Set 36 | The Stack Set to use 37 | --parameters KEY=Value [KEY=Value ...] 38 | Add one or multiple stack parameters 39 | --main-account-parameter 40 | Set MainAccount Parameter 41 | --tags KEY=Value [KEY=Value ...] 42 | Add one or multiple stack tags 43 | --capabilities Cap1 Cap2 [Cap1 Cap2 ...] 44 | Set one or multiple stack capabilities 45 | --config-file CONFIG_FILE [CONFIG_FILE ...], -c CONFIG_FILE [CONFIG_FILE ...] 46 | Set the config files to use 47 | --vars KEY=Value [KEY=Value ...] 48 | Add one or multiple Jinja2 variables 49 | --administration-role-arn ADMINISTRATION_ROLE_ARN 50 | The Administration Role to create the StackSet 51 | --administration-role-name ADMINISTRATION_ROLE_NAME 52 | The Administration Role name that will be translated 53 | to the ARN 54 | --execution-role-name EXECUTION_ROLE_NAME 55 | The Execution role name to use for the CloudFormation 56 | Stack 57 | --organization-variables 58 | Add AWSAccounts, AWSSubAccounts, AWSMainAccount and 59 | AWSRegions as Jinja variables with an Email, Id and 60 | Name field for each account 61 | --organization-region-variables 62 | Add AWSRegions as Jinja variables 63 | --organization-account-variables 64 | Add AWSAccounts, AWSSubAccounts, and AWSMainAccount as 65 | Jinja variables with an Email, Id, and Name field for 66 | each account 67 | ``` 68 | -------------------------------------------------------------------------------- /docs/commands/stack-set_add-instances.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: StackSet Add Instances 3 | weight: 100 4 | --- 5 | 6 | # `formica stack-set add-instances` 7 | 8 | The `formica stack-set add-instances` command allows you to add instances to your StackSet in 9 | various accounts and regions. It will deploy the StackSet in every region of every account 10 | you're adding. 11 | 12 | ## Usage 13 | 14 | ``` 15 | usage: formica stack-set add-instances [-h] [--region REGION] 16 | [--profile PROFILE] 17 | [--stack-set STACK-Set] 18 | [--accounts ACCOUNTS [ACCOUNTS ...]] 19 | [--regions REGIONS [REGIONS ...]] 20 | [--config-file CONFIG_FILE [CONFIG_FILE ...]] 21 | [--all-accounts] [--all-subaccounts] 22 | [--excluded-accounts EXCLUDED_ACCOUNTS [EXCLUDED_ACCOUNTS ...]] 23 | [--all-regions] 24 | [--excluded-regions EXCLUDED_REGIONS [EXCLUDED_REGIONS ...]] 25 | [--main-account] 26 | [--region-order REGION_ORDER [REGION_ORDER ...]] 27 | [--failure-tolerance-count FAILURE_TOLERANCE_COUNT | --failure-tolerance-percentage FAILURE_TOLERANCE_PERCENTAGE] 28 | [--max-concurrent-count MAX_CONCURRENT_COUNT | --max-concurrent-percentage MAX_CONCURRENT_PERCENTAGE] 29 | [--yes] 30 | 31 | Add Stack Set Instances 32 | 33 | options: 34 | -h, --help show this help message and exit 35 | --region REGION The AWS region to use 36 | --profile PROFILE The AWS profile to use 37 | --stack-set STACK-Set, -s STACK-Set 38 | The Stack Set to use 39 | --accounts ACCOUNTS [ACCOUNTS ...] 40 | The Accounts for this operation 41 | --regions REGIONS [REGIONS ...] 42 | The Regions for this operation 43 | --config-file CONFIG_FILE [CONFIG_FILE ...], -c CONFIG_FILE [CONFIG_FILE ...] 44 | Set the config files to use 45 | --all-accounts Use All Accounts of this Org 46 | --all-subaccounts Use Only Subaccounts of this Org 47 | --excluded-accounts EXCLUDED_ACCOUNTS [EXCLUDED_ACCOUNTS ...] 48 | All Accounts excluding these 49 | --all-regions Use all Regions 50 | --excluded-regions EXCLUDED_REGIONS [EXCLUDED_REGIONS ...] 51 | Excluded Regions from deployment 52 | --main-account Deploy to Main Account only 53 | --region-order REGION_ORDER [REGION_ORDER ...] 54 | Order in which to deploy to regions 55 | --failure-tolerance-count FAILURE_TOLERANCE_COUNT 56 | Number of Stacks to fail before failing operation 57 | --failure-tolerance-percentage FAILURE_TOLERANCE_PERCENTAGE 58 | Percentage of Stacks to fail before failing operation 59 | --max-concurrent-count MAX_CONCURRENT_COUNT 60 | Max Number of concurrent accounts to deploy to 61 | --max-concurrent-percentage MAX_CONCURRENT_PERCENTAGE 62 | Max Percentage of concurrent accounts to deploy to 63 | --yes, -y Answer all input questions with yes 64 | ``` 65 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to making participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, gender identity and expression, level of experience, 9 | nationality, personal appearance, race, religion, or sexual identity and 10 | orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | * Using welcoming and inclusive language 18 | * Being respectful of differing viewpoints and experiences 19 | * Gracefully accepting constructive criticism 20 | * Focusing on what is best for the community 21 | * Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | * The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | * Trolling, insulting/derogatory comments, and personal or political attacks 28 | * Public or private harassment 29 | * Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | * Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ## Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or 41 | reject comments, commits, code, wiki edits, issues, and other contributions 42 | that are not aligned to this Code of Conduct, or to ban temporarily or 43 | permanently any contributor for other behaviors that they deem inappropriate, 44 | threatening, offensive, or harmful. 45 | 46 | ## Scope 47 | 48 | This Code of Conduct applies both within project spaces and in public spaces 49 | when an individual is representing the project or its community. Examples of 50 | representing a project or community include using an official project e-mail 51 | address, posting via an official social media account, or acting as an appointed 52 | representative at an online or offline event. Representation of a project may be 53 | further defined and clarified by project maintainers. 54 | 55 | ## Enforcement 56 | 57 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 58 | reported by contacting the project team at [INSERT EMAIL ADDRESS]. All 59 | complaints will be reviewed and investigated and will result in a response that 60 | is deemed necessary and appropriate to the circumstances. The project team is 61 | obligated to maintain confidentiality with regard to the reporter of an incident. 62 | Further details of specific enforcement policies may be posted separately. 63 | 64 | Project maintainers who do not follow or enforce the Code of Conduct in good 65 | faith may face temporary or permanent repercussions as determined by other 66 | members of the project's leadership. 67 | 68 | ## Attribution 69 | 70 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 71 | available at [http://contributor-covenant.org/version/1/4][version] 72 | 73 | [homepage]: http://contributor-covenant.org 74 | [version]: http://contributor-covenant.org/version/1/4/ -------------------------------------------------------------------------------- /docs/commands/describe.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Describe 3 | weight: 100 4 | --- 5 | 6 | # `formica describe` 7 | 8 | Through the describe command you can see the changes that would be performed with the current ChangeSet. The output 9 | created by the describe command is also printed when running [`formica new`]({{< relref "new.md" >}}) or [`formica change`]({{< relref "change.md" >}}). 10 | 11 | ## Example 12 | 13 | Before printing the actual changes the output constists of all parameters, tags and capabilities given to either 14 | [`formica new`]({{< relref "new.md" >}}) or [`formica change`]({{< relref "change.md" >}}) when creating the ChangeSet. 15 | 16 | After that a table with the following action will be printed: 17 | 18 | * **Action**: Action on that resource 19 | * **LogicalId**: Id in the CloudFormationTemplate 20 | * **PhysicalId**: Id of the actual AWS resource (if available), e.g. S3 Bucket name 21 | * **Type**: Type of the Resource 22 | * **Replacement**: True if Resource will be replaced due to action performed against it, empty otherwise 23 | * **Changed**: Any attribute of that resource that will be changed when the ChangeSet gets deployed 24 | 25 | 26 | ``` 27 | root@07e549506145:/app/docs/examples/s3-bucket# formica change --stack formica-example-stack 28 | Removing existing change set 29 | Change set submitted, waiting for CloudFormation to calculate changes ... 30 | Change set created successfully 31 | Deployment metadata: 32 | +---------------+--+ 33 | | Parameters | | 34 | +---------------+--+ 35 | | Tags | | 36 | +---------------+--+ 37 | | Capabilities | | 38 | +---------------+--+ 39 | 40 | Resource Changes: 41 | +--------+-------------------+-------------------------------------------------------+-----------------+-------------+------------+ 42 | | Action | LogicalId | PhysicalId | Type | Replacement | Changed | 43 | +========+===================+=======================================================+=================+=============+============+ 44 | | Remove | DeploymentBucket | formica-example-stack-deploymentbucket-57ouvt2o46yh | AWS::S3::Bucket | | | 45 | +--------+-------------------+-------------------------------------------------------+-----------------+-------------+------------+ 46 | | Modify | DeploymentBucket2 | formica-example-stack-deploymentbucket2-1ov3l9pces9mu | AWS::S3::Bucket | True | BucketName | 47 | +--------+-------------------+-------------------------------------------------------+-----------------+-------------+------------+ 48 | | Add | DeploymentBucket3 | | AWS::S3::Bucket | | | 49 | +--------+-------------------+-------------------------------------------------------+-----------------+-------------+------------+ 50 | ``` 51 | 52 | ## Usage 53 | 54 | ``` 55 | usage: formica describe [-h] [--region REGION] [--profile PROFILE] 56 | [--stack STACK] 57 | [--config-file CONFIG_FILE [CONFIG_FILE ...]] 58 | 59 | Describe the latest change-set of the stack 60 | 61 | options: 62 | -h, --help show this help message and exit 63 | --region REGION The AWS region to use 64 | --profile PROFILE The AWS profile to use 65 | --stack STACK, -s STACK 66 | The Stack to use 67 | --config-file CONFIG_FILE [CONFIG_FILE ...], -c CONFIG_FILE [CONFIG_FILE ...] 68 | Set the config files to use 69 | ``` 70 | -------------------------------------------------------------------------------- /tests/unit/test_s3.py: -------------------------------------------------------------------------------- 1 | from formica.s3 import temporary_bucket 2 | import pytest 3 | from .constants import STACK 4 | 5 | 6 | @pytest.fixture 7 | def bucket(mocker, boto_resource, boto_client): 8 | return boto_resource.return_value.Bucket 9 | 10 | 11 | STRING_BODY = "string" 12 | # MD5 hash of body 13 | STRING_KEY = "b45cffe084dd3d20d928bee85e7b0f21" 14 | BINARY_BODY = "binary".encode() 15 | BINARY_KEY = "9d7183f16acce70658f686ae7f1a4d20" 16 | BUCKET_NAME = "formica-deploy-88dec80484e3155b2c8cf023b635fb31" 17 | FILE_NAME = "testfile" 18 | FILE_BODY = "file-body" 19 | FILE_KEY = "de858a1b070b29a579e2d8861b53ad20" 20 | 21 | 22 | def test_s3_bucket_context(mocker, bucket, uuid4, boto_client, boto_resource): 23 | bucket.return_value.objects.all.return_value = [mocker.Mock(key=STRING_KEY), mocker.Mock(key=BINARY_KEY)] 24 | boto_client.return_value.meta.region_name = "eu-central-1" 25 | boto_client.return_value.get_caller_identity.return_value = {'Account': '1234'} 26 | mock_open = mocker.mock_open(read_data=FILE_BODY.encode()) 27 | mocker.patch('formica.s3.open', mock_open) 28 | 29 | with temporary_bucket(seed=STACK) as temp_bucket: 30 | string_return = temp_bucket.add(STRING_BODY) 31 | binary_return = temp_bucket.add(BINARY_BODY) 32 | file_return = temp_bucket.add_file(FILE_NAME) 33 | temp_bucket.upload() 34 | bucket_name = temp_bucket.name 35 | 36 | assert string_return == STRING_KEY 37 | assert binary_return == BINARY_KEY 38 | assert file_return == FILE_KEY 39 | assert bucket_name == BUCKET_NAME 40 | bucket.assert_called_once_with(BUCKET_NAME) 41 | assert bucket.call_count == 1 42 | assert mock_open.call_count == 2 43 | 44 | location_parameters = {'CreateBucketConfiguration': dict(LocationConstraint='eu-central-1')} 45 | 46 | calls = [mocker.call(Body=STRING_BODY.encode(), Key=STRING_KEY), mocker.call(Body=BINARY_BODY, Key=BINARY_KEY), 47 | mocker.call(Body=mock_open(), Key=FILE_KEY)] 48 | bucket.return_value.create.assert_called_once_with(**location_parameters) 49 | boto_resource.return_value.meta.client.put_bucket_encryption.assert_called_once_with( 50 | Bucket=BUCKET_NAME, 51 | ServerSideEncryptionConfiguration={ 52 | "Rules": [ 53 | { 54 | "ApplyServerSideEncryptionByDefault": { 55 | "SSEAlgorithm": "AES256", 56 | }, 57 | } 58 | ], 59 | } 60 | ) 61 | bucket.return_value.put_object.assert_has_calls(calls) 62 | assert bucket.return_value.put_object.call_count == 3 63 | bucket.return_value.delete_objects.assert_called_once_with( 64 | Delete={'Objects': [{'Key': STRING_KEY}, {'Key': BINARY_KEY}]}) 65 | bucket.return_value.delete.assert_called_once_with() 66 | 67 | 68 | def test_does_not_delete_objects_if_empty(bucket): 69 | bucket.return_value.objects.all.return_value = [] 70 | 71 | with temporary_bucket(seed=STACK): 72 | pass 73 | 74 | bucket.return_value.delete_objects.assert_not_called() 75 | 76 | 77 | def test_does_not_use_s3_api_when_planning(bucket): 78 | bucket.return_value.objects.all.return_value = [] 79 | 80 | with temporary_bucket(seed=STACK) as temp_bucket: 81 | temp_bucket.add(STRING_BODY) 82 | temp_bucket.add(BINARY_BODY) 83 | 84 | bucket.return_value.create.assert_not_called() 85 | bucket.return_value.put_object.assert_not_called() 86 | bucket.return_value.delete_objects.assert_not_called() 87 | -------------------------------------------------------------------------------- /formica/stack_waiter.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import time 3 | from datetime import datetime 4 | import boto3 5 | 6 | import logging 7 | from texttable import Texttable 8 | 9 | EVENT_TABLE_HEADERS = ["Timestamp", "Status", "Type", "Logical ID", "Status reason"] 10 | 11 | TABLE_COLUMN_SIZE = [28, 24, 30, 30, 50] 12 | 13 | SUCCESSFUL_STATES = ["CREATE_COMPLETE", "UPDATE_COMPLETE", "DELETE_COMPLETE"] 14 | FAILED_STATES = [ 15 | "CREATE_FAILED", 16 | "DELETE_FAILED", 17 | "ROLLBACK_FAILED", 18 | "ROLLBACK_COMPLETE", 19 | "UPDATE_FAILED", 20 | "UPDATE_ROLLBACK_FAILED", 21 | "UPDATE_ROLLBACK_COMPLETE", 22 | ] 23 | 24 | logger = logging.getLogger(__name__) 25 | 26 | SLEEP_TIME = 5 27 | 28 | cf = boto3.client("cloudformation") 29 | 30 | 31 | class StackWaiter: 32 | def __init__(self, stack, timeout=0): 33 | self.stack = stack 34 | self.timeout = timeout 35 | 36 | def wait(self, last_event): 37 | header_printed = False 38 | finished = False 39 | canceled = False 40 | start = datetime.now() 41 | while not finished: 42 | stack_events = cf.describe_stack_events(StackName=self.stack)["StackEvents"] 43 | index = next((i for i, v in enumerate(stack_events) if v["EventId"] == last_event)) 44 | last_event = stack_events[0]["EventId"] 45 | new_events = stack_events[0:index] 46 | if new_events: 47 | if not header_printed: 48 | self.print_header() 49 | header_printed = True 50 | self.print_events(new_events) 51 | stack_status = self.stack_status() 52 | if stack_status in SUCCESSFUL_STATES: 53 | finished = True 54 | logger.info("Stack Status Successful: {}".format(stack_status)) 55 | elif stack_status in FAILED_STATES: 56 | logger.info("Stack Status Failed: {}".format(stack_status)) 57 | sys.exit(1) 58 | elif not canceled and self.timeout > 0 and (datetime.now() - start).seconds > (self.timeout * 60): 59 | logger.info("Timeout of {} minute(s) reached. Canceling Update.".format(self.timeout)) 60 | canceled = True 61 | cf.cancel_update_stack(StackName=self.stack) 62 | else: 63 | time.sleep(SLEEP_TIME) 64 | 65 | def stack_status(self): 66 | return cf.describe_stacks(StackName=self.stack)["Stacks"][0]["StackStatus"] 67 | 68 | def __create_table(self): 69 | table = Texttable() 70 | table.set_cols_width(TABLE_COLUMN_SIZE) 71 | return table 72 | 73 | def print_header(self): 74 | if self.timeout > 0: 75 | logger.info("Timeout set to {} minute(s)".format(self.timeout)) 76 | table = self.__create_table() 77 | table.add_rows([EVENT_TABLE_HEADERS]) 78 | table.set_deco(Texttable.BORDER | Texttable.VLINES) 79 | logger.info(table.draw()) 80 | 81 | def print_events(self, events): 82 | table = self.__create_table() 83 | table.set_deco(0) 84 | for event in reversed(events): 85 | table.add_row( 86 | [ 87 | event["Timestamp"].strftime("%Y-%m-%d %H:%M:%S %Z%z"), 88 | event["ResourceStatus"], 89 | event["ResourceType"], 90 | event["LogicalResourceId"], 91 | event.get("ResourceStatusReason", ""), 92 | ] 93 | ) 94 | logger.info(table.draw()) 95 | -------------------------------------------------------------------------------- /docs/commands/stack-set_diff.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: StackSet Diff 3 | weight: 100 4 | --- 5 | 6 | # `formica stack-set diff` 7 | 8 | The `formica stack-set diff` command compares the current local template against the deployed template in the stack-set. 9 | It works the same as [`formica diff`]({{< relref "diff.md" >}}). 10 | 11 | ``` 12 | root@61aaad32daf7:/app/docs/examples/s3-bucket# formica stack-set diff -s teststack 13 | +---------------------------------------------------------+------------------+----------------------------------+-----------------------+ 14 | | Path | From | To | Change Type | 15 | +=========================================================+==================+==================================+=======================+ 16 | | Resources > DeploymentBucket > Properties | Not Present | {'BucketName': 'another-bucket'} | Dictionary Item Added | 17 | +---------------------------------------------------------+------------------+----------------------------------+-----------------------+ 18 | | Resources > DeploymentBucket2 > Properties > BucketName | a-formica-bucket | a-bucket | Values Changed | 19 | +---------------------------------------------------------+------------------+----------------------------------+-----------------------+ 20 | ``` 21 | 22 | ## Usage 23 | 24 | ``` 25 | usage: formica stack-set diff [-h] [--region REGION] [--profile PROFILE] 26 | [--stack-set STACK-Set] 27 | [--config-file CONFIG_FILE [CONFIG_FILE ...]] 28 | [--parameters KEY=Value [KEY=Value ...]] 29 | [--tags KEY=Value [KEY=Value ...]] 30 | [--vars KEY=Value [KEY=Value ...]] 31 | [--organization-variables] 32 | [--organization-region-variables] 33 | [--organization-account-variables] 34 | [--main-account-parameter] 35 | 36 | Diff the StackSet template to the local template 37 | 38 | options: 39 | -h, --help show this help message and exit 40 | --region REGION The AWS region to use 41 | --profile PROFILE The AWS profile to use 42 | --stack-set STACK-Set, -s STACK-Set 43 | The Stack Set to use 44 | --config-file CONFIG_FILE [CONFIG_FILE ...], -c CONFIG_FILE [CONFIG_FILE ...] 45 | Set the config files to use 46 | --parameters KEY=Value [KEY=Value ...] 47 | Add one or multiple stack parameters 48 | --tags KEY=Value [KEY=Value ...] 49 | Add one or multiple stack tags 50 | --vars KEY=Value [KEY=Value ...] 51 | Add one or multiple Jinja2 variables 52 | --organization-variables 53 | Add AWSAccounts, AWSSubAccounts, AWSMainAccount and 54 | AWSRegions as Jinja variables with an Email, Id and 55 | Name field for each account 56 | --organization-region-variables 57 | Add AWSRegions as Jinja variables 58 | --organization-account-variables 59 | Add AWSAccounts, AWSSubAccounts, and AWSMainAccount as 60 | Jinja variables with an Email, Id, and Name field for 61 | each account 62 | --main-account-parameter 63 | Set MainAccount Parameter 64 | ``` 65 | -------------------------------------------------------------------------------- /docs/commands/stack-set_remove-instances.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: StackSet Remove Instances 3 | weight: 100 4 | --- 5 | 6 | # `formica stack-set remove-instances` 7 | 8 | The `formica stack-set remove-instances` command allows you to remove instances from your StackSet for 9 | various accounts and regions. It will remove the stack in every region of every account 10 | you're configuring. 11 | 12 | The `--retain` option makes it possible to remove a stack from the StackSet, but keep it in your 13 | AWS account. 14 | 15 | ## Usage 16 | 17 | ``` 18 | usage: formica stack-set remove-instances [-h] [--region REGION] 19 | [--profile PROFILE] 20 | [--stack-set STACK-Set] 21 | [--accounts ACCOUNTS [ACCOUNTS ...]] 22 | [--regions REGIONS [REGIONS ...]] 23 | [--retain] 24 | [--config-file CONFIG_FILE [CONFIG_FILE ...]] 25 | [--all-accounts] [--all-subaccounts] 26 | [--excluded-accounts EXCLUDED_ACCOUNTS [EXCLUDED_ACCOUNTS ...]] 27 | [--all-regions] 28 | [--excluded-regions EXCLUDED_REGIONS [EXCLUDED_REGIONS ...]] 29 | [--main-account] 30 | [--region-order REGION_ORDER [REGION_ORDER ...]] 31 | [--failure-tolerance-count FAILURE_TOLERANCE_COUNT | --failure-tolerance-percentage FAILURE_TOLERANCE_PERCENTAGE] 32 | [--max-concurrent-count MAX_CONCURRENT_COUNT | --max-concurrent-percentage MAX_CONCURRENT_PERCENTAGE] 33 | [--yes] 34 | 35 | Remove Stack Set Instances 36 | 37 | options: 38 | -h, --help show this help message and exit 39 | --region REGION The AWS region to use 40 | --profile PROFILE The AWS profile to use 41 | --stack-set STACK-Set, -s STACK-Set 42 | The Stack Set to use 43 | --accounts ACCOUNTS [ACCOUNTS ...] 44 | The Accounts for this operation 45 | --regions REGIONS [REGIONS ...] 46 | The Regions for this operation 47 | --retain Retain stacks 48 | --config-file CONFIG_FILE [CONFIG_FILE ...], -c CONFIG_FILE [CONFIG_FILE ...] 49 | Set the config files to use 50 | --all-accounts Use All Accounts of this Org 51 | --all-subaccounts Use Only Subaccounts of this Org 52 | --excluded-accounts EXCLUDED_ACCOUNTS [EXCLUDED_ACCOUNTS ...] 53 | All Accounts excluding these 54 | --all-regions Use all Regions 55 | --excluded-regions EXCLUDED_REGIONS [EXCLUDED_REGIONS ...] 56 | Excluded Regions from deployment 57 | --main-account Deploy to Main Account only 58 | --region-order REGION_ORDER [REGION_ORDER ...] 59 | Order in which to deploy to regions 60 | --failure-tolerance-count FAILURE_TOLERANCE_COUNT 61 | Number of Stacks to fail before failing operation 62 | --failure-tolerance-percentage FAILURE_TOLERANCE_PERCENTAGE 63 | Percentage of Stacks to fail before failing operation 64 | --max-concurrent-count MAX_CONCURRENT_COUNT 65 | Max Number of concurrent accounts to deploy to 66 | --max-concurrent-percentage MAX_CONCURRENT_PERCENTAGE 67 | Max Percentage of concurrent accounts to deploy to 68 | --yes, -y Answer all input questions with yes 69 | ``` 70 | -------------------------------------------------------------------------------- /formica/diff.py: -------------------------------------------------------------------------------- 1 | import collections 2 | import re 3 | import yaml 4 | import logging 5 | from deepdiff import DeepDiff 6 | import boto3 7 | from texttable import Texttable 8 | 9 | from formica.loader import Loader 10 | 11 | logger = logging.getLogger(__name__) 12 | 13 | 14 | class Change: 15 | def __init__(self, path, before, after, type): 16 | self.path = path 17 | self.before = before 18 | self.after = after 19 | self.type = type 20 | 21 | 22 | def convert(data): 23 | if isinstance(data, str): 24 | return data 25 | if isinstance(data, collections.abc.Mapping): 26 | return dict(map(convert, data.items())) 27 | elif isinstance(data, collections.abc.Iterable): 28 | return type(data)(map(convert, data)) 29 | else: 30 | return data 31 | 32 | 33 | def compare_stack(stack, vars=None, parameters={}, tags={}): 34 | client = boto3.client("cloudformation") 35 | template = client.get_template(StackName=stack)["TemplateBody"] 36 | 37 | stack = client.describe_stacks(StackName=stack)["Stacks"][0] 38 | __compare(template, stack, vars, parameters, tags) 39 | 40 | 41 | def compare_stack_set(stack, vars=None, parameters={}, tags={}, main_account_parameter=False): 42 | client = boto3.client("cloudformation") 43 | 44 | stack_set = client.describe_stack_set(StackSetName=stack)["StackSet"] 45 | __compare(stack_set["TemplateBody"], stack_set, vars, parameters, tags, main_account_parameter) 46 | 47 | 48 | def __compare(template, stack, vars=None, parameters={}, tags={}, main_account_parameter=False): 49 | current_parameters = {p["ParameterKey"]: p["ParameterValue"] for p in (stack.get("Parameters", []))} 50 | parameters = {key: str(value) for key, value in parameters.items()} 51 | tags = {key: str(value) for key, value in tags.items()} 52 | current_tags = {p["Key"]: p["Value"] for p in (stack.get("Tags", []))} 53 | 54 | loader = Loader(variables=vars, main_account_parameter=main_account_parameter) 55 | loader.load() 56 | deployed_template = convert(template) 57 | template_parameters = { 58 | key: str(value["Default"]).lower() if type(value["Default"]) == bool else str(value["Default"]) 59 | for key, value in (loader.template_dictionary().get("Parameters", {})).items() 60 | if "Default" in value 61 | } 62 | 63 | template_parameters.update(parameters) 64 | if isinstance(deployed_template, str): 65 | deployed_template = yaml.full_load(deployed_template) 66 | 67 | __generate_table("Parameters", current_parameters, template_parameters) 68 | __generate_table("Tags", current_tags, tags) 69 | __generate_table("Template", deployed_template, convert(loader.template_dictionary())) 70 | 71 | 72 | def __generate_table(header, current, new): 73 | changes = DeepDiff(current, new, ignore_order=False, report_repetition=True, verbose_level=2, view="tree") 74 | table = Texttable(max_width=200) 75 | table.set_cols_dtype(["t", "t", "t", "t"]) 76 | table.add_rows([["Path", "From", "To", "Change Type"]]) 77 | print_diff = False 78 | processed_changes = __collect_changes(changes) 79 | for change in processed_changes: 80 | print_diff = True 81 | path = re.findall("\\['?([\\w-]+)'?\\]", change.path) 82 | table.add_row([" > ".join(path), change.before, change.after, change.type.title().replace("_", " ")]) 83 | logger.info(header + " Diff:") 84 | if print_diff: 85 | logger.info(table.draw() + "\n") 86 | else: 87 | logger.info("No Changes found" + "\n") 88 | 89 | 90 | def __collect_changes(changes): 91 | results = [] 92 | for key, value in changes.items(): 93 | for change in list(value): 94 | results.append(Change(path=change.path(), before=change.t1, after=change.t2, type=key)) 95 | return sorted(results, key=lambda x: x.path) 96 | -------------------------------------------------------------------------------- /docs/commands/diff.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Diff 3 | weight: 100 4 | --- 5 | 6 | # `formica diff` 7 | 8 | Through the diff command you can see exactly what changed in your template compared to what is already deployed in your stack. It uses [DeepDiff](https://github.com/seperman/deepdiff) to compare the two templates and show you detailed results. 9 | 10 | Following is an example where we have two S3 Buckets and want to add a specific BucketName for one and change the BucketName of the second. 11 | 12 | ``` 13 | root@61aaad32daf7:/app/docs/examples/s3-bucket# formica diff --stack teststack 14 | +---------------------------------------------------------+------------------+----------------------------------+-----------------------+ 15 | | Path | From | To | Change Type | 16 | +=========================================================+==================+==================================+=======================+ 17 | | Resources > DeploymentBucket > Properties | Not Present | {'BucketName': 'another-bucket'} | Dictionary Item Added | 18 | +---------------------------------------------------------+------------------+----------------------------------+-----------------------+ 19 | | Resources > DeploymentBucket2 > Properties > BucketName | a-formica-bucket | a-bucket | Values Changed | 20 | +---------------------------------------------------------+------------------+----------------------------------+-----------------------+ 21 | ``` 22 | 23 | As you can see it will show the path of the property that was changed, what it was before and after and what kind of change it was. 24 | 25 | Together with [`formica describe`]({{< relref "describe.md" >}}) you can understand exactly what has changed in your template and how that will influence your deployed stack. 26 | 27 | ## Usage 28 | 29 | ``` 30 | usage: formica diff [-h] [--region REGION] [--profile PROFILE] [--stack STACK] 31 | [--config-file CONFIG_FILE [CONFIG_FILE ...]] 32 | [--vars KEY=Value [KEY=Value ...]] 33 | [--parameters KEY=Value [KEY=Value ...]] 34 | [--tags KEY=Value [KEY=Value ...]] 35 | [--organization-variables] 36 | [--organization-region-variables] 37 | [--organization-account-variables] 38 | [--artifacts ARTIFACTS [ARTIFACTS ...]] 39 | 40 | Print a diff between local and deployed stack 41 | 42 | options: 43 | -h, --help show this help message and exit 44 | --region REGION The AWS region to use 45 | --profile PROFILE The AWS profile to use 46 | --stack STACK, -s STACK 47 | The Stack to use 48 | --config-file CONFIG_FILE [CONFIG_FILE ...], -c CONFIG_FILE [CONFIG_FILE ...] 49 | Set the config files to use 50 | --vars KEY=Value [KEY=Value ...] 51 | Add one or multiple Jinja2 variables 52 | --parameters KEY=Value [KEY=Value ...] 53 | Add one or multiple stack parameters 54 | --tags KEY=Value [KEY=Value ...] 55 | Add one or multiple stack tags 56 | --organization-variables 57 | Add AWSAccounts, AWSSubAccounts, AWSMainAccount and 58 | AWSRegions as Jinja variables with an Email, Id and 59 | Name field for each account 60 | --organization-region-variables 61 | Add AWSRegions as Jinja variables 62 | --organization-account-variables 63 | Add AWSAccounts, AWSSubAccounts, and AWSMainAccount as 64 | Jinja variables with an Email, Id, and Name field for 65 | each account 66 | --artifacts ARTIFACTS [ARTIFACTS ...] 67 | Add one or more artifacts to push to S3 before 68 | deployment 69 | ``` 70 | -------------------------------------------------------------------------------- /tests/integration/test_basic.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | import uuid 3 | import json 4 | import yaml 5 | from path import Path 6 | 7 | import pytest 8 | 9 | 10 | @pytest.fixture 11 | def stack_name(): 12 | return 'formica-it-' + str(uuid.uuid4()) 13 | 14 | 15 | CONFIG_FILE = 'test.config.json' 16 | 17 | TEMPLATE_FILE = """ 18 | Resources: 19 | TestName: 20 | Type: AWS::S3::Bucket 21 | Properties: 22 | Tags: 23 | - Key: Bucket 24 | Value: {{ artifacts['SomeArtifact'].bucket }} 25 | - Key: Name 26 | Value: {{ artifacts['SomeArtifact'].key }} 27 | """ 28 | 29 | 30 | class TestIntegrationBasic(): 31 | def test_integration_basic(self, tmpdir, stack_name): 32 | with Path(tmpdir): 33 | def run_formica(*args): 34 | print() 35 | print("Command: formica {}".format(' '.join(args))) 36 | result = subprocess.check_output(['formica'] + list(args), cwd=str(tmpdir)).decode() 37 | print(result) 38 | return result 39 | 40 | def write_template(content): 41 | with open("test.template.json", 'w') as f: 42 | f.write(content) 43 | 44 | def write_config(content): 45 | with open(CONFIG_FILE, 'w') as f: 46 | f.write(json.dumps(content)) 47 | 48 | # Create a simple FC file and print it to STDOUT 49 | write_template(TEMPLATE_FILE) 50 | write_config({'stack': stack_name, "artifacts": ['SomeArtifact']}) 51 | with open("SomeArtifact", 'w') as f: 52 | f.write("Artifact") 53 | 54 | artifacts_args = ['--artifacts', 'SomeArtifact'] 55 | 56 | template = run_formica('template', *artifacts_args) 57 | assert 'TestName' in template 58 | assert 'AWS::S3::Bucket' in template 59 | 60 | stack_args = ['--stack', stack_name] 61 | 62 | stack_artifact_args = [*stack_args, *artifacts_args] 63 | 64 | # Create a ChangeSet for a new Stack to be deployed 65 | new = run_formica('new', *stack_artifact_args) 66 | assert 'AWS::S3::Bucket' in new 67 | 68 | # Deploy new Stack 69 | run_formica('deploy', '-c', CONFIG_FILE) 70 | 71 | write_template(json.dumps({'Resources': {'TestNameUpdate': {'Type': 'AWS::S3::Bucket'}}})) 72 | 73 | # Diff the current stack 74 | diff = run_formica('diff', *stack_artifact_args) 75 | 76 | assert 'Resources > TestName' in diff 77 | assert 'Dictionary Item Removed' in diff 78 | assert 'Resources > TestNameUpdate' in diff 79 | assert 'Dictionary Item Added' in diff 80 | 81 | # Change Resources in existing stack 82 | change = run_formica('change', '--s3', *stack_artifact_args) 83 | assert 'TestNameUpdate' in change 84 | 85 | # Describe ChangeSet before deploying 86 | describe = run_formica('describe', *stack_args) 87 | assert 'TestNameUpdate' in describe 88 | 89 | # Deploy changes to existing stack 90 | deploy = run_formica('deploy', *stack_artifact_args) 91 | assert 'UPDATE_COMPLETE' in deploy 92 | 93 | # Add Changes again without failing 94 | change = run_formica('change', *stack_artifact_args) 95 | assert "The submitted information didn't contain changes." in change 96 | 97 | # List Resources after deployment 98 | resources = run_formica('resources', *stack_args) 99 | assert 'TestNameUpdate' in resources 100 | 101 | # List all existing stacks 102 | stacks = run_formica('stacks') 103 | assert stack_name in stacks 104 | 105 | remove = run_formica('remove', *stack_args) 106 | assert 'DELETE_COMPLETE' in remove 107 | -------------------------------------------------------------------------------- /docs/commands/new.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: New 3 | weight: 100 4 | --- 5 | 6 | # `formica new` 7 | 8 | Through the new command you can upload your local template to CloudFormation and create a new ChangeSet for a new stack. CloudFormation will create a stack in the **REVIEW_IN_PROGRESS** state until the ChangeSet is deployed. It will fail if the stack already exists. If you decide to not execute the ChangeSet you need to use [`formica remove`]({{< relref "remove.md" >}}) to remove the stack and run `formica new` again. After executing it you have to use `formica change`({{< relref "change.md" >}}) to update the Stack. You can see the full template that will be used by running [`formica template`]({{< relref "template.md" >}}). 9 | 10 | The default name for the created ChangeSet is `STACK_NAME-change-set`, e.g. `formica-stack-change-set`. 11 | 12 | After the change was submitted a description of the changes will be printed. For all details on the information in that description check out [`formica describe`]({{< relref "describe.md" >}}) 13 | 14 | For nested Stacks you have the option to create nested ChangeSets via the `--nested-change-sets` option and `nested_change_sets` config file option. Those will give details about the changes proposed for each nested Stack as well as for the main Stack. 15 | 16 | ## Usage 17 | 18 | ``` 19 | usage: formica new [-h] [--region REGION] [--profile PROFILE] [--stack STACK] 20 | [--parameters KEY=Value [KEY=Value ...]] 21 | [--tags KEY=Value [KEY=Value ...]] 22 | [--capabilities Cap1 Cap2 [Cap1 Cap2 ...]] 23 | [--role-arn ROLE_ARN] [--role-name ROLE_NAME] 24 | [--config-file CONFIG_FILE [CONFIG_FILE ...]] 25 | [--vars KEY=Value [KEY=Value ...]] [--s3] 26 | [--artifacts ARTIFACTS [ARTIFACTS ...]] [--resource-types] 27 | [--organization-variables] 28 | [--organization-region-variables] 29 | [--organization-account-variables] [--upload-artifacts] 30 | [--nested-change-sets] 31 | 32 | Create a change set for a new stack 33 | 34 | options: 35 | -h, --help show this help message and exit 36 | --region REGION The AWS region to use 37 | --profile PROFILE The AWS profile to use 38 | --stack STACK, -s STACK 39 | The Stack to use 40 | --parameters KEY=Value [KEY=Value ...] 41 | Add one or multiple stack parameters 42 | --tags KEY=Value [KEY=Value ...] 43 | Add one or multiple stack tags 44 | --capabilities Cap1 Cap2 [Cap1 Cap2 ...] 45 | Set one or multiple stack capabilities 46 | --role-arn ROLE_ARN Set a separate role ARN to pass to the stack 47 | --role-name ROLE_NAME 48 | Set a role name that will be translated to the ARN 49 | --config-file CONFIG_FILE [CONFIG_FILE ...], -c CONFIG_FILE [CONFIG_FILE ...] 50 | Set the config files to use 51 | --vars KEY=Value [KEY=Value ...] 52 | Add one or multiple Jinja2 variables 53 | --s3 Upload template to S3 before deployment 54 | --artifacts ARTIFACTS [ARTIFACTS ...] 55 | Add one or more artifacts to push to S3 before 56 | deployment 57 | --resource-types Add Resource Types to the ChangeSet 58 | --organization-variables 59 | Add AWSAccounts, AWSSubAccounts, AWSMainAccount and 60 | AWSRegions as Jinja variables with an Email, Id and 61 | Name field for each account 62 | --organization-region-variables 63 | Add AWSRegions as Jinja variables 64 | --organization-account-variables 65 | Add AWSAccounts, AWSSubAccounts, and AWSMainAccount as 66 | Jinja variables with an Email, Id, and Name field for 67 | each account 68 | --upload-artifacts Upload Artifacts when creating the ChangeSet 69 | --nested-change-sets Create a ChangeSet for nested Stacks 70 | ``` 71 | -------------------------------------------------------------------------------- /tests/unit/test_stack_waiter.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from mock import Mock 3 | from datetime import datetime, timedelta 4 | 5 | from formica.stack_waiter import StackWaiter, EVENT_TABLE_HEADERS 6 | from tests.unit.constants import STACK, STACK_EVENTS 7 | 8 | 9 | @pytest.fixture 10 | def time(mocker): 11 | return mocker.patch('formica.stack_waiter.time') 12 | 13 | 14 | @pytest.fixture 15 | def datetime_mock(mocker): 16 | return mocker.patch('formica.stack_waiter.datetime') 17 | 18 | 19 | @pytest.fixture 20 | def logger(mocker): 21 | return mocker.patch('formica.stack_waiter.logger') 22 | 23 | 24 | @pytest.fixture 25 | def stack_waiter(): 26 | return StackWaiter(STACK) 27 | 28 | @pytest.fixture 29 | def client(mocker): 30 | client = mocker.patch('formica.stack_waiter.cf') 31 | return client 32 | 33 | def set_stack_status_returns(client, statuses): 34 | client.describe_stacks.side_effect = [{'Stacks': [{'StackStatus': status}]} for status in 35 | statuses] 36 | 37 | 38 | def set_stack_events(client, events=1): 39 | stack_events = {'StackEvents': [{"EventId": str(num)} for num in range(events)]} 40 | client.describe_stack_events.return_value = stack_events 41 | 42 | 43 | def test_prints_header(time, mocker, client, stack_waiter): 44 | header = mocker.patch.object(StackWaiter, 'print_header') 45 | set_stack_status_returns(client, ['CREATE_COMPLETE']) 46 | client.describe_stack_events.return_value = STACK_EVENTS 47 | stack_waiter.wait('DeploymentBucket3-7c92066b-c2e7-427a-ab29-53b928925473') 48 | header.assert_called() 49 | 50 | 51 | def test_waits_until_successful(client, time, stack_waiter): 52 | set_stack_status_returns(client, ['UPDATE_IN_PROGRESS', 'CREATE_COMPLETE']) 53 | set_stack_events(client) 54 | stack_waiter.wait('0') 55 | assert time.sleep.call_count == 1 56 | time.sleep.assert_called_with(5) 57 | 58 | 59 | def test_waits_until_failed_and_raises(client, time, stack_waiter): 60 | set_stack_status_returns(client, ['UPDATE_IN_PROGRESS', 'CREATE_FAILED']) 61 | set_stack_events(client) 62 | with pytest.raises(SystemExit, match='1'): 63 | stack_waiter.wait('0') 64 | assert time.sleep.call_count == 1 65 | 66 | 67 | def test_waits_until_timeout(client, time, datetime_mock): 68 | first_timestamp = datetime.now() 69 | second_timestamp = datetime.now() + timedelta(0, 50, 0) 70 | last_timestamp = datetime.now() + timedelta(0, 61, 0) 71 | datetime_mock.now.side_effect = [first_timestamp, second_timestamp, last_timestamp] 72 | set_stack_status_returns(client, 73 | ['UPDATE_IN_PROGRESS', 'UPDATE_IN_PROGRESS', 'UPDATE_IN_PROGRESS', 'CREATE_FAILED']) 74 | set_stack_events(client) 75 | stack_waiter = StackWaiter(STACK, timeout=1) 76 | with pytest.raises(SystemExit, match='1'): 77 | stack_waiter.wait('0') 78 | assert time.sleep.call_count == 2 79 | client.cancel_update_stack.assert_called_with(StackName=STACK) 80 | 81 | 82 | def test_prints_new_events(logger, time, client, stack_waiter): 83 | set_stack_status_returns(client, ['CREATE_COMPLETE']) 84 | client.describe_stack_events.return_value = STACK_EVENTS 85 | stack_waiter.wait('DeploymentBucket3-7c92066b-c2e7-427a-ab29-53b928925473') 86 | 87 | logger.info.assert_called() 88 | output = '\n'.join([call[1][0] for call in logger.info.mock_calls]) 89 | to_search = [] 90 | to_search.extend(EVENT_TABLE_HEADERS) 91 | to_search.extend(['UPDATE_COMPLETE', 'DELETE_COMPLETE']) 92 | to_search.extend(['AWS::S3::Bucket', 'AWS::CloudFormation::Stack']) 93 | to_search.extend(['2017-02-06 16:01:16', '2017-02-06 16:01:16', '2017-02-06 16:01:17']) 94 | to_search.extend(['DeploymentBucket14', 'DeploymentBucket18', 'teststack ']) 95 | to_search.extend(['Resource creation Initiated']) 96 | for term in to_search: 97 | assert term in output 98 | 99 | old_events = ['DeploymentBucket3', 'DeploymentBucket15'] 100 | for term in old_events: 101 | assert term not in output 102 | assert 'None' not in output 103 | -------------------------------------------------------------------------------- /tests/unit/test_deploy.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from mock import Mock 3 | 4 | from botocore.exceptions import NoCredentialsError 5 | 6 | from formica import cli 7 | from tests.unit.constants import STACK, STACK_ID, PROFILE, REGION, CHANGESETNAME, EVENT_ID 8 | 9 | 10 | @pytest.fixture 11 | def logger(mocker): 12 | return mocker.patch('formica.cli.logger') 13 | 14 | 15 | def test_catches_common_aws_exceptions(session, stack_waiter): 16 | session.side_effect = NoCredentialsError() 17 | with pytest.raises(SystemExit) as pytest_wrapped_e: 18 | cli.main(['deploy', '--stack', STACK]) 19 | assert pytest_wrapped_e.value.code == 1 20 | 21 | 22 | def test_fails_if_no_stack_given(logger): 23 | with pytest.raises(SystemExit) as pytest_wrapped_e: 24 | cli.main(['deploy']) 25 | assert pytest_wrapped_e.value.code == 1 26 | 27 | logger.error.assert_called() 28 | out = logger.error.call_args[0][0] 29 | assert '--stack' in out 30 | assert '--config-file' in out 31 | 32 | 33 | def test_executes_change_set_and_waits(session, stack_waiter, client, boto_client): 34 | client.describe_change_set.return_value = {'Status': 'CREATE_COMPLETE'} 35 | client.describe_stack_events.return_value = {'StackEvents': [{'EventId': EVENT_ID}]} 36 | client.describe_stacks.return_value = {'Stacks': [{'StackId': STACK_ID}]} 37 | 38 | cli.main(['deploy', '--stack', STACK, '--profile', PROFILE, '--region', REGION]) 39 | boto_client.assert_called_with('cloudformation') 40 | client.describe_stack_events.assert_called_with(StackName=STACK) 41 | client.execute_change_set.assert_called_with(ChangeSetName=CHANGESETNAME, StackName=STACK, DisableRollback=False) 42 | client.describe_change_set.assert_called_with(ChangeSetName=CHANGESETNAME, StackName=STACK) 43 | stack_waiter.assert_called_with(STACK_ID) 44 | stack_waiter.return_value.wait.assert_called_with(EVENT_ID) 45 | 46 | 47 | def test_executes_change_set_and_waits(session, stack_waiter, client, boto_client): 48 | client.describe_change_set.return_value = {'Status': 'CREATE_COMPLETE'} 49 | client.describe_stack_events.return_value = {'StackEvents': [{'EventId': EVENT_ID}]} 50 | client.describe_stacks.return_value = {'Stacks': [{'StackId': STACK_ID}]} 51 | cli.main(['deploy', '--stack', STACK, '--disable-rollback']) 52 | boto_client.assert_called_with('cloudformation') 53 | client.execute_change_set.assert_called_with(ChangeSetName=CHANGESETNAME, StackName=STACK, DisableRollback=True) 54 | 55 | 56 | def test_executes_change_set_with_timeout(stack_waiter, client): 57 | client.describe_change_set.return_value = {'Status': 'CREATE_COMPLETE'} 58 | client.describe_stack_events.return_value = {'StackEvents': [{'EventId': EVENT_ID}]} 59 | client.describe_stacks.return_value = {'Stacks': [{'StackId': STACK_ID}]} 60 | 61 | cli.main(['deploy', '--stack', STACK, '--profile', PROFILE, '--region', REGION, '--timeout', '15']) 62 | stack_waiter.assert_called_with(STACK_ID, timeout=15) 63 | stack_waiter.return_value.wait.assert_called_with(EVENT_ID) 64 | 65 | 66 | def test_does_not_execute_changeset_if_no_changes(stack_waiter, client): 67 | client.describe_change_set.return_value = {'Status': 'FAILED', 68 | "StatusReason": "The submitted information didn't contain changes. Submit different information to create a change set."} 69 | client.describe_stack_events.return_value = {'StackEvents': [{'EventId': EVENT_ID}]} 70 | client.describe_stacks.return_value = {'Stacks': [{'StackId': STACK_ID}]} 71 | 72 | cli.main(['deploy', '--stack', STACK]) 73 | client.execute_change_set.assert_not_called() 74 | stack_waiter.assert_called_with(STACK_ID) 75 | stack_waiter.return_value.wait.assert_called_with(EVENT_ID) 76 | 77 | 78 | def test_does_not_execute_changeset_if_in_failed_state(stack_waiter, client): 79 | client.describe_change_set.return_value = {'Status': 'FAILED'} 80 | client.describe_stack_events.return_value = {'StackEvents': [{'EventId': EVENT_ID}]} 81 | client.describe_stacks.return_value = {'Stacks': [{'StackId': STACK_ID}]} 82 | 83 | with pytest.raises(SystemExit): 84 | cli.main(['deploy', '--stack', STACK]) 85 | client.execute_change_set.assert_not_called() 86 | -------------------------------------------------------------------------------- /formica/s3.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | from contextlib import contextmanager 3 | import logging 4 | from hashlib import md5 5 | from io import BytesIO 6 | 7 | logger = logging.getLogger(__name__) 8 | 9 | # Using MD5 for shorter string names as Sha256 is larger than allowed bucket name characters 10 | HASH = md5 11 | # Use Hash Blocksize to determine number of bytes read 12 | BLOCKSIZE_MULTI = 512 13 | 14 | 15 | class TemporaryS3Bucket(object): 16 | def __init__(self, seed): 17 | self.objects = {} 18 | self.uploaded = False 19 | self.__sts = boto3.client("sts") 20 | self.s3_bucket = None 21 | self.files = {} 22 | self.seed = seed 23 | 24 | def __digest(self, body): 25 | 26 | used_hash = HASH() 27 | for byte_block in iter(lambda: body.read(used_hash.block_size * BLOCKSIZE_MULTI), b""): 28 | used_hash.update(byte_block) 29 | return used_hash.hexdigest() 30 | 31 | def add(self, body): 32 | if type(body) == str: 33 | body = body.encode() 34 | with BytesIO(body) as b: 35 | object_name = self.__digest(b) 36 | self.objects[object_name] = body 37 | return object_name 38 | 39 | def add_file(self, file_name): 40 | with open(file_name, "rb") as f: 41 | object_name = self.__digest(f) 42 | self.files[object_name] = file_name 43 | return object_name 44 | 45 | @property 46 | def name(self): 47 | body_hashes = "".join( 48 | [key for key, _ in self.objects.items()] + [key for key, _ in self.files.items()] 49 | ).encode() 50 | account_id = self.__sts.get_caller_identity()["Account"] 51 | to_hash = self.seed + account_id + self.__sts.meta.region_name + body_hashes.decode() 52 | name_digest_input = BytesIO(to_hash.encode()) 53 | body_hashes_hash = self.__digest(name_digest_input) 54 | return "formica-deploy-{}".format(body_hashes_hash) 55 | 56 | def upload(self): 57 | if not self.uploaded: 58 | s3 = boto3.resource("s3") 59 | self.uploaded = True 60 | self.s3_bucket = s3.Bucket(self.name) 61 | try: 62 | if self.__sts.meta.region_name == "us-east-1": 63 | # To create a bucket in us-east-1 no LocationConstraint should be specified. 64 | # See https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#S3.Bucket.create 65 | self.s3_bucket.create() 66 | else: 67 | self.s3_bucket.create( 68 | CreateBucketConfiguration=dict(LocationConstraint=self.__sts.meta.region_name) 69 | ) 70 | except s3.meta.client.exceptions.BucketAlreadyOwnedByYou: 71 | logger.info("Artifact Bucket already exists") 72 | 73 | s3.meta.client.put_bucket_encryption( 74 | Bucket=self.name, 75 | ServerSideEncryptionConfiguration={ 76 | "Rules": [ 77 | { 78 | "ApplyServerSideEncryptionByDefault": { 79 | "SSEAlgorithm": "AES256", 80 | }, 81 | } 82 | ], 83 | }, 84 | ) 85 | 86 | for name, body in self.objects.items(): 87 | logger.info("Uploading to Bucket: {}/{}".format(self.name, name)) 88 | self.s3_bucket.put_object(Key=name, Body=body) 89 | for name, file_name in self.files.items(): 90 | with open(file_name, "rb") as f: 91 | logger.info("Uploading to Bucket: {}/{}".format(self.name, name)) 92 | self.s3_bucket.put_object(Key=name, Body=f) 93 | 94 | 95 | @contextmanager 96 | def temporary_bucket(seed): 97 | temp_bucket = TemporaryS3Bucket(seed=seed) 98 | try: 99 | yield temp_bucket 100 | finally: 101 | if temp_bucket.uploaded: 102 | to_delete = [dict(Key=obj.key) for obj in temp_bucket.s3_bucket.objects.all()] 103 | if to_delete: 104 | logger.info("Deleting {} Objects from Bucket: {}".format(len(to_delete), temp_bucket.name)) 105 | temp_bucket.s3_bucket.delete_objects(Delete=dict(Objects=to_delete)) 106 | logger.info("Deleting Bucket: {}".format(temp_bucket.name)) 107 | temp_bucket.s3_bucket.delete() 108 | -------------------------------------------------------------------------------- /docs/commands/change.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Change 3 | weight: 100 4 | --- 5 | 6 | # `formica change` 7 | 8 | Through the change command you can upload your local template to CloudFormation and create a new Changeset for an existing stack. It will fail if the stack doesn't already exist. If you want to create the missing stack use the `--create-missing` option. In general try to use `formica new` when creating a new Stack so you don't accidentally creates stacks when running change. For automation it is sometimes useful though to create them if they aren't there yet. 9 | 10 | If you want to create a ChangeSet for a new stack check out `formica new`({{< relref "new.md" >}}). You can see the full template that will be used by running [`formica template`]({{< relref "template.md" >}}). 11 | 12 | The default name for the created ChangeSet is `STACK_NAME-change-set`, e.g. `formica-stack-change-set`. If a ChangeSet exists but wasn't deployed it will be removed before creating a new ChangeSet. 13 | 14 | After the change was submitted a description of the changes will be printed. For all details on the information in that description check out [`formica describe`]({{< relref "describe.md" >}}) 15 | 16 | For nested Stacks you have the option to create nested ChangeSets via the `--nested-change-sets` option and `nested_change_sets` config file option. Those will give details about the changes proposed for each nested Stack as well as for the main Stack. 17 | 18 | ## Usage 19 | 20 | ``` 21 | usage: formica change [-h] [--region REGION] [--profile PROFILE] 22 | [--stack STACK] [--parameters KEY=Value [KEY=Value ...]] 23 | [--tags KEY=Value [KEY=Value ...]] 24 | [--capabilities Cap1 Cap2 [Cap1 Cap2 ...]] 25 | [--role-arn ROLE_ARN] [--role-name ROLE_NAME] 26 | [--config-file CONFIG_FILE [CONFIG_FILE ...]] 27 | [--vars KEY=Value [KEY=Value ...]] [--s3] 28 | [--artifacts ARTIFACTS [ARTIFACTS ...]] 29 | [--resource-types] [--create-missing] 30 | [--organization-variables] 31 | [--organization-region-variables] 32 | [--organization-account-variables] 33 | [--use-previous-template] [--use-previous-parameters] 34 | [--upload-artifacts] [--nested-change-sets] 35 | 36 | Create a change set for an existing stack 37 | 38 | options: 39 | -h, --help show this help message and exit 40 | --region REGION The AWS region to use 41 | --profile PROFILE The AWS profile to use 42 | --stack STACK, -s STACK 43 | The Stack to use 44 | --parameters KEY=Value [KEY=Value ...] 45 | Add one or multiple stack parameters 46 | --tags KEY=Value [KEY=Value ...] 47 | Add one or multiple stack tags 48 | --capabilities Cap1 Cap2 [Cap1 Cap2 ...] 49 | Set one or multiple stack capabilities 50 | --role-arn ROLE_ARN Set a separate role ARN to pass to the stack 51 | --role-name ROLE_NAME 52 | Set a role name that will be translated to the ARN 53 | --config-file CONFIG_FILE [CONFIG_FILE ...], -c CONFIG_FILE [CONFIG_FILE ...] 54 | Set the config files to use 55 | --vars KEY=Value [KEY=Value ...] 56 | Add one or multiple Jinja2 variables 57 | --s3 Upload template to S3 before deployment 58 | --artifacts ARTIFACTS [ARTIFACTS ...] 59 | Add one or more artifacts to push to S3 before 60 | deployment 61 | --resource-types Add Resource Types to the ChangeSet 62 | --create-missing Create the Stack in case it's missing 63 | --organization-variables 64 | Add AWSAccounts, AWSSubAccounts, AWSMainAccount and 65 | AWSRegions as Jinja variables with an Email, Id and 66 | Name field for each account 67 | --organization-region-variables 68 | Add AWSRegions as Jinja variables 69 | --organization-account-variables 70 | Add AWSAccounts, AWSSubAccounts, and AWSMainAccount as 71 | Jinja variables with an Email, Id, and Name field for 72 | each account 73 | --use-previous-template 74 | Use the previously deployed template 75 | --use-previous-parameters 76 | Reuse Stack Parameters not specifically set 77 | --upload-artifacts Upload Artifacts when creating the ChangeSet 78 | --nested-change-sets Create a ChangeSet for nested Stacks 79 | ``` 80 | -------------------------------------------------------------------------------- /tests/unit/test_new.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from formica import cli 4 | from tests.unit.constants import REGION, PROFILE, STACK, TEMPLATE 5 | 6 | 7 | def test_create_changeset_for_new_stack(change_set, client, loader): 8 | loader.return_value.template.return_value = TEMPLATE 9 | cli.main(['new', '--stack', STACK, '--profile', PROFILE, '--region', REGION]) 10 | change_set.assert_called_with(stack=STACK, nested_change_sets=False) 11 | change_set.return_value.create.assert_called_once_with(template=TEMPLATE, change_set_type='CREATE', 12 | parameters={}, tags={}, capabilities=None, 13 | resource_types=False, role_arn=None, s3=False) 14 | change_set.return_value.describe.assert_called_once() 15 | 16 | 17 | def test_new_uses_parameters_for_creation(change_set, client, loader): 18 | loader.return_value.template.return_value = TEMPLATE 19 | cli.main(['new', '--stack', STACK, '--parameters', 'A=B', 'C=D', '--profile', PROFILE, '--region', REGION, ]) 20 | change_set.assert_called_with(stack=STACK, nested_change_sets=False) 21 | change_set.return_value.create.assert_called_once_with(template=TEMPLATE, change_set_type='CREATE', 22 | parameters={'A': 'B', 'C': 'D'}, tags={}, 23 | capabilities=None, resource_types=False, role_arn=None, 24 | s3=False) 25 | 26 | 27 | def test_new_uses_tags_for_creation(change_set, client, loader): 28 | loader.return_value.template.return_value = TEMPLATE 29 | cli.main(['new', '--stack', STACK, '--tags', 'A=C', 'C=D', '--profile', PROFILE, '--region', REGION, ]) 30 | change_set.assert_called_with(stack=STACK, nested_change_sets=False) 31 | change_set.return_value.create.assert_called_once_with(template=TEMPLATE, change_set_type='CREATE', 32 | parameters={}, 33 | tags={'A': 'C', 'C': 'D'}, capabilities=None, 34 | resource_types=False, role_arn=None, s3=False) 35 | 36 | 37 | def test_new_tests_parameter_format(capsys): 38 | with pytest.raises(SystemExit) as pytest_wrapped_e: 39 | cli.main( 40 | ['new', '--stack', STACK, '--parameters', 'A=B', '--profile', PROFILE, '--region', REGION, '--tags', 'CD']) 41 | 42 | out, err = capsys.readouterr() 43 | 44 | assert "needs to be in format KEY=VALUE" in err 45 | assert pytest_wrapped_e.value.code == 2 46 | 47 | 48 | def test_new_uses_capabilities_for_creation(change_set, client, loader): 49 | loader.return_value.template.return_value = TEMPLATE 50 | cli.main(['new', '--stack', STACK, '--capabilities', 'A', 'B']) 51 | change_set.assert_called_with(stack=STACK, nested_change_sets=False) 52 | change_set.return_value.create.assert_called_once_with(template=TEMPLATE, change_set_type='CREATE', 53 | parameters={}, 54 | tags={}, capabilities=['A', 'B'], resource_types=False, 55 | role_arn=None, s3=False) 56 | 57 | 58 | def test_upload_artifacts(change_set, aws_client, loader, temp_bucket_cli, mocker): 59 | mocker.patch('formica.cli.collect_vars').return_value = {} 60 | loader.return_value.template.return_value = TEMPLATE 61 | cli.main(['new', '--stack', STACK, '--upload-artifacts', '--artifacts', 'testfile']) 62 | change_set.assert_called_with(stack=STACK, nested_change_sets=False) 63 | change_set.return_value.create.assert_called_once_with(change_set_type='CREATE', 64 | parameters={}, 65 | tags={}, capabilities=None, resource_types=False, 66 | role_arn=None, s3=False, template=TEMPLATE) 67 | 68 | temp_bucket_cli.add_file.assert_called_once_with('testfile') 69 | temp_bucket_cli.upload.assert_called_once() 70 | 71 | 72 | def test_nested_change_sets(change_set, aws_client, loader): 73 | loader.return_value.template.return_value = TEMPLATE 74 | cli.main(['new', '--stack', STACK, '--nested-change-sets']) 75 | change_set.assert_called_with(stack=STACK, nested_change_sets=True) 76 | change_set.return_value.create.assert_called_once_with(change_set_type='CREATE', 77 | parameters={}, 78 | tags={}, capabilities=None, resource_types=False, 79 | template=TEMPLATE, 80 | role_arn=None, s3=False) 81 | -------------------------------------------------------------------------------- /docs/stack-sets.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Working with Stack Sets 3 | weight: 600 4 | --- 5 | 6 | Stack sets are a fantastic way to manage infrastructure across multiple accounts using CloudFormation. Formica allows you to use the same template building features you're accustomed to with Stacks to use for StackSets. It also gives you all the commands necessary to manage your StackSet instances and update your StackSets. 7 | 8 | You can see all the available StackSet commands by running `formica stack-set --help`. 9 | 10 | Let's go through a quick example how to use it: 11 | 12 | In the example we're creating an AWS Config Recorder across all your accounts and regions. The following template would configure that Recorder and a Role to access all necessary resources. 13 | 14 | 15 | ``` 16 | Resources: 17 | ConfigurationRecorderGlobal: 18 | Type: 'AWS::Config::ConfigurationRecorder' 19 | Properties: 20 | RecordingGroup: 21 | AllSupported: true 22 | IncludeGlobalResourceTypes: true 23 | RoleARN: !GetAtt 'ConfigurationRecorderRole.Arn' 24 | 25 | ConfigurationRecorderRole: 26 | Type: 'AWS::IAM::Role' 27 | Properties: 28 | ManagedPolicyArns: 29 | - 'arn:aws:iam::aws:policy/service-role/AWSConfigRole' 30 | AssumeRolePolicyDocument: 31 | Version: '2012-10-17' 32 | Statement: 33 | - Sid: AssumeRole 34 | Effect: Allow 35 | Principal: 36 | Service: 'config.amazonaws.com' 37 | Action: 'sts:AssumeRole' 38 | ``` 39 | 40 | You can use the same commands like `formica template` to see the resulting template before pushing it to CloudFormation. 41 | 42 | To deploy the stack consistently we're creating a config file as well (`stack-set.config.yaml` would be the typical name): 43 | 44 | ``` 45 | stack-set: config-recorder 46 | capabilities: 47 | - CAPABILITY_IAM 48 | ``` 49 | 50 | With this in place we can now create the StackSet before adding instances to it. 51 | 52 | ``` 53 | formica stack-set create -c stack-set.config.yaml 54 | ``` 55 | 56 | Now we can add instances to it. You have to set the accounts and regions you want a specific stack-set action to be performed in for every command or through the config file. CloudFormation will then update each individual stack in each account and region combination. 57 | 58 | As a first step lets add us-east-1 in our main account. 59 | 60 | ``` 61 | formica stack-set add-instances -c stack-set.config.yaml --regions us-east-1 --main-account 62 | ``` 63 | 64 | As you can see we're not specifying the account id directly (but you can with `--accounts`, but using the --main-account option which means the account we're currently using. Formica will compare the currently deployed instances to the ones you want to add and give you a list of stacks it will actually create (or tell you if all were already added). 65 | 66 | If you deploy to multiple accounts it will even figure out which account/region combinations overlap and deploy those as one action to enable parallelisation across accounts. CloudFormation will fail if you try to add an already existing instance, so we have to be careful to only add the specific ones that aren't already there. 67 | 68 | Now lets add it to all subaccounts. All command line options have equivalent config file options which I'll show at the end of the post: 69 | 70 | ``` 71 | formica stack-set add-instances -c stack-set.config.yaml --regions us-east-1 --all-subaccounts 72 | ``` 73 | 74 | The same process as before will happen, you'll get a list of all instances to be added and the instances will be deployed. 75 | 76 | Now let's add the configuration to deploy AWS Config everywhere into our configuration file. 77 | 78 | ``` 79 | stack-set: config-recorder 80 | capabilities: 81 | - CAPABILITY_IAM 82 | all-regions: true 83 | all-accounts: true 84 | ``` 85 | 86 | And add the instances: 87 | 88 | ``` 89 | formica stack-set add-instances -c stack-set.config.yaml 90 | ``` 91 | 92 | Now if we want to update our StackSet Instances we can run the update command. It accepts the same account and region options as `add-instances`. Before the deployment it will also show you a diff of the currently deployed template and the new one. You can see the same diff with `formica stack-set diff -c stack-set.config.yaml`. 93 | 94 | ``` 95 | formica stack-set update -c stack-set.config.yaml. 96 | ``` 97 | 98 | The update, as all the other commands before, waits for the CloudFormation StackSet operation to finish and report the result. Stopping the command with `ctrl-c` will not end the operation though, just the waiter. 99 | 100 | To remove StackSets, e.g. `eu-central-1` and `us-east-1` in the main account we can use the command `formica stack-set remove-instances -c stack-set.config.yaml --main-account --regions us-east-1 eu-central-1`. 101 | 102 | If you want to remove a StackSet (after removing all its instances) you can run `formica stack-set remove -c stack-set.config.yaml`. 103 | 104 | Check out all the existing formica commands in the [commands documentation]({{< relref "/tools/formica/commands" >}}) and run `--help` on any command you'd like to get additional info on in your workflow. -------------------------------------------------------------------------------- /tests/unit/test_config_file.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from path import Path 3 | import yaml 4 | from uuid import uuid4 5 | from formica import cli 6 | from tests.unit.constants import (REGION, PROFILE, STACK, 7 | CHANGE_SET_PARAMETERS, CHANGE_SET_STACK_TAGS, 8 | FULL_CONFIG_FILE, CHANGE_SET_CAPABILITIES, 9 | ROLE_ARN, VARS) 10 | 11 | 12 | def test_loads_config_file(mocker, tmpdir, session): 13 | stacks = mocker.patch('formica.cli.stacks') 14 | file_name = 'test.config.yaml' 15 | with Path(tmpdir): 16 | with open(file_name, 'w') as f: 17 | f.write(yaml.dump(FULL_CONFIG_FILE)) 18 | cli.main(['stacks', '-c', file_name]) 19 | call_args = stacks.call_args[0][0] 20 | assert call_args.region == REGION 21 | assert call_args.profile == PROFILE 22 | assert call_args.stack == STACK 23 | assert call_args.parameters == CHANGE_SET_PARAMETERS 24 | assert call_args.tags == CHANGE_SET_STACK_TAGS 25 | assert call_args.capabilities == CHANGE_SET_CAPABILITIES 26 | assert call_args.role_arn == ROLE_ARN 27 | assert call_args.vars == VARS 28 | 29 | 30 | def test_loads_multiple_config_files(mocker, tmpdir, session): 31 | stacks = mocker.patch('formica.cli.stacks') 32 | file_name = 'test.config.yaml' 33 | overwrite_file = 'overwrite.config.yaml' 34 | with Path(tmpdir): 35 | with open(file_name, 'w') as f: 36 | f.write(yaml.dump(FULL_CONFIG_FILE)) 37 | with open(overwrite_file, 'w') as f: 38 | f.write(yaml.dump(dict(stack='someotherstacktestvalue', vars=dict(OtherVar=3)))) 39 | cli.main(['stacks', '-c', file_name, overwrite_file]) 40 | call_args = stacks.call_args[0][0] 41 | assert call_args.stack == 'someotherstacktestvalue' 42 | assert call_args.stack != STACK 43 | assert call_args.vars['OtherVar'] == 3 44 | assert call_args.vars['OtherVar'] != VARS['OtherVar'] 45 | 46 | 47 | def test_prioritises_cli_args(mocker, tmpdir, session): 48 | stacks = mocker.patch('formica.cli.new') 49 | cli_stack = str(uuid4()) 50 | file_name = 'test.config.yaml' 51 | with Path(tmpdir): 52 | with open(file_name, 'w') as f: 53 | f.write(yaml.dump(FULL_CONFIG_FILE)) 54 | cli.main(['new', '-s', cli_stack, '-c', file_name]) 55 | call_args = stacks.call_args[0][0] 56 | assert call_args.stack == cli_stack 57 | assert call_args.stack != STACK 58 | 59 | 60 | def test_merges_cli_args_on_load(mocker, tmpdir, session): 61 | stacks = mocker.patch('formica.cli.new') 62 | param1 = str(uuid4()) 63 | param2 = str(uuid4()) 64 | file_name = 'test.config.yaml' 65 | with Path(tmpdir): 66 | with open(file_name, 'w') as f: 67 | f.write(yaml.dump(FULL_CONFIG_FILE)) 68 | cli.main(['new', '--parameters', "A={}".format(param1), "D={}".format(param2), '-c', file_name]) 69 | call_args = stacks.call_args[0][0] 70 | assert call_args.parameters == {"A": param1, "B": 2, 'C': True, 'D': param2} 71 | 72 | 73 | def test_merges_vars(mocker, tmpdir, session): 74 | stacks = mocker.patch('formica.cli.template') 75 | param1 = str(uuid4()) 76 | file_name = 'test.config.yaml' 77 | with Path(tmpdir): 78 | with open(file_name, 'w') as f: 79 | f.write(yaml.dump(FULL_CONFIG_FILE)) 80 | with open('test.template.yaml', 'w') as f: 81 | f.write('{"Description": "{{ OtherVar }}"}') 82 | cli.main(['template', '--vars', "OtherVar={}".format(param1), '-c', file_name]) 83 | call_args = stacks.call_args[0][0] 84 | assert call_args.vars['OtherVar'] == param1 85 | 86 | 87 | def test_exception_with_wrong_config_type(mocker, tmpdir, session, logger): 88 | file_name = 'test.config.yaml' 89 | with Path(tmpdir): 90 | with open(file_name, 'w') as f: 91 | f.write(yaml.dump({'stack': ['test', 'test2']})) 92 | with pytest.raises(SystemExit): 93 | cli.main(['stacks', '-c', file_name]) 94 | logger.error.assert_called_with('Config file parameter stack needs to be of type str') 95 | 96 | 97 | def test_exception_with_forbiddeng_config_argument(mocker, tmpdir, session, logger): 98 | file_name = 'test.config.yaml' 99 | with Path(tmpdir): 100 | with open(file_name, 'w') as f: 101 | f.write(yaml.dump({'stacks': 'somestack'})) 102 | with pytest.raises(SystemExit): 103 | cli.main(['stacks', '-c', file_name]) 104 | logger.error.assert_called_with('Config file parameter stacks is not supported') 105 | 106 | 107 | def test_exception_with_failed_yaml_syntax(mocker, tmpdir, session, logger): 108 | file_name = 'test.config.yaml' 109 | with Path(tmpdir): 110 | with open(file_name, 'w') as f: 111 | f.write("stacks: somestack\nprofile testprofile") 112 | with pytest.raises(SystemExit): 113 | cli.main(['stacks', '-c', file_name]) 114 | logger.error.assert_called() 115 | 116 | 117 | def test_loads_empty_config_file(mocker, tmpdir, session): 118 | stacks = mocker.patch('formica.cli.stacks') 119 | file_name = 'test.config.yaml' 120 | with Path(tmpdir): 121 | with open(file_name, 'w') as f: 122 | f.write('') 123 | cli.main(['stacks', '-c', file_name]) 124 | -------------------------------------------------------------------------------- /docs/commands/stack-set_update.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: StackSet Update 3 | weight: 100 4 | --- 5 | 6 | # `formica stack-set update` 7 | 8 | The `formica stack-set update` command allows you to update a StackSet in your current AWS account 9 | with the template in your current folder. You can use the same parameters as in create, but 10 | can also limit which accounts/regions defined in your StackSet should be updated. This is helpful 11 | for deploying to selected accounts/regions first to see if everything works out fine. 12 | 13 | ## Usage 14 | 15 | ``` 16 | usage: formica stack-set update [-h] [--region REGION] [--profile PROFILE] 17 | [--stack-set STACK-Set] 18 | [--parameters KEY=Value [KEY=Value ...]] 19 | [--main-account-parameter] 20 | [--tags KEY=Value [KEY=Value ...]] 21 | [--capabilities Cap1 Cap2 [Cap1 Cap2 ...]] 22 | [--config-file CONFIG_FILE [CONFIG_FILE ...]] 23 | [--vars KEY=Value [KEY=Value ...]] 24 | [--administration-role-arn ADMINISTRATION_ROLE_ARN] 25 | [--administration-role-name ADMINISTRATION_ROLE_NAME] 26 | [--execution-role-name EXECUTION_ROLE_NAME] 27 | [--accounts ACCOUNTS [ACCOUNTS ...]] 28 | [--regions REGIONS [REGIONS ...]] 29 | [--all-accounts] [--all-subaccounts] 30 | [--excluded-accounts EXCLUDED_ACCOUNTS [EXCLUDED_ACCOUNTS ...]] 31 | [--all-regions] 32 | [--excluded-regions EXCLUDED_REGIONS [EXCLUDED_REGIONS ...]] 33 | [--main-account] 34 | [--region-order REGION_ORDER [REGION_ORDER ...]] 35 | [--failure-tolerance-count FAILURE_TOLERANCE_COUNT | --failure-tolerance-percentage FAILURE_TOLERANCE_PERCENTAGE] 36 | [--max-concurrent-count MAX_CONCURRENT_COUNT | --max-concurrent-percentage MAX_CONCURRENT_PERCENTAGE] 37 | [--organization-variables] 38 | [--organization-region-variables] 39 | [--organization-account-variables] [--yes] 40 | [--create-missing] 41 | 42 | Update a Stack Set 43 | 44 | options: 45 | -h, --help show this help message and exit 46 | --region REGION The AWS region to use 47 | --profile PROFILE The AWS profile to use 48 | --stack-set STACK-Set, -s STACK-Set 49 | The Stack Set to use 50 | --parameters KEY=Value [KEY=Value ...] 51 | Add one or multiple stack parameters 52 | --main-account-parameter 53 | Set MainAccount Parameter 54 | --tags KEY=Value [KEY=Value ...] 55 | Add one or multiple stack tags 56 | --capabilities Cap1 Cap2 [Cap1 Cap2 ...] 57 | Set one or multiple stack capabilities 58 | --config-file CONFIG_FILE [CONFIG_FILE ...], -c CONFIG_FILE [CONFIG_FILE ...] 59 | Set the config files to use 60 | --vars KEY=Value [KEY=Value ...] 61 | Add one or multiple Jinja2 variables 62 | --administration-role-arn ADMINISTRATION_ROLE_ARN 63 | The Administration Role to create the StackSet 64 | --administration-role-name ADMINISTRATION_ROLE_NAME 65 | The Administration Role name that will be translated 66 | to the ARN 67 | --execution-role-name EXECUTION_ROLE_NAME 68 | The Execution role name to use for the CloudFormation 69 | Stack 70 | --accounts ACCOUNTS [ACCOUNTS ...] 71 | The Accounts for this operation 72 | --regions REGIONS [REGIONS ...] 73 | The Regions for this operation 74 | --all-accounts Use All Accounts of this Org 75 | --all-subaccounts Use Only Subaccounts of this Org 76 | --excluded-accounts EXCLUDED_ACCOUNTS [EXCLUDED_ACCOUNTS ...] 77 | All Accounts excluding these 78 | --all-regions Use all Regions 79 | --excluded-regions EXCLUDED_REGIONS [EXCLUDED_REGIONS ...] 80 | Excluded Regions from deployment 81 | --main-account Deploy to Main Account only 82 | --region-order REGION_ORDER [REGION_ORDER ...] 83 | Order in which to deploy to regions 84 | --failure-tolerance-count FAILURE_TOLERANCE_COUNT 85 | Number of Stacks to fail before failing operation 86 | --failure-tolerance-percentage FAILURE_TOLERANCE_PERCENTAGE 87 | Percentage of Stacks to fail before failing operation 88 | --max-concurrent-count MAX_CONCURRENT_COUNT 89 | Max Number of concurrent accounts to deploy to 90 | --max-concurrent-percentage MAX_CONCURRENT_PERCENTAGE 91 | Max Percentage of concurrent accounts to deploy to 92 | --organization-variables 93 | Add AWSAccounts, AWSSubAccounts, AWSMainAccount and 94 | AWSRegions as Jinja variables with an Email, Id and 95 | Name field for each account 96 | --organization-region-variables 97 | Add AWSRegions as Jinja variables 98 | --organization-account-variables 99 | Add AWSAccounts, AWSSubAccounts, and AWSMainAccount as 100 | Jinja variables with an Email, Id, and Name field for 101 | each account 102 | --yes, -y Answer all input questions with yes 103 | --create-missing Create the Stack in case it's missing 104 | ``` 105 | -------------------------------------------------------------------------------- /docs/modules.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Modules 3 | weight: 400 4 | --- 5 | 6 | One of the core features of Formica is its module system. The goal is to make reusing existing CloudFormation components easy. 7 | 8 | Modules are simply subfolders in the current directory. You can then either include all `template` files in that subfolder or include specific `template` files. All resources in the files you include will be automatically added to the template. 9 | 10 | To load modules simply create a Resource with a FROM attribute in your existing `template` files. As with Resources you set a `LogicalName` to that instance of the Module (e.g. in the following example we're using SomeModule). This `LogicalName` can later be used to create multiple instances of the same module as its passed into the module as a variable. It also makes the module syntax feel more like a general CloudFormation resource and helps with naming parts of the template. 11 | 12 | Example for including all `template` files in a subfolder: 13 | 14 | ```yaml 15 | Resources: 16 | SomeModule: 17 | From: ModuleDirectory 18 | ``` 19 | 20 | ```json 21 | { 22 | "Resources": { 23 | "SomeModule": { 24 | "From": "ModuleDirectory" 25 | } 26 | } 27 | } 28 | ``` 29 | 30 | This will automatically load all template files in the `moduledirectory` subfolder. Module directories and template files have to be lower cased and can't contain any whitespace. Otherwise they won't be found properly. 31 | 32 | You can also use nested directories by separating them with `::` which follows the AWS Syntax for the types. The following example will load template from `submoduledirectory` in `moduledirectory`. 33 | 34 | 35 | ```yaml 36 | Resources: 37 | SomeModule: 38 | From: ModuleDirectory::SubModuleDirectory 39 | ``` 40 | 41 | ```json 42 | { 43 | "Resources": { 44 | "SomeModule": { 45 | "From": "ModuleDirectory::SubModuleDirectory" 46 | } 47 | } 48 | } 49 | ``` 50 | 51 | ## Loading only specific templates 52 | 53 | Sometimes you don't want to load the whole module folder, but only specific templates. You can do this by adding the template name at the end of the `From` field. So if you have a file `sometemplate.template.json` in a subfolder you just add `::SomeTemplate` (upper/lower casing is irrelevant as we'll simply downcase). If you have multiple files that start with the template name, but have different extensions (e.g. `sometemplate.template.yaml` and `sometemplate.template.json`) all of them will be loaded. 54 | 55 | Following is an example in yaml and json: 56 | 57 | ```yaml 58 | Resources: 59 | SomeModule: 60 | From: ModuleDirectory::SomeTemplate 61 | ``` 62 | 63 | ```json 64 | { 65 | "Resources": { 66 | "SomeModule": { 67 | "From": "ModuleDirectory::SomeTemplate" 68 | } 69 | } 70 | } 71 | ``` 72 | 73 | Formica will check if there is a folder named `sometemplate`, if that doesn't exist it will try to find template files with that name. Make sure you don't name folders and templates the same in case you want to include them separately as folders have precedence. 74 | 75 | You can check out the [formica-modules](https://github.com/flomotlik/formica-modules) repository for official modules. 76 | 77 | ## Properties 78 | 79 | Similar to CloudFormation Resources you can also pass Properties to the modules. In the following example we're creating a Route53 HostedZone for a domain and pass the domain, target and HostedZone to a the CName file in a DNS module. 80 | 81 | ```yaml 82 | Resources: 83 | SomeCname: 84 | From: DNS::CName 85 | Properties: 86 | Source: flomotlik.me 87 | Target: somewhere.else.me 88 | HostedZone: FloMotlikHostedZone 89 | ``` 90 | 91 | In the module you can then use the variable: 92 | 93 | ```yaml 94 | {{ module_name }}RecordSet: 95 | Type: AWS::Route53::RecordSet 96 | Properties: 97 | HostedZoneName: 98 | Ref: {{ HostedZone }} 99 | Name: {{ Source }} 100 | Type: CNAME 101 | TTL: '43200' 102 | ResourceRecords: 103 | - {{ Target }} 104 | ``` 105 | 106 | ## Building reusable modules 107 | 108 | If you want your modules to be used multiple times in one template (e.g. a module for DNS handling) you need to make sure to properly name and reference your resources. In the above example we've used `module_name` to create a name for the resource that is unique. The `module_name` variable is the `LogicalName` mentioned before that we're setting when configuring a module. 109 | 110 | This makes sure that in the final CloudFormation template the Resources won't just be named `RecordSet` but `WWWRecordSet` for example if we create a cname for www. 111 | 112 | You have to use the `module_name` variable in any case you're referencing a Resource in the module as well, e.g. if you're doing `!Ref RecordSet` you actually have to do `!Ref {{ module_name }}RecordSet`. For best reusability you should also think about creating Parametes in your Module instead of relying too much on variables. This makes sure that a module doesn't depend on any formica variables, but can be set up just as a template with Parameters. 113 | 114 | For more examples check out the [formica-modules](https://github.com/flomotlik/formica-modules) repository. 115 | 116 | ## How to get modules 117 | 118 | As modules are simply subfolders of the current directory you can use any tool to add them. If you're in a git repository [`git submodule`](https://git-scm.com/docs/git-submodule) or [`git subtree`](https://blogs.atlassian.com/2013/05/alternatives-to-git-submodule-git-subtree/) are a great way to set up modules. 119 | 120 | In the following example we'll add the `formica-modules` module into your project: 121 | 122 | ``` 123 | git submodule add https://github.com/flomotlik/formica-modules modules/formica 124 | ``` 125 | 126 | You can then check out specific tags in that submodule so you're specific about which version of the modules you use. 127 | 128 | But in the end the important part is that modules are just subfolders. So any way to add them as a subfolder will work great. -------------------------------------------------------------------------------- /formica/change_set.py: -------------------------------------------------------------------------------- 1 | import json 2 | import sys 3 | 4 | import logging 5 | from formica.s3 import temporary_bucket 6 | from botocore.exceptions import ClientError, WaiterError 7 | from texttable import Texttable 8 | import boto3 9 | 10 | from formica import CHANGE_SET_FORMAT 11 | import time 12 | 13 | CHANGE_SET_HEADER = ["Action", "LogicalId", "PhysicalId", "Type", "Replacement", "Changed"] 14 | 15 | logger = logging.getLogger(__name__) 16 | 17 | cf = boto3.client("cloudformation") 18 | 19 | 20 | class ChangeSet: 21 | def create( 22 | self, 23 | template="", 24 | change_set_type="", 25 | parameters=None, 26 | tags=None, 27 | capabilities=None, 28 | role_arn=None, 29 | s3=False, 30 | resource_types=False, 31 | use_previous_template=False, 32 | use_previous_parameters=False, 33 | ): 34 | optional_arguments = {} 35 | parameters_set = [] 36 | if use_previous_parameters: 37 | stacks = cf.describe_stacks(StackName=self.stack) 38 | parameters_set = [ 39 | {"ParameterKey": p["ParameterKey"], "UsePreviousValue": True} 40 | for p in stacks["Stacks"][0]["Parameters"] 41 | ] 42 | if parameters: 43 | for key, value in parameters.items(): 44 | item = next((x for x in parameters_set if x["ParameterKey"] == key), None) 45 | values = {"ParameterKey": key, "ParameterValue": str(value), "UsePreviousValue": False} 46 | if item: 47 | item.update(values) 48 | else: 49 | parameters_set.append(values) 50 | 51 | if parameters_set: 52 | optional_arguments["Parameters"] = parameters_set 53 | 54 | if tags: 55 | optional_arguments["Tags"] = [{"Key": key, "Value": str(value)} for (key, value) in tags.items()] 56 | if role_arn: 57 | optional_arguments["RoleARN"] = role_arn 58 | if capabilities: 59 | optional_arguments["Capabilities"] = capabilities 60 | if change_set_type == "UPDATE": 61 | self.remove_existing_changeset() 62 | if resource_types: 63 | optional_arguments["ResourceTypes"] = list( 64 | set([resource["Type"] for key, resource in json.loads(template)["Resources"].items()]) 65 | ) 66 | 67 | if use_previous_template: 68 | optional_arguments["UsePreviousTemplate"] = True 69 | self.__change_and_wait(change_set_type, optional_arguments) 70 | else: 71 | if s3: 72 | with temporary_bucket(self.stack) as t: 73 | file_name = t.add(template) 74 | t.upload() 75 | bucket_name = t.name 76 | template_url = "https://{}.s3.amazonaws.com/{}".format(bucket_name, file_name) 77 | self.__change_and_wait(change_set_type, {"TemplateURL": template_url, **optional_arguments}) 78 | else: 79 | self.__change_and_wait(change_set_type, {"TemplateBody": template, **optional_arguments}) 80 | 81 | def __change_and_wait(self, change_set_type, optional_arguments): 82 | try: 83 | cf.create_change_set( 84 | StackName=self.stack, 85 | ChangeSetName=self.name, 86 | ChangeSetType=change_set_type, 87 | **optional_arguments, 88 | IncludeNestedStacks=self.nested_change_sets, 89 | ) 90 | logger.info("Change set submitted, waiting for CloudFormation to calculate changes ...") 91 | waiter = cf.get_waiter("change_set_create_complete") 92 | waiter.wait(ChangeSetName=self.name, StackName=self.stack, WaiterConfig=dict(Delay=10, MaxAttempts=120)) 93 | logger.info("Change set created successfully") 94 | except WaiterError as e: 95 | status_reason = e.last_response.get("StatusReason", "") 96 | logger.info(status_reason) 97 | if "didn't contain changes" not in status_reason: 98 | sys.exit(1) 99 | 100 | def __init__(self, stack, arn="", nested_change_sets=False): 101 | self.name = CHANGE_SET_FORMAT.format(stack=stack) 102 | self.stack = stack 103 | self.change_set_arn = arn 104 | self.nested_change_sets = nested_change_sets 105 | 106 | def describe(self, print_metadata=True): 107 | if self.change_set_arn: 108 | cs_options = dict(ChangeSetName=self.change_set_arn) 109 | else: 110 | cs_options = dict(StackName=self.stack, ChangeSetName=self.name) 111 | change_set = cf.describe_change_set(**cs_options) 112 | table = Texttable(max_width=150) 113 | 114 | if print_metadata: 115 | logger.info("Deployment metadata:") 116 | parameters = ", ".join( 117 | [ 118 | parameter["ParameterKey"] + "=" + parameter["ParameterValue"] 119 | for parameter in change_set.get("Parameters", []) 120 | ] 121 | ) 122 | table.add_row(["Parameters", parameters]) 123 | tags = [tag["Key"] + "=" + tag["Value"] for tag in change_set.get("Tags", [])] 124 | table.add_row(["Tags ", ", ".join(tags)]) 125 | table.add_row(["Capabilities ", ", ".join(change_set.get("Capabilities", []))]) 126 | logger.info(table.draw() + "\n") 127 | logger.info("Resource Changes:") 128 | 129 | table.reset() 130 | table = Texttable(max_width=150) 131 | table.add_rows([CHANGE_SET_HEADER]) 132 | 133 | def __change_detail(change): 134 | target_ = change["Target"] 135 | attribute = target_["Attribute"] 136 | if attribute == "Properties": 137 | return target_.get("Name", "") 138 | else: 139 | return attribute 140 | 141 | for change in change_set["Changes"]: 142 | resource_change = change["ResourceChange"] 143 | table.add_row( 144 | [ 145 | resource_change["Action"], 146 | resource_change["LogicalResourceId"], 147 | resource_change.get("PhysicalResourceId", ""), 148 | resource_change["ResourceType"], 149 | resource_change.get("Replacement", ""), 150 | ", ".join(sorted(set([__change_detail(c) for c in resource_change["Details"]]))), 151 | ] 152 | ) 153 | 154 | logger.info(table.draw()) 155 | 156 | nested_change_sets = [ 157 | (change["ResourceChange"]["LogicalResourceId"], change["ResourceChange"]["ChangeSetId"]) 158 | for change in change_set["Changes"] 159 | if change["ResourceChange"]["ResourceType"] == "AWS::CloudFormation::Stack" 160 | and change["ResourceChange"].get("ChangeSetId") 161 | ] 162 | for nested_change_set in nested_change_sets: 163 | logger.info(f"\nChanges for nested Stack: {nested_change_set[0]}") 164 | ChangeSet(stack=self.stack, arn=nested_change_set[1]).describe(print_metadata=False) 165 | 166 | def remove_existing_changeset(self): 167 | try: 168 | id = cf.describe_change_set(StackName=self.stack, ChangeSetName=self.name)["ChangeSetId"] 169 | logger.info("Removing existing change set") 170 | cf.delete_change_set(ChangeSetName=id) 171 | 172 | # Sleep to let Cloudformation remove 173 | for _ in range(100): 174 | cf.describe_change_set(StackName=self.stack, ChangeSetName=self.name) 175 | time.sleep(5) 176 | raise Exception("Old Change Set could not be removed, please retry") 177 | except ClientError as e: 178 | if e.response["Error"]["Code"] != "ChangeSetNotFound": 179 | raise e 180 | -------------------------------------------------------------------------------- /tests/unit/test_diff.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | import pytest 4 | import yaml 5 | from formica import cli 6 | from uuid import uuid4 7 | 8 | from formica.diff import compare_stack, compare_stack_set 9 | from tests.unit.constants import STACK 10 | 11 | 12 | @pytest.fixture 13 | def client(aws_client): 14 | aws_client.get_template.return_value = {'TemplateBody': template} 15 | aws_client.describe_stacks.return_value = {'Stacks': [{}]} 16 | return aws_client 17 | 18 | 19 | @pytest.fixture 20 | def loader(mocker): 21 | return mocker.patch('formica.diff.Loader') 22 | 23 | 24 | @pytest.fixture 25 | def template(client, mocker): 26 | template = mocker.Mock() 27 | client.get_template.return_value = {'TemplateBody': template} 28 | return template 29 | 30 | 31 | def loader_return(loader, template): 32 | loader.return_value.template_dictionary.return_value = template 33 | 34 | 35 | def template_return(client, template): 36 | client.get_template.return_value = {'TemplateBody': json.dumps(template)} 37 | 38 | 39 | def template_return_string(client, template): 40 | client.get_template.return_value = {'TemplateBody': template} 41 | 42 | 43 | def check_echo(caplog, args): 44 | assert all([arg in caplog.text for arg in args]) 45 | 46 | 47 | def check_no_echo(caplog, args): 48 | assert not any([arg in caplog.text for arg in args]) 49 | 50 | 51 | def uuid(): 52 | return str(uuid4()) 53 | 54 | 55 | def test_loads_stack_data(client, loader, mocker): 56 | compare_stack(STACK) 57 | client.get_template.assert_called_with(StackName=STACK) 58 | client.describe_stacks.assert_called_with(StackName=STACK) 59 | 60 | 61 | def test_loads_stack_set_data(client, loader): 62 | client.describe_stack_set.return_value = {'StackSet': {'TemplateBody': ''}} 63 | compare_stack_set(STACK) 64 | client.describe_stack_set.assert_called_with(StackSetName=STACK) 65 | 66 | 67 | def test_unicode_string_no_diff(loader, client, caplog): 68 | loader_return(loader, {'Resources': u'1234'}) 69 | template_return(client, {'Resources': '1234'}) 70 | compare_stack(STACK) 71 | check_no_echo(caplog, ['1234']) 72 | 73 | 74 | def test_values_changed(loader, client, caplog): 75 | loader_return(loader, {'Resources': '5678'}) 76 | template_return(client, {'Resources': '1234'}) 77 | compare_stack(STACK) 78 | check_echo(caplog, ['Resources', '1234', '5678', 'Values Changed']) 79 | 80 | 81 | def test_dictionary_item_added(loader, client, caplog): 82 | loader_return(loader, {'Resources': '5678'}) 83 | template_return(client, {}) 84 | compare_stack(STACK) 85 | check_echo(caplog, ['Resources', 'not present', '5678', 'Dictionary Item Added']) 86 | 87 | 88 | def test_dictionary_item_removed(loader, client, caplog): 89 | loader_return(loader, {}) 90 | template_return(client, {'Resources': '5678'}) 91 | compare_stack(STACK) 92 | check_echo(caplog, ['Resources', '5678', 'not present', 'Dictionary Item Removed']) 93 | 94 | 95 | def test_type_changed(loader, client, caplog): 96 | loader_return(loader, {'Resources': 5}) 97 | template_return(client, {'Resources': 'abcde'}) 98 | compare_stack(STACK) 99 | check_echo(caplog, ['Resources', 'abcde', '5', 'Type Changes']) 100 | 101 | 102 | def test_iterable_item_added(loader, client, caplog): 103 | loader_return(loader, {'Resources': [1, 2]}) 104 | template_return(client, {'Resources': [1]}) 105 | compare_stack(STACK) 106 | check_echo(caplog, ['Resources > 1', 'not present', '2', 'Iterable Item Added']) 107 | 108 | 109 | def test_iterable_item_removed(loader, client, caplog): 110 | loader_return(loader, {'Resources': [1]}) 111 | template_return(client, {'Resources': [1, 2]}) 112 | compare_stack(STACK) 113 | check_echo(caplog, ['Resources > 1', '2', 'not present', 'Iterable Item Removed']) 114 | 115 | 116 | def test_request_returns_string(loader, client, caplog): 117 | loader_return(loader, {'Resources': u'1234'}) 118 | client.get_template.return_value = {'TemplateBody': yaml.dump({'Resources': '1234'})} 119 | compare_stack(STACK) 120 | check_no_echo(caplog, ['1234']) 121 | 122 | 123 | def test_long_numbers(loader, client, caplog): 124 | id1 = '987497529474523452345234' 125 | id2 = '235462563563456345634563' 126 | loader_return(loader, {'Resources': id1}) 127 | client.get_template.return_value = {'TemplateBody': yaml.dump({'Resources': id2})} 128 | compare_stack(STACK) 129 | check_echo(caplog, [id1, id2]) 130 | 131 | 132 | def test_yaml_tags(loader, client, caplog): 133 | tagged_template = 'Resources: !Not []' 134 | loader_return(loader, {'Resources': {"Not": []}}) 135 | client.get_template.return_value = {'TemplateBody': tagged_template} 136 | compare_stack(STACK) 137 | 138 | 139 | def test_diff_cli_with_vars(template, mocker): 140 | diff = mocker.patch('formica.diff.compare_stack') 141 | cli.main(['diff', '--stack', STACK, '--vars', 'V=1', '--parameters', 'P=2', '--tags', 'T=3']) 142 | diff.assert_called_with(stack=STACK, vars={'V': '1'}, parameters={'P': '2'}, tags={'T': '3'}) 143 | 144 | 145 | def test_diff_parameters(caplog, loader, client): 146 | key = uuid() 147 | before = uuid() 148 | after = uuid() 149 | loader_return(loader, {'Resources': u'1234'}) 150 | client.get_template.return_value = {'TemplateBody': yaml.dump({'Resources': '1234'})} 151 | client.describe_stacks.return_value = { 152 | 'Stacks': [{'Parameters': [{'ParameterKey': key, 'ParameterValue': before}]}]} 153 | compare_stack(STACK, parameters={key: after}) 154 | check_echo(caplog, [key, before, after, 'Values Changed']) 155 | 156 | 157 | def test_diff_parameters_overrides_defaults(caplog, loader, client): 158 | key = uuid() 159 | before = uuid() 160 | after = uuid() 161 | key2 = uuid() 162 | template = {'Resources': '1234', 'Parameters': {key: {'Default': after}, key2: {'Default': after}}} 163 | loader_return(loader, template) 164 | client.get_template.return_value = {'TemplateBody': json.dumps(template)} 165 | client.describe_stacks.return_value = { 166 | 'Stacks': [{'Parameters': [{'ParameterKey': key, 'ParameterValue': before}, 167 | {'ParameterKey': key2, 'ParameterValue': before}]}]} 168 | compare_stack(STACK, parameters={key2: before}) 169 | check_echo(caplog, [key, before, after, 'Values Changed']) 170 | check_no_echo(caplog, [key2]) 171 | 172 | 173 | def test_diff_tags(caplog, loader, client): 174 | key = uuid() 175 | before = uuid() 176 | after = uuid() 177 | loader_return(loader, {'Resources': u'1234'}) 178 | client.get_template.return_value = {'TemplateBody': yaml.dump({'Resources': '1234'})} 179 | client.describe_stacks.return_value = { 180 | 'Stacks': [{'Tags': [{'Key': key, 'Value': before}]}]} 181 | compare_stack(STACK, tags={key: after}) 182 | check_echo(caplog, [key, before, after, 'Values Changed']) 183 | 184 | 185 | def test_diff__on_stack_set(caplog, loader, client): 186 | template = (uuid(), uuid()) 187 | tag_key = uuid() 188 | tag_before = uuid() 189 | tag_after = uuid() 190 | parameter_key = uuid() 191 | parameter_before = uuid() 192 | parameter_after = uuid() 193 | loader_return(loader, {'Resources': template[0]}) 194 | client.get_template.return_value = {} 195 | client.describe_stack_set.return_value = { 196 | 'StackSet': { 197 | 'Parameters': [{'ParameterKey': parameter_key, 'ParameterValue': parameter_before}], 198 | 'Tags': [{'Key': tag_key, 'Value': tag_before}], 199 | 'TemplateBody': yaml.dump({'Resources': template[1]}) 200 | } 201 | } 202 | compare_stack_set(STACK, parameters={parameter_key: parameter_after}, tags={tag_key: tag_after}) 203 | check_echo(caplog, [parameter_key, parameter_before, parameter_after, 'Values Changed']) 204 | check_echo(caplog, [tag_key, tag_before, tag_after, 'Values Changed']) 205 | check_echo(caplog, ['Resources', template[0], template[1], 'Values Changed']) 206 | check_no_echo(caplog, ['No Changes found']) 207 | -------------------------------------------------------------------------------- /tests/unit/test_template.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import yaml 4 | import pytest 5 | from path import Path 6 | 7 | from formica import cli 8 | from .constants import ACCOUNTS, EC2_REGIONS 9 | 10 | 11 | @pytest.fixture 12 | def logger(mocker): 13 | return mocker.patch('formica.cli.logger') 14 | 15 | 16 | def test_template_calls_template(tmpdir, logger): 17 | with Path(tmpdir): 18 | with open('test.template.json', 'w') as f: 19 | f.write('{"Description": "{{ \'test\' | title }}"}') 20 | cli.main(['template']) 21 | logger.info.assert_called() 22 | assert {"Description": "Test"} == json.loads(logger.info.call_args[0][0]) 23 | 24 | 25 | def test_template_calls_template_with_yaml(tmpdir, logger): 26 | with Path(tmpdir): 27 | with open('test.template.json', 'w') as f: 28 | f.write('{"Description": "{{ \'test\' | title }}"}') 29 | cli.main(['template', '--yaml']) 30 | logger.info.assert_called() 31 | assert {"Description": "Test"} == yaml.safe_load(logger.info.call_args[0][0]) 32 | 33 | 34 | def test_with_organization_variables(aws_client, tmpdir, logger, paginators): 35 | aws_client.get_paginator.side_effect = paginators(list_accounts=[ACCOUNTS]) 36 | aws_client.describe_regions.return_value = EC2_REGIONS 37 | aws_client.get_caller_identity.return_value = {'Account': '1234'} 38 | example = '{"Resources": {"ModuleAccounts": {{ AWSAccounts | tojson }}, "ModuleSubAccounts": {{ AWSSubAccounts | tojson }}, "ModuleRegions": {{ AWSRegions | tojson }}, "ModuleMainAccount": {{ AWSMainAccount | tojson }}}}' 39 | with Path(tmpdir): 40 | os.mkdir('moduledir') 41 | with open('moduledir/test.template.json', 'w') as f: 42 | f.write(example) 43 | with open('test.template.json', 'w') as f: 44 | f.write( 45 | '{"Resources": {"AccountsRegionsTest": {"From": "Moduledir"}, "Accounts": {{ AWSAccounts | tojson}}, "SubAccounts": {{ AWSSubAccounts | tojson}}, "Regions": {{ AWSRegions | tojson }} }}') 46 | cli.main(['template', '--organization-variables']) 47 | logger.info.assert_called() 48 | output = logger.info.call_args[0][0] 49 | 50 | actual = yaml.safe_load(output) 51 | expected = {'Resources': {'Accounts': [{'Email': 'email1@test.com', 'Id': '1234', 'Name': 'TestName1'}, 52 | {'Email': 'email2@test.com', 'Id': '5678', 'Name': 'TestName2'}], 53 | 'SubAccounts': [{'Email': 'email2@test.com', 'Id': '5678', 'Name': 'TestName2'}], 54 | 'ModuleAccounts': [{'Email': 'email1@test.com', 'Id': '1234', 'Name': 'TestName1'}, 55 | {'Email': 'email2@test.com', 'Id': '5678', 'Name': 'TestName2'}], 56 | 'ModuleSubAccounts': [ 57 | {'Email': 'email2@test.com', 'Id': '5678', 'Name': 'TestName2'}], 58 | 'ModuleMainAccount': {'Email': 'email1@test.com', 59 | 'Id': '1234', 60 | 'Name': 'TestName1'}, 61 | 'ModuleRegions': ['us-west-1', 'us-west-2'], 'Regions': ['us-west-1', 'us-west-2']}} 62 | assert actual == expected 63 | 64 | 65 | def test_with_organization_region_variables_no_account_variables(aws_client, tmpdir, logger, paginators): 66 | aws_client.get_paginator.side_effect = paginators(list_accounts=[ACCOUNTS]) 67 | aws_client.describe_regions.return_value = EC2_REGIONS 68 | aws_client.get_caller_identity.return_value = {'Account': '1234'} 69 | example = '{"Resources": {"Accounts": {{ AWSAccounts | tojson }}, "SubAccounts": {{ AWSSubAccounts | tojson }}, "Regions": {{ AWSRegions | tojson }}, "MainAccount": {{ AWSMainAccount | tojson }}}}' 70 | with Path(tmpdir): 71 | with open('test.template.json', 'w') as f: 72 | f.write(example) 73 | 74 | # CLI now raises a TypeError because {{AWSAccounts}} and other account variables are null and cannot be json-serialized 75 | with pytest.raises(TypeError): 76 | cli.main(['template', '--organization-region-variables']) 77 | 78 | 79 | def test_with_organization_region_variables(aws_client, tmpdir, logger, paginators): 80 | aws_client.get_paginator.side_effect = paginators(list_accounts=[ACCOUNTS]) 81 | aws_client.describe_regions.return_value = EC2_REGIONS 82 | aws_client.get_caller_identity.return_value = {'Account': '1234'} 83 | example = '{"Resources": {"Regions": {{ AWSRegions | tojson }}}}' 84 | with Path(tmpdir): 85 | with open('test.template.json', 'w') as f: 86 | f.write(example) 87 | 88 | cli.main(['template', '--organization-region-variables']) 89 | logger.info.assert_called() 90 | output = logger.info.call_args[0][0] 91 | 92 | actual = yaml.safe_load(output) 93 | expected = {'Resources': {'Regions': ['us-west-1', 'us-west-2'], 'Regions': ['us-west-1', 'us-west-2']}} 94 | assert actual == expected 95 | 96 | 97 | def test_with_organization_account_variables_no_region_variables(aws_client, tmpdir, logger, paginators): 98 | aws_client.get_paginator.side_effect = paginators(list_accounts=[ACCOUNTS]) 99 | aws_client.describe_regions.return_value = EC2_REGIONS 100 | aws_client.get_caller_identity.return_value = {'Account': '1234'} 101 | example = '{"Resources": {"Accounts": {{ AWSAccounts | tojson }}, "SubAccounts": {{ AWSSubAccounts | tojson }}, "Regions": {{ AWSRegions | tojson }}, "MainAccount": {{ AWSMainAccount | tojson }}}}' 102 | with Path(tmpdir): 103 | with open('test.template.json', 'w') as f: 104 | f.write(example) 105 | 106 | # CLI now raises a TypeError because {{AWSRegions}} is null and cannot be json-serialized 107 | with pytest.raises(TypeError): 108 | cli.main(['template', '--organization-account-variables']) 109 | 110 | 111 | def test_with_organization_account_variables(aws_client, tmpdir, logger, paginators): 112 | aws_client.get_paginator.side_effect = paginators(list_accounts=[ACCOUNTS]) 113 | aws_client.describe_regions.return_value = EC2_REGIONS 114 | aws_client.get_caller_identity.return_value = {'Account': '1234'} 115 | example = '{"Resources": {"Accounts": {{ AWSAccounts | tojson }}, "SubAccounts": {{ AWSSubAccounts | tojson }}, "MainAccount": {{ AWSMainAccount | tojson }}}}' 116 | with Path(tmpdir): 117 | with open('test.template.json', 'w') as f: 118 | f.write(example) 119 | 120 | cli.main(['template', '--organization-account-variables']) 121 | logger.info.assert_called() 122 | output = logger.info.call_args[0][0] 123 | 124 | actual = yaml.safe_load(output) 125 | expected = {'Resources': {'Accounts': [{'Email': 'email1@test.com', 'Id': '1234', 'Name': 'TestName1'}, 126 | {'Email': 'email2@test.com', 'Id': '5678', 'Name': 'TestName2'}], 127 | 'SubAccounts': [{'Email': 'email2@test.com', 'Id': '5678', 'Name': 'TestName2'}], 128 | 'MainAccount': {'Email': 'email1@test.com', 'Id': '1234', 'Name': 'TestName1'}}} 129 | assert actual == expected 130 | 131 | 132 | def test_with_artifacts(aws_client, tmpdir, logger, paginators): 133 | aws_client.get_paginator.side_effect = paginators(list_accounts=[ACCOUNTS]) 134 | aws_client.describe_regions.return_value = EC2_REGIONS 135 | aws_client.meta.region_name = "eu-central-1" 136 | aws_client.get_caller_identity.return_value = {'Account': '1234'} 137 | example = '{"Resources": {"Bucket": {{ artifacts["bucketfile"].bucket }}, "Key": {{ artifacts["bucketfile"].key }} }}' 138 | with Path(tmpdir): 139 | with open('test.template.json', 'w') as f: 140 | f.write(example) 141 | with open('bucketfile', 'w') as f: 142 | f.write("Testfile") 143 | 144 | cli.main(['template', '--artifacts', 'bucketfile']) 145 | logger.info.assert_called() 146 | output = logger.info.call_args[0][0] 147 | 148 | actual = yaml.safe_load(output) 149 | expected = {"Resources": {"Bucket": "formica-deploy-83acc03037c35fdce1aae77faa87d9f2", "Key": "864c71d530a42421476458005e05b2a0" }} 150 | assert actual == expected 151 | -------------------------------------------------------------------------------- /formica/loader.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import sys 4 | import re 5 | 6 | import logging 7 | import yaml 8 | from jinja2 import Environment, FileSystemLoader 9 | from jinja2.exceptions import TemplateSyntaxError, TemplateNotFound, UndefinedError 10 | import arrow 11 | import fnmatch 12 | 13 | from .exceptions import FormicaArgumentException 14 | 15 | from . import yaml_tags 16 | from .helper import main_account_id 17 | 18 | # To silence pyflakes warning of unused import 19 | assert yaml_tags 20 | 21 | logger = logging.getLogger(__name__) 22 | 23 | FILE_TYPES = ["yml", "yaml", "json"] 24 | 25 | RESOURCES_KEY = "Resources" 26 | MODULE_KEY = "From" 27 | 28 | ALLOWED_ATTRIBUTES = { 29 | "AWSTemplateFormatVersion": [str], 30 | "Description": [str], 31 | "Metadata": dict, 32 | "Parameters": dict, 33 | "Mappings": dict, 34 | "Conditions": dict, 35 | "Transform": [str, list], 36 | RESOURCES_KEY: dict, 37 | "Outputs": dict, 38 | } 39 | 40 | 41 | def code_escape(source): 42 | return source.replace("\n", "\\n").replace('"', '\\"') 43 | 44 | 45 | def code_array(source): 46 | lines = source.split("\\n") 47 | return "[" + ",".join(['"%s"' % line for line in lines]) + "]" 48 | 49 | 50 | def mandatory(a): 51 | from jinja2.runtime import Undefined 52 | 53 | if isinstance(a, Undefined) or a is None: 54 | raise FormicaArgumentException("Mandatory variable not set.") 55 | return a 56 | 57 | 58 | def resource(name): 59 | if name is None: 60 | return "" 61 | else: 62 | return "".join(e for e in name.title() if e.isalnum()) 63 | 64 | 65 | def novalue(variable): 66 | return False if variable is False else 0 if variable == 0 else (variable or '{"Ref": "AWS::NoValue"}') 67 | 68 | 69 | class Loader(object): 70 | def __init__(self, path=".", filename="*", variables=None, main_account_parameter=False): 71 | if variables is None: 72 | variables = {} 73 | self.cftemplate = {} 74 | self.path = path 75 | self.filename = filename 76 | self.env = Environment(loader=FileSystemLoader("./", followlinks=True)) 77 | self.env.filters.update( 78 | { 79 | "code_escape": code_escape, 80 | "code_array": code_array, 81 | "mandatory": mandatory, 82 | "resource": resource, 83 | "novalue": novalue, 84 | } 85 | ) 86 | self.variables = variables 87 | self.main_account_parameter = main_account_parameter 88 | 89 | def include_file(self, filename, **args): 90 | source = self.render(filename, **args) 91 | return code_escape(source) 92 | 93 | def load_file(self, filename, **args): 94 | return self.render(filename, **args) 95 | 96 | def list_files(self, filter="*"): 97 | return [t for t in self.env.list_templates() if fnmatch.fnmatch(t, filter)] 98 | 99 | def render(self, filename, **args): 100 | template_path = os.path.normpath("{}/{}".format(self.path, filename)) 101 | template = self.env.get_template(template_path) 102 | variables = {} 103 | variables.update(self.variables) 104 | variables.update(args) 105 | arguments = dict( 106 | code=self.include_file, 107 | file=self.load_file, 108 | files=self.list_files, 109 | now=arrow.now, 110 | utcnow=arrow.utcnow, 111 | **variables, 112 | ) 113 | return template.render(**arguments) 114 | 115 | def template(self, indent=4, sort_keys=True, separators=(",", ":"), dumper=None): 116 | if dumper is not None: 117 | return dumper(self.cftemplate) 118 | return json.dumps(self.cftemplate, indent=indent, sort_keys=sort_keys, separators=separators) 119 | 120 | def template_dictionary(self): 121 | return self.cftemplate 122 | 123 | def merge(self, template, file): 124 | if template: 125 | for key in template.keys(): 126 | new = template[key] 127 | new_type = type(new) 128 | types = ALLOWED_ATTRIBUTES.get(key) 129 | if type(types) != list: 130 | types = [types] 131 | if key in ALLOWED_ATTRIBUTES.keys() and new_type in types: 132 | if new_type == str or new_type == list: 133 | self.cftemplate[key] = new 134 | elif new_type == dict: 135 | for element_key, element_value in template[key].items(): 136 | if ( 137 | key == RESOURCES_KEY 138 | and isinstance(element_value, dict) 139 | and MODULE_KEY in element_value 140 | ): 141 | self.load_module(element_value[MODULE_KEY], element_key, element_value) 142 | else: 143 | self.cftemplate.setdefault(key, {})[element_key] = element_value 144 | else: 145 | logger.info("Key '{}' in file {} is not valid".format(key, file)) 146 | sys.exit(1) 147 | else: 148 | logger.info("File {} is empty".format(file)) 149 | 150 | def load_module(self, module_path, element_key, element_value): 151 | path_elements = module_path.split("::") 152 | dir_pattern = re.compile(fnmatch.translate(path_elements.pop(0)), re.IGNORECASE) 153 | matched_dirs = [dir for dir in os.listdir(self.path) if dir_pattern.match(dir)] 154 | matched_dir = module_path 155 | if matched_dirs: 156 | matched_dir = matched_dirs[0] 157 | 158 | module_path = self.path + "/" + "/".join([matched_dir] + path_elements) 159 | 160 | file_name = "*" 161 | 162 | if not os.path.isdir(module_path): 163 | file_name = module_path.split("/")[-1] 164 | module_path = "/".join(module_path.split("/")[:-1]) 165 | 166 | properties = element_value.get("Properties", {}) 167 | properties["module_name"] = element_key 168 | vars = self.merge_variables(properties) 169 | 170 | loader = Loader(module_path, file_name, vars) 171 | loader.load() 172 | self.merge(loader.template_dictionary(), file=file_name) 173 | 174 | def merge_variables(self, module_vars): 175 | merged_vars = {} 176 | for k, v in self.variables.items(): 177 | merged_vars[k] = v 178 | for k, v in module_vars.items(): 179 | merged_vars[k] = v 180 | return merged_vars 181 | 182 | def load(self): 183 | files = [] 184 | 185 | for file_type in FILE_TYPES: 186 | pattern = re.compile(fnmatch.translate("{}.template.{}".format(self.filename, file_type)), re.IGNORECASE) 187 | files.extend([filename for filename in os.listdir(self.path) if pattern.match(filename)]) 188 | 189 | if not files: 190 | logger.info("Could not find any template files in {}".format(self.path)) 191 | sys.exit(1) 192 | 193 | for file in files: 194 | try: 195 | result = str(self.render(os.path.basename(file), **self.variables)) 196 | template = yaml.full_load(result) 197 | except TemplateNotFound as e: 198 | logger.info("File not found" + ": " + e.message) 199 | logger.info('In: "' + file + '"') 200 | sys.exit(1) 201 | except TemplateSyntaxError as e: 202 | logger.info(e.__class__.__name__ + ": " + e.message) 203 | logger.info('File: "' + (e.filename or file) + '", line ' + str(e.lineno)) 204 | sys.exit(1) 205 | except UndefinedError as e: 206 | logger.info(e.__class__.__name__ + ": " + e.message) 207 | logger.info('In: "' + file + '"') 208 | sys.exit(1) 209 | except FormicaArgumentException as e: 210 | logger.info(e.__class__.__name__ + ": " + e.args[0]) 211 | logger.info('For Template: "' + file + '"') 212 | logger.info("If you use it as a template make sure you're setting all necessary vars") 213 | sys.exit(1) 214 | except yaml.YAMLError as e: 215 | logger.info(e.__str__()) 216 | logger.info("Following is the Yaml document formica is trying to load:") 217 | logger.info("---------------------------------------------------------------------------") 218 | logger.info(result) 219 | logger.info("---------------------------------------------------------------------------") 220 | sys.exit(1) 221 | self.merge(template, file) 222 | 223 | if self.main_account_parameter: 224 | self.cftemplate["Parameters"] = self.cftemplate.get("Parameters") or {} 225 | self.cftemplate["Parameters"]["MainAccount"] = {"Type": "String", "Default": main_account_id()} 226 | --------------------------------------------------------------------------------