├── doc ├── .nojekyll ├── pro.gif ├── favicon.ico ├── mutato.png ├── img │ └── bg-pattern.png ├── mutato-transparent.png ├── _coverpage.md ├── _sidebar.md ├── css │ └── style.css ├── mutato-cdk.md ├── mutato-docker.md ├── index.html ├── mutato-architecture.md ├── mutato-vs-mu.md ├── mutato-workflow.md ├── mutato-concepts.md ├── README.md └── mutato-yaml.md ├── cdk.json ├── .eslintignore ├── lib ├── actions │ ├── index.ts │ ├── interface.ts │ ├── approval.ts │ ├── docker.ts │ └── codebuild.ts ├── parser │ ├── index.ts │ ├── loader.ts │ ├── parser.ts │ └── preprocessor.ts ├── index.ts ├── resources │ ├── index.ts │ ├── network.ts │ ├── storage.ts │ ├── database.ts │ ├── container.ts │ └── service.ts ├── config.ts └── app.ts ├── .dockerignore ├── .vscode ├── settings.json ├── extensions.json └── launch.json ├── .npmignore ├── test ├── sanity.test.ts ├── parser.test.ts └── container.test.ts ├── deploy.sh ├── bin └── mutato.ts ├── jsdoc2md.json ├── patches ├── @aws-cdk+aws-codebuild+1.32.0.patch └── @aws-cdk+app-delivery+1.32.0.patch ├── Dockerfile ├── tsconfig.json ├── .github └── workflows │ └── main.yml ├── mutato.yml ├── LICENSE ├── README.md ├── .devcontainer ├── devcontainer.json └── Dockerfile ├── .gitignore ├── CONTRIBUTING.md └── package.json /doc/.nojekyll: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /cdk.json: -------------------------------------------------------------------------------- 1 | { 2 | "app": "npx ts-node bin/mutato.ts" 3 | } 4 | -------------------------------------------------------------------------------- /.eslintignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | coverage 3 | docs 4 | **/*.d.ts 5 | -------------------------------------------------------------------------------- /doc/pro.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stelligent/mutato/HEAD/doc/pro.gif -------------------------------------------------------------------------------- /doc/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stelligent/mutato/HEAD/doc/favicon.ico -------------------------------------------------------------------------------- /doc/mutato.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stelligent/mutato/HEAD/doc/mutato.png -------------------------------------------------------------------------------- /doc/img/bg-pattern.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stelligent/mutato/HEAD/doc/img/bg-pattern.png -------------------------------------------------------------------------------- /doc/mutato-transparent.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stelligent/mutato/HEAD/doc/mutato-transparent.png -------------------------------------------------------------------------------- /lib/actions/index.ts: -------------------------------------------------------------------------------- 1 | export * from './approval'; 2 | export * from './codebuild'; 3 | export * from './docker'; 4 | -------------------------------------------------------------------------------- /lib/parser/index.ts: -------------------------------------------------------------------------------- 1 | export * from './loader'; 2 | export * from './parser'; 3 | export * from './preprocessor'; 4 | -------------------------------------------------------------------------------- /lib/index.ts: -------------------------------------------------------------------------------- 1 | export * from './app'; 2 | export * from './config'; 3 | export * from './parser'; 4 | export * from './resources'; 5 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | **/* 2 | !lib 3 | !bin 4 | !doc 5 | !tsconfig.json 6 | !package.json 7 | !package-lock.json 8 | !cdk.json 9 | !jsdoc2md.json 10 | !.gitignore 11 | !patches 12 | -------------------------------------------------------------------------------- /lib/resources/index.ts: -------------------------------------------------------------------------------- 1 | export * from './container'; 2 | export * from './database'; 3 | export * from './network'; 4 | export * from './service'; 5 | export * from './storage'; 6 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "eslint.enable": true, 3 | "editor.tabSize": 2, 4 | "editor.rulers": [80], 5 | "editor.codeActionsOnSave": { 6 | "source.fixAll.eslint": true 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | .devcontainer 2 | .github 3 | .vscode 4 | cdk.out 5 | dist 6 | test 7 | .dockerignore 8 | .eslintignore 9 | .gitignore 10 | CONTRIBUTING.md 11 | Dockerfile 12 | jsdoc2md.json 13 | mutato.yml 14 | -------------------------------------------------------------------------------- /doc/_coverpage.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ![](mutato-transparent.png) 4 | 5 | # Mutato 2.0 6 | 7 | a tool for managing your microservices platform 8 | 9 | [GitHub](https://github.com/stelligent/mutato) 10 | [Get Started](#mutato) 11 | -------------------------------------------------------------------------------- /lib/actions/interface.ts: -------------------------------------------------------------------------------- 1 | import * as codePipeline from '@aws-cdk/aws-codepipeline'; 2 | 3 | export interface ActionPropsInterface { 4 | readonly name: string; 5 | } 6 | 7 | export interface ActionInterface { 8 | readonly name: string; 9 | action(suffix: string): codePipeline.IAction; 10 | } 11 | -------------------------------------------------------------------------------- /.vscode/extensions.json: -------------------------------------------------------------------------------- 1 | { 2 | "recommendations": [ 3 | "christian-kohler.npm-intellisense", 4 | "DavidAnson.vscode-markdownlint", 5 | "dbaeumer.vscode-eslint", 6 | "eg2.vscode-npm-script", 7 | "esbenp.prettier-vscode", 8 | "ms-vscode.vscode-typescript-tslint-plugin", 9 | "stkb.rewrap", 10 | "yzhang.markdown-all-in-one" 11 | ] 12 | } 13 | -------------------------------------------------------------------------------- /test/sanity.test.ts: -------------------------------------------------------------------------------- 1 | import chai from 'chai'; 2 | import chaiAsPromised from 'chai-as-promised'; 3 | 4 | chai.use(chaiAsPromised); 5 | 6 | describe('Sanity Check', () => { 7 | it('should always pass sync', () => { 8 | chai.assert(2 + 2 === 4); 9 | }); 10 | 11 | it('should always pass async', async () => { 12 | await chai.assert.isFulfilled(Promise.resolve()); 13 | await chai.assert.isRejected(Promise.reject()); 14 | }); 15 | }); 16 | -------------------------------------------------------------------------------- /deploy.sh: -------------------------------------------------------------------------------- 1 | #! bash -eux 2 | 3 | VAULT_PROFILE=${1-"mutato-dev"} 4 | docker build --no-cache -t stelligent/mutato . 5 | echo "NPM_TOKEN=$NPM_TOKEN" > mutato.env 6 | echo "GITHUB_TOKEN=$GITHUB_TOKEN" >> mutato.env 7 | echo "DOCKER_USERNAME=$DOCKER_USERNAME" >> mutato.env 8 | echo "DOCKER_PASSWORD=$DOCKER_PASSWORD" >> mutato.env 9 | aws-vault exec $VAULT_PROFILE -- env | grep AWS_ >> mutato.env 10 | docker run -it --rm --env-file mutato.env -v `pwd`:/project stelligent/mutato deploy 11 | -------------------------------------------------------------------------------- /bin/mutato.ts: -------------------------------------------------------------------------------- 1 | import debug from 'debug'; 2 | import 'source-map-support/register'; 3 | import * as mutato from '../lib'; 4 | 5 | const _debug = debug('mutato'); 6 | 7 | (async (): Promise => { 8 | _debug('creating a new Mutato App'); 9 | const app = new mutato.App(); 10 | _debug('synthesizing Mutato App'); 11 | await app.synthesizeFromFile(); 12 | })() 13 | .then(() => { 14 | _debug('synthesized with Mutato.'); 15 | }) 16 | .catch((err) => { 17 | _debug('failed to deploy with Mutato: %o', err); 18 | process.exit(1); 19 | }); 20 | -------------------------------------------------------------------------------- /doc/_sidebar.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | - [home](/) 4 | - [concepts](mutato-concepts.md 'Mutato Concepts') 5 | - [workflow](mutato-workflow.md 'Deploy workflow') 6 | - [architecture](mutato-architecture.md 'Pipeline Architecture') 7 | - [mutato.yml](mutato-yaml.md 'YAML Reference specification') 8 | - [mutato container](mutato-docker.md 'Docker Container Documentation') 9 | - [extending mutato](mutato-cdk.md 'Extending Mutato in CDK') 10 | - [mutato vs mu](mutato-vs-mu.md 'Mutato Compared to OG Stelligent Mu') 11 | - [API](api.md 'Low level CDK API documentation') 12 | -------------------------------------------------------------------------------- /jsdoc2md.json: -------------------------------------------------------------------------------- 1 | { 2 | "source": { 3 | "includePattern": ".+\\.ts(doc|x)?$", 4 | "excludePattern": ".+\\.(test|spec).ts" 5 | }, 6 | "plugins": ["plugins/markdown", "node_modules/jsdoc-babel"], 7 | "babel": { 8 | "extensions": ["ts", "tsx"], 9 | "ignore": ["**/*.(test|spec).ts"], 10 | "babelrc": false, 11 | "presets": [ 12 | ["@babel/preset-env", { "targets": { "node": true } }], 13 | "@babel/preset-typescript" 14 | ], 15 | "plugins": [ 16 | "@babel/proposal-class-properties", 17 | "@babel/proposal-object-rest-spread" 18 | ] 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /patches/@aws-cdk+aws-codebuild+1.32.0.patch: -------------------------------------------------------------------------------- 1 | diff --git a/node_modules/@aws-cdk/aws-codebuild/lib/project.js b/node_modules/@aws-cdk/aws-codebuild/lib/project.js 2 | index 059b726..967645d 100644 3 | --- a/node_modules/@aws-cdk/aws-codebuild/lib/project.js 4 | +++ b/node_modules/@aws-cdk/aws-codebuild/lib/project.js 5 | @@ -966,7 +966,7 @@ var BuildEnvironmentVariableType; 6 | /** 7 | * An environment variable in plaintext format. 8 | */ 9 | - BuildEnvironmentVariableType["PLAINTEXT"] = "PLAINTEXT"; 10 | + //BuildEnvironmentVariableType["PLAINTEXT"] = "PLAINTEXT"; 11 | /** 12 | * An environment variable stored in Systems Manager Parameter Store. 13 | */ 14 | -------------------------------------------------------------------------------- /lib/parser/loader.ts: -------------------------------------------------------------------------------- 1 | import debug from 'debug'; 2 | import assert from 'assert'; 3 | import _ from 'lodash'; 4 | import yaml from 'yaml'; 5 | 6 | const _debug = debug('mutato:parser:Loader'); 7 | 8 | /** single document YAML loader */ 9 | export class Loader { 10 | /** 11 | * @param input a single-document YAML string 12 | * @returns YAML document in the form of single JSON object 13 | */ 14 | public load(input: string): object { 15 | _debug('loading input YAML: %s', input); 16 | const document = yaml.parseDocument(input); 17 | _debug('parsed document: %o', document); 18 | assert.ok(_.isEmpty(document.errors), 'failed to parse the input YAML'); 19 | return document.toJSON(); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /doc/css/style.css: -------------------------------------------------------------------------------- 1 | /* COVER */ 2 | section.cover { 3 | color: #fff; 4 | background: url(../img/bg-pattern.png), linear-gradient(to left, #f4842b, #7b4397) !important; 5 | } 6 | 7 | section.cover h1 { 8 | font-size: 5rem; 9 | font-weight: 600; 10 | } 11 | 12 | section.cover h1 a span { 13 | color: #fff; 14 | } 15 | 16 | /* Cover buttons */ 17 | 18 | section.cover .cover-main > p:last-child a { 19 | border: 1px solid #fff; 20 | color: #fff; 21 | } 22 | section.cover .cover-main > p:last-child a:last-child { 23 | background-color: #fff; 24 | color: var(--theme-color,#fff); 25 | } 26 | section.cover .cover-main > p:last-child a:last-child:hover { 27 | color: var(--theme-color,#fff); 28 | opacity: 0.8; 29 | } 30 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:lts 2 | 3 | ADD . /mutato 4 | WORKDIR /mutato 5 | 6 | ENV DEBUG=mutato* 7 | ENV DEBUG_COLORS=0 8 | ENV USER=root 9 | ENV TEMP=/tmp 10 | ENV HOME=/home/root 11 | ENV DEBIAN_FRONTEND=noninteractive 12 | 13 | RUN mkdir -p ${HOME} && chmod a+rwx ${TEMP} ${HOME} \ 14 | && apt-get update -qq \ 15 | && apt-get upgrade -y \ 16 | && apt-get install -y --no-install-recommends git dumb-init \ 17 | && git config --global user.email "support@stelligent.com" \ 18 | && git config --global user.name "mutato-docker" \ 19 | && npm install --allow-root --unsafe-perm && npm run build \ 20 | && rm -rf /var/lib/apt/lists/* 21 | 22 | ENV mutato_opts__git__local=/project 23 | 24 | ENTRYPOINT ["/usr/bin/dumb-init", "--", "/usr/local/bin/npm", "run"] 25 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES2018", 4 | "module": "commonjs", 5 | "lib": ["es2018"], 6 | "declaration": true, 7 | "strict": true, 8 | "noImplicitAny": true, 9 | "strictNullChecks": true, 10 | "noImplicitThis": true, 11 | "alwaysStrict": true, 12 | "noUnusedLocals": false, 13 | "noUnusedParameters": false, 14 | "noImplicitReturns": true, 15 | "noFallthroughCasesInSwitch": false, 16 | "inlineSourceMap": true, 17 | "inlineSources": true, 18 | "experimentalDecorators": true, 19 | "strictPropertyInitialization": false, 20 | "typeRoots": ["./node_modules/@types"], 21 | "resolveJsonModule": true, 22 | "esModuleInterop": true, 23 | "outDir": "out", 24 | "allowJs": true 25 | }, 26 | "exclude": ["cdk.out", "out", "dist", "docs"] 27 | } 28 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "0.2.0", 3 | "configurations": [ 4 | { 5 | "type": "node", 6 | "request": "launch", 7 | "name": "Debug Current Mocha Test", 8 | "program": "${workspaceFolder}/node_modules/mocha/bin/_mocha", 9 | "args": ["--timeout", "0", "--colors", "${file}"], 10 | "internalConsoleOptions": "openOnSessionStart", 11 | "skipFiles": ["/**"] 12 | }, 13 | { 14 | "type": "node", 15 | "request": "launch", 16 | "name": "Debug EntryPoint", 17 | "program": "${workspaceFolder}/bin/mutato.ts", 18 | "preLaunchTask": "npm: build", 19 | "outFiles": ["${workspaceFolder}/out/**/*.js"], 20 | "skipFiles": ["/**"], 21 | "cwd": "${workspaceFolder}", 22 | "env": { "DEBUG": "mutato*" }, 23 | "console": "integratedTerminal" 24 | } 25 | ] 26 | } 27 | -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: Docs 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | 8 | jobs: 9 | build: 10 | runs-on: ubuntu-latest 11 | 12 | steps: 13 | - uses: actions/checkout@v2 14 | - name: Setup Node 15 | uses: actions/setup-node@v1 16 | with: 17 | node-version: 12.x 18 | - name: Cache Modules 19 | uses: actions/cache@v1 20 | with: 21 | path: ~/.npm # npm cache files are stored in `~/.npm` on Linux/macOS 22 | key: ${{ runner.OS }}-node-${{ hashFiles('**/package-lock.json') }} 23 | restore-keys: | 24 | ${{ runner.OS }}-node- 25 | ${{ runner.OS }}- 26 | - run: npm install 27 | - name: Deploy 28 | uses: JamesIves/github-pages-deploy-action@releases/v3 29 | with: 30 | ACCESS_TOKEN: ${{ secrets.GITHUB_TOKEN }} 31 | BRANCH: gh-pages 32 | FOLDER: doc 33 | -------------------------------------------------------------------------------- /mutato.yml: -------------------------------------------------------------------------------- 1 | {% set productionBranch = "master" %} 2 | {% set imageTag = "latest" if git.branch == productionBranch else git.branch %} 3 | 4 | containers: 5 | - docker: 6 | name: mutato 7 | file: Dockerfile 8 | uri: stelligent/mutato:{{ imageTag }} 9 | events: 10 | pre-build: npm-test 11 | {% if git.branch == productionBranch %} 12 | post-build: npm-deploy 13 | {% endif %} 14 | 15 | actions: 16 | - docker: 17 | name: npm-test 18 | container: node:lts 19 | cmd: 20 | - npm install --allow-root --unsafe-perm 21 | - USER=root npm test 22 | 23 | {% if git.branch == productionBranch %} 24 | - docker: 25 | name: npm-deploy 26 | container: node:lts 27 | cmd: 28 | - npm install --allow-root --unsafe-perm 29 | - npm run build 30 | - echo //registry.npmjs.org/:_authToken={{ env("NPM_TOKEN") }} > .npmrc 31 | - npm publish --access=public 32 | {% endif %} 33 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2020 Mphasis Stelligent 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 8 | -------------------------------------------------------------------------------- /patches/@aws-cdk+app-delivery+1.32.0.patch: -------------------------------------------------------------------------------- 1 | diff --git a/node_modules/@aws-cdk/app-delivery/lib/pipeline-deploy-stack-action.js b/node_modules/@aws-cdk/app-delivery/lib/pipeline-deploy-stack-action.js 2 | index 06de0db..29074e5 100644 3 | --- a/node_modules/@aws-cdk/app-delivery/lib/pipeline-deploy-stack-action.js 4 | +++ b/node_modules/@aws-cdk/app-delivery/lib/pipeline-deploy-stack-action.js 5 | @@ -18,7 +18,7 @@ class PipelineDeployStackAction { 6 | const assets = this.stack.node.metadata.filter(md => md.type === cxapi.ASSET_METADATA); 7 | if (assets.length > 0) { 8 | // FIXME: Implement the necessary actions to publish assets 9 | - throw new Error(`Cannot deploy the stack ${this.stack.stackName} because it references ${assets.length} asset(s)`); 10 | + //throw new Error(`Cannot deploy the stack ${this.stack.stackName} because it references ${assets.length} asset(s)`); 11 | } 12 | const createChangeSetRunOrder = props.createChangeSetRunOrder || 1; 13 | const executeChangeSetRunOrder = props.executeChangeSetRunOrder || (createChangeSetRunOrder + 1); 14 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Mutato 2 | 3 |

4 | 5 | The Great Mutato! 6 | 7 |

8 | 9 | [Stelligent Mutato](https://github.com/stelligent/mutato) is an open-source 10 | framework for building containerized micro-services on the AWS ecosystem (e.g 11 | ECS or Fargate). Mutato is designed to leverage [AWS Cloud Development Kit 12 | (CDK)](https://docs.aws.amazon.com/cdk/latest/guide/home.html) constructs which 13 | abstract the complexity of writing a safe and secure CloudFormation file to 14 | deploy and automate micro-service deployments. 15 | 16 | - Documentation: [stelligent.github.io/mutato](https://stelligent.github.io/mutato) 17 | - Contributions: [CONTRIBUTING.md](CONTRIBUTING.md) 18 | - Issues & Bugs: [Github Issues](https://github.com/stelligent/mutato/issues) 19 | - Active Development: [Github Project](https://github.com/stelligent/mutato/projects/3) 20 | - Roadmap: [Github Project](https://github.com/stelligent/mutato/projects/2) 21 | - CDK Roadmap: [Github Project](https://github.com/orgs/aws/projects/7) 22 | - [Watching RFC 49](https://github.com/aws/aws-cdk-rfcs/blob/master/text/0049-continuous-delivery.md) 23 | -------------------------------------------------------------------------------- /lib/resources/network.ts: -------------------------------------------------------------------------------- 1 | import * as ec2 from '@aws-cdk/aws-ec2'; 2 | import * as ecs from '@aws-cdk/aws-ecs'; 3 | import * as cdk from '@aws-cdk/core'; 4 | import debug from 'debug'; 5 | import _ from 'lodash'; 6 | 7 | interface NetworkProps { 8 | vpc?: ec2.VpcProps; 9 | cluster?: ecs.ClusterProps; 10 | } 11 | 12 | /** ECS Cluster and VPC */ 13 | export class Network extends cdk.Construct { 14 | private readonly _props: NetworkProps; 15 | private readonly _debug: debug.IDebugger; 16 | public readonly vpc: ec2.Vpc; 17 | public readonly cluster: ecs.Cluster; 18 | 19 | /** 20 | * @hideconstructor 21 | * @param scope CDK scope 22 | * @param id CDK construct id 23 | * @param props CDK construct parameters 24 | */ 25 | constructor(scope: cdk.Construct, id: string, props?: NetworkProps) { 26 | super(scope, id); 27 | 28 | this._debug = debug(`mutato:constructs:Network:${id}`); 29 | this._props = _.defaults(props, {}); 30 | 31 | this._debug('creating a network construct with props: %o', this._props); 32 | this.vpc = new ec2.Vpc(this, 'VPC', this._props.vpc); 33 | this.cluster = new ecs.Cluster(this, 'Cluster', { 34 | ...this._props.cluster, 35 | vpc: this.vpc, 36 | }); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /.devcontainer/devcontainer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "mutato", 3 | "dockerFile": "Dockerfile", 4 | 5 | // Use 'appPort' to create a container with published ports. If the port isn't working, be sure 6 | // your server accepts connections from all interfaces (0.0.0.0 or '*'), not just localhost. 7 | "appPort": [], 8 | 9 | // Comment out the next line to run as root instead. 10 | "remoteUser": "node", 11 | 12 | // Use 'settings' to set *default* container specific settings.json values on container create. 13 | // You can edit these settings after create using File > Preferences > Settings > Remote. 14 | "settings": { 15 | "terminal.integrated.shell.linux": "/bin/bash" 16 | }, 17 | 18 | // Specifies a command that should be run after the container has been created. 19 | "postCreateCommand": "npm install", 20 | 21 | "forwardPorts": [3000], 22 | 23 | // Add the IDs of extensions you want installed when the container is created in the array below. 24 | "extensions": [ 25 | "dbaeumer.vscode-eslint", 26 | "esbenp.prettier-vscode", 27 | "stkb.rewrap", 28 | "christian-kohler.npm-intellisense", 29 | "DavidAnson.vscode-markdownlint", 30 | "eg2.vscode-npm-script", 31 | "ms-vscode.vscode-typescript-tslint-plugin", 32 | "yzhang.markdown-all-in-one" 33 | ] 34 | } 35 | -------------------------------------------------------------------------------- /doc/mutato-cdk.md: -------------------------------------------------------------------------------- 1 | # Extending Mutato in CDK 2 | 3 | > [!WARNING] Advanced users only! If you do not know what you are doing and do 4 | > not know how to work with CDK, you are at the wrong place. This page is for 5 | > CDK users who wish to integrate and extend Mutato in their CDK applications. 6 | 7 | Mutato at its core is just a CDK application. You can use its constructs and 8 | helpers in any CDK application. To get started: 9 | 10 | ```bash 11 | $ npm install @stelligent/mutato --save 12 | ``` 13 | 14 | And you can start using our constructs or helpers: 15 | 16 | ```TypeScript 17 | import * as mutato from '@stelligent/mutato'; 18 | import * as sns from '@aws-cdk/aws-sns'; 19 | 20 | async function createCustomMutatoApp() { 21 | // mutato.App is a subclass of cdk.App 22 | const app = new mutato.App(); 23 | await app.synthesizeFromFile('/path/to/mutato.yml'); 24 | 25 | // here you have a functioning CDK app, you can use it to add more resources 26 | const customStack = new cdk.Stack(app, 'MyCustomStack', { 27 | description: 'example of a custom stack attached to a Mutato app', 28 | }); 29 | 30 | // add a SNS topic into your custom stack 31 | const topic = new sns.Topic(customStack, 'MyTopic', { 32 | displayName: 'Customer subscription topic' 33 | }); 34 | 35 | // if you wish to customize what mutato synthesizes, you can also use all the 36 | // CDK "escape hatches" or other mechanisms you know of here. 37 | // https://docs.aws.amazon.com/cdk/latest/guide/cfn_layer.html 38 | } 39 | ``` -------------------------------------------------------------------------------- /doc/mutato-docker.md: -------------------------------------------------------------------------------- 1 | # Docker Container Documentation 2 | 3 | Mutato's Docker container is the preferred and recommended way to quickly get up 4 | and running with the toolset. 5 | 6 | The container conveniently packages all the dependencies needed to successfully 7 | deploy with Mutato. 8 | 9 | The container ships with the version of CDK we use in Mutato to ensure no 10 | collision happens between user's environment and what Mutato uses. 11 | 12 | ## Container Commands 13 | 14 | ### `docker run ... stelligent/mutato bootstrap` 15 | 16 | This is the one-time setup needed to be executed for each AWS account mutato is 17 | going to be used. This is currently just the same as `cdk bootstrap`. 18 | 19 | ### `docker run ... stelligent/mutato destroy` 20 | 21 | This destroys all the resources deployed by mutato for the current branch. 22 | 23 | ### `docker run ... stelligent/mutato deploy` 24 | 25 | This deploys _mutato.yml_ for the current branch into your AWS account. 26 | 27 | ### `docker run ... stelligent/mutato synth` 28 | 29 | This is essentially a dry-run of deploy. It only generates the CloudFormation 30 | files used during deployment. This can be used in a CI to catch errors in a 31 | _mutato.yml_ file before it actually deploys. 32 | 33 | ### `docker run ... stelligent/mutato cdk` 34 | 35 | > [!WARNING] Advanced users only! If you do not know what you are doing and do 36 | > not know how to work with CDK, running commands directly through CDK may 37 | > damage your deployed stacks! 38 | 39 | This gives you access to the bundled CDK. 40 | -------------------------------------------------------------------------------- /lib/actions/approval.ts: -------------------------------------------------------------------------------- 1 | import * as codePipelineActions from '@aws-cdk/aws-codepipeline-actions'; 2 | import assert from 'assert'; 3 | import debug from 'debug'; 4 | import _ from 'lodash'; 5 | import { config } from '../config'; 6 | import { ActionInterface, ActionPropsInterface } from './interface'; 7 | 8 | const _debug = debug('mutato:actions:Approval'); 9 | 10 | interface ApprovalProps extends ActionPropsInterface { 11 | emails?: string[]; 12 | } 13 | 14 | /** manual approval action in the pipeline */ 15 | export class Approval implements ActionInterface { 16 | private readonly _props: ApprovalProps; 17 | public readonly name: string; 18 | 19 | /** 20 | * @hideconstructor 21 | * @param props approval parameters 22 | */ 23 | constructor(props: ApprovalProps) { 24 | this._props = _.defaults(props, { order: 1 }); 25 | assert.ok(this._props.name); 26 | this.name = this._props.name; 27 | } 28 | 29 | /** 30 | * creates a manual approval action in the pipeline 31 | * 32 | * @param requester a unique ID used to prevent action duplication 33 | * @returns action construct to be added into a code pipeline 34 | */ 35 | public action( 36 | requester = 'default', 37 | ): codePipelineActions.ManualApprovalAction { 38 | _debug('creating a manual approval with props: %o', this._props); 39 | const git = config.getGithubMetaData(); 40 | return new codePipelineActions.ManualApprovalAction({ 41 | actionName: `${this.name}-${requester}`, 42 | notifyEmails: this._props?.emails, 43 | additionalInformation: this._props?.emails 44 | ? [ 45 | 'an approval action in a Mutato pipeline needs your attention.', 46 | `repository: ${git.repo}. branch: ${git.branch}.`, 47 | 'check your AWS console to make a decision.', 48 | ].join(' ') 49 | : undefined, 50 | }); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /doc/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | mutato - simplify the declaration and administration of the AWS resources necessary to support microservices. 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 |
15 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | -------------------------------------------------------------------------------- /lib/actions/docker.ts: -------------------------------------------------------------------------------- 1 | import * as codeBuild from '@aws-cdk/aws-codebuild'; 2 | import * as codePipeline from '@aws-cdk/aws-codepipeline'; 3 | import * as codePipelineActions from '@aws-cdk/aws-codepipeline-actions'; 4 | import { Container } from '../resources/container'; 5 | import { CodeBuild } from './codebuild'; 6 | import { ActionPropsInterface } from './interface'; 7 | import _ from 'lodash'; 8 | 9 | interface DockerBuildProps extends ActionPropsInterface { 10 | container: Container; 11 | pipeline: codePipeline.Pipeline; 12 | source: codePipeline.Artifact; 13 | sourceAction: codePipelineActions.GitHubSourceAction; 14 | } 15 | 16 | /** "docker build" convenience action */ 17 | export class DockerBuild extends CodeBuild { 18 | /** 19 | * @hideconstructor 20 | * @param props build parameters 21 | */ 22 | constructor(props: DockerBuildProps) { 23 | super({ 24 | ...props, 25 | privileged: true, 26 | buildImage: codeBuild.LinuxBuildImage.STANDARD_2_0, 27 | spec: { 28 | version: 0.2, 29 | phases: { 30 | install: { 'runtime-versions': { docker: 18 } }, 31 | pre_build: { commands: [props.container.loginCommand] }, 32 | build: { commands: [props.container.buildCommand] }, 33 | post_build: { commands: [props.container.pushCommand] }, 34 | }, 35 | }, 36 | }); 37 | } 38 | } 39 | 40 | interface DockerRunProps extends ActionPropsInterface { 41 | container: Container; 42 | pipeline: codePipeline.Pipeline; 43 | source: codePipeline.Artifact; 44 | sourceAction: codePipelineActions.GitHubSourceAction; 45 | cmd?: string | string[]; 46 | privileged?: boolean; 47 | } 48 | 49 | /** "docker run" convenience action */ 50 | export class DockerRun extends CodeBuild { 51 | /** 52 | * @hideconstructor 53 | * @param props run parameters 54 | */ 55 | constructor(props: DockerRunProps) { 56 | super({ 57 | ...props, 58 | spec: { 59 | version: 0.2, 60 | phases: { 61 | build: { 62 | commands: _.isString(props.cmd) ? [props.cmd] : props.cmd, 63 | }, 64 | }, 65 | }, 66 | }); 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /doc/mutato-architecture.md: -------------------------------------------------------------------------------- 1 | # Pipeline Architecture 2 | 3 | > [!NOTE] this page is aimed at advanced audiences, already familiar with CDK. 4 | 5 | Mutato is a TypeScript CDK application. It is executed via CDK's CLI. During 6 | execution, the entrypoint of the CDK application looks for two things: 7 | 8 | 1. A GitHub repository on the disk 9 | 2. A `mutato.yml` in it 10 | 11 | After that, Mutato processes `mutato.yml` and creates all the relevant resources 12 | using CDK constructs. 13 | 14 | Mutato deploys itself using [CDK's 15 | app-delivery](https://docs.aws.amazon.com/cdk/api/latest/docs/app-delivery-readme.html) 16 | module. Mutato's pipeline then manages exactly one other CloudFormation resource 17 | per `environment` tag in the `mutato.yml` file. 18 | 19 | All the containers and actions are owned by the master app-delivery pipeline and 20 | are processed before `environment` CloudFormation resources are updated on every 21 | push to the GitHub repository. 22 | 23 | This is the graph of a Mutato pipeline: 24 | 25 | ```MARKDOWN 26 | ┏━━━━━━━━━━━━━━━━┓ ┏━━━━━━━━━━━━━━━━┓ ┏━━━━━━━━━━━━━━━━━┓ ┏━━━━━━━━━━━━━━━━━┓ ┏━━━━━━━━━━━━━━━━━┓ ┏━━━━━━━━━━━━━━━━━┓ ┏━━━━━━━━━━━━━━━━━┓ ┏━━━━━━━━━━━━━━━━━┓ ┏━━━━━━━━━━━━━━━━━┓ 27 | ┃ Source ┃ ┃ Synth ┃ ┃ Update ┃ ┃ Container ┃ ┃ Container ┃ ┃ Container ┃ ┃ Environment ┃ ┃ Environment ┃ ┃ Environment ┃ ... 28 | ┃ ┃ ┃ ┃ ┃ ┃ ┃ Pre-Build ┃ ┃ Build & Push ┃ ┃ PostBuild ┃ ┃ PreDeploy ┃ ┃ Deploy ┃ ┃ Post-Deploy ┃ ... 29 | ┃ ┌────────────┐ ┃ ┃ ┌────────────┐ ┃ ┃ ┌─────────────┐ ┃ ┃ ┌─────────────┐ ┃ ┃ ┌─────────────┐ ┃ ┃ ┌─────────────┐ ┃ ┃ ┌─────────────┐ ┃ ┃ ┌─────────────┐ ┃ ┃ ┌─────────────┐ ┃ ... 30 | ┃ │ GitHub ┣━╋━━╋━▶ Mutato/CDK ┣━╋━━╋━▶ CFN ┣━╋━━╋━▶ Actions ┣━╋━━╋━▶ CodeBuild ┣━╋━━╋━▶ Actions ┣━╋━━╋━▶ Actions ┣━╋━━╋━▶ CFN ┣━╋━━╋━▶ Actions │ ┃ ... 31 | ┃ └────────────┘ ┃ ┃ └────────────┘ ┃ ┃ └─────────────┘ ┃ ┃ └─────────────┘ ┃ ┃ └─────────────┘ ┃ ┃ └─────────────┘ ┃ ┃ └─────────────┘ ┃ ┃ └─────────────┘ ┃ ┃ └─────────────┘ ┃ ... 32 | ┗━━━━━━━━━━━━━━━━┛ ┗━━━━━━━━━━━━━━━━┛ ┗━━━━━━━━━━━━━━━━━┛ ┗━━━━━━━━━━━━━━━━━┛ ┗━━━━━━━━━━━━━━━━━┛ ┗━━━━━━━━━━━━━━━━━┛ ┗━━━━━━━━━━━━━━━━━┛ ┗━━━━━━━━━━━━━━━━━┛ ┗━━━━━━━━━━━━━━━━━┛ 33 | ``` 34 | 35 | _CFN_ refers to CloudFormation. The last three boxes loop over for how ever many 36 | `environment` tags present in the `mutato.yml` file. 37 | -------------------------------------------------------------------------------- /doc/mutato-vs-mu.md: -------------------------------------------------------------------------------- 1 | # Mutato vs. Mu 2 | 3 | The OG [Stelligent Mu](https://github.com/stelligent/mu) is a framework similar 4 | to Mutato where it aims at deploying microservices with ease. 5 | 6 | Throughout the years thanks to the community of Mu-users, Stelligent received a 7 | lot of valuable feedback which sparked the creation of Mutato. 8 | 9 | Mutato at its core does exactly what Mu does but with a lot of added benefits. 10 | Some of these benefits are listed below. 11 | 12 | ## Extensibility 13 | 14 | The original Mu is written in Go and leaves almost no room for integration into 15 | third party applications. Stelligent has received a lot of requests from its 16 | enterprise users to change Mu so that it can be extended and integrated so 17 | enterprise applications can customize it to their heart's desire. 18 | 19 | Mutato on the other hand is written on top of the powerful AWS Cloud Development 20 | Kit and at its core is a collection of CDK constructs which can be customized to 21 | the teeth! 22 | 23 | ## Multi-Branch Deploys 24 | 25 | In mutato, every single branch is isolated from other branches. This is similar 26 | to how a properly configured CI system reacts to changes in a repository. You 27 | can have entirely different isolated AWS deployments over different branches. 28 | 29 | This feature is absent in the original Mu and was priority number 1 since day 1 30 | for Mutato and its delivery pipeline. 31 | 32 | ## CI-Agnostic 33 | 34 | The OG Stelligent Mu is heavily integrated with CodeBuild and leaves almost no 35 | room to integrate with external CI systems such as Drone CI and Jenkins. Users 36 | of the original Mu has always requested us to add support for Jenkins into Mu 37 | for enterprise applications. 38 | 39 | This is now possible due to the architecture of Mutato and the fact that it is 40 | built upon CDK and its constructs. Mutato uses CodeBuild internally, but it also 41 | offers ad-hoc support foundation for other CI systems. 42 | 43 | ## Nunjucks and Powerful Templating 44 | 45 | [Nunjucks](https://mozilla.github.io/nunjucks/templating.html) is a powerful 46 | templating tool provided by Mozilla. Every `mutato.yml` is a valid Nunjucks file 47 | and this allows `mutato.yml` to include complex conditionals and 48 | environment-specific deploys in a single file. On top of that, Mutato extends 49 | Nunjucks and adds `env`, `cmd` and other helpers to allow users to customize 50 | their template based on their execution environment even more. 51 | 52 | The original Mu is very basic in terms of templating capabilities which would 53 | often result in managing multiple templates for different environments. 54 | -------------------------------------------------------------------------------- /.devcontainer/Dockerfile: -------------------------------------------------------------------------------- 1 | #------------------------------------------------------------------------------------------------------------- 2 | # Copyright (c) Microsoft Corporation. All rights reserved. 3 | # Licensed under the MIT License. See https://go.microsoft.com/fwlink/?linkid=2090316 for license information. 4 | #------------------------------------------------------------------------------------------------------------- 5 | 6 | # For information on the contents of the container image below, see following Dockerfile: 7 | # https://github.com/microsoft/vscode-dev-containers/tree/v0.43.0/containers/javascript-node-12/.devcontainer/Dockerfile 8 | FROM mcr.microsoft.com/vscode/devcontainers/javascript-node:0.43-12 9 | 10 | # The image referenced above includes a non-root user with sudo access. Add 11 | # the "remoteUser" property to devcontainer.json to use it. On Linux, the container 12 | # user's GID/UIDs will be updated to match your local UID/GID when using the image 13 | # or dockerFile property. Update USER_UID/USER_GID below if you are using the 14 | # dockerComposeFile property or want the image itself to start with different ID 15 | # values. See https://aka.ms/vscode-remote/containers/non-root-user for details. 16 | ARG USER_UID=1000 17 | ARG USER_GID=$USER_UID 18 | ARG USERNAME=node 19 | 20 | # [Optional] Update UID/GID if needed 21 | RUN if [ "$USER_GID" != "1000" ] || [ "$USER_UID" != "1000" ]; then \ 22 | groupmod --gid $USER_GID $USERNAME \ 23 | && usermod --uid $USER_UID --gid $USER_GID $USERNAME \ 24 | && chown -R $USER_UID:$USER_GID /home/$USERNAME; \ 25 | fi \ 26 | # 27 | # Avoid having to use "sudo" with "npm -g" when running as non-root user 28 | && SNIPPET='if [ "$(stat -c %U /usr/local/lib/node_modules)" != "$(id -u)" ]; then \ 29 | sudo chown -R $(id -u):root /usr/local/lib/node_modules \ 30 | && sudo chown -R $(id -u):root /usr/local/bin; \ 31 | fi' \ 32 | && echo $SNIPPET >> /home/$USERNAME/.bashrc \ 33 | && echo $SNIPPET >> /home/$USERNAME/.zshrc \ 34 | && chown $USERNAME /home/$USERNAME/.bashrc /home/$USERNAME/.zshrc \ 35 | 36 | 37 | # ************************************************************* 38 | # * Uncomment this section to use RUN instructions to install * 39 | # * any needed dependencies after executing "apt-get update". * 40 | # * See https://docs.docker.com/engine/reference/builder/#run * 41 | # ************************************************************* 42 | # ENV DEBIAN_FRONTEND=noninteractive 43 | # RUN apt-get update \ 44 | # && apt-get -y install --no-reccomends \ 45 | # # 46 | # # Clean up 47 | # && apt-get autoremove -y \ 48 | # && apt-get clean -y \ 49 | # && rm -rf /var/lib/apt/lists/* 50 | # ENV DEBIAN_FRONTEND=dialog 51 | 52 | # Uncomment to default to non-root user 53 | # USER $USER_UID 54 | -------------------------------------------------------------------------------- /doc/mutato-workflow.md: -------------------------------------------------------------------------------- 1 | # Deploy Workflow 2 | 3 | This workflow is to be followed from top to bottom. 4 | 5 | ## prepare your environment 6 | 7 | You need the following environment variables defined: 8 | 9 | - `GITHUB_TOKEN`: your GitHub Personal Access Token 10 | - `AWS_*`: your AWS account credentials 11 | - `DOCKER_USERNAME` and `DOCKER_PASSWORD` _optional_ and only if you are using 12 | Docker Hub for your containers 13 | 14 | ## prepare your `mutato.yml` 15 | 16 | Everything begins with a `mutato.yml` in a branch of a GitHub repository. Once 17 | you have yourself a valid `mutato.yml` file. You can use Mutato's Docker image 18 | to deploy that branch to your AWS account. 19 | 20 | ## bootstrap (one time) 21 | 22 | ```bash 23 | $ docker run ... stelligent/mutato bootstrap 24 | ``` 25 | 26 | Before you can deploy, you need to execute a one-time `bootstrap` script. This 27 | currently just bootstraps CDK in your AWS account. 28 | 29 | > [!TIP] You can skip this if you have other CDK projects in your account. 30 | 31 | ## deploy (one time†) 32 | 33 | ```bash 34 | $ docker run ... stelligent/mutato deploy 35 | ``` 36 | 37 | You may deploy to your account with this command. You only need to execute this 38 | once in your computer's local terminal. This command freezes your environment's 39 | relevant variables and sends them to your AWS account. 40 | 41 | Mutato pipeline is self-healing, meaning that most changes to your `mutato.yml` 42 | automatically updates the pipeline itself in your AWS account. You do not need 43 | to manually run `deploy` every time you make a change, The pipeline detects 44 | changes and updates itself. 45 | 46 | > [!NOTE] __†__ if you ever makes changes to your `mutato.yml` that involves 47 | > adding or removing environment variables through Nunjucks's `env(...)` and 48 | > `cmd(...)` directives, you need to deploy from your computer's terminal again. 49 | 50 | ## push changes 51 | 52 | As mentioned before, the pipeline is self healing for the most part and does not 53 | need your manual intervention anymore. Just push changes to your branch and see 54 | those changes deploy. 55 | 56 | > [!NOTE] If you make changes to environment variables used in your `mutato.yml` 57 | or the core Mutato codebase itself, you have to deploy manually from your 58 | terminal again. 59 | 60 | ## destroy 61 | 62 | > [!NOTE] It is in our development roadmap to automate this procedure. 63 | 64 | Currently deleting stacks is a little bugged (from the CDK side). Deleting must 65 | be done in this order: 66 | 67 | 1. first delete the `Mutato-Pipeline-` stack 68 | 2. iterate and delete stacks of your deployed environments 69 | 70 | In case any of the stacks throw errors while deleting (very likely), after throw 71 | you have to: 72 | 73 | 1. open up all the IAM roles remaining 74 | 2. associate the "Administrator Access" inline policy to them 75 | 3. delete the stack again 76 | 4. it gives you an error again, this time only roles are remaining 77 | 5. delete the stack again while retaining all the roles 78 | 6. stack should delete successfully at this point 79 | 7. manually delete the roles in the IAM console 80 | -------------------------------------------------------------------------------- /lib/parser/parser.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable @typescript-eslint/no-explicit-any */ 2 | import assert from 'assert'; 3 | import debug from 'debug'; 4 | import _ from 'lodash'; 5 | import { Loader } from './loader'; 6 | import { PreProcessor } from './preprocessor'; 7 | 8 | const _debug = debug('mutato:parser:Parser'); 9 | 10 | type ActionSpec = object; 11 | type ResourceSpec = object; 12 | type ContainerSpec = object; 13 | export interface MutatoSpec { 14 | actions: ActionSpec[]; 15 | containers: ContainerSpec[]; 16 | environments: Map; 17 | environmentVariables: { [key: string]: string }; 18 | } 19 | 20 | /** 21 | * mutato.yml parser 22 | * 23 | * this class glues all the components for the parser together 24 | */ 25 | export class Parser { 26 | /** 27 | * parses the input mutato.yml string 28 | * 29 | * @param input mutato.yml as a string 30 | * @returns parsed and organized mutato.yml object 31 | */ 32 | public parse(input: string): MutatoSpec { 33 | _debug('attempting to parse mutato.yml string: %s', input); 34 | 35 | // during the first pass, we just want to figure out what environments we 36 | // are targeting, what containers we are building and what actions we are 37 | // constructing. during this pass, environment specific configuration is 38 | // not supported. 39 | const environmentLoader = new Loader(); 40 | const environmentPreprocessor = new PreProcessor({ environment: '' }); 41 | _debug('first pass preprocessing the YAML string: %s', input); 42 | const yaml = environmentPreprocessor.render(input); 43 | _debug('first pass loading the YAML string: %s', yaml); 44 | const parsed = environmentLoader.load(yaml); 45 | 46 | const actionSpecs = _.get(parsed, 'actions', []); 47 | _debug('actions: %o, ', actionSpecs); 48 | const containerSpecs = _.get(parsed, 'containers', []); 49 | _debug('containers: %o', containerSpecs); 50 | const environmentSpecs = _.get(parsed, 'environments', ['development']); 51 | _debug('environments %o', environmentSpecs); 52 | 53 | const environments = new Map(); 54 | 55 | // second pass parsing to create environment specific resources 56 | environmentSpecs.forEach((env: object) => { 57 | const key = (_.isObject(env) ? _.head(_.keys(env)) : env) as string; 58 | const tag = _.isObject(env) ? _.get(env, key) : {}; 59 | assert.ok(_.isString(key)); 60 | const environmentLoader = new Loader(); 61 | const environmentPreprocessor = new PreProcessor({ environment: key }); 62 | const yaml = environmentPreprocessor.render(input); 63 | _debug('second pass rendered YAML string: %s', yaml); 64 | const parsed = environmentLoader.load(yaml); 65 | _debug('second pass parsed YAML: %o', parsed); 66 | const resources = _.get(parsed, 'resources', []); 67 | resources.push({ environment: { ...tag } }); 68 | _debug('resources for environment "%s": %o', key, resources); 69 | environments.set(key, resources); 70 | }); 71 | 72 | return { 73 | actions: actionSpecs, 74 | containers: containerSpecs, 75 | environmentVariables: environmentPreprocessor.usedEnvironmentVariables, 76 | environments, 77 | }; 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /lib/resources/storage.ts: -------------------------------------------------------------------------------- 1 | import * as s3 from '@aws-cdk/aws-s3'; 2 | import * as sqs from '@aws-cdk/aws-sqs'; 3 | import * as cdk from '@aws-cdk/core'; 4 | import assert from 'assert'; 5 | import debug from 'debug'; 6 | import _ from 'lodash'; 7 | import { Service } from './service'; 8 | 9 | enum StorageProvider { 10 | S3 = 's3', 11 | SQS = 'sqs', 12 | } 13 | 14 | interface StorageProps { 15 | provider?: StorageProvider; 16 | config?: s3.BucketProps | sqs.QueueProps; 17 | } 18 | 19 | /** Storage provider for service constructs */ 20 | export class Storage extends cdk.Construct { 21 | public readonly props: StorageProps; 22 | public readonly resource: s3.Bucket | sqs.Queue; 23 | private readonly _debug: debug.Debugger; 24 | 25 | /** 26 | * @hideconstructor 27 | * @param scope CDK construct scope 28 | * @param id CDK construct ID 29 | * @param props storage configuration 30 | */ 31 | constructor(scope: cdk.Construct, id: string, props: StorageProps) { 32 | super(scope, id); 33 | 34 | this._debug = debug(`tato:constructs:storage:${id}`); 35 | this.props = _.defaults(props, { 36 | provider: StorageProvider.S3, 37 | }); 38 | 39 | this._debug('creating storage with props: %o', this.props); 40 | switch (this.props.provider) { 41 | case StorageProvider.S3: 42 | this.resource = new s3.Bucket(this, 'Bucket', { 43 | removalPolicy: cdk.RemovalPolicy.DESTROY, 44 | encryption: s3.BucketEncryption.S3_MANAGED, 45 | ...(this.props.config as s3.BucketProps), 46 | }); 47 | break; 48 | case StorageProvider.SQS: 49 | this.resource = new sqs.Queue(this, 'Queue', { 50 | encryption: sqs.QueueEncryption.KMS_MANAGED, 51 | ...(this.props.config as sqs.QueueProps), 52 | }); 53 | break; 54 | default: 55 | assert.fail('storage type not supported'); 56 | } 57 | } 58 | 59 | grantAccess(service: Service): void { 60 | assert.ok(service.resource, 'service resource does not exist'); 61 | // this is a hack but it works, adds env vars to taskDefinition after synth 62 | const _addEnv = (key: string, val: string): void => { 63 | _.set( 64 | service.resource.taskDefinition, 65 | `defaultContainer.props.environment["${key}_${this.resource.node.id}"]`, 66 | val, 67 | ); 68 | }; 69 | switch (typeof this.resource) { 70 | case typeof s3.Bucket: 71 | const b = this.resource as s3.Bucket; 72 | b.grantReadWrite(service.resource.taskDefinition.taskRole); 73 | b.grantDelete(service.resource.taskDefinition.taskRole); 74 | b.grantPut(service.resource.taskDefinition.taskRole); 75 | _addEnv(`STORAGE_BUCKET_ARN`, b.bucketArn); 76 | _addEnv(`STORAGE_BUCKET_NAME`, b.bucketName); 77 | _addEnv(`STORAGE_BUCKET_DOMAIN`, b.bucketDomainName); 78 | case typeof sqs.Queue: 79 | const q = this.resource as sqs.Queue; 80 | q.grantConsumeMessages(service.resource.taskDefinition.taskRole); 81 | q.grantSendMessages(service.resource.taskDefinition.taskRole); 82 | q.grantPurge(service.resource.taskDefinition.taskRole); 83 | _addEnv(`STORAGE_QUEUE_ARN`, q.queueArn); 84 | _addEnv(`STORAGE_QUEUE_URL`, q.queueUrl); 85 | _addEnv(`STORAGE_QUEUE_NAME`, q.queueName); 86 | break; 87 | default: 88 | assert.fail('storage type not supported'); 89 | } 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /lib/actions/codebuild.ts: -------------------------------------------------------------------------------- 1 | import * as codeBuild from '@aws-cdk/aws-codebuild'; 2 | import * as codePipeline from '@aws-cdk/aws-codepipeline'; 3 | import * as codePipelineActions from '@aws-cdk/aws-codepipeline-actions'; 4 | import * as cdk from '@aws-cdk/core'; 5 | import assert from 'assert'; 6 | import debug from 'debug'; 7 | import _ from 'lodash'; 8 | import { config } from '../config'; 9 | import { Container } from '../resources/container'; 10 | import { ActionInterface, ActionPropsInterface } from './interface'; 11 | 12 | const _debug = debug('mutato:actions:CodeBuild'); 13 | 14 | interface CodeBuildProps extends ActionPropsInterface { 15 | buildImage?: codeBuild.IBuildImage; 16 | container?: Container; 17 | pipeline: codePipeline.Pipeline; 18 | source: codePipeline.Artifact; 19 | sourceAction: codePipelineActions.GitHubSourceAction; 20 | spec: object | string; 21 | privileged?: boolean; 22 | } 23 | 24 | /** manual approval action in the pipeline */ 25 | export class CodeBuild implements ActionInterface { 26 | private readonly _props: CodeBuildProps; 27 | public readonly name: string; 28 | 29 | /** 30 | * @hideconstructor 31 | * @param props codebuild parameters 32 | */ 33 | constructor(props: CodeBuildProps) { 34 | this._props = _.defaults(props, { order: 1, privileged: false }); 35 | assert.ok(this._props.pipeline); 36 | assert.ok(this._props.source); 37 | assert.ok(this._props.spec); 38 | this.name = this._props.name; 39 | } 40 | 41 | /** 42 | * creates a codebuild approval action in the pipeline 43 | * 44 | * @param requester a unique ID used to prevent action duplication 45 | * @returns action construct to be added into a code pipeline 46 | */ 47 | public action(requester = 'default'): codePipelineActions.CodeBuildAction { 48 | _debug('creating a code build action with props: %o', this._props); 49 | const project = new codeBuild.PipelineProject( 50 | cdk.Stack.of(this._props.pipeline), 51 | `action-project-${this.name}-${requester}`, 52 | { 53 | environment: { 54 | buildImage: this._props.buildImage 55 | ? this._props.buildImage 56 | : this._props.container 57 | ? this._props.container.repo 58 | ? codeBuild.LinuxBuildImage.fromEcrRepository( 59 | this._props.container.repo, 60 | ) 61 | : codeBuild.LinuxBuildImage.fromDockerRegistry( 62 | this._props.container?.getImageUri( 63 | this._props.pipeline, 64 | true /* latest */, 65 | ), 66 | ) 67 | : undefined, 68 | privileged: this._props.privileged, 69 | environmentVariables: config.toBuildEnvironmentMap(), 70 | }, 71 | buildSpec: _.isObject(this._props.spec) 72 | ? codeBuild.BuildSpec.fromObject(this._props.spec) 73 | : codeBuild.BuildSpec.fromSourceFilename(this._props.spec), 74 | }, 75 | ); 76 | 77 | this._props.container?.repo?.grantPullPush(project); 78 | const action = new codePipelineActions.CodeBuildAction({ 79 | actionName: `${this.name}-${requester}`, 80 | input: this._props.source, 81 | environmentVariables: { 82 | mutato_opts__git__commit: { 83 | value: this._props.sourceAction.variables.commitId, 84 | }, 85 | }, 86 | project, 87 | }); 88 | 89 | return action; 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /doc/mutato-concepts.md: -------------------------------------------------------------------------------- 1 | # Concepts 2 | 3 | Mutato is an extremely opinionated framework out of the box, while leaving total 4 | control and extendibility through [AWS CDK](https://aws.amazon.com/cdk/). 5 | 6 | Mutato works over a single GitHub repository and a single branch of that repo. 7 | 8 | The combination of a Github repo and a branch is considered a _project_ in this 9 | toolset. That means you can have separate projects over different branches of 10 | your repositories. 11 | 12 | > [!NOTE] Keep in mind that the practicality of having separate projects over 13 | > multiple branches is almost non existent. Ideally you always want to have a 14 | > single Mutato pipeline over your _master_ branch and update your AWS resources 15 | > on pushes to that branch only. 16 | 17 | > [!WARNING] AWS multi-account and multi-region deploys are not supported yet. 18 | 19 | ## environments 20 | 21 | An environment isolates _a set of AWS resources_ inside a single CloudFormation 22 | file. You'd normally use environments to separate development resources from 23 | production resources. Stelligent recommends projects to use at least three 24 | different environments: 25 | 26 | - __development__: to be used by developers to rapidly deploying changes 27 | - __acceptance__: to be used by QA engineers to regularly testing pre-release 28 | - __production__: to be used for user-facing resources post-release 29 | 30 | If you do not specify different environments in your _mutato.yml_, it is assumed 31 | that you are just developing a non production application and Mutato gives you a 32 | default development environment for it. 33 | 34 | ## containers 35 | 36 | Containers are at at the heart of Mutato. You can define containers to be built 37 | as a part of your project in your AWS account. Currently you can choose to have 38 | containers stored either on AWS ECR or Docker Hub. 39 | 40 | > [!TIP] You do not have to have containers defined if you are just using third 41 | > party containers off of Docker Hub (e.g the [nginx 42 | > container](https://hub.docker.com/_/nginx)). 43 | 44 | At the moment only Docker containers are supported. It is in our development 45 | roadmap to support other types of containers in future (e.g [_LXC 46 | containers_](https://linuxcontainers.org/)). 47 | 48 | ## actions 49 | 50 | Actions are one-off tasks that run in your Mutato pipeline on every push to the 51 | repository's deployed branch. Think of this similar to how GitHub actions work, 52 | but in a broader sense. Mutato actions are not limited to Docker tasks. 53 | 54 | > [!TIP] You can use actions to define automated unit tests. Both in and out of 55 | > container tests are supported. 56 | 57 | Currently you can have a CodeBuild job, a [Manual 58 | Approval](https://docs.aws.amazon.com/codepipeline/latest/userguide/approvals-action-add.html), 59 | or an in-container Docker command to be executed in the pipeline. It is in our 60 | development roadmap to support other types of actions such as Jenkins jobs and 61 | External CI jobs (e.g Drone CI). 62 | 63 | Actions are triggered by events in the pipeline. 64 | 65 | ## events 66 | 67 | Events are specific points in time in your Mutato pipeline. You can have actions 68 | attached to these events to control the flow of your Mutato pipeline and stop 69 | the execution whenever an action fails. 70 | 71 | Events trigger actions in the pipeline. 72 | 73 | > [!TIP] Use events to stop your Mutato pipeline early from flowing once unit 74 | > tests fail or code linting does not succeed. 75 | -------------------------------------------------------------------------------- /lib/parser/preprocessor.ts: -------------------------------------------------------------------------------- 1 | import assert from 'assert'; 2 | import cp from 'child_process'; 3 | import debug from 'debug'; 4 | import _ from 'lodash'; 5 | import ms from 'ms'; 6 | import nunjucks from 'nunjucks'; 7 | import { config } from '../config'; 8 | 9 | const _debug = debug('mutato:parser:PreProcessor'); 10 | type StringMap = { [key: string]: string }; 11 | 12 | /** 13 | * global env function of our Nunjucks Environment 14 | * 15 | * @param name environment variable name 16 | * @returns environment variable value, empty string if not found 17 | * @ignore 18 | */ 19 | function nunjucks_env_global(name: string): string { 20 | _debug('attempting to resolve environment variable %s', name); 21 | assert.ok(_.isString(name)); 22 | return _.get(process.env, name, ''); 23 | } 24 | 25 | /** 26 | * global cmd function of our Nunjucks Environment 27 | * 28 | * @param command shell command execute. can contain shell operators 29 | * @returns string output of the executed command the output is trimmed 30 | * from whitespace and newlines (trailing newline as well) 31 | * @ignore 32 | */ 33 | function nunjucks_cmd_global(command: string): string { 34 | _debug('attempting to execute command %s', command); 35 | assert.ok(_.isString(command)); 36 | return _.trim( 37 | cp.execSync(command, { 38 | encoding: 'utf8', 39 | timeout: ms(config.opts.preprocessor.timeout), 40 | }), 41 | ); 42 | } 43 | 44 | /** 45 | * mutato.yml template pre processor 46 | * 47 | * internally this class sets up a custom Nunjucks Environment with useful 48 | * helper functions and processes input Nunjucks templates 49 | */ 50 | export class PreProcessor { 51 | private readonly env: nunjucks.Environment; 52 | private readonly ctx: object; 53 | public readonly usedEnvironmentVariables: { [key: string]: string } = {}; 54 | 55 | /** 56 | * @hideconstructor 57 | * @param context additional context variables given to Nunjucks 58 | * @todo implement the "ssm" Nunjucks global function 59 | * @todo implement the "asm" Nunjucks global function 60 | */ 61 | constructor(context: StringMap = {}) { 62 | this.env = new nunjucks.Environment(null, { 63 | autoescape: false, 64 | noCache: true, 65 | throwOnUndefined: true, 66 | watch: false, 67 | }); 68 | this.ctx = { 69 | ...context, 70 | build_time: Date.now(), 71 | git: { 72 | ...config.getGithubMetaData(), 73 | commit: config.opts.git.commit, 74 | }, 75 | }; 76 | _debug('a new preprocessor is initialized with context: %o', this.ctx); 77 | 78 | this.env.addGlobal('env', (name: string) => { 79 | const resolved = nunjucks_env_global(name); 80 | _.set(this.usedEnvironmentVariables, name, resolved); 81 | return resolved; 82 | }); 83 | this.env.addGlobal('cmd', nunjucks_cmd_global); 84 | } 85 | 86 | /** @returns {object} build context of preprocessor */ 87 | public get context(): object { 88 | return this.ctx; 89 | } 90 | 91 | /** 92 | * renders the input string template through our Nunjucks Environment 93 | * 94 | * @param input unprocessed input template string 95 | * @returns processed template 96 | */ 97 | public render(input: string): string { 98 | _debug('attempting to render a string through Nunjucks: %s', input); 99 | return this.env.renderString(input, this.ctx); 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /lib/config.ts: -------------------------------------------------------------------------------- 1 | import { BuildEnvironmentVariable } from '@aws-cdk/aws-codebuild'; 2 | import assert from 'assert'; 3 | import cp from 'child_process'; 4 | import debug from 'debug'; 5 | import _ from 'lodash'; 6 | import parseGithubUrl from 'parse-github-url'; 7 | import rc from 'rc'; 8 | import traverse from 'traverse'; 9 | import parse = require('parse-strings-in-object'); 10 | 11 | const log = debug('mutato:config'); 12 | 13 | /** 14 | * @param name "rc" namespace 15 | * @param defaults default configuration object 16 | * @returns overridden configuration with "rc" 17 | */ 18 | function rcTyped(name: string, defaults: T): T { 19 | const userConfig = rc(name, defaults); 20 | const parsedConfig = parse(userConfig); 21 | return parsedConfig as T; 22 | } 23 | 24 | // args passed to cp.execSync down when we extract defaults from environment 25 | const gitC = _.get(process.env, 'mutato_opts__git__local', process.cwd()); 26 | const gitRemoteCmd = `git -C "${gitC}" config --get remote.origin.url || true`; 27 | const gitBranchCmd = `git -C "${gitC}" rev-parse --abbrev-ref HEAD || true`; 28 | const gitCommitCmd = `git -C "${gitC}" rev-parse HEAD || true`; 29 | 30 | type StringEnvironmentVariableMap = { [key: string]: string }; 31 | type BuildEnvironmentVariableMap = { [key: string]: BuildEnvironmentVariable }; 32 | 33 | log('extracting configuration from environment: %o', process.env); 34 | export const config = rcTyped('mutato', { 35 | opts: { 36 | git: { 37 | local: gitC, 38 | commit: cp 39 | .execSync(gitCommitCmd, { encoding: 'utf8', timeout: 1000 }) 40 | .trim(), 41 | remote: cp 42 | .execSync(gitRemoteCmd, { encoding: 'utf8', timeout: 1000 }) 43 | .trim(), 44 | branch: cp 45 | .execSync(gitBranchCmd, { encoding: 'utf8', timeout: 1000 }) 46 | .trim(), 47 | secret: _.get(process.env, 'GITHUB_TOKEN', ''), 48 | }, 49 | docker: { 50 | user: _.get(process.env, 'DOCKER_USERNAME', ''), 51 | pass: _.get(process.env, 'DOCKER_PASSWORD', ''), 52 | }, 53 | preprocessor: { 54 | timeout: '10s', 55 | }, 56 | bundle: { 57 | bucket: '', 58 | object: '', 59 | }, 60 | }, 61 | getGithubMetaData() { 62 | const meta = parseGithubUrl(this.opts.git.remote); 63 | /** @todo properly handle non Github repositories */ 64 | assert.ok(meta, 'only Github remotes are supported'); 65 | assert.ok(meta?.name, 'Github repo could not be determined'); 66 | assert.ok(meta?.owner, 'Github owner could not be determined'); 67 | const repo = meta?.name as string; 68 | const owner = meta?.owner as string; 69 | const branch = this.opts.git.branch; 70 | const id = `${owner}-${repo}-${branch}`.replace(/[^A-Za-z0-9-]/g, '-'); 71 | return { 72 | repo: meta?.name as string, 73 | owner: meta?.owner as string, 74 | branch: this.opts.git.branch, 75 | /** this can be used in CDK names and IDs to uniquely ID a resource */ 76 | identifier: id, 77 | }; 78 | }, 79 | toStringEnvironmentMap() { 80 | return traverse(this).reduce(function (acc, x) { 81 | if (this.isLeaf && this.key !== '_' && !_.isFunction(x)) 82 | acc[`mutato_${this.path.join('__')}`] = `${x}`; 83 | return _.omit(acc, [ 84 | 'mutato_opts__git__local', 85 | 'mutato_opts__git__commit', 86 | 'mutato_opts__bundle__bucket', 87 | 'mutato_opts__bundle__object', 88 | ]); 89 | }, {}) as StringEnvironmentVariableMap; 90 | }, 91 | toBuildEnvironmentMap(variables?: StringEnvironmentVariableMap) { 92 | return _.transform( 93 | variables ? variables : this.toStringEnvironmentMap(), 94 | (result: BuildEnvironmentVariableMap, value, key) => { 95 | result[key] = { value }; 96 | }, 97 | {}, 98 | ); 99 | }, 100 | }); 101 | 102 | log('Mutato configuration: %o', config); 103 | -------------------------------------------------------------------------------- /doc/README.md: -------------------------------------------------------------------------------- 1 | # Mutato 2 | 3 | > [!NOTE] If you are coming from the original [Stelligent 4 | > Mu](https://github.com/stelligent/mu), keep in mind that Mutato is a successor 5 | > to Mu. While there are many similarities between the two projects, currently 6 | > we are not planning to provide any compatibilities between the two. 7 | 8 | [Stelligent Mutato](https://github.com/stelligent/mutato) is an open-source 9 | framework for building containerized micro-services on the AWS ecosystem (e.g 10 | ECS or Fargate). Mutato is designed to leverage [AWS Cloud Development Kit 11 | (CDK)](https://docs.aws.amazon.com/cdk/latest/guide/home.html) constructs which 12 | abstract the complexity of writing a safe and secure CloudFormation file to 13 | deploy and automate micro-service deployments. 14 | 15 | ## Getting Started 16 | 17 | Create a simple _mutato.yml_ file in your Github repository: 18 | 19 | ```YAML 20 | environments: 21 | - acceptance 22 | - production 23 | resources: 24 | - service: 25 | provider: fargate 26 | container: nginx:latest 27 | - network: 28 | vpc: 29 | maxAZs: {{ 1 if environment == "acceptance" else 3 }} 30 | ``` 31 | 32 | [Obtain](https://github.com/settings/tokens) a [GitHub Personal Access 33 | Token](https://help.github.com/en/actions/configuring-and-managing-workflows/authenticating-with-the-github_token). 34 | We assume you have this available in your `$PATH` under `$GITHUB_TOKEN`. Execute 35 | the following to deploy your microservice: 36 | 37 | ```bash 38 | $ cd /path/to/your/github/repo 39 | $ echo "GITHUB_TOKEN=$GITHUB_TOKEN" > mutato.env 40 | $ aws-vault exec -- env | grep AWS_ >> mutato.env 41 | $ docker run -it --rm --env-file mutato.env -v `pwd`:/project stelligent/mutato bootstrap 42 | $ docker run -it --rm --env-file mutato.env -v `pwd`:/project stelligent/mutato deploy 43 | ``` 44 | 45 | This gives you a load balanced NGINX server in two separate environments and the 46 | _acceptance_ environment has its VPC AZ capacity set to 1 to reduce costs. 47 | 48 | ## Where To Head Next 49 | 50 | You want to familiarize yourself with the [concepts](mutato-concepts.md) first. 51 | 52 | After that, you need to know the [workflow](mutato-workflow.md) of a Mutato 53 | enabled project. The following documentation comes next: 54 | 55 | - To learn more about what you can write in _mutato.yml_, head over to its 56 | [reference schema](mutato-yaml.md) page. 57 | - To learn more about what you can do with the Mutato Docker container, head 58 | over [to its page](mutato-docker.md). 59 | - To learn more about extensibility of Mutato, read [its extensibility 60 | documentation](mutato-cdk.md). 61 | - If you are looking for the auto generated low level CDK api documentation, [go 62 | to API](api.md) 63 | 64 | ## AWS Environment 65 | 66 | We highly recommend you use [aws-vault](https://github.com/99designs/aws-vault) 67 | to manage your AWS environment. If you choose not to, you should manually create 68 | your `mutato.env` file with at least the following environment variables: 69 | 70 | - _AWS_DEFAULT_REGION_ 71 | - _AWS_ACCESS_KEY_ID_ 72 | - _AWS_SECRET_ACCESS_KEY_ 73 | 74 | Consult [Docker run](https://docs.docker.com/engine/reference/commandline/run)'s 75 | documentation on the format of the `mutato.env` file. Consult [AWS 76 | CLI](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html)'s 77 | documentation for the environment variables you may need to set manually. 78 | 79 | ## Supported Platforms 80 | 81 | Mutato has been mostly developed and tested on Linux 64bit (Ubuntu 18.04 LTS). 82 | The core codebase is in typescript and uses CDK, therefore in theory it should 83 | run anywhere that is capable of running a Docker container or node and npm, but 84 | the official support is mostly provided for the Linux 64bit platform. 85 | 86 | ## Stability 87 | 88 | Currently Mutato is under active development. Expect an alpha quality software 89 | and things rapidly changing until we announce our stable v1.0. 90 | 91 | Do not use this in production yet! 92 | 93 | ...or do. test in production amirite? 94 | 95 | ![At that part, I'm a pro](pro.gif) 96 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.js 2 | !jest.config.js 3 | *.d.ts 4 | node_modules 5 | 6 | # CDK asset staging directory 7 | .cdk.staging 8 | cdk.out 9 | 10 | # Autogenerated Typedoc documentation 11 | docs/**/* 12 | !docs/*.md 13 | 14 | # Created by https://www.gitignore.io/api/node,linux,macos,visualstudiocode,typescript 15 | # Edit at https://www.gitignore.io/?templates=node,linux,macos,visualstudiocode,typescript 16 | 17 | ### Linux ### 18 | *~ 19 | 20 | # temporary files which can be created if a process still has a handle open of a deleted file 21 | .fuse_hidden* 22 | 23 | # KDE directory preferences 24 | .directory 25 | 26 | # Linux trash folder which might appear on any partition or disk 27 | .Trash-* 28 | 29 | # .nfs files are created when an open file is removed but is still being accessed 30 | .nfs* 31 | 32 | ### macOS ### 33 | # General 34 | .DS_Store 35 | .AppleDouble 36 | .LSOverride 37 | 38 | # Icon must end with two \r 39 | Icon 40 | 41 | # Thumbnails 42 | ._* 43 | 44 | # Files that might appear in the root of a volume 45 | .DocumentRevisions-V100 46 | .fseventsd 47 | .Spotlight-V100 48 | .TemporaryItems 49 | .Trashes 50 | .VolumeIcon.icns 51 | .com.apple.timemachine.donotpresent 52 | 53 | # Directories potentially created on remote AFP share 54 | .AppleDB 55 | .AppleDesktop 56 | Network Trash Folder 57 | Temporary Items 58 | .apdisk 59 | 60 | ### Node ### 61 | # Logs 62 | logs 63 | *.log 64 | npm-debug.log* 65 | yarn-debug.log* 66 | yarn-error.log* 67 | lerna-debug.log* 68 | 69 | # Diagnostic reports (https://nodejs.org/api/report.html) 70 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json 71 | 72 | # Runtime data 73 | pids 74 | *.pid 75 | *.seed 76 | *.pid.lock 77 | 78 | # Directory for instrumented libs generated by jscoverage/JSCover 79 | lib-cov 80 | 81 | # Coverage directory used by tools like istanbul 82 | coverage 83 | *.lcov 84 | 85 | # nyc test coverage 86 | .nyc_output 87 | 88 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) 89 | .grunt 90 | 91 | # Bower dependency directory (https://bower.io/) 92 | bower_components 93 | 94 | # node-waf configuration 95 | .lock-wscript 96 | 97 | # Compiled binary addons (https://nodejs.org/api/addons.html) 98 | build/Release 99 | 100 | # Dependency directories 101 | node_modules/ 102 | jspm_packages/ 103 | 104 | # TypeScript v1 declaration files 105 | typings/ 106 | 107 | # TypeScript cache 108 | *.tsbuildinfo 109 | 110 | # Optional npm cache directory 111 | .npm 112 | 113 | # Optional eslint cache 114 | .eslintcache 115 | 116 | # Optional REPL history 117 | .node_repl_history 118 | 119 | # Output of 'npm pack' 120 | *.tgz 121 | 122 | # Yarn Integrity file 123 | .yarn-integrity 124 | 125 | # dotenv environment variables file 126 | .env 127 | .env.test 128 | 129 | # parcel-bundler cache (https://parceljs.org/) 130 | .cache 131 | 132 | # next.js build output 133 | .next 134 | 135 | # nuxt.js build output 136 | .nuxt 137 | 138 | # rollup.js default build output 139 | dist/ 140 | 141 | # Uncomment the public line if your project uses Gatsby 142 | # https://nextjs.org/blog/next-9-1#public-directory-support 143 | # https://create-react-app.dev/docs/using-the-public-folder/#docsNav 144 | # public 145 | 146 | # Storybook build outputs 147 | .out 148 | .storybook-out 149 | 150 | # vuepress build output 151 | .vuepress/dist 152 | 153 | # Serverless directories 154 | .serverless/ 155 | 156 | # FuseBox cache 157 | .fusebox/ 158 | 159 | # DynamoDB Local files 160 | .dynamodb/ 161 | 162 | # Temporary folders 163 | tmp/ 164 | temp/ 165 | 166 | #!! ERROR: typescript is undefined. Use list command to see defined gitignore types !!# 167 | 168 | ### VisualStudioCode ### 169 | .vscode/* 170 | !.vscode/settings.json 171 | !.vscode/tasks.json 172 | !.vscode/launch.json 173 | !.vscode/extensions.json 174 | 175 | ### VisualStudioCode Patch ### 176 | # Ignore all local history of files 177 | .history 178 | 179 | # End of https://www.gitignore.io/api/node,linux,macos,visualstudiocode,typescript 180 | 181 | # TSC artifacts 182 | out 183 | doc/api.md 184 | !bin/*.ts 185 | !bin/*.js 186 | mutato.env 187 | -------------------------------------------------------------------------------- /doc/mutato-yaml.md: -------------------------------------------------------------------------------- 1 | # YAML Reference Specification 2 | 3 | _mutato.yml_ is a 4 | [Nunjucks](https://mozilla.github.io/nunjucks/templating.html) enabled file. You 5 | can use all that Nunjucks has to offer. 6 | 7 | Mutato offers two builtin Nunjucks functions: 8 | 9 | - `env("string")`: that resolves to environment variable `string` 10 | - `cmd("string")`: that resolves to the output of shell command `string` 11 | 12 | Mutato also offers a special `environment` Nunjucks variable inside the 13 | _resources_ section to provide environment specific configuration. 14 | 15 | Example use of Nunjucks is provided below in the reference YAML. 16 | 17 | ```YAML 18 | # this section defines the isolated environments "resources" will deploy to. 19 | # best practice is to separate development and production environments on AWS. 20 | environments: 21 | # an environment can simply be a string 22 | - acceptance 23 | # an environment can also be an object 24 | - production: 25 | # events guard the deployment of an environment. use it to run tests 26 | events: 27 | # two supported events are: "pre-deploy" and "post-deploy" 28 | pre-deploy: production-pre-deploy-action 29 | post-deploy: production-post-deploy-action 30 | # this section defines actions that are triggered on certain events 31 | # "actions" section is an array of objects 32 | actions: 33 | # use docker action to run a simple docker command. your GitHub's source is 34 | # automatically mounted in command's current working directory 35 | - docker: 36 | name: production-post-deploy-action 37 | # you can reference containers on DockerHub just by their name 38 | container: ubuntu 39 | cmd: exit 0 40 | 41 | # use approval action to wait for a real human's approval in the pipeline 42 | - approval: 43 | name: production-pre-deploy-action 44 | # optionally provide a list of emails to notify 45 | emails: 46 | - sysadmin1@example.com 47 | - sysadmin2@example.com 48 | 49 | # use codebuild action execute a CodeBuild build spec 50 | - codebuild: 51 | name: container-pre-build-action 52 | # this can also be an inline object here 53 | spec: /path/to/buildspec.yml 54 | privileged: true 55 | 56 | - approval: 57 | name: container-pre-build-action 58 | # this section defines containers used by actions and resources 59 | # "containers" section is an array of objects 60 | containers: 61 | # array key is container's type. currently only "docker" is supported 62 | - docker: 63 | # a required name for your container 64 | name: mutato 65 | # path to a docker file to build in the pipeline 66 | file: Dockerfile 67 | events: 68 | # two supported events are: "pre-build" and "post-build" 69 | pre-build: container-pre-build-action 70 | post-build: container-post-build-action 71 | 72 | # you can also specify a container on the DockerHub 73 | - docker: 74 | name: latest-nginx 75 | uri: nginx:latest 76 | # this section defines the actual resources deployed in each environment 77 | # you can use environment specific resources definitions here 78 | resources: 79 | # a service can be any of: 80 | # - "fargate": container running in Fargate 81 | # - "classic": container running in EC2 ECS 82 | # - "fargate-task": Fargate container triggered by CloudWatch rate expressions 83 | # - "classic-task": EC2 ECS container triggered by CloudWatch rate expressions 84 | # - "fargate-queue": Fargate container triggered by SQS messages 85 | # - "classic-queue": EC2 ECS container triggered by SQS messages 86 | - service: 87 | provider: fargate 88 | # you can reference your container from the "container" section here by 89 | # name or you can just provide a DockerHub uri 90 | container: mutato 91 | 92 | # you can create a storage for your container, all your "service" containers 93 | # get access to "storage"s in runtime via environment variables 94 | - storage: 95 | provider: s3 96 | # this is how "env" function can be used 97 | name: bucket-{{ env("BUCKET_NAME") }} 98 | 99 | - storage: 100 | provider: sqs 101 | # this is how "cmd" function can be used 102 | name: queue-{{ cmd("echo $BUCKET_NAME") }} 103 | 104 | # you can also create a database for your service definitions. similar to 105 | # storage definitions, you get access to these in runtime via environment 106 | # variables into your containers 107 | - database: 108 | provider: dynamo 109 | 110 | # you can perform environment specific configuration through Nunjucks 111 | - network: 112 | vpc: 113 | # this is how "environment" Nunjucks variable can be used 114 | maxAZs: {{ 1 if environment == "acceptance" else 3 }} 115 | ``` 116 | -------------------------------------------------------------------------------- /test/parser.test.ts: -------------------------------------------------------------------------------- 1 | import chai from 'chai'; 2 | import chaiAsPromised from 'chai-as-promised'; 3 | import { Parser } from '../lib'; 4 | 5 | chai.use(chaiAsPromised); 6 | 7 | describe('Parser Tests', () => { 8 | it('should not throw when parsing an empty schema', () => { 9 | const parser = new Parser(); 10 | chai.assert.doesNotThrow(() => { 11 | parser.parse(''); 12 | }); 13 | }); 14 | 15 | it('should make a development environment by default', () => { 16 | const parser = new Parser(); 17 | const parsed = parser.parse(''); 18 | chai.assert.isTrue(parsed.environments.size === 1); 19 | chai.assert.isTrue(parsed.environments.has('development')); 20 | const development = parsed.environments.get('development'); 21 | chai.assert.deepEqual(development, [{ environment: {} }]); 22 | chai.assert.deepEqual(parsed.actions, []); 23 | chai.assert.deepEqual(parsed.containers, []); 24 | }); 25 | 26 | it('should be able to parse environments', () => { 27 | const parser = new Parser(); 28 | const parsed = parser.parse(` 29 | environments: 30 | - develop: 31 | events: 32 | pre-deploy: action-name 33 | post-deploy: action-name 34 | - acceptance 35 | - production`); 36 | chai.assert.deepEqual(parsed.actions, []); 37 | chai.assert.deepEqual(parsed.containers, []); 38 | chai.assert.isTrue(parsed.environments.size === 3); 39 | chai.assert.isTrue(parsed.environments.has('develop')); 40 | chai.assert.isTrue(parsed.environments.has('acceptance')); 41 | chai.assert.isTrue(parsed.environments.has('production')); 42 | chai.assert.deepEqual(parsed.environments.get('develop'), [ 43 | { 44 | environment: { 45 | events: { 46 | 'pre-deploy': 'action-name', 47 | 'post-deploy': 'action-name', 48 | }, 49 | }, 50 | }, 51 | ]); 52 | chai.assert.deepEqual(parsed.environments.get('acceptance'), [ 53 | { environment: {} }, 54 | ]); 55 | chai.assert.deepEqual(parsed.environments.get('production'), [ 56 | { environment: {} }, 57 | ]); 58 | }); 59 | 60 | it('should support env() and cmd() in environments', () => { 61 | const parser = new Parser(); 62 | const parsed = parser.parse(` 63 | environments: 64 | - acceptance: 65 | user: {{ env("USER") }} 66 | - production: 67 | data: {{ cmd("whoami | xargs echo") }}`); 68 | chai.assert.isTrue(parsed.environments.size === 2); 69 | chai.assert.isTrue(parsed.environments.has('acceptance')); 70 | chai.assert.isTrue(parsed.environments.has('production')); 71 | chai.assert.deepEqual(parsed.environments.get('acceptance'), [ 72 | { 73 | environment: { user: process.env.USER }, 74 | }, 75 | ]); 76 | chai.assert.deepEqual(parsed.environments.get('production'), [ 77 | { 78 | environment: { data: process.env.USER }, 79 | }, 80 | ]); 81 | }); 82 | 83 | it('should support environment specific configuration', () => { 84 | const parser = new Parser(); 85 | const parsed = parser.parse(` 86 | environments: 87 | - acceptance 88 | - production 89 | containers: 90 | - nginx: 91 | uri: nginx:latest 92 | actions: 93 | - action-name-1: 94 | foo: bar 95 | - action-name-2: 96 | test: val 97 | resources: 98 | - service: 99 | name: web-server-{{ environment }} 100 | provider: fargate 101 | container: nginx 102 | - network: 103 | cluster: 104 | maxAzs: {{ 1 if environment == "acceptance" else 3 }}`); 105 | chai.assert.isTrue(parsed.environments.size === 2); 106 | chai.assert.isTrue(parsed.environments.has('acceptance')); 107 | chai.assert.isTrue(parsed.environments.has('production')); 108 | chai.assert.deepEqual(parsed.actions, [ 109 | { 'action-name-1': { foo: 'bar' } }, 110 | { 'action-name-2': { test: 'val' } }, 111 | ]); 112 | chai.assert.deepEqual(parsed.containers, [ 113 | { nginx: { uri: 'nginx:latest' } }, 114 | ]); 115 | chai.assert.deepEqual(parsed.environments.get('acceptance'), [ 116 | { 117 | service: { 118 | name: 'web-server-acceptance', 119 | provider: 'fargate', 120 | container: 'nginx', 121 | }, 122 | }, 123 | { network: { cluster: { maxAzs: 1 } } }, 124 | { environment: {} }, 125 | ]); 126 | chai.assert.deepEqual(parsed.environments.get('production'), [ 127 | { 128 | service: { 129 | name: 'web-server-production', 130 | provider: 'fargate', 131 | container: 'nginx', 132 | }, 133 | }, 134 | { network: { cluster: { maxAzs: 3 } } }, 135 | { environment: {} }, 136 | ]); 137 | }); 138 | }); 139 | -------------------------------------------------------------------------------- /lib/resources/database.ts: -------------------------------------------------------------------------------- 1 | import * as ddb from '@aws-cdk/aws-dynamodb'; 2 | import * as rds from '@aws-cdk/aws-rds'; 3 | import * as ec2 from '@aws-cdk/aws-ec2'; 4 | import * as cdk from '@aws-cdk/core'; 5 | import assert from 'assert'; 6 | import debug from 'debug'; 7 | import _ from 'lodash'; 8 | import { Network } from './network'; 9 | import { Service } from './service'; 10 | 11 | enum DatabaseProvider { 12 | RDS = 'rds', 13 | RDSCluster = 'rds-cluster', 14 | Dynamo = 'dynamo', 15 | } 16 | 17 | interface DatabaseProps { 18 | provider?: DatabaseProvider; 19 | config?: 20 | | ddb.TableProps 21 | | rds.DatabaseClusterProps 22 | | rds.DatabaseInstanceProps; 23 | network: Network; 24 | } 25 | 26 | /** Database provider for service constructs */ 27 | export class Database extends cdk.Construct { 28 | public readonly props: DatabaseProps; 29 | public readonly resource: 30 | | ddb.Table 31 | | rds.DatabaseInstance 32 | | rds.DatabaseCluster; 33 | private readonly _debug: debug.Debugger; 34 | 35 | /** 36 | * @hideconstructor 37 | * @param scope CDK construct scope 38 | * @param id CDK construct ID 39 | * @param props database configuration 40 | */ 41 | constructor(scope: cdk.Construct, id: string, props: DatabaseProps) { 42 | super(scope, id); 43 | 44 | this._debug = debug(`mutato:constructs:database:${id}`); 45 | this.props = _.defaults(props, { 46 | provider: DatabaseProvider.Dynamo, 47 | }); 48 | 49 | this._debug('creating storage with props: %o', this.props); 50 | switch (this.props.provider) { 51 | case DatabaseProvider.Dynamo: 52 | this.resource = new ddb.Table(this, 'Table', { 53 | billingMode: ddb.BillingMode.PAY_PER_REQUEST, 54 | partitionKey: { name: 'id', type: ddb.AttributeType.STRING }, 55 | ...(this.props.config as ddb.TableProps), 56 | }); 57 | break; 58 | case DatabaseProvider.RDS: 59 | this.resource = new rds.DatabaseInstance(this, 'Instance', { 60 | engine: rds.DatabaseInstanceEngine.AURORA, 61 | instanceClass: ec2.InstanceType.of( 62 | ec2.InstanceClass.BURSTABLE2, 63 | ec2.InstanceSize.SMALL, 64 | ), 65 | masterUsername: 'admin', 66 | ...(this.props.config as rds.DatabaseInstanceProps), 67 | vpc: this.props.network.vpc, 68 | }); 69 | // todo: fix this wide network permission 70 | this.resource.connections.allowFromAnyIpv4(ec2.Port.allTraffic()); 71 | break; 72 | case DatabaseProvider.RDSCluster: 73 | this.resource = new rds.DatabaseCluster(this, 'Cluster', { 74 | engine: rds.DatabaseClusterEngine.AURORA, 75 | masterUser: { 76 | username: 'admin', 77 | }, 78 | instanceProps: { 79 | vpc: this.props.network.vpc, 80 | instanceType: ec2.InstanceType.of( 81 | ec2.InstanceClass.BURSTABLE2, 82 | ec2.InstanceSize.SMALL, 83 | ), 84 | }, 85 | ...(this.props.config as rds.DatabaseClusterProps), 86 | }); 87 | // todo: fix this wide network permission 88 | this.resource.connections.allowFromAnyIpv4(ec2.Port.allTraffic()); 89 | break; 90 | default: 91 | assert.fail('storage type not supported'); 92 | } 93 | } 94 | 95 | grantAccess(service: Service): void { 96 | assert.ok(service.resource, 'service resource does not exist'); 97 | // this is a hack but it works, adds env vars to taskDefinition after synth 98 | const _addEnv = (key: string, val: string | number): void => { 99 | _.set( 100 | service.resource.taskDefinition, 101 | `defaultContainer.props.environment["${key}_${this.resource.node.id}"]`, 102 | val, 103 | ); 104 | }; 105 | switch (typeof this.resource) { 106 | case typeof ddb.Table: 107 | const dynamo = this.resource as ddb.Table; 108 | dynamo.grantFullAccess(service.resource.taskDefinition.taskRole); 109 | _addEnv(`DATABASE_TABLE_ARN`, dynamo.tableArn); 110 | _addEnv(`DATABASE_TABLE_NAME`, dynamo.tableName); 111 | break; 112 | case typeof rds.DatabaseInstance: 113 | const rdsi = this.resource as rds.DatabaseInstance; 114 | _addEnv(`DATABASE_ADDRESS`, rdsi.dbInstanceEndpointAddress); 115 | _addEnv(`DATABASE_PORT`, rdsi.dbInstanceEndpointPort); 116 | // TODO: add user/pass here 117 | break; 118 | case typeof rds.DatabaseCluster: 119 | const rdsc = this.resource as rds.DatabaseCluster; 120 | _addEnv(`DATABASE_ADDRESS`, rdsc.clusterEndpoint.hostname); 121 | _addEnv(`DATABASE_PORT`, rdsc.clusterEndpoint.port); 122 | // TODO: add user/pass here 123 | break; 124 | default: 125 | assert.fail('storage type not supported'); 126 | } 127 | } 128 | } 129 | -------------------------------------------------------------------------------- /test/container.test.ts: -------------------------------------------------------------------------------- 1 | import * as cdkAssert from '@aws-cdk/assert'; 2 | import * as cdk from '@aws-cdk/core'; 3 | import chai from 'chai'; 4 | import chaiAsPromised from 'chai-as-promised'; 5 | import { Container } from '../lib/resources/container'; 6 | 7 | chai.use(chaiAsPromised); 8 | 9 | describe('Container Construct Tests', () => { 10 | describe('DockerHub Tests', () => { 11 | it('should not create an ECR repository when a tag is given', () => { 12 | const app = new cdk.App(); 13 | const stack = new cdk.Stack(app, 'MyTestStack'); 14 | const construct = new Container(stack, 'MyTestContainer', { 15 | file: 'Dockerfile', 16 | uri: 'stelligent/mutato', 17 | }); 18 | cdkAssert 19 | .expect(stack) 20 | .notTo(cdkAssert.haveResource('AWS::ECR::Repository')); 21 | chai.assert.equal(construct.props.uri, 'stelligent/mutato'); 22 | }); 23 | 24 | it('should ba able to generate a build command with a tag', () => { 25 | const app = new cdk.App(); 26 | const stack = new cdk.Stack(app, 'MyTestStack'); 27 | const construct = new Container(stack, 'MyTestContainer', { 28 | file: 'Dockerfile', 29 | uri: 'stelligent/mutato', 30 | }); 31 | chai.assert.equal(construct.props.uri, 'stelligent/mutato'); 32 | chai 33 | .expect(construct.buildCommand) 34 | .to.be.equal( 35 | `docker build -t stelligent/mutato:latest -t stelligent/mutato:$mutato_opts__git__commit -f Dockerfile .`, 36 | ); 37 | const construct2 = new Container(stack, 'MyTestContainer2', { 38 | buildArgs: { 39 | key1: 'val1', 40 | key2: 'val2', 41 | }, 42 | file: 'Dockerfile2', 43 | context: 'Context2', 44 | uri: 'stelligent/mutato', 45 | }); 46 | chai 47 | .expect(construct2.buildCommand) 48 | .to.be.equal( 49 | `docker build --build-arg key1="val1" --build-arg key2="val2" -t stelligent/mutato:latest -t stelligent/mutato:$mutato_opts__git__commit -f Dockerfile2 Context2`, 50 | ); 51 | }); 52 | 53 | it('should ba able to generate a push command without a tag', () => { 54 | const app = new cdk.App(); 55 | const stack = new cdk.Stack(app, 'MyTestStack'); 56 | const construct = new Container(stack, 'MyTestContainer', { 57 | file: 'Dockerfile', 58 | uri: 'stelligent/mutato', 59 | }); 60 | chai.assert.equal(construct.props.uri, 'stelligent/mutato'); 61 | const uri = construct.getImageUri(); 62 | chai 63 | .expect(construct.pushCommand) 64 | .to.be.equal( 65 | `docker push ${uri}:latest && docker push ${uri}:$mutato_opts__git__commit`, 66 | ); 67 | }); 68 | }); 69 | 70 | describe('ECR Tests', () => { 71 | it('should create an ECR repository when no tag is given', () => { 72 | const app = new cdk.App(); 73 | const stack = new cdk.Stack(app, 'MyTestStack'); 74 | new Container(stack, 'MyTestContainer', { 75 | file: 'Dockerfile', 76 | }); 77 | cdkAssert 78 | .expect(stack) 79 | .to(cdkAssert.haveResource('AWS::ECR::Repository')); 80 | }); 81 | 82 | it('should ba able to generate a build command without a tag', () => { 83 | const app = new cdk.App(); 84 | const stack = new cdk.Stack(app, 'MyTestStack'); 85 | const construct = new Container(stack, 'MyTestContainer', { 86 | file: 'Dockerfile', 87 | }); 88 | 89 | const uri1 = construct.getImageUri(); 90 | chai 91 | .expect(construct.buildCommand) 92 | .to.be.equal( 93 | `docker build -t ${uri1}:latest -t ${uri1}:$mutato_opts__git__commit -f Dockerfile .`, 94 | ); 95 | const construct2 = new Container(stack, 'MyTestContainer2', { 96 | buildArgs: { 97 | key1: 'val1', 98 | key2: 'val2', 99 | }, 100 | file: 'Dockerfile2', 101 | context: 'Context2', 102 | }); 103 | const uri2 = construct2.getImageUri(); 104 | chai 105 | .expect(construct2.buildCommand) 106 | .to.be.equal( 107 | `docker build --build-arg key1="val1" --build-arg key2="val2" -t ${uri2}:latest -t ${uri2}:$mutato_opts__git__commit -f Dockerfile2 Context2`, 108 | ); 109 | }); 110 | 111 | it('should ba able to generate a push command without a tag', () => { 112 | const app = new cdk.App(); 113 | const stack = new cdk.Stack(app, 'MyTestStack'); 114 | const construct = new Container(stack, 'MyTestContainer', { 115 | file: 'Dockerfile', 116 | }); 117 | 118 | const uri = construct.getImageUri(); 119 | chai 120 | .expect(construct.pushCommand) 121 | .to.be.equal( 122 | `docker push ${uri}:latest && docker push ${uri}:$mutato_opts__git__commit`, 123 | ); 124 | }); 125 | }); 126 | }); 127 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to Mutato 2 | 3 | Help wanted! We'd love your contributions to Mutato. Please review the following 4 | guidelines before contributing. Also, feel free to propose changes to these 5 | guidelines by updating this file and submitting a pull request. 6 | 7 | - [I have a question...](#questions) 8 | - [I found a bug...](#bugs) 9 | - [I have a feature request...](#features) 10 | - [I have a contribution to share...](#process) 11 | 12 | ## Have a Question? 13 | 14 | Please don't open a GitHub issue for questions about how to use mutato, as the 15 | goal is to use issues for managing bugs and feature requests. Issues that are 16 | related to general support will be closed and redirected to our gitter room. 17 | 18 | For all support related questions, please ask the question in our gitter room: 19 | [stelligent/mutato](https://gitter.im/stelligent/mutato). 20 | 21 | ## Found a Bug? 22 | 23 | If you've identified a bug in mutato, please [submit an issue](#issue) to our 24 | GitHub repo: 25 | [stelligent/mutato](https://github.com/stelligent/mutato/issues/new). Please 26 | also feel free to submit a [Pull Request](#pr) with a fix for the bug! 27 | 28 | ## Have a Feature Request? 29 | 30 | All feature requests should start with [submitting an issue](#issue) documenting 31 | the user story and acceptance criteria. Again, feel free to submit a [Pull 32 | Request](#pr) with a proposed implementation of the feature. 33 | 34 | ## Ready to Contribute! 35 | 36 | ### Create an issue 37 | 38 | Before submitting a new issue, please search the issues to make sure there isn't 39 | a similar issue doesn't already exist. 40 | 41 | Assuming no existing issues exist, please ensure you include the following bits 42 | of information when submitting the issue to ensure we can quickly reproduce your 43 | issue: 44 | 45 | - Version of mutato 46 | - Platform (Linux, OS X, Windows) 47 | - The complete `mutato.yml` file used 48 | - The complete command that was executed 49 | - Any output from the command 50 | - Details of the expected results and how they differed from the actual results 51 | 52 | We may have additional questions and will communicate through the GitHub issue, 53 | so please respond back to our questions to help reproduce and resolve the issue 54 | as quickly as possible. 55 | 56 | New issues can be created with in our [GitHub 57 | repo](https://github.com/stelligent/mutato/issues/new). 58 | 59 | ### Pull Requests 60 | 61 | Pull requests should target the `develop` branch. Ensure you have a successful 62 | build for your branch. Please also reference the issue from the description of 63 | the pull request using [special keyword 64 | syntax](https://help.github.com/articles/closing-issues-via-commit-messages/) to 65 | auto close the issue when the PR is merged. For example, include the phrase 66 | `fixes #14` in the PR description to have issue #14 auto close. 67 | 68 | ### Styleguide 69 | 70 | When submitting code, please make every effort to follow existing conventions 71 | and style in order to keep the code as readable as possible. Here are a few 72 | points to keep in mind: 73 | 74 | - Please run `npm run-script lint:check` before committing to ensure code aligns 75 | with go standards. 76 | - Ensure all Classes, and Functions are documented appropriately as for the 77 | [JSDoc](https://devdocs.io/jsdoc/) standard. See the project `README.md` for 78 | these instructions. 79 | - All dependencies must be defined in the `package.json` file and pinned to a 80 | patch range. This is accomplished via a command like `npm install --save-dev 81 | {PKG}` for development dependencies or `npm install {PKG}` otherwise 82 | 83 | Also, consider the original design principles: 84 | 85 | - **Polyglot** - There will be no prescribed language or framework for 86 | developing the microservices. The only requirement will be that the service 87 | will be run inside a container and exposed via an HTTP endpoint. 88 | - **Cloud Provider** - At this point, the tool will assume AWS for the cloud 89 | provider and will not be written in a cloud agnostic manner. However, this 90 | does not preclude refactoring to add support for other providers at a later 91 | time. 92 | - **Declarative** - All resource administration will be handled in a declarative 93 | vs. imperative manner. A file will be used to declared the desired state of 94 | the resources and the tool will simply assert the actual state matches the 95 | desired state. The tool will accomplish this by generating CloudFormation 96 | templates. 97 | - **Stateless** - The tool will not maintain its own state. Rather, it will rely 98 | on the CloudFormation stacks to determine the state of the platform. 99 | - **Secure** - All security will be managed by AWS IAM credentials. No 100 | additional authentication or authorization mechanisms will be introduced. 101 | 102 | ### License 103 | 104 | By contributing your code, you agree to license your contribution under the 105 | terms of the [MIT License](LICENSE). 106 | 107 | All files are released with the MIT license. 108 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@stelligent/mutato", 3 | "version": "2.1.0", 4 | "description": "simplify the declaration and administration of the AWS resources necessary to support microservices.", 5 | "license": "MIT", 6 | "repository": { 7 | "type": "git", 8 | "url": "git+https://github.com/stelligent/mutato.git" 9 | }, 10 | "main": "out/lib/index.js", 11 | "types": "out/lib/index.d.ts", 12 | "bin": { 13 | "mutato": "out/bin/mutato.js" 14 | }, 15 | "scripts": { 16 | "postinstall": "npx patch-package", 17 | "build": "npm-run-all lint:write compile:emit docs:build", 18 | "clean": "git clean -xdff -e node_modules", 19 | "lint": "npm-run-all compile:check lint:check", 20 | "test": "npm-run-all build test:mocha", 21 | "compile:emit": "tsc", 22 | "compile:check": "tsc --noEmit", 23 | "test:mocha": "nyc mocha test/**/*.test.ts 2>npm-test.log", 24 | "lint:check": "eslint --quiet {lib,bin,test}/**/*.ts", 25 | "lint:write": "eslint --quiet --fix {lib,bin,test}/**/*.ts", 26 | "docs:serve": "docsify serve doc/", 27 | "docs:build": "jsdoc2md --files ./lib/**/*.ts --no-cache --configure ./jsdoc2md.json > ./doc/api.md", 28 | "deps:upgrade": "npx ncu -u -x @types/node", 29 | "bootstrap": "npx cdk bootstrap", 30 | "destroy": "npx cdk destroy --verbose Mutato-Pipeline*", 31 | "deploy": "npx cdk deploy --verbose Mutato-Pipeline*", 32 | "synth": "npx cdk synth --verbose -o $(ts-node -e \"console.log(require('./lib/config.ts').config.opts.git.local)\")/dist", 33 | "diff": "npx cdk diff --verbose Mutato-Pipeline*", 34 | "cdk": "npx cdk" 35 | }, 36 | "devDependencies": { 37 | "@aws-cdk/assert": "^1.32.0", 38 | "@babel/cli": "^7.8.4", 39 | "@babel/core": "^7.9.0", 40 | "@babel/plugin-proposal-class-properties": "^7.8.3", 41 | "@babel/plugin-proposal-object-rest-spread": "^7.9.0", 42 | "@babel/preset-env": "^7.9.0", 43 | "@babel/preset-typescript": "^7.9.0", 44 | "@istanbuljs/nyc-config-typescript": "^1.0.1", 45 | "@types/chai": "^4.2.11", 46 | "@types/chai-as-promised": "^7.1.2", 47 | "@types/debug": "^4.1.5", 48 | "@types/lodash": "^4.14.149", 49 | "@types/mocha": "^7.0.2", 50 | "@types/ms": "^0.7.31", 51 | "@types/node": "^12.12.34", 52 | "@types/nunjucks": "^3.1.3", 53 | "@types/parse-github-url": "^1.0.0", 54 | "@types/rc": "^1.1.0", 55 | "@types/traverse": "^0.6.32", 56 | "@types/yaml": "^1.2.0", 57 | "@typescript-eslint/eslint-plugin": "^2.27.0", 58 | "@typescript-eslint/parser": "^2.27.0", 59 | "chai": "^4.2.0", 60 | "chai-as-promised": "^7.1.1", 61 | "choma": "^1.2.1", 62 | "docsify-cli": "^4.4.0", 63 | "eslint": "^6.8.0", 64 | "eslint-config-prettier": "^6.10.1", 65 | "eslint-plugin-jsdoc": "^22.1.0", 66 | "eslint-plugin-prettier": "^3.1.2", 67 | "husky": "^4.2.3", 68 | "jsdoc-babel": "^0.5.0", 69 | "jsdoc-to-markdown": "^5.0.3", 70 | "mocha": "^7.1.1", 71 | "npm-run-all": "^4.1.5", 72 | "nyc": "^15.0.1", 73 | "prettier": "^2.0.4", 74 | "ts-node": "^8.8.2", 75 | "typescript": "^3.8.3" 76 | }, 77 | "dependencies": { 78 | "@aws-cdk/app-delivery": "^1.32.0", 79 | "@aws-cdk/aws-applicationautoscaling": "^1.32.0", 80 | "@aws-cdk/aws-codebuild": "^1.32.0", 81 | "@aws-cdk/aws-codepipeline": "^1.32.0", 82 | "@aws-cdk/aws-codepipeline-actions": "^1.32.0", 83 | "@aws-cdk/aws-dynamodb": "^1.32.0", 84 | "@aws-cdk/aws-ec2": "^1.32.0", 85 | "@aws-cdk/aws-ecr": "^1.32.0", 86 | "@aws-cdk/aws-ecs": "^1.32.0", 87 | "@aws-cdk/aws-ecs-patterns": "^1.32.0", 88 | "@aws-cdk/aws-iam": "^1.32.0", 89 | "@aws-cdk/aws-lambda": "^1.32.0", 90 | "@aws-cdk/aws-rds": "^1.32.0", 91 | "@aws-cdk/aws-s3": "^1.32.0", 92 | "@aws-cdk/aws-s3-assets": "^1.32.0", 93 | "@aws-cdk/aws-sqs": "^1.32.0", 94 | "@aws-cdk/core": "^1.32.0", 95 | "aws-cdk": "^1.32.0", 96 | "debug": "^4.1.1", 97 | "docker-parse-image": "^3.0.1", 98 | "gitignore-globs": "^0.1.1", 99 | "lodash": "^4.17.19", 100 | "ms": "^2.1.2", 101 | "nunjucks": "^3.2.1", 102 | "parse-github-url": "^1.0.2", 103 | "parse-strings-in-object": "^2.0.0", 104 | "patch-package": "^6.2.2", 105 | "rc": "^1.2.8", 106 | "source-map-support": "^0.5.16", 107 | "traverse": "^0.6.6", 108 | "yaml": "^1.8.3" 109 | }, 110 | "mocha": { 111 | "bail": true, 112 | "fullTrace": true, 113 | "recursive": true, 114 | "inlineDiffs": true, 115 | "require": [ 116 | "choma", 117 | "ts-node/register", 118 | "source-map-support/register" 119 | ] 120 | }, 121 | "nyc": { 122 | "extends": "@istanbuljs/nyc-config-typescript", 123 | "reporter": [ 124 | "html", 125 | "text-summary" 126 | ], 127 | "all": false 128 | }, 129 | "prettier": { 130 | "semi": true, 131 | "tabWidth": 2, 132 | "printWidth": 80, 133 | "singleQuote": true, 134 | "trailingComma": "all" 135 | }, 136 | "eslintConfig": { 137 | "root": true, 138 | "parser": "@typescript-eslint/parser", 139 | "parserOptions": { 140 | "project": "tsconfig.json" 141 | }, 142 | "extends": [ 143 | "plugin:jsdoc/recommended", 144 | "plugin:@typescript-eslint/recommended", 145 | "prettier/@typescript-eslint", 146 | "plugin:prettier/recommended" 147 | ], 148 | "rules": { 149 | "jsdoc/require-param-type": 0, 150 | "jsdoc/require-returns-type": 0, 151 | "@typescript-eslint/camelcase": 0 152 | }, 153 | "settings": { 154 | "jsdoc": { 155 | "mode": "typescript" 156 | } 157 | } 158 | }, 159 | "husky": { 160 | "hooks": { 161 | "pre-commit": "tsc --noEmit && lint-staged" 162 | } 163 | }, 164 | "lint-staged": { 165 | "*.{js,ts}": [ 166 | "eslint --fix", 167 | "git add" 168 | ] 169 | } 170 | } 171 | -------------------------------------------------------------------------------- /lib/resources/container.ts: -------------------------------------------------------------------------------- 1 | import * as ecr from '@aws-cdk/aws-ecr'; 2 | import * as cdk from '@aws-cdk/core'; 3 | import * as assert from 'assert'; 4 | import debug from 'debug'; 5 | import _ from 'lodash'; 6 | import { config } from '../config'; 7 | // eslint-disable-next-line @typescript-eslint/no-var-requires 8 | const dockerImageParse = require('docker-parse-image'); 9 | 10 | interface ContainerProps { 11 | /** build time parameters passed to "docker build" */ 12 | buildArgs?: { [key: string]: string }; 13 | /** path to Dockerfile (default: Dockerfile) */ 14 | file?: string; 15 | /** path to build context (default: current working directory) */ 16 | context?: string; 17 | /** image's push URI. leave empty if using AWS ECR */ 18 | uri?: string; 19 | /** optional array of tags to apply if we are building the container */ 20 | tags?: string[]; 21 | } 22 | 23 | /** 24 | * a construct abstracting a single Dockerfile. This class does not participate 25 | * in authentication, building, or pushing the actual image of the container. 26 | */ 27 | export class Container extends cdk.Construct { 28 | public readonly props: ContainerProps; 29 | public readonly needsBuilding: boolean; 30 | public readonly repo?: ecr.Repository; 31 | private readonly _debug: debug.Debugger; 32 | private readonly _ecrRepoName?: string; 33 | 34 | /** 35 | * @hideconstructor 36 | * @param scope CDK scope 37 | * @param id CDK construct id 38 | * @param props CDK construct parameters 39 | */ 40 | constructor(scope: cdk.Construct, id: string, props: ContainerProps) { 41 | super(scope, id); 42 | 43 | this._debug = debug(`mutato:constructs:container:${id}`); 44 | this.props = _.defaults(props, { 45 | buildArgs: {}, 46 | context: '.', 47 | file: '', 48 | uri: '', 49 | tags: [], 50 | }); 51 | 52 | this._debug('creating a container construct with props: %o', this.props); 53 | assert.ok(this.props.context); 54 | assert.ok(_.isString(this.props.uri)); 55 | 56 | if (this.props.file) { 57 | if (this.props.uri) { 58 | this._debug('container is building for Docker Hub'); 59 | assert.ok(config.opts.docker.user && config.opts.docker.pass); 60 | const { tag } = dockerImageParse(this.props.uri); 61 | // if a tag is given in the URI, remove it and add it to "tags" 62 | if (tag) this.props.tags?.push(tag); 63 | this.props.uri = this.props.uri?.replace(`:${tag}`, ''); 64 | } else { 65 | this._debug('container is building for AWS ECR'); 66 | const git = config.getGithubMetaData(); 67 | this._ecrRepoName = `mutato/${git.identifier}`; 68 | this.repo = new ecr.Repository(this, 'repository', { 69 | removalPolicy: cdk.RemovalPolicy.DESTROY, 70 | repositoryName: this._ecrRepoName, 71 | }); 72 | const uri = this.repo.repositoryUri; 73 | this._debug('overriding container uri to: %s', uri); 74 | this.props.uri = uri; 75 | } 76 | } 77 | 78 | assert.ok(this.props.uri); 79 | if (_.isEmpty(this.props.tags)) this.props.tags = ['latest']; 80 | this.props.tags?.push('$mutato_opts__git__commit'); 81 | this.needsBuilding = !!this.props.file; 82 | this._debug('uri: %s, tags: %o', this.getImageUri(this), this._tags); 83 | } 84 | 85 | /** 86 | * @param caller optional construct in a different stack needing to access the 87 | * image URI without referencing the stack that is building the container. 88 | * @param latest whether we should return the image with its latest GIT tag. 89 | * @returns Get the container image's URI for use in ECS. Optionally caller 90 | * can be used to get a portable URI independent of the stack building this 91 | * container with a precondition that caller exists in the same AWS region and 92 | * account. 93 | */ 94 | getImageUri(caller?: cdk.Construct, latest = false): string { 95 | const tag = latest ? `:${config.opts.git.commit}` : ''; 96 | if (caller && this.repo) { 97 | // little hack so we can use this container cross-stacks without circular 98 | // errors thrown by CloudFormation. the build order is guaranteed so this 99 | // is safe here. 100 | const stack = cdk.Stack.of(caller); 101 | const suffix = stack.urlSuffix; 102 | const region = stack.region; 103 | const account = stack.account; 104 | const repo = this._ecrRepoName; 105 | const uri = `${account}.dkr.ecr.${region}.${suffix}/${repo}${tag}`; 106 | return uri; 107 | } else 108 | return this.needsBuilding 109 | ? // Docker Hub images we are building 110 | `${this.props.uri}${tag}` 111 | : // Off the shelf images 112 | (this.props.uri as string); 113 | } 114 | 115 | /** @returns shell command containing "docker login" */ 116 | get loginCommand(): string { 117 | const region = cdk.Stack.of(this).region; 118 | return this.repo 119 | ? `$(aws ecr get-login --no-include-email --region ${region})` 120 | : config.opts.docker.user && config.opts.docker.pass 121 | ? `docker login -u ${config.opts.docker.user} -p ${config.opts.docker.pass}` 122 | : 'echo "skipping docker login (credentials not provided)"'; 123 | } 124 | 125 | /** @returns shell command containing "docker build" */ 126 | get buildCommand(): string { 127 | assert.ok(this.needsBuilding, 'container is not part of the pipeline'); 128 | const buildArgs = _.reduce( 129 | this.props.buildArgs, 130 | (accumulate, value, key) => `${accumulate} --build-arg ${key}="${value}"`, 131 | '', 132 | ).trim(); 133 | const file = this.props.file; 134 | const context = this.props.context; 135 | const tagArgs = this._tags.map((tag) => `-t ${tag}`).join(' '); 136 | return `docker build ${buildArgs} ${tagArgs} -f ${file} ${context}`; 137 | } 138 | 139 | /** @returns shell command containing "docker push" */ 140 | get pushCommand(): string { 141 | assert.ok(this.needsBuilding, 'container is not part of the pipeline'); 142 | return this._tags.map((tag) => `docker push ${tag}`).join(' && '); 143 | } 144 | 145 | private get _tags(): string[] { 146 | const imageUri = this.getImageUri(); 147 | return _.uniq(this.props.tags).map((tag) => `${imageUri}:${tag}`); 148 | } 149 | } 150 | -------------------------------------------------------------------------------- /lib/resources/service.ts: -------------------------------------------------------------------------------- 1 | import * as ecs from '@aws-cdk/aws-ecs'; 2 | import * as sqs from '@aws-cdk/aws-sqs'; 3 | import * as ecsPatterns from '@aws-cdk/aws-ecs-patterns'; 4 | import * as iam from '@aws-cdk/aws-iam'; 5 | import * as cdk from '@aws-cdk/core'; 6 | import * as appAutoScaling from '@aws-cdk/aws-applicationautoscaling'; 7 | import assert from 'assert'; 8 | import debug from 'debug'; 9 | import _ from 'lodash'; 10 | import { Container } from './container'; 11 | import { Network } from './network'; 12 | import { Storage } from './storage'; 13 | import { config } from '../config'; 14 | 15 | enum ServiceProvider { 16 | Fargate = 'fargate', 17 | Classic = 'classic', 18 | FargateTask = 'fargate-task', 19 | ClassicTask = 'classic-task', 20 | FargateQueue = 'fargate-queue', 21 | ClassicQueue = 'classic-queue', 22 | } 23 | 24 | interface ServiceProps { 25 | provider?: ServiceProvider; 26 | container: Container; 27 | storage?: Storage; 28 | network: Network; 29 | rate?: string; 30 | config?: 31 | | ecsPatterns.ScheduledEc2TaskProps 32 | | ecsPatterns.ScheduledFargateTaskProps 33 | | ecsPatterns.QueueProcessingEc2ServiceProps 34 | | ecsPatterns.QueueProcessingFargateServiceProps 35 | | ecsPatterns.ApplicationLoadBalancedEc2ServiceProps 36 | | ecsPatterns.ApplicationLoadBalancedFargateServiceProps; 37 | } 38 | 39 | /** ECS service construct */ 40 | export class Service extends cdk.Construct { 41 | public readonly props: ServiceProps; 42 | public readonly resource: 43 | | ecsPatterns.ScheduledEc2Task 44 | | ecsPatterns.ScheduledFargateTask 45 | | ecsPatterns.QueueProcessingEc2Service 46 | | ecsPatterns.QueueProcessingFargateService 47 | | ecsPatterns.ApplicationLoadBalancedFargateService 48 | | ecsPatterns.ApplicationLoadBalancedEc2Service; 49 | private readonly _debug: debug.Debugger; 50 | 51 | /** 52 | * @hideconstructor 53 | * @param scope CDK construct scope 54 | * @param id CDK construct ID 55 | * @param props service configuration 56 | */ 57 | constructor(scope: cdk.Construct, id: string, props: ServiceProps) { 58 | super(scope, id); 59 | 60 | this._debug = debug(`mutato:constructs:service:${id}`); 61 | this.props = _.defaults(props, { 62 | provider: ServiceProvider.Fargate, 63 | rate: 'rate(1 day)', 64 | }); 65 | 66 | this._debug('creating a service construct with props: %o', this.props); 67 | assert.ok(_.isString(this.props.provider)); 68 | assert.ok(_.isObject(this.props.container)); 69 | assert.ok(_.isObject(this.props.network)); 70 | 71 | const imageUri = this.props.container.getImageUri(this, true /* latest */); 72 | // a unique container to force an ECS cluster update to pull down changes 73 | const containerName = `container-${config.opts.git.commit}-${Date.now()}`; 74 | const ecrPullPolicy = new iam.PolicyStatement({ 75 | effect: iam.Effect.ALLOW, 76 | actions: [ 77 | 'ecr:GetAuthorizationToken', 78 | 'ecr:BatchCheckLayerAvailability', 79 | 'ecr:GetDownloadUrlForLayer', 80 | 'ecr:BatchGetImage', 81 | ], 82 | // TODO fix this and limit the scope 83 | resources: ['*'], 84 | }); 85 | 86 | switch (this.props.provider) { 87 | case ServiceProvider.Fargate: 88 | this.resource = new ecsPatterns.ApplicationLoadBalancedFargateService( 89 | this, 90 | `Fargate`, 91 | { 92 | cluster: this.props.network.cluster, 93 | taskImageOptions: { 94 | image: ecs.ContainerImage.fromRegistry(imageUri), 95 | environment: { MUTATO_CONTAINER_NAME: containerName }, 96 | containerName, 97 | }, 98 | ...(this.props 99 | .config as ecsPatterns.ApplicationLoadBalancedFargateServiceProps), 100 | }, 101 | ); 102 | break; 103 | case ServiceProvider.Classic: 104 | this.resource = new ecsPatterns.ApplicationLoadBalancedEc2Service( 105 | this, 106 | `Classic`, 107 | { 108 | cluster: this.props.network.cluster, 109 | taskImageOptions: { 110 | image: ecs.ContainerImage.fromRegistry(imageUri), 111 | environment: { MUTATO_CONTAINER_NAME: containerName }, 112 | containerName, 113 | }, 114 | ...(this.props 115 | .config as ecsPatterns.ApplicationLoadBalancedEc2ServiceProps), 116 | }, 117 | ); 118 | break; 119 | case ServiceProvider.FargateTask: 120 | assert.ok(this.props.rate, 'CloudWatch rate expression must be set'); 121 | this.resource = new ecsPatterns.ScheduledFargateTask( 122 | this, 123 | `FargateTask`, 124 | { 125 | cluster: this.props.network.cluster, 126 | scheduledFargateTaskImageOptions: { 127 | image: ecs.ContainerImage.fromRegistry(imageUri), 128 | environment: { MUTATO_CONTAINER_NAME: containerName }, 129 | }, 130 | schedule: appAutoScaling.Schedule.expression( 131 | this.props.rate as string, 132 | ), 133 | ...(this.props.config as ecsPatterns.ScheduledFargateTaskProps), 134 | }, 135 | ); 136 | break; 137 | case ServiceProvider.ClassicTask: 138 | assert.ok(this.props.rate, 'CloudWatch rate expression must be set'); 139 | this.resource = new ecsPatterns.ScheduledEc2Task(this, `ClassicTask`, { 140 | cluster: this.props.network.cluster, 141 | scheduledEc2TaskImageOptions: { 142 | image: ecs.ContainerImage.fromRegistry(imageUri), 143 | environment: { MUTATO_CONTAINER_NAME: containerName }, 144 | }, 145 | schedule: appAutoScaling.Schedule.expression( 146 | this.props.rate as string, 147 | ), 148 | ...(this.props.config as ecsPatterns.ScheduledEc2TaskProps), 149 | }); 150 | break; 151 | case ServiceProvider.FargateQueue: 152 | assert.ok(this.props.storage, 'storage must be set'); 153 | assert.ok(this.props.storage?.resource, 'storage resource must be set'); 154 | this.resource = new ecsPatterns.QueueProcessingFargateService( 155 | this, 156 | 'FargateQueue', 157 | { 158 | image: ecs.ContainerImage.fromRegistry(imageUri), 159 | environment: { MUTATO_CONTAINER_NAME: containerName }, 160 | queue: this.props.storage?.resource as sqs.Queue, 161 | cluster: this.props.network.cluster, 162 | ...(this.props 163 | .config as ecsPatterns.QueueProcessingFargateServiceProps), 164 | }, 165 | ); 166 | break; 167 | case ServiceProvider.ClassicQueue: 168 | assert.ok(this.props.storage, 'storage must be set'); 169 | assert.ok(this.props.storage?.resource, 'storage resource must be set'); 170 | this.resource = new ecsPatterns.QueueProcessingEc2Service( 171 | this, 172 | 'ClassicQueue', 173 | { 174 | image: ecs.ContainerImage.fromRegistry(imageUri), 175 | environment: { MUTATO_CONTAINER_NAME: containerName }, 176 | queue: this.props.storage?.resource as sqs.Queue, 177 | cluster: this.props.network.cluster, 178 | ...(this.props 179 | .config as ecsPatterns.QueueProcessingEc2ServiceProps), 180 | }, 181 | ); 182 | break; 183 | default: 184 | assert.fail('storage type not supported'); 185 | } 186 | 187 | this.resource.taskDefinition.addToExecutionRolePolicy(ecrPullPolicy); 188 | } 189 | } 190 | -------------------------------------------------------------------------------- /lib/app.ts: -------------------------------------------------------------------------------- 1 | import * as cicd from '@aws-cdk/app-delivery'; 2 | import * as codeBuild from '@aws-cdk/aws-codebuild'; 3 | import * as codePipeline from '@aws-cdk/aws-codepipeline'; 4 | import * as codePipelineActions from '@aws-cdk/aws-codepipeline-actions'; 5 | import * as s3Assets from '@aws-cdk/aws-s3-assets'; 6 | import * as s3 from '@aws-cdk/aws-s3'; 7 | import * as cdk from '@aws-cdk/core'; 8 | import * as iam from '@aws-cdk/aws-iam'; 9 | import assert from 'assert'; 10 | import debug from 'debug'; 11 | import fs from 'fs'; 12 | import _ from 'lodash'; 13 | import path from 'path'; 14 | import * as Actions from './actions'; 15 | import { config } from './config'; 16 | import { MutatoSpec, Parser } from './parser'; 17 | import { Container } from './resources/container'; 18 | import { Network } from './resources/network'; 19 | import { Service } from './resources/service'; 20 | import { Storage } from './resources/storage'; 21 | import { Database } from './resources/database'; 22 | // eslint-disable-next-line @typescript-eslint/no-var-requires 23 | const toGlob = require('gitignore-globs'); 24 | 25 | /** 26 | * This class holds together a Mutato Pipeline (a Stack) and Mutato Resources (a 27 | * Stack deployed through the Mutato Pipeline stack). This is the entry point to 28 | * all Mutato powered microservice infrastructures. 29 | */ 30 | export class App extends cdk.App { 31 | private readonly _parser = new Parser(); 32 | private readonly _debug = debug('mutato:App'); 33 | private static MUTATO_YML = path.resolve(config.opts.git.local, 'mutato.yml'); 34 | 35 | /** 36 | * initializes this Mutato App from a valid Mutato YAML file 37 | * 38 | * @param file Mutato YAML file path. By default it looks under your 39 | * current working directory for mutato.yml 40 | */ 41 | public async synthesizeFromFile(file = App.MUTATO_YML): Promise { 42 | this._debug('synthesizing Mutato app from file: %s', file); 43 | const yamlString = await fs.promises.readFile(file, { encoding: 'utf8' }); 44 | this.synthesizeFromString(yamlString); 45 | } 46 | 47 | /** 48 | * initializes this Mutato stack from a valid Mutato YAML string 49 | * 50 | * @param string Mutato YAML string 51 | */ 52 | public async synthesizeFromString(string: string): Promise { 53 | this._debug('synthesizing Mutato app from string: %s', string); 54 | const ymlObject = this._parser.parse(string); 55 | await this.synthesizeFromObject(ymlObject as MutatoSpec); 56 | } 57 | 58 | /** 59 | * initializes this Mutato stack from a valid Mutato YAML object (converted to JSON) 60 | * 61 | * @param spec a valid Mutato YAML object 62 | */ 63 | public async synthesizeFromObject(spec: MutatoSpec): Promise { 64 | this._debug('synthesizing Mutato app from object: %o', spec); 65 | assert.ok(_.isObject(spec)); 66 | 67 | const git = config.getGithubMetaData(); 68 | this._debug('git meta data extracted: %o', git); 69 | 70 | const __ = function (name: string): string { 71 | return `${name}-${git.identifier}`; 72 | }; 73 | 74 | this._debug('creating a stack (Mutato Pipeline)'); 75 | const pipelineStack = new cdk.Stack(this, 'MutatoPipeline', { 76 | description: 'pipeline that manages deploy of mutato.yml resources', 77 | stackName: __('Mutato-Pipeline'), 78 | }); 79 | 80 | this._debug('creating a CodePipeline to manage Mutato resources'); 81 | const pipeline = new codePipeline.Pipeline(pipelineStack, __('pipeline'), { 82 | restartExecutionOnUpdate: true, 83 | }); 84 | 85 | this._debug('creating an artifact to store Github source'); 86 | const githubSource = new codePipeline.Artifact('S'); 87 | this._debug('creating an action that pulls source from Github'); 88 | const source = new codePipelineActions.GitHubSourceAction({ 89 | variablesNamespace: 'GH', 90 | actionName: 'GitHub', 91 | output: githubSource, 92 | owner: git.owner, 93 | repo: git.repo, 94 | branch: git.branch, 95 | /** @todo add SSM here to read github token from */ 96 | oauthToken: cdk.SecretValue.plainText(config.opts.git.secret), 97 | }); 98 | this._debug('adding Github action to the pipeline'); 99 | pipeline.addStage({ 100 | stageName: 'Mutato-Source', 101 | actions: [source], 102 | }); 103 | 104 | let variables: { [key: string]: codeBuild.BuildEnvironmentVariable } = {}; 105 | // explanation on WTF is going on here: if "bundle" isn't configured through 106 | // environment variables, that means user is executing mutato outside of the 107 | // CodeBuild environment. in that case, we capture mutato's source into .zip 108 | // and send it to CodeBuild as a CDK asset. CodeBuild won't re-run this code 109 | // since "config.opts.bundle" is defined for it. 110 | if (!process.env.CODEBUILD_BUILD_ID) { 111 | this._debug('running outside of CodeBuild, package up mutato'); 112 | assert.ok(!config.opts.bundle.bucket && !config.opts.bundle.object); 113 | this._debug('freezing the list of env vars to send to CodeBuild'); 114 | const envFile = _.map( 115 | { 116 | ...config.toStringEnvironmentMap(), 117 | ...spec.environmentVariables, 118 | USER: 'root', 119 | DEBUG: 'mutato*', 120 | DEBUG_COLORS: '0', 121 | }, 122 | (v, k) => `export ${k}=${v};`, 123 | ).join('\n'); 124 | this._debug('variables env file: %s', envFile); 125 | await fs.promises.writeFile('.env', envFile, { encoding: 'utf-8' }); 126 | assert.ok(fs.existsSync('.env')); 127 | const bundle = new s3Assets.Asset(pipelineStack, __('mutato-asset'), { 128 | exclude: _.concat('.git', 'mutato.yml', toGlob('.gitignore'), '!.env'), 129 | path: process.cwd(), 130 | }); 131 | variables = config.toBuildEnvironmentMap({ 132 | mutato_opts__bundle__bucket: bundle.s3BucketName, 133 | mutato_opts__bundle__object: bundle.s3ObjectKey, 134 | }); 135 | } else { 136 | // the first time user deploys through their terminal, this causes a loop 137 | // from Synth -> Update -> Synth again because the underlying CFN template 138 | // is changing from using a CDN param (CDK asset) to an inline bucket. but 139 | // the result is the same, therefore it goes out of the loop 140 | assert.ok(config.opts.bundle.bucket && config.opts.bundle.object); 141 | const bundle = `s3://${config.opts.bundle.bucket}/${config.opts.bundle.object}`; 142 | this._debug('running inside CodeBuild, using mutato bundle: %s', bundle); 143 | // so that it is not destroyed when synth stage passes for the first time 144 | s3.Bucket.fromBucketName( 145 | pipelineStack, 146 | __('mutato-bucket'), 147 | config.opts.bundle.bucket, 148 | ); 149 | variables = config.toBuildEnvironmentMap({ 150 | mutato_opts__bundle__bucket: config.opts.bundle.bucket, 151 | mutato_opts__bundle__object: config.opts.bundle.object, 152 | }); 153 | } 154 | 155 | this._debug('creating a CodeBuild project that synthesizes myself'); 156 | const project = new codeBuild.PipelineProject(pipelineStack, 'build', { 157 | environment: { 158 | buildImage: codeBuild.LinuxBuildImage.fromDockerRegistry('node:lts'), 159 | environmentVariables: variables, 160 | }, 161 | buildSpec: codeBuild.BuildSpec.fromObject({ 162 | version: 0.2, 163 | phases: { 164 | build: { 165 | commands: [ 166 | '/bin/bash', 167 | // make sure mutato knows where user's repo is mounted 168 | 'export mutato_opts__git__local=`pwd`', 169 | // install AWS CLI 170 | 'mkdir -p /aws-cli && cd /aws-cli', 171 | 'curl "s3.amazonaws.com/aws-cli/awscli-bundle.zip" -o "awscli-bundle.zip"', 172 | 'unzip awscli-bundle.zip', 173 | './awscli-bundle/install -i /usr/local/aws -b /usr/local/bin/aws', 174 | // create the mutato bundle address 175 | `export BUNDLE="s3://$mutato_opts__bundle__bucket/$mutato_opts__bundle__object"`, 176 | // pull down mutato's bundle used to create this pipeline 177 | 'mkdir -p /mutato && cd /mutato', 178 | 'aws s3 cp "$BUNDLE" .', 179 | 'unzip $(basename "$BUNDLE")', 180 | // prepare the environment 181 | 'chmod +x .env && . ./.env && rm .env', 182 | // do cdk synth, mutato knows about user's repo over env vars 183 | 'npm install && npm run synth', 184 | // show the user what changes they just pushed 185 | 'npm run --silent cdk -- diff || true', 186 | ], 187 | }, 188 | }, 189 | artifacts: { 'base-directory': 'dist', files: '**/*' }, 190 | }), 191 | }); 192 | 193 | // band-aid for admin permission issues during deploy. FIXME 194 | this._debug('granting admin permission to the synthesize build stage'); 195 | project.addToRolePolicy( 196 | new iam.PolicyStatement({ 197 | resources: ['*'], 198 | actions: ['*'], 199 | }), 200 | ); 201 | 202 | this._debug('creating an artifact to store synthesized self'); 203 | const synthesizedApp = new codePipeline.Artifact(); 204 | this._debug('creating an action for the pipeline to actually build self'); 205 | const buildAction = new codePipelineActions.CodeBuildAction({ 206 | actionName: 'CodeBuild', 207 | project, 208 | input: githubSource, 209 | outputs: [synthesizedApp], 210 | environmentVariables: { 211 | mutato_opts__git__commit: { value: source.variables.commitId }, 212 | }, 213 | }); 214 | 215 | this._debug('adding self build action to the pipeline'); 216 | pipeline.addStage({ 217 | stageName: 'Mutato-Synthesize', 218 | actions: [buildAction], 219 | }); 220 | 221 | this._debug('adding a self update stage'); 222 | pipeline.addStage({ 223 | stageName: 'Mutato-Update', 224 | actions: [ 225 | new cicd.PipelineDeployStackAction({ 226 | stack: pipelineStack, 227 | input: synthesizedApp, 228 | adminPermissions: true, 229 | }), 230 | ], 231 | }); 232 | 233 | const containerSpecs = spec.containers; 234 | this._debug('containers specs: %o', containerSpecs); 235 | const containers = _.map(containerSpecs, (containerSpec) => { 236 | const type = _.head(_.keys(containerSpec)) as string; 237 | assert.ok(type === 'docker'); 238 | const prop = _.get(containerSpec, type); 239 | const name = _.get(prop, 'name', 'default'); 240 | const construct = new Container(pipelineStack, name, prop); 241 | return construct; 242 | }) as Container[]; 243 | const queryContainer = ( 244 | nameOrUri: string, 245 | requester: string, 246 | ): Container => { 247 | this._debug('resolving container: %s', nameOrUri); 248 | const container = _.find(containers, (c) => c.node.id === nameOrUri); 249 | return container 250 | ? container 251 | : new Container(pipelineStack, `volatile-${requester}-${nameOrUri}`, { 252 | uri: nameOrUri, 253 | }); 254 | }; 255 | 256 | const actionSpecs = spec.actions; 257 | this._debug('action specs: %o', actionSpecs); 258 | const actions = _.map(actionSpecs, (actionSpec) => { 259 | const type = _.head(_.keys(actionSpec)) as string; 260 | const prop = _.get(actionSpec, type); 261 | const name = _.get(prop, 'name'); 262 | assert.ok(name, 'actions must have a name'); 263 | switch (type) { 264 | case 'docker': 265 | return new Actions.DockerRun({ 266 | name, 267 | ...prop, 268 | pipeline, 269 | source: githubSource, 270 | sourceAction: source, 271 | container: queryContainer(prop.container, name), 272 | }); 273 | case 'codebuild': 274 | return new Actions.CodeBuild({ 275 | name, 276 | ...prop, 277 | pipeline, 278 | source: githubSource, 279 | sourceAction: source, 280 | container: _.isString(prop.container) 281 | ? queryContainer(prop.container, name) 282 | : undefined, 283 | }); 284 | case 'approval': 285 | return new Actions.Approval({ name, ...prop }); 286 | default: 287 | assert.fail(`action type not supported: ${type}`); 288 | } 289 | }); 290 | 291 | this._debug('checking to see if we have any containers to build'); 292 | const pipelineContainers = containers.filter((c) => c.needsBuilding); 293 | if (pipelineContainers.length > 0) { 294 | this._debug('we are building containers, adding its stages'); 295 | const containersStage = pipeline.addStage({ 296 | stageName: 'Mutato-Containers', 297 | }); 298 | 299 | const havePreBuild = !!containerSpecs 300 | .map((c) => _.head(_.values(c))) 301 | .filter((p) => _.get(p, 'events["pre-build"]') && _.get(p, 'file')) 302 | ?.length; 303 | this._debug('container pre build events found: %s', havePreBuild); 304 | let containerPreBuildStage: codePipeline.IStage; 305 | if (havePreBuild) { 306 | containerPreBuildStage = pipeline.addStage({ 307 | stageName: 'Mutato-Containers-Pre-Build', 308 | placement: { 309 | rightBefore: containersStage, 310 | }, 311 | }); 312 | } 313 | 314 | const havePostBuild = !!containerSpecs 315 | .map((c) => _.head(_.values(c))) 316 | .filter((p) => _.get(p, 'events["post-build"]') && _.get(p, 'file')) 317 | ?.length; 318 | this._debug('container post build events found: %s', havePostBuild); 319 | let containerPostBuildStage: codePipeline.IStage; 320 | if (havePostBuild) { 321 | containerPostBuildStage = pipeline.addStage({ 322 | stageName: 'Mutato-Containers-Post-Build', 323 | placement: { 324 | justAfter: containersStage, 325 | }, 326 | }); 327 | } 328 | 329 | pipelineContainers.forEach((container) => { 330 | const events = _.get( 331 | containerSpecs 332 | .map((c) => _.head(_.values(c))) 333 | .find((c) => _.get(c, 'name', 'default') === container.node.id) || 334 | {}, 335 | 'events', 336 | {}, 337 | ); 338 | 339 | const preBuildEventSpecs = _.get(events, 'pre-build') as string[]; 340 | const preBuildEvents = 341 | (_.isString(preBuildEventSpecs) 342 | ? [preBuildEventSpecs] 343 | : preBuildEventSpecs) || []; 344 | preBuildEvents 345 | .map((ev) => 346 | actions.find((actionFactory) => actionFactory.name === ev), 347 | ) 348 | .forEach((actionFactory) => 349 | containerPreBuildStage?.addAction( 350 | actionFactory?.action( 351 | `${container.node.id}-pre-build`, 352 | ) as codePipeline.IAction, 353 | ), 354 | ); 355 | 356 | containersStage.addAction( 357 | new Actions.DockerBuild({ 358 | name: `build-${container.node.id}`, 359 | source: githubSource, 360 | sourceAction: source, 361 | container, 362 | pipeline, 363 | }).action(container.node.id), 364 | ); 365 | 366 | const postBuildEventSpecs = _.get(events, 'post-build') as string[]; 367 | const postBuildEvents = 368 | (_.isString(postBuildEventSpecs) 369 | ? [postBuildEventSpecs] 370 | : postBuildEventSpecs) || []; 371 | postBuildEvents 372 | .map((ev) => 373 | actions.find((actionFactory) => actionFactory.name === ev), 374 | ) 375 | .forEach((actionFactory) => 376 | containerPostBuildStage?.addAction( 377 | actionFactory?.action( 378 | `${container.node.id}-post-build`, 379 | ) as codePipeline.IAction, 380 | ), 381 | ); 382 | }); 383 | } 384 | 385 | Array.from(spec.environments.keys()).forEach((envName) => { 386 | const resources = spec.environments.get(envName); 387 | if (resources?.length === 1) { 388 | this._debug('environment is empty, skipping it: %s', envName); 389 | return; 390 | } 391 | const queryConstruct = (type: string): object[] => 392 | _.filter( 393 | (resources?.filter( 394 | (c) => _.head(_.keys(c)) === type, 395 | ) as object[]).map((c) => _.get(c, type)), 396 | ); 397 | const environment = _.head(queryConstruct('environment')); 398 | this._debug('creating environment: %s / %o', envName, environment); 399 | 400 | this._debug('creating a stack (Mutato Resources)'); 401 | const envStack = new cdk.Stack(this, `MutatoResources-${envName}`, { 402 | description: `application resources for environment: ${envName}`, 403 | stackName: __(`Mutato-App-${envName}`), 404 | }); 405 | 406 | const networkSpecs = queryConstruct('network'); 407 | assert.ok(networkSpecs.length <= 1); 408 | const networkProp = _.head(networkSpecs); 409 | const networkName = `network-${envName}`; 410 | const networkConstruct = new Network(envStack, networkName, networkProp); 411 | 412 | const storages = queryConstruct('storage').map((props) => { 413 | const storageName = _.get(props, 'name', `storage-${envName}`); 414 | return new Storage(envStack, storageName, props); 415 | }); 416 | 417 | const databases = queryConstruct('database').map((props) => { 418 | const databaseName = _.get(props, 'name', `database-${envName}`); 419 | return new Database(envStack, databaseName, { 420 | ...props, 421 | network: networkConstruct, 422 | }); 423 | }); 424 | 425 | queryConstruct('service').forEach((props) => { 426 | const serviceName = _.get(props, 'name', `service-${envName}`); 427 | const containerNameOrUri = _.get(props, 'container', 'default'); 428 | const service = new Service(envStack, serviceName, { 429 | ...props, 430 | network: networkConstruct, 431 | container: queryContainer(containerNameOrUri, serviceName), 432 | }); 433 | storages.forEach((storage) => storage.grantAccess(service)); 434 | databases.forEach((database) => database.grantAccess(service)); 435 | }); 436 | 437 | this._debug('adding environment deploy stage'); 438 | const deployStage = pipeline.addStage({ 439 | stageName: `Mutato-${envName}-Deploy`, 440 | actions: [ 441 | new cicd.PipelineDeployStackAction({ 442 | stack: envStack, 443 | input: synthesizedApp, 444 | adminPermissions: true, 445 | }), 446 | ], 447 | }); 448 | 449 | const havePreDeploy = !!_.get(environment, 'events["pre-deploy"]'); 450 | if (havePreDeploy) { 451 | const preDeployEventSpecs = _.get( 452 | environment, 453 | 'events["pre-deploy"]', 454 | ) as string[]; 455 | const preDeployEvents = 456 | (_.isString(preDeployEventSpecs) 457 | ? [preDeployEventSpecs] 458 | : preDeployEventSpecs) || []; 459 | pipeline.addStage({ 460 | stageName: `Mutato-${envName}-Pre-Deploy`, 461 | placement: { rightBefore: deployStage }, 462 | actions: preDeployEvents.map( 463 | (ev) => 464 | actions 465 | .find((actionFactory) => actionFactory.name === ev) 466 | ?.action(`${envName}-pre-deploy`) as codePipeline.IAction, 467 | ), 468 | }); 469 | } 470 | 471 | const havePostDeploy = !!_.get(environment, 'events["post-deploy"]'); 472 | if (havePostDeploy) { 473 | const postDeployEventSpecs = _.get( 474 | environment, 475 | 'events["post-deploy"]', 476 | ) as string[]; 477 | const postDeployEvents = 478 | (_.isString(postDeployEventSpecs) 479 | ? [postDeployEventSpecs] 480 | : postDeployEventSpecs) || []; 481 | pipeline.addStage({ 482 | stageName: `Mutato-${envName}-Post-Deploy`, 483 | placement: { justAfter: deployStage }, 484 | actions: postDeployEvents.map( 485 | (ev) => 486 | actions 487 | .find((actionFactory) => actionFactory.name === ev) 488 | ?.action(`${envName}-post-deploy`) as codePipeline.IAction, 489 | ), 490 | }); 491 | } 492 | }); 493 | } 494 | } 495 | --------------------------------------------------------------------------------