├── .eslintignore ├── .eslintrc ├── .gitignore ├── .nvmrc ├── .terraform-version ├── .travis.yml ├── LEARNING.md ├── LICENSE.txt ├── README.md ├── aws-lambda-serverless-reference-Hero.png ├── aws └── bootstrap.yml ├── layers ├── repeat │ └── repeat │ │ └── index.js └── vendor │ └── nodejs │ ├── package.json │ └── yarn.lock ├── package.json ├── serverless.yml ├── src └── server │ ├── base.js │ ├── layers.js │ └── xray.js ├── terraform ├── main.tf ├── role-ci.tf ├── scripts │ └── dev.js ├── variables.tf └── versions.tf └── yarn.lock /.eslintignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | -------------------------------------------------------------------------------- /.eslintrc: -------------------------------------------------------------------------------- 1 | --- 2 | extends: 3 | - formidable/configurations/es6-node 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .git 2 | 3 | .DS_Store 4 | .project 5 | .vscode 6 | node_modules 7 | npm-debug.log* 8 | yarn-error.log* 9 | package-lock.json 10 | 11 | aws/.service* 12 | 13 | .serverless 14 | .terraform 15 | _warmup 16 | -------------------------------------------------------------------------------- /.nvmrc: -------------------------------------------------------------------------------- 1 | 12.14.0 -------------------------------------------------------------------------------- /.terraform-version: -------------------------------------------------------------------------------- 1 | 0.12.18 2 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | 3 | node_js: 4 | - "12" 5 | 6 | branches: 7 | only: 8 | - master 9 | 10 | env: 11 | global: 12 | - TF_VERSION=0.12.18 13 | 14 | before_install: 15 | - wget https://releases.hashicorp.com/terraform/${TF_VERSION}/terraform_${TF_VERSION}_linux_amd64.zip -O /tmp/terraform.zip 16 | - sudo unzip -d /usr/local/bin/ /tmp/terraform.zip 17 | 18 | install: 19 | # Fail if lockfile outdated. 20 | # https://yarnpkg.com/lang/en/docs/cli/install/#toc-yarn-install-frozen-lockfile 21 | - yarn install --frozen-lockfile 22 | 23 | script: 24 | - yarn --version 25 | - yarn run check:ci 26 | -------------------------------------------------------------------------------- /LEARNING.md: -------------------------------------------------------------------------------- 1 | Learning 🎓 2 | ========== 3 | 4 | This project's primary audience is experienced cloud infrastructure folks looking to production-ize a modern Serverless stack in a single AWS account. But, it's also an appropriate place to learn the various parts along the way! 5 | 6 | ## Contents 7 | 8 | 9 | 10 | 11 | 12 | - [Projects](#projects) 13 | - [Getting Started](#getting-started) 14 | - [Exercises](#exercises) 15 | - [Environments](#environments) 16 | - [Lambda Exercises](#lambda-exercises) 17 | - [Infrastructure Exercises](#infrastructure-exercises) 18 | 19 | 20 | 21 | ## Projects 22 | 23 | We have **three** main reference projects from which to learn: 24 | 25 | * [aws-lambda-serverless-reference](https://github.com/FormidableLabs/aws-lambda-serverless-reference): (**This project!**) Our most basic Serverless reference with a full production infrastructure. 26 | * [aws-lambda-dogs](https://github.com/FormidableLabs/aws-lambda-dogs): A simple REST API Serverless reference using [json-server](https://github.com/typicode/json-server). If you don't need VPC or are unsure of where to begin, start here. 27 | * [badges](https://github.com/FormidableLabs/badges): Our advanced reference project that includes more complex enhancements like: per-PR environments, github-flow support, promote-to-production artifacts, etc. Once you've mastered manual CF + TF + SLS deploys, come over here and see how far automation can take us! 28 | 29 | ## Getting Started 30 | 31 | 1. (_For Formidables only_). Contact Roemer to get three AWS IAM users. The `-developer` and `-admin` users will be manually attached to the appropriate `sandbox` stage groups (`tf-simple-reference-sandbox-developer`, `tf-simple-reference-sandbox-admin`) for use with `serverless` commands. 32 | 1. `FIRST.LAST` user: An AWS superadmin that can do things like deploy the bootstrap CloudFormation and service Terraform infrastructures. This user reflects a DevOps lead in a client project. Should only be used for `yarn cf:*` and `yarn tf:*` commands. 33 | 2. `FIRST.LAST-developer` user: A user with IAM group permissions for the specific serverless project + `STAGE` to update an existing deployment. This reflects an engineer on a client project with limited privileges or maybe a CI system for staging or whatnot. Should only be used for `yarn lambda:*` commands. 34 | 3. `FIRST.LAST-admin` user: A user with IAM group permissions for the specific serverless project + `STAGE` to update + create/delete any deployment. This reflects an engineer on a client project with full privileges over the serverless application, but not the supporting cloud infrastructure. Should only be used for `yarn lambda:*` commands. 35 | 36 | 2. Then, follow all the basic directions for installation in the [README](./README.md). 37 | 1. (_For Formidables only_). Please set up `aws-vault` per the instructions for your authentication method. 38 | 39 | 3. Check that your credentials are set up for all appropriate work: 40 | 41 | ```sh 42 | # 1. Check CloudFormation as superadmin 43 | $ STAGE=sandbox aws-vault exec FIRST.LAST --no-session -- \ 44 | yarn cf:bootstrap:status 45 | ... 46 | "UPDATE_COMPLETE" # (or some other status) 47 | 48 | # 2. Check Terraform as superadmin 49 | # 2.a. Init to appropriate stage backend. 50 | $ STAGE=sandbox aws-vault exec FIRST.LAST --no-session -- \ 51 | yarn tf:service:init --reconfigure 52 | ... 53 | Terraform has been successfully initialized! 54 | 55 | # 2.b. Run plan to see any changes from our code vs. actual infrastructure. 56 | $ STAGE=sandbox aws-vault exec FIRST.LAST --no-session -- \ 57 | yarn tf:service:plan 58 | ... 59 | No changes. Infrastructure is up-to-date. 60 | 61 | # 3. Check Serverless as `-developer` 62 | $ STAGE=sandbox aws-vault exec FIRST.LAST-developer --no-session -- \ 63 | yarn lambda:info 64 | ... 65 | Service Information 66 | service: sls-simple-reference 67 | stage: sandbox 68 | region: us-east-1 69 | stack: sls-simple-reference-sandbox 70 | ... 71 | ``` 72 | 73 | ## Exercises 74 | 75 | ### Environments 76 | 77 | > `tl:dr`: use `sandbox` and only do `yarn lambda:*` commands for intro exercises. 78 | 79 | **Lambda** 80 | 81 | All of the introductory exercises are performed in the pre-existing `sandbox` environment using only `yarn lambda:*` commands to do serverless deployments as an `-admin` or `-developer` user. This means that you should just let folks know in an appropriate channel that you are "taking over" the environment. 82 | 83 | **Infrastructure** (_CloudFormation + Terraform_) 84 | 85 | Your superadmin user should be reserved only for work on _new_ environments. If you choose to do a new environment, it is _very important_ that you talk to Tyler or Roemer and refactor this application to grep and comment out all sections titled `OPTION` in `serverless.yml` and `terraform/main.tf` to remove all the extra bells and whistles (particularly VPC which is slow and complicates things) in a temporary branch to make your infrastructure work much easier and more focused. 86 | 87 | ### Lambda Exercises 88 | 89 | 1. Completely review and read the `README.md` one more time and ask questions in a slack channel! 90 | 91 | 1. Do a deployment of the serverless branch as-is off `master` branch: 92 | 93 | ```sh 94 | $ STAGE=sandbox aws-vault exec FIRST.LAST-developer --no-session -- \ 95 | yarn lambda:deploy 96 | ``` 97 | 98 | ... then go and kick the tires on a URL! 99 | 100 | The endpoints created will be listed under `endpoints` in the console output. Please note that the endpoints MUST end with `/`. If the output is `https://ii178wi5hi.execute-api.us-east-1.amazonaws.com/sandbox/base`, then you must use `https://ii178wi5hi.execute-api.us-east-1.amazonaws.com/sandbox/base/` (notice the trailing `/`) in your browser. 101 | 102 | 1. Delete the serverless application and re-deploy it as-is off `master` branch: 103 | 104 | ```sh 105 | # Be careful! 106 | $ STAGE=sandbox aws-vault exec FIRST.LAST-admin --no-session -- \ 107 | yarn lambda:_delete 108 | 109 | # Now, the `lambda:deploy` command needs an `-admin` user to create. 110 | $ STAGE=sandbox aws-vault exec FIRST.LAST-admin --no-session -- \ 111 | yarn lambda:deploy 112 | ``` 113 | 114 | 1. Create a temporary branch and edit something in `src/server/base.js` that will be visible from a public URL. Then deploy it and check the results! 115 | 116 | ```sh 117 | $ STAGE=sandbox aws-vault exec FIRST.LAST-developer --no-session -- \ 118 | yarn lambda:deploy 119 | ``` 120 | 121 | 1. See if there are any [open issues](https://github.com/FormidableLabs/aws-lambda-serverless-reference/issues) that need just a serverless / application code fix and try to open a pull request! Or, be a saint and update all the root project dependencies in `package.json` to keep us up to date, verify everything still works, and open a pull request. 122 | 123 | ### Infrastructure Exercises 124 | 125 | Once you've got the basics of serverless deployment down, you can head over to [aws-lambda-dogs Infrastructure Exercises](https://github.com/FormidableLabs/aws-lambda-dogs/blob/master/LEARNING.md#infrastructure-exercises) to do things with your AWS superadmin user to the infrastructure (without having to deal with a VPC). 126 | 127 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2019 Formidable Labs 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![AWS Lambda Serverless Reference — Formidable, We build the modern web](https://raw.githubusercontent.com/FormidableLabs/aws-lambda-serverless-reference/master/aws-lambda-serverless-reference-Hero.png)](https://formidable.com/open-source/) 2 | 3 | [![Travis Status][trav_img]][trav_site] 4 | [![Maintenance Status][maintenance-image]](#maintenance-status) 5 | 6 | A simple "hello world" reference app for the [`FormidableLabs/serverless/aws`][FormidableLabs/serverless/aws] Terraform module, using the [serverless][] framework targeting an AWS Lambda deploy. 7 | 8 | ## Contents 9 | 10 | 11 | 12 | 13 | 14 | - [Overview](#overview) 15 | - [Audience](#audience) 16 | - [Stack](#stack) 17 | - [Naming](#naming) 18 | - [Stages](#stages) 19 | - [Environment Variables](#environment-variables) 20 | - [User Roles](#user-roles) 21 | - [Installation](#installation) 22 | - [Node.js (Runtime)](#nodejs-runtime) 23 | - [AWS (Deployment)](#aws-deployment) 24 | - [AWS Tools](#aws-tools) 25 | - [Terraform](#terraform) 26 | - [AWS Credentials](#aws-credentials) 27 | - [In Environment](#in-environment) 28 | - [Saved to Local Disk](#saved-to-local-disk) 29 | - [AWS Vault](#aws-vault) 30 | - [Development](#development) 31 | - [Node.js](#nodejs) 32 | - [Lambda Offline](#lambda-offline) 33 | - [Support Stack Provisioning (Superuser)](#support-stack-provisioning-superuser) 34 | - [Bootstrap Stack](#bootstrap-stack) 35 | - [Service Stack](#service-stack) 36 | - [Serverless Deployment (IAM Roles)](#serverless-deployment-iam-roles) 37 | - [Admin Deployment](#admin-deployment) 38 | - [User Deployment](#user-deployment) 39 | - [`terraform-aws-serverless` Development](#terraform-aws-serverless-development) 40 | 41 | 42 | 43 | ## Overview 44 | 45 | Getting a `serverless` application into the cloud "the right way" can be a challenge. To this end, we start with a super-simple, "hello world" Express app targeting AWS Lambda using serverless. Along the way, this reference project takes care of all of the **tough** supporting pieces that go into a production-ready, best-practices-following cloud infrastructure like: 46 | 47 | - Local development workflows. 48 | - Terraform stack controlling IAM permissions and cloud resources to support a vanilla `serverless` application. 49 | - Remote state management for Terraform. 50 | - Serverless application deployment and production lifecycle management. 51 | 52 | Using this project as a template, you can hopefully take a new `serverless` application and set up "everything else" to support it in AWS the right way, from the start. 53 | 54 | A great starting point is our [introduction blog post](https://formidable.com/blog/2019/locking-down-aws-serverless-applications-the-right-way/) that explains existing privilege approaches, motivations for this project, and some quick example infrastructures. 55 | 56 | ### Audience 57 | 58 | This reference application is meant for developers / architects who are already familiar with AWS infrastructures (and CloudFormation), Terraform, and Serverless framework applications. This project will hopefully provide some guidance / examples to get the whole shebang all the way to a multi-environment deployment and support a team of administrators and engineers for the application. 59 | 60 | For folks (particularly [Formidables](https://formidable.com/)) interested in learning more about how we construct our production cloud infrastructures, head over to our [learning page](./LEARNING.md). 61 | 62 | ### Stack 63 | 64 | We use very simple, very common tools to allow a mostly vanilla Express server to run in localdev / Docker like a normal Node.js HTTP server and _also_ as a Lambda function exposed via API Gateway. 65 | 66 | Tech stack: 67 | 68 | * [`express`](serverless-http): A server. 69 | 70 | Infrastructure stack: 71 | 72 | * [serverless][]: Build / deployment framework for getting code to Lambda. 73 | * [serverless-http][]: Bridge to make a vanilla Express server run on Lambda. 74 | 75 | Infrastructure tools: 76 | 77 | * AWS [CloudFormation][]: Create AWS cloud resources using YAML. The `serverless` framework creates a CloudFormation stack of Lambda-supporting resources as part of a normal deployment. This project also uses a small CloudFormation stack to bootstrap an S3 bucket and DynamoDB to handle Terraform state. 78 | * HashiCorp [Terraform][]: Create AWS cloud resources using [HCL][]. Typically more flexible and expressive than CloudFormation. We have a simple Terraform stack that uses the [`FormidableLabs/serverless/aws`][FormidableLabs/serverless/aws] module to set up a production-ready set of resources (IAM, monitoring, etc.) to support the resources/stack generated by `serverless`. 79 | 80 | ### Naming 81 | 82 | We use a naming convention in cloud resources and `yarn` tasks to separate some various high level things: 83 | 84 | * `cf`: AWS CloudFormation specific names. 85 | * `tf`: Terraform specific names. 86 | * `sls`: Serverless framework names. 87 | 88 | ### Stages 89 | 90 | Development hits a local machine, and when programmatically named, is usually referred to as: 91 | 92 | * `localdev`: A development-only setup running on a local machine. 93 | 94 | We target four different stages/environments of AWS hosted deployments: 95 | 96 | * `sandbox`: A loose environment where developers can manually push / check things / break things with impunity. Typically deployed from developer laptops. 97 | * `development`: Tracks feature development branches. Typically deployed by CI on merges to `develop` branch if using git flow workflow. 98 | * `staging`: A near-production environment to validate changes before committing to actual production. Typically deployed by CI for release candidate branches before merging to `master`. 99 | * `production`: The real deal. Typically deployed by CI after a merge to `master`. 100 | 101 | Note that these are completely arbitrary groups, both in composition and naming. There a sensible set of groups if you need just some starting point. But the final group (or even one if you want) is totally up to you! 102 | 103 | All of our `yarn run ` tasks should be run with a `STAGE=` prefix. The default is to assume `STAGE=localdev` and only commands like `yarn run node:localdev` or `yarn run lambda:localdev` can run without specification successfully. For commands actually targeting AWS, please prefix like: 104 | 105 | ```sh 106 | $ STAGE=sandbox yarn run 107 | $ STAGE=development yarn run 108 | $ STAGE=stage yarn run 109 | $ STAGE=production yarn run 110 | ``` 111 | 112 | _Note_: We separate the `STAGE` variable from `NODE_ENV` because often there are build implications of `NODE_ENV` that are distinct from our notion of deploy target environments. 113 | 114 | ### Environment Variables 115 | 116 | Our task runner scheme is a bash + `yarn` based system crafted around the following environment variables (with defaults): 117 | 118 | * `STAGE`: `localdev` 119 | * `SERVICE_NAME`: `simple-reference` (The name of the application/service in the cloud.) 120 | * `AWS_REGION`: `us-east-1` 121 | 122 | ... and some minor localdev only ones: 123 | 124 | * `AWS_XRAY_CONTEXT_MISSING`: `LOG_ERROR` (Have Xray not error in localdev) 125 | * `SERVER_PORT`: `3000` 126 | * `SERVER_HOST`: `0.0.0.0` 127 | 128 | ... and some implied ones: 129 | 130 | * `FUNCTION_NAME`: The name of a given Lambda function. In this project, the main one is `server`. 131 | 132 | If your project supports Windows, you will want to have a more general / permissive approach. 133 | 134 | ### User Roles 135 | 136 | We rely on IAM roles to limit privileges to the minimum necessary to provision, update, and deploy the service. Typically this involves creating personalized users in the AWS console, and then assigning them groups for varying appropriate degrees of privilege. Here are the relevant ones for this reference project: 137 | 138 | * **Superuser - Support Stack**: A privileged user that can create the initial bootstrap CloudFormation stack and Terraform service module that will support a Serverless application. It should not be used for Serverless deploys. 139 | * **IAM Groups - Serverless App**: The [`FormidableLabs/serverless/aws`][FormidableLabs/serverless/aws] module provides IAM groups and support for different types of users to create/update/delete the Serverless application. The IAM groups created are: 140 | * `tf-${SERVICE_NAME}-${STAGE}-admin`: Can create/delete/update the Severless app. 141 | * `tf-${SERVICE_NAME}-${STAGE}-developer`: Can deploy the Severless app. 142 | * `tf-${SERVICE_NAME}-${STAGE}-ci`: Can deploy the Severless app. 143 | 144 | > ℹ️ **Note**: Our cloud infrastructure is based on an approach of a single shared AWS account (with many limited IAM users). A more secure and differently complex option is to use _separate_ AWS accounts for different stages/environments for infrastructures/applications. We discuss these approaches more in our [introductory blog post](https://formidable.com/blog/2019/locking-down-aws-serverless-applications-the-right-way/#existing-privilege-approaches) for the `FormidableLabs/serverless/aws` Terraform module. 145 | > 146 | > In practice, many real world projects will segregate at least the ultimate production infrastructure to a separate AWS account and potentially utilize multiple infrastructures within a shared non-production AWS account. There are many ways to implement a robust production privilege approach, and this reference project implements just one of them! 147 | 148 | ## Installation 149 | 150 | ### Node.js (Runtime) 151 | 152 | Our application is a Node.js server. 153 | 154 | First, make sure you have our version of node (determined by `.nvmrc`) that matches our Lambda target (you will need to have [`nvm`](https://github.com/creationix/nvm) installed): 155 | 156 | ```sh 157 | $ nvm use 158 | ``` 159 | 160 | Then, `yarn` install the Node.js dependencies: 161 | 162 | ```sh 163 | $ yarn install 164 | ``` 165 | 166 | ### AWS (Deployment) 167 | 168 | #### AWS Tools 169 | 170 | Certain administrative / development work require the AWS CLI tools to prepare and deploy our staging / production services. To get those either do: 171 | 172 | ```sh 173 | # Install via Python 174 | $ sudo pip install awscli --ignore-installed six 175 | 176 | # Or brew 177 | $ brew install awscli 178 | ``` 179 | 180 | After this you should be able to type: 181 | 182 | ```sh 183 | $ aws --version 184 | ``` 185 | 186 | #### Terraform 187 | 188 | Install [tfenv][] from Homebrew: `brew install tfenv`. Then, in the root of the repo, run `tfenv install` to download and use the pinned version of Terraform for this project. 189 | 190 | Note that `tfenv` conflicts with Homebrew `terraform` and must be uninstalled first. You can still use `tfenv` to install and use the latest Terraform version in projects that don't have a `.terraform-version` file. 191 | 192 | #### AWS Credentials 193 | 194 | To work with this reference app, you need AWS credentials for your specific user (aka, `FIRST.LAST`). To create the bootstrap and service support stacks, that user will need to be a superuser. To deploy `serverless` applications, the user will need to be attached to given `tf-${SERVICE_NAME}-${STAGE}-(admin|developer)` IAM groups after the service stack is created. 195 | 196 | Once you have a user + access + secret keys, you need to make them available to commands requiring them. There are a couple of options: 197 | 198 | ##### In Environment 199 | 200 | You can append the following two environment variables to any command like: 201 | 202 | ```sh 203 | $ AWS_ACCESS_KEY_ID=INSERT \ 204 | AWS_SECRET_ACCESS_KEY=INSERT \ 205 | STAGE=sandbox \ 206 | yarn run lambda:info 207 | ``` 208 | 209 | This has the advantage of not storing secrets on disk. The disadvantage is needing to keep the secrets around to paste and/or `export` into every new terminal. 210 | 211 | ##### Saved to Local Disk 212 | 213 | Another option is to store the secrets on disk. You can configure your `~/.aws` credentials like: 214 | 215 | ```sh 216 | $ mkdir -p ~/.aws 217 | $ touch ~/.aws/credentials 218 | ``` 219 | 220 | Then add a `default` entry if you only anticipate working on this one project or a named profile entry of your username (aka, `FIRST.LAST`): 221 | 222 | ```sh 223 | $ vim ~/.aws/credentials 224 | [default|FIRST.LAST] 225 | aws_access_key_id = INSERT 226 | aws_secret_access_key = INSERT 227 | ``` 228 | 229 | If you are using a named profile, then export it into the environment in any terminal you are working in: 230 | 231 | ```sh 232 | $ export AWS_PROFILE="FIRST.LAST" 233 | $ STAGE=sandbox yarn run lambda:info 234 | ``` 235 | 236 | Or, you can declare the variable inline: 237 | 238 | ```sh 239 | $ AWS_PROFILE="FIRST.LAST"\ 240 | STAGE=sandbox \ 241 | yarn run lambda:info 242 | ``` 243 | 244 | ##### AWS Vault 245 | 246 | The most secure mix of the two above options is to install and use 247 | [aws-vault](https://github.com/99designs/aws-vault). Once you've followed the installation instructions, you can set up and use a profile like: 248 | 249 | ```sh 250 | # Store AWS credentials for a profile named "FIRST.LAST" 251 | $ aws-vault add FIRST.LAST 252 | Enter Access Key Id: INSERT 253 | Enter Secret Key: INSERT 254 | 255 | # Execute a command with temporary creds 256 | $ STAGE=sandbox aws-vault exec FIRST.LAST -- yarn run lambda:info 257 | ``` 258 | 259 | > ⚠️ **Warning**: Certain IAM role creation commands do not work with the default `aws-vault` setup if you have MFA set up (which you should). 260 | 261 | The following commands that definitely need extra command support: 262 | 263 | * `yarn tf:service:apply` 264 | * `yarn tf:service:_delete` 265 | * `yarn lambda:deploy` 266 | 267 | We have a [research ticket](https://github.com/FormidableLabs/aws-lambda-serverless-reference/issues/38) to better handle sessions with MFA, but in the meantime you can simply add the `--no-session` flag to any `aws-vault` commands that need it. E.g. 268 | 269 | ```sh 270 | $ STAGE=sandbox aws-vault exec FIRST.LAST --no-session -- 271 | 272 | # E.g. 273 | $ STAGE=sandbox aws-vault exec FIRST.LAST --no-session -- STAGE=sandbox yarn tf:service:apply 274 | ``` 275 | 276 | In practice, it is probably easier in the meantime to just always add the `--no-session` flag when using `aws-vault exec`. 277 | 278 | ## Development 279 | 280 | We have several options for developing a service locally, with different advantages. Here's a quick list of application ports / running commands: 281 | 282 | * `3000`: Node server via `nodemon`. (`yarn node:localdev`) 283 | * `3001`: Lambda offline local simulation. (`yarn lambda:localdev`) 284 | 285 | ### Node.js 286 | 287 | Run the server straight up in your terminal with Node.js via `nodemon` for instant restarts on changes. Note that because we effectively run the same server at different URL bases you must separately specify them. 288 | 289 | ```sh 290 | # Base server 291 | $ yarn node:localdev 292 | 293 | # Different scenarios that reuse base.js 294 | $ BASE_URL=/canary yarn node:localdev 295 | $ BASE_URL=/vpc yarn node:localdev 296 | $ BASE_URL=/layers yarn node:localdev 297 | 298 | # Other scenarios that use a different js file. 299 | $ SCENARIO=xray yarn node:localdev 300 | ``` 301 | 302 | See it in action!: 303 | 304 | - [http://127.0.0.1:3000/base/hello.json](http://127.0.0.1:3000/base/hello.json) 305 | 306 | Or from the command line: 307 | 308 | ```sh 309 | $ curl -X POST "http://127.0.0.1:3000/base/hello.json" \ 310 | -H "Content-Type: application/json" 311 | ``` 312 | 313 | ### Lambda Offline 314 | 315 | Run the server in a Lambda simulation via the [`serverless-offline`](https://github.com/dherault/serverless-offline) plugin. In contrast to `node:localdev` above, _all_ routes and functions are loaded together. 316 | 317 | ```sh 318 | $ yarn lambda:localdev 319 | ``` 320 | 321 | See it in action!: 322 | 323 | - [http://127.0.0.1:3001/base/hello.json](http://127.0.0.1:3001/base/hello.json) 324 | 325 | ## Support Stack Provisioning (Superuser) 326 | 327 | This section discusses getting AWS resources provisioned to support Terraform and then Serverless. 328 | 329 | The basic overview is: 330 | 331 | 1. **Bootstrap Stack**: Use AWS CloudFormation to provision resources to manage Terraform state. 332 | 2. **Service Stack**: Use Terraform to provision resources / permissions to accompany a Serverless deploy. 333 | 334 | after this, _then_ we are ready to deploy a standard `serverless` application with full support! 335 | 336 | ### Bootstrap Stack 337 | 338 | This step creates an S3 bucket and DynamoDB data store to enable Terraform to remotely manage it's state. We do this via AWS CloudFormation. 339 | 340 | All commands in this section should be run by an AWS superuser. The configuration for all of this section is controlled by: [`aws/bootstrap.yml`](./aws/bootstrap.yml). Commands and resources created are all prefixed with `cf` as a project-specific choice for ease of identification in the AWS console (vs. Terraform vs. Serverless-generated). 341 | 342 | **Create** the CloudFormation stack: 343 | 344 | ```sh 345 | # Provision stack. 346 | $ STAGE=sandbox yarn run cf:bootstrap:create 347 | { 348 | "StackId": "arn:aws:cloudformation:${AWS_REGION}:${AWS_ACCOUNT}:stack/cf-${SERVICE_NAME}-${STAGE}-bootstrap/HASH" 349 | } 350 | 351 | # Check status until reach `CREATE_COMPLETE` 352 | $ STAGE=sandbox yarn run cf:bootstrap:status 353 | "CREATE_COMPLETE" 354 | ``` 355 | 356 | Once this is complete, you can move on to provisioning the service stack section. The remaining commands below are only if you need to update / delete the bootstrap stack, which shouldn't happen that often. 357 | 358 | **Update** the CloudFormation stack: 359 | 360 | ```sh 361 | # Update, then check status. 362 | $ STAGE=sandbox yarn run cf:bootstrap:update 363 | $ STAGE=sandbox yarn run cf:bootstrap:status 364 | ``` 365 | 366 | **Delete** the CloudFormation stack: 367 | 368 | The bootstrap stack should only be deleted _after_ you have removed all of the `-admin|-developer|-ci` groups from users and deleted the Serverless and Terraform service stacks. 369 | 370 | ```sh 371 | # **WARNING**: Use with extreme caution!!! 372 | $ STAGE=sandbox yarn run cf:bootstrap:_delete 373 | 374 | # Check status. (A status or error with `does not exist` when done). 375 | $ STAGE=sandbox yarn run cf:bootstrap:status 376 | An error occurred (ValidationError) when calling the DescribeStacks operation: Stack with id cf-SERVICE_NAME-STAGE does not exist 377 | ``` 378 | 379 | ### Service Stack 380 | 381 | This step provisions a Terraform stack to provide us with IAM groups and other AWS resources to support and enhance a Serverless provision (in the next section). 382 | 383 | All commands in this section should be run by an AWS superuser. The configuration for all of this section is controlled by: [`terraform/main.tf`](./terraform/main.tf). Commands and resources created are all prefixed with `tf` as a project-specific choice for ease of identification. 384 | 385 | > ℹ️ **Note**: We use the `terraform` CLI program under the hood directly for all of our Terraform work. This is simple and good for learning, but in a real world infrastructure has several limitations (such as the pain of remembering to re-`init` environments on switching, etc.). Consequently, if you're looking to maintain multiple environments with Terraform in the real world, consider more flexible meta tools like [terragrunt](https://github.com/gruntwork-io/terragrunt). 386 | 387 | **Init** your local Terraform state. 388 | 389 | This needs to be run once to be able to run any other Terraform commands. 390 | 391 | ```sh 392 | $ STAGE=sandbox yarn run tf:service:init --reconfigure 393 | ``` 394 | 395 | > ⚠️ **Warning**: You need to run `yarn run tf:service:init` **every** time you change `STAGE` or other core environmental setup before you can mutate anything with the stack (like `yarn run tf:service:apply`). Failure to do so will result in bad things like incorrect stage variables applied to an old, stale stage in the underlying Terraform local disk cache. 396 | 397 | > ℹ️ **Note**: We suggest using the `--reconfigure` flag every time you run `init` when switching environments so that the remote state (in S3) remains the source of truth and accidental stuff you do on local disk doesn't end up corrupting things. 398 | 399 | **Plan** the Terraform stack. 400 | 401 | Terraform allows you to see what's going to happen / change in your cloud infrastructure before actually committing to it, so it is _always_ a good idea to run a plan before any Terraform mutating command. 402 | 403 | ```sh 404 | $ STAGE=sandbox yarn run tf:service:plan 405 | ``` 406 | 407 | **Apply** the Terraform stack: 408 | 409 | This creates / updates as appropriate. 410 | 411 | ```sh 412 | # Type in `yes` to go forward 413 | $ STAGE=sandbox yarn run tf:service:apply 414 | 415 | # YOLO: run without checking first 416 | $ STAGE=sandbox yarn run tf:service:apply -auto-approve 417 | 418 | # **WARNING**: If using `aws-vault`, remember `--no-session`! 419 | $ STAGE=sandbox aws-vault exec FIRST.LAST --no-session -- STAGE=sandbox yarn tf:service:apply 420 | ``` 421 | 422 | **Delete** the Terraform stack: 423 | 424 | The service stack should only be deleted _after_ you have removed all of the `-admin|-developer|-ci` groups from users and deleted the Serverless stack. 425 | 426 | ```sh 427 | # **WARNING**: Use with extreme caution!!! 428 | # Type in `yes` to go forward 429 | $ STAGE=sandbox yarn run tf:service:_delete 430 | 431 | # YOLO: run without checking first 432 | $ STAGE=sandbox yarn run tf:service:_delete -auto-approve 433 | 434 | # **WARNING**: If using `aws-vault`, remember `--no-session`! 435 | $ STAGE=sandbox aws-vault exec FIRST.LAST --no-session -- STAGE=sandbox yarn tf:service:_delete 436 | ``` 437 | 438 | **Visualize** the Terraform stack: 439 | 440 | These are Mac-based instructions, but analogous steps are available on other platforms. First, you'll need GraphViz for the `dot` tool: 441 | 442 | ```sh 443 | $ brew install graphviz 444 | ``` 445 | 446 | From there, you can visualize with: 447 | 448 | ```sh 449 | # Generate SVG 450 | $ STAGE=sandbox yarn run -s tf:terraform graph | dot -Tsvg > ~/Desktop/infrastructure.svg 451 | ``` 452 | 453 | ## Serverless Deployment (IAM Roles) 454 | 455 | This section discusses developers getting code and secrets deployed (manually from local machines to an AWS `development` playground or automated via CI). 456 | 457 | All commands in this section should be run by AWS users with attached IAM groups provisioned by our support stack of `tf-${SERVICE_NAME}-${STAGE}-(admin|developer|ci)`. The configuration for this section is controlled by: [`serverless.yml`](./serverless.yml) 458 | 459 | > ⚠️ **Prod/Real World Warning**: This reference application deploys from local laptops for ease of instruction. However, our laptops are usually a different operating system than the target Lambda Linux execution environment. This is an issue for binary dependencies in `node_modules` which are OS-specific and zipped up and shipped with the Lambda application. 460 | > 461 | > Our reference application presently does not have binary dependencies, but as a best practice for a real world Lambda application, you should not package and deploy from a different OS than your target Lambda execution environment. This means if locally deploying using an appropriate Docker setup for packaging, or using a CI/CD system that matches the Lambda OS to package and deploy the application. 462 | 463 | ### Admin Deployment 464 | 465 | These actions are reserved for `-admin` users. 466 | 467 | **Create** the Lambda app. The first time through a `deploy`, an `-admin` user 468 | is required (to effect the underlying CloudFormation changes). 469 | 470 | ```sh 471 | $ STAGE=sandbox yarn run lambda:deploy 472 | 473 | # **WARNING**: If using `aws-vault`, remember `--no-session`! 474 | $ STAGE=sandbox aws-vault exec FIRST.LAST --no-session -- STAGE=sandbox yarn lambda:deploy 475 | 476 | # Check on app and endpoints. 477 | $ STAGE=sandbox yarn run lambda:info 478 | ``` 479 | 480 | **Delete** the Lambda app. 481 | 482 | ```sh 483 | # **WARNING**: Use with extreme caution!!! 484 | $ STAGE=sandbox yarn run lambda:_delete 485 | 486 | # Confirm (with expected error). 487 | $ STAGE=sandbox yarn lambda:info 488 | ... 489 | 490 | Serverless Error --------------------------------------- 491 | 492 | Stack with id sls-${SERVICE_NAME}-${STAGE} does not exist 493 | ``` 494 | 495 | **Metrics**: 496 | 497 | ```sh 498 | # Show metrics for an application 499 | $ STAGE=sandbox yarn run lambda:metrics 500 | ``` 501 | 502 | ### User Deployment 503 | 504 | These actions can be performed by any user (`-admin|developer|ci`). 505 | 506 | Get server **information**: 507 | 508 | ```sh 509 | $ STAGE=sandbox yarn run lambda:info 510 | ... 511 | endpoints: 512 | ANY - https://HASH.execute-api.AWS_REGION.amazonaws.com/STAGE/base 513 | ANY - https://HASH.execute-api.AWS_REGION.amazonaws.com/STAGE/base/{proxy+} 514 | ANY - https://HASH.execute-api.AWS_REGION.amazonaws.com/STAGE/xray 515 | ANY - https://HASH.execute-api.AWS_REGION.amazonaws.com/STAGE/xray/{proxy+} 516 | ... 517 | ``` 518 | 519 | See the **logs**: 520 | 521 | ```sh 522 | $ STAGE=sandbox yarn run lambda:logs -f FUNCTION_NAME 523 | ``` 524 | 525 | _Note_: To see the logs in the AWS console, you unfortunately cannot just click on "CloudWatch > Logs" and see the relevant potential ones listed because a wildcard would be needed for `log:DescribeLogGroups|Streams`. However, if you know the log group generated name, and we **do** here, you can fill in the blanks and navigate to: 526 | 527 | ``` 528 | https://console.aws.amazon.com/cloudwatch/home?#logStream:group=/aws/lambda/sls-SERVICE_NAME-STAGE-FUNCTION_NAME;streamFilter=typeLogStreamPrefix 529 | ``` 530 | 531 | **Update** the Lambda server. 532 | 533 | ```sh 534 | $ STAGE=sandbox yarn run lambda:deploy 535 | 536 | # **WARNING**: If using `aws-vault`, remember `--no-session`! 537 | $ STAGE=sandbox aws-vault exec FIRST.LAST --no-session -- STAGE=sandbox yarn lambda:deploy 538 | ``` 539 | 540 | **Rollback** to a previous Lambda deployment: 541 | 542 | If something has gone wrong, you can see the list of available states to 543 | roll back to with: 544 | 545 | ```sh 546 | $ STAGE=sandbox yarn run lambda:rollback 547 | ``` 548 | 549 | Then choose a datestamp and add with the `-t` flag like: 550 | 551 | ```sh 552 | $ STAGE=sandbox yarn run lambda:rollback -t 2019-02-07T00:35:56.362Z 553 | ``` 554 | 555 | ## `terraform-aws-serverless` Development 556 | 557 | _For contributors to [FormidableLabs/serverless/aws][]_ 558 | 559 | To test out a local branch of `terraform-aws-serverless`, first clone it to the relative path of `../terraform-aws-serverless` from this project's checkout. Then run the command: 560 | 561 | ```sh 562 | $ yarn _dev:on 563 | ``` 564 | 565 | To switch to the local modules instead of published ones. Next, make sure to re-initialize Terraform to pick up the local modules: 566 | 567 | ```sh 568 | $ STAGE=sandbox yarn run tf:service:init --reconfigure 569 | ``` 570 | 571 | Then, do all your testing of the local module. When you're ready to unwind the changes, you can do: 572 | 573 | ```sh 574 | $ yarn _dev:off 575 | ``` 576 | 577 | To switch back to the published version. Typically then you'll do a version bump if we've published a new module. Then again re-initialize: 578 | 579 | ```sh 580 | $ STAGE=sandbox yarn run tf:service:init --reconfigure 581 | ``` 582 | 583 | ...and you're up and running the published module again! 584 | 585 | [serverless]: https://serverless.com/ 586 | [serverless-http]: https://github.com/dougmoscrop/serverless-http 587 | [Terraform]: https://www.terraform.io 588 | [CloudFormation]: https://aws.amazon.com/cloudformation/ 589 | [tfenv]: https://github.com/tfutils/tfenv 590 | [HCL]: https://www.terraform.io/docs/configuration/syntax.html 591 | [FormidableLabs/serverless/aws]: https://registry.terraform.io/modules/FormidableLabs/serverless/aws 592 | 593 | [trav_img]: https://api.travis-ci.com/FormidableLabs/aws-lambda-serverless-reference.svg 594 | [trav_site]: https://travis-ci.com/FormidableLabs/aws-lambda-serverless-reference 595 | 596 | 597 | ## Maintenance Status 598 | 599 | **Stable:** Formidable is not planning to develop any new features for this project. We are still responding to bug reports and security concerns. We are still welcoming PRs for this project, but PRs that include new features should be small and easy to integrate and should not include breaking changes. 600 | 601 | [maintenance-image]: https://img.shields.io/badge/maintenance-stable-yellow.svg?color=yellow&style=flat 602 | -------------------------------------------------------------------------------- /aws-lambda-serverless-reference-Hero.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FormidableLabs/aws-lambda-serverless-reference/5ed26dde4b917bd1554fb33658395849c5ccd241/aws-lambda-serverless-reference-Hero.png -------------------------------------------------------------------------------- /aws/bootstrap.yml: -------------------------------------------------------------------------------- 1 | --- 2 | ############################################################################# 3 | # Bootstrap 4 | ############################################################################# 5 | # This template provides base resources for a Terraform stack to store and 6 | # manage remote state. 7 | ############################################################################# 8 | AWSTemplateFormatVersion: "2010-09-09" 9 | 10 | Description: "Terraform bootstrap" 11 | 12 | # Custom parameters 13 | Parameters: 14 | ServiceName: 15 | Description: Base name of the service 16 | Type: String 17 | 18 | # Our arbitrary choices for supported, isolated statges. 19 | Stage: 20 | Description: Operational stage (e.g., development, staging, production) 21 | Type: String 22 | 23 | Resources: 24 | # Enable Terraform state in AWS 25 | # https://www.terraform.io/docs/backends/types/s3.html 26 | TerraformState: 27 | Type: AWS::S3::Bucket 28 | Properties: 29 | BucketName: !Sub "cf-${ServiceName}-${Stage}-terraform-state" 30 | # > It is highly recommended that you enable Bucket Versioning on the S3 31 | # > bucket to allow for state recovery in the case of accidental deletions 32 | # > and human error. 33 | VersioningConfiguration: 34 | Status: Enabled 35 | Tags: 36 | - 37 | Key: "Service" 38 | Value: !Sub "${ServiceName}" 39 | - 40 | Key: "Stage" 41 | Value: !Sub "${Stage}" 42 | 43 | TerraformLocks: 44 | Type: AWS::DynamoDB::Table 45 | Properties: 46 | TableName: !Sub "cf-${ServiceName}-${Stage}-terraform-locks" 47 | BillingMode: PAY_PER_REQUEST 48 | KeySchema: 49 | - AttributeName: LockID 50 | KeyType: HASH 51 | AttributeDefinitions: 52 | - AttributeName: LockID 53 | AttributeType: S 54 | Tags: 55 | - 56 | Key: "Service" 57 | Value: !Sub "${ServiceName}" 58 | - 59 | Key: "Stage" 60 | Value: !Sub "${Stage}" -------------------------------------------------------------------------------- /layers/repeat/repeat/index.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | // Repeate a string. 4 | const repeat = (str, times) => str.repeat(times); 5 | 6 | module.exports = { 7 | repeat 8 | }; 9 | -------------------------------------------------------------------------------- /layers/vendor/nodejs/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "aws-lambda-serverless-reference-layers-vendor", 3 | "version": "0.0.1", 4 | "description": "Common vendor libraries for reference app", 5 | "main": "index.js", 6 | "license": "MIT", 7 | "dependencies": { 8 | "figlet": "^1.2.3" 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /layers/vendor/nodejs/yarn.lock: -------------------------------------------------------------------------------- 1 | # THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. 2 | # yarn lockfile v1 3 | 4 | 5 | figlet@^1.2.3: 6 | version "1.2.3" 7 | resolved "https://registry.yarnpkg.com/figlet/-/figlet-1.2.3.tgz#7d25df546f41fc411c2a8b88012d48d55de72129" 8 | integrity sha512-+F5zdvZ66j77b8x2KCPvWUHC0UCKUMWrewxmewgPlagp3wmDpcrHMbyv/ygq/6xoxBPGQA+UJU3SMoBzKoROQQ== 9 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "aws-lambda-serverless-reference", 3 | "version": "0.0.1", 4 | "description": "A reference application for AWS + serverless framework.", 5 | "repository": "https://github.com/FormidableLabs/aws-lambda-serverless-reference", 6 | "author": "Ryan Roemer ", 7 | "license": "MIT", 8 | "scripts": { 9 | "postinstall": "cd layers/vendor/nodejs && yarn", 10 | "_dev:on": "node terraform/scripts/dev.js --on", 11 | "_dev:off": "node terraform/scripts/dev.js --off", 12 | "env": "echo export STAGE=${STAGE:-localdev}; echo export SERVICE_NAME=simple-reference; echo export AWS_REGION=${AWS_REGION:-us-east-1}; echo export AWS_XRAY_CONTEXT_MISSING=LOG_ERROR", 13 | "build:toc-files": "find . -name '*.md' -type f -not -path './node_modules/*' -not -path './terraform/.terraform/*' -exec grep -l '