├── .gitignore ├── .prettierignore ├── .prettierrc ├── LICENSE ├── README.md ├── contrib ├── release.sh └── terraform-example │ ├── main.tf │ ├── permissions.tf │ └── variables.tf ├── dist └── .gitkeep ├── doc ├── graphs-badge.png ├── graphs-long.png └── graphs-short.png ├── package-lock.json ├── package.json ├── src └── index.ts └── tsconfig.json /.gitignore: -------------------------------------------------------------------------------- 1 | # General 2 | node_modules/ 3 | npm-debug.log 4 | .env 5 | .DS_Store 6 | .history 7 | .vscode 8 | 9 | # Project 10 | /dist/* 11 | -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- 1 | .history 2 | *.json 3 | *.js 4 | -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "singleQuote": true, 3 | "trailingComma": "all", 4 | "bracketSpacing": true, 5 | "jsxBracketSameLine": false, 6 | "printWidth": 120 7 | } 8 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Jarno Rantanen 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # terraform-monitor-lambda 2 | 3 | Monitors a Terraform repository and reports on [configuration drift](https://www.hashicorp.com/blog/detecting-and-managing-drift-with-terraform): changes that are in the repo, but not in the deployed infra, or vice versa. Hooks up to dashboards and alerts via [CloudWatch](https://aws.amazon.com/cloudwatch/) or [InfluxDB](https://docs.influxdata.com/influxdb/). 4 | 5 | ## Background 6 | 7 | Terraform is a great tool for defining infrastructure as code. As the team working on the infra grows, however, it becomes more likely that someone forgets to push changes to version control which have already been applied to infrastructure. This will cause others to see differences on their next `terraform apply` which they have no knowledge of. Also, this will reduce the usefulness of your Terraform configuration as the documentation of your infrastructure, because whatever is currently deployed _does not necessarily match_ what's in your configuration. 8 | 9 | ## Requirements 10 | 11 | This project will only be useful if you host your Terraform configuration in a repo on [GitHub](https://github.com/), and use Terraform with the [S3 backend](https://www.terraform.io/docs/backends/types/s3.html). 12 | 13 | It will probably work with other VCS's & backend types with minor modifications, but your out-of-box experience will not be as smooth. 14 | 15 | ## Setup 16 | 17 | ### Setting up manually 18 | 19 | 1. Download the [latest release](https://github.com/futurice/terraform-monitor-lambda/releases) 20 | 1. Log into AWS Lambda 21 | 1. Create a new function from the release zipfile 22 | 1. Put in [configuration](#configuration) via environment variables 23 | 1. Grant [necessary IAM permissions](contrib/terraform-example/permissions.tf) for the Lambda user 24 | 1. Add an invocation schedule (e.g. once per hour) 25 | 26 | ### Setting up with Terraform 27 | 28 | Because you're already sold on Terraform, setting up this project using Terraform probably sounds like a good idea! An [example setup](contrib/terraform-example) is included. 29 | 30 | ## Configuration 31 | 32 | The Lambda function expects configuration via environment variables as follows: 33 | 34 | ```bash 35 | # These should match the Terraform S3 backend configuration: 36 | TERRAFORM_MONITOR_S3_BUCKET=my-bucket 37 | TERRAFORM_MONITOR_S3_KEY=terraform 38 | 39 | # GitHub repo which contains your Terraform files, and API token with access to it: 40 | TERRAFORM_MONITOR_GITHUB_REPO=user/infra 41 | TERRAFORM_MONITOR_GITHUB_TOKEN=123abc 42 | 43 | # (Optional) AWS CloudWatch metric name to which metrics should be shipped: 44 | TERRAFORM_MONITOR_CLOUDWATCH_NAMESPACE=TerraformMonitor 45 | 46 | # (Optional) Configuration for an InfluxDB instance to which metrics should be shipped: 47 | TERRAFORM_MONITOR_INFLUXDB_URL=https://db.example.com 48 | TERRAFORM_MONITOR_INFLUXDB_DB=my_metrics 49 | TERRAFORM_MONITOR_INFLUXDB_AUTH=user:pass 50 | TERRAFORM_MONITOR_INFLUXDB_MEASUREMENT=terraform_monitor 51 | 52 | # (Optional) AWS config for when running outside of Lambda: 53 | AWS_SECRET_ACCESS_KEY=abcdef 54 | AWS_ACCESS_KEY_ID=ABCDEF 55 | AWS_REGION=eu-central-1 56 | ``` 57 | 58 | In addition, you may sometimes need the following: 59 | 60 | ```bash 61 | TERRAFORM_MONITOR_DEBUG=1 # with this flag enabled, full output from Terraform commands is written to logs 62 | TERRAFORM_MONITOR_ALWAYS_INIT=1 # with this flag enabled, "terraform init" will always be ran, even when the results could be cached 63 | ``` 64 | 65 | ## Graphs 66 | 67 | With the optional CloudWatch/InfluxDB support, it's simple enough to make a badge onto your wall display that goes red when there's configuration drift: 68 | 69 | ![badge](doc/graphs-badge.png) 70 | 71 | ...or a full-blown dashboard showing what's been going on with your Terraform infrastructure recently: 72 | 73 | ![short](doc/graphs-short.png) 74 | 75 | ...or over a longer time period: 76 | 77 | ![short](doc/graphs-long.png) 78 | 79 | ## Alarms 80 | 81 | It can be a good idea to set up alarms for conditions such as: 82 | 83 | - Committed and deployed configuration being out of sync for a while 84 | - Not receiving new values from the Lambda function for a while 85 | 86 | This will depend on which metrics backend you're using. 87 | 88 | - If using CloudWatch, see [Using Amazon CloudWatch Alarms](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/AlarmThatSendsEmail.html). 89 | - If using InfluxDB, you'll have to set up alarms using a separate monitoring tool. [Grafana has decent alert support](http://docs.grafana.org/alerting/rules/), for example. 90 | 91 | ## Security 92 | 93 | Your AWS account probably contains sensitive things. And understandably, you should be cautious of using code from a stranger on the Internet, when that code can have access to your whole infrastructure. 94 | 95 | This project aims to alleviate those concerns in two ways. 96 | 97 | ### 1. Running with limited permissions 98 | 99 | The Lambda function doesn't expect to have full privileges on the AWS account. To the contrary, it assumes a very limited set of permissions; the only required one is read-only access to the bucket that contains your Terraform state. 100 | 101 | ### 2. Simple to audit 102 | 103 | The Lambda function is defined in a [single easy-to-read file](src/), in strictly-typed TypeScript. Also, it uses zero external dependencies from npm. The only other dependencies are Terraform itself, and the `aws-sdk` which is built in to the Lambda JS environment. 104 | 105 | ## Development 106 | 107 | To get a standard development environment that matches Lambda's pretty well, consider using Docker: 108 | 109 | ```console 110 | $ docker run --rm -it -v $(pwd):/app -w /app --env-file .env node:8.10.0 bash 111 | > apt-get update && apt-get install -y zip 112 | > ./node_modules/.bin/ts-node src/index.ts 113 | ``` 114 | 115 | See [above](#configuration) for an example `.env` file to use. 116 | 117 | ## Release 118 | 119 | Releasing a new version is automated via a script, which asks a question (semver bump), and does the following: 120 | 121 | ```console 122 | $ ./contrib/release.sh 123 | Checking for clean working copy... OK 124 | Parsing git remote... OK 125 | Verifying GitHub API access... OK 126 | Running pre-release QA tasks... OK 127 | Building Lambda function... OK 128 | 129 | This release is major/minor/patch: patch 130 | 131 | Committing and tagging new release... OK 132 | Pushing tag to GitHub... OK 133 | Renaming release zipfile... OK 134 | Creating release on GitHub... OK 135 | Uploading release zipfile... OK 136 | Cleaning up... OK 137 | 138 | New release: https://github.com/futurice/terraform-monitor-lambda/releases/tag/v1.0.0 139 | ``` 140 | -------------------------------------------------------------------------------- /contrib/release.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e # exit on error 4 | PATH="$PATH:./node_modules/.bin" # allows us to run "npm binaries" 5 | SELF_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" # https://stackoverflow.com/a/246128 6 | cd "$SELF_DIR/.." # run all commands at project root, regardless of PWD 7 | 8 | WORK_DIR="dist" 9 | BUILD_ZIP="$WORK_DIR/lambda.zip" # note: this has to match what's in package.json 10 | OK="\033[1;32mOK\033[0m" 11 | 12 | echo -n "Checking for clean working copy... " # npm version will fail otherwise 13 | if [ "$(git diff-index HEAD)" != "" ]; then 14 | echo -e "ERROR\n\nThere's uncommitted changes in the working copy" 15 | exit 1 16 | fi 17 | echo -e "$OK" 18 | 19 | echo -n "Parsing git remote... " 20 | github_raw="$(git config --get remote.origin.url | sed 's/.*://' | sed 's/\..*//')" # e.g. "git@github.com:user/project.git" => "user/project" 21 | github_user="$(echo "$github_raw" | cut -d / -f 1)" 22 | github_project="$(echo "$github_raw" | cut -d / -f 2)" 23 | if [[ ! "$github_user" =~ ^[[:alnum:]-]+$ ]]; then 24 | echo -e "ERROR\n\nCan't seem to determine GitHub user name reliably: \"$github_user\"" 25 | exit 1 26 | fi 27 | if [[ ! "$github_project" =~ ^[[:alnum:]-]+$ ]]; then 28 | echo -e "ERROR\n\nCan't seem to determine GitHub project name reliably: \"$github_project\"" 29 | exit 1 30 | fi 31 | echo -e "$OK" 32 | 33 | echo -n "Verifying GitHub API access... " 34 | github_test="$(curl -s -n -o /dev/null -w "%{http_code}" https://api.github.com/user)" 35 | if [ "$github_test" != "200" ]; then 36 | echo -e "ERROR\n\nPlease ensure that:" 37 | echo "* You've set up a Personal access token for the GitHub API (https://github.com/settings/tokens/new)" 38 | echo "* The resulting token is listed in your ~/.netrc file (under \"machine api.github.com\" and \"machine uploads.github.com\")" 39 | exit 1 40 | fi 41 | echo -e "$OK" 42 | 43 | echo -n "Running pre-release QA tasks... " 44 | npm run lint > /dev/null 45 | echo -e "$OK" 46 | 47 | echo -n "Building Lambda function... " 48 | npm run build > /dev/null 49 | echo -e "$OK" 50 | 51 | echo 52 | echo -n "This release is major/minor/patch: " 53 | read version_bump 54 | echo 55 | 56 | echo -n "Committing and tagging new release... " 57 | version_tag="$(npm version -m "Release %s" "$version_bump")" 58 | echo -e "$OK" 59 | 60 | echo -n "Pushing tag to GitHub... " 61 | git push --quiet origin "$version_tag" 62 | echo -e "$OK" 63 | 64 | release_zip="$github_project-$version_tag.zip" 65 | echo -n "Renaming release zipfile... " 66 | mv "$BUILD_ZIP" "$WORK_DIR/$release_zip" 67 | echo -e "$OK" 68 | 69 | echo -n "Creating release on GitHub... " # https://developer.github.com/v3/repos/releases/ 70 | curl -o curl-out -s -n -X POST "https://api.github.com/repos/$github_user/$github_project/releases" --data "{\"tag_name\":\"$version_tag\"}" 71 | release_upload_url="$(cat curl-out | node -p 'JSON.parse(fs.readFileSync(0)).upload_url' | sed 's/{.*//')" 72 | release_html_url="$(cat curl-out | node -p 'JSON.parse(fs.readFileSync(0)).html_url')" 73 | if [[ ! "$release_upload_url" =~ ^https:// ]]; then 74 | echo ERROR 75 | cat curl-out 76 | exit 1 77 | fi 78 | echo -e "$OK" 79 | 80 | echo -n "Uploading release zipfile... " 81 | release_upload_result="$(curl -o /dev/null -w "%{http_code}" -s -n "$release_upload_url?name=$release_zip" --data-binary @"$WORK_DIR/$release_zip" -H "Content-Type: application/octet-stream")" 82 | if [ "$release_upload_result" != "201" ]; then 83 | echo -e "ERROR\n\nRelease upload gave unexpected HTTP status: \"$release_upload_result\"" 84 | exit 1 85 | fi 86 | echo -e "$OK" 87 | 88 | echo -n "Cleaning up... " 89 | rm curl-out 90 | echo -e "$OK" 91 | 92 | echo 93 | echo "New release: $release_html_url" 94 | echo 95 | -------------------------------------------------------------------------------- /contrib/terraform-example/main.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | lambda_function_name = "TerraformMonitor" 3 | lambda_zipfile_name = "terraform-monitor-lambda-v1.2.0.zip" # see https://github.com/futurice/terraform-monitor-lambda/releases 4 | } 5 | 6 | resource "aws_lambda_function" "this" { 7 | function_name = "${local.lambda_function_name}" 8 | filename = "${substr("${path.module}/${local.lambda_zipfile_name}", length(path.cwd) + 1, -1)}" # see https://github.com/hashicorp/terraform/issues/7613#issuecomment-332238441 9 | source_code_hash = "${base64sha256(file("${path.module}/${local.lambda_zipfile_name}"))}" 10 | handler = "index.handler" 11 | timeout = 600 # 10 minutes 12 | memory_size = 512 # running big external binaries like Terraform's needs a bit more memory 13 | runtime = "nodejs8.10" 14 | role = "${aws_iam_role.this.arn}" 15 | description = "${var.default_resource_comment}" 16 | 17 | environment { 18 | variables = { 19 | TERRAFORM_MONITOR_S3_BUCKET = "${var.terraform_monitor_s3_bucket}" 20 | TERRAFORM_MONITOR_S3_KEY = "${var.terraform_monitor_s3_key}" 21 | 22 | TERRAFORM_MONITOR_GITHUB_REPO = "${var.terraform_monitor_github_repo}" 23 | TERRAFORM_MONITOR_GITHUB_TOKEN = "${var.terraform_monitor_github_token}" 24 | 25 | TERRAFORM_MONITOR_CLOUDWATCH_NAMESPACE = "${var.terraform_monitor_cloudwatch_namespace}" 26 | 27 | TERRAFORM_MONITOR_INFLUXDB_URL = "${var.terraform_monitor_influxdb_url}" 28 | TERRAFORM_MONITOR_INFLUXDB_DB = "${var.terraform_monitor_influxdb_db}" 29 | TERRAFORM_MONITOR_INFLUXDB_AUTH = "${var.terraform_monitor_influxdb_auth}" 30 | TERRAFORM_MONITOR_INFLUXDB_MEASUREMENT = "${var.terraform_monitor_influxdb_measurement}" 31 | } 32 | } 33 | } 34 | 35 | # Add the scheduled execution rules & permissions: 36 | 37 | resource "aws_cloudwatch_event_rule" "this" { 38 | name = "${local.lambda_function_name}_InvocationSchedule" 39 | schedule_expression = "${var.terraform_monitor_schedule_expression}" 40 | } 41 | 42 | resource "aws_cloudwatch_event_target" "this" { 43 | rule = "${aws_cloudwatch_event_rule.this.name}" 44 | target_id = "${aws_cloudwatch_event_rule.this.name}" 45 | arn = "${aws_lambda_function.this.arn}" 46 | } 47 | 48 | resource "aws_lambda_permission" "this" { 49 | statement_id = "${local.lambda_function_name}_ScheduledInvocation" 50 | action = "lambda:InvokeFunction" 51 | function_name = "${aws_lambda_function.this.function_name}" 52 | principal = "events.amazonaws.com" 53 | source_arn = "${aws_cloudwatch_event_rule.this.arn}" 54 | } 55 | -------------------------------------------------------------------------------- /contrib/terraform-example/permissions.tf: -------------------------------------------------------------------------------- 1 | resource "aws_iam_role" "this" { 2 | name = "${local.lambda_function_name}_AllowLambdaExec" 3 | 4 | assume_role_policy = < dist/index.js && browserify -p tsify --node --external aws-sdk src/index.ts >> dist/index.js && (cd dist && zip lambda.zip index.js)" 17 | }, 18 | "repository": { 19 | "type": "git", 20 | "url": "git+https://github.com/futurice/terraform-monitor-lambda.git" 21 | }, 22 | "author": "Jarno Rantanen ", 23 | "license": "MIT", 24 | "bugs": { 25 | "url": "https://github.com/futurice/terraform-monitor-lambda/issues" 26 | }, 27 | "homepage": "https://github.com/futurice/terraform-monitor-lambda#readme", 28 | "devDependencies": { 29 | "@types/aws-sdk": "^2.7.0", 30 | "@types/node": "^10.12.0", 31 | "aws-sdk": "^2.338.0", 32 | "browserify": "^16.2.3", 33 | "check-node-version": "^3.2.0", 34 | "prettier": "^1.14.3", 35 | "ts-node": "^7.0.1", 36 | "tsify": "^4.0.0", 37 | "typescript": "^3.1.3" 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | import { S3, CloudWatch } from 'aws-sdk'; // @see https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/ 2 | import { StandardUnit } from 'aws-sdk/clients/cloudwatch'; 3 | import { access, createWriteStream } from 'fs'; 4 | import { get, request } from 'https'; 5 | import { exec, spawn } from 'child_process'; 6 | import { parse } from 'url'; 7 | import { assert } from 'console'; 8 | 9 | // Define the Lambda runtime environment (alias made during build process) 10 | declare var lambda: { 11 | handler: (event: object, context: object, callback: (error?: Error | null) => void) => void; 12 | }; 13 | 14 | // Determine if we're running in a Lambda, or a regular-old CLI 15 | if (typeof lambda === 'undefined') { 16 | main(); 17 | } else { 18 | lambda.handler = (_, __, callback) => main().then(callback, callback); 19 | } 20 | 21 | // Read config from environment and make it globally available 22 | const config = { 23 | DEBUG: !!process.env.TERRAFORM_MONITOR_DEBUG, 24 | ALWAYS_INIT: !!process.env.TERRAFORM_MONITOR_ALWAYS_INIT, 25 | S3_BUCKET: process.env.TERRAFORM_MONITOR_S3_BUCKET || '', 26 | S3_KEY: process.env.TERRAFORM_MONITOR_S3_KEY || '', 27 | GITHUB_REPO: process.env.TERRAFORM_MONITOR_GITHUB_REPO || '', 28 | GITHUB_TOKEN: process.env.TERRAFORM_MONITOR_GITHUB_TOKEN || '', 29 | SCRATCH_SPACE: process.env.TERRAFORM_MONITOR_SCRATCH_SPACE || '/tmp', // @see https://aws.amazon.com/lambda/faqs/ "scratch space" 30 | CLOUDWATCH_NAMESPACE: process.env.TERRAFORM_MONITOR_CLOUDWATCH_NAMESPACE || '', 31 | INFLUXDB_URL: process.env.TERRAFORM_MONITOR_INFLUXDB_URL || '', 32 | INFLUXDB_DB: process.env.TERRAFORM_MONITOR_INFLUXDB_DB || '', 33 | INFLUXDB_AUTH: process.env.TERRAFORM_MONITOR_INFLUXDB_AUTH || '', 34 | INFLUXDB_MEASUREMENT: process.env.TERRAFORM_MONITOR_INFLUXDB_MEASUREMENT || '', 35 | }; 36 | 37 | // @see https://www.terraform.io/docs/commands/plan.html#detailed-exitcode 38 | enum TerraformStatus { 39 | CLEAN = 0, // Succeeded with empty diff (no changes) 40 | ERROR = 1, // Error 41 | DIRTY = 2, // Succeeded with non-empty diff (changes present) 42 | } 43 | 44 | // These are the metrics which are eventually collected from Terraform 45 | type TerraformMetrics = { 46 | terraformStatus: TerraformStatus; 47 | resourceCount: number; 48 | refreshTime: number; 49 | totalTime: number; 50 | scratchSpaceBytes: number; 51 | pendingAdd: number; 52 | pendingChange: number; 53 | pendingDestroy: number; 54 | pendingTotal: number; 55 | }; 56 | 57 | // @see https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html 58 | const s3 = new S3(); 59 | 60 | // @see https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/CloudWatch.html 61 | var cloudwatch = new CloudWatch(); 62 | 63 | // Log to console; return null for convenient returns with an || expression 64 | function log(...args: any[]): null { 65 | console.log.apply(console.log, args); 66 | return null; 67 | } 68 | 69 | // Returns a Promise that resolves when the run is complete 70 | function main(): Promise { 71 | let then = Date.now(); 72 | return Promise.resolve() 73 | .then(() => 74 | Promise.all([ 75 | getTerraformVersion().then(installTerraform), 76 | getRepoHead().then(fetchRepo), 77 | getScratchSpaceUsage(), 78 | ]), 79 | ) 80 | .then(([terraformBin, repoPath, scratchSpaceBytes]) => 81 | Promise.resolve() 82 | .then(() => terraformInit(terraformBin, repoPath)) 83 | .then(() => terraformPlan(terraformBin, repoPath)) 84 | .then(metrics => ({ ...metrics, scratchSpaceBytes, totalTime: Date.now() - then })) 85 | .then(shipMetrics), 86 | ) 87 | .catch(err => log('ERROR', err)) 88 | .then(() => null); 89 | } 90 | 91 | // Gets the Terraform state from S3 and reports the exact version being used 92 | // @example "0.11.8" 93 | function getTerraformVersion(): Promise { 94 | return s3 95 | .getObject({ 96 | Bucket: config.S3_BUCKET, 97 | Key: config.S3_KEY, 98 | }) 99 | .promise() 100 | .then(data => JSON.parse(data.Body + '').terraform_version) 101 | .then(version => log(`Terraform state has version "${version}"`) || version); 102 | } 103 | 104 | // Installs the requested version of Terraform, if not already installed. 105 | // Resolves with the path to its binary. 106 | // @example "/tmp/terraform_0.11.8_linux_amd64/terraform" 107 | function installTerraform(version: string): Promise { 108 | const file = `terraform_${version}_linux_amd64`; 109 | const url = `https://releases.hashicorp.com/terraform/${version}/${file}.zip`; 110 | const zip = `${config.SCRATCH_SPACE}/${file}.zip`; 111 | const out = `${config.SCRATCH_SPACE}/${file}`; 112 | const bin = `${out}/terraform`; 113 | return Promise.resolve() 114 | .then(() => new Promise(resolve => access(bin, resolve))) 115 | .then( 116 | res => 117 | res instanceof Error // fs.access() returns an Error if the file doesn't exist 118 | ? new Promise((resolve, reject) => { 119 | const file = createWriteStream(zip); 120 | const request = get(url, response => response.pipe(file)); 121 | request.on('error', reject); 122 | file.on('close', resolve); 123 | }) 124 | .then(() => execShell(`unzip -o ${zip} -d ${out}`)) 125 | .then(() => log(`Downloaded new Terraform binary: ${bin}`)) 126 | : log(`Using cached Terraform binary: ${bin}`), 127 | ) 128 | .then(() => bin); 129 | } 130 | 131 | // Promises the simple shell-output of the given command. 132 | // Unless ignoreStderrOutput is true, automatically rejects if the command writes to stderr. 133 | // @example execShell("ls -la") 134 | function execShell(command: string, ignoreStderrOutput = false): Promise { 135 | return new Promise((resolve, reject) => { 136 | exec(command, (error, stdout, stderr) => { 137 | if (error) reject(new Error(`Could not exec command "${command}": "${error}"`)); 138 | if (stderr.length && !ignoreStderrOutput) 139 | reject(new Error(`Command "${command}" produced output on stderr: "${stderr}"`)); 140 | if (typeof stdout !== 'string') reject(new Error(`Command "${command}" produced non-string stdout: "${stdout}"`)); 141 | resolve(stdout); 142 | }); 143 | }); 144 | } 145 | 146 | // Promises the outputs and exit code of the given command. 147 | // Note that as opposed to execShell(), this doesn't reject if the process exits non-zero. 148 | // @see https://nodejs.org/api/child_process.html#child_process_child_process_spawn_command_args_options 149 | function execProcess( 150 | opt: Partial<{ 151 | command: string; 152 | args: string[]; 153 | env: { 154 | [key: string]: string; 155 | }; 156 | cwd: string; 157 | }> = {}, 158 | ): Promise<{ code: number; stdout: string; stderr: string }> { 159 | let stdout = ''; 160 | let stderr = ''; 161 | return new Promise((resolve, reject) => { 162 | const proc = spawn(opt.command || 'false', opt.args || [], { 163 | cwd: opt.cwd || undefined, 164 | env: Object.assign({}, process.env, opt.env || {}), 165 | }); 166 | proc.stdout.on('data', data => (stdout += data)); 167 | proc.stderr.on('data', data => (stderr += data)); 168 | proc.on('exit', code => resolve({ code, stdout, stderr })); 169 | proc.on('error', reject); 170 | }); 171 | } 172 | 173 | // @see https://www.terraform.io/docs/commands/init.html 174 | function terraformInit(terraformBin: string, repoPath: string): Promise { 175 | return Promise.resolve() 176 | .then(() => (config.ALWAYS_INIT ? Promise.reject() : checkPathExists(`${repoPath}/.terraform`))) 177 | .then( 178 | () => log('Terraform init already performed'), 179 | () => 180 | Promise.resolve() 181 | .then(() => log('Terraform init running...')) 182 | .then(() => 183 | execProcess({ 184 | cwd: repoPath, 185 | command: terraformBin, 186 | args: [ 187 | 'init', 188 | '-input=false', 189 | '-lock=false', // since we won't be making any changes, it's not necessary to lock the state, and thus we can safely crash without leaving it locked 190 | '-no-color', 191 | ], 192 | }), 193 | ) 194 | .then(res => { 195 | if (res.code || config.DEBUG) log(res.stdout + res.stderr); 196 | if (res.code) throw new Error(`Terraform init failed (exit code ${res.code})`); 197 | log(`Terraform init finished`); 198 | }), 199 | ); 200 | } 201 | 202 | // @see https://www.terraform.io/docs/commands/plan.html 203 | function terraformPlan(terraformBin: string, repoPath: string) { 204 | log('Terraform plan running...'); 205 | const then = Date.now(); 206 | return Promise.resolve() 207 | .then(() => 208 | execProcess({ 209 | cwd: repoPath, 210 | command: terraformBin, 211 | args: [ 212 | 'plan', 213 | '-detailed-exitcode', // @see https://www.terraform.io/docs/commands/plan.html#detailed-exitcode 214 | '-input=false', 215 | '-lock=false', // since we won't be making any changes, it's not necessary to lock the state, and thus we can safely crash without leaving it locked 216 | '-no-color', 217 | ], 218 | }), 219 | ) 220 | .then(res => { 221 | let resourceCount = 0, 222 | pendingAdd = 0, 223 | pendingChange = 0, 224 | pendingDestroy = 0, 225 | pendingTotal = 0; 226 | const terraformStatus = res.code; 227 | const refreshTime = Date.now() - then; 228 | if (terraformStatus === 1 || config.DEBUG) log(res.stdout + res.stderr); 229 | if (terraformStatus === 1) { 230 | log(`Terraform plan failed`); 231 | pendingTotal = 1; // for ease of monitoring, consider Terraform failing a "pending change" 232 | } else { 233 | log(`Terraform plan finished`); 234 | const refresh = / Refreshing state.../; 235 | const plan = /^Plan: (\d+) to add, (\d+) to change, (\d+) to destroy./; 236 | res.stdout.split('\n').forEach(line => { 237 | if (line.match(refresh)) resourceCount++; 238 | if (line.match(plan)) { 239 | const [, a, b, c] = line.match(plan); 240 | pendingAdd = parseInt(a, 10); 241 | pendingChange = parseInt(b, 10); 242 | pendingDestroy = parseInt(c, 10); 243 | } 244 | }); 245 | pendingTotal = pendingAdd + pendingChange + pendingDestroy; 246 | } 247 | return { 248 | terraformStatus, 249 | resourceCount, 250 | refreshTime, 251 | pendingAdd, 252 | pendingChange, 253 | pendingDestroy, 254 | pendingTotal, 255 | }; 256 | }); 257 | } 258 | 259 | // Retrieves the current HEAD for the given branch on GitHub 260 | // @example "b719dc5f5ebf92894e3a50052ad73c4c9b8cbd9d" 261 | function getRepoHead(branch = 'master'): Promise { 262 | return new Promise((resolve, reject) => 263 | request( 264 | { 265 | hostname: 'api.github.com', 266 | path: `/repos/${config.GITHUB_REPO}/branches/${branch}`, 267 | headers: { 268 | Authorization: `token ${config.GITHUB_TOKEN}`, 269 | 'User-Agent': 'terraform_monitor', // @see https://developer.github.com/v3/#user-agent-required 270 | }, 271 | }, 272 | res => { 273 | let rawData = ''; 274 | res.setEncoding('utf8'); 275 | res.on('data', chunk => (rawData += chunk)); 276 | res.on('end', () => { 277 | try { 278 | resolve(JSON.parse(rawData)); 279 | } catch (err) { 280 | reject(new Error(`Could not parse response as JSON:\n${rawData}`)); 281 | } 282 | }); 283 | }, 284 | ) 285 | .on('error', reject) 286 | .end(), 287 | ) 288 | .then((res: any) => res.commit.sha as string) // TODO: Cast "res" to unknown and inspect 289 | .then(head => log(`Head for "${config.GITHUB_REPO}/${branch}" is "${head}"`) || head); 290 | } 291 | 292 | // Promises the number of bytes of scratch space we're currently using (probably under /tmp) 293 | function getScratchSpaceUsage(): Promise { 294 | return Promise.resolve() 295 | .then(() => execShell(`du --summarize --bytes ${config.SCRATCH_SPACE}`)) // e.g. "258828124 /tmp" 296 | .then(out => out.split('\t')) // "du" uses tabs as a delimiter 297 | .then( 298 | ([bytes]) => 299 | isNaN(parseInt(bytes, 10)) 300 | ? Promise.reject(`Could not parse scratch space ${config.SCRATCH_SPACE} usage bytes from "${bytes}"`) 301 | : parseInt(bytes, 10), 302 | ) 303 | .then(bytes => log(`Currently using ${bytes} bytes of scratch space under ${config.SCRATCH_SPACE}`) || bytes); 304 | } 305 | 306 | // Fetches the repo, at the given commit, to the scratch space, if not already fetched. 307 | // Resolves with the path to the repo. 308 | // @example "/tmp/repo/john-doe-terraform-infra-7b5cbf69999c86555fd6086e8c5e2e233f673b69" 309 | function fetchRepo(repoHead: string): Promise { 310 | const zipPath = `${config.SCRATCH_SPACE}/repo.zip`; 311 | const outPath = `${config.SCRATCH_SPACE}/repo`; 312 | const expectedExtractedPath = `${outPath}/${config.GITHUB_REPO.replace('/', '-')}-${repoHead}`; 313 | return Promise.resolve(expectedExtractedPath) 314 | .then(checkPathExists) 315 | .then( 316 | path => log(`Using cached Terraform repository: ${path}`) || path, 317 | () => 318 | new Promise((resolve, reject) => { 319 | request( 320 | { 321 | hostname: 'api.github.com', 322 | path: `/repos/${config.GITHUB_REPO}/zipball/${repoHead}`, 323 | headers: { 324 | Authorization: `token ${config.GITHUB_TOKEN}`, 325 | 'User-Agent': 'terraform_monitor', // @see https://developer.github.com/v3/#user-agent-required 326 | }, 327 | }, 328 | res => { 329 | if (!res.headers.location) { 330 | reject(new Error(`Expecting a redirect from GitHub API, got ${res.statusCode} "${res.statusMessage}"`)); 331 | return; 332 | } 333 | const file = createWriteStream(zipPath); 334 | const { protocol, port, hostname, path } = parse(res.headers.location); 335 | request( 336 | { 337 | protocol, 338 | port: port || undefined, 339 | hostname, 340 | path, 341 | headers: { 342 | Authorization: `token ${config.GITHUB_TOKEN}`, 343 | 'User-Agent': 'terraform_monitor', // @see https://developer.github.com/v3/#user-agent-required 344 | }, 345 | }, 346 | res => res.pipe(file), 347 | ) 348 | .on('error', reject) 349 | .end(); 350 | file.on('close', resolve); 351 | }, 352 | ) 353 | .on('error', reject) 354 | .end(); 355 | }) 356 | .then(() => execShell(`unzip -o ${zipPath} -d ${outPath}`)) 357 | .then(() => expectedExtractedPath) 358 | .then(checkPathExists) 359 | .then(path => log(`Fetched Terraform repository: ${path}`) || path), 360 | ); 361 | } 362 | 363 | // Promises to check that the given path exists and is readable. 364 | // Resolves with the path that was given. 365 | function checkPathExists(path: string): Promise { 366 | return new Promise((resolve, reject) => 367 | access( 368 | path, 369 | err => (err ? reject(new Error(`Expected path "${path}" doesn't exist or is not readable`)) : resolve(path)), 370 | ), 371 | ); 372 | } 373 | 374 | // @see https://stackoverflow.com/a/24398129 375 | function pad(input: string | number | boolean, padToLength: number, padLeft: boolean = false, padString = ' ') { 376 | const pad = padString.repeat(padToLength + 1); 377 | if (padLeft) { 378 | return (pad + input).slice(-pad.length); 379 | } else { 380 | return (input + pad).substring(0, pad.length); 381 | } 382 | } 383 | 384 | // @see https://github.com/Microsoft/TypeScript/pull/12253#issuecomment-263132208 385 | // That is, this can behave strangely with strange objects. You have been warned. 386 | function keys(object: T): (keyof T)[] { 387 | return Object.keys(object).filter(key => object.hasOwnProperty(key)) as any; 388 | } 389 | 390 | // Ships the given metrics as appropriate 391 | function shipMetrics(metrics: TerraformMetrics) { 392 | return Promise.all([ 393 | shipMetricsToConsole(metrics), 394 | config.CLOUDWATCH_NAMESPACE ? shipMetricsToCloudWatch(metrics) : Promise.resolve(), 395 | config.INFLUXDB_URL ? shipMetricsToInfluxDb(metrics) : Promise.resolve(), 396 | ]); 397 | } 398 | 399 | // Pretty-prints the given metrics to the console 400 | function shipMetricsToConsole(metrics: TerraformMetrics): void { 401 | const maxKeyLen = keys(metrics) 402 | .map(key => key.length) 403 | .reduce((a, b) => Math.max(a, b), 0); 404 | const maxValLen = keys(metrics) 405 | .map(key => (metrics[key] + '').length) 406 | .reduce((a, b) => Math.max(a, b), 0); 407 | log( 408 | 'Collected metrics:\n' + 409 | keys(metrics) 410 | .map(key => ` ${pad(key + ':', maxKeyLen)} ${pad(metrics[key], maxValLen - 1, true)}`) 411 | .join('\n'), 412 | ); 413 | } 414 | 415 | // @see https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/CloudWatch.html#putMetricData-property 416 | function shipMetricsToCloudWatch(metrics: TerraformMetrics) { 417 | const Dimensions = [{ Name: 'GitHubRepo', Value: config.GITHUB_REPO }]; 418 | const data: CloudWatch.Types.PutMetricDataInput = { 419 | MetricData: keys(metrics).map(key => ({ 420 | MetricName: key, 421 | Dimensions, // the same dimensions apply to all metrics 422 | Unit: getMetricsUnit(key), 423 | Value: metrics[key], 424 | })), 425 | Namespace: config.CLOUDWATCH_NAMESPACE, 426 | }; 427 | return Promise.resolve() 428 | .then(() => log('Shipping metrics to CloudWatch...')) 429 | .then(() => cloudwatch.putMetricData(data).promise()) 430 | .then(() => log(`Metrics shipped to CloudWatch`)); 431 | } 432 | 433 | // Can be used to implement exhaustiveness checks in TS. 434 | // Returns "any" for convenience. 435 | function assertExhausted(value: void): any { 436 | throw new Error(`Runtime behaviour doesn't match type definitions (value was "${value}")`); 437 | } 438 | 439 | // Chooses the correct CloudWatch unit for the given metric 440 | function getMetricsUnit(key: keyof TerraformMetrics): StandardUnit { 441 | switch (key) { 442 | case 'terraformStatus': 443 | return 'None'; 444 | case 'resourceCount': 445 | case 'pendingAdd': 446 | case 'pendingChange': 447 | case 'pendingDestroy': 448 | case 'pendingTotal': 449 | return 'Count'; 450 | case 'refreshTime': 451 | case 'totalTime': 452 | return 'Milliseconds'; 453 | case 'scratchSpaceBytes': 454 | return 'Bytes'; 455 | default: 456 | return assertExhausted(key); 457 | } 458 | } 459 | 460 | // @see https://docs.influxdata.com/influxdb/ 461 | function shipMetricsToInfluxDb(metrics: TerraformMetrics) { 462 | return Promise.resolve() 463 | .then(() => log('Shipping metrics to InfluxDB...')) 464 | .then(() => 465 | influxSend( 466 | config.INFLUXDB_URL, 467 | config.INFLUXDB_DB, 468 | influxLine(config.INFLUXDB_MEASUREMENT, { gitHubRepo: config.GITHUB_REPO }, metrics), 469 | config.INFLUXDB_AUTH, 470 | ), 471 | ) 472 | .then(() => log(`Metrics shipped to InfluxDB`)); 473 | } 474 | 475 | // @see https://docs.influxdata.com/influxdb/v1.6/write_protocols/line_protocol_reference/ 476 | // @example "weather,location=us-midwest temperature=82,bug_concentration=98 1465839830100000000" 477 | function influxLine( 478 | measurement: string, 479 | tags: { [tag: string]: string }, 480 | fields: { [field: string]: string | number | boolean }, 481 | timestampInMs?: number, 482 | ): string { 483 | assert(measurement, `Measurement name required, "${measurement}" given`); 484 | assert(Object.keys(fields).length, 'At least 1 field required, 0 given'); 485 | const tagString: string = Object.keys(tags) 486 | .map(tag => `${influxEscape(tag, 'TAG_KEY')}=${influxEscape(tags[tag], 'TAG_VALUE')}`) 487 | .join(','); 488 | const tagSeparator = Object.keys(tags).length ? ',' : ''; 489 | const fieldString: string = Object.keys(fields) 490 | .map(field => `${influxEscape(field, 'FIELD_KEY')}=${influxEscape(fields[field], 'FIELD_VALUE')}`) 491 | .join(','); 492 | const timeString: string = timestampInMs ? ` ${timestampInMs * 1e6}` : ''; // convert from milliseconds to nanoseconds 493 | return `${influxEscape(measurement, 'MEASUREMENT')}${tagSeparator}${tagString} ${fieldString}${timeString}`; 494 | } 495 | 496 | // @see https://docs.influxdata.com/influxdb/v1.6/write_protocols/line_protocol_tutorial/#special-characters-and-keywords 497 | function influxEscape( 498 | input: string | number | boolean, 499 | context: 'TAG_KEY' | 'TAG_VALUE' | 'FIELD_KEY' | 'MEASUREMENT' | 'FIELD_VALUE', 500 | ): string { 501 | switch (context) { 502 | case 'MEASUREMENT': 503 | return (input + '').replace(/,/g, '\\,').replace(/ /g, '\\ '); 504 | case 'TAG_KEY': 505 | case 'TAG_VALUE': 506 | case 'FIELD_KEY': 507 | return (input + '') 508 | .replace(/,/g, '\\,') 509 | .replace(/=/g, '\\=') 510 | .replace(/ /g, '\\ '); 511 | case 'FIELD_VALUE': 512 | return typeof input === 'number' || typeof input === 'boolean' 513 | ? input + '' 514 | : input 515 | .replace(/"/g, '\\"') 516 | .replace(/^/, '"') 517 | .replace(/$/, '"'); 518 | default: 519 | return assertExhausted(context); 520 | } 521 | } 522 | 523 | // @see https://github.com/jareware/heroku-metrics-to-influxdb/blob/master/src/influxdb.ts 524 | function influxSend( 525 | dbUrl: string, // e.g. "https://my-influxdb.example.com/" 526 | dbName: string, // e.g. "my_metrics_db" 527 | lines: string | string[], // see influxLine() 528 | auth?: string, // e.g. "user:pass" 529 | ): Promise { 530 | const url = (dbUrl + '').replace(/\/*$/, '/write?db=' + dbName); 531 | const data = typeof lines === 'string' ? lines : lines.join('\n'); 532 | const { protocol, port, hostname, path } = parse(url); 533 | return new Promise((resolve, reject) => { 534 | const req = request( 535 | { 536 | protocol, 537 | port: port || undefined, 538 | hostname, 539 | method: 'POST', 540 | path, 541 | auth, 542 | }, 543 | res => { 544 | if (res.statusCode && res.statusCode >= 200 && res.statusCode < 300) { 545 | resolve(data); 546 | } else { 547 | reject(new Error(`Unexpected response from InfluxDB: ${res.statusCode} "${res.statusMessage}"`)); 548 | } 549 | }, 550 | ); 551 | req.on('error', reject); 552 | req.write(data); 553 | req.end(); 554 | }); 555 | } 556 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "es5", 4 | "module": "commonjs", 5 | "allowJs": false, 6 | "downlevelIteration": true, 7 | "strict": true, 8 | "noUnusedLocals": true, 9 | "noUnusedParameters": true, 10 | "types": [ 11 | "node" 12 | ], 13 | "lib": [ 14 | "es2015", 15 | "dom" // for Blob et al (see https://github.com/Microsoft/TypeScript/issues/14897) 16 | ] 17 | }, 18 | "include": [ 19 | "src/**/*.ts" 20 | ], 21 | "exclude": [ 22 | "node_modules" 23 | ] 24 | } 25 | --------------------------------------------------------------------------------