├── .gitignore ├── Dockerfile ├── LICENSE ├── README.md ├── app ├── database │ ├── __init__.py │ ├── base.py │ ├── postgresql_database.py │ └── sqlite_database.py ├── drifter.py └── settings.py ├── requirements.txt └── tf ├── main.tf └── variables.tf /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | db.sqlite3 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # Environments 85 | .env 86 | .venv 87 | env/ 88 | venv/ 89 | ENV/ 90 | env.bak/ 91 | venv.bak/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | .spyproject 96 | 97 | # Rope project settings 98 | .ropeproject 99 | 100 | # mkdocs documentation 101 | /site 102 | 103 | # mypy 104 | .mypy_cache/ 105 | 106 | .vscode -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:3.8 2 | 3 | RUN apk add --update --no-cache --virtual=run-deps \ 4 | python3 \ 5 | ca-certificates \ 6 | py3-psycopg2 \ 7 | git \ 8 | && rm -rf /var/cache/apk/* 9 | 10 | ENV DEBUG False 11 | ENV DB_NAME drifter.db 12 | ENV DB_TYPE sqlite 13 | ENV TMP_FOLDER /tmp 14 | 15 | WORKDIR /opt/app 16 | CMD ["python3", "-u", "drifter.py"] 17 | 18 | COPY requirements.txt /opt/app/ 19 | RUN pip3 install --no-cache-dir -r /opt/app/requirements.txt 20 | 21 | COPY app /opt/app/ 22 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Digirati Ltd 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Drifter 2 | 3 | A tool that can detect and report on configuration drift between the latest repo version of AWS infrastructure Terraform and the deployed result. 4 | 5 | This is loosely based upon https://github.com/futurice/terraform-monitor-lambda 6 | 7 | ## Environment Variables 8 | 9 | | Name | Description | Default | 10 | |-------------------------|-----------------------------------------------------------------------------------------|---------| 11 | | DEBUG | Enable debug output | False | 12 | | TERRAFORM_S3_BUCKET | The S3 bucket that the Terraform is stored in (used to detect Terraform version in use) | | 13 | | TERRAFORM_S3_KEY | The key of the Terraform remote state in S3 (see `TERRAFORM_S3_BUCKET`, above) | | 14 | | TERRAFORM_GITHUB_REPO | GitHub repository in format `user/repo` | | 15 | | TERRAFORM_GITHUB_BRANCH | GitHub repository branch to use | master | 16 | | TERRAFORM_GITHUB_FOLDER | Subfolder within GitHub repository for Terraform | | 17 | | TERRAFORM_GITHUB_TOKEN | GitHub access token for the repo defined in `TERRAFORM_GITHUB_REPO` | | 18 | | CLOUDWATCH_NAMESPACE | AWS CloudWatch metric namespace where metrics should be shipped | | 19 | | AWS_REGION | AWS Region name | | 20 | | SLACK_WEBHOOK_URL | Slack Webhook URL to emit messages to | | 21 | | TMP_FOLDER | Temporary folder to use | /tmp | 22 | 23 | The beady-eyed amongst you may note that there are additional settings for configuring a database - this is reserved for expansion and is currently unused. 24 | 25 | ## Permissions - AWS 26 | 27 | From an AWS point of view these are handled by the Terraform. We don't know the scope of the Terraform that we will be asked to check, but we do know that we don't want to be able to change anything, so in the Terraform packaged with this module the Drifter task is given `arn:aws:iam::aws:policy/ReadOnlyAccess` which is a pre-rolled AWS policy that gives read-only access to all resource types. 28 | 29 | ## Permissions - GitHub 30 | 31 | For GitHub, the access token given to Drifter must have READ access to the Terraform source repository. In Digirati's case, we'd simply add the `CI` team with READ access to the repository Teams list. 32 | 33 | ## Terraform 34 | 35 | Terraform module for scheduled checking using Drifter, with notifications sent to a Slack webhook and metrics emitted to Cloudwatch. 36 | 37 | | Variable | Description | Default | 38 | |-------------------------|---------------------------------------------------------------------|-------------------| 39 | | prefix | Prefix to give to AWS resources | | 40 | | slack_webhook_url | Slack Webhook URL for notifications | | 41 | | terraform_identifier | Identifier for the Drifter task (e.g. `my-tf-repo-master`) | | 42 | | terraform_s3_bucket | Name of S3 bucket that the Terraform resides in | | 43 | | terraform_s3_key | S3 Key of the Terraform remote state file | terraform.tfstate | 44 | | terraform_github_repo | GitHub repository in format `user/repo` | | 45 | | terraform_github_branch | GitHub repository branch to use | master | 46 | | terraform_github_folder | Subfolder within GitHub repository for Terraform | | 47 | | terraform_github_token | GitHub access token for the repo defined in `terraform_github_repo` | | 48 | | cloudwatch_namespace | AWS CloudWatch metric namespace where metrics should be shipped | | 49 | | tmp_folder | Temporary folder to use | /tmp | 50 | | log_group_name | CloudWatch log group name that the container will emit logs to | | 51 | | region | AWS Region for resources | | 52 | | account_id | AWS account ID | | 53 | | cluster_id | The cluster on which to run the scheduled ECS task | | 54 | | cron_expression | Cron scheduling expression in form `cron(x x x x x x)` | | 55 | 56 | ### Example 57 | 58 | ``` 59 | module "drifter_estate" { 60 | source = "git::https://github.com/digirati-labs/drifter.git/tree/master/tf/" 61 | slack_webhook_url = "${var.slack_webhook_status}" 62 | terraform_identifier = "my-terraform-repo-master" 63 | terraform_s3_bucket = "my-state-bucket" 64 | terraform_github_repo = "my-github-user/my-terraform-repo" 65 | terraform_github_token = "${data.aws_ssm_parameter.terraform_github_token.value}" 66 | cloudwatch_namespace = "terraform-drift" 67 | log_group_name = "${var.log_group_name}" 68 | prefix = "${var.prefix}" 69 | region = "${var.region}" 70 | account_id = "${var.account_id}" 71 | cluster_id = "${module.metropolis_cluster.cluster_id}" 72 | cron_expression = "cron(0 0 * * ? *)" 73 | } 74 | 75 | ``` 76 | -------------------------------------------------------------------------------- /app/database/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/digirati-labs/drifter/8ad8703fda8704d7fc5264c13b4eede9324082be/app/database/__init__.py -------------------------------------------------------------------------------- /app/database/base.py: -------------------------------------------------------------------------------- 1 | import abc 2 | 3 | class Database(object, metaclass=abc.ABCMeta): 4 | 5 | @abc.abstractmethod 6 | def initialise(self, settings): 7 | raise NotImplementedError("must define initialise() to use this base class") 8 | 9 | -------------------------------------------------------------------------------- /app/database/postgresql_database.py: -------------------------------------------------------------------------------- 1 | import psycopg2 2 | import psycopg2.extras 3 | import abc 4 | from .base import Database 5 | from logzero import logger 6 | 7 | class PostgreSqlDatabase(Database): 8 | 9 | def initialise(self, settings): 10 | logger.info("postgresql_database: initialise()") 11 | con = None 12 | 13 | create = False 14 | 15 | self.connection_string = "dbname='%s' user='%s' host='%s' password='%s'" % \ 16 | (settings["dbname"], settings["user"], settings["host"], settings["password"]) 17 | 18 | try: 19 | con = psycopg2.connect(self.connection_string) 20 | cur = con.cursor() 21 | cur.execute("SELECT * FROM active") 22 | except psycopg2.Error: 23 | # no active table 24 | create = True 25 | finally: 26 | if con: 27 | con.close() 28 | 29 | if create: 30 | self.create_schema() 31 | else: 32 | logger.info("postgresql_database: schema ready") 33 | 34 | 35 | def create_schema(self): 36 | logger.debug("postgresql_database: create_schema()") 37 | con = None 38 | 39 | try: 40 | con = psycopg2.connect(self.connection_string) 41 | cur = con.cursor() 42 | cur.execute("CREATE TABLE active (environment_group CHARACTER VARYING(500) NOT NULL, environment CHARACTER VARYING(500) NOT NULL, endpoint_group CHARACTER VARYING(500) NOT NULL, endpoint CHARACTER VARYING(500) NOT NULL, timestamp INTEGER NOT NULL, message CHARACTER VARYING(500) NOT NULL, url CHARACTER VARYING(500) NOT NULL)") 43 | con.commit() 44 | except psycopg2.Error as e: 45 | logger.error("postgresql_database: problem during create_schema() - %s" % str(e)) 46 | finally: 47 | if con: 48 | con.close() -------------------------------------------------------------------------------- /app/database/sqlite_database.py: -------------------------------------------------------------------------------- 1 | import sqlite3 2 | import abc 3 | from .base import Database 4 | 5 | from logzero import logger 6 | 7 | class SqliteDatabase(Database): 8 | 9 | def initialise(self, settings): 10 | logger.info("sqlite_database: initialise()") 11 | con = None 12 | 13 | create = False 14 | 15 | self.db_name = settings["db_name"] 16 | 17 | try: 18 | con = sqlite3.connect(self.db_name) 19 | cur = con.cursor() 20 | cur.execute("SELECT * FROM active") 21 | _ = cur.fetchone() 22 | except sqlite3.Error: 23 | # no active table 24 | create = True 25 | finally: 26 | if con: 27 | con.close() 28 | 29 | if create: 30 | self.create_schema() 31 | else: 32 | logger.info("sqlite_database: schema ready") 33 | 34 | 35 | def create_schema(self): 36 | logger.debug("sqlite_database: create_schema()") 37 | con = None 38 | 39 | try: 40 | con = sqlite3.connect(self.db_name) 41 | cur = con.cursor() 42 | cur.execute("CREATE TABLE active (environment_group TEXT, environment TEXT, endpoint_group TEXT, endpoint TEXT, timestamp INTEGER, message TEXT, url TEXT)") 43 | con.commit() 44 | except sqlite3.Error as e: 45 | logger.error("sqlite_database: problem during create_schema() - %s" % str(e)) 46 | finally: 47 | if con: 48 | con.close() -------------------------------------------------------------------------------- /app/drifter.py: -------------------------------------------------------------------------------- 1 | from logzero import logger 2 | import logging 3 | import logzero 4 | from urllib.parse import urlparse 5 | from datetime import datetime, timezone 6 | from dateutil.relativedelta import relativedelta 7 | from concurrent.futures.thread import ThreadPoolExecutor 8 | import re 9 | import json 10 | import time 11 | import requests 12 | import signal 13 | import os 14 | import boto3 15 | import uuid 16 | import subprocess 17 | import settings 18 | 19 | requested_to_quit = False 20 | last_summary_emitted = 0 21 | 22 | endpoint_definitions = None 23 | alert_definitions = None 24 | metrics_definitions = None 25 | db = None 26 | 27 | 28 | def main(): 29 | logger.info("starting...") 30 | 31 | setup_signal_handling() 32 | 33 | global db 34 | db = settings.get_database() 35 | 36 | # get terraform version from state found with s3 bucket/key 37 | terraform_version = get_terraform_version(settings.TERRAFORM_S3_BUCKET, settings.TERRAFORM_S3_KEY) 38 | 39 | # install appropriate terraform version 40 | terraform_bin = install_terraform(terraform_version) 41 | 42 | # get current head of terraform repository 43 | # fetch that version as an archive and unzip it 44 | repo_folder = fetch_current_repo_head() 45 | 46 | # terraform init (with parameters) 47 | if not terraform_initialise(terraform_bin, repo_folder): 48 | return 49 | 50 | # terraform plan (with parameters) 51 | metrics = terraform_plan(terraform_bin, repo_folder) 52 | if metrics is None: 53 | return 54 | 55 | # ship metrics 56 | ship_metrics_to_console(metrics) 57 | 58 | if settings.CLOUDWATCH_NAMESPACE: 59 | ship_metrics_to_cloudwatch(metrics) 60 | 61 | if settings.SLACK_WEBHOOK_URL and deduplicate_alert(metrics): 62 | alert_slack(pretty_print_metrics(metrics)) 63 | 64 | 65 | def signal_handler(signum, frame): 66 | logger.info(f"Caught signal {signum}") 67 | global requested_to_quit 68 | requested_to_quit = True 69 | 70 | 71 | def setup_signal_handling(): 72 | logger.info("setting up signal handling") 73 | signal.signal(signal.SIGTERM, signal_handler) 74 | signal.signal(signal.SIGINT, signal_handler) 75 | 76 | 77 | def get_file_or_s3(uri): 78 | logger.info(f"getting file URI {uri}") 79 | 80 | if uri.lower().startswith("s3://"): 81 | s3 = boto3.resource("s3") 82 | parse_result = urlparse(uri) 83 | s3_object = s3.Object(parse_result.netloc, parse_result.path.lstrip("/")) 84 | return s3_object.get()["Body"].read().decode("utf-8") 85 | 86 | return open(uri).read() 87 | 88 | 89 | def download_file(url, filename, headers={}): 90 | with requests.get(url, stream=True, headers=headers) as r: 91 | r.raise_for_status() 92 | with open(filename, 'wb') as f: 93 | for chunk in r.iter_content(chunk_size=8192): 94 | if chunk: 95 | f.write(chunk) 96 | 97 | 98 | def get_terraform_version(bucket, key): 99 | logger.info(f"getting Terraform version from remote state at s3://{bucket}/{key}") 100 | 101 | remote_state = json.loads(get_file_or_s3(f"s3://{bucket}/{key}")) 102 | 103 | version = remote_state["terraform_version"] 104 | logger.debug(f"terraform version = {version}") 105 | 106 | return version 107 | 108 | 109 | def install_terraform(version): 110 | logger.info(f"installing Terraform version {version}") 111 | 112 | filename = f"terraform_{version}_linux_amd64" 113 | url = f"https://releases.hashicorp.com/terraform/{version}/{filename}.zip" 114 | zip = f"{settings.TMP_FOLDER}/{filename}.zip" 115 | out_path = f"{settings.TMP_FOLDER}/{filename}" 116 | bin = f"{out_path}/terraform" 117 | 118 | logger.debug(f"downloading Terraform from {url}") 119 | download_file(url=url, filename=zip) 120 | 121 | logger.debug(f"making output directory {out_path}") 122 | os.mkdir(out_path) 123 | 124 | logger.debug(f"unzipping archive {zip} to {out_path}") 125 | zip_output = subprocess.Popen( 126 | f"unzip -o {zip} -d {out_path}", 127 | cwd=settings.TMP_FOLDER, 128 | shell=True, 129 | stdout=subprocess.PIPE, 130 | stderr=subprocess.PIPE 131 | ).stderr.read() 132 | 133 | logger.debug(f"zip stderr output was: {zip_output}") 134 | 135 | return bin 136 | 137 | 138 | def fetch_current_repo_head(): 139 | logger.info(f"getting current HEAD of {settings.TERRAFORM_GITHUB_REPO}") 140 | 141 | api_url = f"https://api.github.com/repos/{settings.TERRAFORM_GITHUB_REPO}/branches/{settings.TERRAFORM_GITHUB_BRANCH}" 142 | 143 | r=requests.get(api_url, headers={ 144 | "Authorization": f"token {settings.TERRAFORM_GITHUB_TOKEN}", 145 | "User-Agent": f"Drifter (Terraform monitor)" 146 | }) 147 | 148 | parsed_json = json.loads(r.text) 149 | 150 | repo_sha = parsed_json["commit"]["sha"] 151 | 152 | logger.info(f"commit SHA was {repo_sha}") 153 | 154 | zip = f"{settings.TMP_FOLDER}/repo.zip" 155 | out_path = f"{settings.TMP_FOLDER}/repo" 156 | 157 | modified_repo_name = settings.TERRAFORM_GITHUB_REPO.replace("/", "-") 158 | full_repo_path = f"{out_path}/{modified_repo_name}-{repo_sha}" 159 | 160 | if os.path.isdir(full_repo_path): 161 | logger.info(f"skipping download as it already exists in {settings.TMP_FOLDER}") 162 | else: 163 | api_url = f"https://api.github.com/repos/{settings.TERRAFORM_GITHUB_REPO}/zipball/{repo_sha}" 164 | 165 | logger.debug(f"downloading repo from {api_url}") 166 | 167 | download_file(url=api_url, filename=zip, headers={ 168 | "Authorization": f"token {settings.TERRAFORM_GITHUB_TOKEN}", 169 | "User-Agent": f"Drifter (Terraform monitor)" 170 | }) 171 | 172 | logger.debug(f"making output directory {out_path}") 173 | os.mkdir(out_path) 174 | 175 | logger.debug(f"unzipping archive {zip} to {out_path}") 176 | zip_output = subprocess.Popen( 177 | f"unzip -o {zip} -d {out_path}", 178 | cwd=settings.TMP_FOLDER, 179 | shell=True, 180 | stdout=subprocess.PIPE, 181 | stderr=subprocess.PIPE 182 | ).stderr.read() 183 | 184 | logger.debug(f"zip stderr output was: {zip_output}") 185 | 186 | return full_repo_path 187 | 188 | 189 | def terraform_initialise(terraform_bin, repo_folder): 190 | logger.info(f"initialising Terraform ({terraform_bin})") 191 | 192 | candidate_folder = repo_folder 193 | if settings.TERRAFORM_GITHUB_FOLDER: 194 | candidate_folder = f"{candidate_folder}/{settings.TERRAFORM_GITHUB_FOLDER}" 195 | logger.info(f"using candidate repo folder {candidate_folder}") 196 | 197 | child = subprocess.Popen( 198 | f"{terraform_bin} init -input=false -lock=false -no-color", 199 | cwd=candidate_folder, 200 | shell=True, 201 | stdout=subprocess.PIPE, 202 | stderr=subprocess.PIPE 203 | ) 204 | 205 | init_output = get_utf8(child.stdout.read()) 206 | init_error = get_utf8(child.stderr.read()) 207 | 208 | if len(init_error) > 0: 209 | logger.info(f"terraform init failed. output was: {init_output}") 210 | logger.info(f"error was: {init_error}") 211 | 212 | alert_slack(f"problem during initialise: {init_error}") 213 | return False 214 | 215 | logger.debug(f"terraform init output was: {init_output}") 216 | return True 217 | 218 | 219 | def terraform_plan(terraform_bin, repo_folder): 220 | logger.info(f"planning Terraform ({terraform_bin}) using {repo_folder}") 221 | 222 | candidate_folder = repo_folder 223 | if settings.TERRAFORM_GITHUB_FOLDER: 224 | candidate_folder = f"{candidate_folder}/{settings.TERRAFORM_GITHUB_FOLDER}" 225 | logger.info(f"using candidate repo folder {candidate_folder}") 226 | 227 | plan_start_time = time.time() 228 | 229 | child = subprocess.Popen( 230 | f"{terraform_bin} plan --detailed-exitcode -input=false -lock=false -no-color", 231 | cwd=candidate_folder, 232 | shell=True, 233 | stdout=subprocess.PIPE, 234 | stderr=subprocess.PIPE, 235 | universal_newlines=True 236 | ) 237 | 238 | stdout, stderr = child.communicate() 239 | 240 | exit_code = child.poll() 241 | 242 | plan_time_taken = time.time() - plan_start_time 243 | 244 | plan_output = get_utf8(stdout) 245 | plan_error = get_utf8(stderr) 246 | 247 | if exit_code == 1: 248 | # plan failed 249 | logger.info(f"terraform plan failed. output was: {plan_output}") 250 | logger.info(f"error was: {plan_error}") 251 | return None 252 | else: 253 | # plan finished 254 | logger.debug(f"terraform plan output was: {plan_output}") 255 | 256 | logger.info(f"plan finished") 257 | 258 | resource_count = 0 259 | pending_add = 0 260 | pending_change = 0 261 | pending_destroy = 0 262 | pending_total = 0 263 | 264 | resource_regex = re.compile(r".*Refreshing state\.\.\.") 265 | plan_regex = re.compile(r".*Plan: (\d+) to add, (\d+) to change, (\d+) to destroy\.") 266 | 267 | for plan_line in plan_output.split("\n"): 268 | # count number of resources 269 | if resource_regex.match(plan_line): 270 | resource_count = resource_count + 1 271 | 272 | m = plan_regex.match(plan_line) 273 | if m: 274 | pending_add = int(m.group(1)) 275 | pending_change = int(m.group(2)) 276 | pending_destroy = int(m.group(3)) 277 | 278 | logger.debug(f"line: {plan_line}") 279 | logger.debug(f"pending add: {pending_add}") 280 | logger.debug(f"pending change: {pending_change}") 281 | logger.debug(f"pending destroy: {pending_destroy}") 282 | 283 | pending_total = pending_add + pending_change + pending_destroy 284 | 285 | # logger.debug(f"pending total: {pending_total}") 286 | 287 | # logger.debug(f"plan time taken: {plan_time_taken}") 288 | # logger.debug(f"terraform_status: {exit_code}") 289 | 290 | return { 291 | "terraform_status": exit_code, 292 | "resource_count": resource_count, 293 | "pending_add": pending_add, 294 | "pending_change": pending_change, 295 | "pending_destroy": pending_destroy, 296 | "pending_total": pending_total, 297 | "plan_time": plan_time_taken 298 | } 299 | 300 | 301 | def get_utf8(input): 302 | try: 303 | input = input.decode("utf-8") 304 | except (UnicodeDecodeError, AttributeError): 305 | pass 306 | 307 | return input 308 | 309 | 310 | def ship_metrics_to_console(metrics): 311 | logger.info(f"ship_metrics_to_console") 312 | 313 | message = pretty_print_metrics(metrics) 314 | logger.info(message) 315 | 316 | 317 | def ship_metrics_to_cloudwatch(metrics): 318 | logger.info(f"shipping metrics to cloudwatch") 319 | cloudwatch = boto3.client("cloudwatch", settings.AWS_REGION) 320 | 321 | cloudwatch.put_metric_data( 322 | MetricData=[ 323 | { 324 | "MetricName": "Pending-Add", 325 | "Dimensions": [ 326 | { 327 | "Name": "GitHubRepo", 328 | "Value": settings.TERRAFORM_GITHUB_REPO 329 | } 330 | ], 331 | "Unit": "Count", 332 | "Value": metrics["pending_add"] 333 | }, 334 | { 335 | "MetricName": "Pending-Change", 336 | "Dimensions": [ 337 | { 338 | "Name": "GitHubRepo", 339 | "Value": settings.TERRAFORM_GITHUB_REPO 340 | } 341 | ], 342 | "Unit": "Count", 343 | "Value": metrics["pending_change"] 344 | }, 345 | { 346 | "MetricName": "Pending-Destroy", 347 | "Dimensions": [ 348 | { 349 | "Name": "GitHubRepo", 350 | "Value": settings.TERRAFORM_GITHUB_REPO 351 | } 352 | ], 353 | "Unit": "Count", 354 | "Value": metrics["pending_destroy"] 355 | }, 356 | { 357 | "MetricName": "Pending-Total", 358 | "Dimensions": [ 359 | { 360 | "Name": "GitHubRepo", 361 | "Value": settings.TERRAFORM_GITHUB_REPO 362 | } 363 | ], 364 | "Unit": "Count", 365 | "Value": metrics["pending_total"] 366 | } 367 | ], 368 | Namespace=settings.CLOUDWATCH_NAMESPACE 369 | ) 370 | 371 | 372 | def deduplicate_alert(metrics): 373 | logger.info(f"deduplicating alert") 374 | return True 375 | 376 | 377 | def alert_slack(message): 378 | logger.info(f"alerting to Slack") 379 | 380 | _ = requests.post(settings.SLACK_WEBHOOK_URL, json={"text": message, "link_names": 1}) 381 | 382 | 383 | def get_relative_time(start_time, end_time): 384 | return relativedelta(microsecond=int(round((end_time-start_time) * 1000000))) 385 | 386 | 387 | def pretty_print_metrics(metrics): 388 | logger.info(f"pretty_print_metrics") 389 | 390 | attrs = ["years", "months", "days", "hours", "minutes", "seconds", "microsecond"] 391 | human_readable = lambda delta: ["%d %s" % (getattr(delta, attr), getattr(delta, attr) > 1 and attr or attr[:-1]) 392 | for attr in attrs if getattr(delta, attr)] 393 | 394 | pending_message = f'Repository: {settings.TERRAFORM_GITHUB_REPO}, Branch: {settings.TERRAFORM_GITHUB_BRANCH}, Folder: {settings.TERRAFORM_GITHUB_FOLDER}\nPending: Add={metrics["pending_add"]}, Change={metrics["pending_change"]}, Destroy={metrics["pending_destroy"]}, Total={metrics["pending_total"]}' 395 | 396 | changes_message = "No changes detected." 397 | 398 | if metrics["terraform_status"] == 2: 399 | changes_message = f"Drift detected! {pending_message}" 400 | 401 | resources_message = f'Resource count: {metrics["resource_count"]}' 402 | 403 | delta = relativedelta(seconds=metrics["plan_time"]) 404 | 405 | time_taken_human_readable = ", ".join(human_readable(delta)) 406 | timing_message = f"Plan took {time_taken_human_readable}." 407 | 408 | return f"{changes_message}\n{resources_message}\n{timing_message}" 409 | 410 | 411 | if __name__ == "__main__": 412 | if settings.DEBUG: 413 | logzero.loglevel(logging.DEBUG) 414 | else: 415 | logzero.loglevel(logging.INFO) 416 | 417 | main() 418 | -------------------------------------------------------------------------------- /app/settings.py: -------------------------------------------------------------------------------- 1 | import os 2 | import database 3 | import re 4 | import distutils.util 5 | from database.sqlite_database import SqliteDatabase 6 | from database.postgresql_database import PostgreSqlDatabase 7 | 8 | DEBUG = bool(distutils.util.strtobool(os.getenv("DEBUG", "True"))) 9 | DB_TYPE = os.getenv("DB_TYPE") 10 | TERRAFORM_S3_BUCKET = os.getenv("TERRAFORM_S3_BUCKET") 11 | TERRAFORM_S3_KEY = os.getenv("TERRAFORM_S3_KEY") 12 | TERRAFORM_GITHUB_REPO = os.getenv("TERRAFORM_GITHUB_REPO") 13 | TERRAFORM_GITHUB_BRANCH = os.getenv("TERRAFORM_GITHUB_BRANCH", default="master") 14 | TERRAFORM_GITHUB_TOKEN = os.getenv("TERRAFORM_GITHUB_TOKEN") 15 | TERRAFORM_GITHUB_FOLDER = os.getenv("TERRAFORM_GITHUB_FOLDER") 16 | CLOUDWATCH_NAMESPACE = os.getenv("CLOUDWATCH_NAMESPACE") 17 | AWS_REGION = os.getenv("AWS_REGION") 18 | SLACK_WEBHOOK_URL = os.getenv("SLACK_WEBHOOK_URL") 19 | TMP_FOLDER = os.getenv("TMP_FOLDER", default="/tmp") 20 | 21 | 22 | def get_database(): 23 | if DB_TYPE == "postgresql": 24 | return get_database_postgresql() 25 | else: 26 | return get_database_sqlite() 27 | 28 | 29 | def get_database_postgresql(): 30 | db = PostgreSqlDatabase() 31 | db.initialise({ 32 | "dbname": os.getenv("DB_NAME"), 33 | "user": os.getenv("DB_USER"), 34 | "host": os.getenv("DB_HOST"), 35 | "password": os.getenv("DB_PASSWORD") 36 | }) 37 | 38 | return db 39 | 40 | 41 | def get_database_sqlite(): 42 | db = SqliteDatabase() 43 | db.initialise({ 44 | "db_name": os.getenv("DB_NAME") 45 | }) 46 | 47 | return db -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | logzero 2 | requests 3 | python-dateutil 4 | boto3 5 | -------------------------------------------------------------------------------- /tf/main.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | identifier = "drifter-${var.terraform_identifier}" 3 | } 4 | 5 | module "drifter_task" { 6 | source = "git::https://github.com/digirati-co-uk/terraform-aws-modules.git//tf/modules/services/tasks/base/?ref=v1.0" 7 | 8 | environment_variables = { 9 | "DEBUG" = "True" 10 | "SLACK_WEBHOOK_URL" = "${var.slack_webhook_url}" 11 | "TERRAFORM_S3_BUCKET" = "${var.terraform_s3_bucket}" 12 | "TERRAFORM_S3_KEY" = "${var.terraform_s3_key}" 13 | "TERRAFORM_GITHUB_REPO" = "${var.terraform_github_repo}" 14 | "TERRAFORM_GITHUB_BRANCH" = "${var.terraform_github_branch}" 15 | "TERRAFORM_GITHUB_FOLDER" = "${var.terraform_github_folder}" 16 | "TERRAFORM_GITHUB_TOKEN" = "${var.terraform_github_token}" 17 | "CLOUDWATCH_NAMESPACE" = "${var.cloudwatch_namespace}" 18 | "AWS_REGION" = "${var.region}" 19 | "TMP_FOLDER" = "${var.tmp_folder}" 20 | } 21 | 22 | environment_variables_length = 11 23 | 24 | prefix = "${var.prefix}" 25 | log_group_name = "${var.log_group_name}" 26 | log_group_region = "${var.region}" 27 | log_prefix = "${var.prefix}-${local.identifier}" 28 | 29 | family = "${var.prefix}-${local.identifier}" 30 | 31 | container_name = "${var.prefix}-${local.identifier}" 32 | 33 | cpu_reservation = 0 34 | memory_reservation = 128 35 | 36 | docker_image = "${var.drifter_docker_image}" 37 | } 38 | 39 | module "drifter" { 40 | source = "git::https://github.com/digirati-co-uk/terraform-aws-modules.git//tf/modules/services/tasks/scheduled/?ref=v1.0" 41 | 42 | family = "${var.prefix}-${local.identifier}" 43 | task_role_name = "${module.drifter_task.role_name}" 44 | region = "${var.region}" 45 | account_id = "${var.account_id}" 46 | cluster_arn = "${var.cluster_id}" 47 | schedule_expression = "${var.cron_expression}" 48 | desired_count = 1 49 | task_definition_arn = "${module.drifter_task.task_definition_arn}" 50 | } 51 | 52 | data "aws_s3_bucket" "terraform_s3_bucket" { 53 | bucket = "${var.terraform_s3_bucket}" 54 | } 55 | 56 | data "aws_iam_policy_document" "drifter_abilities" { 57 | statement { 58 | actions = [ 59 | "s3:ListBucket", 60 | ] 61 | 62 | resources = [ 63 | "${data.aws_s3_bucket.terraform_s3_bucket.arn}", 64 | ] 65 | } 66 | 67 | statement { 68 | actions = [ 69 | "s3:GetObject", 70 | ] 71 | 72 | resources = [ 73 | "${data.aws_s3_bucket.terraform_s3_bucket.arn}/*", 74 | ] 75 | } 76 | 77 | statement { 78 | actions = [ 79 | "cloudwatch:PutMetricData", 80 | ] 81 | 82 | resources = [ 83 | "*", 84 | ] 85 | } 86 | } 87 | 88 | resource "aws_iam_role_policy" "drifter_abilities" { 89 | name = "${var.prefix}-drifter-${local.identifier}-abilities" 90 | role = "${module.drifter_task.role_name}" 91 | policy = "${data.aws_iam_policy_document.drifter_abilities.json}" 92 | } 93 | 94 | data "aws_iam_policy" "readonly" { 95 | arn = "arn:aws:iam::aws:policy/ReadOnlyAccess" 96 | } 97 | 98 | resource "aws_iam_role_policy_attachment" "drifter_readonly" { 99 | role = "${module.drifter_task.role_name}" 100 | policy_arn = "${data.aws_iam_policy.readonly.arn}" 101 | } 102 | -------------------------------------------------------------------------------- /tf/variables.tf: -------------------------------------------------------------------------------- 1 | variable "slack_webhook_url" {} 2 | variable "terraform_identifier" {} 3 | variable "terraform_s3_bucket" {} 4 | 5 | variable "terraform_s3_key" { 6 | default = "terraform.tfstate" 7 | } 8 | 9 | variable "terraform_github_repo" {} 10 | 11 | variable "terraform_github_branch" { 12 | default = "master" 13 | } 14 | 15 | variable "terraform_github_folder" { 16 | default = "" 17 | } 18 | 19 | variable "terraform_github_token" {} 20 | variable "cloudwatch_namespace" {} 21 | 22 | variable "tmp_folder" { 23 | default = "/tmp" 24 | } 25 | 26 | variable "log_group_name" {} 27 | variable "prefix" {} 28 | variable "region" {} 29 | 30 | variable "drifter_docker_image" { 31 | default = "digirati/drifter:latest" 32 | } 33 | 34 | variable "account_id" {} 35 | variable "cluster_id" {} 36 | variable "cron_expression" {} 37 | --------------------------------------------------------------------------------