├── LICENSE ├── README.md └── packt-sre-code └── packt-sre ├── IAC ├── .gitignore ├── 1-codecommit │ ├── backend.tf │ ├── main.tf │ ├── outputs.tf │ └── provider.tf ├── 10-eks │ ├── backend.tf │ ├── configmap.yaml │ ├── main.tf │ ├── outputs.tf │ ├── provider.tf │ ├── worker-iam.tf │ └── workers.tf ├── 11-aurora │ ├── backend.tf │ ├── main.tf │ ├── outputs.tf │ ├── provider.tf │ ├── terraform.tfvars │ └── variables.tf ├── 12-appmesh │ ├── pyglobal-mesh-virtual-node.yml │ ├── pyglobal-mesh.yml │ └── pyglobal-namepsace.yml ├── 2-codebuild │ ├── backend.tf │ ├── files │ │ └── policy.json │ ├── main.tf │ └── provider.tf ├── 3-ecr │ ├── backend.tf │ ├── main.tf │ └── provider.tf ├── 4-rds │ ├── backend.tf │ ├── main.tf │ ├── outputs.tf │ ├── provider.tf │ ├── terraform.tfvars │ └── variables.tf ├── 5-ecs │ ├── backend.tf │ ├── main.tf │ ├── outputs.tf │ ├── terraform.tfvars │ └── variables.tf ├── 6-ecs-task │ ├── backend.tf │ ├── main.tf │ └── provider.tf ├── 7-multiregion-ecs │ ├── backend.tf │ ├── ecs-module.tf │ └── output.tf ├── 8-multiregion-rds │ ├── backend.tf │ ├── output.tf │ └── rds-module.tf └── 9-ecs-task │ ├── backend.tf │ ├── main.tf │ └── provider.tf ├── build ├── alb-bucket-policy └── buildspec.yml ├── cars-ms-pipeline.json ├── py-auth ├── auth │ ├── awsHelper.py │ ├── decoder.py │ ├── dev_cfg.ini │ └── main.py └── requirements.txt ├── py-cars ├── .dockerignore ├── .gitignore ├── Dockerfile ├── README.md ├── buildspec.yml ├── cars │ ├── __init__.py │ ├── awsHelper.py │ ├── dbHelper.py │ ├── local_cfg.ini │ ├── main.py │ └── schema.sql ├── k8 │ └── std │ │ ├── cars-deployment.yaml │ │ ├── cars-service.yaml │ │ └── iam-policy.json ├── k8commands.txt ├── requirements.txt ├── skaffold.yaml └── tests │ └── api-tests.py ├── pysimple ├── .dockerignore ├── .gitignore ├── Dockerfile ├── README.md ├── buildspec.yml ├── carLister │ ├── __init__.py │ ├── awsHelper.py │ ├── dbHelper.py │ ├── local_cfg.ini │ ├── main.py │ └── schema.sql ├── requirements.txt └── tests │ └── api-tests.py └── rds-cdc ├── lambda-py.py ├── local_cfg.ini └── main.py /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Packt 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | # Site Reliability Engineering on AWS [Video] 5 | This is the code repository for [Site Reliability Engineering on AWS](https://www.packtpub.com/cloud-networking/site-reliability-engineering-on-aws-video). It contains all the supporting project files necessary to work through the video course from start to finish. 6 | ## About the Video Course 7 | Reliability in AWS includes the ability of a system to recover from infrastructure or service disruptions. It's essential to acquire computing resources to meet the demand, and mitigate disruptions such as configuration issues or transient network problems. 8 | 9 | In this course, you will first explore the key concepts and core services of AWS and Site Reliability Engineering (SRE). We show you step-by-step how to implement a real-world application that is built via the reliability principles defined within the AWS Well-Architected Framework using the SRE approach. So you can increase the reliability of application architectures on AWS by implementing resilience infrastructure and application resilience. 10 | 11 | You will be covering some common architectural patterns used every day by real-world AWS solution architects to build reliable systems and implement fault tolerance into an application architecture running on AWS. While learning how to further increase the reliability of application architectures on AWS by implementing multi-region solutions for disaster recovery on a global scale. 12 | 13 | By the end of this course, you will have gained a variety of AWS architecture skills that you can then apply to the real world. 14 | 15 |

What You Will Learn

16 |
17 |
24 | 25 | ## Instructions and Navigation 26 | ### Assumed Knowledge 27 | To fully benefit from the coverage included in this course, you will need:
28 | 30 | 31 | ### Technical Requirements 32 | This course has the following software requirements:
33 | 37 | 38 | ## Related Products 39 | * [AWS Certified Developer - Associate Certification [Video]](https://www.packtpub.com/virtualization-and-cloud/aws-certified-developer-associate-certification-video) 40 | 41 | * [AWS Certified Security – Specialty [Video]](https://www.packtpub.com/cloud-networking/aws-certified-security-specialty-video) 42 | 43 | * [AWS Certified Cloud Practitioner (CLF-C01) [Video]](https://www.packtpub.com/cloud-networking/aws-certified-cloud-practitioner-clf-c01-video) 44 | 45 | -------------------------------------------------------------------------------- /packt-sre-code/packt-sre/IAC/.gitignore: -------------------------------------------------------------------------------- 1 | .terraform 2 | terraform.tfstate 3 | **/.terraform 4 | **/terraform.tfstate -------------------------------------------------------------------------------- /packt-sre-code/packt-sre/IAC/1-codecommit/backend.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | backend "s3" { 3 | bucket = "tfstate-widgets-com" 4 | key = "codecommit/euwest2/terraform.tfstate" 5 | region = "eu-west-2" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /packt-sre-code/packt-sre/IAC/1-codecommit/main.tf: -------------------------------------------------------------------------------- 1 | resource "aws_codecommit_repository" "pysimple" { 2 | repository_name = "pysimple" 3 | description = "This is the initial move and improve code repository" 4 | } 5 | -------------------------------------------------------------------------------- /packt-sre-code/packt-sre/IAC/1-codecommit/outputs.tf: -------------------------------------------------------------------------------- 1 | output "clone_url" { 2 | value = aws_codecommit_repository.pysimple.clone_url_http 3 | 4 | } 5 | -------------------------------------------------------------------------------- /packt-sre-code/packt-sre/IAC/1-codecommit/provider.tf: -------------------------------------------------------------------------------- 1 | provider "aws" { 2 | version = "~> 2.0" 3 | region = "eu-west-2" 4 | } 5 | -------------------------------------------------------------------------------- /packt-sre-code/packt-sre/IAC/10-eks/backend.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | backend "s3" { 3 | bucket = "tfstate-widgets-com" 4 | key = "eks/euwest2/terraform.tfstate" 5 | region = "eu-west-2" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /packt-sre-code/packt-sre/IAC/10-eks/configmap.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: aws-auth 5 | namespace: kube-system 6 | data: 7 | mapRoles: | 8 | - rolearn: arn:aws:iam::915793320862:role/eks-main-node 9 | username: system:node:{{EC2PrivateDNSName}} 10 | groups: 11 | - system:bootstrappers 12 | - system:nodes -------------------------------------------------------------------------------- /packt-sre-code/packt-sre/IAC/10-eks/main.tf: -------------------------------------------------------------------------------- 1 | #setup any local variables 2 | locals { 3 | vpc_id = "vpc-05939a73c8684c026" 4 | cluster_subnets = ["subnet-071fabf574452338c", "subnet-0b05b60e1a8d9a65b","subnet-0b0e88e7e28517c5e","subnet-08a8928bd3ccbc36e"] 5 | worker_subnets = ["subnet-071fabf574452338c", "subnet-0b05b60e1a8d9a65b"] 6 | cluster_name = "tf-eks-pyglobal-eu-west-2" 7 | rds_instance_name = "cars" 8 | eks_security_group = "sg-0b2e88cc0874e81b9" 9 | alb_arn = "arn:aws:elasticloadbalancing:eu-west-2:915793320862:loadbalancer/app/tf-ecs-simplepy-alb-eu-west-2/8d55184c3c904bd0" 10 | alb_name = "tf-ecs-simplepy-alb-eu-west-2" 11 | alb_target_arn = "arn:aws:elasticloadbalancing:eu-west-2:915793320862:targetgroup/tf-ecs-simplepy-eu-west-2/93d77d67f48af047" 12 | main-node-userdata = < claims['exp']: 49 | logging.error('Token has expired') 50 | result = {"error": "Token has expired"} 51 | return result 52 | # and the Audience (use claims['client_id'] if verifying an access token) 53 | if claims['token_use'] == 'id': 54 | if claims['aud'] != cognito_clinet_id: 55 | logging.error('Token was not issued for this audience') 56 | result = {"error": "Token was not issued for this audience"} 57 | return result 58 | if claims['token_use'] == 'access': 59 | if claims['client_id'] != cognito_clinet_id: 60 | logging.error('Token was not issued for this audience') 61 | result = {"error": "Token was not issued for this audience"} 62 | return result 63 | 64 | result['data'] = claims 65 | return result 66 | '''''' -------------------------------------------------------------------------------- /packt-sre-code/packt-sre/py-auth/auth/dev_cfg.ini: -------------------------------------------------------------------------------- 1 | [user_user_pool] 2 | id=eu-west-2_zMGwyN5WF 3 | region=eu-west-2 4 | key_url=https://cognito-idp.eu-west-2.amazonaws.com/my-id/.well-known/jwks.json 5 | 6 | [cognito_app] 7 | client_id= 8 | client_secret= 9 | domain=https://myauth.com 10 | -------------------------------------------------------------------------------- /packt-sre-code/packt-sre/py-auth/auth/main.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from aws_xray_sdk.core import xray_recorder 3 | from aws_xray_sdk.core.context import Context 4 | from aws_xray_sdk.ext.flask.middleware import XRayMiddleware 5 | from aws_xray_sdk.core import patch_all 6 | patch_all() 7 | 8 | 9 | from flask import request, Response, redirect,abort,make_response,jsonify 10 | from flask import Flask 11 | from eventlet import wsgi, monkey_patch, listen 12 | import logging 13 | import os 14 | import json 15 | from configparser import ConfigParser 16 | import awsHelper as cog 17 | import decoder as jwt 18 | import sys 19 | import requests 20 | from functools import wraps 21 | #logging config 22 | logging.basicConfig(format='%(asctime)s %(levelname)-8s %(message)s',level=logging.INFO,datefmt='%Y-%m-%d %H:%M:%S') 23 | logger = logging.getLogger(__name__) 24 | 25 | #globals 26 | MODULE = "widgets-auth" 27 | HOST = "0.0.0.0" 28 | PORT = "8000" 29 | REDIRECT = f"http://127.0.0.1:{PORT}/api/v0.1/auth/login" 30 | PROFILE = "default" 31 | REGION = "eu-west-2" 32 | PROFILE = "aws-dev" 33 | REGION = "eu-west-2" 34 | ENV="dev" 35 | CONFIG_FILE = f"{ENV}_cfg.ini" 36 | 37 | #initiliase flask 38 | app = Flask(__name__) 39 | app.secret_key = os.urandom(24) 40 | cidp = cog.createClient(REGION,"cognito-idp") 41 | xray_recorder.configure( 42 | sampling=False, 43 | context_missing='LOG_ERROR', 44 | service='pyapp', 45 | context=Context() 46 | ) 47 | XRayMiddleware(app, xray_recorder) 48 | 49 | 50 | def check_token(f): 51 | @wraps(f) 52 | def wrap(*args, **kwargs): 53 | allowed = False 54 | keys = USER_POOL['keys'] 55 | idtoken = request.cookies.get("idtoken") 56 | atoken = request.cookies.get("access_token") 57 | if idtoken == None and atoken == None: 58 | return abort(400, description="no identity or access token") 59 | if idtoken != None: 60 | i_tokenObject = jwt.decode_cognito_token(idtoken, COGNITO_CFG['client_id'], keys) 61 | if 'error' in i_tokenObject: 62 | logging.error('failed admin pool token auth, generating error') 63 | result = {'error': "token not valid", "result": "fail"} 64 | return abort(401, description="invalid id_token") 65 | if atoken != None: 66 | a_tokenObject = jwt.decode_cognito_token(atoken, COGNITO_CFG['client_id'], keys) 67 | if 'error' in a_tokenObject: 68 | logging.error('failed admin pool token auth, generating error') 69 | result = {'error' : "token not valid", "result": "fail"} 70 | return abort(401,description="invalid access_token") 71 | action = request.method 72 | resource = request.endpoint 73 | resource_url = request.base_url 74 | user_pool_id = str(i_tokenObject['data']['iss']).split("/")[-1] 75 | first_group = i_tokenObject['data']['cognito:groups'][0] 76 | inferred_scope = f"{first_group}.{resource}.{action}" 77 | resource_response = cog.decribe_rs(cidp, user_pool_id, resource_url) 78 | try: 79 | error = resource_response['error'] 80 | except Exception as e: 81 | logging.info(f"checking {inferred_scope} in {resource_response['ResourceServer']['Scopes']}") 82 | for scope in resource_response['ResourceServer']['Scopes']: 83 | if scope['ScopeName'] == inferred_scope: 84 | logging.info('matched inferred scope') 85 | allowed = True 86 | else: 87 | if "ResourceNotFoundException" in resource_response['error']: 88 | logging.info(f"not a protect resource:passing") 89 | allowed = True 90 | else: 91 | logging.info(f"error for resource:{error}") 92 | allowed = True 93 | if allowed: 94 | return f(*args, **kwargs) 95 | else: 96 | return abort(403, description="Inferred permission denied") 97 | return wrap 98 | 99 | def get_keys(url): 100 | try: 101 | result = requests.get(url, timeout=1) 102 | except Exception as e: 103 | logging.error(f'failed to get userpool JWK {str(e)}') 104 | raise e 105 | else: 106 | logging.debug(result.json()) 107 | return result.json() 108 | 109 | def get_config(filename,section): 110 | parser = ConfigParser() 111 | parser.read(filename) 112 | config = {} 113 | if parser.has_section(section): 114 | params = parser.items(section) 115 | for param in params: 116 | config[param[0]] = param[1] 117 | logging.info(f"added config section {section} from {filename}") 118 | else: 119 | logging.error(f'Section {section} not found in the {filename} file') 120 | sys.exit(1) 121 | return config 122 | 123 | USER_POOL = get_config(CONFIG_FILE, "user_user_pool") 124 | USER_POOL['keys'] = get_keys(USER_POOL['key_url']) 125 | USER_POOL_ID = USER_POOL['id'] 126 | COGNITO_CFG = get_config(CONFIG_FILE, "cognito_app") 127 | 128 | @app.route('/api//auth/login',methods=["POST"]) 129 | @xray_recorder.capture('login') 130 | def loginUser(version): 131 | user_pool = "admins" 132 | result = {} 133 | headers = {} 134 | username = request.authorization.username 135 | password = request.authorization.password 136 | authObject = cog.login(cidp,username,password,user_pool,COGNITO_CFG['client_id'],COGNITO_CFG['client_secret']) 137 | if 'error' in authObject: 138 | if 'User is disabled' in str(authObject['error']): 139 | result['error'] = "user disabled" 140 | else: 141 | result['error'] = str(authObject['error']) 142 | status = 401 143 | result['result'] = 'fail' 144 | else: 145 | result['result'] = "ok" 146 | result['data'] = authObject['AuthenticationResult'] 147 | status = 200 148 | lresponse = Response(json.dumps(result), status=status, mimetype='application/json',headers=headers) 149 | if status == 200: 150 | lresponse.set_cookie("idtoken",authObject['AuthenticationResult']['IdToken'],httponly=True,expires=None) 151 | lresponse.set_cookie("atoken", authObject['AuthenticationResult']['AccessToken'], httponly=True, expires=None) 152 | return lresponse 153 | 154 | 155 | @app.route('/api//auth/whoami',methods=["GET"]) 156 | @xray_recorder.capture('whoami') 157 | @check_token 158 | def whoami(version): 159 | user_pool_id = USER_POOL['id'] 160 | keys = USER_POOL['keys'] 161 | headers = {} 162 | idtoken = request.cookies.get("idtoken") 163 | accesstoken = request.cookies.get("atoken") 164 | i_tokenObject = jwt.decode_cognito_token(idtoken,COGNITO_CFG['client_id'],keys) 165 | a_tokenObject = jwt.decode_cognito_token(accesstoken, COGNITO_CFG['client_id'], keys) 166 | user_pool_id = str(i_tokenObject['data']['iss']).split("/")[-1] 167 | result = { 'result' : "ok", "data": {"id": i_tokenObject, "access" : a_tokenObject}, "userpool": user_pool_id } 168 | status = 200 169 | return Response(json.dumps(result), status=status, mimetype='application/json',headers=headers) 170 | 171 | @app.route('/api//auth/resource',methods=["GET"]) 172 | @check_token 173 | @xray_recorder.capture('resource_server_check') 174 | def resource_servers(version): 175 | headers = {} 176 | user_pool_id = USER_POOL['id'] 177 | keys = USER_POOL['keys'] 178 | resource_type = request.args.get('type') 179 | idtoken = request.cookies.get("idtoken") 180 | tokenObject = jwt.decode_cognito_token(idtoken, COGNITO_CFG['client_id'], keys) 181 | user_pool_id = str(tokenObject['data']['iss']).split("/")[-1] 182 | resource = cog.decribe_rs(cidp,user_pool_id,resource_type) 183 | result = { 'result' : "ok", "data": resource } 184 | status = 200 185 | return Response(json.dumps(result), status=status, mimetype='application/json',headers=headers) 186 | 187 | @app.route('/api//protected',methods=["GET"]) 188 | @check_token 189 | @xray_recorder.capture('protected') 190 | def protected(version): 191 | headers = {} 192 | result ={"result":"ok"} 193 | user_pool_id = USER_POOL['id'] 194 | keys = USER_POOL['keys'] 195 | status = 200 196 | return Response(json.dumps(result), status=status, mimetype='application/json',headers=headers) 197 | 198 | @app.route('/api//auth/signout',methods=["DELETE"]) 199 | @check_token 200 | def signout(version): 201 | headers = {} 202 | user_pool_id = USER_POOL['id'] 203 | keys = USER_POOL['keys'] 204 | idtoken = request.cookies.get("idtoken") 205 | tokenObject = jwt.decode_cognito_token(idtoken, COGNITO_CFG['client_id'], keys) 206 | username = tokenObject['data']['cognito:username'] 207 | signout_response = cog.admin_signout(cidp, user_pool_id, username) 208 | try: 209 | error = signout_response['error'] 210 | except Exception as e: 211 | logging.info(f"{username}signed out ") 212 | status = 200 213 | result = {"result": "ok"} 214 | else: 215 | status = 500 216 | logging.error(f"failed to sign out {username} with {error}") 217 | result = {"error": error} 218 | return Response(json.dumps(result), status=status, mimetype='application/json',headers=headers) 219 | 220 | @app.route('/api//welcome',methods=["GET"]) 221 | def home(version): 222 | headers = {} 223 | result = "Welcome to cars" 224 | user_pool_id = USER_POOL['id'] 225 | keys = USER_POOL['keys'] 226 | status = 200 227 | return Response(result, status=status, mimetype='text/html',headers=headers) 228 | 229 | 230 | 231 | def main(): 232 | logging.info(f"running {MODULE}") 233 | wsgi.server(listen(('', int(PORT))), app) 234 | #app.run(host='0.0.0.0', port=PORT) 235 | 236 | 237 | if __name__ == "__main__": 238 | main() 239 | 240 | 241 | -------------------------------------------------------------------------------- /packt-sre-code/packt-sre/py-auth/requirements.txt: -------------------------------------------------------------------------------- 1 | atomicwrites==1.4.0 2 | attrs==19.3.0 3 | aws-xray-sdk==2.6.0 4 | boto3==1.13.17 5 | botocore==1.16.17 6 | certifi==2020.4.5.1 7 | chardet==3.0.4 8 | click==7.1.2 9 | colorama==0.4.3 10 | dnspython==1.16.0 11 | docutils==0.15.2 12 | eventlet==0.25.2 13 | Flask==1.1.2 14 | future==0.18.2 15 | greenlet==0.4.15 16 | idna==2.9 17 | importlib-metadata==1.6.0 18 | itsdangerous==1.1.0 19 | Jinja2==2.11.2 20 | jmespath==0.10.0 21 | jsonpickle==1.4.1 22 | MarkupSafe==1.1.1 23 | monotonic==1.5 24 | more-itertools==8.3.0 25 | packaging==20.4 26 | pluggy==0.13.1 27 | psycopg2-binary==2.8.5 28 | py==1.8.1 29 | pyparsing==2.4.7 30 | pytest==5.4.2 31 | pytest-dependency==0.5.1 32 | python-dateutil==2.8.1 33 | requests==2.23.0 34 | s3transfer==0.3.3 35 | six==1.14.0 36 | urllib3==1.25.9 37 | wcwidth==0.1.9 38 | Werkzeug==1.0.1 39 | wrapt==1.12.1 40 | zipp==3.1.0 41 | -------------------------------------------------------------------------------- /packt-sre-code/packt-sre/py-cars/.dockerignore: -------------------------------------------------------------------------------- 1 | *.xlsx 2 | *.png 3 | venv 4 | .venv 5 | .git 6 | docs 7 | tests 8 | .idea -------------------------------------------------------------------------------- /packt-sre-code/packt-sre/py-cars/.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | .idea/ 131 | -------------------------------------------------------------------------------- /packt-sre-code/packt-sre/py-cars/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.8.0-slim-buster 2 | LABEL team="development" \ 3 | maintainer="Ian" \ 4 | email="ian@widgets.com" 5 | MAINTAINER "ian@widgets.com" 6 | ENV PATH=$PATH:/root/.local/bin 7 | RUN apt-get update \ 8 | && apt-get install gcc python3-dev musl-dev -y \ 9 | && apt-get clean 10 | WORKDIR /app 11 | COPY ./requirements.txt /app/requirements.txt 12 | RUN pip3 install --user -r requirements.txt 13 | COPY cars/ /app 14 | EXPOSE 8001 15 | ENTRYPOINT [ "python3" ] 16 | CMD [ "main.py" ] -------------------------------------------------------------------------------- /packt-sre-code/packt-sre/py-cars/README.md: -------------------------------------------------------------------------------- 1 | # aws-simple-py 2 | simple repo for packt course 3 | 4 | -------------------------------------------------------------------------------- /packt-sre-code/packt-sre/py-cars/buildspec.yml: -------------------------------------------------------------------------------- 1 | version: 0.2 2 | 3 | phases: 4 | install: 5 | commands: 6 | - echo Entering install phase... 7 | - pip3 install pytest pytest_dependency 8 | pre_build: 9 | commands: 10 | - echo Logging in to Amazon ECR... 11 | - $(aws ecr get-login --no-include-email --region $AWS_DEFAULT_REGION) 12 | build: 13 | commands: 14 | - echo Build started on `date` 15 | - echo Building the Docker image... 16 | - docker build -t $IMAGE_REPO_NAME:$IMAGE_TAG . 17 | - docker tag $IMAGE_REPO_NAME:$IMAGE_TAG $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$IMAGE_REPO_NAME:$IMAGE_TAG 18 | - docker tag $IMAGE_REPO_NAME:$IMAGE_TAG $AWS_ACCOUNT_ID.dkr.ecr.$AWS_SECOND_REGION.amazonaws.com/$IMAGE_REPO_NAME:$IMAGE_TAG 19 | 20 | post_build: 21 | commands: 22 | - echo Build completed on `date` starting test 23 | - docker run --name test -p 8001:8001 -d --rm --env HOST=$HOST --env DB=$DB --env DB_USER=$DB_USER --env DB_PASS=$DB_PASS --env CODEBUILD_BUILD_ID=$CODEBUILD_BUILD_ID $IMAGE_REPO_NAME:$IMAGE_TAG 24 | - sleep 15s 25 | - docker ps 26 | - docker logs test 27 | - curl http://127.0.0.1:8001/api/v0.1/health 28 | - python3 -m pytest tests/api-tests.py 29 | - echo Pushing the Docker image... 30 | - docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$IMAGE_REPO_NAME:$IMAGE_TAG 31 | - $(aws ecr get-login --no-include-email --region $AWS_SECOND_REGION) 32 | - docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_SECOND_REGION.amazonaws.com/$IMAGE_REPO_NAME:$IMAGE_TAG 33 | - echo "creating imagedefinitions.json for ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/${IMAGE_REPO_NAME}:${IMAGE_TAG}" 34 | - export JSON_FMT='[{"name":"%s","imageUri":"%s"}]' 35 | - printf $JSON_FMT "$IMAGE_REPO_NAME" "${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/${IMAGE_REPO_NAME}:${IMAGE_TAG}" > imagedefinitions.json 36 | - cat imagedefinitions.json 37 | artifacts: 38 | files: 39 | - imagedefinitions.json -------------------------------------------------------------------------------- /packt-sre-code/packt-sre/py-cars/cars/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PacktPublishing/Site-Reliability-Engineering-on-AWS/f22f86ecba8e662f8b5e189b202346cbebef05cd/packt-sre-code/packt-sre/py-cars/cars/__init__.py -------------------------------------------------------------------------------- /packt-sre-code/packt-sre/py-cars/cars/awsHelper.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | 4 | import logging 5 | import boto3 6 | import json 7 | from os import environ 8 | import requests 9 | URL = "http://169.254.169.254/latest/meta-data/public-ipv4" 10 | IN_AWS = False 11 | 12 | try: 13 | response = requests.get(URL,timeout=1) 14 | except Exception as e: 15 | logging.error(f"cannot connect to url:{URL} assuming not in AWS") 16 | else: 17 | print(response.txt) 18 | logging.info(f"got {response.text} from meta data server") 19 | IN_AWS = True 20 | 21 | 22 | #GlOBALS 23 | MODULE = "AWS-helper" 24 | PROFILE = "default" 25 | REGION = "eu-west-2" 26 | 27 | # logging config 28 | logging.basicConfig(format='%(asctime)s %(levelname)-8s %(message)s', level=logging.INFO, datefmt='%Y-%m-%d %H:%M:%S') 29 | logger = logging.getLogger(__name__) 30 | 31 | def createClient(region,service): 32 | logging.info(f"ID:{environ.get('CODEBUILD_BUILD_ID')}") 33 | if environ.get('CODEBUILD_BUILD_ID') is not None: 34 | logging.info('running in CodeBuild') 35 | return boto3.client(service, region_name=region) 36 | else: 37 | if IN_AWS: 38 | logging.info('running in AWS') 39 | return boto3.client(service, region_name=region) 40 | elif environ.get('ECS_RUN') is not None: 41 | logging.info('running in ECS') 42 | return boto3.client(service, region_name=region) 43 | else: 44 | logging.info(f'using AWS profile {PROFILE} for service:{service}') 45 | session = boto3.Session(profile_name=PROFILE) 46 | return session.client(service, region_name=region) 47 | 48 | 49 | def put_cloudwatch_metric(client,mname,a_name,c_name,m_value,uri,namespace): 50 | dimensions = [{'Name':'service','Value':a_name},{'Name':'component','Value':c_name},{'Name':'uri','Value':uri}] 51 | entry = {'MetricName':mname,'Dimensions':dimensions,'Unit':'None','Value':m_value} 52 | try: 53 | response = client.put_metric_data(MetricData=[entry],Namespace=namespace) 54 | except Exception as e: 55 | logging.error(f"put metric data error:{str(e)}") 56 | return 57 | else: 58 | logging.info(f"response:{response}") 59 | return response 60 | 61 | def describe_rds_instance(client,instance_name): 62 | try: 63 | response = client.describe_db_instances(DBInstanceIdentifier=instance_name) 64 | except Exception as e: 65 | logging.error(f"error:{str(e)}") 66 | return 67 | else: 68 | logging.info(f"response:{response}") 69 | return response['DBInstances'][0]['DBInstanceArn'] 70 | 71 | 72 | 73 | 74 | 75 | -------------------------------------------------------------------------------- /packt-sre-code/packt-sre/py-cars/cars/dbHelper.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | 4 | import logging 5 | import psycopg2 6 | 7 | 8 | #GlOBALS 9 | MODULE = "postgres-helper" 10 | 11 | # logging config 12 | logging.basicConfig(format='%(asctime)s %(levelname)-8s %(message)s', level=logging.INFO, datefmt='%Y-%m-%d %H:%M:%S') 13 | logger = logging.getLogger(__name__) 14 | 15 | 16 | def connect_to_db(dbconfig): 17 | try: 18 | conn = psycopg2.connect(**dbconfig) 19 | except (Exception, psycopg2.DatabaseError) as e: 20 | logging.error(f"connection error to {dbconfig['host']}:{dbconfig['database']} error {str(e)}") 21 | return 22 | else: 23 | logging.info(f"connected to {dbconfig['host']}:{dbconfig['database']}") 24 | return conn 25 | 26 | def get_db_version(conn): 27 | cur =conn.cursor() 28 | try: 29 | cur.execute('SELECT version()') 30 | except (Exception, psycopg2.DatabaseError) as e: 31 | logging.error(f"execution error {str(e)}") 32 | return 33 | else: 34 | db_version = cur.fetchone() 35 | logging.info(f"db versions {db_version}") 36 | cur.close() 37 | return db_version 38 | 39 | def init_db(conn,schema_file): 40 | conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT) 41 | cur = conn.cursor() 42 | try: 43 | cur.execute(open(schema_file, "r").read()) 44 | except Exception as e: 45 | logging.error(f"initdb execution error {str(e)}") 46 | return 47 | else: 48 | logging.info(f"tables created") 49 | cur.close() 50 | return 51 | 52 | def update_car_availbility(conn,reg_id,status): 53 | updated_rows = 0 54 | sql = f"UPDATE car_availability set available = {status} WHERE car_id = '{reg_id}'" 55 | cur = conn.cursor() 56 | try: 57 | cur.execute(sql) 58 | except Exception as e: 59 | conn.rollback() 60 | logging.error(f" update availability execution error {str(e)}") 61 | raise Exception(f'update car availability error with {str(e)}') 62 | return 63 | else: 64 | updated_rows = cur.rowcount 65 | conn.commit() 66 | finally: 67 | cur.close() 68 | return updated_rows 69 | 70 | def update_car(conn,car_det): 71 | updated_rows = 0 72 | sql = f"UPDATE cars set make = '{car_det[1]}' , model = '{car_det[2]}' , colour = '{car_det[3]}', capacity ='{car_det[4]}' WHERE number_plate = '{car_det[0]}'" 73 | cur = conn.cursor() 74 | try: 75 | cur.execute(sql) 76 | except Exception as e: 77 | conn.rollback() 78 | logging.error(f" update execution error {str(e)}") 79 | raise Exception(f'update car error with {str(e)}') 80 | return 81 | else: 82 | updated_rows = cur.rowcount 83 | conn.commit() 84 | finally: 85 | cur.close() 86 | return updated_rows 87 | 88 | def add_new_car(conn, car_det): 89 | sql = f"INSERT INTO cars VALUES('{car_det[0]}','{car_det[1]}','{car_det[2]}','{car_det[3]}','{car_det[4]}');" 90 | logging.debug(f"running sql {sql}") 91 | cur = conn.cursor() 92 | try: 93 | cur.execute(sql) 94 | except Exception as e: 95 | conn.rollback() 96 | logging.debug(f"db:add car execution error 1 {str(e)}") 97 | raise Exception(f'add car error with {str(e)}') 98 | else: 99 | sql = f"INSERT INTO car_availability (car_id, available) VALUES('{car_det[0]}','true');" 100 | logging.info(f"running sql {sql}") 101 | try: 102 | cur.execute(sql) 103 | except Exception as e: 104 | conn.rollback() 105 | logging.debug(f"add car availability execution error 1 {str(e)}") 106 | raise Exception(f'add car availabilty error with {str(e)}') 107 | else: 108 | logging.info(f"updated car_availability") 109 | logging.info(f"last notice {str(conn.notices)}") 110 | conn.commit() 111 | cur.close() 112 | 113 | def get_car(conn, reg_id): 114 | results = {} 115 | sql = f"SELECT * FROM cars WHERE number_plate = '{reg_id}';" 116 | cur = conn.cursor() 117 | try: 118 | cur.execute(sql) 119 | except Exception as e: 120 | logging.debug(f"get car execution error {str(e)}") 121 | raise Exception(f'get car error with {str(e)}') 122 | return 123 | else: 124 | car_records = cur.fetchall() 125 | for row in car_records: 126 | results[row[0]] = {"make":row[1],"model":row[2],"colour":row[3],"capacity":row[4]} 127 | finally: 128 | cur.close() 129 | return results 130 | 131 | 132 | def get_available_cars(conn): 133 | results = {} 134 | free_list = [] 135 | sql = f"SELECT car_id FROM car_availability WHERE available = TRUE ;" 136 | cur = conn.cursor() 137 | try: 138 | cur.execute(sql) 139 | except Exception as e: 140 | logging.debug(f"get car availability execution error {str(e)}") 141 | raise Exception(f'get car availability error with {str(e)}') 142 | return 143 | else: 144 | car_records = cur.fetchall() 145 | for row in car_records: 146 | free_list.append(row[0]) 147 | finally: 148 | results['result'] = free_list 149 | cur.close() 150 | return results 151 | 152 | def get_cars(conn): 153 | results = {} 154 | sql = f"SELECT number_plate FROM cars;" 155 | cur = conn.cursor() 156 | plate_list = [] 157 | try: 158 | cur.execute(sql) 159 | except Exception as e: 160 | logging.error(f"execution error {str(e)}") 161 | raise Exception(f'error with {str(e)}') 162 | return 163 | else: 164 | car_records = cur.fetchall() 165 | for row in car_records: 166 | plate_list.append(row[0]) 167 | finally: 168 | results['result'] = plate_list 169 | cur.close() 170 | return results 171 | 172 | def add_booking(conn, booking_det): 173 | sql = f"INSERT INTO bookings (user_name,car_id,pickup,drop_off,booking_title,description) VALUES('{booking_det[0]}','{booking_det[1]}','{booking_det[2]}','{booking_det[3]}','{booking_det[4]}','{booking_det[5]}') RETURNING booking_id;" 174 | logging.info(f"running sql {sql}") 175 | cur = conn.cursor() 176 | id = None 177 | try: 178 | cur.execute(sql) 179 | except Exception as e: 180 | conn.rollback() 181 | logging.error(f"add booking execution error 1 {str(e)}") 182 | raise Exception(f'add booking error with {str(e)}') 183 | else: 184 | id = cur.fetchone()[0] 185 | conn.commit() 186 | finally: 187 | cur.close() 188 | return id 189 | 190 | def get_bookings(conn): 191 | results = {} 192 | sql = f"SELECT booking_id,booking_title,user_name,approved,created,car_id FROM bookings ORDER by booking_id;" 193 | cur = conn.cursor() 194 | booking_list = [] 195 | try: 196 | cur.execute(sql) 197 | except Exception as e: 198 | logging.error(f"execution error for booking list {str(e)}") 199 | raise Exception(f'error with booking list: {str(e)}') 200 | return 201 | else: 202 | booking_records = cur.fetchall() 203 | for row in booking_records: 204 | booking_list.append({"id":row[0],"title":row[1],"raised_by":row[2],"approved":row[3],"created":str(row[4]),"car-registration":str(row[5])}) 205 | finally: 206 | results['result'] = booking_list 207 | cur.close() 208 | return results 209 | 210 | def get_booking(conn,current_booking_id): 211 | results = {} 212 | sql = f"SELECT booking_id,booking_title,user_name,approved,created,car_id FROM bookings WHERE booking_id = {current_booking_id};" 213 | cur = conn.cursor() 214 | try: 215 | cur.execute(sql) 216 | except Exception as e: 217 | logging.error(f"execution error for booking list {str(e)}") 218 | raise Exception(f'error with booking list: {str(e)}') 219 | return 220 | else: 221 | booking_records = cur.fetchall() 222 | for row in booking_records: 223 | results[row[0]] = {"title":row[1],"raised_by":row[2],"approved":row[3],"created":str(row[4]),"car-registration":str(row[5])} 224 | finally: 225 | cur.close() 226 | return results 227 | 228 | def approve_booking(conn,working_booking_id): 229 | sql = f"UPDATE bookings SET approved = 'true' WHERE booking_id = {working_booking_id} RETURNING car_id,drop_off;" 230 | logging.debug(f"running sql {sql}") 231 | cur = conn.cursor() 232 | try: 233 | cur.execute(sql) 234 | except Exception as e: 235 | conn.rollback() 236 | logging.debug(f"failed to approved booking {working_booking_id} {str(e)}") 237 | raise Exception(f'approval booking error with {str(e)}') 238 | else: 239 | row = id = cur.fetchone() 240 | sql = f"UPDATE car_availability SET available = 'false', expires = '{row[1]}' WHERE car_id = '{row[0]}';" 241 | logging.info(f"running sql {sql}") 242 | try: 243 | cur.execute(sql) 244 | except Exception as e: 245 | conn.rollback() 246 | logging.debug(f"update car availability execution error 1 {str(e)}") 247 | raise Exception(f'update car availability error with {str(e)}') 248 | conn.commit() 249 | cur.close() 250 | 251 | 252 | 253 | 254 | -------------------------------------------------------------------------------- /packt-sre-code/packt-sre/py-cars/cars/local_cfg.ini: -------------------------------------------------------------------------------- 1 | [postgresql] 2 | host=192.168.182.128 3 | database=cars 4 | user=carsa 5 | password=letmein12 -------------------------------------------------------------------------------- /packt-sre-code/packt-sre/py-cars/cars/main.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | from aws_xray_sdk.core import xray_recorder 4 | from aws_xray_sdk.core.context import Context 5 | from aws_xray_sdk.ext.flask.middleware import XRayMiddleware 6 | from aws_xray_sdk.core import patch_all 7 | patch_all() 8 | 9 | import logging 10 | import dbHelper as db 11 | import awsHelper as aws 12 | from configparser import ConfigParser 13 | import json 14 | from flask import request, Response, redirect,abort,make_response,jsonify 15 | from flask import Flask 16 | from eventlet import wsgi, monkey_patch, listen 17 | import os 18 | import sys 19 | 20 | 21 | #GlOBALS 22 | MODULE = "cars" 23 | CONFIG_FILE = "local_cfg.ini" 24 | SCHEMA = "schema.sql" 25 | PORT = "8001" \ 26 | "" 27 | DB_HEALTH = 0 28 | REGION = "eu-west-2" 29 | PROFILE = "default" 30 | 31 | try: 32 | DB_HOST = os.environ['HOST'] 33 | DB_NAME = os.environ['DB'] 34 | DB_USER = os.environ['DB_USER'] 35 | DB_PASS = os.environ['DB_PASS'] 36 | except KeyError as e: 37 | logging.error(f'Environment variable {e.args[0]} not set') 38 | DB_CONFIG = None 39 | else: 40 | DB_CONFIG = {"host":DB_HOST,"database":DB_NAME,"user":DB_USER,"password":DB_PASS} 41 | logging.info('using environment variables') 42 | 43 | 44 | # logging config 45 | logging.basicConfig(format='%(asctime)s %(levelname)-8s %(message)s', level=logging.INFO, datefmt='%Y-%m-%d %H:%M:%S') 46 | logger = logging.getLogger(__name__) 47 | 48 | #initialise flask 49 | app = Flask(__name__) 50 | app.secret_key = os.urandom(24) 51 | xray_recorder.configure( 52 | sampling=False, 53 | context_missing='LOG_ERROR', 54 | service='pycar', 55 | context=Context() 56 | ) 57 | XRayMiddleware(app, xray_recorder) 58 | 59 | def get_config(filename='local_cfg.ini', section='postgresql'): 60 | parser = ConfigParser() 61 | parser.read(filename) 62 | config = {} 63 | if parser.has_section(section): 64 | params = parser.items(section) 65 | for param in params: 66 | config[param[0]] = param[1] 67 | else: 68 | raise Exception(f'Section {section} not found in the {filename} file') 69 | return config 70 | 71 | #create DB connection 72 | if DB_CONFIG == None: 73 | logging.info('using local config file') 74 | DB_CONFIG = get_config() 75 | logging.info(f"connecting to database:{DB_CONFIG['database']} host:{DB_CONFIG['host']}") 76 | CONN = db.connect_to_db(DB_CONFIG) 77 | if CONN is not None: 78 | db.get_db_version(CONN) 79 | else: 80 | logging.error(f"cannot connect to {DB_CONFIG['host']}") 81 | sys.exit(1) 82 | 83 | #initialise cloudwatch client 84 | CWM = aws.createClient(REGION,"cloudwatch") 85 | RDS = aws.createClient(REGION,"rds") 86 | RDS_ARN = aws.describe_rds_instance(RDS,DB_CONFIG['database']) 87 | if RDS_ARN is not None: 88 | logging.info(f"cloudwatch logging configured for {RDS_ARN}") 89 | CW_LOGGING = True 90 | else: 91 | logging.info(f"cloudwatch logging NOT configured for RDS instance:{DB_CONFIG['database']}") 92 | CW_LOGGING = False 93 | 94 | 95 | @app.route("/api//health", methods=["GET"]) 96 | @xray_recorder.capture('cars-health') 97 | def health(version): 98 | if DB_HEALTH <= 3: 99 | return Response(json.dumps({"DB_errors":DB_HEALTH,"newkey":"bad"}, sort_keys=True), status=200, mimetype='application/json') 100 | else: 101 | return Response(json.dumps({"DB_errors":DB_HEALTH,"newkey":"good"}, sort_keys=True), status=500, mimetype='application/json') 102 | 103 | 104 | @app.route("/api//car/", methods=["POST"]) 105 | @xray_recorder.capture('post-car') 106 | def add_new_car(version): 107 | global DB_HEALTH 108 | body = request.json 109 | if body is not None: 110 | try: 111 | reg = body['registration'] 112 | make = body['make'] 113 | model = body['make'] 114 | colour = body['colour'] 115 | capacity = body['capacity'] 116 | except KeyError as e: 117 | logging.error(f'new car error: {str(e)}: {type(e)}') 118 | results = {"error": f"Incorrect Payload missing key {str(e)}"} 119 | status = 400 120 | else: 121 | car_details =[reg,make,model,colour,capacity] 122 | try: 123 | db.add_new_car(CONN,car_details) 124 | except Exception as e: 125 | if 'already exists.' in str(e): 126 | logging.error(f'duplicate registration details {car_details[0]}') 127 | status = 400 128 | results = {"error": f'duplicate registration details {car_details[0]}'} 129 | else: 130 | logging.error(f'error with car insert {str(e)}') 131 | DB_HEALTH += 1 132 | status = 500 133 | results = {"error": 'something went wrong'} 134 | if CW_LOGGING: 135 | metric = "SLI-DB-Failed-Requests" 136 | uri = "POST /car" 137 | aws.put_cloudwatch_metric(CWM,metric,'pysimple',RDS_ARN,1,uri,'pysimple') 138 | else: 139 | logging.info(f"added new entry {car_details[0]}") 140 | status = 200 141 | results ={"success": status} 142 | if CW_LOGGING: 143 | metric = "SLI-DB-Success-Requests" 144 | uri = "POST /car" 145 | aws.put_cloudwatch_metric(CWM,metric,'pysimple',RDS_ARN,1,uri,'pysimple') 146 | else: 147 | status = 400 148 | logging.error('missing payload') 149 | results = {"error": "empty payload"} 150 | return Response(json.dumps(results, sort_keys=True), status=status, mimetype='application/json') 151 | 152 | 153 | @app.route("/api//car/", methods=["PUT"]) 154 | @xray_recorder.capture('put-car') 155 | def update_car(version,car_id): 156 | global DB_HEALTH 157 | reg = car_id 158 | body = request.json 159 | if body is not None: 160 | try: 161 | make = body['make'] 162 | model = body['make'] 163 | colour = body['colour'] 164 | capacity = body['capacity'] 165 | except KeyError as e: 166 | logging.error(f'new car error: {str(e)}: {type(e)}') 167 | results = {"error": f"Incorrect Payload missing key {str(e)}"} 168 | status = 400 169 | else: 170 | car_details = [reg, make, model, colour, capacity] 171 | try: 172 | rows = db.update_car(CONN,car_details) 173 | except Exception as e: 174 | logging.error(f'update error {str(e)}') 175 | DB_HEALTH += 1 176 | status = 500 177 | results = {"error": 'something went wrong'} 178 | if CW_LOGGING: 179 | metric = "SLI-DB-Failed-Requests" 180 | uri = "PUT /car" 181 | aws.put_cloudwatch_metric(CWM, metric, 'pysimple', RDS_ARN, 1, uri, 'pysimple') 182 | else: 183 | if rows == 0: 184 | logging.error(f"error: updated {rows} for entry {reg}") 185 | status = 400 186 | results = {"error": f"car {reg} not found"} 187 | else: 188 | logging.info(f"updated {rows} for entry {reg}") 189 | status = 200 190 | results = {"success": status} 191 | if CW_LOGGING: 192 | metric = "SLI-DB-Success-Requests" 193 | uri = "PUT /car" 194 | aws.put_cloudwatch_metric(CWM, metric, 'pysimple', RDS_ARN, 1, uri, 'pysimple') 195 | else: 196 | status = 400 197 | logging.error('missing payload') 198 | results = {"error": "empty payload"} 199 | return Response(json.dumps(results, sort_keys=True), status=status, mimetype='application/json') 200 | 201 | 202 | @app.route("/api//car/", methods=["GET"]) 203 | @xray_recorder.capture('get-car') 204 | def get_car(version,car_id): 205 | global DB_HEALTH 206 | try: 207 | results = db.get_car(CONN,car_id) 208 | except Exception as e: 209 | logging.error(f'failed to get registration details {car_id} error:{str(e)}') 210 | results = {} 211 | DB_HEALTH += 1 212 | status = 500 213 | if CW_LOGGING: 214 | metric = "SLI-DB-Failed-Requests" 215 | uri = "GET /car" 216 | aws.put_cloudwatch_metric(CWM, metric, 'pysimple', RDS_ARN, 1, uri, 'pysimple') 217 | else: 218 | logging.info(f"got entry for {car_id}") 219 | status = 200 220 | if CW_LOGGING: 221 | metric = "SLI-DB-Success-Requests" 222 | uri = "GET /car" 223 | aws.put_cloudwatch_metric(CWM, metric, 'pysimple', RDS_ARN, 1, uri, 'pysimple') 224 | return Response(json.dumps(results, sort_keys=True), status=status, mimetype='application/json') 225 | 226 | 227 | @app.route("/api//cars/", methods=["GET"]) 228 | @xray_recorder.capture('get-cars') 229 | def get_cars(version): 230 | global DB_HEALTH 231 | try: 232 | results = db.get_cars(CONN) 233 | except Exception as e: 234 | logging.error(f'failed to get registration details error:{str(e)}') 235 | results = {} 236 | DB_HEALTH += 1 237 | status = 500 238 | else: 239 | logging.info(f"got car list") 240 | status = 200 241 | return Response(json.dumps(results, sort_keys=True), status=status, mimetype='application/json') 242 | 243 | 244 | @app.route("/api//cars/availble", methods=["GET"]) 245 | @xray_recorder.capture('get-availble-cars') 246 | def get_availble_cars(version): 247 | global DB_HEALTH 248 | try: 249 | results = db.get_available_cars(CONN) 250 | except Exception as e: 251 | logging.error(f'failed to get list of available cars error:{str(e)}') 252 | results = {} 253 | DB_HEALTH += 1 254 | status = 500 255 | else: 256 | logging.info(f"got available car list") 257 | status = 200 258 | return Response(json.dumps(results, sort_keys=True), status=status, mimetype='application/json') 259 | 260 | 261 | def update_car_availbility(conn,car_id,status): 262 | global DB_HEALTH 263 | try: 264 | result = db.update_car_availbility(conn,car_id,status) 265 | except Exception as e: 266 | DB_HEALTH += 1 267 | logging.error(f'failed to update car availbility {car_id} error:{str(e)}') 268 | return {} 269 | else: 270 | logging.info(f"updated car availbility {car_id}:{status}") 271 | return result 272 | 273 | 274 | def main(): 275 | logging.info(f"running {MODULE}") 276 | wsgi.server(listen(('', int(PORT))), app) 277 | 278 | if __name__ == "__main__": 279 | main() -------------------------------------------------------------------------------- /packt-sre-code/packt-sre/py-cars/cars/schema.sql: -------------------------------------------------------------------------------- 1 | DROP FUNCTION IF EXISTS reset_availbility CASCADE; 2 | DROP TRIGGER IF EXISTS reset_availbility ON car_availability CASCADE; 3 | DROP TABLE IF EXISTS car_availability; 4 | DROP TABLE IF EXISTS bookings; 5 | DROP TABLE IF EXISTS cars; 6 | 7 | CREATE TABLE cars ( 8 | number_plate TEXT PRIMARY KEY, 9 | make TEXT NOT NULL, 10 | model TEXT NOT NULL, 11 | colour TEXT NOT NULL, 12 | capacity TEXT NOT NULL 13 | ); 14 | 15 | CREATE TABLE bookings ( 16 | booking_id SERIAL PRIMARY KEY, 17 | user_name TEXT NOT NULL, 18 | car_id TEXT NOT NULL, 19 | created TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, 20 | pickup TIMESTAMP NOT NULL, 21 | drop_off TIMESTAMP NOT NULL, 22 | booking_title TEXT NOT NULL, 23 | description TEXT NOT NULL, 24 | approved BOOLEAN NOT NULL DEFAULT false, 25 | FOREIGN KEY (car_id) REFERENCES cars (number_plate) 26 | ); 27 | 28 | CREATE TABLE car_availability ( 29 | car_id TEXT NOT NULL, 30 | available BOOLEAN NOT NULL, 31 | expires TIMESTAMP DEFAULT NULL, 32 | PRIMARY KEY(car_id), 33 | FOREIGN KEY (car_id) REFERENCES cars (number_plate) 34 | ); 35 | 36 | CREATE FUNCTION reset_availbility() RETURNS trigger 37 | LANGUAGE plpgsql 38 | AS $$ 39 | DECLARE 40 | row_count int; 41 | BEGIN 42 | UPDATE car_availability SET available = 'true', expires = NULL WHERE expires < NOW() - INTERVAL '1 days'; 43 | IF found THEN 44 | GET DIAGNOSTICS row_count = ROW_COUNT; 45 | RAISE NOTICE 'Updated % row(s) FROM car_availability', row_count; 46 | END IF; 47 | RETURN NULL; 48 | END; 49 | $$; 50 | 51 | CREATE TRIGGER reset_availbility 52 | AFTER INSERT ON car_availability 53 | EXECUTE PROCEDURE reset_availbility(); -------------------------------------------------------------------------------- /packt-sre-code/packt-sre/py-cars/k8/std/cars-deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | creationTimestamp: null 5 | labels: 6 | app: cars2-deployment 7 | name: cars2-deployment 8 | namespace: pycars 9 | spec: 10 | replicas: 2 11 | selector: 12 | matchLabels: 13 | app: pycars 14 | strategy: {} 15 | template: 16 | metadata: 17 | creationTimestamp: null 18 | labels: 19 | app: pycars 20 | spec: 21 | containers: 22 | - image: 915793320862.dkr.ecr.eu-west-2.amazonaws.com/py-cars 23 | name: pycars 24 | ports: 25 | - containerPort: 8001 26 | env: 27 | - name: "HOST" 28 | value: "cars.cu6dyxkvr5xc.eu-west-2.rds.amazonaws.com" 29 | - name: "DB" 30 | value: "cars" 31 | - name: "DB_USER" 32 | value: "carsa" 33 | - name: "DB_PASS" 34 | value: "LetmeinAWS!!" 35 | - name: "ECS_RUN" 36 | value: "true" 37 | -------------------------------------------------------------------------------- /packt-sre-code/packt-sre/py-cars/k8/std/cars-service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: pycars-service 5 | namespace: pycars 6 | spec: 7 | selector: 8 | app: pycars 9 | ports: 10 | - protocol: TCP 11 | port: 8001 12 | targetPort: 8001 -------------------------------------------------------------------------------- /packt-sre-code/packt-sre/py-cars/k8/std/iam-policy.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement": [ 4 | { 5 | "Sid": "VisualEditor0", 6 | "Effect": "Allow", 7 | "Action": [ 8 | "logs:CreateLogStream", 9 | "cloudwatch:PutMetricData", 10 | "rds:DescribeDBInstances", 11 | "logs:CreateLogGroup", 12 | "logs:PutLogEvents", 13 | "rds:DescribeDBClusters" 14 | ], 15 | "Resource": "*" 16 | } 17 | ] 18 | } -------------------------------------------------------------------------------- /packt-sre-code/packt-sre/py-cars/k8commands.txt: -------------------------------------------------------------------------------- 1 | aws ecr get-login-password --region eu-west-2 | docker login --username AWS --password-stdin xx.dkr.ecr.eu-west-2.amazonaws.com 2 | 3 | helm upgrade -i appmesh-controller eks/appmesh-controller --namespace appmesh-system 4 | 5 | 6 | helm upgrade -i appmesh-inject eks/appmesh-inject --namespace appmesh-system --set mesh.create=true --set mesh.name=global 7 | 8 | eksctl utils associate-iam-oidc-provider --region=eu-west-2 --cluster meshtest --approve 9 | 10 | eksctl create iamserviceaccount --cluster meshtest --namespace appmesh-system --name appmesh-controller --attach-policy-arn arn:aws:iam::aws:policy/AWSCloudMapFullAccess,arn:aws:iam::aws:policy/AWSAppMeshFullAccess --override-existing-serviceaccounts --approve 11 | 12 | helm upgrade -i appmesh-controller eks/appmesh-controller --namespace appmesh-system --set region=eu-west-2 --set serviceAccount.create=false --set serviceAccount.name=appmesh-controller 13 | 14 | kubectl get deployment appmesh-controller -n appmesh-system -o json | jq -r ".spec.template.spec.containers[].image" | cut -f2 -d ':' 15 | 16 | kubectl describe mesh pyglobal 17 | aws appmesh describe-mesh --mesh-name pyglobal -------------------------------------------------------------------------------- /packt-sre-code/packt-sre/py-cars/requirements.txt: -------------------------------------------------------------------------------- 1 | atomicwrites==1.4.0 2 | attrs==19.3.0 3 | aws-xray-sdk==2.6.0 4 | boto3==1.13.17 5 | botocore==1.16.17 6 | certifi==2020.4.5.1 7 | chardet==3.0.4 8 | click==7.1.2 9 | colorama==0.4.3 10 | dnspython==1.16.0 11 | docutils==0.15.2 12 | eventlet==0.25.2 13 | Flask==1.1.2 14 | future==0.18.2 15 | greenlet==0.4.15 16 | idna==2.9 17 | importlib-metadata==1.6.0 18 | itsdangerous==1.1.0 19 | Jinja2==2.11.2 20 | jmespath==0.10.0 21 | jsonpickle==1.4.1 22 | MarkupSafe==1.1.1 23 | monotonic==1.5 24 | more-itertools==8.3.0 25 | packaging==20.4 26 | pluggy==0.13.1 27 | psycopg2-binary==2.8.5 28 | py==1.8.1 29 | pyparsing==2.4.7 30 | pytest==5.4.2 31 | pytest-dependency==0.5.1 32 | python-dateutil==2.8.1 33 | requests==2.23.0 34 | s3transfer==0.3.3 35 | six==1.14.0 36 | urllib3==1.25.9 37 | wcwidth==0.1.9 38 | Werkzeug==1.0.1 39 | wrapt==1.12.1 40 | zipp==3.1.0 41 | -------------------------------------------------------------------------------- /packt-sre-code/packt-sre/py-cars/skaffold.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: skaffold/v2beta5 2 | kind: Config 3 | metadata: 4 | name: py-cars 5 | build: 6 | artifacts: 7 | - image: xx.dkr.ecr.eu-west-2.amazonaws.com/py-cars 8 | sync: 9 | infer: 10 | - '**/*.py' 11 | - '**/*.yaml' 12 | - 'src/local.cfg.ini' 13 | - 'src/schema.sql' 14 | 15 | deploy: 16 | kubectl: 17 | manifests: 18 | - k8/std/cars-deployment.yaml 19 | - k8/std/cars-service.yaml 20 | -------------------------------------------------------------------------------- /packt-sre-code/packt-sre/py-cars/tests/api-tests.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | import pytest 4 | import pytest_dependency 5 | import requests 6 | import logging 7 | import json 8 | import random 9 | import string 10 | 11 | 12 | # logging config 13 | logging.basicConfig(format='%(asctime)s %(levelname)-8s %(message)s', level=logging.INFO, datefmt='%Y-%m-%d %H:%M:%S') 14 | logger = logging.getLogger(__name__) 15 | 16 | HOST="http://tf-ecs-simplepy-alb-140830557.eu-west-2.elb.amazonaws.com" 17 | CARS_MS = "http://127.0.0.1:8001" 18 | VER="v0.1" 19 | URI = f"{CARS_MS}/api/{VER}" 20 | HEADERS = {'Content-Type': 'application/json'} 21 | 22 | 23 | def randomString(stringLength=8): 24 | letters = string.ascii_lowercase 25 | return ''.join(random.choice(letters) for i in range(stringLength)) 26 | 27 | CAR_REG = f"car-{randomString(4)}" 28 | logging.info(f"testing with {CAR_REG}") 29 | 30 | def get_data(url,method,headers,in_data): 31 | try: 32 | if method == "get": 33 | response = requests.get(url,headers=headers) 34 | elif method == "post": 35 | response = requests.post(url,headers=headers,data=json.dumps(in_data)) 36 | elif method == "patch": 37 | response = requests.patch(url,headers=headers,data=json.dumps(in_data)) 38 | elif method == "put": 39 | response = requests.put(url,headers=headers,data=json.dumps(in_data)) 40 | else: 41 | response = None 42 | logging.error(f"no supported method:{method}") 43 | except Exception as e: 44 | logging.error(f"error for {url} error:{str(e)}") 45 | response = None 46 | else: 47 | logging.info(f"response code{response.status_code} json:{str(response.json)}") 48 | return response 49 | 50 | @pytest.mark.dependency() 51 | def test_post_car_data_returns_a_200_response_code_and_json_payload(): 52 | url = f"{URI}/car" 53 | method = "post" 54 | data = {"registration": CAR_REG, "make": "ford", "colour": "pink", "capacity": "4"} 55 | response = get_data(url, method,HEADERS,data) 56 | assert response.status_code == 200, "response code was not 200" 57 | assert response.headers['Content-Type'] == "application/json", "response Content-Type was not json" 58 | assert response.json is not None, "response data is JSON" 59 | 60 | @pytest.mark.dependency(depends=['test_post_car_data_returns_a_200_response_code_and_json_payload']) 61 | def test_get_cars_data_returns_a_200_response_code_and_json_payload(): 62 | url = f"{URI}/cars" 63 | method = "get" 64 | response = get_data(url, method, HEADERS,{}) 65 | jdata = response.json() 66 | assert response.status_code == 200, "response code was not 200" 67 | assert response.headers['Content-Type'] == "application/json", "response Content-Type was not json" 68 | assert jdata is not None, "response data is JSON" 69 | assert CAR_REG in jdata['result'], "test registration is not in list" 70 | 71 | @pytest.mark.dependency(depends=['test_post_car_data_returns_a_200_response_code_and_json_payload']) 72 | def test_get_car_data_returns_a_200_response_code_and_json_payload(): 73 | url = f"{URI}/car/{CAR_REG}" 74 | method = "get" 75 | response = get_data(url, method,HEADERS,{}) 76 | assert response.status_code == 200, "response code was not 200" 77 | assert response.headers['Content-Type'] == "application/json", "response Content-Type was not json" 78 | assert response.json is not None, "response data is JSON" 79 | 80 | @pytest.mark.dependency() 81 | @pytest.mark.dependency(depends=['test_post_car_data_returns_a_200_response_code_and_json_payload']) 82 | def test_put_car_data_returns_a_200_response_code_and_json_payload(): 83 | url = f"{URI}/car/{CAR_REG}" 84 | method = "put" 85 | data = {"make":"ford","colour":"aqua-maroon","capacity":"4"} 86 | response = get_data(url, method,HEADERS,data) 87 | assert response.status_code == 200, "response code was not 200" 88 | assert response.headers['Content-Type'] == "application/json", "response Content-Type was not json" 89 | assert response.json is not None, "response data is JSON" 90 | 91 | @pytest.mark.dependency(depends=['test_put_car_data_returns_a_200_response_code_and_json_payload']) 92 | def test_get_car_data_returns_a_200_response_code_and_json_payload_and_has_updated_colour(): 93 | url = f"{URI}/car/{CAR_REG}" 94 | method = "get" 95 | response = get_data(url, method,HEADERS,{}) 96 | jdata = response.json() 97 | assert response.status_code == 200, "response code was not 200" 98 | assert response.headers['Content-Type'] == "application/json", "response Content-Type was not json" 99 | assert response.json is not None, "response data is JSON" 100 | assert jdata[CAR_REG]['colour'] == 'aqua-maroon', "colour data has not been updated" 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | -------------------------------------------------------------------------------- /packt-sre-code/packt-sre/pysimple/.dockerignore: -------------------------------------------------------------------------------- 1 | *.xlsx 2 | *.png 3 | venv 4 | .venv 5 | .git 6 | docs 7 | tests 8 | .idea -------------------------------------------------------------------------------- /packt-sre-code/packt-sre/pysimple/.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | .idea/ 131 | -------------------------------------------------------------------------------- /packt-sre-code/packt-sre/pysimple/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.8.0-slim-buster 2 | LABEL team="development" \ 3 | maintainer="Ian" \ 4 | email="ian@widgets.com" 5 | MAINTAINER "ian@widgets.com" 6 | RUN apt-get update \ 7 | && apt-get install gcc python3-dev musl-dev -y \ 8 | && apt-get clean 9 | WORKDIR /app 10 | COPY ./requirements.txt /app/requirements.txt 11 | RUN pip3 install --user -r requirements.txt 12 | COPY carLister/ /app 13 | EXPOSE 8080 14 | ENTRYPOINT [ "python3" ] 15 | CMD [ "main.py" ] -------------------------------------------------------------------------------- /packt-sre-code/packt-sre/pysimple/README.md: -------------------------------------------------------------------------------- 1 | # aws-simple-py 2 | simple repo for packt course 3 | -------------------------------------------------------------------------------- /packt-sre-code/packt-sre/pysimple/buildspec.yml: -------------------------------------------------------------------------------- 1 | version: 0.2 2 | 3 | phases: 4 | pre_build: 5 | commands: 6 | - echo Logging in to Amazon ECR... 7 | - $(aws ecr get-login --no-include-email --region $AWS_DEFAULT_REGION) 8 | build: 9 | commands: 10 | - echo Build started on `date` 11 | - echo Building the Docker image... 12 | - docker build -t $IMAGE_REPO_NAME:$IMAGE_TAG . 13 | - docker tag $IMAGE_REPO_NAME:$IMAGE_TAG $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$IMAGE_REPO_NAME:$IMAGE_TAG 14 | post_build: 15 | commands: 16 | - echo Build completed on `date` 17 | - echo Pushing the Docker image... 18 | - docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$IMAGE_REPO_NAME:$IMAGE_TAG -------------------------------------------------------------------------------- /packt-sre-code/packt-sre/pysimple/carLister/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PacktPublishing/Site-Reliability-Engineering-on-AWS/f22f86ecba8e662f8b5e189b202346cbebef05cd/packt-sre-code/packt-sre/pysimple/carLister/__init__.py -------------------------------------------------------------------------------- /packt-sre-code/packt-sre/pysimple/carLister/awsHelper.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | 4 | import logging 5 | import boto3 6 | import json 7 | from os import environ 8 | import requests 9 | URL = "http://169.254.169.254/latest/meta-data/public-ipv4" 10 | IN_AWS = False 11 | 12 | try: 13 | response = requests.get(URL,timeout=1) 14 | except Exception as e: 15 | logging.error(f"cannot connect to url:{URL} assuming not in AWS") 16 | else: 17 | print(response.txt) 18 | logging.info(f"got {response.text} from meta data server") 19 | IN_AWS = True 20 | 21 | 22 | #GlOBALS 23 | MODULE = "AWS-helper" 24 | PROFILE = "default" 25 | REGION = "eu-west-2" 26 | 27 | # logging config 28 | logging.basicConfig(format='%(asctime)s %(levelname)-8s %(message)s', level=logging.INFO, datefmt='%Y-%m-%d %H:%M:%S') 29 | logger = logging.getLogger(__name__) 30 | 31 | def createClient(region,service): 32 | if environ.get('CODEBUILD_BUILD_ID') is not None: 33 | return boto3.client(service, region_name=region) 34 | elif IN_AWS: 35 | return boto3.client(service, region_name=region) 36 | elif environ.get('ECS_RUN') is not None: 37 | return boto3.client(service, region_name=region) 38 | else: 39 | logging.info(f'using AWS profile {PROFILE} for service:{service}') 40 | session = boto3.Session(profile_name=PROFILE) 41 | return session.client(service, region_name=region) 42 | 43 | 44 | def put_cloudwatch_metric(client,mname,a_name,c_name,m_value,uri,namespace): 45 | dimensions = [{'Name':'service','Value':a_name},{'Name':'component','Value':c_name},{'Name':'uri','Value':uri}] 46 | entry = {'MetricName':mname,'Dimensions':dimensions,'Unit':'None','Value':m_value} 47 | try: 48 | response = client.put_metric_data(MetricData=[entry],Namespace=namespace) 49 | except Exception as e: 50 | logging.error(f"put metric data error:{str(e)}") 51 | return 52 | else: 53 | logging.info(f"response:{response}") 54 | return response 55 | 56 | def describe_rds_instance(client,instance_name): 57 | try: 58 | response = client.describe_db_instances(DBInstanceIdentifier=instance_name) 59 | except Exception as e: 60 | logging.error(f"error:{str(e)}") 61 | return 62 | else: 63 | logging.info(f"response:{response}") 64 | return response['DBInstances'][0]['DBInstanceArn'] 65 | 66 | 67 | 68 | 69 | 70 | -------------------------------------------------------------------------------- /packt-sre-code/packt-sre/pysimple/carLister/dbHelper.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | 4 | import logging 5 | import psycopg2 6 | 7 | 8 | #GlOBALS 9 | MODULE = "postgres-helper" 10 | 11 | # logging config 12 | logging.basicConfig(format='%(asctime)s %(levelname)-8s %(message)s', level=logging.INFO, datefmt='%Y-%m-%d %H:%M:%S') 13 | logger = logging.getLogger(__name__) 14 | 15 | 16 | def connect_to_db(dbconfig): 17 | try: 18 | conn = psycopg2.connect(**dbconfig) 19 | except (Exception, psycopg2.DatabaseError) as e: 20 | logging.error(f"connection error to {dbconfig['host']}:{dbconfig['database']} error {str(e)}") 21 | return 22 | else: 23 | logging.info(f"connected to {dbconfig['host']}:{dbconfig['database']}") 24 | return conn 25 | 26 | def get_db_version(conn): 27 | cur =conn.cursor() 28 | try: 29 | cur.execute('SELECT version()') 30 | except (Exception, psycopg2.DatabaseError) as e: 31 | logging.error(f"execution error {str(e)}") 32 | return 33 | else: 34 | db_version = cur.fetchone() 35 | logging.info(f"db versions {db_version}") 36 | cur.close() 37 | return db_version 38 | 39 | def init_db(conn,schema_file): 40 | conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT) 41 | cur = conn.cursor() 42 | try: 43 | cur.execute(open(schema_file, "r").read()) 44 | except Exception as e: 45 | logging.error(f"initdb execution error {str(e)}") 46 | return 47 | else: 48 | logging.info(f"tables created") 49 | cur.close() 50 | return 51 | 52 | def update_car_availbility(conn,reg_id,status): 53 | updated_rows = 0 54 | sql = f"UPDATE car_availability set available = {status} WHERE car_id = '{reg_id}'" 55 | cur = conn.cursor() 56 | try: 57 | cur.execute(sql) 58 | except Exception as e: 59 | conn.rollback() 60 | logging.error(f" update availability execution error {str(e)}") 61 | raise Exception(f'update car availability error with {str(e)}') 62 | return 63 | else: 64 | updated_rows = cur.rowcount 65 | conn.commit() 66 | finally: 67 | cur.close() 68 | return updated_rows 69 | 70 | def update_car(conn,car_det): 71 | updated_rows = 0 72 | sql = f"UPDATE cars set make = '{car_det[1]}' , model = '{car_det[2]}' , colour = '{car_det[3]}', capacity ='{car_det[4]}' WHERE number_plate = '{car_det[0]}'" 73 | cur = conn.cursor() 74 | try: 75 | cur.execute(sql) 76 | except Exception as e: 77 | conn.rollback() 78 | logging.error(f" update execution error {str(e)}") 79 | raise Exception(f'update car error with {str(e)}') 80 | return 81 | else: 82 | updated_rows = cur.rowcount 83 | conn.commit() 84 | finally: 85 | cur.close() 86 | return updated_rows 87 | 88 | def add_new_car(conn, car_det): 89 | sql = f"INSERT INTO cars VALUES('{car_det[0]}','{car_det[1]}','{car_det[2]}','{car_det[3]}','{car_det[4]}');" 90 | logging.debug(f"running sql {sql}") 91 | cur = conn.cursor() 92 | try: 93 | cur.execute(sql) 94 | except Exception as e: 95 | conn.rollback() 96 | logging.debug(f"db:add car execution error 1 {str(e)}") 97 | raise Exception(f'add car error with {str(e)}') 98 | else: 99 | sql = f"INSERT INTO car_availability (car_id, available) VALUES('{car_det[0]}','true');" 100 | logging.info(f"running sql {sql}") 101 | try: 102 | cur.execute(sql) 103 | except Exception as e: 104 | conn.rollback() 105 | logging.debug(f"add car availability execution error 1 {str(e)}") 106 | raise Exception(f'add car availabilty error with {str(e)}') 107 | else: 108 | logging.info(f"updated car_availability") 109 | logging.info(f"last notice {str(conn.notices)}") 110 | conn.commit() 111 | cur.close() 112 | 113 | def get_car(conn, reg_id): 114 | results = {} 115 | sql = f"SELECT * FROM cars WHERE number_plate = '{reg_id}';" 116 | cur = conn.cursor() 117 | try: 118 | cur.execute(sql) 119 | except Exception as e: 120 | logging.debug(f"get car execution error {str(e)}") 121 | raise Exception(f'get car error with {str(e)}') 122 | return 123 | else: 124 | car_records = cur.fetchall() 125 | for row in car_records: 126 | results[row[0]] = {"make":row[1],"model":row[2],"colour":row[3],"capacity":row[4]} 127 | finally: 128 | cur.close() 129 | return results 130 | 131 | 132 | def get_available_cars(conn): 133 | results = {} 134 | free_list = [] 135 | sql = f"SELECT car_id FROM car_availability WHERE available = TRUE ;" 136 | cur = conn.cursor() 137 | try: 138 | cur.execute(sql) 139 | except Exception as e: 140 | logging.debug(f"get car availability execution error {str(e)}") 141 | raise Exception(f'get car availability error with {str(e)}') 142 | return 143 | else: 144 | car_records = cur.fetchall() 145 | for row in car_records: 146 | free_list.append(row[0]) 147 | finally: 148 | results['result'] = free_list 149 | cur.close() 150 | return results 151 | 152 | def get_cars(conn): 153 | results = {} 154 | sql = f"SELECT number_plate FROM cars;" 155 | cur = conn.cursor() 156 | plate_list = [] 157 | try: 158 | cur.execute(sql) 159 | except Exception as e: 160 | logging.error(f"execution error {str(e)}") 161 | raise Exception(f'error with {str(e)}') 162 | return 163 | else: 164 | car_records = cur.fetchall() 165 | for row in car_records: 166 | plate_list.append(row[0]) 167 | finally: 168 | results['result'] = plate_list 169 | cur.close() 170 | return results 171 | 172 | def add_booking(conn, booking_det): 173 | sql = f"INSERT INTO bookings (user_name,car_id,pickup,drop_off,booking_title,description) VALUES('{booking_det[0]}','{booking_det[1]}','{booking_det[2]}','{booking_det[3]}','{booking_det[4]}','{booking_det[5]}') RETURNING booking_id;" 174 | logging.info(f"running sql {sql}") 175 | cur = conn.cursor() 176 | id = None 177 | try: 178 | cur.execute(sql) 179 | except Exception as e: 180 | conn.rollback() 181 | logging.error(f"add booking execution error 1 {str(e)}") 182 | raise Exception(f'add booking error with {str(e)}') 183 | else: 184 | id = cur.fetchone()[0] 185 | conn.commit() 186 | finally: 187 | cur.close() 188 | return id 189 | 190 | def get_bookings(conn): 191 | results = {} 192 | sql = f"SELECT booking_id,booking_title,user_name,approved,created,car_id FROM bookings ORDER by booking_id;" 193 | cur = conn.cursor() 194 | booking_list = [] 195 | try: 196 | cur.execute(sql) 197 | except Exception as e: 198 | logging.error(f"execution error for booking list {str(e)}") 199 | raise Exception(f'error with booking list: {str(e)}') 200 | return 201 | else: 202 | booking_records = cur.fetchall() 203 | for row in booking_records: 204 | booking_list.append({"id":row[0],"title":row[1],"raised_by":row[2],"approved":row[3],"created":str(row[4]),"car-registration":str(row[5])}) 205 | finally: 206 | results['result'] = booking_list 207 | cur.close() 208 | return results 209 | 210 | def get_booking(conn,current_booking_id): 211 | results = {} 212 | sql = f"SELECT booking_id,booking_title,user_name,approved,created,car_id FROM bookings WHERE booking_id = {current_booking_id};" 213 | cur = conn.cursor() 214 | try: 215 | cur.execute(sql) 216 | except Exception as e: 217 | logging.error(f"execution error for booking list {str(e)}") 218 | raise Exception(f'error with booking list: {str(e)}') 219 | return 220 | else: 221 | booking_records = cur.fetchall() 222 | for row in booking_records: 223 | results[row[0]] = {"title":row[1],"raised_by":row[2],"approved":row[3],"created":str(row[4]),"car-registration":str(row[5])} 224 | finally: 225 | cur.close() 226 | return results 227 | 228 | def approve_booking(conn,working_booking_id): 229 | sql = f"UPDATE bookings SET approved = 'true' WHERE booking_id = {working_booking_id} RETURNING car_id,drop_off;" 230 | logging.debug(f"running sql {sql}") 231 | cur = conn.cursor() 232 | try: 233 | cur.execute(sql) 234 | except Exception as e: 235 | conn.rollback() 236 | logging.debug(f"failed to approved booking {working_booking_id} {str(e)}") 237 | raise Exception(f'approval booking error with {str(e)}') 238 | else: 239 | row = id = cur.fetchone() 240 | sql = f"UPDATE car_availability SET available = 'false', expires = '{row[1]}' WHERE car_id = '{row[0]}';" 241 | logging.info(f"running sql {sql}") 242 | try: 243 | cur.execute(sql) 244 | except Exception as e: 245 | conn.rollback() 246 | logging.debug(f"update car availability execution error 1 {str(e)}") 247 | raise Exception(f'update car availability error with {str(e)}') 248 | conn.commit() 249 | cur.close() 250 | 251 | 252 | 253 | 254 | -------------------------------------------------------------------------------- /packt-sre-code/packt-sre/pysimple/carLister/local_cfg.ini: -------------------------------------------------------------------------------- 1 | [postgresql] 2 | host=192.168.182.128 3 | database=cars 4 | user=carsa 5 | password=letmein12 -------------------------------------------------------------------------------- /packt-sre-code/packt-sre/pysimple/carLister/main.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | 4 | import logging 5 | import dbHelper as db 6 | import awsHelper as aws 7 | from configparser import ConfigParser 8 | import json 9 | from flask import request, Response, redirect,abort,make_response,jsonify 10 | from flask import Flask 11 | from eventlet import wsgi, monkey_patch, listen 12 | import os 13 | import sys 14 | 15 | 16 | #GlOBALS 17 | MODULE = "car-lister" 18 | CONFIG_FILE = "local_cfg.ini" 19 | SCHEMA = "schema.sql" 20 | PORT = "8080" 21 | DB_HEALTH = 0 22 | REGION = "eu-west-2" 23 | 24 | try: 25 | DB_HOST = os.environ['HOST'] 26 | DB_NAME = os.environ['DB'] 27 | DB_USER = os.environ['DB_USER'] 28 | DB_PASS = os.environ['DB_PASS'] 29 | except KeyError as e: 30 | logging.error(f'Environment variable {e.args[0]} not set') 31 | DB_CONFIG = None 32 | else: 33 | DB_CONFIG = {"host":DB_HOST,"database":DB_NAME,"user":DB_USER,"password":DB_PASS} 34 | logging.info('using environment variables') 35 | 36 | 37 | # logging config 38 | logging.basicConfig(format='%(asctime)s %(levelname)-8s %(message)s', level=logging.INFO, datefmt='%Y-%m-%d %H:%M:%S') 39 | logger = logging.getLogger(__name__) 40 | 41 | #initialise flask 42 | app = Flask(__name__) 43 | app.secret_key = os.urandom(24) 44 | 45 | def get_config(filename='local_cfg.ini', section='postgresql'): 46 | parser = ConfigParser() 47 | parser.read(filename) 48 | config = {} 49 | if parser.has_section(section): 50 | params = parser.items(section) 51 | for param in params: 52 | config[param[0]] = param[1] 53 | else: 54 | raise Exception(f'Section {section} not found in the {filename} file') 55 | return config 56 | 57 | #create DB connection 58 | if DB_CONFIG == None: 59 | logging.info('using local config file') 60 | DB_CONFIG = get_config() 61 | logging.info(f"connecting to database:{DB_CONFIG['database']} host:{DB_CONFIG['host']}") 62 | CONN = db.connect_to_db(DB_CONFIG) 63 | if CONN is not None: 64 | db.get_db_version(CONN) 65 | else: 66 | logging.error(f"cannot connect to {DB_CONFIG['host']}") 67 | sys.exit(1) 68 | 69 | #initialise cloudwatch client 70 | CWM = aws.createClient(REGION,"cloudwatch") 71 | RDS = aws.createClient(REGION,"rds") 72 | RDS_ARN = aws.describe_rds_instance(RDS,DB_CONFIG['database']) 73 | if RDS_ARN is not None: 74 | logging.info(f"cloudwatch logging configured for {RDS_ARN}") 75 | CW_LOGGING = True 76 | else: 77 | logging.info(f"cloudwatch logging NOT configured for RDS instance:{DB_CONFIG['database']}") 78 | CW_LOGGING = False 79 | 80 | @app.route("/api//health", methods=["GET"]) 81 | def health(version): 82 | if DB_HEALTH <= 3: 83 | return Response(json.dumps({"DB_errors":DB_HEALTH}, sort_keys=True), status=200, mimetype='application/json') 84 | else: 85 | return Response(json.dumps({"DB_errors":DB_HEALTH}, sort_keys=True), status=500, mimetype='application/json') 86 | 87 | 88 | 89 | @app.route("/api//car/", methods=["POST"]) 90 | def add_new_car(version): 91 | global DB_HEALTH 92 | body = request.json 93 | if body is not None: 94 | try: 95 | reg = body['registration'] 96 | make = body['make'] 97 | model = body['make'] 98 | colour = body['colour'] 99 | capacity = body['capacity'] 100 | except KeyError as e: 101 | logging.error(f'new car error: {str(e)}: {type(e)}') 102 | results = {"error": f"Incorrect Payload missing key {str(e)}"} 103 | status = 400 104 | else: 105 | car_details =[reg,make,model,colour,capacity] 106 | try: 107 | db.add_new_car(CONN,car_details) 108 | except Exception as e: 109 | if 'already exists.' in str(e): 110 | logging.error(f'duplicate registration details {car_details[0]}') 111 | status = 400 112 | results = {"error": f'duplicate registration details {car_details[0]}'} 113 | else: 114 | logging.error(f'error with car insert {str(e)}') 115 | DB_HEALTH += 1 116 | status = 500 117 | results = {"error": 'something went wrong'} 118 | if CW_LOGGING: 119 | metric = "SLI-DB-Failed-Requests" 120 | uri = "POST /car" 121 | aws.put_cloudwatch_metric(CWM,metric,'pysimple',RDS_ARN,1,uri,'pysimple') 122 | else: 123 | logging.info(f"added new entry {car_details[0]}") 124 | status = 200 125 | results ={"success": status} 126 | if CW_LOGGING: 127 | metric = "SLI-DB-Success-Requests" 128 | uri = "POST /car" 129 | aws.put_cloudwatch_metric(CWM,metric,'pysimple',RDS_ARN,1,uri,'pysimple') 130 | else: 131 | status = 400 132 | logging.error('missing payload') 133 | results = {"error": "empty payload"} 134 | return Response(json.dumps(results, sort_keys=True), status=status, mimetype='application/json') 135 | 136 | @app.route("/api//car/", methods=["PUT"]) 137 | def update_car(version,car_id): 138 | global DB_HEALTH 139 | reg = car_id 140 | body = request.json 141 | if body is not None: 142 | try: 143 | make = body['make'] 144 | model = body['make'] 145 | colour = body['colour'] 146 | capacity = body['capacity'] 147 | except KeyError as e: 148 | logging.error(f'new car error: {str(e)}: {type(e)}') 149 | results = {"error": f"Incorrect Payload missing key {str(e)}"} 150 | status = 400 151 | else: 152 | car_details = [reg, make, model, colour, capacity] 153 | try: 154 | rows = db.update_car(CONN,car_details) 155 | except Exception as e: 156 | logging.error(f'update error {str(e)}') 157 | DB_HEALTH += 1 158 | status = 500 159 | results = {"error": 'something went wrong'} 160 | if CW_LOGGING: 161 | metric = "SLI-DB-Failed-Requests" 162 | uri = "PUT /car" 163 | aws.put_cloudwatch_metric(CWM, metric, 'pysimple', RDS_ARN, 1, uri, 'pysimple') 164 | else: 165 | if rows == 0: 166 | logging.error(f"error: updated {rows} for entry {reg}") 167 | status = 400 168 | results = {"error": f"car {reg} not found"} 169 | else: 170 | logging.info(f"updated {rows} for entry {reg}") 171 | status = 200 172 | results = {"success": status} 173 | if CW_LOGGING: 174 | metric = "SLI-DB-Success-Requests" 175 | uri = "PUT /car" 176 | aws.put_cloudwatch_metric(CWM, metric, 'pysimple', RDS_ARN, 1, uri, 'pysimple') 177 | else: 178 | status = 400 179 | logging.error('missing payload') 180 | results = {"error": "empty payload"} 181 | return Response(json.dumps(results, sort_keys=True), status=status, mimetype='application/json') 182 | 183 | 184 | @app.route("/api//car/", methods=["GET"]) 185 | def get_car(version,car_id): 186 | global DB_HEALTH 187 | try: 188 | results = db.get_car(CONN,car_id) 189 | except Exception as e: 190 | logging.error(f'failed to get registration details {car_id} error:{str(e)}') 191 | results = {} 192 | DB_HEALTH += 1 193 | status = 500 194 | if CW_LOGGING: 195 | metric = "SLI-DB-Failed-Requests" 196 | uri = "GET /car" 197 | aws.put_cloudwatch_metric(CWM, metric, 'pysimple', RDS_ARN, 1, uri, 'pysimple') 198 | else: 199 | logging.info(f"got entry for {car_id}") 200 | status = 200 201 | if CW_LOGGING: 202 | metric = "SLI-DB-Success-Requests" 203 | uri = "GET /car" 204 | aws.put_cloudwatch_metric(CWM, metric, 'pysimple', RDS_ARN, 1, uri, 'pysimple') 205 | return Response(json.dumps(results, sort_keys=True), status=status, mimetype='application/json') 206 | 207 | @app.route("/api//cars/", methods=["GET"]) 208 | def get_cars(version): 209 | global DB_HEALTH 210 | try: 211 | results = db.get_cars(CONN) 212 | except Exception as e: 213 | logging.error(f'failed to get registration details error:{str(e)}') 214 | results = {} 215 | DB_HEALTH += 1 216 | status = 500 217 | else: 218 | logging.info(f"got car list") 219 | status = 200 220 | return Response(json.dumps(results, sort_keys=True), status=status, mimetype='application/json') 221 | 222 | @app.route("/api//cars/availble", methods=["GET"]) 223 | def get_availble_cars(version): 224 | global DB_HEALTH 225 | try: 226 | results = db.get_available_cars(CONN) 227 | except Exception as e: 228 | logging.error(f'failed to get list of available cars error:{str(e)}') 229 | results = {} 230 | DB_HEALTH += 1 231 | status = 500 232 | else: 233 | logging.info(f"got available car list") 234 | status = 200 235 | return Response(json.dumps(results, sort_keys=True), status=status, mimetype='application/json') 236 | 237 | def update_car_availbility(conn,car_id,status): 238 | global DB_HEALTH 239 | try: 240 | result = db.update_car_availbility(conn,car_id,status) 241 | except Exception as e: 242 | DB_HEALTH += 1 243 | logging.error(f'failed to update car availbility {car_id} error:{str(e)}') 244 | return {} 245 | else: 246 | logging.info(f"updated car availbility {car_id}:{status}") 247 | return result 248 | 249 | @app.route("/api//booking/", methods=["POST"]) 250 | def add_booking(version): 251 | global DB_HEALTH 252 | body = request.json 253 | print(body) 254 | if body is not None: 255 | try: 256 | user_name = body['username'] 257 | car_id = body['car_registration'] 258 | pickup = body['pickup'] 259 | drop_off = body['drop-off'] 260 | booking_title = body['title'] 261 | booking_description = body['description'] 262 | except KeyError as e: 263 | logging.error(f'new booking error: {str(e)}: {type(e)}') 264 | results = {"error": f"Incorrect Payload missing key {str(e)}"} 265 | status = 400 266 | else: 267 | booking_details = [user_name, car_id, pickup, drop_off, booking_title, booking_description] 268 | try: 269 | booking_id = db.add_booking(CONN,booking_details) 270 | except Exception as e: 271 | logging.error(f'update error {str(e)}') 272 | DB_HEALTH += 1 273 | status = 500 274 | results = {"error": 'something went wrong'} 275 | else: 276 | logging.info(f"added new booking entry {booking_id}:{booking_description}") 277 | results = {"success" : booking_id} 278 | status = 200 279 | else: 280 | status = 400 281 | logging.error('missing payload') 282 | results = {"error": "empty payload"} 283 | return Response(json.dumps(results, sort_keys=True), status=status, mimetype='application/json') 284 | 285 | 286 | @app.route("/api//bookings/", methods=["GET"]) 287 | def get_bookings(version): 288 | global DB_HEALTH 289 | try: 290 | results = db.get_bookings(CONN) 291 | except Exception as e: 292 | logging.error(f'failed to get list of bookings error:{str(e)}') 293 | DB_HEALTH += 1 294 | status = 500 295 | results = {} 296 | else: 297 | logging.info(f"got bookings list") 298 | status = 200 299 | return Response(json.dumps(results, sort_keys=True), status=status, mimetype='application/json') 300 | 301 | @app.route("/api//booking/", methods=["GET"]) 302 | def get_booking(version,booking_id): 303 | global DB_HEALTH 304 | try: 305 | results = db.get_booking(CONN,booking_id) 306 | except Exception as e: 307 | logging.error(f'failed to get bookings error:{str(e)}') 308 | DB_HEALTH += 1 309 | status = 500 310 | results = {"error":"something went wrong"} 311 | else: 312 | logging.info(f"got bookings {booking_id}") 313 | status = 200 314 | return Response(json.dumps(results, sort_keys=True), status=status, mimetype='application/json') 315 | 316 | 317 | @app.route("/api//admin/booking/approve/", methods=["PATCH"]) 318 | def approve_booking(version,booking_id): 319 | global DB_HEALTH 320 | try: 321 | results = db.approve_booking(CONN,booking_id) 322 | except Exception as e: 323 | logging.error(f'failed to approve bookings error:{str(e)}') 324 | status = 500 325 | DB_HEALTH += 1 326 | results = {"error": "something went wrong"} 327 | else: 328 | logging.info(f"approved bookings {booking_id}") 329 | results = {"success": 200} 330 | status = 200 331 | return Response(json.dumps(results, sort_keys=True), status=status, mimetype='application/json') 332 | 333 | def main(): 334 | logging.info(f"running {MODULE}") 335 | wsgi.server(listen(('', int(PORT))), app) 336 | 337 | if __name__ == "__main__": 338 | main() -------------------------------------------------------------------------------- /packt-sre-code/packt-sre/pysimple/carLister/schema.sql: -------------------------------------------------------------------------------- 1 | DROP FUNCTION IF EXISTS reset_availbility CASCADE; 2 | DROP TRIGGER IF EXISTS reset_availbility ON car_availability CASCADE; 3 | DROP TABLE IF EXISTS car_availability; 4 | DROP TABLE IF EXISTS bookings; 5 | DROP TABLE IF EXISTS cars; 6 | 7 | CREATE TABLE cars ( 8 | number_plate TEXT PRIMARY KEY, 9 | make TEXT NOT NULL, 10 | model TEXT NOT NULL, 11 | colour TEXT NOT NULL, 12 | capacity TEXT NOT NULL 13 | ); 14 | 15 | CREATE TABLE bookings ( 16 | booking_id SERIAL PRIMARY KEY, 17 | user_name TEXT NOT NULL, 18 | car_id TEXT NOT NULL, 19 | created TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, 20 | pickup TIMESTAMP NOT NULL, 21 | drop_off TIMESTAMP NOT NULL, 22 | booking_title TEXT NOT NULL, 23 | description TEXT NOT NULL, 24 | approved BOOLEAN NOT NULL DEFAULT false, 25 | FOREIGN KEY (car_id) REFERENCES cars (number_plate) 26 | ); 27 | 28 | CREATE TABLE car_availability ( 29 | car_id TEXT NOT NULL, 30 | available BOOLEAN NOT NULL, 31 | expires TIMESTAMP DEFAULT NULL, 32 | PRIMARY KEY(car_id), 33 | FOREIGN KEY (car_id) REFERENCES cars (number_plate) 34 | ); 35 | 36 | CREATE FUNCTION reset_availbility() RETURNS trigger 37 | LANGUAGE plpgsql 38 | AS $$ 39 | DECLARE 40 | row_count int; 41 | BEGIN 42 | UPDATE car_availability SET available = 'true', expires = NULL WHERE expires < NOW() - INTERVAL '1 days'; 43 | IF found THEN 44 | GET DIAGNOSTICS row_count = ROW_COUNT; 45 | RAISE NOTICE 'Updated % row(s) FROM car_availability', row_count; 46 | END IF; 47 | RETURN NULL; 48 | END; 49 | $$; 50 | 51 | CREATE TRIGGER reset_availbility 52 | AFTER INSERT ON car_availability 53 | EXECUTE PROCEDURE reset_availbility(); -------------------------------------------------------------------------------- /packt-sre-code/packt-sre/pysimple/requirements.txt: -------------------------------------------------------------------------------- 1 | atomicwrites==1.4.0 2 | attrs==19.3.0 3 | boto3==1.13.17 4 | botocore==1.16.17 5 | certifi==2020.4.5.1 6 | chardet==3.0.4 7 | click==7.1.2 8 | colorama==0.4.3 9 | dnspython==1.16.0 10 | docutils==0.15.2 11 | eventlet==0.25.2 12 | Flask==1.1.2 13 | greenlet==0.4.15 14 | idna==2.9 15 | importlib-metadata==1.6.0 16 | itsdangerous==1.1.0 17 | Jinja2==2.11.2 18 | jmespath==0.10.0 19 | MarkupSafe==1.1.1 20 | monotonic==1.5 21 | more-itertools==8.3.0 22 | packaging==20.4 23 | pluggy==0.13.1 24 | psycopg2-binary==2.8.5 25 | py==1.8.1 26 | pyparsing==2.4.7 27 | pytest==5.4.2 28 | python-dateutil==2.8.1 29 | requests==2.23.0 30 | s3transfer==0.3.3 31 | six==1.14.0 32 | urllib3==1.25.9 33 | wcwidth==0.1.9 34 | Werkzeug==1.0.1 35 | zipp==3.1.0 36 | -------------------------------------------------------------------------------- /packt-sre-code/packt-sre/pysimple/tests/api-tests.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | import pytest 4 | import requests 5 | import logging 6 | 7 | # logging config 8 | logging.basicConfig(format='%(asctime)s %(levelname)-8s %(message)s', level=logging.INFO, datefmt='%Y-%m-%d %H:%M:%S') 9 | logger = logging.getLogger(__name__) 10 | 11 | HOST="http://tf-ecs-simplepy-alb-140830557.eu-west-2.elb.amazonaws.com" 12 | VER="v0.1" 13 | URI = f"{HOST}/api/{VER}" 14 | 15 | def get_data(url,method): 16 | try: 17 | if method == "get": 18 | response = requests.get(url) 19 | elif method == "post": 20 | response = requests.post(url) 21 | elif method == "patch": 22 | response = requests.patch(url) 23 | else: 24 | response = None 25 | logging.error(f"no supported method:{method}") 26 | except Exception as e: 27 | logging.error(f"error for {url} error:{str(e)}") 28 | response = None 29 | return response 30 | 31 | def test_get_cars_data_returns_a_200_response_code_and_json_payload(get_cars_data): 32 | assert get_cars_data.status_code == 200, "response code was not 200" 33 | assert get_cars_data.headers['Content-Type'] == "application/json", "response Content-Type was not json" 34 | assert get_cars_data.json is not None, "response data is JSON" 35 | 36 | @pytest.fixture 37 | def get_cars_data(): 38 | url = f"{URI}/cars" 39 | method = "get" 40 | return get_data(url,method) 41 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /packt-sre-code/packt-sre/rds-cdc/lambda-py.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | import boto3 3 | import base64 4 | from json import loads 5 | 6 | dynamodb_client = boto3.client('dynamodb') 7 | 8 | # The block below creates the DDB table with the specified column names. 9 | table_name = "data_change" 10 | row_id = "id" 11 | row_timestamp = "time" 12 | try: 13 | response = dynamodb_client.create_table( 14 | AttributeDefinitions=[ 15 | { 16 | 'AttributeName': row_id, 17 | 'AttributeType': 'S', 18 | }, 19 | { 20 | 'AttributeName': row_timestamp, 21 | 'AttributeType': 'S', 22 | } 23 | ], 24 | KeySchema=[ 25 | { 26 | 'AttributeName': row_id, 27 | 'KeyType': 'HASH', 28 | }, 29 | { 30 | 'AttributeName': row_timestamp, 31 | 'KeyType': 'RANGE', 32 | }, 33 | ], 34 | ProvisionedThroughput={ 35 | 'ReadCapacityUnits': 1, 36 | 'WriteCapacityUnits': 1, 37 | }, 38 | TableName=table_name 39 | ) 40 | except dynamodb_client.exceptions.ResourceInUseException: 41 | # Table is created, skip 42 | pass 43 | 44 | 45 | def lambda_handler(event, context): 46 | payload = event['Records'] 47 | output = [] 48 | success = 0 49 | failure = 0 50 | for record in payload: 51 | 52 | try: 53 | payload = base64.b64decode(record['kinesis']['data']) 54 | except Exception as e: 55 | failure += 1 56 | output.append({'recordId': record['eventID'], 'result': 'DeliveryFailed'}) 57 | print(str(e)) 58 | else: 59 | data_item = loads(payload) 60 | print("data:{}".format(data_item)) 61 | if data_item['data'][0] == "table": 62 | ddb_item = { row_id: { 'S': data_item[row_id] }, 63 | "resource" : { 'S': str(data_item['data'][0]) }, 64 | "resource_name" : { 'S': str(data_item['data'][1]) }, 65 | "action" : { 'S': str(data_item['data'][2]) }, 66 | "detail" : { 'S': ", ".join(data_item['data'][3:-1]) }, 67 | row_timestamp: { 'S': data_item[row_timestamp] } 68 | } 69 | else: 70 | ddb_item = None 71 | if ddb_item is not None: 72 | try: 73 | dynamodb_client.put_item(TableName=table_name, Item=ddb_item) 74 | except Exception as e: 75 | print("error storing message {}".format(str(e))) 76 | failure += 1 77 | else: 78 | success += 1 79 | output.append({'recordId': record['eventID'], 'result': 'Ok'}) 80 | else: 81 | print("cognito message not stored") 82 | 83 | print('Successfully delivered {0} records, failed to deliver {1} records'.format(success, failure)) 84 | return {'records': output} -------------------------------------------------------------------------------- /packt-sre-code/packt-sre/rds-cdc/local_cfg.ini: -------------------------------------------------------------------------------- 1 | [changedata] 2 | stream_name=repl_stream 3 | region=eu-west-2 4 | 5 | [postgresql] 6 | host=192.168.182.128 7 | database=cars 8 | user=carsa 9 | password=letmein12 -------------------------------------------------------------------------------- /packt-sre-code/packt-sre/rds-cdc/main.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | 4 | import boto3 5 | import json 6 | from configparser import ConfigParser 7 | from os import environ 8 | import random 9 | import calendar 10 | import time 11 | from datetime import datetime 12 | import psycopg2 13 | from psycopg2.extras import LogicalReplicationConnection 14 | import logging 15 | import uuid 16 | 17 | 18 | #globals 19 | MODULE = "RDS-REPL" 20 | PROFILE = "default" 21 | REGION = "eu-west-2" 22 | FILENAME = "local_cfg.ini" 23 | IN_AWS = False 24 | 25 | 26 | # logging config 27 | logging.basicConfig(format='%(asctime)s %(levelname)-8s %(message)s', level=logging.INFO, datefmt='%Y-%m-%d %H:%M:%S') 28 | logger = logging.getLogger(__name__) 29 | 30 | def get_config(filename='local_cfg.ini', section='changedata'): 31 | parser = ConfigParser() 32 | parser.read(filename) 33 | config = {} 34 | if parser.has_section(section): 35 | params = parser.items(section) 36 | for param in params: 37 | config[param[0]] = param[1] 38 | else: 39 | raise Exception(f'Section {section} not found in the {filename} file') 40 | return config 41 | 42 | logging.info('using local config file') 43 | MAIN_CONFIG = get_config(FILENAME,'changedata') 44 | DB_CONFIG = get_config(FILENAME,'postgresql') 45 | 46 | def createClient(region,service): 47 | if environ.get('CODEBUILD_BUILD_ID') is not None: 48 | logging.info('running in CodeBuild') 49 | return boto3.client(service, region_name=region) 50 | else: 51 | if IN_AWS: 52 | logging.info('running in AWS') 53 | return boto3.client(service, region_name=region) 54 | elif environ.get('ECS_RUN') is not None: 55 | logging.info('running in ECS') 56 | return boto3.client(service, region_name=region) 57 | else: 58 | logging.info(f'using AWS profile {PROFILE} for service:{service}') 59 | session = boto3.Session(profile_name=PROFILE) 60 | return session.client(service, region_name=region) 61 | 62 | KINESIS_CLIENT = createClient(MAIN_CONFIG['region'],'kinesis') 63 | 64 | def connect_to_db(dbconfig): 65 | try: 66 | conn = psycopg2.connect(**dbconfig,connection_factory=LogicalReplicationConnection) 67 | except (Exception, psycopg2.DatabaseError) as e: 68 | logging.error(f"connection error to {dbconfig['host']}:{dbconfig['database']} error {str(e)}") 69 | return 70 | else: 71 | logging.info(f"connected to {dbconfig['host']}:{dbconfig['database']}") 72 | return conn 73 | 74 | def consume(msg): 75 | msg_list = msg.payload.split(" ") 76 | nw_msg = {"id": str(uuid.uuid1()),"time":str(datetime.utcnow()),"data":msg_list} 77 | try: 78 | result =KINESIS_CLIENT.put_record(StreamName=MAIN_CONFIG['stream_name'], Data=json.dumps(nw_msg), PartitionKey="default") 79 | except Exception as e: 80 | logging.error(f"failed to send msg error:{str(e)}") 81 | else: 82 | logging.info(json.dumps(nw_msg),result) 83 | 84 | 85 | def main(): 86 | logging.info(f"running {MODULE}") 87 | CONN = connect_to_db(DB_CONFIG) 88 | cur = CONN.cursor() 89 | cur.drop_replication_slot('cdc_test_slot') 90 | cur.create_replication_slot('cdc_test_slot', output_plugin ='test_decoding') 91 | cur.start_replication(slot_name = 'cdc_test_slot', decode= True) 92 | cur.consume_stream(consume) 93 | 94 | 95 | if __name__ == "__main__": 96 | main() 97 | 98 | --------------------------------------------------------------------------------