├── .gitignore ├── LICENSE ├── README.rst ├── rdspg ├── __init__.py ├── main.py ├── rds.py ├── terraform_cluster.jinja └── terraform_instance.jinja ├── setup.py └── tox.ini /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | 49 | # Translations 50 | *.mo 51 | *.pot 52 | 53 | # Django stuff: 54 | *.log 55 | local_settings.py 56 | 57 | # Flask stuff: 58 | instance/ 59 | .webassets-cache 60 | 61 | # Scrapy stuff: 62 | .scrapy 63 | 64 | # Sphinx documentation 65 | docs/_build/ 66 | 67 | # PyBuilder 68 | target/ 69 | 70 | # Jupyter Notebook 71 | .ipynb_checkpoints 72 | 73 | # pyenv 74 | .python-version 75 | 76 | # celery beat schedule file 77 | celerybeat-schedule 78 | 79 | # SageMath parsed files 80 | *.sage.py 81 | 82 | # dotenv 83 | .env 84 | 85 | # virtualenv 86 | .venv 87 | venv/ 88 | ENV/ 89 | 90 | # Spyder project settings 91 | .spyderproject 92 | .spyproject 93 | 94 | # Rope project settings 95 | .ropeproject 96 | 97 | # mkdocs documentation 98 | /site 99 | 100 | # mypy 101 | .mypy_cache/ 102 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Xiuming Chen 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | rdspg 2 | ====== 3 | 4 | Command-line toolkit to help understand information about your AWS RDS Parameter Groups. 5 | 6 | Installation 7 | ------------ 8 | 9 | .. code:: bash 10 | 11 | pip install rdspg 12 | 13 | Purpose 14 | ------- 15 | 16 | When it comes to analyzing parameter groups for RDS, AWS suggested in a `blog post `_ that it could only be done using `diff`: 17 | 18 | There is no AWS CLI command to compare two parameter groups simultaneously; this feature is only available by using the RDS console. 19 | You can then compare the plain text files that list the parameter groups using a Linux tool such as the diff command, or a source code editor like Notepad++. 20 | 21 | I think we can do better. This tool is to help us make that task a lot easier. Also adding a few other features to help analyzing changes. 22 | 23 | Permission Config 24 | ----------------- 25 | This tool needs certain IAM permissions in order to work. Example policy: 26 | 27 | :: 28 | 29 | { 30 | "Version":"2012-10-17", 31 | "Statement":[ 32 | { 33 | "Effect":"Allow", 34 | "Action":[ 35 | "rds:DescribeDBInstances", 36 | "rds:DescribeDBParameters", 37 | "rds:DescribeDBParameterGroups", 38 | "rds:DescribeDBClusters", 39 | "rds:DescribeDBClusterParameterGroups", 40 | "rds:DescribeDBClusterParameters", 41 | "rds:ListTagsForResource" 42 | ], 43 | "Resource":"*" 44 | } 45 | ] 46 | } 47 | 48 | Usage 49 | ----- 50 | * Listing Parameter Groups: 51 | 52 | :: 53 | 54 | $ rdspg list 55 | DBParameterGroupName DBParameterGroupFamily Description 56 | ------------------------------- ------------------------ ---------------------------------------------------------- 57 | default.aurora-postgresql9.6 aurora-postgresql9.6 Default parameter group for aurora-postgresql9.6 58 | default.aurora5.6 aurora5.6 Default parameter group for aurora5.6 59 | default.postgres9.3 postgres9.3 Default parameter group for postgres9.3 60 | default.postgres9.4 postgres9.4 Default parameter group for postgres9.4 61 | default.postgres9.5 postgres9.5 Default parameter group for postgres9.5 62 | default.postgres9.6 postgres9.6 Default parameter group for postgres9.6 63 | my-parameter-group postgres9.6 My Parameter Group 64 | 65 | * Getting parameters in parameter group, filtering out default values: 66 | 67 | :: 68 | 69 | $ rdspg get my-replica 70 | ParameterName ParameterValue ApplyMethod ApplyType 71 | ------------------------------- ---------------- ------------- ----------- 72 | autovacuum_analyze_scale_factor 0.1 immediate dynamic 73 | checkpoint_segments 512 immediate dynamic 74 | checkpoint_timeout 300 immediate dynamic 75 | checkpoint_warning 60 immediate dynamic 76 | default_statistics_target 100 immediate dynamic 77 | hot_standby_feedback 1 immediate dynamic 78 | log_autovacuum_min_duration 0 immediate dynamic 79 | log_connections 1 immediate dynamic 80 | log_disconnections 1 immediate dynamic 81 | 82 | * Getting a mapping of parameter group -> instances: 83 | 84 | :: 85 | 86 | $ rdspg mapping 87 | ParameterGroup DBInstances 88 | ------------------- --------------------------------- 89 | default.postgres9.4 90 | default.postgres9.5 db-replica-9-5-a,db-replica-9-5-b 91 | default.postgres9.6 db-replica-9-6-a,db-replica-9-6-b 92 | 93 | * Compare differences between two parameter groups: 94 | 95 | :: 96 | 97 | $ rdspg diff my-replica-a my-replica-b 98 | ParameterName my-replica-a my-replica-b 99 | ------------------- -------------- --------------------- 100 | checkpoint_timeout 300 450 101 | checkpoint_warning 60 102 | checkpoint_segments 512 32 103 | 104 | * Export parameter group in terraform template format: 105 | 106 | :: 107 | 108 | $ rdspg terraform my-parameter-group 109 | resource "aws_db_parameter_group" "my-parameter-group" { 110 | name = "my-parameter-group" 111 | family = "postgres9.5" 112 | description = "My awesome parameter group" 113 | 114 | parameter { 115 | name = "autovacuum_analyze_scale_factor" 116 | value = "0.01" 117 | apply_method = "immediate" 118 | } 119 | 120 | parameter { 121 | name = "autovacuum_vacuum_scale_factor" 122 | value = "0.01" 123 | apply_method = "immediate" 124 | } 125 | 126 | } 127 | 128 | * All the commands work for db clusters with ``--cluster`` flag 129 | 130 | :: 131 | 132 | $ rdspg list --cluster 133 | DBClusterParameterGroupName DBParameterGroupFamily Description 134 | ----------------------------- ------------------------ -------------------------------------------------------- 135 | customers-p-cluster aurora-postgresql9.6 Managed by Terraform 136 | default.aurora-postgresql1 aurora-postgresql1 Default cluster parameter group for aurora-postgresql1 137 | default.aurora-postgresql9.6 aurora-postgresql9.6 Default cluster parameter group for aurora-postgresql9.6 138 | default.aurora5.6 aurora5.6 Default cluster parameter group for aurora5.6 139 | -------------------------------------------------------------------------------- /rdspg/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cxmcc/rdspg/ef30925551170b7fbc467dbacd233bb5be1e75c1/rdspg/__init__.py -------------------------------------------------------------------------------- /rdspg/main.py: -------------------------------------------------------------------------------- 1 | import click 2 | import rds 3 | import tabulate 4 | import os 5 | import jinja2 6 | 7 | 8 | def params_to_kv(params): 9 | out = {} 10 | for param in params: 11 | key = param['ParameterName'] 12 | value = param.get('ParameterValue') 13 | out[key] = value 14 | return out 15 | 16 | 17 | def params_list_to_dict(params, detail=False): 18 | headers = ['ParameterName', 'ParameterValue', 'ApplyMethod', 'ApplyType'] 19 | if detail: 20 | headers += ['AllowedValues', 'DataType', 'Source'] 21 | out = [] 22 | for param in params: 23 | p = list(param.get(h) for h in headers) 24 | out.append(p) 25 | return out, headers 26 | 27 | 28 | def calculate_diff(params_a, params_b): 29 | params_a = params_to_kv(params_a) 30 | params_b = params_to_kv(params_b) 31 | out = [] 32 | for k, v in params_a.iteritems(): 33 | if v != params_b.get(k): 34 | out.append((k, v, params_b.get(k, ''))) 35 | if k in params_b: 36 | del params_b[k] 37 | for k, v in params_b.iteritems(): 38 | out.append((k, None, v)) 39 | return out 40 | 41 | 42 | def only_important_columns_pg(pgs): 43 | for pg in pgs: 44 | for k in ('DBParameterGroupArn', 'DBClusterParameterGroupArn'): 45 | if k in pg: 46 | del pg[k] 47 | return pgs 48 | 49 | 50 | def only_user_params(params): 51 | out = [] 52 | for param in params: 53 | if param['Source'] not in ('system', 'engine-default'): 54 | out.append(param) 55 | return out 56 | 57 | 58 | def only_important_columns(params): 59 | columns = ('Description', 'DataType', 'IsModifiable', 60 | 'AllowedValues', 'Source') 61 | for param in params: 62 | for k in columns: 63 | if k in param: 64 | del param[k] 65 | return params 66 | 67 | 68 | def terraform(parameter_group_name, info, params, tags, cluster=False): 69 | if cluster: 70 | template_file = 'terraform_cluster.jinja' 71 | else: 72 | template_file = 'terraform_instance.jinja' 73 | 74 | def render(context): 75 | curr_dir = os.path.dirname(__file__) 76 | template_file_path = os.path.join(curr_dir, template_file) 77 | path, filename = os.path.split(template_file_path) 78 | return jinja2.Environment( 79 | loader=jinja2.FileSystemLoader(path or './') 80 | ).get_template(filename).render(context) 81 | 82 | context = {'parameter_group_name': parameter_group_name, 83 | 'params': params, 'info': info, 'tags': tags} 84 | return render(context) 85 | 86 | 87 | @click.group() 88 | def cli(): 89 | pass 90 | 91 | 92 | @cli.command(name='mapping') 93 | @click.option('--cluster', is_flag=True, default=False) 94 | @click.option('--no-header', is_flag=True, default=False) 95 | def cmd_mapping(cluster, no_header): 96 | api = rds.get_api(cluster=cluster) 97 | mapping = api.generate_pg_to_db_mapping() 98 | if no_header: 99 | kwargs = {'tablefmt': 'plain'} 100 | else: 101 | headers = ('ParameterGroup', 'DBInstances') 102 | kwargs = {'tablefmt': 'simple', 'headers': headers} 103 | output = tabulate.tabulate(mapping, numalign='right', **kwargs) 104 | click.echo(output) 105 | 106 | 107 | @cli.command(name='list') 108 | @click.option('--cluster', is_flag=True, default=False) 109 | @click.option('--detail', is_flag=True, default=False) 110 | @click.option('--no-header', is_flag=True, default=False) 111 | def cmd_list(cluster, detail, no_header): 112 | api = rds.get_api(cluster=cluster) 113 | pgs = api.get_parameter_groups() 114 | if not detail: 115 | pgs = only_important_columns_pg(pgs) 116 | if no_header: 117 | kwargs = {'tablefmt': 'plain'} 118 | else: 119 | kwargs = {'tablefmt': 'simple', 'headers': 'keys'} 120 | output = tabulate.tabulate(pgs, numalign='right', **kwargs) 121 | click.echo(output) 122 | 123 | 124 | @cli.command(name='get') 125 | @click.argument('parameter-group') 126 | @click.option('--cluster', is_flag=True, default=False) 127 | @click.option('--all-params', is_flag=True, default=False) 128 | @click.option('--detail', is_flag=True, default=False) 129 | @click.option('--no-header', is_flag=True, default=False) 130 | def cmd_get(cluster, parameter_group, all_params, detail, no_header): 131 | api = rds.get_api(cluster=cluster) 132 | params = api.get_parameters(parameter_group) 133 | if not all_params: 134 | params = only_user_params(params) 135 | if not detail: 136 | params = only_important_columns(params) 137 | params, headers = params_list_to_dict(params, detail=detail) 138 | if no_header: 139 | kwargs = {'tablefmt': 'plain'} 140 | else: 141 | kwargs = {'tablefmt': 'simple', 'headers': headers} 142 | output = tabulate.tabulate(params, numalign='right', **kwargs) 143 | click.echo(output) 144 | 145 | 146 | @cli.command(name='diff') 147 | @click.argument('parameter-group-a') 148 | @click.argument('parameter-group-b') 149 | @click.option('--cluster', is_flag=True, default=False) 150 | @click.option('--all-params', is_flag=True, default=False) 151 | @click.option('--no-header', is_flag=True, default=False) 152 | def cmd_diff(cluster, parameter_group_a, parameter_group_b, 153 | all_params, no_header): 154 | api = rds.get_api(cluster=cluster) 155 | params_a = api.get_parameters(parameter_group_a) 156 | params_b = api.get_parameters(parameter_group_b) 157 | if not all_params: 158 | params_a = only_user_params(params_a) 159 | params_b = only_user_params(params_b) 160 | diff = calculate_diff(params_a, params_b) 161 | headers = ['ParameterName', parameter_group_a, parameter_group_b] 162 | if no_header: 163 | kwargs = {'tablefmt': 'plain'} 164 | else: 165 | kwargs = {'tablefmt': 'simple', 'headers': headers} 166 | output = tabulate.tabulate(diff, numalign='right', **kwargs) 167 | click.echo(output) 168 | 169 | 170 | @cli.command(name='terraform') 171 | @click.option('--cluster', is_flag=True, default=False) 172 | @click.argument('parameter-group') 173 | def cmd_terraform(cluster, parameter_group): 174 | api = rds.get_api(cluster=cluster) 175 | params = api.get_parameters(parameter_group) 176 | info = api.get_pg_info(parameter_group) 177 | if cluster: 178 | arn = info['DBClusterParameterGroupArn'] 179 | else: 180 | arn = info['DBParameterGroupArn'] 181 | tags = api.list_tags(arn) 182 | params = only_user_params(params) 183 | template = terraform(parameter_group, info, params, tags) 184 | click.echo(template) 185 | 186 | 187 | def main(): 188 | cli() 189 | 190 | 191 | if __name__ == '__main__': 192 | main() 193 | -------------------------------------------------------------------------------- /rdspg/rds.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | 3 | 4 | def get_api(cluster=False): 5 | if cluster: 6 | return RDSClusterAPI() 7 | else: 8 | return RDSInstanceAPI() 9 | 10 | 11 | class RDSAPI: 12 | def __init__(self): 13 | self.client = boto3.client('rds') 14 | 15 | def get_parameters(self, name): 16 | raise NotImplementedError 17 | 18 | def get_dbs(self): 19 | raise NotImplementedError 20 | 21 | def get_parameter_groups(self): 22 | raise NotImplementedError 23 | 24 | def get_pg_info(self): 25 | raise NotImplementedError 26 | 27 | def generate_pg_to_db_mapping(self): 28 | pgs = self.get_parameter_groups() 29 | dbs = self.get_dbs() 30 | mapping = {} 31 | for pg in pgs: 32 | pg_name = pg[self.FIELD_PARAMETER_GROUP_NAME] 33 | mapping[pg_name] = [] 34 | for db in dbs: 35 | db_name = db[self.FIELD_DB_ID] 36 | pg_list = db[self.FIELD_PARAMETER_GROUP] 37 | pg_name = pg_list[0][self.FIELD_PARAMETER_GROUP_NAME] 38 | mapping[pg_name].append(db_name) 39 | out = [] 40 | for k, v in sorted(mapping.items()): 41 | if v == []: 42 | value = '' 43 | else: 44 | value = ','.join(v) 45 | out.append((k, value)) 46 | return out 47 | 48 | def list_tags(self, arn): 49 | resp = self.client.list_tags_for_resource(ResourceName=arn) 50 | return resp.get('TagList', []) 51 | 52 | 53 | class RDSInstanceAPI(RDSAPI): 54 | FIELD_PARAMETER_GROUP_NAME = 'DBParameterGroupName' 55 | FIELD_DB_ID = 'DBInstanceIdentifier' 56 | FIELD_PARAMETER_GROUP = 'DBParameterGroups' 57 | 58 | def get_parameters(self, name): 59 | paginator = self.client.get_paginator('describe_db_parameters') 60 | out = [] 61 | for page in paginator.paginate(DBParameterGroupName=name): 62 | out += page['Parameters'] 63 | return out 64 | 65 | def get_dbs(self): 66 | paginator = self.client.get_paginator('describe_db_instances') 67 | out = [] 68 | for page in paginator.paginate(): 69 | out += page['DBInstances'] 70 | return out 71 | 72 | def get_parameter_groups(self): 73 | paginator = self.client.get_paginator('describe_db_parameter_groups') 74 | out = [] 75 | for page in paginator.paginate(): 76 | out += page['DBParameterGroups'] 77 | return out 78 | 79 | def get_pg_info(self, name): 80 | resp = self.client.describe_db_parameter_groups( 81 | DBParameterGroupName=name 82 | ) 83 | info = resp['DBParameterGroups'][0] 84 | return info 85 | 86 | 87 | class RDSClusterAPI(RDSAPI): 88 | FIELD_PARAMETER_GROUP_NAME = 'DBClusterParameterGroupName' 89 | FIELD_DB_ID = 'DBClusterIdentifier' 90 | FIELD_PARAMETER_GROUP = 'DBClusterParameterGroups' 91 | 92 | def get_parameters(self, name): 93 | # Can't be paginated 94 | resp = self.client.describe_db_cluster_parameters( 95 | DBClusterParameterGroupName=name, 96 | ) 97 | return resp['Parameters'] 98 | 99 | def get_dbs(self): 100 | # Can't be paginated 101 | resp = self.client.describe_db_clusters() 102 | return resp['DBClusters'] 103 | 104 | def get_parameter_groups(self): 105 | # Can't be paginated 106 | resp = self.client.describe_db_cluster_parameter_groups() 107 | return resp['DBClusterParameterGroups'] 108 | 109 | def get_pg_info(self, name): 110 | resp = self.client.describe_db_cluster_parameter_groups( 111 | DBClusterParameterGroupName=name 112 | ) 113 | info = resp['DBClusterParameterGroups'][0] 114 | return info 115 | -------------------------------------------------------------------------------- /rdspg/terraform_cluster.jinja: -------------------------------------------------------------------------------- 1 | resource "aws_rds_cluster_parameter_group" "{{ parameter_group_name }}" { 2 | name = "{{ parameter_group_name }}" 3 | family = "{{ info['DBParameterGroupFamily'] }}" 4 | {% if info['Description'] %} description = "{{ info['Description'] }}" 5 | {% endif %} 6 | {% if params %} 7 | {% for param in params %} 8 | parameter { 9 | name = "{{ param['ParameterName'] }}" 10 | value = "{{ param['ParameterValue'] }}" 11 | apply_method = "{{ param['ApplyMethod'] }}" 12 | } 13 | {% endfor %} 14 | {% endif %} 15 | {% if tags %} 16 | tags { 17 | {% for tag in tags %} 18 | {{ tag['Key']}} = "{{ tag['Value'] }}" 19 | {% endfor %} 20 | } 21 | {% endif %} 22 | } 23 | -------------------------------------------------------------------------------- /rdspg/terraform_instance.jinja: -------------------------------------------------------------------------------- 1 | resource "aws_db_parameter_group" "{{ parameter_group_name }}" { 2 | name = "{{ parameter_group_name }}" 3 | family = "{{ info['DBParameterGroupFamily'] }}" 4 | {% if info['Description'] %} description = "{{ info['Description'] }}" 5 | {% endif %} 6 | {% if params %} 7 | {% for param in params %} 8 | parameter { 9 | name = "{{ param['ParameterName'] }}" 10 | value = "{{ param['ParameterValue'] }}" 11 | apply_method = "{{ param['ApplyMethod'] }}" 12 | } 13 | {% endfor %} 14 | {% endif %} 15 | {% if tags %} 16 | tags { 17 | {% for tag in tags %} 18 | {{ tag['Key']}} = "{{ tag['Value'] }}" 19 | {% endfor %} 20 | } 21 | {% endif %} 22 | } 23 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from setuptools import setup 4 | 5 | install_requires = [ 6 | 'boto3>=1.4.5', 7 | 'click>=6.7', 8 | 'tabulate>=0.7.7', 9 | 'Jinja2>=2.9.6' 10 | ] 11 | 12 | classifiers = [ 13 | 'Development Status :: 4 - Beta', 14 | 'Environment :: Console', 15 | 'Topic :: System :: Clustering', 16 | ] 17 | 18 | with open('README.rst', 'r') as f: 19 | long_description = f.read() 20 | 21 | description = ('Command-line toolkit to help understand information ' 22 | 'about your AWS RDS Parameter Groups.') 23 | 24 | setup( 25 | name='rdspg', 26 | version='0.1.9', 27 | description=description, 28 | long_description=long_description, 29 | author='Xiuming Chen', 30 | author_email='cc@cxm.cc', 31 | url='https://github.com/cxmcc/rdspg', 32 | packages=['rdspg'], 33 | package_data={'rdspg': ['terraform_*.jinja']}, 34 | entry_points={'console_scripts': ['rdspg = rdspg.main:main']}, 35 | install_requires=install_requires, 36 | keywords=['AWS', 'RDS', 'Parameter Groups', 'Relational Database Service'], 37 | classifiers=classifiers, 38 | ) 39 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py27, py35, py36 3 | 4 | [flake8] 5 | show-pep8 = True 6 | 7 | [testenv] 8 | deps= 9 | flake8 10 | boto3 11 | click 12 | tabulate 13 | commands = 14 | {envpython} setup.py install 15 | flake8 16 | --------------------------------------------------------------------------------