├── .coveragerc ├── .gitignore ├── .travis.yml ├── CHANGELOG.md ├── LICENSE ├── OSSMETADATA ├── README.md ├── manifest.ini ├── setup.cfg ├── setup.py ├── swag_client ├── __about__.py ├── __init__.py ├── backend.py ├── backends │ ├── __init__.py │ ├── dynamodb.py │ ├── file.py │ └── s3.py ├── cli.py ├── compat.py ├── data │ └── aws_accounts.json ├── exceptions.py ├── migrations │ ├── __init__.py │ ├── migrations.py │ └── versions │ │ ├── __init__.py │ │ └── v2.py ├── schemas │ ├── __init__.py │ ├── services │ │ └── __init__.py │ ├── v1.py │ ├── v2.py │ └── validators.py ├── swag.py ├── tests │ ├── __init__.py │ ├── conftest.py │ ├── test_swag.py │ └── vectors │ │ ├── invalid_accounts_v1.json │ │ ├── invalid_accounts_v2.json │ │ ├── valid_accounts_v1.json │ │ └── valid_accounts_v2.json └── util.py └── tox.ini /.coveragerc: -------------------------------------------------------------------------------- 1 | [report] 2 | include = swag_client/*.py 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ### Python template 2 | # Byte-compiled / optimized / DLL files 3 | __pycache__/ 4 | *.py[cod] 5 | *$py.class 6 | 7 | # C extensions 8 | *.so 9 | 10 | # Distribution / packaging 11 | .Python 12 | env/ 13 | build/ 14 | develop-eggs/ 15 | dist/ 16 | downloads/ 17 | eggs/ 18 | .eggs/ 19 | lib/ 20 | lib64/ 21 | parts/ 22 | sdist/ 23 | var/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *,cover 47 | .pytest_cache/ 48 | 49 | # Translations 50 | *.mo 51 | *.pot 52 | 53 | # Django stuff: 54 | *.log 55 | 56 | # Sphinx documentation 57 | docs/_build/ 58 | 59 | # PyBuilder 60 | target/ 61 | ### VirtualEnv template 62 | # Virtualenv 63 | # http://iamzed.com/2009/05/07/a-primer-on-virtualenv/ 64 | .Python 65 | [Bb]in 66 | [Ii]nclude 67 | [Ll]ib 68 | [Ss]cripts 69 | pyvenv.cfg 70 | pip-selfcheck.json 71 | 72 | # Created by .ignore support plugin (hsz.mobi) 73 | 74 | # IntelliJ 75 | .idea 76 | 77 | venv/ 78 | venv 79 | 80 | accounts.json -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | 3 | matrix: 4 | include: 5 | - python: "3.6" 6 | 7 | before_install: 8 | - sudo rm -f /etc/boto.cfg 9 | 10 | cache: 11 | directories: 12 | - .pip_download_cache 13 | 14 | env: 15 | global: 16 | - PIP_DOWNLOAD_CACHE=".pip_download_cache" 17 | 18 | before_script: 19 | - pip install --upgrade pip 20 | - pip install --upgrade setuptools 21 | - pip install .[tests] 22 | - python setup.py develop 23 | 24 | script: 25 | - coverage run -m py.test || exit 1 26 | 27 | after_success: 28 | - coveralls 29 | 30 | notifications: 31 | email: 32 | - secops@netflix.com 33 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | Changelog 2 | ========= 3 | 4 | 0.3.7 - 2017-02-07 5 | ------------------ 6 | 7 | - Fixed an issue where swag would pull in a beta version of the marshmallow dependency. 8 | 9 | 0.3.0 - 2017-11-15 10 | ------------------ 11 | 12 | - Added the ability to test-run changes via the dry-run keyword. 13 | - Fixed get_service_enabled function such that both v1 and v2 formats are correctly handled. 14 | - Added v1 support to the get_service function. 15 | - Upgrade click_log and moved to a application configuration object for CLI. 16 | - Added a list_service command to CLI. 17 | - Added a deploy_service command to CLI. 18 | - Moved caching to a backend controlled option. -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /OSSMETADATA: -------------------------------------------------------------------------------- 1 | osslifecycle=active 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # swag_client 2 | 3 | ![PyPi](http://img.shields.io/pypi/v/swag-client.svg?style=flat) 4 | 5 | [![Build Status](https://travis-ci.com/Netflix-Skunkworks/swag-client.svg?branch=master)](https://travis-ci.com/Netflix-Skunkworks/swag-client) 6 | 7 | ![OSS Status](https://img.shields.io/badge/NetflixOSS-active-brightgreen.svg) 8 | 9 | 10 | SWAG is a collection of repositories used to keep track of the metadata describing cloud accounts. Originally built to store data on AWS, swag is flexible enough to be used for multiple providers (AWS, GCP, etc.,). 11 | 12 | For applications that manage or deploy to several different accounts/environments SWAG provides a centralized metadata store. 13 | 14 | Managing your cloud accounts with SWAG allows for consuming application to focus on business logic, removing hardcoded lists or properties from your application. 15 | 16 | 17 | #### Related Projects 18 | These projects are part of the SWAG family and either manipulate or utilize SWAG data. 19 | 20 | [swag-api](https://github.com/Netflix-Skunkworks/swag-api) - Rest API and UI for SWAG Data 21 | 22 | [swag-functions](https://github.com/Netflix-Skunkworks/swag-functions) - Transformation functions for SWAG Data 23 | 24 | ## Installation 25 | 26 | swag-client is available on pypi: 27 | 28 | ```bash 29 | pip install swag_client 30 | ``` 31 | 32 | 33 | ## Basic Usage 34 | If no options are passed to the SWAGManager it is assumed that you are using the `file` backend and the `account` namespace. 35 | 36 | ```python 37 | from swag_client.backend import SWAGManager 38 | from swag_client.util import parse_swag_config_options 39 | 40 | swag_opts = {} 41 | swag = SWAGManager(**parse_swag_config_options(swag_opts)) 42 | 43 | account = swag.get_by_name('account') 44 | ``` 45 | 46 | Configure SWAG by passing the client additional keyword arguments: 47 | 48 | ```python 49 | from swag_client.backend import SWAGManager 50 | from swag_client.util import parse_swag_config_options 51 | 52 | swag_opts = { 53 | 'swag.type': 's3', 54 | 'swag.bucket_name': s3_bucket_name 55 | } 56 | 57 | swag = SWAGManager(**parse_swag_config_options(swag_opts)) 58 | ``` 59 | 60 | Additionally SWAG can be used a singleton, allowing a single instance (and it's cache) to be shared across an application. 61 | 62 | Declare swag in a file like ``extensions.py``: 63 | ``` 64 | from swag_client.backend import SWAGManager 65 | from swag_client.util import parse_swag_config_options 66 | 67 | swag = SWAGManager() 68 | ``` 69 | 70 | 71 | When you're ready to configure swag: 72 | ``` 73 | from extensions import swag 74 | 75 | swag_opts = { 76 | 'swag.type': 's3', 77 | 'swag.bucket_name': s3_bucket_name 78 | } 79 | 80 | swag.configure(**parse_swag_config_options(swag_opts)) 81 | ``` 82 | 83 | 84 | ### Example Account JSON (v2) 85 | Below is an example of the bare minimum JSON that will be created by SWAG with `schema_version=2`. 86 | 87 | ``` 88 | [{ 89 | "aliases": ["test"], 90 | "contacts": ["admins@test.net"], 91 | "description": "This is just a test.", 92 | "email": "test@example.net", 93 | "environment": "test", 94 | "id": "012345678910", 95 | "name": "testaccount", 96 | "owner": "ExampleCorp", 97 | "provider": "aws", 98 | "sensitive": false 99 | }] 100 | ``` 101 | 102 | Additionally SWAG has the ability to store metadata for `services` that may need to be tied to an account. 103 | 104 | ``` 105 | [{ 106 | "id": "012345678910", 107 | ... 108 | "services": [ 109 | { 110 | "name": "myService", 111 | "metadata": { 112 | "name": "testService" 113 | }, 114 | "status": [ 115 | { 116 | "region": "all", 117 | "enabled": true 118 | } 119 | ] 120 | } 121 | ] 122 | }] 123 | ``` 124 | 125 | This service metadata for serving as a single source of truth for applications that have to routinely manage/monitor/process multiple accounts. 126 | 127 | 128 | ### Filtering 129 | 130 | Regardless of backend, SWAG uses the `jmespath` syntax for filtering data. 131 | 132 | ```python 133 | swag.get("[?id=='012345678910']") # valid jmespath filter 134 | ``` 135 | 136 | More information on jmespath filtering: http://jmespath.org/tutorial.html 137 | 138 | ### Old-style (deprecated) 139 | 140 | SWAG also supports an older style calling convention. This convention only supports the S3 backend, additionally these functions are now deprecated and will be removed in the future. 141 | 142 | ```python 143 | from swag_client.swag import get_all_accounts 144 | get_all_accounts(bucket='your-swag-bucket').get('accounts') 145 | ``` 146 | 147 | or to filter by a service tag: 148 | 149 | ```python 150 | service = {'services': {'YOURSERVICE': {'enabled': True, 'randomflag': True}}} 151 | get_all_accounts(bucket='your-swag-bucket', **service).get('accounts') 152 | ``` 153 | 154 | ## Versioning 155 | 156 | All SWAG metadata is versioned. The most current version of the `account` metadata schema is `v2`. SWAG further provides the ability to transform data between schema version as necessary. 157 | 158 | ## Backends 159 | 160 | SWAG supports multiple backends, included in the package are file, s3, and dynamodb backends. Additional backends can easily be created and integrated into SWAG. 161 | 162 | 163 | #### Global Backend Options 164 | 165 | | Key | Type | Required | Description | 166 | | --- | ---- | -------- | ----------- | 167 | | swag.type | str | false | Type of backend to use (Default: 'file') | 168 | | swag.namespace | str | false | Namespace for metadata (Default: 'accounts') | 169 | | swag.schema_version | int | false | Schema version that will be returned to the caller. (Default: 'v2') | 170 | | swag.cache_expires | int | false | Number of seconds to cache backend results (Default: 60) | 171 | 172 | ### S3 Backend 173 | 174 | The S3 backend uses AWS S3 to SWAG metadata. 175 | 176 | #### Backend Options 177 | 178 | | Key | Type | Required | Description | 179 | | --- | ---- | -------- | ----------- | 180 | | swag.bucket_name | str | true | Raw S3 bucket name | 181 | | swag.data_file | str | false | Full S3 key of file | 182 | | swag.region | str | false | Region the bucket exists in. (Default: us-east-1) 183 | 184 | 185 | #### Permissions 186 | 187 | IAM Permissions required: 188 | 189 | ``` 190 | { 191 | "Action": ["s3:GetObject"], 192 | "Effect": ["Allow"], 193 | "Resource: ["arn:aws:s3:::/"] 194 | } 195 | ``` 196 | 197 | If you wish to update SWAG metadata you will also need the following permissions: 198 | 199 | ``` 200 | { 201 | "Action": ["s3:PutObject"], 202 | "Effect": ["Allow"], 203 | "Resource: ["arn:aws:s3:::/"] 204 | } 205 | ``` 206 | 207 | 208 | ### File Backend 209 | The file backend uses a file on the local filesystem. This backend is often useful for testing purposes but it not scalable to multiple clients. 210 | 211 | 212 | #### Backend Options 213 | | Key | Type | Required | Description | 214 | | --- | ---- | -------- | ----------- | 215 | | swag.data_dir | str | false | Directory to store data (Default: cwd()) | 216 | | swag.data_file | str | false | Full path to data file | 217 | 218 | 219 | ### DynamoDB Backend 220 | The DynamoDB backend leverages AWSs DynamoDB as a key value store for SWAG metadata. 221 | 222 | SWAG expects a Dynamodb Table already exists. 223 | 224 | Create a new table via the AWS console named `accounts` with a primary key of `id`. 225 | 226 | #### Backend Options 227 | 228 | | Key | Type | Required | Description | 229 | | --- | ---- | -------- | ----------- | 230 | | swag.region | str | false | Region dynamodb table exists | 231 | 232 | Note the above options except region is only needed if not SWAG table has been created. 233 | 234 | #### Permissions 235 | 236 | Minimum Permissions required: 237 | 238 | ``` 239 | { 240 | "Version": "2012-10-17", 241 | "Statement": [ 242 | { 243 | "Sid": "DescribeQueryTable", 244 | "Effect": "Allow", 245 | "Action": [ 246 | "dynamodb:DescribeTable", 247 | "dynamodb:Query", 248 | "dynamodb:Scan" 249 | ], 250 | "Resource": "*" 251 | } 252 | ] 253 | } 254 | ``` 255 | 256 | If you wish SWAG to modify your table you will need the following additional permissions: 257 | 258 | ``` 259 | { 260 | "Version": "2012-10-17", 261 | "Statement": [ 262 | { 263 | "Sid": "PutUpdateDeleteTable", 264 | "Effect": "Allow", 265 | "Action": [ 266 | "dynamodb:PutItem", 267 | "dynamodb:UpdateItem", 268 | "dynamodb:DeleteItem" 269 | ], 270 | "Resource": "*" 271 | } 272 | ] 273 | } 274 | ``` 275 | 276 | 277 | ### CLI Usage 278 | 279 | Upon installation the swag_client creates a `swag` entrypoint which invokes the swag_client cli, example usage: 280 | 281 | Examples: 282 | 283 | ```bash 284 | swag --help 285 | ``` 286 | 287 | 288 | ### Extended SWAG Schema (Version 2) 289 | The following describes the usage of all native fields included within the SWAG schema. 290 | 291 | 292 | | Name | Type | Description | 293 | | ---- | ---- | ----------- | 294 | | schemaVersion | int | Describes the current schema version | 295 | | id | str | Unique ID of the account | 296 | | name | str | Canonical name, according to the account naming standard | 297 | | contacts | list(str) | List of team DLs that are majority stakeholders for the account | 298 | | provider | str | One of: AWS, GCP, Azure | 299 | | type | str | See schema context for field validation | 300 | | status | list(dict) | See status schema | 301 | | services | list(dict) | See service schema | 302 | | environment | str | See schema context for field validation | 303 | | sensitive | bool | Signifies if the account holds a special significance; (in scope for PCI, holds PII, contains sensitive key material, etc.,) | 304 | | description | str | Brief description about the account's intended use. | 305 | | owner | str | See schema context for field validation | 306 | | aliases | list(str) | List of other names this account may be referred to as | 307 | 308 | ### Schmea Context for Field Validation 309 | The V2 schema performs validation checks on certain fields to ensure values are within a defined list. Some of these are optional and configurable to allow users to specify values that make sense for their use case. 310 | 311 | The allowed values for `owner`, `environment` and `type` can be set during SWAGManager initialization by passing a `swag.schema_context` object as part of the swag_opts. 312 | 313 | If you do not specify a schema_context entry for a field then any value is permitted. 314 | 315 | 316 | ``` 317 | swag_opts = { 318 | 'swag.schema_context': { 319 | 'owner': ['netflix', 'dvd', 'aws', 'third-party'], 320 | 'environment': ['test', 'prod'], 321 | 'type': ['billing', 'security', 'shared-service', 'service'] 322 | } 323 | } 324 | ``` 325 | 326 | #### Service Schema 327 | 328 | | Name | Type | Description | 329 | | ---- | ---- | ----------- | 330 | | name | str | Name of the service | 331 | | regions | list(str) | List of regions - empty list indicates all regions | 332 | | roles | list(dict) | List of roles that control access to this service. See role schema | 333 | | metadata | dict | Service Level metadata 334 | 335 | 336 | #### Status Schema 337 | 338 | | Name | Type | Description | 339 | | ---- | ---- | ----------- | 340 | | region | str | Status per-region | 341 | | status | str | One of: Created, In-progress, Ready, Deprecated, In-active, Deleted | 342 | | notes | list(dict) | See notes schema | 343 | 344 | 345 | #### Notes Schema 346 | 347 | | Name | Type | Description | 348 | | ---- | ---- | ----------- | 349 | | date | date | Date note was created | 350 | | text | str | Free text field with additional information | 351 | 352 | #### Roles Schema 353 | 354 | | Name | Type | Description | 355 | | ---- | ---- | ----------- | 356 | | id | str | Id of the role | 357 | | policyUrl | str | URL with link to role permissions | 358 | | roleName | str | Corresponding AWS role (if any) 359 | | googleGroup | str | Corresponding google group | 360 | | secondaryApprover | str | DL needed to approve role | 361 | 362 | 363 | #### Definitions 364 | ##### Status 365 | 366 | Created - Account has been created but infrastructure has not yet been established 367 | 368 | In-progress - Account infrastructure is currently being deployed 369 | 370 | Ready - Account is ready for deployment 371 | 372 | Deprecated - Account has been marked as deprecated, no new services should be deployed into this account 373 | 374 | In-active - Account has been evacuated of all services 375 | 376 | Deleted - Account has been marked as deleted 377 | 378 | ##### Type 379 | Billing Account(s) - The billing account in a multiple account architecture provides a central account for billing aggregation across the environment. Typically, no AWS resources (e.g. instances) run in the billing account. 380 | 381 | Security Account(s) - The security account in a multiple account architecture provides environments for central log collection and analysis (e.g. CloudTrail, VPC Flow Logs) and security monitoring (e.g. configuration monitoring with Security Monkey, active security scanning and testing with other tools). 382 | 383 | Shared Service Account(s) - Shared service accounts provide infrastructure, data, and services across the organization. All service/resource accounts will typically have network connectivity to shared service accounts. 384 | 385 | Service/Resource Account Groups - Service/resource account groups host the bulk of systems and applications. Account groups are created based on various dimensions and will typically have production and test elements (separate accounts). Service/resource accounts may optionally have connectivity to other service/resource accounts. 386 | 387 | See [sample_accounts.json](https://github.com/Netflix-Skunkworks/swag-client/blob/masVxter/sample_accounts.json) an example of the current json data created by SWAG. 388 | 389 | 390 | -------------------------------------------------------------------------------- /manifest.ini: -------------------------------------------------------------------------------- 1 | include setup.py README.md MANIFEST.in LICENSE 2 | global-exclude *~ -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | description-file = README.md 3 | 4 | [wheel] 5 | universal = 1 6 | 7 | [egg_info] 8 | tag_build = 9 | tag_date = 0 10 | tag_svn_revision = 0 11 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | """ 2 | SWAG Client 3 | =========== 4 | 5 | Is a python client to interface with the SWAG service. 6 | It also provides some helpful functions for dealing with accounts (aws or otherwise) 7 | """ 8 | import sys 9 | import os.path 10 | 11 | from setuptools import setup, find_packages 12 | 13 | ROOT = os.path.realpath(os.path.join(os.path.dirname(__file__))) 14 | 15 | sys.path.insert(0, ROOT) 16 | 17 | about = {} 18 | with open(os.path.join(ROOT, "swag_client", "__about__.py")) as f: 19 | exec(f.read(), about) 20 | 21 | 22 | install_requires = [ 23 | 'marshmallow>=3.5.0', 24 | 'boto3>=1.3.7', 25 | 'tabulate>=0.7.7', 26 | 'dogpile.cache>=0.6.4', 27 | 'click>=6.7', 28 | 'click-log>=0.2.1', 29 | 'jmespath>=0.9.3', 30 | 'deepdiff>=3.3.0', 31 | 'retrying>=1.3.3', 32 | 'simplejson>=3.16.0' 33 | ] 34 | 35 | tests_require = [ 36 | 'pytest==3.1.3', 37 | 'moto', 38 | 'coveralls==1.1' 39 | ] 40 | 41 | setup( 42 | name=about["__title__"], 43 | version=about["__version__"], 44 | author=about["__author__"], 45 | author_email=about["__email__"], 46 | url=about["__uri__"], 47 | description=about["__summary__"], 48 | long_description='See README.md', 49 | packages=find_packages(), 50 | include_package_data=True, 51 | zip_safe=False, 52 | install_requires=install_requires, 53 | extras_require={ 54 | 'tests': tests_require 55 | }, 56 | entry_points={ 57 | 'console_scripts': [ 58 | 'swag-client = swag_client.cli:cli', 59 | ], 60 | 'swag_client.backends': [ 61 | 'file = swag_client.backends.file:FileSWAGManager', 62 | 's3 = swag_client.backends.s3:S3SWAGManager', 63 | 'dynamodb = swag_client.backends.dynamodb:DynamoDBSWAGManager' 64 | ] 65 | }, 66 | keywords=['aws', 'account_management'] 67 | ) 68 | -------------------------------------------------------------------------------- /swag_client/__about__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, division, print_function 2 | 3 | __all__ = [ 4 | "__title__", "__summary__", "__uri__", "__version__", "__author__", 5 | "__email__", "__license__", "__copyright__", 6 | ] 7 | 8 | __title__ = "swag-client" 9 | __summary__ = ("Cloud multi-account metadata management tool.") 10 | __uri__ = "https://github.com/Netflix-Skunkworks/swag-client" 11 | 12 | __version__ = "3.0.0" 13 | 14 | __author__ = "The swag developers" 15 | __email__ = "oss@netflix.com" 16 | 17 | __license__ = "Apache License, Version 2.0" 18 | __copyright__ = "Copyright 2017 {0}".format(__author__) 19 | -------------------------------------------------------------------------------- /swag_client/__init__.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from logging import NullHandler 3 | 4 | from .exceptions import InvalidSWAGDataException 5 | 6 | logging.getLogger(__name__).addHandler(NullHandler()) 7 | 8 | -------------------------------------------------------------------------------- /swag_client/backend.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module:: swag_client.backend 3 | :platform: Unix 4 | .. author:: Kevin Glisson (kglisson@netflix.com) 5 | """ 6 | import logging 7 | import pkg_resources 8 | 9 | import jmespath 10 | 11 | from swag_client.schemas import v1, v2 12 | from swag_client.util import parse_swag_config_options 13 | from swag_client.exceptions import InvalidSWAGDataException 14 | 15 | logger = logging.getLogger(__name__) 16 | 17 | 18 | def validate(item, namespace='accounts', version=2, context=None): 19 | """Validate item against version schema. 20 | 21 | Args: 22 | item: data object 23 | namespace: backend namespace 24 | version: schema version 25 | context: schema context object 26 | """ 27 | if namespace == 'accounts': 28 | if version == 2: 29 | schema = v2.AccountSchema(context=context) 30 | return schema.load(item) 31 | elif version == 1: 32 | return v1.AccountSchema().load(item) 33 | raise InvalidSWAGDataException('Schema version is not supported. Version: {}'.format(version)) 34 | raise InvalidSWAGDataException('Namespace not supported. Namespace: {}'.format(namespace)) 35 | 36 | 37 | def one(items): 38 | """Fetches one item from a list. Throws exception if there are multiple items.""" 39 | if items: 40 | if len(items) > 1: 41 | raise InvalidSWAGDataException('Attempted to fetch one item, but multiple items found.') 42 | return items[0] 43 | 44 | 45 | def get(name): 46 | for ep in pkg_resources.iter_entry_points('swag_client.backends'): 47 | if ep.name == name: 48 | return ep.load() 49 | 50 | 51 | class SWAGManager(object): 52 | """Manages swag backends.""" 53 | def __init__(self, *args, **kwargs): 54 | if kwargs: 55 | self.configure(*args, **kwargs) 56 | 57 | def configure(self, *args, **kwargs): 58 | """Configures a SWAG manager. Overrides existing configuration.""" 59 | 60 | self.version = kwargs['schema_version'] 61 | self.namespace = kwargs['namespace'] 62 | self.backend = get(kwargs['type'])(*args, **kwargs) 63 | self.context = kwargs.pop('schema_context', {}) 64 | 65 | def create(self, item, dry_run=None): 66 | """Create a new item in backend.""" 67 | return self.backend.create(validate(item, version=self.version, context=self.context), dry_run=dry_run) 68 | 69 | def delete(self, item, dry_run=None): 70 | """Delete an item in backend.""" 71 | return self.backend.delete(item, dry_run=dry_run) 72 | 73 | def update(self, item, dry_run=None): 74 | """Update an item in backend.""" 75 | return self.backend.update(validate(item, version=self.version, context=self.context), dry_run=dry_run) 76 | 77 | def get(self, search_filter): 78 | """Fetch one item from backend.""" 79 | return one(self.get_all(search_filter)) 80 | 81 | def get_all(self, search_filter=None): 82 | """Fetch all data from backend.""" 83 | items = self.backend.get_all() 84 | 85 | if not items: 86 | if self.version == 1: 87 | return {self.namespace: []} 88 | return [] 89 | 90 | if search_filter: 91 | items = jmespath.search(search_filter, items) 92 | 93 | return items 94 | 95 | def health_check(self): 96 | """Performs a health check specific to backend technology.""" 97 | return self.backend.health_check() 98 | 99 | def get_service_enabled(self, name, accounts_list=None, search_filter=None, region=None): 100 | """Get a list of accounts where a service has been enabled.""" 101 | if not accounts_list: 102 | accounts = self.get_all(search_filter=search_filter) 103 | else: 104 | accounts = accounts_list 105 | 106 | if self.version == 1: 107 | accounts = accounts['accounts'] 108 | 109 | enabled = [] 110 | for account in accounts: 111 | if self.version == 1: 112 | account_filter = "accounts[?id=='{id}']".format(id=account['id']) 113 | else: 114 | account_filter = "[?id=='{id}']".format(id=account['id']) 115 | 116 | service = self.get_service(name, search_filter=account_filter) 117 | 118 | if self.version == 1: 119 | if service: 120 | service = service['enabled'] # no region information available in v1 121 | else: 122 | if not region: 123 | service_filter = "status[?enabled]" 124 | else: 125 | service_filter = "status[?(region=='{region}' || region=='all') && enabled]".format(region=region) 126 | 127 | service = jmespath.search(service_filter, service) 128 | 129 | if service: 130 | enabled.append(account) 131 | 132 | return enabled 133 | 134 | def get_service(self, name, search_filter): 135 | """Fetch service metadata.""" 136 | if self.version == 1: 137 | service_filter = "service.{name}".format(name=name) 138 | return jmespath.search(service_filter, self.get(search_filter)) 139 | else: 140 | service_filter = "services[?name=='{}']".format(name) 141 | return one(jmespath.search(service_filter, self.get(search_filter))) 142 | 143 | def get_service_name(self, name, search_filter): 144 | """Fetch account name as referenced by a particular service. """ 145 | service_filter = "services[?name=='{}'].metadata.name".format(name) 146 | return one(jmespath.search(service_filter, self.get(search_filter))) 147 | 148 | def get_by_name(self, name, alias=None): 149 | """Fetch all accounts with name specified, optionally include aliases.""" 150 | search_filter = "[?name=='{}']".format(name) 151 | 152 | if alias: 153 | if self.version == 1: 154 | search_filter = "accounts[?name=='{name}' || contains(alias, '{name}')]".format(name=name) 155 | 156 | elif self.version == 2: 157 | search_filter = "[?name=='{name}' || contains(aliases, '{name}')]".format(name=name) 158 | 159 | return self.get_all(search_filter) 160 | -------------------------------------------------------------------------------- /swag_client/backends/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Netflix-Skunkworks/swag-client/20f02f5547b6dd633806ec7d5f6dd7b22aad3760/swag_client/backends/__init__.py -------------------------------------------------------------------------------- /swag_client/backends/dynamodb.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | import boto3 4 | from botocore.exceptions import ClientError 5 | from dogpile.cache import make_region 6 | 7 | from swag_client.backend import SWAGManager 8 | 9 | logger = logging.getLogger(__file__) 10 | 11 | dynamodb_region = make_region() 12 | 13 | 14 | class DynamoDBSWAGManager(SWAGManager): 15 | def __init__(self, namespace, **kwargs): 16 | """Create a DynamoDb based SWAG backend.""" 17 | self.namespace = namespace 18 | resource = boto3.resource('dynamodb', region_name=kwargs['region']) 19 | self.table = resource.Table(namespace) 20 | 21 | if not dynamodb_region.is_configured: 22 | dynamodb_region.configure( 23 | 'dogpile.cache.memory', 24 | expiration_time=kwargs['cache_expires'] 25 | ) 26 | 27 | def create(self, item, dry_run=None): 28 | """Creates a new item in file.""" 29 | logger.debug('Creating new item. Item: {item} Table: {namespace}'.format( 30 | item=item, 31 | namespace=self.namespace 32 | )) 33 | 34 | if not dry_run: 35 | self.table.put_item(Item=item) 36 | 37 | return item 38 | 39 | def delete(self, item, dry_run=None): 40 | """Deletes item in file.""" 41 | logger.debug('Deleting item. Item: {item} Table: {namespace}'.format( 42 | item=item, 43 | namespace=self.namespace 44 | )) 45 | 46 | if not dry_run: 47 | self.table.delete_item(Key={'id': item['id']}) 48 | 49 | return item 50 | 51 | def update(self, item, dry_run=None): 52 | """Updates item info in file.""" 53 | logger.debug('Updating item. Item: {item} Table: {namespace}'.format( 54 | item=item, 55 | namespace=self.namespace 56 | )) 57 | 58 | if not dry_run: 59 | self.table.put_item(Item=item) 60 | 61 | return item 62 | 63 | @dynamodb_region.cache_on_arguments() 64 | def get_all(self): 65 | """Gets all items in file.""" 66 | logger.debug('Fetching items. Table: {namespace}'.format( 67 | namespace=self.namespace 68 | )) 69 | 70 | rows = [] 71 | 72 | result = self.table.scan() 73 | 74 | while True: 75 | next_token = result.get('LastEvaluatedKey', None) 76 | rows += result['Items'] 77 | 78 | if next_token: 79 | result = self.table.scan(ExclusiveStartKey=next_token) 80 | else: 81 | break 82 | 83 | return rows 84 | 85 | def health_check(self): 86 | """Gets a single item to determine if Dynamo is functioning.""" 87 | logger.debug('Health Check on Table: {namespace}'.format( 88 | namespace=self.namespace 89 | )) 90 | 91 | try: 92 | self.get_all() 93 | return True 94 | 95 | except ClientError as e: 96 | logger.exception(e) 97 | logger.error('Error encountered with Database. Assume unhealthy') 98 | return False 99 | -------------------------------------------------------------------------------- /swag_client/backends/file.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import simplejson as json 4 | import logging 5 | from io import open 6 | 7 | from dogpile.cache import make_region 8 | 9 | from swag_client.backend import SWAGManager 10 | from swag_client.util import append_item, remove_item 11 | 12 | logger = logging.getLogger(__name__) 13 | 14 | try: 15 | from json.errors import JSONDecodeError 16 | except ImportError: 17 | JSONDecodeError = ValueError 18 | 19 | 20 | file_region = make_region() 21 | 22 | 23 | def load_file(data_file): 24 | """Tries to load JSON from data file.""" 25 | try: 26 | with open(data_file, 'r', encoding='utf-8') as f: 27 | return json.loads(f.read()) 28 | 29 | except JSONDecodeError as e: 30 | return [] 31 | 32 | 33 | def save_file(data_file, data, dry_run=None): 34 | """Writes JSON data to data file.""" 35 | if dry_run: 36 | return 37 | 38 | with open(data_file, 'w', encoding='utf-8') as f: 39 | if sys.version_info > (3, 0): 40 | f.write(json.dumps(data)) 41 | else: 42 | f.write(json.dumps(data).decode('utf-8')) 43 | 44 | 45 | class FileSWAGManager(SWAGManager): 46 | def __init__(self, namespace, **kwargs): 47 | """Create a file based SWAG backend.""" 48 | self.namespace = namespace 49 | self.version = kwargs['schema_version'] 50 | 51 | if not file_region.is_configured: 52 | file_region.configure( 53 | 'dogpile.cache.memory', 54 | expiration_time=kwargs['cache_expires'] 55 | ) 56 | 57 | if not kwargs.get('data_file'): 58 | self.data_file = os.path.join(kwargs['data_dir'], self.namespace + '.json') 59 | else: 60 | self.data_file = kwargs['data_file'] 61 | 62 | if not os.path.isfile(self.data_file): 63 | logger.warning( 64 | 'Backend file does not exist, creating... Path: {data_file}'.format(data_file=self.data_file) 65 | ) 66 | 67 | save_file(self.data_file, []) 68 | 69 | def create(self, item, dry_run=None): 70 | """Creates a new item in file.""" 71 | logger.debug('Creating new item. Item: {item} Path: {data_file}'.format( 72 | item=item, 73 | data_file=self.data_file 74 | )) 75 | 76 | items = load_file(self.data_file) 77 | items = append_item(self.namespace, self.version, item, items) 78 | save_file(self.data_file, items, dry_run=dry_run) 79 | 80 | return item 81 | 82 | def delete(self, item, dry_run=None): 83 | """Deletes item in file.""" 84 | logger.debug('Deleting item. Item: {item} Path: {data_file}'.format( 85 | item=item, 86 | data_file=self.data_file 87 | )) 88 | 89 | items = load_file(self.data_file) 90 | items = remove_item(self.namespace, self.version, item, items) 91 | save_file(self.data_file, items, dry_run=dry_run) 92 | 93 | return item 94 | 95 | def update(self, item, dry_run=None): 96 | """Updates item info in file.""" 97 | logger.debug('Updating item. Item: {item} Path: {data_file}'.format( 98 | item=item, 99 | data_file=self.data_file 100 | )) 101 | self.delete(item, dry_run=dry_run) 102 | return self.create(item, dry_run=dry_run) 103 | 104 | @file_region.cache_on_arguments() 105 | def get_all(self): 106 | """Gets all items in file.""" 107 | logger.debug('Fetching items. Path: {data_file}'.format( 108 | data_file=self.data_file 109 | )) 110 | 111 | return load_file(self.data_file) 112 | 113 | 114 | def health_check(self): 115 | """Checks to make sure the file is there.""" 116 | logger.debug('Health Check on file for: {namespace}'.format( 117 | namespace=self.namespace 118 | )) 119 | 120 | return os.path.isfile(self.data_file) 121 | -------------------------------------------------------------------------------- /swag_client/backends/s3.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import simplejson as json 3 | import logging 4 | 5 | import boto3 6 | from botocore.exceptions import ClientError 7 | from retrying import retry 8 | 9 | from dogpile.cache import make_region 10 | 11 | from swag_client.backend import SWAGManager 12 | from swag_client.util import append_item, remove_item 13 | 14 | logger = logging.getLogger(__name__) 15 | 16 | try: 17 | from json.errors import JSONDecodeError 18 | except ImportError: 19 | JSONDecodeError = ValueError 20 | 21 | 22 | s3_region = make_region() 23 | 24 | 25 | @retry(stop_max_attempt_number=3, wait_exponential_multiplier=1000, wait_exponential_max=10000) 26 | def _get_from_s3(client, bucket, data_file): 27 | return client.get_object(Bucket=bucket, Key=data_file)['Body'].read() 28 | 29 | 30 | @retry(stop_max_attempt_number=3, wait_exponential_multiplier=1000, wait_exponential_max=10000) 31 | def _put_to_s3(client, bucket, data_file, body): 32 | return client.put_object(Bucket=bucket, Key=data_file, Body=body, ContentType='application/json', 33 | CacheControl='no-cache, no-store, must-revalidate') 34 | 35 | 36 | def load_file(client, bucket, data_file): 37 | """Tries to load JSON data from S3.""" 38 | logger.debug('Loading item from s3. Bucket: {bucket} Key: {key}'.format( 39 | bucket=bucket, 40 | key=data_file 41 | )) 42 | 43 | # If the file doesn't exist, then return an empty dict: 44 | try: 45 | data = _get_from_s3(client, bucket, data_file) 46 | 47 | except ClientError as ce: 48 | if ce.response['Error']['Code'] == 'NoSuchKey': 49 | return {} 50 | 51 | else: 52 | raise ce 53 | 54 | if sys.version_info > (3,): 55 | data = data.decode('utf-8') 56 | 57 | return json.loads(data) 58 | 59 | 60 | def save_file(client, bucket, data_file, items, dry_run=None): 61 | """Tries to write JSON data to data file in S3.""" 62 | logger.debug('Writing {number_items} items to s3. Bucket: {bucket} Key: {key}'.format( 63 | number_items=len(items), 64 | bucket=bucket, 65 | key=data_file 66 | )) 67 | 68 | if not dry_run: 69 | return _put_to_s3(client, bucket, data_file, json.dumps(items)) 70 | 71 | 72 | class S3SWAGManager(SWAGManager): 73 | def __init__(self, namespace, **kwargs): 74 | """Create a S3 based SWAG backend.""" 75 | self.namespace = namespace 76 | self.version = kwargs['schema_version'] 77 | 78 | if kwargs.get('data_file'): 79 | self.data_file = kwargs['data_file'] 80 | else: 81 | self.data_file = self.namespace + '.json' 82 | 83 | self.bucket_name = kwargs['bucket_name'] 84 | self.client = boto3.client('s3', region_name=kwargs['region']) 85 | 86 | if not s3_region.is_configured: 87 | s3_region.configure( 88 | 'dogpile.cache.memory', 89 | expiration_time=kwargs['cache_expires'] 90 | ) 91 | 92 | def create(self, item, dry_run=None): 93 | """Creates a new item in file.""" 94 | logger.debug('Creating new item. Item: {item} Path: {data_file}'.format( 95 | item=item, 96 | data_file=self.data_file 97 | )) 98 | 99 | items = load_file(self.client, self.bucket_name, self.data_file) 100 | items = append_item(self.namespace, self.version, item, items) 101 | save_file(self.client, self.bucket_name, self.data_file, items, dry_run=dry_run) 102 | 103 | return item 104 | 105 | def delete(self, item, dry_run=None): 106 | """Deletes item in file.""" 107 | logger.debug('Deleting item. Item: {item} Path: {data_file}'.format( 108 | item=item, 109 | data_file=self.data_file 110 | )) 111 | 112 | items = load_file(self.client, self.bucket_name, self.data_file) 113 | items = remove_item(self.namespace, self.version, item, items) 114 | save_file(self.client, self.bucket_name, self.data_file, items, dry_run=dry_run) 115 | 116 | def update(self, item, dry_run=None): 117 | """Updates item info in file.""" 118 | logger.debug('Updating item. Item: {item} Path: {data_file}'.format( 119 | item=item, 120 | data_file=self.data_file 121 | )) 122 | self.delete(item, dry_run=dry_run) 123 | return self.create(item, dry_run=dry_run) 124 | 125 | @s3_region.cache_on_arguments() 126 | def get_all(self): 127 | """Gets all items in file.""" 128 | logger.debug('Fetching items. Path: {data_file}'.format( 129 | data_file=self.data_file 130 | )) 131 | 132 | return load_file(self.client, self.bucket_name, self.data_file) 133 | 134 | def health_check(self): 135 | """Uses head object to make sure the file exists in S3.""" 136 | logger.debug('Health Check on S3 file for: {namespace}'.format( 137 | namespace=self.namespace 138 | )) 139 | 140 | try: 141 | self.client.head_object(Bucket=self.bucket_name, Key=self.data_file) 142 | return True 143 | except ClientError as e: 144 | logger.debug('Error encountered with S3. Assume unhealthy') 145 | -------------------------------------------------------------------------------- /swag_client/cli.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | import time 4 | import simplejson as json 5 | 6 | import boto3 7 | import click 8 | import click_log 9 | from tabulate import tabulate 10 | 11 | from swag_client.backend import SWAGManager 12 | from swag_client.__about__ import __version__ 13 | from swag_client.migrations import run_migration 14 | from swag_client.util import parse_swag_config_options 15 | from swag_client.exceptions import InvalidSWAGDataException 16 | 17 | 18 | log = logging.getLogger('swag_client') 19 | click_log.basic_config(log) 20 | 21 | 22 | class CommaList(click.ParamType): 23 | name = 'commalist' 24 | 25 | def convert(self, value, param, ctx): 26 | return value.split(',') 27 | 28 | 29 | def create_swag_from_ctx(ctx): 30 | """Creates SWAG client from the current context.""" 31 | swag_opts = {} 32 | if ctx.type == 'file': 33 | swag_opts = { 34 | 'swag.type': 'file', 35 | 'swag.data_dir': ctx.data_dir, 36 | 'swag.data_file': ctx.data_file 37 | } 38 | elif ctx.type == 's3': 39 | swag_opts = { 40 | 'swag.type': 's3', 41 | 'swag.bucket_name': ctx.bucket_name, 42 | 'swag.data_file': ctx.data_file, 43 | 'swag.region': ctx.region 44 | } 45 | elif ctx.type == 'dynamodb': 46 | swag_opts = { 47 | 'swag.type': 'dynamodb', 48 | 'swag.region': ctx.region 49 | } 50 | return SWAGManager(**parse_swag_config_options(swag_opts)) 51 | 52 | 53 | class AppContext(object): 54 | def __init__(self): 55 | self.namespace = None 56 | self.region = None 57 | self.type = None 58 | self.data_dir = None 59 | self.data_file = None 60 | self.bucket_name = None 61 | self.dry_run = None 62 | 63 | 64 | pass_context = click.make_pass_decorator(AppContext, ensure=True) 65 | 66 | 67 | @click.group() 68 | @click.option('--namespace', default='accounts') 69 | @click.option('--dry-run', type=bool, default=False, is_flag=True, help='Run command without persisting anything.') 70 | @click_log.simple_verbosity_option(log) 71 | @click.version_option(version=__version__) 72 | @pass_context 73 | def cli(ctx, namespace, dry_run): 74 | if not ctx.namespace: 75 | ctx.namespace = namespace 76 | 77 | if not ctx.dry_run: 78 | ctx.dry_run = dry_run 79 | 80 | 81 | @cli.group() 82 | @click.option('--region', default='us-east-1', help='Region the table is located in.') 83 | @pass_context 84 | def dynamodb(ctx, region): 85 | if not ctx.region: 86 | ctx.region = region 87 | 88 | ctx.type = 'dynamodb' 89 | 90 | 91 | @cli.group() 92 | @click.option('--data-dir', help='Directory to store data.', default=os.getcwd()) 93 | @click.option('--data-file') 94 | @pass_context 95 | def file(ctx, data_dir, data_file): 96 | """Use the File SWAG Backend""" 97 | if not ctx.file: 98 | ctx.data_file = data_file 99 | 100 | if not ctx.data_dir: 101 | ctx.data_dir = data_dir 102 | 103 | ctx.type = 'file' 104 | 105 | 106 | @cli.group() 107 | @click.option('--bucket-name', help='Name of the bucket you wish to operate on.') 108 | @click.option('--data-file', help='Key name of the file to operate on.') 109 | @click.option('--region', default='us-east-1', help='Region the bucket is located in.') 110 | @pass_context 111 | def s3(ctx, bucket_name, data_file, region): 112 | """Use the S3 SWAG backend.""" 113 | if not ctx.data_file: 114 | ctx.data_file = data_file 115 | 116 | if not ctx.bucket_name: 117 | ctx.bucket_name = bucket_name 118 | 119 | if not ctx.region: 120 | ctx.region = region 121 | 122 | ctx.type = 's3' 123 | 124 | 125 | @cli.command() 126 | @pass_context 127 | def list(ctx): 128 | """List SWAG account info.""" 129 | if ctx.namespace != 'accounts': 130 | click.echo( 131 | click.style('Only account data is available for listing.', fg='red') 132 | ) 133 | return 134 | 135 | swag = create_swag_from_ctx(ctx) 136 | accounts = swag.get_all() 137 | _table = [[result['name'], result.get('id')] for result in accounts] 138 | click.echo( 139 | tabulate(_table, headers=["Account Name", "Account Number"]) 140 | ) 141 | 142 | 143 | @cli.command() 144 | @click.option('--name', help='Name of the service to list.') 145 | @pass_context 146 | def list_service(ctx, name): 147 | """Retrieve accounts pertaining to named service.""" 148 | swag = create_swag_from_ctx(ctx) 149 | accounts = swag.get_service_enabled(name) 150 | 151 | _table = [[result['name'], result.get('id')] for result in accounts] 152 | click.echo( 153 | tabulate(_table, headers=["Account Name", "Account Number"]) 154 | ) 155 | 156 | 157 | @cli.command() 158 | @click.option('--start-version', default=1, help='Starting version.') 159 | @click.option('--end-version', default=2, help='Ending version.') 160 | @pass_context 161 | def migrate(ctx, start_version, end_version): 162 | """Transition from one SWAG schema to another.""" 163 | if ctx.type == 'file': 164 | if ctx.data_file: 165 | file_path = ctx.data_file 166 | else: 167 | file_path = os.path.join(ctx.data_file, ctx.namespace + '.json') 168 | 169 | # todo make this more like alemebic and determine/load versions automatically 170 | with open(file_path, 'r') as f: 171 | data = json.loads(f.read()) 172 | 173 | data = run_migration(data, start_version, end_version) 174 | with open(file_path, 'w') as f: 175 | f.write(json.dumps(data)) 176 | 177 | 178 | @cli.command() 179 | @pass_context 180 | def propagate(ctx): 181 | """Transfers SWAG data from one backend to another""" 182 | data = [] 183 | if ctx.type == 'file': 184 | if ctx.data_file: 185 | file_path = ctx.data_file 186 | else: 187 | file_path = os.path.join(ctx.data_dir, ctx.namespace + '.json') 188 | 189 | with open(file_path, 'r') as f: 190 | data = json.loads(f.read()) 191 | 192 | swag_opts = { 193 | 'swag.type': 'dynamodb' 194 | } 195 | 196 | swag = SWAGManager(**parse_swag_config_options(swag_opts)) 197 | 198 | for item in data: 199 | time.sleep(2) 200 | swag.create(item, dry_run=ctx.dry_run) 201 | 202 | 203 | @cli.command() 204 | @pass_context 205 | @click.argument('data', type=click.File()) 206 | def create(ctx, data): 207 | """Create a new SWAG item.""" 208 | swag = create_swag_from_ctx(ctx) 209 | data = json.loads(data.read()) 210 | 211 | for account in data: 212 | swag.create(account, dry_run=ctx.dry_run) 213 | 214 | 215 | @cli.command() 216 | @pass_context 217 | @click.argument('data', type=click.File()) 218 | def update(ctx, data): 219 | """Updates a given record.""" 220 | swag = create_swag_from_ctx(ctx) 221 | data = json.loads(data.read()) 222 | 223 | for account in data: 224 | swag.update(account, dry_run=ctx.dry_run) 225 | 226 | 227 | @cli.command() 228 | @pass_context 229 | @click.argument('name') 230 | @click.option('--path', type=str, default='', help='JMESPath string to filter accounts to be targeted. Default is all accounts.') 231 | @click.option('--regions', type=CommaList(), default='all', 232 | help='AWS regions that should be configured. These are comma delimited (e.g. us-east-1, us-west-2, eu-west-1). Default: all') 233 | @click.option('--disabled', type=bool, default=False, is_flag=True, help='Service should be marked as enabled.') 234 | def deploy_service(ctx, path, name, regions, disabled): 235 | """Deploys a new service JSON to multiple accounts. NAME is the service name you wish to deploy.""" 236 | enabled = False if disabled else True 237 | 238 | swag = create_swag_from_ctx(ctx) 239 | accounts = swag.get_all(search_filter=path) 240 | log.debug('Searching for accounts. Found: {} JMESPath: `{}`'.format(len(accounts), path)) 241 | for a in accounts: 242 | try: 243 | if not swag.get_service(name, search_filter="[?id=='{id}']".format(id=a['id'])): 244 | log.info('Found an account to update. AccountName: {name} AccountNumber: {number}'.format(name=a['name'], number=a['id'])) 245 | status = [] 246 | for region in regions: 247 | status.append( 248 | { 249 | 'enabled': enabled, 250 | 'region': region 251 | 252 | } 253 | ) 254 | a['services'].append( 255 | { 256 | 'name': name, 257 | 'status': status 258 | } 259 | ) 260 | 261 | swag.update(a, dry_run=ctx.dry_run) 262 | except InvalidSWAGDataException as e: 263 | log.warning('Found a data quality issue. AccountName: {name} AccountNumber: {number}'.format(name=a['name'], number=a['id'])) 264 | 265 | log.info('Service has been deployed to all matching accounts.') 266 | 267 | 268 | @cli.command() 269 | @pass_context 270 | @click.argument('data', type=click.File()) 271 | def seed_aws_data(ctx, data): 272 | """Seeds SWAG from a list of known AWS accounts.""" 273 | swag = create_swag_from_ctx(ctx) 274 | for k, v in json.loads(data.read()).items(): 275 | for account in v['accounts']: 276 | data = { 277 | 'description': 'This is an AWS owned account used for {}'.format(k), 278 | 'id': account['account_id'], 279 | 'contacts': [], 280 | 'owner': 'aws', 281 | 'provider': 'aws', 282 | 'sensitive': False, 283 | 'email': 'support@amazon.com', 284 | 'name': k + '-' + account['region'] 285 | } 286 | 287 | click.echo(click.style( 288 | 'Seeded Account. AccountName: {}'.format(data['name']), fg='green') 289 | ) 290 | 291 | swag.create(data, dry_run=ctx.dry_run) 292 | 293 | @cli.command() 294 | @pass_context 295 | @click.option('--owner', type=str, required=True, help='The owner for the account schema.') 296 | def seed_aws_organization(ctx, owner): 297 | """Seeds SWAG from an AWS organziation.""" 298 | swag = create_swag_from_ctx(ctx) 299 | accounts = swag.get_all() 300 | _ids = [result.get('id') for result in accounts] 301 | 302 | client = boto3.client('organizations') 303 | paginator = client.get_paginator('list_accounts') 304 | response_iterator = paginator.paginate() 305 | 306 | count = 0 307 | for response in response_iterator: 308 | for account in response['Accounts']: 309 | if account['Id'] in _ids: 310 | click.echo(click.style( 311 | 'Ignoring Duplicate Account. AccountId: {} already exists in SWAG'.format(account['Id']), fg='yellow') 312 | ) 313 | continue 314 | 315 | if account['Status'] == 'SUSPENDED': 316 | status = 'deprecated' 317 | else: 318 | status = 'created' 319 | 320 | data = { 321 | 'id': account['Id'], 322 | 'name': account['Name'], 323 | 'description': 'Account imported from AWS organization.', 324 | 'email': account['Email'], 325 | 'owner': owner, 326 | 'provider': 'aws', 327 | 'contacts': [], 328 | 'sensitive': False, 329 | 'status': [{'region': 'all', 'status': status}] 330 | } 331 | 332 | click.echo(click.style( 333 | 'Seeded Account. AccountName: {}'.format(data['name']), fg='green') 334 | ) 335 | 336 | count += 1 337 | swag.create(data, dry_run=ctx.dry_run) 338 | 339 | click.echo('Seeded {} accounts to SWAG.'.format(count)) 340 | 341 | # todo perhaps there is a better way of dynamically adding subcommands? 342 | file.add_command(list) 343 | file.add_command(migrate) 344 | file.add_command(propagate) 345 | file.add_command(create) 346 | file.add_command(seed_aws_data) 347 | file.add_command(seed_aws_organization) 348 | file.add_command(update) 349 | file.add_command(deploy_service) 350 | file.add_command(list_service) 351 | dynamodb.add_command(list) 352 | dynamodb.add_command(create) 353 | dynamodb.add_command(update) 354 | dynamodb.add_command(seed_aws_data) 355 | dynamodb.add_command(seed_aws_organization) 356 | dynamodb.add_command(deploy_service) 357 | dynamodb.add_command(list_service) 358 | s3.add_command(list) 359 | s3.add_command(create) 360 | s3.add_command(update) 361 | s3.add_command(seed_aws_data) 362 | s3.add_command(seed_aws_organization) 363 | s3.add_command(deploy_service) 364 | s3.add_command(list_service) 365 | 366 | 367 | -------------------------------------------------------------------------------- /swag_client/compat.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Python 2/3 compatibility module.""" 3 | import sys 4 | 5 | PY2 = int(sys.version[0]) == 2 6 | 7 | if PY2: 8 | text_type = unicode # noqa 9 | binary_type = str 10 | string_types = (str, unicode) # noqa 11 | unicode = unicode # noqa 12 | basestring = basestring # noqa 13 | else: 14 | text_type = str 15 | binary_type = bytes 16 | string_types = (str,) 17 | unicode = str 18 | basestring = (str, bytes) 19 | -------------------------------------------------------------------------------- /swag_client/data/aws_accounts.json: -------------------------------------------------------------------------------- 1 | { 2 | "CloudTrailLogs": { 3 | "url": "https://docs.aws.amazon.com/awscloudtrail/latest/userguide/cloudtrail-supported-regions.html", 4 | "accounts": [ 5 | { 6 | "region": "ap-south-1", 7 | "account_id": "977081816279" 8 | }, 9 | { 10 | "region": "ap-northeast-1", 11 | "account_id": "216624486486" 12 | }, 13 | { 14 | "region": "ap-northeast-2", 15 | "account_id": "492519147666" 16 | }, 17 | { 18 | "region": "ap-southeast-1", 19 | "account_id": "903692715234" 20 | }, 21 | { 22 | "region": "ap-southeast-2", 23 | "account_id": "284668455005" 24 | }, 25 | { 26 | "region": "ca-central-1", 27 | "account_id": "819402241893" 28 | }, 29 | { 30 | "region": "eu-central-1", 31 | "account_id": "035351147821" 32 | }, 33 | { 34 | "region": "eu-west-1", 35 | "account_id": "859597730677" 36 | }, 37 | { 38 | "region": "sa-east-1", 39 | "account_id": "814480443879" 40 | }, 41 | { 42 | "region": "us-east-1", 43 | "account_id": "086441151436" 44 | }, 45 | { 46 | "region": "us-east-2", 47 | "account_id": "475085895292" 48 | }, 49 | { 50 | "region": "us-west-1", 51 | "account_id": "388731089494" 52 | }, 53 | { 54 | "region": "us-west-2", 55 | "account_id": "113285607260" 56 | } 57 | ] 58 | }, 59 | "ELBLogs": { 60 | "url": "https://docs.aws.amazon.com/ElasticLoadBalancing/latest/DeveloperGuide/enable-access-logs.html", 61 | "accounts": [ 62 | { 63 | "region": "us-east-1", 64 | "account_id": "127311923021" 65 | }, 66 | { 67 | "region": "us-east-2", 68 | "account_id": "033677994240" 69 | }, 70 | { 71 | "region": "us-west-1", 72 | "account_id": "027434742980" 73 | }, 74 | { 75 | "region": "us-west-2", 76 | "account_id": "797873946194" 77 | }, 78 | { 79 | "region": "eu-west-1", 80 | "account_id": "156460612806" 81 | }, 82 | { 83 | "region": "eu-central-1", 84 | "account_id": "054676820928" 85 | }, 86 | { 87 | "region": "ap-northeast-1", 88 | "account_id": "582318560864" 89 | }, 90 | { 91 | "region": "ap-northeast-2", 92 | "account_id": "600734575887" 93 | }, 94 | { 95 | "region": "ap-southeast-1", 96 | "account_id": "114774131450" 97 | }, 98 | { 99 | "region": "ap-southeast-2", 100 | "account_id": "783225319266" 101 | }, 102 | { 103 | "region": "sa-east-1", 104 | "account_id": "507241528517" 105 | }, 106 | { 107 | "region": "ap-south-1", 108 | "account_id": "718504428378" 109 | }, 110 | { 111 | "region": "us-gov-west-1", 112 | "account_id": "048591011584" 113 | }, 114 | { 115 | "region": "cn-north-1", 116 | "account_id": "638102146993" 117 | } 118 | ] 119 | }, 120 | "RedShiftLogs": { 121 | "url": "https://docs.aws.amazon.com/redshift/latest/mgmt/db-auditing.html", 122 | "accounts": [ 123 | { 124 | "region": "us-east-1", 125 | "account_id": "193672423079" 126 | }, 127 | { 128 | "region": "us-east-2", 129 | "account_id": "391106570357" 130 | }, 131 | { 132 | "region": "us-west-2", 133 | "account_id": "902366379725" 134 | }, 135 | { 136 | "region": "eu-central-1", 137 | "account_id": "53454850223" 138 | }, 139 | { 140 | "region": "eu-west-1", 141 | "account_id": "210876761215" 142 | }, 143 | { 144 | "region": "ca-central-1", 145 | "account_id": "907379612154" 146 | }, 147 | { 148 | "region": "ap-northeast-1", 149 | "account_id": "404641285394" 150 | }, 151 | { 152 | "region": "ap-northeast-2", 153 | "account_id": "760740231472" 154 | }, 155 | { 156 | "region": "ap-south-1", 157 | "account_id": "865932855811" 158 | }, 159 | { 160 | "region": "ap-southeast-1", 161 | "account_id": "361669875840" 162 | }, 163 | { 164 | "region": "ap-southeast-2", 165 | "account_id": "762762565011" 166 | } 167 | ] 168 | } 169 | } -------------------------------------------------------------------------------- /swag_client/exceptions.py: -------------------------------------------------------------------------------- 1 | """SWAG exception classes""" 2 | 3 | 4 | class InvalidSWAGDataException(Exception): 5 | pass 6 | 7 | 8 | class SWAGException(Exception): 9 | pass 10 | 11 | 12 | class InvalidSWAGBackendError(SWAGException, ImportError): 13 | pass 14 | 15 | 16 | class MissingSWAGParameter(SWAGException): 17 | pass 18 | -------------------------------------------------------------------------------- /swag_client/migrations/__init__.py: -------------------------------------------------------------------------------- 1 | from swag_client.migrations.versions import v2 2 | 3 | 4 | def run_migration(data, version_start, version_end): 5 | """Runs migration against a data set.""" 6 | items = [] 7 | if version_start == 1 and version_end == 2: 8 | for item in data['accounts']: 9 | items.append(v2.upgrade(item)) 10 | 11 | if version_start == 2 and version_end == 1: 12 | for item in data: 13 | items.append(v2.downgrade(item)) 14 | items = {'accounts': items} 15 | return items 16 | -------------------------------------------------------------------------------- /swag_client/migrations/migrations.py: -------------------------------------------------------------------------------- 1 | from swag_client.migrations.versions import v2 2 | 3 | 4 | def run_migration(data, version_start, version_end): 5 | """Runs migration against a data set.""" 6 | items = [] 7 | if version_start == 1 and version_end == 2: 8 | for item in data['accounts']: 9 | items.append(v2.upgrade(item)) 10 | 11 | if version_start == 2 and version_end == 1: 12 | for item in data: 13 | items.append(v2.downgrade(item)) 14 | items = {'accounts': items} 15 | return items 16 | -------------------------------------------------------------------------------- /swag_client/migrations/versions/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Netflix-Skunkworks/swag-client/20f02f5547b6dd633806ec7d5f6dd7b22aad3760/swag_client/migrations/versions/__init__.py -------------------------------------------------------------------------------- /swag_client/migrations/versions/v2.py: -------------------------------------------------------------------------------- 1 | """Contains the revision information for a given SWAG Table""" 2 | 3 | revision = 'v2' 4 | down_revision = None 5 | 6 | 7 | def upgrade(account): 8 | """Transforms data from a v1 format to a v2 format""" 9 | environ = 'test' 10 | if 'prod' in account['tags']: 11 | environ = 'prod' 12 | 13 | owner = 'netflix' 14 | if not account['ours']: 15 | owner = 'third-party' 16 | 17 | services = [] 18 | if account['metadata'].get('s3_name'): 19 | services.append( 20 | dict( 21 | name='s3', 22 | metadata=dict( 23 | name=account['metadata']['s3_name'] 24 | ), 25 | status=[ 26 | dict( 27 | region='all', 28 | enabled=True 29 | ) 30 | ] 31 | ) 32 | ) 33 | 34 | if account['metadata'].get('cloudtrail_index'): 35 | services.append( 36 | dict( 37 | name='cloudtrail', 38 | metadata=dict( 39 | esIndex=account['metadata']['cloudtrail_index'], 40 | kibanaUrl=account['metadata']['cloudtrail_kibana_url'] 41 | ), 42 | status=[ 43 | dict( 44 | region='all', 45 | enabled=True 46 | ) 47 | ] 48 | ) 49 | ) 50 | 51 | if account.get('bastion'): 52 | services.append( 53 | dict( 54 | name='bastion', 55 | metadata=dict( 56 | hostname=account['bastion'] 57 | ), 58 | status=[ 59 | dict( 60 | region='all', 61 | enabled=True 62 | ) 63 | ] 64 | ) 65 | ) 66 | 67 | for service in account['services'].keys(): 68 | s = dict( 69 | name=service, 70 | status=[ 71 | dict( 72 | region='all', 73 | enabled=account['services'][service].get('enabled', True) 74 | ) 75 | ] 76 | ) 77 | 78 | if service == 'spinnaker': 79 | s['metadata'] = {'name': account['services'][service]['name']} 80 | 81 | if service == 'lazyfalcon': 82 | if account['services'][service].get('owner'): 83 | s['metadata'] = {'owner': account['services'][service]['owner']} 84 | 85 | if service == 'titus': 86 | s['metadata'] = {'stacks': account['services'][service]['stacks']} 87 | 88 | services.append(s) 89 | 90 | if account['metadata'].get('project_id'): 91 | item_id = account['metadata']['project_id'] 92 | elif account['metadata'].get('account_number'): 93 | item_id = account['metadata']['account_number'] 94 | else: 95 | raise Exception('No id found, are you sure this is in v1 swag format.') 96 | 97 | status = [] 98 | if account['type'] == 'aws': 99 | status = [ 100 | { 101 | 'region': 'us-east-1', 102 | 'status': 'ready' 103 | }, 104 | { 105 | 'region': 'us-west-2', 106 | 'status': 'ready' 107 | }, 108 | { 109 | 'region': 'eu-west-1', 110 | 'status': 'ready' 111 | }, 112 | { 113 | 'region': 'us-east-2', 114 | 'status': 'in-active' 115 | }, 116 | { 117 | 'region': 'us-west-1', 118 | 'status': 'in-active' 119 | }, 120 | { 121 | 'region': 'ca-central-1', 122 | 'status': 'in-active' 123 | }, 124 | { 125 | 'region': 'ap-south-1', 126 | 'status': 'in-active' 127 | }, 128 | { 129 | 'region': 'ap-northeast-2', 130 | 'status': 'in-active' 131 | }, 132 | { 133 | 'region': 'ap-northeast-1', 134 | 'status': 'in-active' 135 | }, 136 | { 137 | 'region': 'ap-southeast-1', 138 | 'status': 'in-active' 139 | }, 140 | { 141 | 'region': 'ap-southeast-2', 142 | 'status': 'in-active' 143 | }, 144 | { 145 | 'region': 'eu-west-2', 146 | 'status': 'in-active' 147 | }, 148 | { 149 | 'region': 'eu-central-1', 150 | 'status': 'in-active' 151 | }, 152 | { 153 | 'region': 'sa-east-1', 154 | 'status': 'in-active' 155 | }, 156 | ] 157 | 158 | return dict( 159 | id=item_id, 160 | email=account['metadata'].get('email'), 161 | name=account['name'], 162 | contacts=account['owners'], 163 | provider=account['type'], 164 | status=status, 165 | tags=list(set(account['tags'])), 166 | environment=environ, 167 | description=account['description'], 168 | sensitive=account['cmc_required'], 169 | owner=owner, 170 | aliases=account['alias'], 171 | services=services, 172 | account_status=account['account_status'] 173 | ) 174 | 175 | 176 | def downgrade(account): 177 | """Transforms data from v2 format to a v1 format""" 178 | d_account = dict(schema_version=1, metadata={'email': account['email']}, 179 | tags=list(set([account['environment']] + account.get('tags', [])))) 180 | 181 | v1_services = {} 182 | for service in account.get('services', []): 183 | if service['name'] == 's3': 184 | if service['metadata'].get('name'): 185 | d_account['metadata']['s3_name'] = service['metadata']['name'] 186 | 187 | elif service['name'] == 'cloudtrail': 188 | d_account['metadata']['cloudtrail_index'] = service['metadata']['esIndex'] 189 | d_account['metadata']['cloudtrail_kibana_url'] = service['metadata']['kibanaUrl'] 190 | 191 | elif service['name'] == 'bastion': 192 | d_account['bastion'] = service['metadata']['hostname'] 193 | 194 | elif service['name'] == 'titus': 195 | v1_services['titus'] = { 196 | 'stacks': service['metadata']['stacks'], 197 | 'enabled': service['status'][0]['enabled'] 198 | } 199 | 200 | elif service['name'] == 'spinnaker': 201 | v1_services['spinnaker'] = { 202 | 'name': service['metadata'].get('name', account["name"]), 203 | 'enabled': service['status'][0]['enabled'] 204 | } 205 | 206 | elif service['name'] == 'awwwdit': 207 | v1_services['awwwdit'] = { 208 | 'enabled': service['status'][0]['enabled'] 209 | } 210 | 211 | elif service['name'] == 'security_monkey': 212 | v1_services['security_monkey'] = { 213 | 'enabled': service['status'][0]['enabled'] 214 | } 215 | 216 | elif service['name'] == 'poseidon': 217 | v1_services['poseidon'] = { 218 | 'enabled': service['status'][0]['enabled'] 219 | } 220 | 221 | elif service['name'] == 'rolliepollie': 222 | v1_services['rolliepollie'] = { 223 | 'enabled': service['status'][0]['enabled'] 224 | } 225 | 226 | elif service['name'] == 'lazyfalcon': 227 | owner = None 228 | 229 | if service.get('metadata'): 230 | if service['metadata'].get('owner'): 231 | owner = service['metadata']['owner'] 232 | 233 | v1_services['lazyfalcon'] = { 234 | 'enabled': service['status'][0]['enabled'], 235 | 'owner': owner 236 | } 237 | 238 | if account['provider'] == 'aws': 239 | d_account['metadata']['account_number'] = account['id'] 240 | 241 | elif account['provider'] == 'gcp': 242 | d_account['metadata']['project_id'] = account['id'] 243 | 244 | d_account['id'] = account['provider'] + '-' + account['id'] 245 | d_account['cmc_required'] = account['sensitive'] 246 | d_account['name'] = account['name'] 247 | d_account['alias'] = account['aliases'] 248 | d_account['description'] = account['description'] 249 | d_account['owners'] = account['contacts'] 250 | d_account['type'] = account['provider'] 251 | d_account['ours'] = True if account['owner'] == 'netflix' else False 252 | d_account['netflix'] = True if account['owner'] == 'netflix' else False 253 | d_account['services'] = v1_services 254 | d_account['account_status'] = account['account_status'] 255 | 256 | return d_account 257 | -------------------------------------------------------------------------------- /swag_client/schemas/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Netflix-Skunkworks/swag-client/20f02f5547b6dd633806ec7d5f6dd7b22aad3760/swag_client/schemas/__init__.py -------------------------------------------------------------------------------- /swag_client/schemas/services/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Netflix-Skunkworks/swag-client/20f02f5547b6dd633806ec7d5f6dd7b22aad3760/swag_client/schemas/services/__init__.py -------------------------------------------------------------------------------- /swag_client/schemas/v1.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module:: swag_client.schemas.v1 3 | :platform: Unix 4 | .. author:: Kevin Glisson (kglisson@netflix.com) 5 | .. author:: Mike Grima (mgrima@netflix.com) 6 | """ 7 | from marshmallow import Schema, fields, validates_schema 8 | from marshmallow.validate import Length, OneOf 9 | 10 | from swag_client.schemas.validators import validate_fqdn 11 | 12 | 13 | class AWSAccountSchema(Schema): 14 | account_number = fields.String(required=True, validate=Length(max=12, min=12)) # equal keyword not yet available in mainline marshmallow 15 | cloudtrail_index = fields.String() 16 | cloudtrail_kibana_url = fields.String() 17 | s3_name = fields.String() 18 | email = fields.Email(required=True) 19 | 20 | 21 | class GoogleProjectSchema(Schema): 22 | project_id = fields.String(required=True, validate=Length(max=30)) 23 | project_number = fields.Integer(required=True) 24 | project_name = fields.String(required=True) 25 | 26 | 27 | TYPES = {'aws': AWSAccountSchema, 'gcp': GoogleProjectSchema} 28 | 29 | 30 | # Here are define top level fields that every account would need 31 | class AccountSchema(Schema): 32 | id = fields.String(required=True) 33 | name = fields.String(required=True) 34 | email = fields.String(required=False) 35 | type = fields.String(required=True, validate=OneOf(TYPES.keys())) 36 | metadata = fields.Dict(required=True) 37 | tags = fields.List(fields.String()) 38 | services = fields.Dict() 39 | cmc_required = fields.Boolean(required=True) 40 | description = fields.String(required=True) 41 | owners = fields.List(fields.Email(required=True), required=True) 42 | alias = fields.List(fields.String(), required=True) 43 | bastion = fields.String(validate=validate_fqdn) 44 | 45 | # Set to True if your company owns the account. 46 | # Set to False if this is a partner account your apps may need to know about. 47 | ours = fields.Boolean(required=True) 48 | 49 | schema_version = fields.Integer(missing='v1') 50 | account_status = fields.String(missing='created') 51 | 52 | @validates_schema 53 | def validate_metadata(self, data, partial=False, many=False, unknown=False): 54 | TYPES[data['type']](many=True).load([data['metadata']]) 55 | -------------------------------------------------------------------------------- /swag_client/schemas/v2.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | from marshmallow import Schema, fields, validates_schema 3 | from marshmallow.exceptions import ValidationError 4 | from marshmallow.validate import OneOf 5 | 6 | 7 | PROVIDERS = ['aws', 'gcp', 'azure'] 8 | ACCOUNT_STATUSES = ['created', 'in-progress', 'ready', 'deprecated', 'deleted', 'in-active'] 9 | 10 | 11 | class NoteSchema(Schema): 12 | date = fields.Str() 13 | text = fields.Str(required=True) 14 | 15 | 16 | class RoleSchema(Schema): 17 | policyUrl = fields.Str() 18 | roleName = fields.Str() 19 | id = fields.Str(required=True) 20 | secondaryApprover = fields.Str(default=None, missing=None) 21 | googleGroup = fields.Str(required=True) 22 | 23 | 24 | class AccountStatusSchema(Schema): 25 | region = fields.Str(required=True) 26 | status = fields.Str(validate=OneOf(ACCOUNT_STATUSES), missing='created') 27 | notes = fields.Nested(NoteSchema, many=True, missing=[]) 28 | 29 | 30 | class ServiceStatusSchema(Schema): 31 | region = fields.Str(required=True) 32 | enabled = fields.Boolean(missing=False) 33 | notes = fields.Nested(NoteSchema, many=True, missing=[]) 34 | 35 | 36 | class ServiceSchema(Schema): 37 | name = fields.Str(required=True) 38 | status = fields.Nested(ServiceStatusSchema, many=True, required=True) 39 | roles = fields.Nested(RoleSchema, many=True, missing=[]) 40 | metadata = fields.Dict(missing={}) 41 | 42 | 43 | class RegionSchema(Schema): 44 | status = fields.Str(validate=OneOf(ACCOUNT_STATUSES), missing='created') 45 | az_mapping = fields.Dict() 46 | 47 | 48 | class AccountSchema(Schema): 49 | schemaVersion = fields.Str(missing='2') 50 | id = fields.Str(required=True) 51 | name = fields.Str(required=True) 52 | contacts = fields.List(fields.Email(), missing=[]) 53 | provider = fields.Str(validate=OneOf(PROVIDERS), missing='aws') 54 | type = fields.Str(missing='service') 55 | tags = fields.List(fields.Str(), missing=[]) 56 | status = fields.Nested(AccountStatusSchema, many=True, missing=[]) 57 | email = fields.Email(required=True) 58 | environment = fields.Str(missing='prod') 59 | services = fields.Nested(ServiceSchema, many=True, missing=[]) 60 | sensitive = fields.Bool(missing=False) 61 | description = fields.Str(required=True) 62 | owner = fields.Str(missing='netflix') 63 | aliases = fields.List(fields.Str(), missing=[]) 64 | account_status = fields.Str(validate=OneOf(ACCOUNT_STATUSES), missing='created') 65 | domain = fields.Str() 66 | sub_domain = fields.Str() 67 | regions = fields.Dict() 68 | org_id = fields.Str() 69 | 70 | @validates_schema 71 | def validate_type(self, data, partial=False, many=False, unknown=False): 72 | """Performs field validation against the schema context 73 | if values have been provided to SWAGManager via the 74 | swag.schema_context config object. 75 | 76 | If the schema context for a given field is empty, then 77 | we assume any value is valid for the given schema field. 78 | """ 79 | fields_to_validate = ['type', 'environment', 'owner'] 80 | for field in fields_to_validate: 81 | value = data.get(field) 82 | allowed_values = self.context.get(field) 83 | if allowed_values and value not in allowed_values: 84 | raise ValidationError('Must be one of {}'.format(allowed_values), field_names=field) 85 | 86 | @validates_schema 87 | def validate_account_status(self, data, partial=False, many=False, unknown=False): 88 | """Performs field validation for account_status. If any 89 | region is not deleted, account_status cannot be deleted 90 | """ 91 | deleted_status = 'deleted' 92 | region_status = data.get('status') 93 | account_status = data.get('account_status') 94 | for region in region_status: 95 | if region['status'] != deleted_status and account_status == deleted_status: 96 | raise ValidationError('Account Status cannot be "deleted" if a region is not "deleted"') 97 | 98 | @validates_schema 99 | def validate_regions_schema(self, data, partial=False, many=False, unknown=False): 100 | """Performs field validation for regions. This should be 101 | a dict with region names as the key and RegionSchema as the value 102 | """ 103 | region_schema = RegionSchema() 104 | supplied_regions = data.get('regions', {}) 105 | for region in supplied_regions.keys(): 106 | result = region_schema.validate(supplied_regions[region]) 107 | if len(result.keys()) > 0: 108 | raise ValidationError(result) 109 | 110 | -------------------------------------------------------------------------------- /swag_client/schemas/validators.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module:: swag_client.schemas.validators 3 | :platform: Unix 4 | .. author:: Kevin Glisson (kglisson@netflix.com) 5 | .. author:: Mike Grima (mgrima@netflix.com) 6 | """ 7 | from marshmallow.validate import Validator 8 | from marshmallow.exceptions import ValidationError 9 | 10 | 11 | def validate_fqdn(n): 12 | if '.' not in n: 13 | raise ValidationError("{0} is not a FQDN".format(n)) 14 | 15 | 16 | class IsDigit(Validator): 17 | def __call__(self, value): 18 | if not value.isdigit(): 19 | raise ValidationError("Must be a string of digits: {}".format(value)) 20 | return True 21 | -------------------------------------------------------------------------------- /swag_client/swag.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module:: swag_client.swag 3 | :platform: Unix 4 | .. author:: Kevin Glisson (kglisson@netflix.com) 5 | .. author:: Mike Grima (mgrima@netflix.com) 6 | """ 7 | from swag_client.backend import SWAGManager 8 | from swag_client.util import is_sub_dict, deprecated, parse_swag_config_options 9 | 10 | 11 | @deprecated('This has been deprecated.') 12 | def get_by_name(account_name, bucket, region='us-west-2', json_path='accounts.json', alias=None): 13 | """Given an account name, attempts to retrieve associated account info.""" 14 | for account in get_all_accounts(bucket, region, json_path)['accounts']: 15 | if 'aws' in account['type']: 16 | if account['name'] == account_name: 17 | return account 18 | elif alias: 19 | for a in account['alias']: 20 | if a == account_name: 21 | return account 22 | 23 | 24 | @deprecated('This has been deprecated.') 25 | def get_by_aws_account_number(account_number, bucket, region='us-west-2', json_path='accounts.json'): 26 | """Given an account number (or ID), attempts to retrieve associated account info.""" 27 | for account in get_all_accounts(bucket, region, json_path)['accounts']: 28 | if 'aws' in account['type']: 29 | if account['metadata']['account_number'] == account_number: 30 | return account 31 | 32 | 33 | @deprecated('This has been deprecated.') 34 | def get_all_accounts(bucket, region='us-west-2', json_path='accounts.json', **filters): 35 | """Fetches all the accounts from SWAG.""" 36 | swag_opts = { 37 | 'swag.type': 's3', 38 | 'swag.bucket_name': bucket, 39 | 'swag.region': region, 40 | 'swag.data_file': json_path, 41 | 'swag.schema_version': 1 42 | } 43 | 44 | swag = SWAGManager(**parse_swag_config_options(swag_opts)) 45 | accounts = swag.get_all() 46 | accounts = [account for account in accounts['accounts'] if is_sub_dict(filters, account)] 47 | return {'accounts': accounts} 48 | 49 | 50 | 51 | 52 | -------------------------------------------------------------------------------- /swag_client/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Netflix-Skunkworks/swag-client/20f02f5547b6dd633806ec7d5f6dd7b22aad3760/swag_client/tests/__init__.py -------------------------------------------------------------------------------- /swag_client/tests/conftest.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import boto3 4 | import pytest 5 | import tempfile 6 | 7 | from mock import patch 8 | from moto import mock_s3, mock_dynamodb2 9 | 10 | 11 | @pytest.fixture(scope='function') 12 | def aws_credentials(): 13 | """Mocked AWS Credentials for moto.""" 14 | os.environ['AWS_ACCESS_KEY_ID'] = 'testing' 15 | os.environ['AWS_SECRET_ACCESS_KEY'] = 'testing' 16 | os.environ['AWS_SECURITY_TOKEN'] = 'testing' 17 | os.environ['AWS_SESSION_TOKEN'] = 'testing' 18 | 19 | 20 | @pytest.fixture(scope='function') 21 | def retry(): 22 | """Mock the retry library so that it doesn't retry.""" 23 | def mock_retry_decorator(*args, **kwargs): 24 | def retry(func): 25 | return func 26 | return retry 27 | 28 | patch_retry = patch('retrying.retry', mock_retry_decorator) 29 | yield patch_retry.start() 30 | 31 | patch_retry.stop() 32 | 33 | 34 | @pytest.fixture(scope='function') 35 | def s3(aws_credentials, retry): 36 | with mock_s3(): 37 | yield boto3.client('s3') 38 | 39 | 40 | @pytest.fixture(scope='function') 41 | def dynamodb(aws_credentials): 42 | with mock_dynamodb2(): 43 | yield boto3.resource('dynamodb', region_name='us-east-1') 44 | 45 | 46 | @pytest.fixture(scope='function') 47 | def temp_file_name(): 48 | """A temporary file for function scope.""" 49 | with tempfile.NamedTemporaryFile(delete=True) as f: 50 | yield f.name 51 | 52 | 53 | @pytest.fixture(scope='session') 54 | def vector_path(): 55 | cwd = os.path.dirname(os.path.realpath(__file__)) 56 | return os.path.join(cwd, 'vectors') 57 | 58 | 59 | @pytest.yield_fixture(scope='function') 60 | def s3_bucket_name(s3): 61 | s3.create_bucket(Bucket='swag.test.backend') 62 | yield 'swag.test.backend' 63 | 64 | 65 | @pytest.yield_fixture(scope='function') 66 | def dynamodb_table(dynamodb): 67 | table = dynamodb.create_table( 68 | TableName='accounts', 69 | KeySchema=[ 70 | { 71 | 'AttributeName': 'id', 72 | 'KeyType': 'HASH' 73 | } 74 | ], 75 | AttributeDefinitions=[ 76 | { 77 | 'AttributeName': 'id', 78 | 'AttributeType': 'S' 79 | } 80 | ], 81 | ProvisionedThroughput={ 82 | 'ReadCapacityUnits': 1, 83 | 'WriteCapacityUnits': 1 84 | }) 85 | 86 | table.meta.client.get_waiter('table_exists').wait(TableName='accounts') 87 | yield 88 | -------------------------------------------------------------------------------- /swag_client/tests/test_swag.py: -------------------------------------------------------------------------------- 1 | from deepdiff import DeepDiff 2 | from marshmallow.exceptions import ValidationError 3 | import pytest 4 | 5 | 6 | def test_upgrade_1_to_2(): 7 | from swag_client.migrations.versions.v2 import upgrade, downgrade 8 | 9 | a = { 10 | "bastion": "testaccount.net", 11 | "metadata": { 12 | "s3_name": "testaccounts3", 13 | "cloudtrail_index": "cloudtrail_testaccount[yyyymm]", 14 | "cloudtrail_kibana_url": "http://testaccount.cloudtrail.dashboard.net", 15 | "email": "testaccount@test.net", 16 | "account_number": "012345678910" 17 | }, 18 | "schema_version": 1, 19 | "owners": [ 20 | "admins@test.net" 21 | ], 22 | "ours": True, 23 | "description": "LOL, Test account", 24 | "cmc_required": False, 25 | "tags": [ 26 | "testing", 27 | "test" 28 | ], 29 | "netflix": True, 30 | "id": "aws-012345678910", 31 | "name": "testaccount", 32 | "type": "aws", 33 | "alias": [ 34 | "test", 35 | ], 36 | "services": { 37 | "rolliepollie": { 38 | "enabled": True 39 | }, 40 | "awwwdit": { 41 | "enabled": True 42 | } 43 | }, 44 | "account_status": "ready" 45 | } 46 | 47 | v2 = upgrade(a) 48 | 49 | v1 = downgrade(v2) 50 | assert not DeepDiff(v1, a, ignore_order=True) 51 | 52 | 53 | def test_file_backend_get_all(vector_path): 54 | from swag_client.backend import SWAGManager 55 | from swag_client.util import parse_swag_config_options 56 | 57 | swag_opts = { 58 | 'swag.data_dir': vector_path, 59 | 'swag.namespace': 'valid_accounts_v2', 60 | 'swag.cache_expires': 0 61 | } 62 | 63 | swag = SWAGManager(**parse_swag_config_options(swag_opts)) 64 | assert len(swag.get_all()) == 2 65 | 66 | 67 | def test_file_backend_get_service_enabled(vector_path): 68 | from swag_client.backend import SWAGManager 69 | from swag_client.util import parse_swag_config_options 70 | 71 | swag_opts = { 72 | 'swag.data_dir': vector_path, 73 | 'swag.namespace': 'valid_accounts_v2', 74 | 'swag.cache_expires': 0 75 | } 76 | 77 | swag = SWAGManager(**parse_swag_config_options(swag_opts)) 78 | 79 | enabled = swag.get_service_enabled('myService') 80 | assert len(enabled) == 1 81 | 82 | enabled = swag.get_service_enabled('myService', region='us-east-1') 83 | assert len(enabled) == 1 84 | 85 | enabled = swag.get_service_enabled('myService1') 86 | assert len(enabled) == 0 87 | 88 | enabled = swag.get_service_enabled('myService1', region='us-east-1') 89 | assert len(enabled) == 0 90 | 91 | enabled = swag.get_service_enabled('myService2', region='us-east-1') 92 | assert len(enabled) == 1 93 | 94 | enabled = swag.get_service_enabled('myService2') 95 | assert len(enabled) == 1 96 | 97 | 98 | def test_file_backend_get_service_enabled_v1(vector_path): 99 | from swag_client.backend import SWAGManager 100 | from swag_client.util import parse_swag_config_options 101 | 102 | swag_opts = { 103 | 'swag.data_dir': vector_path, 104 | 'swag.namespace': 'valid_accounts_v1', 105 | 'swag.cache_expires': 0, 106 | 'swag.schema_version': 1 107 | } 108 | 109 | swag = SWAGManager(**parse_swag_config_options(swag_opts)) 110 | 111 | enabled = swag.get_service_enabled('myService') 112 | assert len(enabled) == 1 113 | 114 | enabled = swag.get_service_enabled('myService', region='us-east-1') 115 | assert len(enabled) == 1 116 | 117 | enabled = swag.get_service_enabled('myService1') 118 | assert len(enabled) == 1 119 | 120 | enabled = swag.get_service_enabled('myService1', region='us-east-1') 121 | assert len(enabled) == 1 122 | 123 | enabled = swag.get_service_enabled('myService2', region='us-east-1') 124 | assert len(enabled) == 0 125 | 126 | enabled = swag.get_service_enabled('myService2') 127 | assert len(enabled) == 0 128 | 129 | 130 | def test_file_backend_update(temp_file_name): 131 | from swag_client.backend import SWAGManager 132 | from swag_client.util import parse_swag_config_options 133 | 134 | swag_opts = { 135 | 'swag.data_file': str(temp_file_name), 136 | 'swag.cache_expires': 0 137 | } 138 | 139 | swag = SWAGManager(**parse_swag_config_options(swag_opts)) 140 | 141 | account = { 142 | 'aliases': ['test'], 143 | 'contacts': ['admins@test.net'], 144 | 'description': 'LOL, Test account', 145 | 'email': 'testaccount@test.net', 146 | 'environment': 'test', 147 | 'id': '012345678910', 148 | 'name': 'testaccount', 149 | 'owner': 'netflix', 150 | 'provider': 'aws', 151 | 'sensitive': False 152 | } 153 | 154 | swag.create(account) 155 | 156 | account['aliases'] = ['test', 'prod'] 157 | swag.update(account) 158 | 159 | account = swag.get("[?id=='{id}']".format(id=account['id'])) 160 | assert account['aliases'] == ['test', 'prod'] 161 | 162 | 163 | def test_file_backend_delete(temp_file_name): 164 | from swag_client.backend import SWAGManager 165 | from swag_client.util import parse_swag_config_options 166 | 167 | swag_opts = { 168 | 'swag.data_file': str(temp_file_name), 169 | 'swag.cache_expires': 0 170 | } 171 | 172 | swag = SWAGManager(**parse_swag_config_options(swag_opts)) 173 | 174 | account = { 175 | 'aliases': ['test'], 176 | 'contacts': ['admins@test.net'], 177 | 'description': 'LOL, Test account', 178 | 'email': 'testaccount@test.net', 179 | 'environment': 'test', 180 | 'id': '012345678910', 181 | 'name': 'testaccount', 182 | 'owner': 'netflix', 183 | 'provider': 'aws', 184 | 'sensitive': False 185 | } 186 | 187 | swag.create(account) 188 | swag.delete(account) 189 | assert not swag.get("[?id=='012345678910']") 190 | 191 | 192 | def test_file_backend_create(temp_file_name): 193 | from swag_client.backend import SWAGManager 194 | from swag_client.util import parse_swag_config_options 195 | 196 | swag_opts = { 197 | 'swag.data_file': str(temp_file_name), 198 | 'swag.cache_expires': 0 199 | } 200 | 201 | swag = SWAGManager(**parse_swag_config_options(swag_opts)) 202 | 203 | account = { 204 | 'aliases': ['test'], 205 | 'contacts': ['admins@test.net'], 206 | 'description': 'LOL, Test account', 207 | 'email': 'testaccount@test.net', 208 | 'environment': 'test', 209 | 'id': '012345678910', 210 | 'name': 'testaccount', 211 | 'owner': 'netflix', 212 | 'provider': 'aws', 213 | 'sensitive': False 214 | } 215 | 216 | assert not swag.get_all() 217 | item = swag.create(account) 218 | assert swag.get("[?id=='{id}']".format(id=item['id'])) 219 | 220 | 221 | def test_file_backend_get(vector_path): 222 | from swag_client.backend import SWAGManager 223 | from swag_client.util import parse_swag_config_options 224 | 225 | swag_opts = { 226 | 'swag.data_dir': vector_path, 227 | 'swag.namespace': 'valid_accounts_v2', 228 | 'swag.cache_expires': 0 229 | } 230 | 231 | swag = SWAGManager(**parse_swag_config_options(swag_opts)) 232 | 233 | assert swag.get("[?id=='012345678910']") 234 | 235 | 236 | def test_backend_get_by_name(vector_path): 237 | from swag_client.backend import SWAGManager 238 | from swag_client.util import parse_swag_config_options 239 | 240 | swag_opts = { 241 | 'swag.data_dir': vector_path, 242 | 'swag.namespace': 'valid_accounts_v2', 243 | 'swag.cache_expires': 0 244 | } 245 | 246 | swag = SWAGManager(**parse_swag_config_options(swag_opts)) 247 | 248 | assert swag.get_by_name('testaccount') 249 | assert not swag.get_by_name('test') 250 | assert swag.get_by_name('test', alias=True) 251 | assert swag.get_by_name('testaccount', alias=True) 252 | 253 | 254 | def test_backend_get_service_name(vector_path): 255 | from swag_client.backend import SWAGManager 256 | from swag_client.util import parse_swag_config_options 257 | 258 | swag_opts = { 259 | 'swag.data_dir': vector_path, 260 | 'swag.namespace': 'valid_accounts_v2', 261 | 'swag.cache_expires': 0 262 | } 263 | 264 | swag = SWAGManager(**parse_swag_config_options(swag_opts)) 265 | assert swag.get_service_name('myService', "[?name=='testaccount']") == 'testaccount' 266 | 267 | 268 | def test_s3_backend_get_all(s3_bucket_name): 269 | from swag_client.backend import SWAGManager 270 | from swag_client.util import parse_swag_config_options 271 | 272 | swag_opts = { 273 | 'swag.type': 's3', 274 | 'swag.bucket_name': s3_bucket_name, 275 | 'swag.cache_expires': 0 276 | } 277 | 278 | swag = SWAGManager(**parse_swag_config_options(swag_opts)) 279 | 280 | account = { 281 | 'aliases': ['test'], 282 | 'contacts': ['admins@test.net'], 283 | 'description': 'LOL, Test account', 284 | 'email': 'testaccount@test.net', 285 | 'environment': 'test', 286 | 'id': '012345678910', 287 | 'name': 'testaccount', 288 | 'owner': 'netflix', 289 | 'provider': 'aws', 290 | 'sensitive': False 291 | } 292 | 293 | swag.create(account) 294 | assert len(swag.get_all()) == 1 295 | 296 | 297 | def test_s3_backend_update(s3_bucket_name): 298 | from swag_client.backend import SWAGManager 299 | from swag_client.util import parse_swag_config_options 300 | 301 | swag_opts = { 302 | 'swag.type': 's3', 303 | 'swag.bucket_name': s3_bucket_name, 304 | 'swag.cache_expires': 0 305 | } 306 | 307 | swag = SWAGManager(**parse_swag_config_options(swag_opts)) 308 | 309 | account = { 310 | 'aliases': ['test'], 311 | 'contacts': ['admins@test.net'], 312 | 'description': 'LOL, Test account', 313 | 'email': 'testaccount@test.net', 314 | 'environment': 'test', 315 | 'id': '012345678910', 316 | 'name': 'testaccount', 317 | 'owner': 'netflix', 318 | 'provider': 'aws', 319 | 'sensitive': False 320 | } 321 | 322 | swag.create(account) 323 | 324 | account['aliases'] = ['test', 'prod'] 325 | swag.update(account) 326 | 327 | item = swag.get("[?id=='{id}']".format(id=account['id'])) 328 | 329 | assert item['aliases'] == ['test', 'prod'] 330 | 331 | 332 | def test_s3_backend_delete(s3_bucket_name): 333 | from swag_client.backend import SWAGManager 334 | from swag_client.util import parse_swag_config_options 335 | 336 | swag_opts = { 337 | 'swag.type': 's3', 338 | 'swag.bucket_name': s3_bucket_name, 339 | 'swag.cache_expires': 0 340 | } 341 | 342 | swag = SWAGManager(**parse_swag_config_options(swag_opts)) 343 | 344 | account = { 345 | 'aliases': ['test'], 346 | 'contacts': ['admins@test.net'], 347 | 'description': 'LOL, Test account', 348 | 'email': 'testaccount@test.net', 349 | 'environment': 'test', 350 | 'id': '012345678910', 351 | 'name': 'testaccount', 352 | 'owner': 'netflix', 353 | 'provider': 'aws', 354 | 'sensitive': False 355 | } 356 | 357 | swag.create(account) 358 | 359 | account = { 360 | 'aliases': ['test'], 361 | 'contacts': ['admins@test.net'], 362 | 'description': 'LOL, Test account', 363 | 'email': 'testaccount@test.net', 364 | 'environment': 'test', 365 | 'id': '012345678911', 366 | 'name': 'testaccount', 367 | 'owner': 'netflix', 368 | 'provider': 'aws', 369 | 'sensitive': False 370 | } 371 | swag.create(account) 372 | 373 | assert len(swag.get_all()) == 2 374 | 375 | swag.delete(account) 376 | assert len(swag.get_all()) == 1 377 | 378 | 379 | def test_s3_backend_delete_v1(s3_bucket_name): 380 | from swag_client.backend import SWAGManager 381 | from swag_client.util import parse_swag_config_options 382 | 383 | swag_opts = { 384 | 'swag.type': 's3', 385 | 'swag.bucket_name': s3_bucket_name, 386 | 'swag.schema_version': 1, 387 | 'swag.cache_expires': 0 388 | } 389 | 390 | swagv1 = SWAGManager(**parse_swag_config_options(swag_opts)) 391 | 392 | account = { 393 | "bastion": "testaccount.net", 394 | "metadata": { 395 | "s3_name": "testaccounts3", 396 | "cloudtrail_index": "cloudtrail_testaccount[yyyymm]", 397 | "cloudtrail_kibana_url": "http://testaccount.cloudtrail.dashboard.net", 398 | "email": "testaccount@test.net", 399 | "account_number": "012345678910" 400 | }, 401 | "schema_version": 1, 402 | "owners": [ 403 | "admins@test.net" 404 | ], 405 | "ours": True, 406 | "email": "bob@example.com", 407 | "description": "LOL, Test account", 408 | "cmc_required": False, 409 | "tags": [ 410 | "testing" 411 | ], 412 | "id": "aws-012345678910", 413 | "name": "testaccount", 414 | "type": "aws", 415 | "alias": [ 416 | "test", 417 | ], 418 | "services": { 419 | "rolliepollie": { 420 | "enabled": True 421 | }, 422 | "awwwdit": { 423 | "enabled": True 424 | } 425 | } 426 | } 427 | 428 | swagv1.create(account) 429 | 430 | assert len(swagv1.get_all()['accounts']) == 1 431 | swagv1.delete(account) 432 | assert len(swagv1.get_all()['accounts']) == 0 433 | 434 | 435 | def test_s3_backend_create(s3_bucket_name): 436 | from swag_client.backend import SWAGManager 437 | from swag_client.util import parse_swag_config_options 438 | 439 | swag_opts = { 440 | 'swag.type': 's3', 441 | 'swag.bucket_name': s3_bucket_name, 442 | 'swag.cache_expires': 0 443 | } 444 | 445 | swag = SWAGManager(**parse_swag_config_options(swag_opts)) 446 | 447 | account = { 448 | 'aliases': ['test'], 449 | 'contacts': ['admins@test.net'], 450 | 'description': 'LOL, Test account', 451 | 'email': 'testaccount@test.net', 452 | 'environment': 'test', 453 | 'id': '012345678910', 454 | 'name': 'testaccount', 455 | 'owner': 'netflix', 456 | 'provider': 'aws', 457 | 'sensitive': False 458 | } 459 | 460 | assert not swag.get_all() 461 | item = swag.create(account) 462 | assert swag.get("[?id=='{id}']".format(id=item['id'])) 463 | 464 | 465 | def test_s3_backend_get(s3_bucket_name): 466 | from swag_client.backend import SWAGManager 467 | from swag_client.util import parse_swag_config_options 468 | 469 | swag_opts = { 470 | 'swag.type': 's3', 471 | 'swag.bucket_name': s3_bucket_name, 472 | 'swag.cache_expires': 0 473 | } 474 | 475 | swag = SWAGManager(**parse_swag_config_options(swag_opts)) 476 | 477 | account = { 478 | 'aliases': ['test'], 479 | 'contacts': ['admins@test.net'], 480 | 'description': 'LOL, Test account', 481 | 'email': 'testaccount@test.net', 482 | 'environment': 'test', 483 | 'id': '012345678910', 484 | 'name': 'testaccount', 485 | 'owner': 'netflix', 486 | 'provider': 'aws', 487 | 'sensitive': False 488 | } 489 | 490 | swag.create(account) 491 | assert swag.get("[?id=='012345678910']") 492 | 493 | 494 | def test_dynamodb_backend_get(dynamodb_table): 495 | from swag_client.backend import SWAGManager 496 | from swag_client.util import parse_swag_config_options 497 | 498 | swag_opts = { 499 | 'swag.type': 'dynamodb', 500 | 'swag.namespace': 'accounts', 501 | 'swag.cache_expires': 0 502 | } 503 | 504 | swag = SWAGManager(**parse_swag_config_options(swag_opts)) 505 | 506 | account = { 507 | 'aliases': ['test'], 508 | 'contacts': ['admins@test.net'], 509 | 'description': 'LOL, Test account', 510 | 'email': 'testaccount@test.net', 511 | 'environment': 'test', 512 | 'id': '012345678910', 513 | 'name': 'testaccount', 514 | 'owner': 'netflix', 515 | 'provider': 'aws', 516 | 'sensitive': False 517 | } 518 | 519 | swag.create(account) 520 | assert swag.get("[?id=='012345678910']") 521 | 522 | 523 | def test_dynamodb_backend_get_all(dynamodb_table): 524 | from swag_client.backend import SWAGManager 525 | from swag_client.util import parse_swag_config_options 526 | 527 | swag_opts = { 528 | 'swag.type': 'dynamodb', 529 | 'swag.namespace': 'accounts', 530 | 'swag.cache_expires': 0 531 | } 532 | swag = SWAGManager(**parse_swag_config_options(swag_opts)) 533 | 534 | account = { 535 | 'aliases': ['test'], 536 | 'contacts': ['admins@test.net'], 537 | 'description': 'LOL, Test account', 538 | 'email': 'testaccount@test.net', 539 | 'environment': 'test', 540 | 'id': '012345678910', 541 | 'name': 'testaccount', 542 | 'owner': 'netflix', 543 | 'provider': 'aws', 544 | 'sensitive': False 545 | } 546 | 547 | swag.create(account) 548 | assert len(swag.get_all()) == 1 549 | 550 | 551 | def test_dynamodb_backend_update(dynamodb_table): 552 | from swag_client.backend import SWAGManager 553 | from swag_client.util import parse_swag_config_options 554 | 555 | swag_opts = { 556 | 'swag.type': 'dynamodb', 557 | 'swag.namespace': 'accounts', 558 | 'swag.key_type': 'HASH', 559 | 'swag.key_attribute': 'id', 560 | 'swag.cache_expires': 0 561 | } 562 | swag = SWAGManager(**parse_swag_config_options(swag_opts)) 563 | 564 | account = { 565 | 'aliases': ['test'], 566 | 'contacts': ['admins@test.net'], 567 | 'description': 'LOL, Test account', 568 | 'email': 'testaccount@test.net', 569 | 'environment': 'test', 570 | 'id': '012345678910', 571 | 'name': 'testaccount', 572 | 'owner': 'netflix', 573 | 'provider': 'aws', 574 | 'sensitive': False 575 | } 576 | 577 | swag.create(account) 578 | 579 | account['aliases'] = ['test', 'prod'] 580 | swag.update(account) 581 | 582 | assert swag.get("[?id=='{id}']".format(id=account['id'])) 583 | 584 | 585 | def test_dynamodb_backend_delete(dynamodb_table): 586 | from swag_client.backend import SWAGManager 587 | from swag_client.util import parse_swag_config_options 588 | 589 | swag_opts = { 590 | 'swag.type': 'dynamodb', 591 | 'swag.namespace': 'accounts', 592 | 'swag.cache_expires': 0 593 | } 594 | 595 | swag = SWAGManager(**parse_swag_config_options(swag_opts)) 596 | 597 | account = { 598 | 'aliases': ['test'], 599 | 'contacts': ['admins@test.net'], 600 | 'description': 'LOL, Test account', 601 | 'email': 'testaccount@test.net', 602 | 'environment': 'test', 603 | 'id': '012345678910', 604 | 'name': 'testaccount', 605 | 'owner': 'netflix', 606 | 'provider': 'aws', 607 | 'sensitive': False 608 | } 609 | 610 | swag.create(account) 611 | swag.delete(account) 612 | assert not swag.get("[?id=='012345678910']") 613 | 614 | 615 | def test_dynamodb_backend_create(dynamodb_table): 616 | from swag_client.backend import SWAGManager 617 | from swag_client.util import parse_swag_config_options 618 | 619 | swag_opts = { 620 | 'swag.type': 'dynamodb', 621 | 'swag.namespace': 'accounts', 622 | 'swag.cache_expires': 0 623 | } 624 | 625 | swag = SWAGManager(**parse_swag_config_options(swag_opts)) 626 | 627 | account = { 628 | 'aliases': ['test'], 629 | 'contacts': ['admins@test.net'], 630 | 'description': 'LOL, Test account', 631 | 'email': 'testaccount@test.net', 632 | 'environment': 'test', 633 | 'id': '012345678910', 634 | 'name': 'testaccount', 635 | 'owner': 'netflix', 636 | 'provider': 'aws', 637 | 'sensitive': False 638 | } 639 | 640 | assert not swag.get_all() 641 | item = swag.create(account) 642 | assert swag.get("[?id=='{id}']".format(id=item['id'])) 643 | 644 | 645 | # test backwards compatibility 646 | def test_get_all_accounts(s3_bucket_name): 647 | from swag_client.swag import get_all_accounts 648 | 649 | from swag_client.backend import SWAGManager 650 | from swag_client.util import parse_swag_config_options 651 | 652 | swag_opts = { 653 | 'swag.type': 's3', 654 | 'swag.bucket_name': s3_bucket_name, 655 | 'swag.schema_version': 1, 656 | 'swag.cache_expires': 0 657 | } 658 | 659 | swagv1 = SWAGManager(**parse_swag_config_options(swag_opts)) 660 | 661 | account = { 662 | "bastion": "test2.net", 663 | "metadata": { 664 | "s3_name": "testaccounts3", 665 | "cloudtrail_index": "cloudtrail_testaccount[yyyymm]", 666 | "cloudtrail_kibana_url": "http://testaccount.cloudtrail.dashboard.net", 667 | "email": "testaccount@test.net", 668 | "account_number": "012345678910" 669 | }, 670 | "schema_version": 1, 671 | "owners": [ 672 | "admins@test.net" 673 | ], 674 | "ours": True, 675 | "description": "LOL, Test account", 676 | "cmc_required": False, 677 | "email": "joe@example.com", 678 | "tags": [ 679 | "testing" 680 | ], 681 | "id": "aws-012345678910", 682 | "name": "testaccount", 683 | "type": "aws", 684 | "alias": [ 685 | "test", 686 | ] 687 | } 688 | 689 | swagv1.create(account) 690 | 691 | data = get_all_accounts(s3_bucket_name) 692 | assert len(data['accounts']) == 1 693 | 694 | data = get_all_accounts(s3_bucket_name, 695 | **{'owners': ['admins@test.net']}) 696 | 697 | assert len(data['accounts']) == 1 698 | 699 | data = get_all_accounts(s3_bucket_name, bastion="test2.net") 700 | assert len(data['accounts']) == 1 701 | 702 | 703 | def test_downgrade_spinnaker(): 704 | """Test without any metadata name set -- should default to the account name""" 705 | from swag_client.migrations.versions.v2 import downgrade 706 | account_spinnaker = { 707 | "email": "spinnakertestaccount@test.com", 708 | "services": [ 709 | { 710 | "metadata": {}, 711 | "status": [ 712 | { 713 | "region": "all", 714 | "notes": [], 715 | "enabled": True 716 | } 717 | ], 718 | "name": "spinnaker" 719 | } 720 | ], 721 | "type": "service", 722 | "aliases": [], 723 | "description": "Spinnaker Test for Downgrade", 724 | "schemaVersion": "2", 725 | "id": "098765432110", 726 | "name": "testspinnaker", 727 | "owner": "netflix", 728 | "contacts": [ 729 | "test@test.com" 730 | ], 731 | "status": [ 732 | { 733 | "status": "created", 734 | "region": "all", 735 | "notes": [] 736 | } 737 | ], 738 | "sensitive": False, 739 | "provider": "aws", 740 | "tags": [], 741 | "environment": "test", 742 | "account_status": "ready" 743 | } 744 | 745 | v1 = downgrade(account_spinnaker) 746 | assert v1["services"]["spinnaker"]["name"] == "testspinnaker" 747 | 748 | # With the name set: 749 | account_spinnaker["services"][0]["metadata"]["name"] = "lolaccountname" 750 | v1 = downgrade(account_spinnaker) 751 | assert v1["services"]["spinnaker"]["name"] == "lolaccountname" 752 | 753 | 754 | 755 | def test_get_by_name(s3_bucket_name): 756 | from swag_client.swag import get_by_name 757 | 758 | from swag_client.backend import SWAGManager 759 | from swag_client.util import parse_swag_config_options 760 | 761 | swag_opts = { 762 | 'swag.type': 's3', 763 | 'swag.bucket_name': s3_bucket_name, 764 | 'swag.schema_version': 1, 765 | 'swag.cache_expires': 0 766 | } 767 | 768 | swagv1 = SWAGManager(**parse_swag_config_options(swag_opts)) 769 | 770 | account = { 771 | "bastion": "testaccount.net", 772 | "metadata": { 773 | "s3_name": "testaccounts3", 774 | "cloudtrail_index": "cloudtrail_testaccount[yyyymm]", 775 | "cloudtrail_kibana_url": "http://testaccount.cloudtrail.dashboard.net", 776 | "email": "testaccount@test.net", 777 | "account_number": "012345678910" 778 | }, 779 | "schema_version": 1, 780 | "owners": [ 781 | "admins@test.net" 782 | ], 783 | "email": "joe@example.com", 784 | "ours": True, 785 | "description": "LOL, Test account", 786 | "cmc_required": False, 787 | "tags": [ 788 | "testing" 789 | ], 790 | "id": "aws-012345678910", 791 | "name": "testaccount", 792 | "type": "aws", 793 | "alias": [ 794 | "test", 795 | ] 796 | } 797 | 798 | swagv1.create(account) 799 | 800 | # Test getting account named: 'testaccount' 801 | account = get_by_name('testaccount', s3_bucket_name) 802 | assert account['name'] == 'testaccount' 803 | 804 | # Test by getting account that does not exist: 805 | assert not get_by_name('does not exist', s3_bucket_name) 806 | 807 | # With alias 808 | account = get_by_name('test', s3_bucket_name, alias=True) 809 | assert account['metadata']['account_number'] == '012345678910' 810 | 811 | 812 | def test_get_by_aws_account_number(s3_bucket_name): 813 | from swag_client.swag import get_by_aws_account_number 814 | 815 | from swag_client.backend import SWAGManager 816 | from swag_client.util import parse_swag_config_options 817 | 818 | swag_opts = { 819 | 'swag.type': 's3', 820 | 'swag.bucket_name': s3_bucket_name, 821 | 'swag.schema_version': 1, 822 | 'swag.cache_expires': 0 823 | } 824 | 825 | swagv1 = SWAGManager(**parse_swag_config_options(swag_opts)) 826 | 827 | account = { 828 | "bastion": "testaccount.net", 829 | "metadata": { 830 | "s3_name": "testaccounts3", 831 | "cloudtrail_index": "cloudtrail_testaccount[yyyymm]", 832 | "cloudtrail_kibana_url": "http://testaccount.cloudtrail.dashboard.net", 833 | "email": "testaccount@test.net", 834 | "account_number": "012345678910" 835 | }, 836 | "schema_version": 1, 837 | "owners": [ 838 | "admins@test.net" 839 | ], 840 | "ours": True, 841 | "email": "bob@example.com", 842 | "description": "LOL, Test account", 843 | "cmc_required": False, 844 | "tags": [ 845 | "testing" 846 | ], 847 | "id": "aws-012345678910", 848 | "name": "testaccount", 849 | "type": "aws", 850 | "alias": [ 851 | "test", 852 | ] 853 | } 854 | 855 | swagv1.create(account) 856 | 857 | # Test getting account # 012345678910 858 | account = get_by_aws_account_number('012345678910', s3_bucket_name) 859 | assert account['name'] == 'testaccount' 860 | 861 | # Test by getting account that does not exist: 862 | assert not get_by_aws_account_number('thisdoesnotexist', s3_bucket_name) 863 | 864 | 865 | def test_schema_context_validation_type_field(): 866 | """Test schema context validation for type field""" 867 | from swag_client.backend import SWAGManager 868 | from swag_client.util import parse_swag_config_options 869 | 870 | swag_opts = { 871 | 'swag.schema_context': { 872 | 'type': ['billing', 'security', 'shared-service', 'service'], 873 | } 874 | } 875 | swag = SWAGManager(**parse_swag_config_options(swag_opts)) 876 | 877 | data = { 878 | "aliases": ["test"], 879 | "contacts": ["admins@test.net"], 880 | "description": "This is just a test.", 881 | "email": "test@example.net", 882 | "environment": "dev", 883 | "id": "012345678910", 884 | "name": "testaccount", 885 | "owner": "netflix", 886 | "provider": "aws", 887 | } 888 | 889 | # Test with invalid account type 890 | with pytest.raises(ValidationError): 891 | data['type'] = 'bad_type' 892 | swag.create(data) 893 | 894 | # Test with a valid account type 895 | data['type'] = 'billing' 896 | account = swag.create(data) 897 | assert account.get('type') == 'billing' 898 | 899 | 900 | def test_schema_context_validation_environment_field(): 901 | """Test schema context validation for environment field""" 902 | from swag_client.backend import SWAGManager 903 | from swag_client.util import parse_swag_config_options 904 | 905 | swag_opts = { 906 | 'swag.schema_context': { 907 | 'environment': ['test', 'prod'] 908 | } 909 | } 910 | 911 | swag = SWAGManager(**parse_swag_config_options(swag_opts)) 912 | 913 | data = { 914 | "aliases": ["test"], 915 | "contacts": ["admins@test.net"], 916 | "description": "This is just a test.", 917 | "email": "test@example.net", 918 | "id": "012345678910", 919 | "name": "testaccount", 920 | "owner": "netflix", 921 | "provider": "aws", 922 | } 923 | 924 | # Test with invalid environment 925 | with pytest.raises(ValidationError): 926 | data['environment'] = 'bad_environment' 927 | swag.create(data) 928 | 929 | # Test with a valid environment 930 | data['environment'] = 'test' 931 | account = swag.create(data) 932 | assert account.get('environment') == 'test' 933 | 934 | def test_schema_context_validation_owner_field(): 935 | """Test schema context validation for owner field""" 936 | from swag_client.backend import SWAGManager 937 | from swag_client.util import parse_swag_config_options 938 | 939 | swag_opts = { 940 | 'swag.schema_context': { 941 | 'owner': ['netflix', 'dvd', 'aws', 'third-party'] 942 | } 943 | } 944 | swag = SWAGManager(**parse_swag_config_options(swag_opts)) 945 | 946 | data = { 947 | "aliases": ["test"], 948 | "contacts": ["admins@test.net"], 949 | "description": "This is just a test.", 950 | "email": "test@example.net", 951 | "id": "012345678910", 952 | "name": "testaccount", 953 | "environment": "test", 954 | "provider": "aws", 955 | } 956 | 957 | # Test with invalid owner 958 | with pytest.raises(ValidationError): 959 | data['owner'] = 'bad_owner' 960 | swag.create(data) 961 | 962 | # Test with a valid owner 963 | data['owner'] = 'netflix' 964 | account = swag.create(data) 965 | assert account.get('owner') == 'netflix' 966 | 967 | def test_schema_validation_account_status_field(s3_bucket_name): 968 | """Test schema context validation for owner field""" 969 | from swag_client.backend import SWAGManager 970 | from swag_client.util import parse_swag_config_options 971 | 972 | swag_opts = { 973 | 'swag.type': 's3', 974 | 'swag.bucket_name': s3_bucket_name, 975 | 'swag.schema_version': 2, 976 | 'swag.cache_expires': 0 977 | } 978 | swag = SWAGManager(**parse_swag_config_options(swag_opts)) 979 | 980 | data = { 981 | "aliases": ["test"], 982 | "contacts": ["admins@test.net"], 983 | "description": "This is just a test.", 984 | "email": "test@example.net", 985 | "id": "012345678910", 986 | "name": "testaccount", 987 | "environment": "test", 988 | "provider": "aws", 989 | "status": [ 990 | { 991 | "region": "us-west-2", 992 | "status": "created", 993 | "notes": [] 994 | } 995 | ], 996 | "account_status": "deleted" 997 | } 998 | 999 | # Test with invalid account_status 1000 | with pytest.raises(ValidationError): 1001 | swag.create(data) 1002 | 1003 | # Test with a valid account_status 1004 | data['account_status'] = 'created' 1005 | account = swag.create(data) 1006 | assert account.get('account_status') == 'created' 1007 | 1008 | def test_region_validation_field(s3_bucket_name): 1009 | """Test schema context validation for owner field""" 1010 | from swag_client.backend import SWAGManager 1011 | from swag_client.util import parse_swag_config_options 1012 | 1013 | swag_opts = { 1014 | 'swag.type': 's3', 1015 | 'swag.bucket_name': s3_bucket_name, 1016 | 'swag.schema_version': 2, 1017 | 'swag.cache_expires': 0 1018 | } 1019 | swag = SWAGManager(**parse_swag_config_options(swag_opts)) 1020 | 1021 | data = { 1022 | "aliases": ["test"], 1023 | "contacts": ["admins@test.net"], 1024 | "description": "This is just a test.", 1025 | "email": "test@example.net", 1026 | "id": "012345678910", 1027 | "name": "testaccount", 1028 | "environment": "test", 1029 | "provider": "aws", 1030 | "status": [ 1031 | { 1032 | "region": "us-west-2", 1033 | "status": "created", 1034 | "notes": [] 1035 | } 1036 | ], 1037 | "account_status": "created", 1038 | "regions": { 1039 | "us-east-1": { 1040 | "status": "created", 1041 | "az_mapping": [] 1042 | } 1043 | } 1044 | } 1045 | 1046 | # Test with invalid account_status 1047 | with pytest.raises(ValidationError): 1048 | swag.create(data) 1049 | 1050 | # Test with a valid account_status 1051 | data['regions']['us-east-1']['az_mapping'] = {} 1052 | account = swag.create(data) 1053 | assert account.get('account_status') == 'created' 1054 | 1055 | 1056 | -------------------------------------------------------------------------------- /swag_client/tests/vectors/invalid_accounts_v1.json: -------------------------------------------------------------------------------- 1 | { 2 | "data": [ 3 | { 4 | "account": "012345678910", 5 | "description": "Test account 1", 6 | "cloudtrail": "http://wouldnt/you/like/to/know", 7 | "cloudtrail_index": "cloudtrail_test1", 8 | "alias": [ 9 | "testaccnt1" 10 | ], 11 | "bastion": "test1.net", 12 | "owner": "someadmin@netflix.com", 13 | "type": "aws", 14 | "name": "test1@test" 15 | }, 16 | { 17 | "account": "109876543210", 18 | "description": "Test account 2", 19 | "cloudtrail": "http://wouldnt/you/like/to/know", 20 | "cloudtrail_index": "cloudtrail_test2", 21 | "alias": [ 22 | "testaccnt2" 23 | ], 24 | "bastion": "test2.net", 25 | "owner": "someadmin@netflix.com", 26 | "type": "aws", 27 | "name": "test2@test" 28 | } 29 | ] 30 | } 31 | -------------------------------------------------------------------------------- /swag_client/tests/vectors/invalid_accounts_v2.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Netflix-Skunkworks/swag-client/20f02f5547b6dd633806ec7d5f6dd7b22aad3760/swag_client/tests/vectors/invalid_accounts_v2.json -------------------------------------------------------------------------------- /swag_client/tests/vectors/valid_accounts_v1.json: -------------------------------------------------------------------------------- 1 | { 2 | "accounts": [ 3 | { 4 | "description": "Test account 1", 5 | "alias": [ 6 | "testaccnt1" 7 | ], 8 | "bastion": "test1.net", 9 | "owners": [ 10 | "someadmin@test.net" 11 | ], 12 | "type": "aws", 13 | "name": "test1@test", 14 | "cmc_required": false, 15 | "schema_version": 1, 16 | "ours": true, 17 | "service": { 18 | "myService": { 19 | "enabled": true 20 | }, 21 | "myService1": { 22 | "enabled": false 23 | } 24 | }, 25 | "metadata": { 26 | "s3_name": "testaccount1", 27 | "cloudtrail_index": "cloudtrail_testaccount1[yyyymm]", 28 | "cloudtrail_kibana_url": "http://wouldnt/you/like/to/know", 29 | "email": "testaccount1@test.net", 30 | "account_number": "012345678910" 31 | }, 32 | "id": "aws-012345678910" 33 | }, 34 | { 35 | "description": "Test account 2", 36 | "alias": [ 37 | "testaccnt2" 38 | ], 39 | "bastion": "test2.net", 40 | "owners": [ 41 | "someadmin@netflix.com" 42 | ], 43 | "type": "aws", 44 | "name": "test2@test", 45 | "cmc_required": false, 46 | "schema_version": 1, 47 | "ours": true, 48 | "service": { 49 | "myService": { 50 | "enabled": false 51 | }, 52 | "myService1": { 53 | "enabled": true 54 | } 55 | }, 56 | "metadata": { 57 | "s3_name": "testaccount2", 58 | "cloudtrail_index": "cloudtrail_testaccount2[yyyymm]", 59 | "cloudtrail_kibana_url": "http://wouldnt/you/like/to/know2", 60 | "email": "testaccount2@test.net", 61 | "account_number": "109876543210" 62 | }, 63 | "id": "aws-109876543210" 64 | } 65 | ] 66 | } 67 | -------------------------------------------------------------------------------- /swag_client/tests/vectors/valid_accounts_v2.json: -------------------------------------------------------------------------------- 1 | [{ 2 | "aliases": ["test"], 3 | "contacts": ["admins@test.net"], 4 | "description": "LOL, Test account", 5 | "email": "testaccount@test.net", 6 | "environment": "test", 7 | "id": "012345678910", 8 | "name": "testaccount", 9 | "owner": "netflix", 10 | "provider": "aws", 11 | "sensitive": false, 12 | "services": [ 13 | { 14 | "name": "myService", 15 | "metadata": { 16 | "name": "testaccount" 17 | }, 18 | "status": [ 19 | { 20 | "enabled": true, 21 | "region": "all" 22 | } 23 | ] 24 | }, 25 | { 26 | "name": "myService1", 27 | "metadata": {}, 28 | "status": [ 29 | { 30 | "enabled": false, 31 | "region": "all" 32 | } 33 | ] 34 | }, 35 | { 36 | "name": "myService2", 37 | "metadata": {}, 38 | "status": [ 39 | { 40 | "enabled": true, 41 | "region": "us-east-1" 42 | } 43 | ] 44 | } 45 | ] 46 | }, { 47 | "aliases": ["test2"], 48 | "contacts": ["admins@test2.net"], 49 | "description": "LOL, Test account", 50 | "email": "test2account@test.net", 51 | "environment": "test", 52 | "id": "0123452323", 53 | "name": "testaccount2", 54 | "owner": "netflix", 55 | "provider": "aws", 56 | "sensitive": false, 57 | "services": [ 58 | { 59 | "name": "s3", 60 | "metadata": { 61 | "name": "testaccount2" 62 | }, 63 | "status": [ 64 | { 65 | "enabled": true, 66 | "region": "all" 67 | } 68 | ] 69 | }, 70 | { 71 | "name": "cloudtrail", 72 | "metadata": { 73 | "kibanaUrl": "http://testaccount.cloudtrail.dashboard.net", 74 | "esIndex": "cloudtrail_testaccount[yyyymm]" 75 | }, 76 | "status": [ 77 | { 78 | "enabled": true, 79 | "region": "all" 80 | } 81 | ] 82 | } 83 | ] 84 | }] 85 | -------------------------------------------------------------------------------- /swag_client/util.py: -------------------------------------------------------------------------------- 1 | import os 2 | import warnings 3 | import jmespath 4 | 5 | from marshmallow import Schema, fields 6 | from marshmallow.validate import OneOf 7 | 8 | 9 | class OptionsSchema(Schema): 10 | type = fields.String(missing='file', validate=OneOf(['file', 's3', 'dynamodb'])) 11 | namespace = fields.String(missing='accounts') 12 | schema_version = fields.Integer(missing=2) # default version to return data as 13 | cache_expires = fields.Integer(missing=60) 14 | schema_context = fields.Dict(missing={}) 15 | 16 | 17 | class FileOptionsSchema(OptionsSchema): 18 | """Option schema for the file backend.""" 19 | data_dir = fields.String(missing=os.getcwd()) 20 | data_file = fields.String() 21 | 22 | 23 | class S3OptionsSchema(OptionsSchema): 24 | """Option schema for the S3 backend.""" 25 | bucket_name = fields.String(required=True) 26 | data_file = fields.String() 27 | region = fields.String(missing='us-east-1', validate=OneOf(['us-east-1', 'us-west-2', 'eu-west-1'])) 28 | 29 | 30 | class DynamoDBOptionsSchema(OptionsSchema): 31 | """Option schema for the DynamoDB backend.""" 32 | key_attribute = fields.String(missing='id') 33 | key_type = fields.String(missing='HASH') 34 | read_units = fields.Integer(missing=1) 35 | write_units = fields.Integer(missing=1) 36 | region = fields.String(missing='us-east-1', validate=OneOf(['us-east-1', 'us-west-2', 'eu-west-1'])) 37 | 38 | 39 | def parse_swag_config_options(config): 40 | """Ensures that options passed to the backend are valid.""" 41 | options = {} 42 | for key, val in config.items(): 43 | if key.startswith('swag.backend.'): 44 | options[key[12:]] = val 45 | if key.startswith('swag.'): 46 | options[key[5:]] = val 47 | 48 | if options.get('type') == 's3': 49 | return S3OptionsSchema().load(options) 50 | elif options.get('type') == 'dynamodb': 51 | return DynamoDBOptionsSchema().load(options) 52 | else: 53 | return FileOptionsSchema().load(options) 54 | 55 | 56 | def deprecated(message): 57 | """Deprecated function decorator.""" 58 | def wrapper(fn): 59 | def deprecated_method(*args, **kargs): 60 | warnings.warn(message, DeprecationWarning, 2) 61 | return fn(*args, **kargs) 62 | # TODO: use decorator ? functools.wrapper ? 63 | deprecated_method.__name__ = fn.__name__ 64 | deprecated_method.__doc__ = "%s\n\n%s" % (message, fn.__doc__) 65 | return deprecated_method 66 | return wrapper 67 | 68 | 69 | def append_item(namespace, version, item, items): 70 | if version == 1: 71 | if items: 72 | items[namespace].append(item) 73 | else: 74 | items = {namespace: [item]} 75 | 76 | else: 77 | if items: 78 | items.append(item) 79 | else: 80 | items = [item] 81 | 82 | return items 83 | 84 | 85 | def remove_item(namespace, version, item, items): 86 | if version == 1: 87 | # NOTE only supports aws providers 88 | path = "{namespace}[?id!='{id}']".format(id=item['id'], namespace=namespace) 89 | return jmespath.search(path, items) 90 | else: 91 | return jmespath.search("[?id!='{id}']".format(id=item['id']), items) 92 | 93 | 94 | def is_sub_dict(sub_dict, dictionary): 95 | """Legacy filter for determining if a given dict is present.""" 96 | for key in sub_dict.keys(): 97 | if key not in dictionary: 98 | return False 99 | if (type(sub_dict[key]) is not dict) and (sub_dict[key] != dictionary[key]): 100 | return False 101 | if (type(sub_dict[key]) is dict) and (not is_sub_dict(sub_dict[key], dictionary[key])): 102 | return False 103 | return True 104 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py27,py36 3 | 4 | [testenv] 5 | deps=pytest 6 | moto 7 | commands=pytest --------------------------------------------------------------------------------