├── .bumpversion.cfg ├── .gitignore ├── .pre-commit-config.yaml ├── .pyup.yml ├── .readthedocs.yml ├── .travis.yml ├── AUTHORS ├── CHANGELOG.rst ├── LICENSE ├── MANIFEST.in ├── OSSMETADATA ├── README.rst ├── analysis-localhost.json ├── baseline-localhost.json ├── config.yml ├── dev-requirements.in ├── dev-requirements.txt ├── diffy ├── __init__.py ├── _version.py ├── accounts.py ├── common │ ├── __init__.py │ ├── managers.py │ └── utils.py ├── config.py ├── core.py ├── exceptions.py ├── extensions.py ├── filters.py ├── plugins │ ├── __init__.py │ ├── base │ │ ├── __init__.py │ │ ├── manager.py │ │ └── v1.py │ ├── bases │ │ ├── __init__.py │ │ ├── analysis.py │ │ ├── collection.py │ │ ├── inventory.py │ │ ├── payload.py │ │ ├── persistence.py │ │ └── target.py │ ├── diffy_aws │ │ ├── __init__.py │ │ ├── auto_scaling.py │ │ ├── plugin.py │ │ ├── s3.py │ │ ├── ssm.py │ │ ├── sts.py │ │ └── tests │ │ │ ├── __init__.py │ │ │ └── conftest.py │ ├── diffy_local │ │ ├── __init__.py │ │ ├── plugin.py │ │ └── tests │ │ │ ├── __init__.py │ │ │ └── conftest.py │ └── diffy_osquery │ │ ├── __init__.py │ │ ├── plugin.py │ │ └── tests │ │ ├── __init__.py │ │ └── conftest.py └── schema.py ├── diffy_api ├── __init__.py ├── analysis │ └── views.py ├── baseline │ └── views.py ├── common │ ├── __init__.py │ ├── health.py │ └── util.py ├── core.py ├── extensions.py ├── factory.py ├── plugins │ ├── __init__.py │ └── views.py ├── schemas.py └── tasks │ ├── __init__.py │ └── views.py ├── diffy_cli ├── __init__.py ├── core.py └── utils │ ├── __init__.py │ ├── dynamic_click.py │ └── json_schema.py ├── docs ├── Makefile ├── changelog.rst ├── conf.py ├── developer │ ├── index.rst │ └── plugins │ │ └── index.rst ├── doing-a-release.rst ├── faq.rst ├── images │ ├── diffy.png │ └── diffy_small.png ├── index.rst ├── internals │ └── index.rst.py ├── license │ └── index.rst ├── production │ └── index.rst ├── quickstart │ └── index.rst ├── requirements.txt └── security.rst ├── localhost-localhost.json ├── requirements.in ├── requirements.txt ├── setup.cfg ├── setup.py ├── tests ├── __init__.py ├── conf.py ├── config.yml ├── conftest.py ├── test_analysis.py └── test_baseline.py ├── web-requirements.in └── web-requirements.txt /.bumpversion.cfg: -------------------------------------------------------------------------------- 1 | [bumpversion] 2 | current_version = 0.1.0 3 | commit = True 4 | tag = True 5 | 6 | [bumpversion:file:diffy/_version.py] 7 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # Created by https://www.gitignore.io/api/vim,python 3 | 4 | ### Python ### 5 | # Byte-compiled / optimized / DLL files 6 | __pycache__/ 7 | *.py[cod] 8 | *$py.class 9 | 10 | # C extensions 11 | *.so 12 | 13 | # Distribution / packaging 14 | .Python 15 | build/ 16 | develop-eggs/ 17 | dist/ 18 | downloads/ 19 | eggs/ 20 | .eggs/ 21 | lib/ 22 | lib64/ 23 | parts/ 24 | sdist/ 25 | var/ 26 | wheels/ 27 | *.egg-info/ 28 | .installed.cfg 29 | *.egg 30 | 31 | # PyInstaller 32 | # Usually these files are written by a python script from a template 33 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 34 | *.manifest 35 | *.spec 36 | 37 | # Installer logs 38 | pip-log.txt 39 | pip-delete-this-directory.txt 40 | 41 | # Unit test / coverage reports 42 | htmlcov/ 43 | .tox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | .pytest_cache/ 48 | nosetests.xml 49 | coverage.xml 50 | *.cover 51 | .hypothesis/ 52 | 53 | # Translations 54 | *.mo 55 | *.pot 56 | 57 | # Flask stuff: 58 | instance/ 59 | .webassets-cache 60 | 61 | # Scrapy stuff: 62 | .scrapy 63 | 64 | # Sphinx documentation 65 | docs/_build/ 66 | 67 | # Jupyter Notebook 68 | .ipynb_checkpoints 69 | 70 | # pyenv 71 | .python-version 72 | 73 | # celery beat schedule file 74 | celerybeat-schedule.* 75 | 76 | # SageMath parsed files 77 | *.sage.py 78 | 79 | # Environments 80 | .env 81 | .venv 82 | env/ 83 | venv/ 84 | ENV/ 85 | env.bak/ 86 | venv.bak/ 87 | 88 | # Spyder project settings 89 | .spyderproject 90 | .spyproject 91 | 92 | # Rope project settings 93 | .ropeproject 94 | 95 | # mkdocs documentation 96 | /site 97 | 98 | # mypy 99 | .mypy_cache/ 100 | 101 | ### Vim ### 102 | # swap 103 | .sw[a-p] 104 | .*.sw[a-p] 105 | # session 106 | Session.vim 107 | # temporary 108 | .netrwhist 109 | *~ 110 | # auto-generated tag files 111 | tags 112 | 113 | .idea/* 114 | .DS_Store 115 | *.log 116 | 117 | 118 | # End of https://www.gitignore.io/api/vim,python 119 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: git://github.com/pre-commit/pre-commit-hooks 3 | rev: v1.4.0 4 | hooks: 5 | - id: check-merge-conflict 6 | - id: detect-private-key 7 | - repo: https://github.com/ambv/black 8 | rev: stable 9 | hooks: 10 | - id: black 11 | language_version: python3.6 12 | -------------------------------------------------------------------------------- /.pyup.yml: -------------------------------------------------------------------------------- 1 | update: insecure 2 | label_prs: update 3 | -------------------------------------------------------------------------------- /.readthedocs.yml: -------------------------------------------------------------------------------- 1 | build: 2 | image: latest 3 | 4 | python: 5 | version: 3.6 6 | pip_install: true 7 | extra_requirements: 8 | - dev 9 | 10 | requirements_file: requirements.txt -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | dist: bionic 3 | 4 | matrix: 5 | include: 6 | - python: "3.6" 7 | 8 | cache: pip 9 | 10 | before_install: 11 | - sudo rm -f /etc/boto.cfg 12 | 13 | #install: 14 | #- pip install -e git+git://github.com/forestmonster/moto@1.3.5#egg=moto-1.3.5 15 | 16 | before_script: 17 | #- pip install --upgrade pip 18 | #- pip install --upgrade setuptools 19 | - pip install -e ".[dev]" 20 | 21 | script: 22 | - coverage run -m py.test || exit 1 23 | # - mypy -p diffy --ignore-missing-imports || exit 1 24 | - bandit -r . -ll -ii -x tests/,docs 25 | 26 | after_success: 27 | - coveralls 28 | 29 | notifications: 30 | email: 31 | - fmonsen@netflix.com 32 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Netflix-Skunkworks/diffy/1d8c1093023832be40d04021906f8779b37e8521/AUTHORS -------------------------------------------------------------------------------- /CHANGELOG.rst: -------------------------------------------------------------------------------- 1 | Changelog 2 | ========= 3 | 4 | 5 | 0.1.0 - `master` 6 | ~~~~~~~~~~~~~~~~ 7 | 8 | .. note:: This version is not yet released and is under active development 9 | 10 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2018 Netflix, Inc. 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | include README.rst 3 | include requirements.txt 4 | include dev-requirements.txt 5 | recursive-exclude tests * -------------------------------------------------------------------------------- /OSSMETADATA: -------------------------------------------------------------------------------- 1 | osslifecycle=active 2 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | Diffy (DEPRECATED) 2 | ===== 3 | 4 | **Diffy has been deprecated** at Netflix. This software is no longer maintained or supported. 5 | 6 | .. image:: docs/images/diffy_small.png 7 | :align: right 8 | 9 | .. image:: http://unmaintained.tech/badge.svg 10 | :target: http://unmaintained.tech 11 | :alt: No Maintenance Intended 12 | 13 | .. image:: https://travis-ci.org/Netflix-Skunkworks/diffy.svg?branch=master 14 | :target: https://travis-ci.org/Netflix-Skunkworks/diffy 15 | 16 | .. image:: https://img.shields.io/gitter/room/nwjs/nw.js.svg?style=flat-square 17 | :target: https://gitter.im/diffy/diffy 18 | 19 | .. image:: https://img.shields.io/pypi/v/diffy.svg?style=flat-square 20 | :target: https://pypi.python.org/pypi/diffy 21 | :alt: PyPi version 22 | 23 | .. image:: https://img.shields.io/pypi/pyversions/diffy.svg?style=flat-square 24 | :target: https://pypi.org/project/diffy 25 | :alt: Supported Python versions 26 | 27 | .. image:: https://img.shields.io/pypi/l/diffy.svg?style=flat-square 28 | :target: https://choosealicense.com/licenses 29 | :alt: License 30 | 31 | .. image:: https://img.shields.io/pypi/status/diffy.svg?style=flat-square 32 | :target: https://pypi.python.org/pypi/diffy 33 | :alt: Status 34 | 35 | .. image:: https://img.shields.io/readthedocs/diffy.svg?style=flat-square 36 | :target: https://readthedocs.org/projects/diffy/badge/?version=latest 37 | :alt: RTD 38 | 39 | 40 | Diffy is a digital forensics and incident response (DFIR) tool that was developed by 41 | Netflix's Security Intelligence and Response Team (SIRT). 42 | 43 | Diffy allows a forensic investigator to quickly scope a compromise across cloud 44 | instances during an incident, and triage those instances for followup actions. 45 | Diffy is currently focused on Linux instances running within Amazon Web 46 | Services (AWS), but owing to our plugin structure, could support multiple 47 | platforms and cloud providers. 48 | 49 | It's called "Diffy" because it helps a human investigator to identify the 50 | *differences* between instances, and because `Alex`_ pointed out that "The 51 | Difforensicator" was unnecessarily tricky. 52 | 53 | See `Releases`_ for recent changes. See `our Read the Docs site`_ for 54 | well-formatted documentation. 55 | 56 | .. _Alex: https://www.linkedin.com/in/maestretti/ 57 | .. _Releases: https://github.com/Netflix-Skunkworks/diffy/releases 58 | .. _our Read the Docs site: http://diffy.readthedocs.io/ 59 | 60 | Supported Technologies 61 | ---------------------- 62 | 63 | - AWS (AWS Systems Manager / SSM) 64 | - Local 65 | - osquery 66 | 67 | Each technology has its own plugins for targeting, collection and persistence. 68 | 69 | 70 | Features 71 | -------- 72 | 73 | - Efficiently highlights outliers in security-relevant instance behavior. For 74 | example, you can use Diffy to tell you which of your instances are listening 75 | on an unexpected port, are running an unusual process, include a strange 76 | crontab entry, or have inserted a surprising kernel module. 77 | - Uses one, or both, of two methods to highlight differences: 78 | 79 | - Collection of a "functional" baseline from a "clean" running instance, 80 | against which your instance group is compared, and 81 | - Collection of a "clustered" baseline, in which all instances are surveyed, 82 | and outliers are made obvious. 83 | 84 | - Uses a modular plugin-based architecture. We currently include plugins for 85 | collection using osquery via AWS Systems Manager (formerly known as Simple 86 | Systems Manager or SSM). 87 | 88 | 89 | Installation 90 | ------------ 91 | 92 | Via pip:: 93 | 94 | pip install diffy 95 | 96 | 97 | Roadmap 98 | ------- 99 | 100 | **Diffy has been deprecated at Netflix.** This software is no longer maintained or supported. 101 | 102 | 103 | Why python 3 only? 104 | ~~~~~~~~~~~~~~~~~~ 105 | 106 | Please see `Guido's guidance 107 | `_ 108 | regarding the Python 2.7 end of life date. 109 | -------------------------------------------------------------------------------- /analysis-localhost.json: -------------------------------------------------------------------------------- 1 | [{"instance_id": "localhost", "status": "success", "collected_at": "2018-11-30 23:17:37", "stdout": [{"address": "0.0.0.0", "cmdline": "com.docker.vpnkit --ethernet fd:3 --port fd:4 --diagnostics fd:5 --pcap fd:6 --vsock-path vms/0/connect --host-names host.docker.internal,docker.for.mac.host.internal,docker.for.mac.localhost --gateway-names gateway.docker.internal,docker.for.mac.gateway.internal,docker.for.mac.http.internal --vm-names docker-for-desktop --listen-backlog 32 --mtu 1500 --allowed-bind-addresses 0.0.0.0 --http /Users/kglisson/Library/Group Containers/group.com.docker/http_proxy.json --dhcp /Users/kglisson/Library/Group Containers/group.com.docker/dhcp.json --port-max-idle-time 300 --max-connections 2000 --gateway-ip 192.168.65.1 --host-ip 192.168.65.2 --lowest-ip 192.168.65.3 --highest-ip 192.168.65.254 --log-destination asl --udpv4-forwards 123:127.0.0.1:53020 --gc-compact-interval 1800", "name": "com.docker.vpnkit", "pid": "2219", "port": "6379"}, {"address": "0.0.0.0", "cmdline": "/Applications/Dropbox.app/Contents/MacOS/Dropbox /firstrunupdate 1723", "name": "Dropbox", "pid": "70500", "port": "17500"}], "diff": [{"address": "0.0.0.0", "cmdline": "com.docker.vpnkit --ethernet fd:3 --port fd:4 --diagnostics fd:5 --pcap fd:6 --vsock-path vms/0/connect --host-names host.docker.internal,docker.for.mac.host.internal,docker.for.mac.localhost --gateway-names gateway.docker.internal,docker.for.mac.gateway.internal,docker.for.mac.http.internal --vm-names docker-for-desktop --listen-backlog 32 --mtu 1500 --allowed-bind-addresses 0.0.0.0 --http /Users/kglisson/Library/Group Containers/group.com.docker/http_proxy.json --dhcp /Users/kglisson/Library/Group Containers/group.com.docker/dhcp.json --port-max-idle-time 300 --max-connections 2000 --gateway-ip 192.168.65.1 --host-ip 192.168.65.2 --lowest-ip 192.168.65.3 --highest-ip 192.168.65.254 --log-destination asl --udpv4-forwards 123:127.0.0.1:53020 --gc-compact-interval 1800", "name": "com.docker.vpnkit", "pid": "2219", "port": "6379"}, {"address": "0.0.0.0", "cmdline": "/Applications/Dropbox.app/Contents/MacOS/Dropbox /firstrunupdate 1723", "name": "Dropbox", "pid": "70500", "port": "17500"}]}, {"instance_id": "localhost", "status": "success", "collected_at": "2018-11-30 23:17:37", "stdout": [], "diff": {}}] -------------------------------------------------------------------------------- /baseline-localhost.json: -------------------------------------------------------------------------------- 1 | {"instance_id": "localhost", "status": "success", "collected_at": "2018-11-30 23:15:31", "stdout": []} -------------------------------------------------------------------------------- /config.yml: -------------------------------------------------------------------------------- 1 | # General Diffy configuration 2 | DIFFY_SWAG_ENABLED: True 3 | SWAG_BUCKET_NAME: example 4 | 5 | 6 | # Plugin specific options 7 | 8 | # AWS 9 | DIFFY_AWS_S3_BUCKET: bucket-blah 10 | DIFFY_AWS_SSM_IAM_ROLE: DiffyRole -------------------------------------------------------------------------------- /dev-requirements.in: -------------------------------------------------------------------------------- 1 | -r requirements.txt 2 | -r web-requirements.txt 3 | bandit 4 | bumpversion 5 | codecov 6 | moto 7 | mypy 8 | pip-tools 9 | pre-commit 10 | pre-commit-hooks 11 | pytest 12 | pytest-cov 13 | pytest-flask 14 | Sphinx 15 | sphinx-autodoc-annotation 16 | -------------------------------------------------------------------------------- /dev-requirements.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile 3 | # To update, run: 4 | # 5 | # pip-compile --index-url=https://pypi.org/simple dev-requirements.in 6 | # 7 | alabaster==0.7.12 # via sphinx 8 | aniso8601==6.0.0 9 | aspy.yaml==1.3.0 # via pre-commit 10 | attrs==19.1.0 11 | aws-xray-sdk==0.95 12 | babel==2.8.0 # via sphinx 13 | bandit==1.6.2 14 | blinker==1.4 15 | boto3==1.9.131 16 | boto==2.49.0 17 | botocore==1.12.131 18 | bumpversion==0.5.3 19 | certifi==2019.11.28 # via requests 20 | cffi==1.13.2 # via cryptography 21 | cfgv==2.0.1 # via pre-commit 22 | chardet==3.0.4 # via requests 23 | click-log==0.3.2 24 | click==7.0 25 | codecov==2.0.15 26 | cookies==2.2.1 27 | coverage==5.0.1 # via codecov, pytest-cov 28 | croniter==0.3.29 29 | cryptography==2.8 30 | decorator==4.4.0 31 | deepdiff==4.0.6 32 | docker==4.1.0 33 | docutils==0.14 34 | dogpile.cache==0.7.1 35 | ecdsa==0.14.1 # via python-jose 36 | entrypoints==0.3 # via flake8 37 | flake8==3.7.9 # via pre-commit-hooks 38 | flask-restful==0.3.7 39 | flask-rq2==18.3 40 | flask==1.0.2 41 | future==0.18.2 # via python-jose 42 | fuzzywuzzy==0.17.0 43 | gitdb2==2.0.6 # via gitpython 44 | gitpython==3.0.5 # via bandit 45 | gunicorn==19.9.0 46 | identify==1.4.9 # via pre-commit 47 | idna==2.8 # via requests 48 | imagesize==1.2.0 # via sphinx 49 | importlib-metadata==1.3.0 # via pluggy, pre-commit, pytest 50 | importlib-resources==1.0.2 # via pre-commit 51 | inflection==0.3.1 52 | itsdangerous==1.1.0 53 | jinja2==2.10.1 54 | jmespath==0.9.4 55 | jsondiff==1.1.2 56 | jsonpickle==1.1 57 | jsonschema==3.0.1 58 | markupsafe==1.1.1 59 | marshmallow-jsonschema==0.5.0 60 | marshmallow==2.19.2 61 | mccabe==0.6.1 # via flake8 62 | mock==3.0.5 63 | more-itertools==8.0.2 # via pytest, zipp 64 | mypy-extensions==0.4.3 # via mypy 65 | mypy==0.761 66 | nodeenv==1.3.3 # via pre-commit 67 | ordered-set==3.1 68 | packaging==19.2 # via pytest, sphinx 69 | pbr==5.4.4 # via stevedore 70 | pip-tools==4.3.0 71 | pluggy==0.13.1 # via pytest 72 | pre-commit-hooks==2.4.0 73 | pre-commit==1.20.0 74 | py==1.8.1 # via pytest 75 | pyaml==19.12.0 76 | pycodestyle==2.5.0 # via flake8 77 | pycparser==2.19 # via cffi 78 | pycryptodome==3.9.4 # via python-jose 79 | pyflakes==2.1.1 # via flake8 80 | pygments==2.5.2 # via sphinx 81 | pyparsing==2.4.6 # via packaging 82 | pyrsistent==0.14.11 83 | pytest-cov==2.8.1 84 | pytest-flask==0.15.0 85 | pytest==5.3.2 86 | python-dateutil==2.8.0 87 | python-jose==2.0.2 88 | python-levenshtein==0.12.0 89 | pytz==2019.1 90 | pyyaml==5.1 91 | raven[flask]==6.10.0 92 | redis==3.2.1 93 | requests==2.22.0 # via aws-xray-sdk, codecov, docker, responses, sphinx 94 | responses==0.10.9 95 | retrying==1.3.3 96 | rq-scheduler==0.9 97 | rq==1.0 98 | ruamel.yaml.clib==0.2.0 # via ruamel.yaml 99 | ruamel.yaml==0.16.5 # via pre-commit-hooks 100 | s3transfer==0.2.0 101 | simplejson==3.16.0 102 | six==1.12.0 103 | smmap2==2.0.5 # via gitdb2 104 | snowballstemmer==2.0.0 # via sphinx 105 | sphinx-autodoc-annotation==1.0.post1 106 | sphinx==2.3.1 107 | sphinxcontrib-applehelp==1.0.1 # via sphinx 108 | sphinxcontrib-devhelp==1.0.1 # via sphinx 109 | sphinxcontrib-htmlhelp==1.0.2 # via sphinx 110 | sphinxcontrib-jsmath==1.0.1 # via sphinx 111 | sphinxcontrib-qthelp==1.0.2 # via sphinx 112 | sphinxcontrib-serializinghtml==1.1.3 # via sphinx 113 | stevedore==1.31.0 # via bandit 114 | swag-client==0.4.6 115 | tabulate==0.8.3 116 | toml==0.10.0 # via pre-commit, pre-commit-hooks 117 | typed-ast==1.4.0 # via mypy 118 | typing-extensions==3.7.4.1 # via mypy 119 | urllib3==1.24.1 120 | virtualenv==16.7.9 # via pre-commit 121 | wcwidth==0.1.7 # via pytest 122 | websocket-client==0.57.0 # via docker 123 | werkzeug==0.15.2 124 | wrapt==1.11.2 # via aws-xray-sdk 125 | xmltodict==0.12.0 126 | zipp==0.6.0 # via importlib-metadata 127 | -------------------------------------------------------------------------------- /diffy/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Netflix-Skunkworks/diffy/1d8c1093023832be40d04021906f8779b37e8521/diffy/__init__.py -------------------------------------------------------------------------------- /diffy/_version.py: -------------------------------------------------------------------------------- 1 | __version__ = "0.1.0" 2 | -------------------------------------------------------------------------------- /diffy/accounts.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: diffy.accounts 3 | :platform: Unix 4 | :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | """ 7 | import logging 8 | from rapidfuzz import process 9 | 10 | from honeybee.extensions import swag 11 | from honeybee.exceptions import ResolveException 12 | 13 | log = logging.getLogger(__name__) 14 | 15 | 16 | def get_fuzzy_accounts(identifier, accounts): 17 | """Attempts to fuzz identifier and provides suggestions. 18 | 19 | :param identifier: identifier to fuzz 20 | :param accounts: list of accounts to compare 21 | :return: tuple. Possible accounts that could match. 22 | """ 23 | # collect all possibilities 24 | choices = [] 25 | for a in accounts: 26 | choices.append(a["name"]) 27 | choices += a["aliases"] 28 | 29 | return process.extract(identifier, choices, limit=3) 30 | 31 | 32 | def valid_account(identifier): 33 | """Validates account identifier. 34 | 35 | :param identifier: identifier to validate 36 | :return: bool. ``True`` 37 | """ 38 | account = get_account_id(identifier) 39 | if account: 40 | return True 41 | 42 | 43 | def get_account_name(identifier): 44 | """Fetches account name from SWAG. 45 | 46 | :param identifier: identifier to fetch 47 | """ 48 | log.debug(f"Fetching account information. Name: {identifier}") 49 | account_data = swag.get(f"[?id=='{identifier}']") 50 | 51 | if not account_data: 52 | raise ResolveException( 53 | f"Unable to find any account information. Identifier: {identifier}" 54 | ) 55 | 56 | return account_data["name"] 57 | 58 | 59 | def get_account_id(identifier): 60 | """Fetches account id from SWAG. 61 | 62 | :param identifier: identifier to fetch 63 | """ 64 | log.debug(f"Fetching account information. Identifier: {identifier}") 65 | account_data = swag.get(f"[?id=='{identifier}']") 66 | 67 | if not account_data: 68 | account_data = swag.get_by_name(identifier, alias=True) 69 | 70 | if not account_data: 71 | raise ResolveException( 72 | f"Unable to find any account information. Identifier: {identifier}" 73 | ) 74 | 75 | if len(account_data) > 1: 76 | raise ResolveException( 77 | f"Unable to resolve to a single account. Accounts: {len(account_data)} Identifier: {identifier}" 78 | ) 79 | 80 | return account_data[0]["id"] 81 | return account_data["id"] 82 | -------------------------------------------------------------------------------- /diffy/common/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Netflix-Skunkworks/diffy/1d8c1093023832be40d04021906f8779b37e8521/diffy/common/__init__.py -------------------------------------------------------------------------------- /diffy/common/managers.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: diffy.common.managers 3 | :platform: Unix 4 | :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | 7 | .. moduleauthor:: Kevin Glisson 8 | """ 9 | import logging 10 | from diffy.exceptions import InvalidConfiguration 11 | 12 | logger = logging.getLogger(__name__) 13 | 14 | 15 | # inspired by https://github.com/getsentry/sentry 16 | class InstanceManager(object): 17 | def __init__(self, class_list=None, instances=True): 18 | if class_list is None: 19 | class_list = [] 20 | self.instances = instances 21 | self.update(class_list) 22 | 23 | def get_class_list(self): 24 | return self.class_list 25 | 26 | def add(self, class_path): 27 | self.cache = None 28 | if class_path not in self.class_list: 29 | self.class_list.append(class_path) 30 | 31 | def remove(self, class_path): 32 | self.cache = None 33 | self.class_list.remove(class_path) 34 | 35 | def update(self, class_list): 36 | """ 37 | Updates the class list and wipes the cache. 38 | """ 39 | self.cache = None 40 | self.class_list = class_list 41 | 42 | def all(self): 43 | """ 44 | Returns a list of cached instances. 45 | """ 46 | class_list = list(self.get_class_list()) 47 | if not class_list: 48 | self.cache = [] 49 | return [] 50 | 51 | if self.cache is not None: 52 | return self.cache 53 | 54 | results = [] 55 | for cls_path in class_list: 56 | module_name, class_name = cls_path.rsplit(".", 1) 57 | try: 58 | module = __import__(module_name, {}, {}, class_name) 59 | cls = getattr(module, class_name) 60 | if self.instances: 61 | results.append(cls()) 62 | else: 63 | results.append(cls) 64 | 65 | except InvalidConfiguration as e: 66 | logger.warning(f"Plugin '{class_name}' may not work correctly. {e}") 67 | 68 | except Exception as e: 69 | logger.exception(f"Unable to import {cls_path}. Reason: {e}") 70 | continue 71 | 72 | self.cache = results 73 | 74 | return results 75 | -------------------------------------------------------------------------------- /diffy/common/utils.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: diffy.common.utils 3 | :platform: Unix 4 | :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | .. moduleauthor:: Forest Monsen 7 | .. moduleauthor:: Kevin Glisson 8 | """ 9 | import logging 10 | import pkg_resources 11 | 12 | 13 | logger = logging.getLogger(__name__) 14 | 15 | 16 | def chunk(l, n): 17 | """Chunk a list to sublists.""" 18 | for i in range(0, len(l), n): 19 | yield l[i : i + n] 20 | 21 | 22 | def install_plugins(): 23 | """ 24 | Installs plugins associated with diffy 25 | :return: 26 | """ 27 | from diffy.plugins.base import register 28 | 29 | # entry_points={ 30 | # 'diffy.plugins': [ 31 | # 'ssm = diffy_aws.plugin:SSMCollectionPlugin' 32 | # ], 33 | # }, 34 | for ep in pkg_resources.iter_entry_points("diffy.plugins"): 35 | logger.info(f"Loading plugin {ep.name}") 36 | try: 37 | plugin = ep.load() 38 | except Exception: 39 | import traceback 40 | 41 | logger.error(f"Failed to load plugin {ep.name}:{traceback.format_exc()}") 42 | else: 43 | register(plugin) 44 | -------------------------------------------------------------------------------- /diffy/config.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: diffy.config 3 | :platform: Unix 4 | :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | """ 7 | import os 8 | import errno 9 | import yaml 10 | import logging 11 | from typing import Union, Any, Dict, Iterable 12 | 13 | from pathlib import Path 14 | 15 | from swag_client.util import parse_swag_config_options 16 | 17 | from diffy.extensions import swag 18 | 19 | from distutils.util import strtobool 20 | 21 | logger = logging.getLogger(__name__) 22 | 23 | 24 | AVAILABLE_REGIONS = ["us-east-1", "us-west-2", "eu-west-1"] 25 | 26 | 27 | def configure_swag() -> None: 28 | """Configures SWAG if enabled.""" 29 | if CONFIG["DIFFY_SWAG_ENABLED"]: 30 | swag_config = CONFIG.get_namespace("SWAG_") 31 | logger.debug(str(swag_config)) 32 | swag_config = {"swag." + k: v for k, v in swag_config.items()} 33 | swag.configure(**parse_swag_config_options(swag_config)) 34 | CONFIG["DIFFY_ACCOUNTS"] = swag.get_service_enabled("diffy") 35 | 36 | 37 | def valid_region(region) -> bool: 38 | if region in CONFIG["DIFFY_REGIONS"]: 39 | return True 40 | return False 41 | 42 | 43 | def consume_envvars(defaults: dict) -> dict: 44 | for k, v in defaults.items(): 45 | v = os.environ.get(k, v) 46 | if isinstance(v, str): 47 | try: 48 | v = bool(strtobool(v)) 49 | except ValueError: 50 | pass 51 | defaults[k] = v 52 | return defaults 53 | 54 | 55 | class ConfigAttribute(object): 56 | """Makes an attribute forward to the config""" 57 | 58 | def __init__(self, name, get_converter=None): 59 | self.__name__ = name 60 | self.get_converter = get_converter 61 | 62 | def __get__(self, obj, type=None): 63 | if obj is None: 64 | return self 65 | rv = obj.config[self.__name__] 66 | if self.get_converter is not None: 67 | rv = self.get_converter(rv) 68 | return rv 69 | 70 | def __set__(self, obj, value): 71 | obj.config[self.__name__] = value 72 | 73 | 74 | class Config(dict): 75 | def __init__(self, root_path: str = None, defaults: dict = None) -> None: 76 | dict.__init__(self, defaults or {}) 77 | self.root_path = root_path or os.getcwd() 78 | super().__init__() 79 | 80 | def from_envvar(self, variable_name: str, silent: bool = False) -> bool: 81 | """Loads a configuration from an environment variable pointing to 82 | a configuration file. This is basically just a shortcut with nicer 83 | error messages for this line of code:: 84 | 85 | config.from_yaml(os.environ['YOURAPPLICATION_SETTINGS']) 86 | 87 | :param variable_name: name of the environment variable 88 | :param silent: set to ``True`` if you want silent failure for missing 89 | files. 90 | :return: bool. ``True`` if able to load config, ``False`` otherwise. 91 | """ 92 | rv = os.environ.get(variable_name) 93 | if not rv: 94 | if silent: 95 | return False 96 | raise RuntimeError( 97 | f"The environment variable {variable_name} is not set " 98 | "and as such configuration could not be " 99 | "loaded. Set this variable and make it " 100 | "point to a configuration file" 101 | ) 102 | return self.from_yaml(rv, silent=silent) 103 | 104 | def from_yaml(self, filename: str, silent: bool = False) -> bool: 105 | """Updates the values in the config from a YAML file. This function 106 | behaves as if the YAML object was a dictionary and passed to the 107 | :meth:`from_mapping` function. 108 | 109 | :param filename: the filename of the YAML file. This can either be an 110 | absolute filename or a filename relative to the 111 | root path. 112 | :param silent: set to ``True`` if you want silent failure for missing 113 | files. 114 | """ 115 | filename = os.path.join(self.root_path, filename) 116 | 117 | try: 118 | with open(filename) as yaml_file: 119 | obj = yaml.safe_load(yaml_file.read()) 120 | except IOError as e: 121 | if silent and e.errno in (errno.ENOENT, errno.EISDIR): 122 | return False 123 | e.strerror = f"Unable to load configuration file ({e})" 124 | raise e 125 | return self.from_mapping(obj) 126 | 127 | def from_mapping(self, *mapping, **kwargs) -> bool: 128 | """Updates the config like :meth:`update` ignoring items with non-upper 129 | keys.""" 130 | mappings = [] 131 | if len(mapping) == 1: 132 | if hasattr(mapping[0], "items"): 133 | mappings.append(mapping[0].items()) 134 | else: 135 | mappings.append(mapping[0]) 136 | elif len(mapping) > 1: 137 | raise TypeError( 138 | f"expected at most 1 positional argument, got {len(mapping)}" 139 | ) 140 | mappings.append(kwargs.items()) 141 | for mapping in mappings: 142 | for (key, value) in mapping: 143 | if key.isupper(): 144 | self[key] = value 145 | return True 146 | 147 | def get_namespace( 148 | self, namespace: str, lowercase: bool = True, trim_namespace: bool = True 149 | ) -> dict: 150 | """Returns a dictionary containing a subset of configuration options 151 | that match the specified namespace/prefix. Example usage:: 152 | 153 | config['IMAGE_STORE_TYPE'] = 'fs' 154 | config['IMAGE_STORE_PATH'] = '/var/app/images' 155 | config['IMAGE_STORE_BASE_URL'] = 'http://img.website.com' 156 | image_store_config = config.get_namespace('IMAGE_STORE_') 157 | 158 | The resulting dictionary `image_store_config` would look like:: 159 | 160 | { 161 | 'type': 'fs', 162 | 'path': '/var/app/images', 163 | 'base_url': 'http://img.website.com' 164 | } 165 | 166 | This is often useful when configuration options map directly to 167 | keyword arguments in functions or class constructors. 168 | 169 | :param namespace: a configuration namespace 170 | :param lowercase: a flag indicating if the keys of the resulting 171 | dictionary should be lowercase 172 | :param trim_namespace: a flag indicating if the keys of the resulting 173 | dictionary should not include the namespace 174 | 175 | """ 176 | rv = {} 177 | for k, v in self.items(): 178 | if not k.startswith(namespace): 179 | continue 180 | if trim_namespace: 181 | key = k[len(namespace) :] 182 | else: 183 | key = k 184 | if lowercase: 185 | key = key.lower() 186 | rv[key] = v 187 | return rv 188 | 189 | def __repr__(self): 190 | return f"<{self.__class__.__name__} {dict.__repr__(self)}>" 191 | 192 | 193 | DEFAULTS: Dict[str, Union[Iterable[Any], Path, str, bool, None]] = { 194 | # DIFFY_ACCOUNTS: If SWAG is enabled (see below), we'll populate this list 195 | # with the accounts in which Diffy will operate. 196 | 'DIFFY_ACCOUNTS': [], 197 | # DIFFY_REGIONS: The regions to which Diffy has access. 198 | 'DIFFY_REGIONS': AVAILABLE_REGIONS, 199 | # DIFFY_DEFAULT_REGION: The default region in which Diffy will operate to 200 | # baseline and analyze host differences. 201 | 'DIFFY_DEFAULT_REGION': 'us-west-2', 202 | # DIFFY_SWAG_ENABLED: Whether to utilize SWAG for translation of AWS 203 | # account names to numbers. See 204 | # https://github.com/Netflix-Skunkworks/swag-client 205 | 'DIFFY_SWAG_ENABLED': False, 206 | # DIFFY_LOCAL_FILE_DIRECTORY: When saving results to a local file, we use 207 | # this directory to build the final location of the output. 208 | 'DIFFY_LOCAL_FILE_DIRECTORY': Path(__file__).resolve().parent.parent.absolute(), 209 | # DIFFY_AWS_PERSISTENCE_BUCKET: An AWS S3 bucket name describing the 210 | # location where Diffy will save its output. 211 | 'DIFFY_AWS_PERSISTENCE_BUCKET': 'mybucket', 212 | # DIFFY_AWS_ASSUME_ROLE: An AWS IAM role into which Diffy will assume to 213 | # take its actions. 214 | 'DIFFY_AWS_ASSUME_ROLE': 'Diffy', 215 | # DIFFY_PAYLOAD_LOCAL_COMMANDS: A set of raw commands that Diffy will send 216 | # to the local host, if local collection is specified. 217 | 'DIFFY_PAYLOAD_LOCAL_COMMANDS': [ 218 | 'osqueryi --json "SELECT address, port, name, pid, cmdline FROM listening_ports, processes USING (pid) WHERE protocol = 6 and family = 2 AND address NOT LIKE \'127.0.0.%\'"', 219 | 'osqueryi --json "SELECT * FROM crontab"' 220 | ], 221 | # DIFFY_PAYLOAD_OSQUERY_KEY: An AWS S3 key prefix describing the download 222 | # location of your osquery binary. 223 | 'DIFFY_PAYLOAD_OSQUERY_KEY': 'osquery-download', 224 | # DIFFY_PAYLOAD_OSQUERY_REGION: The default region of the S3 bucket from 225 | # where Diffy will download your osquery binary. 226 | 'DIFFY_PAYLOAD_OSQUERY_REGION': 'us-west-2', 227 | # DIFFY_PAYLOAD_OSQUERY_COMMANDS: The commands that Diffy will send to the 228 | # host to be run. 229 | 'DIFFY_PAYLOAD_OSQUERY_COMMANDS': [ 230 | 'osqueryi --json "SELECT * FROM crontab"', 231 | "osqueryi --json \"SELECT address, port, name, pid, cmdline FROM listening_ports, processes USING (pid) WHERE protocol = 6 and family = 2 AND address NOT LIKE '127.0.0.%'\"", 232 | ], 233 | # DIFFY_PERSISTENCE_PLUGIN: The default plugin to use to save Diffy 234 | # results. 235 | "DIFFY_PERSISTENCE_PLUGIN": "local-file", 236 | # DIFFY_TARGET_PLUGIN: The default targeting plugin. Target plugins locate 237 | # and identify hosts to baseline or to analyze. 238 | "DIFFY_TARGET_PLUGIN": "auto-scaling-target", 239 | # DIFFY_PAYLOAD_PLUGIN: The default plugin to use for creating a payload to 240 | # send to remote hosts for execution. 241 | "DIFFY_PAYLOAD_PLUGIN": "local-command", 242 | # DIFFY_COLLECTION_PLUGIN: The default plugin to use for collection of 243 | # baseline and analysis results. 244 | "DIFFY_COLLECTION_PLUGIN": "ssm-collection", 245 | # DIFFY_ANALYSIS_PLUGIN: The default analysis plugin to apply to Diffy 246 | # results. 247 | "DIFFY_ANALYSIS_PLUGIN": "local-simple", 248 | # SWAG_TYPE: The storage protocol of SWAG data. 249 | "SWAG_TYPE": "s3", 250 | # SWAG_BUCKET_NAME: The AWS S3 bucket location of SWAG data. 251 | "SWAG_BUCKET_NAME": None, 252 | # SWAG_DATA_FILE: Name of the file containing SWAG data. 253 | "SWAG_DATA_FILE": "v2/accounts.json", 254 | # RQ_REDIS_URL: URL of the Redis queue utilized for Diffy dispatch. 255 | "RQ_REDIS_URL": None, 256 | # LOG_FILE: An output file for Diffy API logs. 257 | "LOG_FILE": None, 258 | } 259 | 260 | 261 | # os environ takes precedence over default 262 | _defaults = consume_envvars(DEFAULTS) 263 | 264 | CONFIG = Config(defaults=_defaults) 265 | -------------------------------------------------------------------------------- /diffy/core.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: diffy.core 3 | :platform: Unix 4 | :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | .. moduleauthor:: Kevin Glisson 7 | """ 8 | import logging 9 | 10 | logger = logging.getLogger(__name__) 11 | 12 | 13 | def analysis( 14 | target_key: str, 15 | target_plugin: dict, 16 | payload_plugin: dict, 17 | collection_plugin: dict, 18 | persistence_plugin: dict, 19 | analysis_plugin: dict, 20 | **kwargs, 21 | ) -> dict: 22 | """Creates a new analysis.""" 23 | logger.debug(f'Attempting to collect targets with {target_plugin}. TargetKey: {target_key}') 24 | target_plugin['options'].update(kwargs) 25 | targets = target_plugin['plugin'].get(target_key, **target_plugin['options']) 26 | 27 | logger.debug(f'Generating payload with {payload_plugin}') 28 | payload_plugin['options'].update(kwargs) 29 | commands = payload_plugin['plugin'].generate(None, **kwargs) 30 | 31 | logger.debug(f'Attempting to collect data from targets with {collection_plugin}. NumberTargets: {len(targets)}') 32 | collection_plugin['options'].update(kwargs) 33 | results = collection_plugin['plugin'].get(targets, commands, **collection_plugin['options']) 34 | 35 | logger.debug(f'Persisting result data with {persistence_plugin}.') 36 | 37 | # TODO how does this work for non-local analysis? 38 | items = [] 39 | for k, v in results.items(): 40 | for i in v: 41 | instance_id = i["instance_id"] 42 | key = f"{target_key}-{instance_id}" 43 | 44 | items.append(i) 45 | persistence_plugin["plugin"].save(None, key, i) 46 | 47 | logger.debug('Running analysis.') 48 | 49 | results = analysis_plugin["plugin"].run( 50 | items, 51 | baseline=persistence_plugin["plugin"].get("baseline", target_key), 52 | syntax="compact", 53 | ) 54 | 55 | persistence_plugin["options"].update(kwargs) 56 | persistence_plugin["plugin"].save( 57 | "analysis", target_key, results, **persistence_plugin["options"] 58 | ) 59 | return {"analysis": results} 60 | 61 | 62 | def baseline( 63 | target_key: str, 64 | target_plugin: dict, 65 | payload_plugin: dict, 66 | collection_plugin: dict, 67 | persistence_plugin: dict, 68 | **kwargs, 69 | ) -> dict: 70 | """Creates a new baseline.""" 71 | target_plugin["options"].update(kwargs) 72 | targets = target_plugin["plugin"].get(target_key, **target_plugin["options"])[:1] 73 | 74 | logger.debug("Generating payload.") 75 | 76 | payload_plugin["options"].update(kwargs) 77 | commands = payload_plugin["plugin"].generate(None, **payload_plugin["options"]) 78 | 79 | logger.debug( 80 | f"Attempting to collect data from targets. NumberTargets: {len(targets)}" 81 | ) 82 | 83 | collection_plugin["options"].update(kwargs) 84 | results = collection_plugin["plugin"].get( 85 | targets, commands, **collection_plugin["options"] 86 | ) 87 | 88 | logger.debug("Persisting result data") 89 | 90 | baselines = [] 91 | for k, v in results.items(): 92 | persistence_plugin["options"].update(kwargs) 93 | persistence_plugin["plugin"].save( 94 | "baseline", target_key, v[0], **persistence_plugin["options"] 95 | ) # only one baseline 96 | baselines.append({target_key: v[0]}) 97 | 98 | return {"baselines": baselines} 99 | -------------------------------------------------------------------------------- /diffy/exceptions.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: diffy.exceptions 3 | :platform: Unix 4 | :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | 7 | .. moduleauthor:: Kevin Glisson 8 | """ 9 | 10 | 11 | class DiffyException(Exception): 12 | """Base Diffy exception. Catch this to catch all Diffy related errors.""" 13 | 14 | def __init__(self, *args, **kwargs): 15 | self.message = kwargs.pop("message", None) 16 | super().__init__(self.message) 17 | 18 | def __repr__(self): 19 | return f"" 20 | 21 | 22 | class InvalidConfiguration(DiffyException): 23 | """Raised on configuration issues.""" 24 | 25 | def __init__(self, *args, **kwargs): 26 | """ 27 | :param args: Exception arguments 28 | :param kwargs: Exception kwargs 29 | """ 30 | self.message = kwargs.pop("message", None) 31 | super().__init__(self.message) 32 | 33 | def __repr__(self): 34 | return f"" 35 | 36 | 37 | class PendingException(DiffyException): 38 | """Raised when an action is still pending.""" 39 | 40 | def __init__(self, *args, **kwargs): 41 | """ 42 | :param args: Exception arguments 43 | :param kwargs: Exception kwargs 44 | """ 45 | self.message = kwargs.pop("message", None) 46 | super().__init__(self.message) 47 | 48 | def __repr__(self): 49 | return f"" 50 | 51 | 52 | class SchemaError(DiffyException): 53 | """ 54 | Raised on schema issues, relevant probably when creating or changing a plugin schema 55 | :param schema_error: The schema error that was raised 56 | :param args: Exception arguments 57 | :param kwargs: Exception kwargs 58 | """ 59 | 60 | def __init__(self, schema_error, *args, **kwargs): 61 | kwargs["message"] = f"Schema error: {schema_error}" 62 | super().__init__(*args, **kwargs) 63 | 64 | def __repr__(self): 65 | return f"" 66 | 67 | 68 | class BadArguments(DiffyException): 69 | """ 70 | Raised on schema data validation issues 71 | :param validation_error: The validation error message 72 | :param args: Exception arguments 73 | :param kwargs: Exception kwargs 74 | """ 75 | 76 | def __init__(self, validation_error, *args, **kwargs): 77 | kwargs["message"] = f"Error with sent data: {validation_error}" 78 | super().__init__(*args, **kwargs) 79 | 80 | def __repr__(self): 81 | return f"" 82 | 83 | 84 | class TargetNotFound(DiffyException): 85 | """ 86 | Raised when a target plugin cannot find the target specified. 87 | :param target_key: Key used for targeting 88 | :param plugin_slug: Plugin attempting to target 89 | :param args: Exception arguments 90 | :param kwargs: Exception kwargs 91 | """ 92 | 93 | def __init__(self, target_key, plugin_slug, *args, **kwargs): 94 | options = "" 95 | for k, v in kwargs.items(): 96 | options += f"{k}: {v} " 97 | 98 | kwargs[ 99 | "message" 100 | ] = f"Could not find target. key: {target_key} slug: {plugin_slug} {options}" 101 | super().__init__(*args, **kwargs) 102 | 103 | def __repr__(self): 104 | return f"" 105 | -------------------------------------------------------------------------------- /diffy/extensions.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: diffy.extensions 3 | :platform: Unix 4 | :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | """ 7 | from swag_client.backend import SWAGManager 8 | 9 | swag = SWAGManager() 10 | -------------------------------------------------------------------------------- /diffy/filters.py: -------------------------------------------------------------------------------- 1 | import re 2 | import logging 3 | 4 | AWS_REGEX = re.compile( 5 | r"(AWS|aws)?_(SECRET|secret|Secret)?_?(ACCESS|access|Access)?_?(KEY|key|Key)(?P[A-Za-z0-9/\+=]{41})" 6 | ) 7 | 8 | 9 | def replace(m): 10 | return "AWS_SECRET_ACCESS_KEY=" + len(m.group("secret")) * "*" 11 | 12 | 13 | class AWSFilter(logging.Filter): 14 | def filter(self, record): 15 | record.msg = re.sub(AWS_REGEX, replace, record.msg) 16 | return True 17 | -------------------------------------------------------------------------------- /diffy/plugins/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Netflix-Skunkworks/diffy/1d8c1093023832be40d04021906f8779b37e8521/diffy/plugins/__init__.py -------------------------------------------------------------------------------- /diffy/plugins/base/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: diffy.plugins.base 3 | :platform: Unix 4 | :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | 7 | .. moduleauthor:: Kevin Glisson 8 | """ 9 | from __future__ import absolute_import, print_function 10 | 11 | from diffy.plugins.base.manager import PluginManager 12 | from diffy.plugins.base.v1 import * # noqa 13 | 14 | plugins = PluginManager() 15 | register = plugins.register 16 | unregister = plugins.unregister 17 | -------------------------------------------------------------------------------- /diffy/plugins/base/manager.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: diffy.plugins.base.manager 3 | :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more 4 | :license: Apache, see LICENSE for more details. 5 | 6 | .. moduleauthor:: Kevin Glisson (kglisson@netflix.com) 7 | """ 8 | import logging 9 | from diffy.common.managers import InstanceManager 10 | 11 | 12 | logger = logging.getLogger(__name__) 13 | 14 | 15 | # inspired by https://github.com/getsentry/sentry 16 | class PluginManager(InstanceManager): 17 | def __iter__(self): 18 | return iter(self.all()) 19 | 20 | def __len__(self): 21 | return sum(1 for i in self.all()) 22 | 23 | def all(self, version=1, plugin_type=None): 24 | for plugin in sorted( 25 | super(PluginManager, self).all(), key=lambda x: x.get_title() 26 | ): 27 | if not plugin.type == plugin_type and plugin_type: 28 | continue 29 | if not plugin.is_enabled(): 30 | continue 31 | if version is not None and plugin.__version__ != version: 32 | continue 33 | yield plugin 34 | 35 | def get(self, slug): 36 | for plugin in self.all(version=1): 37 | if plugin.slug == slug: 38 | return plugin 39 | for plugin in self.all(version=2): 40 | if plugin.slug == slug: 41 | return plugin 42 | logger.error( 43 | f"Unable to find slug: {slug} in self.all version 1: {self.all(version=1)} or version 2: {self.all(version=2)}" 44 | ) 45 | raise KeyError(slug) 46 | 47 | def first(self, func_name, *args, **kwargs): 48 | version = kwargs.pop("version", 1) 49 | for plugin in self.all(version=version): 50 | try: 51 | result = getattr(plugin, func_name)(*args, **kwargs) 52 | except Exception as e: 53 | logger.error( 54 | "Error processing %s() on %r: %s", 55 | func_name, 56 | plugin.__class__, 57 | e, 58 | extra={"func_arg": args, "func_kwargs": kwargs}, 59 | exc_info=True, 60 | ) 61 | continue 62 | 63 | if result is not None: 64 | return result 65 | 66 | def register(self, cls): 67 | self.add("%s.%s" % (cls.__module__, cls.__name__)) 68 | return cls 69 | 70 | def unregister(self, cls): 71 | self.remove("%s.%s" % (cls.__module__, cls.__name__)) 72 | return cls 73 | -------------------------------------------------------------------------------- /diffy/plugins/base/v1.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: diffy.plugins.base.v1 3 | :platform: Unix 4 | :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | 7 | .. moduleauthor:: Kevin Glisson 8 | """ 9 | import logging 10 | from threading import local 11 | from typing import List, Tuple, Optional, Any 12 | 13 | from marshmallow_jsonschema import JSONSchema 14 | 15 | 16 | logger = logging.getLogger(__name__) 17 | 18 | 19 | # stolen from https://github.com/getsentry/sentry/ 20 | class PluginMount(type): 21 | def __new__(cls, name, bases, attrs): 22 | new_cls = type.__new__(cls, name, bases, attrs) 23 | if IPlugin in bases: 24 | return new_cls 25 | if new_cls.title is None: 26 | new_cls.title = new_cls.__name__ 27 | if not new_cls.slug: 28 | new_cls.slug = new_cls.title.replace(" ", "-").lower() 29 | return new_cls 30 | 31 | 32 | class IPlugin(local): 33 | """ 34 | Plugin interface. Should not be inherited from directly. 35 | A plugin should be treated as if it were a singleton. The owner does not 36 | control when or how the plugin gets instantiated, nor is it guaranteed that 37 | it will happen, or happen more than once. 38 | >>> from diffy.plugins import Plugin 39 | >>> 40 | >>> class MyPlugin(Plugin): 41 | >>> def get_title(self): 42 | >>> return 'My Plugin' 43 | As a general rule all inherited methods should allow ``**kwargs`` to ensure 44 | ease of future compatibility. 45 | """ 46 | 47 | # Generic plugin information 48 | title: Optional[str] = None 49 | slug: Optional[str] = None 50 | description: Optional[str] = None 51 | version: Optional[str] = None 52 | author: Optional[str] = None 53 | author_url: Optional[str] = None 54 | resource_links = () 55 | 56 | _schema: Any = None 57 | 58 | # Global enabled state 59 | enabled: bool = True 60 | can_disable: bool = True 61 | 62 | def validate_options(self, options: dict) -> Any: 63 | """ 64 | Validates given options against defined schema. 65 | >>> plugin.validate_options(options) 66 | """ 67 | return self._schema(strict=True).load(options).data 68 | 69 | @property 70 | def json_schema(self): 71 | """Returns JSON Schema of current plugin schema.""" 72 | return JSONSchema().dump(self._schema()).data 73 | 74 | def is_enabled(self) -> bool: 75 | """ 76 | Returns a boolean representing if this plugin is enabled. 77 | If ``project`` is passed, it will limit the scope to that project. 78 | >>> plugin.is_enabled() 79 | """ 80 | if not self.enabled: 81 | return False 82 | if not self.can_disable: 83 | return True 84 | 85 | return True 86 | 87 | def get_title(self) -> Optional[str]: 88 | """ 89 | Returns the general title for this plugin. 90 | >>> plugin.get_title() 91 | """ 92 | return self.title 93 | 94 | def get_description(self) -> Optional[str]: 95 | """ 96 | Returns the description for this plugin. This is shown on the plugin configuration 97 | page. 98 | >>> plugin.get_description() 99 | """ 100 | return self.description 101 | 102 | def get_resource_links(self) -> List[Any]: 103 | """ 104 | Returns a list of tuples pointing to various resources for this plugin. 105 | >>> def get_resource_links(self): 106 | >>> return [ 107 | >>> ('Documentation', 'https://diffy.readthedocs.io'), 108 | >>> ('Bug Tracker', 'https://github.com/Netflix/diffy/issues'), 109 | >>> ('Source', 'https://github.com/Netflix/diffy'), 110 | >>> ] 111 | """ 112 | return self.resource_links 113 | 114 | 115 | class Plugin(IPlugin): 116 | """ 117 | A plugin should be treated as if it were a singleton. The owner does not 118 | control when or how the plugin gets instantiated, nor is it guaranteed that 119 | it will happen, or happen more than once. 120 | """ 121 | 122 | __version__ = 1 123 | __metaclass__ = PluginMount 124 | -------------------------------------------------------------------------------- /diffy/plugins/bases/__init__.py: -------------------------------------------------------------------------------- 1 | from .persistence import PersistencePlugin # noqa 2 | from .analysis import AnalysisPlugin # noqa 3 | from .collection import CollectionPlugin # noqa 4 | from .target import TargetPlugin # noqa 5 | from .payload import PayloadPlugin # noqa 6 | from .inventory import InventoryPlugin # noqa 7 | -------------------------------------------------------------------------------- /diffy/plugins/bases/analysis.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: diffy.plugins.bases.analysis 3 | :platform: Unix 4 | :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | .. moduleauthor:: Kevin Glisson 7 | """ 8 | from diffy.plugins.base import Plugin 9 | from diffy.schema import PluginOptionSchema 10 | 11 | 12 | class AnalysisPlugin(Plugin): 13 | type = "analysis" 14 | _schema = PluginOptionSchema 15 | 16 | def run(self, items, **kwargs): 17 | raise NotImplementedError 18 | -------------------------------------------------------------------------------- /diffy/plugins/bases/collection.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: diffy.plugins.bases.collection 3 | :platform: Unix 4 | :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | .. moduleauthor:: Kevin Glisson 7 | """ 8 | from diffy.plugins.base import Plugin 9 | from diffy.schema import PluginOptionSchema 10 | 11 | 12 | class CollectionPlugin(Plugin): 13 | type = "collection" 14 | _schema = PluginOptionSchema 15 | 16 | def get(self, targets, incident, commands, **kwargs): 17 | raise NotImplementedError 18 | -------------------------------------------------------------------------------- /diffy/plugins/bases/inventory.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: diffy.plugins.bases.inventory 3 | :platform: Unix 4 | :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | .. moduleauthor:: Forest Monsen 7 | """ 8 | from diffy.plugins.base import Plugin 9 | from diffy.schema import PluginOptionSchema 10 | 11 | 12 | class InventoryPlugin(Plugin): 13 | type = "inventory" 14 | _schema = PluginOptionSchema 15 | 16 | def get(self, **kwargs): 17 | raise NotImplementedError 18 | -------------------------------------------------------------------------------- /diffy/plugins/bases/payload.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: diffy.plugins.bases.payload 3 | :platform: Unix 4 | :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | .. moduleauthor:: Kevin Glisson 7 | """ 8 | from diffy.plugins.base import Plugin 9 | from diffy.schema import PluginOptionSchema 10 | 11 | 12 | class PayloadPlugin(Plugin): 13 | type = "payload" 14 | _schema = PluginOptionSchema 15 | 16 | def generate(self, incident, **kwargs): 17 | raise NotImplementedError 18 | -------------------------------------------------------------------------------- /diffy/plugins/bases/persistence.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: diffy.plugins.bases.persistence 3 | :platform: Unix 4 | :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | .. moduleauthor:: Kevin Glisson 7 | """ 8 | from diffy.plugins.base import Plugin 9 | from diffy.schema import PluginOptionSchema 10 | 11 | 12 | class PersistencePlugin(Plugin): 13 | type = "persistence" 14 | _schema = PluginOptionSchema 15 | 16 | def get(self, file_type, key, **kwargs): 17 | raise NotImplementedError 18 | 19 | def get_all(self, **kwargs): 20 | raise NotImplementedError 21 | 22 | def save(self, file_type, key, item, **kwargs): 23 | raise NotImplementedError 24 | -------------------------------------------------------------------------------- /diffy/plugins/bases/target.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: diffy.plugins.bases.target 3 | :platform: Unix 4 | :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | .. moduleauthor:: Kevin Glisson 7 | """ 8 | from diffy.plugins.base import Plugin 9 | from diffy.schema import PluginOptionSchema 10 | 11 | 12 | class TargetPlugin(Plugin): 13 | type = "target" 14 | _schema = PluginOptionSchema 15 | 16 | def get(self, key, **kwargs): 17 | raise NotImplementedError 18 | -------------------------------------------------------------------------------- /diffy/plugins/diffy_aws/__init__.py: -------------------------------------------------------------------------------- 1 | from diffy._version import __version__ # noqa 2 | -------------------------------------------------------------------------------- /diffy/plugins/diffy_aws/auto_scaling.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: diffy.plugins.diffy_aws.autoscaling 3 | :platform: Unix 4 | :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | .. moduleauthor:: Forest Monsen 7 | .. moduleauthor:: Kevin Glisson 8 | """ 9 | import logging 10 | from typing import List 11 | 12 | from retrying import retry 13 | from botocore.exceptions import ClientError 14 | 15 | from .sts import sts_client 16 | 17 | logger = logging.getLogger(__name__) 18 | 19 | 20 | def retry_throttled(exception): 21 | """ 22 | Determines if this exception is due to throttling 23 | :param exception: 24 | :return: 25 | """ 26 | logger.debug(exception) 27 | if isinstance(exception, ClientError): 28 | if exception.response["Error"]["Code"] == "ThrottlingException": 29 | return True 30 | return False 31 | 32 | 33 | @sts_client("autoscaling") 34 | @retry( 35 | retry_on_exception=retry_throttled, 36 | stop_max_attempt_number=7, 37 | wait_exponential_multiplier=1000, 38 | ) 39 | def describe_auto_scaling_group(group_name: str, **kwargs) -> List[str]: 40 | """Uses boto to query for command status.""" 41 | logger.debug(f"Describing autoscaling group. AutoScalingGroupName: {group_name}") 42 | 43 | return kwargs["client"].describe_auto_scaling_groups( 44 | AutoScalingGroupNames=[group_name] 45 | )["AutoScalingGroups"] 46 | -------------------------------------------------------------------------------- /diffy/plugins/diffy_aws/plugin.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: diffy.plugins.diffy_s3.plugin 3 | :platform: Unix 4 | :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | .. moduleauthor:: Kevin Glisson 7 | """ 8 | import logging 9 | from typing import List 10 | 11 | import boto3 12 | from botocore.exceptions import ClientError, NoCredentialsError 13 | from marshmallow import fields 14 | 15 | from diffy.config import CONFIG 16 | from diffy.exceptions import TargetNotFound 17 | from diffy.schema import DiffyInputSchema 18 | from diffy.plugins import diffy_aws as aws 19 | from diffy.plugins.bases import PersistencePlugin, TargetPlugin, CollectionPlugin 20 | 21 | from .s3 import save_file, load_file 22 | from .ssm import process 23 | from .auto_scaling import describe_auto_scaling_group 24 | 25 | 26 | logger = logging.getLogger(__name__) 27 | 28 | 29 | def get_default_aws_account_number() -> dict: 30 | """Retrieves current account number""" 31 | sts = boto3.client('sts') 32 | accountId = '1234' 33 | try: 34 | accountId = sts.get_caller_identity()['Account'] 35 | except (ClientError, NoCredentialsError) as e: 36 | logger.debug(f'Failed to get AWS AccountID, using Prod: {e}') 37 | return accountId 38 | 39 | 40 | 41 | class AWSSchema(DiffyInputSchema): 42 | account_number = fields.String( 43 | default=get_default_aws_account_number, missing=get_default_aws_account_number 44 | ) 45 | region = fields.String( 46 | default=CONFIG["DIFFY_DEFAULT_REGION"], missing=CONFIG["DIFFY_DEFAULT_REGION"] 47 | ) 48 | 49 | 50 | class S3PersistencePlugin(PersistencePlugin): 51 | title = "s3" 52 | slug = "s3-persistence" 53 | description = "Persist diffy collection results to S3." 54 | version = aws.__version__ 55 | 56 | author = "Kevin Glisson" 57 | author_url = "https://github.com/netflix/diffy.git" 58 | 59 | def get(self, key: str, **kwargs) -> dict: 60 | """Fetches a result from S3.""" 61 | logger.debug(f"Retrieving file from S3. Bucket: {self.bucket_name} Key: {key}") 62 | return load_file(key) 63 | 64 | # TODO 65 | def get_all(self, **kwargs): 66 | """Fetches all results from S3.""" 67 | pass 68 | 69 | def save(self, key: str, item: str, **kwargs) -> dict: 70 | """Saves a result to S3.""" 71 | logger.debug(f"Saving file to S3. Bucket: {self.bucket_name} Item: {item}") 72 | return save_file(key, item) 73 | 74 | 75 | class AutoScalingTargetPlugin(TargetPlugin): 76 | title = "auto scaling" 77 | slug = "auto-scaling-target" 78 | description = ( 79 | "Uses Auto Scaling Groups to determine which instances to target for analysis" 80 | ) 81 | version = aws.__version__ 82 | 83 | author = "Kevin Glisson" 84 | author_url = "https://github.com/netflix/diffy.git" 85 | 86 | _schema = AWSSchema 87 | 88 | def get(self, key: str, **kwargs) -> List[str]: 89 | """Fetches instances to target for collection.""" 90 | logger.debug(f"Fetching instances for Auto Scaling Group. GroupName: {key}") 91 | groups = describe_auto_scaling_group( 92 | key, account_number=kwargs["account_number"], region=kwargs["region"] 93 | ) 94 | logger.debug(groups) 95 | 96 | if not groups: 97 | raise TargetNotFound(target_key=key, plugin_slug=self.slug, **kwargs) 98 | 99 | return [x["InstanceId"] for x in groups[0]["Instances"]] 100 | 101 | 102 | class SSMCollectionPlugin(CollectionPlugin): 103 | title = "ssm" 104 | slug = "ssm-collection" 105 | description = "Uses SSM to collection information for analysis." 106 | version = aws.__version__ 107 | 108 | author = "Kevin Glisson" 109 | author_url = "https://github.com/netflix/diffy.git" 110 | 111 | _schema = AWSSchema 112 | 113 | def get(self, targets: List[str], commands: List[str], **kwargs) -> dict: 114 | """Queries an target via SSM.""" 115 | logger.debug(f"Querying instances. Instances: {targets}") 116 | return process( 117 | targets, 118 | commands, 119 | incident_id=kwargs["incident_id"], 120 | account_number=kwargs["account_number"], 121 | region=kwargs["region"], 122 | ) 123 | -------------------------------------------------------------------------------- /diffy/plugins/diffy_aws/s3.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: diffy.plugins.diffy_aws.s3 3 | :platform: Unix 4 | :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | .. moduleauthor:: Kevin Glisson 7 | """ 8 | import json 9 | import logging 10 | 11 | from retrying import retry 12 | from botocore.exceptions import ClientError 13 | 14 | from .sts import sts_client 15 | 16 | from diffy.config import CONFIG 17 | 18 | logger = logging.getLogger(__name__) 19 | 20 | 21 | @retry( 22 | stop_max_attempt_number=3, 23 | wait_exponential_multiplier=1000, 24 | wait_exponential_max=10000, 25 | ) 26 | def _get_from_s3(client, bucket, data_file): 27 | return client.get_object(Bucket=bucket, Key=data_file)["Body"].read() 28 | 29 | 30 | @retry( 31 | stop_max_attempt_number=3, 32 | wait_exponential_multiplier=1000, 33 | wait_exponential_max=10000, 34 | ) 35 | def _put_to_s3(client, bucket, data_file, body): 36 | return client.put_object( 37 | Bucket=bucket, 38 | Key=data_file, 39 | Body=body, 40 | ContentType="application/json", 41 | CacheControl="no-cache, no-store, must-revalidate", 42 | ) 43 | 44 | 45 | @sts_client("s3") 46 | def load_file(key: str, **kwargs) -> dict: 47 | """Tries to load JSON data from S3.""" 48 | bucket = CONFIG.get("DIFFY_AWS_PERSISTENCE_BUCKET") 49 | logger.debug(f"Loading item from s3. Bucket: {bucket} Key: {key}") 50 | try: 51 | data = _get_from_s3(kwargs["client"], bucket, key) 52 | 53 | data = data.decode("utf-8") 54 | 55 | return json.loads(data) 56 | except ClientError as e: 57 | logger.exception(e) 58 | assert False 59 | 60 | 61 | @sts_client("s3") 62 | def save_file(key: str, item: str, dry_run=None, **kwargs) -> dict: 63 | """Tries to write JSON data to data file in S3.""" 64 | bucket = CONFIG.get("DIFFY_AWS_PERSISTENCE_BUCKET") 65 | logger.debug(f"Writing item to s3. Bucket: {bucket} Key: {key}") 66 | 67 | if not dry_run: 68 | return _put_to_s3(kwargs["client"], bucket, key, item) 69 | return {} 70 | -------------------------------------------------------------------------------- /diffy/plugins/diffy_aws/ssm.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: diffy.plugins.diffy_aws.ssm 3 | :platform: Unix 4 | :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | .. moduleauthor:: Forest Monsen 7 | .. moduleauthor:: Kevin Glisson 8 | """ 9 | import json 10 | import logging 11 | from base64 import urlsafe_b64encode 12 | from typing import List, Tuple 13 | 14 | from botocore.exceptions import ClientError 15 | 16 | from retrying import retry 17 | 18 | from diffy.filters import AWSFilter 19 | from diffy.exceptions import PendingException 20 | from diffy.common.utils import chunk 21 | from .sts import sts_client 22 | 23 | logger = logging.getLogger(__name__) 24 | logger.addFilter(AWSFilter()) 25 | 26 | 27 | # TODO process all commands 28 | def encode_command(string: str) -> str: 29 | """Base64-encode and prepare an SSM command string, to avoid shell interpolation. 30 | 31 | :string: String. A command or commands to be encoded. 32 | :returns: String. Command(s) prepared for execution. 33 | """ 34 | encoded = urlsafe_b64encode(string.encode("utf-8")) 35 | command = f"eval $(echo {encoded} | base64 --decode)" 36 | return command 37 | 38 | 39 | @sts_client("ssm") 40 | def send_commands( 41 | instance_ids: List[str], commands: List[str], **kwargs 42 | ) -> Tuple[str, str]: 43 | """Send SSM command to target instances. 44 | 45 | :client: Object. AWS SSM client. 46 | :instance_ids: List. InstanceIds to affect. 47 | :incident: String. Incident name or comment. 48 | :commands: List(String). Commands to send to instance. 49 | :returns: Tuple(String, String). GUID uniquely identifying the SSM command and the current status. 50 | """ 51 | logger.debug("Sending command(s) to instance(s).") 52 | 53 | try: 54 | response = kwargs["client"].send_command( 55 | InstanceIds=instance_ids, 56 | DocumentName="AWS-RunShellScript", 57 | Comment=kwargs.get("incident_id", ""), 58 | Parameters={"commands": commands}, 59 | ) 60 | except ClientError as ex: 61 | if ex.response["Error"]["Code"] == "InvalidInstanceId": 62 | code = ex.response["Error"]["Code"] 63 | logger.error(f"AWS doesn't have a record of this instance (got {code}).") 64 | raise ex 65 | 66 | logger.debug(f"Command Response: {response}") 67 | 68 | return response["Command"]["CommandId"], response["Command"]["Status"] 69 | 70 | 71 | def process(instances: List[str], commands: List[str], **kwargs) -> dict: 72 | """Dispatch an SSM command to each instance in a list.""" 73 | # boto limits us to 50 per 74 | command_ids = {} 75 | for c in chunk(instances, 50): 76 | logger.debug( 77 | f"Sending command. Instances: {c} Command: {json.dumps(commands, indent=2)}" 78 | ) 79 | command_id, status = send_commands(c, commands, **kwargs) 80 | command_ids[command_id] = [ 81 | {"instance_id": i, "status": status, "stdout": ""} for i in c 82 | ] 83 | 84 | return poll(command_ids, **kwargs) 85 | 86 | 87 | def retry_throttled(exception) -> bool: 88 | """ 89 | Determines if this exception is due to throttling 90 | 91 | :param exception: 92 | :return: 93 | """ 94 | if isinstance(exception, ClientError): 95 | if exception.response["Error"]["Code"] != "InvocationDoesNotExist": 96 | return True 97 | return False 98 | 99 | 100 | @sts_client("ssm") 101 | @retry( 102 | retry_on_exception=retry_throttled, 103 | stop_max_attempt_number=7, 104 | wait_exponential_multiplier=1000, 105 | ) 106 | def get_command_invocation(command_id: str, instance_id: str, **kwargs) -> dict: 107 | """Uses boto to query for command status.""" 108 | logger.debug( 109 | f"Getting command status. CommandId: {command_id} InstanceId: {instance_id}" 110 | ) 111 | 112 | return kwargs["client"].get_command_invocation( 113 | CommandId=command_id, InstanceId=instance_id 114 | ) 115 | 116 | 117 | def is_completed(status: str) -> bool: 118 | """Determines if the status is deemed to be completed.""" 119 | if status in ["Success", "TimedOut", "Cancelled", "Failed"]: 120 | return True 121 | return False 122 | 123 | 124 | def retry_pending(exception) -> bool: 125 | """Determines if exception is due to commands stuck in pending state and retries.""" 126 | return isinstance(exception, PendingException) 127 | 128 | 129 | # TODO is this the most efficient polling wise? 130 | @retry(retry_on_exception=retry_pending, wait_exponential_multiplier=1000) 131 | def poll(command_ids: dict, **kwargs) -> dict: 132 | """Query the SSM endpoint to determine whether a command has completed. 133 | 134 | :returns: Dict. Results of command(s) 135 | command_ids = { 136 | 'command_id': [ 137 | { 138 | 'instance_id': 'i-123343243', 139 | 'status': 'success', 140 | 'collected_at' : 'dtg' 141 | 'stdout': {} 142 | } 143 | ] 144 | } 145 | """ 146 | for cid, instances in command_ids.items(): 147 | for i in instances: 148 | response = get_command_invocation(cid, i["instance_id"], **kwargs) 149 | 150 | if not is_completed(response["Status"]): 151 | raise PendingException("SSM command is not yet completed.") 152 | 153 | logger.debug( 154 | f"Command completed. Response: {json.dumps(response, indent=2)}" 155 | ) 156 | 157 | i["status"] = response["Status"] 158 | i["collected_at"] = response["ExecutionEndDateTime"] 159 | 160 | if i["status"] != "Failed": 161 | i["stdout"] = json.loads(response["StandardOutputContent"]) 162 | else: 163 | i["stderr"] = response["StandardErrorContent"] 164 | logger.error( 165 | f'Failed to fetch command output. Instance: {i["instance_id"]} Reason: {response["StandardErrorContent"]}' 166 | ) 167 | 168 | return command_ids 169 | -------------------------------------------------------------------------------- /diffy/plugins/diffy_aws/sts.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: diffy.plugins.diffy_aws.sts 3 | :platform: Unix 4 | :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | .. moduleauthor:: Kevin Glisson 7 | """ 8 | import logging 9 | from functools import wraps 10 | 11 | import boto3 12 | 13 | from diffy.config import CONFIG 14 | 15 | logger = logging.getLogger(__name__) 16 | 17 | 18 | def sts_client(service, service_type="client"): 19 | def decorator(f): 20 | @wraps(f) 21 | def decorated_function(*args, **kwargs): 22 | sts = boto3.client("sts") 23 | 24 | account_number = kwargs["account_number"] 25 | role = CONFIG.get("DIFFY_AWS_ASSUME_ROLE", "Diffy") 26 | 27 | arn = f"arn:aws:iam::{account_number}:role/{role}" 28 | 29 | kwargs.pop("account_number") 30 | 31 | # TODO add incident specific information to RoleSessionName 32 | logger.debug(f"Assuming role. Arn: {arn}") 33 | role = sts.assume_role(RoleArn=arn, RoleSessionName="diffy") 34 | 35 | if service_type == "client": 36 | client = boto3.client( 37 | service, 38 | region_name=kwargs["region"], 39 | aws_access_key_id=role["Credentials"]["AccessKeyId"], 40 | aws_secret_access_key=role["Credentials"]["SecretAccessKey"], 41 | aws_session_token=role["Credentials"]["SessionToken"], 42 | ) 43 | kwargs["client"] = client 44 | elif service_type == "resource": 45 | resource = boto3.resource( 46 | service, 47 | region_name=kwargs["region"], 48 | aws_access_key_id=role["Credentials"]["AccessKeyId"], 49 | aws_secret_access_key=role["Credentials"]["SecretAccessKey"], 50 | aws_session_token=role["Credentials"]["SessionToken"], 51 | ) 52 | kwargs["resource"] = resource 53 | return f(*args, **kwargs) 54 | 55 | return decorated_function 56 | 57 | return decorator 58 | -------------------------------------------------------------------------------- /diffy/plugins/diffy_aws/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Netflix-Skunkworks/diffy/1d8c1093023832be40d04021906f8779b37e8521/diffy/plugins/diffy_aws/tests/__init__.py -------------------------------------------------------------------------------- /diffy/plugins/diffy_aws/tests/conftest.py: -------------------------------------------------------------------------------- 1 | from diffy.tests.conftest import * 2 | -------------------------------------------------------------------------------- /diffy/plugins/diffy_local/__init__.py: -------------------------------------------------------------------------------- 1 | from diffy._version import __version__ # noq 2 | -------------------------------------------------------------------------------- /diffy/plugins/diffy_local/plugin.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: diffy.plugins.diffy_simple.plugin 3 | :platform: Unix 4 | :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | .. moduleauthor:: Kevin Glisson 7 | """ 8 | import os 9 | import subprocess 10 | import shlex 11 | import datetime 12 | import json 13 | import logging 14 | from typing import List 15 | 16 | from jsondiff import diff 17 | 18 | from diffy.config import CONFIG 19 | from diffy.exceptions import BadArguments 20 | from diffy.plugins import diffy_local as local 21 | from diffy.plugins.bases import AnalysisPlugin, PersistencePlugin, PayloadPlugin, CollectionPlugin, TargetPlugin 22 | 23 | 24 | logger = logging.getLogger(__name__) 25 | 26 | 27 | def get_local_file_path(file_type: str, key: str) -> str: 28 | """Creates the full path for given local file.""" 29 | if file_type: 30 | file_name = f"{file_type}-{key}.json" 31 | else: 32 | file_name = f"{key}.json" 33 | 34 | return os.path.join(CONFIG.get("DIFFY_LOCAL_FILE_DIRECTORY"), file_name) 35 | 36 | 37 | class SimpleAnalysisPlugin(AnalysisPlugin): 38 | title = "simple" 39 | slug = "local-simple" 40 | description = "Perform simple differential analysis on collection results." 41 | version = local.__version__ 42 | 43 | author = "Kevin Glisson" 44 | author_url = "https://github.com/Netflix-Skunkworks/diffy.git" 45 | 46 | def run(self, items: List[dict], **kwargs) -> List[dict]: 47 | """Run simple difference calculation on results based on a baseline.""" 48 | logger.debug("Performing simple local baseline analysis.") 49 | 50 | if not kwargs.get("baseline"): 51 | raise BadArguments("Cannot run simple analysis. No baseline found.") 52 | 53 | for i in items: 54 | i["diff"] = diff(kwargs["baseline"]["stdout"], i["stdout"]) 55 | 56 | return items 57 | 58 | 59 | class ClusterAnalysisPlugin(AnalysisPlugin): 60 | title = "cluster" 61 | slug = "local-cluster" 62 | description = "Perform cluster analysis on collection results." 63 | version = local.__version__ 64 | 65 | author = "Kevin Glisson" 66 | author_url = "https://github.com/Netflix-Skunkworks/diffy.git" 67 | 68 | def run(self, items: List[dict], **kwargs) -> List[dict]: 69 | """Run cluster calculation on results based on a baseline.""" 70 | logger.debug("Performing simple local cluster analysis.") 71 | return items 72 | 73 | 74 | class FilePersistencePlugin(PersistencePlugin): 75 | title = "file" 76 | slug = "local-file" 77 | description = "Store results locally for further analysis." 78 | version = local.__version__ 79 | 80 | author = "Kevin Glisson" 81 | author_url = "https://github.com/Netflix-Skunkworks/diffy.git" 82 | 83 | def get(self, file_type: str, key: str, **kwargs) -> dict: 84 | """Fetch data from local file system.""" 85 | path = get_local_file_path(file_type, key) 86 | logging.debug(f"Reading persistent data. Path: {path}") 87 | 88 | if os.path.exists(path): 89 | with open(path, "r") as f: 90 | return json.load(f) 91 | 92 | def get_all(self, file_type: str) -> List[dict]: 93 | """Fetches all files matching given prefix""" 94 | path = os.path.join(CONFIG.get("DIFFY_LOCAL_FILE_DIRECTORY")) 95 | 96 | items = [] 97 | for p in [os.path.abspath(x) for x in os.listdir(path)]: 98 | file = p.split("/")[-1] 99 | if file.startswith(file_type) and file.endswith(".json"): 100 | with open(p, "r") as f: 101 | items.append(json.load(f)) 102 | return items 103 | 104 | def save(self, file_type: str, key: str, item: dict, **kwargs) -> None: 105 | """Save data to local file system.""" 106 | path = get_local_file_path(file_type, key) 107 | logging.debug(f"Writing persistent data. Path: {path}") 108 | 109 | with open(path, "w") as f: 110 | json.dump(item, f) 111 | 112 | 113 | class CommandPayloadPlugin(PayloadPlugin): 114 | title = "command" 115 | slug = "local-command" 116 | description = "Sends command without any modification." 117 | version = local.__version__ 118 | 119 | author = "Kevin Glisson" 120 | author_url = "https://github.com/Netflix-Skunkworks/diffy.git" 121 | 122 | def generate(self, incident: str, **kwargs) -> dict: 123 | return CONFIG.get('DIFFY_PAYLOAD_LOCAL_COMMANDS') 124 | 125 | 126 | class LocalShellCollectionPlugin(CollectionPlugin): 127 | title = 'command' 128 | slug = 'local-shell-collection' 129 | description = 'Executes payload commands via local shell.' 130 | version = local.__version__ 131 | 132 | author = 'Alex Maestretti' 133 | author_url = 'https://github.com/Netflix-Skunkworks/diffy.git' 134 | 135 | def get(self, targets: List[str], commands: List[str], **kwargs) -> dict: 136 | """Queries local system target via subprocess shell. 137 | 138 | :returns command results as dict { 139 | 'command_id': [ 140 | { 141 | 'instance_id': 'i-123343243', 142 | 'status': 'success', 143 | 'collected_at' : 'dtg' 144 | 'stdout': {json osquery result} 145 | } 146 | ... 147 | ] 148 | } 149 | """ 150 | # TODO: check if we are root, warn user if not we may not get a full baseline 151 | results = {} 152 | for idx, cmd in enumerate(commands): 153 | logger.debug(f'Querying local system with: {cmd}') 154 | # format command which is a string with an osqueryi shell command into a list of args for subprocess 155 | formatted_cmd = shlex.split(cmd) 156 | 157 | # TODO support python37 158 | process_result = subprocess.run(formatted_cmd, stdout=subprocess.PIPE) # python36 only 159 | stdout = process_result.stdout.decode('utf-8') 160 | 161 | # TODO: check return status and pass stderr if needed 162 | results[idx] = [{ 163 | 'instance_id': 'localhost', 164 | 'status': 'success', 165 | 'collected_at': datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"), 166 | 'stdout': json.loads(stdout) 167 | }] 168 | logger.debug(f'Results[{idx}] : {format(json.dumps(stdout, indent=2))}') 169 | return results 170 | 171 | 172 | class LocalTargetPlugin(TargetPlugin): 173 | title = 'command' 174 | slug = 'local-target' 175 | description = 'Targets the local system for collection.' 176 | version = local.__version__ 177 | 178 | author = 'Alex Maestretti' 179 | author_url = 'https://github.com/Netflix-Skunkworks/diffy.git' 180 | 181 | def get(self, key, **kwargs): 182 | return 'local' # returns arbitrary value that is ignored by local-collection 183 | -------------------------------------------------------------------------------- /diffy/plugins/diffy_local/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Netflix-Skunkworks/diffy/1d8c1093023832be40d04021906f8779b37e8521/diffy/plugins/diffy_local/tests/__init__.py -------------------------------------------------------------------------------- /diffy/plugins/diffy_local/tests/conftest.py: -------------------------------------------------------------------------------- 1 | from diffy.tests.conftest import * # noqa 2 | -------------------------------------------------------------------------------- /diffy/plugins/diffy_osquery/__init__.py: -------------------------------------------------------------------------------- 1 | from diffy._version import __version__ # noq 2 | -------------------------------------------------------------------------------- /diffy/plugins/diffy_osquery/plugin.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: diffy.plugins.diffy_osquery.plugin 3 | :platform: Unix 4 | :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | .. moduleauthor:: Kevin Glisson 7 | """ 8 | import logging 9 | from typing import List 10 | from shutil import which 11 | from boto3 import Session 12 | 13 | from diffy.config import CONFIG 14 | from diffy.exceptions import BadArguments 15 | from diffy.plugins import diffy_osquery as osquery 16 | from diffy.plugins.bases import PayloadPlugin 17 | 18 | 19 | logger = logging.getLogger(__name__) 20 | 21 | 22 | class OSQueryPayloadPlugin(PayloadPlugin): 23 | title = "osquery" 24 | slug = "osquery-payload" 25 | description = "Uses osquery as part of the collection payload." 26 | version = osquery.__version__ 27 | 28 | author = "Kevin Glisson" 29 | author_url = "https://github.com/netflix/diffy.git" 30 | 31 | def generate(self, incident: str, **kwargs) -> List[str]: 32 | """Generates the commands that will be run on the host.""" 33 | logger.debug("Generating osquery payload.") 34 | session = Session() 35 | 36 | # If osquery isn't present, obtain an osquery binary from S3. 37 | if not which("osqueryi"): 38 | # We run these commands with Diffy credentials so as to not pollute 39 | # the on-instance credentials. 40 | creds = session.get_credentials() 41 | region = kwargs.get("region", CONFIG.get("DIFFY_PAYLOAD_OSQUERY_REGION")) 42 | key = kwargs.get("key", CONFIG.get("DIFFY_PAYLOAD_OSQUERY_KEY")) 43 | 44 | if not region: 45 | raise BadArguments( 46 | "DIFFY_PAYLOAD_OSQUERY_REGION required for use with OSQuery plugin." 47 | ) 48 | 49 | if not key: 50 | raise BadArguments( 51 | "DIFFY_PAYLOAD_OSQUERY_KEY required for use with OSQuery plugin." 52 | ) 53 | 54 | # If we've downloaded our own osquery collection binary, create a 55 | # symbolic link, allowing us to use relative commands elsewhere. 56 | commands: List[str] = [ 57 | f"export AWS_ACCESS_KEY_ID={creds.access_key}", 58 | f"export AWS_SECRET_ACCESS_KEY={creds.secret_key}", 59 | f"export AWS_SESSION_TOKEN={creds.token}", 60 | f"cd $(mktemp -d -t binaries-{incident}-`date +%s`-XXXXXX)", 61 | f"aws s3 --region {region} cp s3://{key} ./latest.tar.bz2 --quiet", 62 | "tar xvf latest.tar.bz2 &>/dev/null", 63 | "export PATH=${PATH}:${HOME}/.local/bin", 64 | "mkdir -p ${HOME}/.local/bin", 65 | "ln -s ./usr/bin/osqueryi ${HOME}/.local/bin/osqueryi", 66 | ] 67 | else: 68 | commands: List[str] = [ 69 | f"cd $(mktemp -d -t binaries-{incident}-`date +%s`-XXXXXX)" 70 | ] 71 | 72 | commands += CONFIG.get("DIFFY_PAYLOAD_OSQUERY_COMMANDS") 73 | return commands 74 | -------------------------------------------------------------------------------- /diffy/plugins/diffy_osquery/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Netflix-Skunkworks/diffy/1d8c1093023832be40d04021906f8779b37e8521/diffy/plugins/diffy_osquery/tests/__init__.py -------------------------------------------------------------------------------- /diffy/plugins/diffy_osquery/tests/conftest.py: -------------------------------------------------------------------------------- 1 | from diffy.tests.conftest import * # noqa 2 | -------------------------------------------------------------------------------- /diffy/schema.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: diffy.plugins.schema 3 | :platform: Unix 4 | :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | .. moduleauthor:: Kevin Glisson 7 | """ 8 | 9 | from inflection import underscore, camelize 10 | from marshmallow import fields, Schema, post_load, pre_load, post_dump 11 | from marshmallow.exceptions import ValidationError 12 | 13 | from diffy.config import CONFIG 14 | from diffy.plugins.base import plugins 15 | 16 | 17 | class DiffySchema(Schema): 18 | """ 19 | Base schema from which all diffy schema's inherit 20 | """ 21 | 22 | __envelope__ = True 23 | 24 | def under(self, data, many=None): 25 | items = [] 26 | if many: 27 | for i in data: 28 | items.append({underscore(key): value for key, value in i.items()}) 29 | return items 30 | return {underscore(key): value for key, value in data.items()} 31 | 32 | def camel(self, data, many=None): 33 | items = [] 34 | if many: 35 | for i in data: 36 | items.append( 37 | { 38 | camelize(key, uppercase_first_letter=False): value 39 | for key, value in i.items() 40 | } 41 | ) 42 | return items 43 | return { 44 | camelize(key, uppercase_first_letter=False): value 45 | for key, value in data.items() 46 | } 47 | 48 | def wrap_with_envelope(self, data, many): 49 | if many: 50 | if "total" in self.context.keys(): 51 | return dict(total=self.context["total"], items=data) 52 | return data 53 | 54 | 55 | class DiffyInputSchema(DiffySchema): 56 | @pre_load(pass_many=True) 57 | def preprocess(self, data, many): 58 | return self.under(data, many=many) 59 | 60 | 61 | class DiffyOutputSchema(DiffySchema): 62 | @pre_load(pass_many=True) 63 | def preprocess(self, data, many): 64 | if many: 65 | data = self.unwrap_envelope(data, many) 66 | return self.under(data, many=many) 67 | 68 | def unwrap_envelope(self, data, many): 69 | if many: 70 | if data["items"]: 71 | self.context["total"] = data["total"] 72 | else: 73 | self.context["total"] = 0 74 | data = {"items": []} 75 | 76 | return data["items"] 77 | 78 | return data 79 | 80 | @post_dump(pass_many=True) 81 | def post_process(self, data, many): 82 | if data: 83 | data = self.camel(data, many=many) 84 | if self.__envelope__: 85 | return self.wrap_with_envelope(data, many=many) 86 | else: 87 | return data 88 | 89 | 90 | def resolve_plugin_slug(slug): 91 | """Attempts to resolve plugin to slug.""" 92 | plugin = plugins.get(slug) 93 | 94 | if not plugin: 95 | raise ValidationError(f"Could not find plugin. Slug: {slug}") 96 | 97 | return plugin 98 | 99 | 100 | class PluginOptionSchema(Schema): 101 | options = fields.Dict(missing={}) 102 | 103 | 104 | class PluginSchema(DiffyInputSchema): 105 | options = fields.Dict(missing={}) 106 | 107 | @post_load 108 | def post_load(self, data): 109 | data["plugin"] = resolve_plugin_slug(data["slug"]) 110 | data["options"] = data["plugin"].validate_options(data["options"]) 111 | return data 112 | 113 | 114 | class TargetPluginSchema(PluginSchema): 115 | slug = fields.String( 116 | missing=CONFIG["DIFFY_TARGET_PLUGIN"], 117 | default=CONFIG["DIFFY_TARGET_PLUGIN"], 118 | required=True, 119 | ) 120 | 121 | 122 | class PersistencePluginSchema(PluginSchema): 123 | slug = fields.String( 124 | missing=CONFIG["DIFFY_PERSISTENCE_PLUGIN"], 125 | default=CONFIG["DIFFY_PERSISTENCE_PLUGIN"], 126 | required=True, 127 | ) 128 | 129 | 130 | class CollectionPluginSchema(PluginSchema): 131 | slug = fields.String( 132 | missing=CONFIG["DIFFY_COLLECTION_PLUGIN"], 133 | default=CONFIG["DIFFY_COLLECTION_PLUGIN"], 134 | required=True, 135 | ) 136 | 137 | 138 | class PayloadPluginSchema(PluginSchema): 139 | slug = fields.String( 140 | missing=CONFIG["DIFFY_PAYLOAD_PLUGIN"], 141 | default=CONFIG["DIFFY_PAYLOAD_PLUGIN"], 142 | required=True, 143 | ) 144 | 145 | 146 | class AnalysisPluginSchema(PluginSchema): 147 | slug = fields.String( 148 | missing=CONFIG["DIFFY_ANALYSIS_PLUGIN"], 149 | default=CONFIG["DIFFY_ANALYSIS_PLUGIN"], 150 | required=True, 151 | ) 152 | -------------------------------------------------------------------------------- /diffy_api/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: diffy 3 | :platform: Unix 4 | :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | 7 | .. moduleauthor:: Kevin Glisson 8 | 9 | 10 | """ 11 | import time 12 | from flask import g, request 13 | 14 | from diffy_api import factory 15 | from diffy_api.baseline.views import mod as baseline_bp 16 | from diffy_api.analysis.views import mod as analysis_bp 17 | from diffy_api.tasks.views import mod as task_bp 18 | 19 | DIFFY_BLUEPRINTS = (baseline_bp, analysis_bp, task_bp) 20 | 21 | 22 | def create_app(config=None): 23 | app = factory.create_app( 24 | app_name=__name__, blueprints=DIFFY_BLUEPRINTS, config=config 25 | ) 26 | configure_hook(app) 27 | return app 28 | 29 | 30 | def configure_hook(app): 31 | """ 32 | 33 | :param app: 34 | :return: 35 | """ 36 | from flask import jsonify 37 | from werkzeug.exceptions import HTTPException 38 | 39 | @app.errorhandler(Exception) 40 | def handle_error(e): 41 | code = 500 42 | if isinstance(e, HTTPException): 43 | code = e.code 44 | 45 | app.logger.exception(e) 46 | return jsonify(error=str(e)), code 47 | 48 | @app.before_request 49 | def before_request(): 50 | g.request_start_time = time.time() 51 | 52 | @app.after_request 53 | def after_request(response): 54 | # Return early if we don't have the start time 55 | if not hasattr(g, "request_start_time"): 56 | return response 57 | 58 | # Get elapsed time in milliseconds 59 | elapsed = time.time() - g.request_start_time 60 | elapsed = int(round(1000 * elapsed)) 61 | 62 | # Collect request/response tags 63 | # tags = { 64 | # 'endpoint': request.endpoint, 65 | # 'request_method': request.method.lower(), 66 | # 'status_code': response.status_code 67 | # } 68 | 69 | # Record our response time metric 70 | app.logger.debug( 71 | f"Request Info: Elapsed: {elapsed} Status Code: {response.status_code} Endpoint: {request.endpoint} Method: {request.method}" 72 | ) 73 | return response 74 | -------------------------------------------------------------------------------- /diffy_api/analysis/views.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: diffy.analysis.views 3 | :platform: Unix 4 | :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | .. moduleauthor:: Kevin Glisson 7 | """ 8 | from flask import Blueprint, current_app, request 9 | from flask_restful import reqparse, Api, Resource 10 | 11 | from diffy.plugins.base import plugins 12 | from diffy.exceptions import TargetNotFound 13 | 14 | from diffy_api.core import async_analysis 15 | from diffy_api.common.util import validate_schema 16 | from diffy_api.schemas import analysis_input_schema, task_output_schema 17 | 18 | mod = Blueprint("analysis", __name__) 19 | api = Api(mod) 20 | 21 | 22 | class AnalysisList(Resource): 23 | """Defines the 'baselines' endpoints""" 24 | 25 | def __init__(self): 26 | self.reqparse = reqparse.RequestParser() 27 | super(AnalysisList, self).__init__() 28 | 29 | def get(self): 30 | """ 31 | .. http:get:: /analysis 32 | The current list of analysiss 33 | 34 | **Example request**: 35 | .. sourcecode:: http 36 | GET /analysis HTTP/1.1 37 | Host: example.com 38 | Accept: application/json, text/javascript 39 | 40 | **Example response**: 41 | .. sourcecode:: http 42 | HTTP/1.1 200 OK 43 | Vary: Accept 44 | Content-Type: text/javascript 45 | 46 | # TODO 47 | 48 | :statuscode 200: no error 49 | :statuscode 403: unauthenticated 50 | """ 51 | data = plugins.get(current_app.config["DIFFY_PERSISTENCE_PLUGIN"]).get_all( 52 | "analysis" 53 | ) 54 | return data, 200 55 | 56 | @validate_schema(analysis_input_schema, task_output_schema) 57 | def post(self, data=None): 58 | """ 59 | .. http:post:: /analysis 60 | The current list of analysiss 61 | 62 | **Example request**: 63 | .. sourcecode:: http 64 | GET /analysis HTTP/1.1 65 | Host: example.com 66 | Accept: application/json, text/javascript 67 | 68 | **Example response**: 69 | .. sourcecode:: http 70 | HTTP/1.1 200 OK 71 | Vary: Accept 72 | Content-Type: text/javascript 73 | 74 | # TODO 75 | 76 | :statuscode 200: no error 77 | :statuscode 403: unauthenticated 78 | """ 79 | try: 80 | return async_analysis.queue(request.json) 81 | except TargetNotFound as ex: 82 | return {"message": ex.message}, 404 83 | 84 | 85 | class Analysis(Resource): 86 | """Defines the 'baselines' endpoints""" 87 | 88 | def __init__(self): 89 | super(Analysis, self).__init__() 90 | 91 | def get(self, key): 92 | """ 93 | .. http:get:: /analysis 94 | The current list of analysiss 95 | 96 | **Example request**: 97 | .. sourcecode:: http 98 | GET /analysis HTTP/1.1 99 | Host: example.com 100 | Accept: application/json, text/javascript 101 | 102 | **Example response**: 103 | .. sourcecode:: http 104 | HTTP/1.1 200 OK 105 | Vary: Accept 106 | Content-Type: text/javascript 107 | 108 | # TODO 109 | 110 | :statuscode 200: no error 111 | :statuscode 403: unauthenticated 112 | """ 113 | plugin_slug = current_app.config["DIFFY_PERSISTENCE_PLUGIN"] 114 | p = plugins.get(plugin_slug) 115 | return p.get("analysis", key) 116 | 117 | 118 | api.add_resource(AnalysisList, "/analysis", endpoint="analysisList") 119 | api.add_resource(Analysis, "/analysis/", endpoint="analysis") 120 | -------------------------------------------------------------------------------- /diffy_api/baseline/views.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: diffy.baseline.views 3 | :platform: Unix 4 | :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | .. moduleauthor:: Kevin Glisson 7 | """ 8 | from flask import Blueprint, current_app, request 9 | from flask_restful import Api, Resource 10 | 11 | from diffy.plugins.base import plugins 12 | from diffy.exceptions import TargetNotFound 13 | from diffy_api.core import async_baseline 14 | from diffy_api.common.util import validate_schema 15 | from diffy_api.schemas import ( 16 | baseline_input_schema, 17 | baseline_output_schema, 18 | task_output_schema, 19 | ) 20 | 21 | 22 | mod = Blueprint("baselines", __name__) 23 | api = Api(mod) 24 | 25 | 26 | class BaselineList(Resource): 27 | """Defines the 'baselines' endpoints""" 28 | 29 | def __init__(self): 30 | super(BaselineList, self).__init__() 31 | 32 | @validate_schema(None, baseline_output_schema) 33 | def get(self): 34 | """ 35 | .. http:get:: /baselines 36 | The current list of baselines 37 | 38 | **Example request**: 39 | .. sourcecode:: http 40 | GET /baselines HTTP/1.1 41 | Host: example.com 42 | Accept: application/json, text/javascript 43 | 44 | **Example response**: 45 | .. sourcecode:: http 46 | HTTP/1.1 200 OK 47 | Vary: Accept 48 | Content-Type: text/javascript 49 | 50 | # TODO 51 | 52 | :statuscode 200: no error 53 | :statuscode 403: unauthenticated 54 | """ 55 | data = plugins.get(current_app.config["DIFFY_PERSISTENCE_PLUGIN"]).get_all( 56 | "baseline" 57 | ) 58 | return data, 200 59 | 60 | @validate_schema(baseline_input_schema, task_output_schema) 61 | def post(self, data=None): 62 | """ 63 | .. http:post:: /baselines 64 | The current list of baselines 65 | 66 | **Example request**: 67 | .. sourcecode:: http 68 | POST /baselines HTTP/1.1 69 | Host: example.com 70 | Accept: application/json, text/javascript 71 | 72 | **Example response**: 73 | .. sourcecode:: http 74 | HTTP/1.1 200 OK 75 | Vary: Accept 76 | Content-Type: text/javascript 77 | 78 | # TODO 79 | 80 | :statuscode 200: no error 81 | :statuscode 403: unauthenticated 82 | """ 83 | try: 84 | return async_baseline.queue(request.json) 85 | except TargetNotFound as ex: 86 | return {"message": ex.message}, 404 87 | 88 | 89 | class Baseline(Resource): 90 | """Defines the 'baselines' endpoints""" 91 | 92 | def __init__(self): 93 | super(Baseline, self).__init__() 94 | 95 | def get(self, key): 96 | """ 97 | .. http:get:: /baselines 98 | The current list of baselines 99 | 100 | **Example request**: 101 | .. sourcecode:: http 102 | GET /baselines HTTP/1.1 103 | Host: example.com 104 | Accept: application/json, text/javascript 105 | 106 | **Example response**: 107 | .. sourcecode:: http 108 | HTTP/1.1 200 OK 109 | Vary: Accept 110 | Content-Type: text/javascript 111 | 112 | # TODO 113 | 114 | :statuscode 200: no error 115 | :statuscode 403: unauthenticated 116 | """ 117 | return plugins.get(current_app.config["DIFFY_PERSISTENCE_PLUGIN"]).get( 118 | "baseline", key 119 | ) 120 | 121 | 122 | api.add_resource(Baseline, "/baselines/") 123 | api.add_resource(BaselineList, "/baselines", endpoint="baselines") 124 | -------------------------------------------------------------------------------- /diffy_api/common/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Netflix-Skunkworks/diffy/1d8c1093023832be40d04021906f8779b37e8521/diffy_api/common/__init__.py -------------------------------------------------------------------------------- /diffy_api/common/health.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: diffy.common.health 3 | :platform: Unix 4 | :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | .. moduleauthor:: Kevin Glisson 7 | """ 8 | from flask import Blueprint 9 | 10 | mod = Blueprint("healthCheck", __name__) 11 | 12 | 13 | @mod.route("/healthcheck") 14 | def health(): 15 | return "ok" 16 | -------------------------------------------------------------------------------- /diffy_api/common/util.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | from flask import request, current_app 4 | from functools import wraps 5 | from inflection import camelize 6 | 7 | from diffy_api.extensions import sentry 8 | 9 | 10 | def format_errors(messages: List[str]) -> dict: 11 | errors = {} 12 | for k, v in messages.items(): 13 | key = camelize(k, uppercase_first_letter=False) 14 | 15 | if isinstance(v, dict): 16 | errors[key] = format_errors(v) 17 | 18 | elif isinstance(v, list): 19 | errors[key] = v[0] 20 | 21 | return errors 22 | 23 | 24 | def wrap_errors(messages): 25 | errors = dict(message="Validation Error.") 26 | if messages.get("_schema"): 27 | errors["reasons"] = {"Schema": {"rule": messages["_schema"]}} 28 | else: 29 | errors["reasons"] = format_errors(messages) 30 | return errors 31 | 32 | 33 | def unwrap_pagination(data, output_schema): 34 | if not output_schema: 35 | return data 36 | 37 | if isinstance(data, dict): 38 | if "total" in data.keys(): 39 | if data.get("total") == 0: 40 | return data 41 | 42 | marshaled_data = {"total": data["total"]} 43 | marshaled_data["items"] = output_schema.dump(data["items"], many=True).data 44 | return marshaled_data 45 | 46 | return output_schema.dump(data).data 47 | 48 | elif isinstance(data, list): 49 | marshaled_data = {"total": len(data)} 50 | marshaled_data["items"] = output_schema.dump(data, many=True).data 51 | return marshaled_data 52 | 53 | return output_schema.dump(data).data 54 | 55 | 56 | def validate_schema(input_schema, output_schema): 57 | def decorator(f): 58 | @wraps(f) 59 | def decorated_function(*args, **kwargs): 60 | if input_schema: 61 | if request.get_json(): 62 | request_data = request.get_json() 63 | else: 64 | request_data = request.args 65 | 66 | data, errors = input_schema.load(request_data) 67 | 68 | if errors: 69 | return wrap_errors(errors), 400 70 | 71 | kwargs["data"] = data 72 | 73 | try: 74 | resp = f(*args, **kwargs) 75 | except Exception as e: 76 | sentry.captureException() 77 | current_app.logger.exception(e) 78 | return dict(message=str(e)), 500 79 | 80 | if isinstance(resp, tuple): 81 | return resp[0], resp[1] 82 | 83 | if not resp: 84 | return dict(message="No data found"), 404 85 | 86 | return unwrap_pagination(resp, output_schema), 200 87 | 88 | return decorated_function 89 | 90 | return decorator 91 | -------------------------------------------------------------------------------- /diffy_api/core.py: -------------------------------------------------------------------------------- 1 | """Core API functions.""" 2 | from diffy.core import baseline, analysis 3 | from diffy_api.extensions import rq 4 | from diffy_api.schemas import baseline_input_schema, analysis_input_schema 5 | 6 | 7 | @rq.job() 8 | def async_baseline(kwargs): 9 | """Wrap our standard baseline task.""" 10 | # we can't pickle our objects for remote works so we pickle the raw request 11 | # and then load it here. 12 | data = baseline_input_schema.load(kwargs).data 13 | return baseline(**data) 14 | 15 | 16 | @rq.job() 17 | def async_analysis(kwargs): 18 | """Wrap our standard analysis task.""" 19 | # we can't pickle our objects for remote works so we pickle the raw request 20 | # and then load it here. 21 | data = analysis_input_schema.load(kwargs).data 22 | return analysis(**data) 23 | -------------------------------------------------------------------------------- /diffy_api/extensions.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: diffy_api.extensions 3 | :platform: Unix 4 | :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | """ 7 | from raven.contrib.flask import Sentry 8 | 9 | sentry = Sentry() 10 | 11 | from flask_rq2 import RQ 12 | 13 | rq = RQ() 14 | -------------------------------------------------------------------------------- /diffy_api/factory.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: diffy.factory 3 | :platform: Unix 4 | :synopsis: This module contains all the needed functions to allow 5 | the factory app creation. 6 | 7 | :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more 8 | :license: Apache, see LICENSE for more details. 9 | .. moduleauthor:: Kevin Glisson 10 | 11 | """ 12 | from pprint import pformat 13 | 14 | from logging import Formatter, StreamHandler, DEBUG 15 | from logging.handlers import RotatingFileHandler 16 | 17 | from flask import Flask 18 | 19 | from diffy.config import Config, consume_envvars, DEFAULTS 20 | from diffy.common.utils import install_plugins 21 | 22 | from diffy_api.common.health import mod as health 23 | from diffy_api.extensions import sentry, rq 24 | 25 | 26 | DEFAULT_BLUEPRINTS = (health,) 27 | 28 | API_VERSION = 1 29 | 30 | 31 | def create_app(app_name=None, blueprints=None, config=None): 32 | """ 33 | Diffy application factory 34 | 35 | :param config: 36 | :param app_name: 37 | :param blueprints: 38 | :return: 39 | """ 40 | if not blueprints: 41 | blueprints = DEFAULT_BLUEPRINTS 42 | else: 43 | blueprints = blueprints + DEFAULT_BLUEPRINTS 44 | 45 | if not app_name: 46 | app_name = __name__ 47 | 48 | app = Flask(app_name) 49 | configure_app(app, config) 50 | configure_blueprints(app, blueprints) 51 | configure_extensions(app) 52 | configure_logging(app) 53 | 54 | if app.logger.isEnabledFor(DEBUG): 55 | p_config = pformat(app.config) 56 | app.logger.debug(f"Current Configuration: {p_config}") 57 | 58 | return app 59 | 60 | 61 | def configure_app(app, config=None): 62 | """ 63 | Different ways of configuration 64 | 65 | :param app: 66 | :param config: 67 | :return: 68 | """ 69 | install_plugins() 70 | 71 | _defaults = consume_envvars(DEFAULTS) 72 | default_config = Config(defaults=_defaults) 73 | 74 | if config: 75 | default_config.from_yaml(config) 76 | 77 | app.config.update(default_config) 78 | app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False 79 | 80 | 81 | def configure_extensions(app): 82 | """ 83 | Attaches and configures any needed flask extensions 84 | to our app. 85 | 86 | :param app: 87 | """ 88 | sentry.init_app(app) 89 | rq.init_app(app) 90 | 91 | 92 | def configure_blueprints(app, blueprints): 93 | """ 94 | We prefix our APIs with their given version so that we can support 95 | multiple concurrent API versions. 96 | 97 | :param app: 98 | :param blueprints: 99 | """ 100 | for blueprint in blueprints: 101 | app.register_blueprint(blueprint, url_prefix=f"/api/{API_VERSION}") 102 | 103 | 104 | def configure_logging(app): 105 | """ 106 | Sets up application wide logging. 107 | 108 | :param app: 109 | """ 110 | if app.config.get("LOG_FILE"): 111 | handler = RotatingFileHandler( 112 | app.config.get("LOG_FILE"), maxBytes=10_000_000, backupCount=100 113 | ) 114 | 115 | handler.setFormatter( 116 | Formatter( 117 | "%(asctime)s %(levelname)s: %(message)s " "[in %(pathname)s:%(lineno)d]" 118 | ) 119 | ) 120 | 121 | handler.setLevel(app.config.get("LOG_LEVEL", "DEBUG")) 122 | app.logger.setLevel(app.config.get("LOG_LEVEL", "DEBUG")) 123 | app.logger.addHandler(handler) 124 | 125 | stream_handler = StreamHandler() 126 | 127 | stream_handler.setFormatter( 128 | Formatter( 129 | "%(asctime)s %(levelname)s: %(message)s " "[in %(pathname)s:%(lineno)d]" 130 | ) 131 | ) 132 | 133 | stream_handler.setLevel(app.config.get("LOG_LEVEL", "DEBUG")) 134 | app.logger.addHandler(stream_handler) 135 | -------------------------------------------------------------------------------- /diffy_api/plugins/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Netflix-Skunkworks/diffy/1d8c1093023832be40d04021906f8779b37e8521/diffy_api/plugins/__init__.py -------------------------------------------------------------------------------- /diffy_api/plugins/views.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Netflix-Skunkworks/diffy/1d8c1093023832be40d04021906f8779b37e8521/diffy_api/plugins/views.py -------------------------------------------------------------------------------- /diffy_api/schemas.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: diffy.schemas 3 | :platform: Unix 4 | :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | .. moduleauthor:: Kevin Glisson 7 | """ 8 | from marshmallow import fields 9 | 10 | from diffy.schema import ( 11 | TargetPluginSchema, 12 | PersistencePluginSchema, 13 | CollectionPluginSchema, 14 | PayloadPluginSchema, 15 | AnalysisPluginSchema, 16 | DiffyInputSchema, 17 | DiffyOutputSchema, 18 | ) 19 | 20 | 21 | class BaselineSchema(DiffyInputSchema): 22 | target_key = fields.String(required=True) 23 | incident_id = fields.String(required=True) 24 | target_plugin = fields.Nested(TargetPluginSchema, missing={}) 25 | persistence_plugin = fields.Nested(PersistencePluginSchema, missing={}) 26 | collection_plugin = fields.Nested(CollectionPluginSchema, missing={}) 27 | payload_plugin = fields.Nested(PayloadPluginSchema, missing={}) 28 | 29 | 30 | class AnalysisSchema(BaselineSchema): 31 | analysis_plugin = fields.Nested(AnalysisPluginSchema, missing={}) 32 | # TODO: move incident_id from BaselineSchema here 33 | 34 | 35 | class TaskOutputSchema(DiffyOutputSchema): 36 | id = fields.String(attribute="id") 37 | created_at = fields.DateTime() 38 | args = fields.Dict() 39 | status = fields.String(attribute="status") 40 | 41 | 42 | class TaskInputSchema(DiffyInputSchema): 43 | id = fields.String(required=True) 44 | 45 | 46 | baseline_input_schema = BaselineSchema() 47 | baseline_output_schema = BaselineSchema() 48 | analysis_input_schema = AnalysisSchema() 49 | analysis_output_schema = AnalysisSchema() 50 | task_output_schema = TaskOutputSchema() 51 | task_list_output_schema = TaskOutputSchema(many=True) 52 | task_input_schema = TaskInputSchema() 53 | -------------------------------------------------------------------------------- /diffy_api/tasks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Netflix-Skunkworks/diffy/1d8c1093023832be40d04021906f8779b37e8521/diffy_api/tasks/__init__.py -------------------------------------------------------------------------------- /diffy_api/tasks/views.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: diffy.tasks.views 3 | :platform: Unix 4 | :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | .. moduleauthor:: Kevin Glisson 7 | """ 8 | from flask import Blueprint 9 | from flask_restful import Api, Resource 10 | 11 | from diffy_api.extensions import rq 12 | from diffy_api.common.util import validate_schema 13 | from diffy_api.schemas import task_output_schema, task_list_output_schema 14 | 15 | 16 | mod = Blueprint("tasks", __name__) 17 | api = Api(mod) 18 | 19 | 20 | class TaskList(Resource): 21 | """Defines the 'taskss' endpoints""" 22 | 23 | def __init__(self): 24 | super(TaskList, self).__init__() 25 | 26 | @validate_schema(None, task_list_output_schema) 27 | def get(self): 28 | """ 29 | .. http:get:: /tasks 30 | The current list of tasks 31 | 32 | **Example request**: 33 | .. sourcecode:: http 34 | GET /tasks HTTP/1.1 35 | Host: example.com 36 | Accept: application/json, text/javascript 37 | 38 | **Example response**: 39 | .. sourcecode:: http 40 | HTTP/1.1 200 OK 41 | Vary: Accept 42 | Content-Type: text/javascript 43 | 44 | # TODO 45 | 46 | :statuscode 200: no error 47 | :statuscode 403: unauthenticated 48 | """ 49 | queue = rq.get_queue() 50 | data = queue.get_jobs() 51 | return data, 200 52 | 53 | 54 | class Task(Resource): 55 | """Defines the 'taskss' endpoints""" 56 | 57 | def __init__(self): 58 | super(Task, self).__init__() 59 | 60 | @validate_schema(None, task_output_schema) 61 | def get(self, task_id): 62 | """ 63 | .. http:get:: /tasks 64 | The current list of tasks 65 | 66 | **Example request**: 67 | .. sourcecode:: http 68 | GET /tasks HTTP/1.1 69 | Host: example.com 70 | Accept: application/json, text/javascript 71 | 72 | **Example response**: 73 | .. sourcecode:: http 74 | HTTP/1.1 200 OK 75 | Vary: Accept 76 | Content-Type: text/javascript 77 | 78 | # TODO 79 | 80 | :statuscode 200: no error 81 | :statuscode 403: unauthenticated 82 | """ 83 | queue = rq.get_queue() 84 | return queue.fetch_job(task_id) 85 | 86 | 87 | api.add_resource(Task, "/tasks/") 88 | api.add_resource(TaskList, "/tasks", endpoint="tasks") 89 | -------------------------------------------------------------------------------- /diffy_cli/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Netflix-Skunkworks/diffy/1d8c1093023832be40d04021906f8779b37e8521/diffy_cli/__init__.py -------------------------------------------------------------------------------- /diffy_cli/core.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import json 3 | import yaml 4 | import click 5 | import click_log 6 | 7 | from tabulate import tabulate 8 | 9 | from diffy.filters import AWSFilter 10 | 11 | from diffy.config import CONFIG, configure_swag 12 | from diffy.common.utils import install_plugins 13 | from diffy._version import __version__ 14 | from diffy.plugins.base import plugins 15 | from diffy.core import analysis, baseline 16 | from diffy.exceptions import DiffyException 17 | from diffy_cli.utils.dynamic_click import CORE_COMMANDS, func_factory, params_factory 18 | 19 | log = logging.getLogger("diffy") 20 | log.addFilter(AWSFilter()) 21 | 22 | click_log.basic_config(log) 23 | 24 | install_plugins() 25 | 26 | 27 | def plugin_command_factory(): 28 | """Dynamically generate plugin groups for all plugins, and add all basic command to it""" 29 | for p in plugins.all(): 30 | plugin_name = p.slug 31 | help = f"Options for '{plugin_name}'" 32 | group = click.Group(name=plugin_name, help=help) 33 | for name, description in CORE_COMMANDS.items(): 34 | callback = func_factory(p, name) 35 | pretty_opt = click.Option( 36 | ["--pretty/--not-pretty"], help="Output a pretty version of the JSON" 37 | ) 38 | params = [pretty_opt] 39 | command = click.Command( 40 | name, 41 | callback=callback, 42 | help=description.format(plugin_name), 43 | params=params, 44 | ) 45 | group.add_command(command) 46 | 47 | plugins_group.add_command(group) 48 | 49 | 50 | def get_plugin_properties(json_schema): 51 | for k, v in json_schema["definitions"].items(): 52 | return v["properties"] 53 | 54 | 55 | def add_plugins_args(f): 56 | """Adds installed plugin options.""" 57 | schemas = [] 58 | if isinstance(f, click.Command): 59 | for p in plugins.all(): 60 | schemas.append(get_plugin_properties(p.json_schema)) 61 | f.params.extend(params_factory(schemas)) 62 | else: 63 | if not hasattr(f, "__click_params__"): 64 | f.__click_params__ = [] 65 | 66 | for p in plugins.all(): 67 | schemas.append(get_plugin_properties(p.json_schema)) 68 | f.__click_params__.extend(params_factory(schemas)) 69 | return f 70 | 71 | 72 | class YAML(click.ParamType): 73 | name = "yaml" 74 | 75 | def convert(self, value: str, param: str, ctx: object) -> dict: 76 | try: 77 | with open(value, "rb") as f: 78 | return yaml.safe_load(f.read()) 79 | except (IOError, OSError) as e: 80 | self.fail(f"Could not open file: {value}") 81 | 82 | 83 | def get_plugin_callback(ctx: object, param: str, value: str) -> object: 84 | """Ensures that the plugin selected is available.""" 85 | for p in plugins.all(plugin_type=param.name.split("_")[0]): 86 | if p.slug == value: 87 | return {"plugin": p, "options": {}} 88 | 89 | raise click.BadParameter( 90 | f"Could not find appropriate plugin. Param: {param.name} Value: {value}" 91 | ) 92 | 93 | 94 | @click.group() 95 | @click_log.simple_verbosity_option(log) 96 | @click.option("--config", type=YAML(), help="Configuration file to use.") 97 | @click.option( 98 | "--dry-run", 99 | type=bool, 100 | default=False, 101 | is_flag=True, 102 | help="Run command without persisting anything.", 103 | ) 104 | @click.version_option(version=__version__) 105 | @click.pass_context 106 | def diffy_cli(ctx, config, dry_run): 107 | return 108 | 109 | 110 | # if not ctx.dry_run: 111 | # ctx.dry_run = dry_run 112 | # 113 | # if config: 114 | # CONFIG.from_yaml(config) 115 | # 116 | # log.debug(f'Current context. DryRun: {ctx.dry_run} Config: {json.dumps(CONFIG, indent=2)}') 117 | 118 | 119 | @diffy_cli.group("plugins") 120 | def plugins_group(): 121 | pass 122 | 123 | 124 | @plugins_group.command("list") 125 | def list_plugins(): 126 | """Shows all available plugins""" 127 | table = [] 128 | for p in plugins.all(): 129 | table.append([p.title, p.slug, p.version, p.author, p.description]) 130 | click.echo( 131 | tabulate(table, headers=["Title", "Slug", "Version", "Author", "Description"]) 132 | ) 133 | 134 | 135 | @diffy_cli.group() 136 | def new(): 137 | pass 138 | 139 | 140 | @new.command('baseline') 141 | @click.argument('target-key') 142 | @click.option('--target-plugin', default='local-target', callback=get_plugin_callback) 143 | # TODO: Make 'target' and 'inventory' plugins mutually exclusive 144 | # TODO: Consider an 'async' flag 145 | @click.option('--inventory-plugin', callback=get_plugin_callback) 146 | @click.option('--persistence-plugin', default='local-file', callback=get_plugin_callback) 147 | @click.option('--payload-plugin', default='local-command', callback=get_plugin_callback) 148 | @click.option('--collection-plugin', default='local-shell-collection', callback=get_plugin_callback) 149 | @click.option('--incident-id', default='None') 150 | @add_plugins_args 151 | def baseline_command( 152 | target_key, 153 | target_plugin, 154 | inventory_plugin, 155 | persistence_plugin, 156 | payload_plugin, 157 | collection_plugin, 158 | incident_id, 159 | **kwargs, 160 | ): 161 | """Creates a new baseline based on the given ASG.""" 162 | if inventory_plugin: 163 | for item in inventory_plugin['plugin'].get(): 164 | target_plugin['options'] = item['options'] 165 | baseline( 166 | item['target'], 167 | item['target_plugin'], 168 | payload_plugin, 169 | collection_plugin, 170 | persistence_plugin, 171 | incident_id=incident_id, 172 | **kwargs, 173 | ) 174 | 175 | baselines = baseline( 176 | target_key, 177 | target_plugin, 178 | payload_plugin, 179 | collection_plugin, 180 | persistence_plugin, 181 | incident_id=incident_id, 182 | **kwargs, 183 | ) 184 | click.secho(json.dumps(baselines), fg="green") 185 | 186 | 187 | @new.command('analysis') 188 | @click.argument('target-key') 189 | @click.option('--analysis-plugin', default='local-simple', callback=get_plugin_callback) 190 | @click.option('--payload-plugin', default='local-command', callback=get_plugin_callback) 191 | @click.option('--target-plugin', default='local-target', callback=get_plugin_callback) 192 | @click.option('--persistence-plugin', default='local-file', callback=get_plugin_callback) 193 | @click.option('--collection-plugin', default='local-shell-collection', callback=get_plugin_callback) 194 | @click.option('--incident-id', default='') 195 | @add_plugins_args 196 | def analysis_command(target_key, analysis_plugin, target_plugin, persistence_plugin, collection_plugin, payload_plugin, 197 | incident_id, **kwargs): 198 | """Creates a new analysis based on collected data.""" 199 | result = analysis( 200 | target_key, 201 | target_plugin, 202 | payload_plugin, 203 | collection_plugin, 204 | persistence_plugin, 205 | analysis_plugin, 206 | **kwargs, 207 | ) 208 | 209 | for r in result['analysis']: 210 | if r['diff']: 211 | click.secho( 212 | r['instance_id'] + ': Differences found.', 213 | fg='red') 214 | click.secho(json.dumps(r['diff'], indent=2), fg='red') 215 | else: 216 | click.secho(r["instance_id"] + ": No Differences Found.", fg="green") 217 | 218 | 219 | def entry_point(): 220 | """The entry that CLI is executed from""" 221 | try: 222 | configure_swag() 223 | plugin_command_factory() 224 | diffy_cli(obj={"dry_run": True}) 225 | except DiffyException as e: 226 | click.secho(f"ERROR: {e.message}", bold=True, fg="red") 227 | exit(1) 228 | 229 | 230 | if __name__ == "__main__": 231 | entry_point() 232 | -------------------------------------------------------------------------------- /diffy_cli/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Netflix-Skunkworks/diffy/1d8c1093023832be40d04021906f8779b37e8521/diffy_cli/utils/__init__.py -------------------------------------------------------------------------------- /diffy_cli/utils/dynamic_click.py: -------------------------------------------------------------------------------- 1 | import json 2 | from functools import partial 3 | from typing import List 4 | 5 | import click 6 | 7 | from diffy_cli.utils.json_schema import ( 8 | COMPLEX_TYPES, 9 | json_schema_to_click_type, 10 | handle_oneof, 11 | ) 12 | 13 | CORE_COMMANDS = { 14 | "required": "'{}' required schema", 15 | "schema": "'{}' full schema", 16 | "metadata": "'{}' metadata", 17 | "defaults": "'{}' default values", 18 | } 19 | 20 | 21 | # TODO figure out how to validate across opts 22 | def validate_schema_callback(ctx, param, value): 23 | """Ensures options passed fulfill what plugins are expecting.""" 24 | return value 25 | 26 | 27 | def params_factory(schemas: List[dict]) -> list: 28 | """ 29 | Generates list of :class:`click.Option` based on a JSON schema 30 | 31 | :param schemas: JSON schemas to operate on 32 | :return: Lists of created :class:`click.Option` object to be added to a :class:`click.Command` 33 | """ 34 | params = [] 35 | unique_decls = [] 36 | for schema in schemas: 37 | for prpty, prpty_schema in schema.items(): 38 | multiple = False 39 | choices = None 40 | 41 | if any(char in prpty for char in ["@"]): 42 | continue 43 | 44 | if prpty_schema.get("type") in COMPLEX_TYPES: 45 | continue 46 | 47 | if prpty_schema.get("duplicate"): 48 | continue 49 | 50 | elif not prpty_schema.get("oneOf"): 51 | click_type, description, choices = json_schema_to_click_type( 52 | prpty_schema 53 | ) 54 | else: 55 | click_type, multiple, description = handle_oneof(prpty_schema["oneOf"]) 56 | # Not all oneOf schema can be handled by click 57 | if not click_type: 58 | continue 59 | 60 | # Convert bool values into flags 61 | if click_type == click.BOOL: 62 | param_decls = [get_flag_param_decals_from_bool(prpty)] 63 | click_type = None 64 | else: 65 | param_decls = [get_param_decals_from_name(prpty)] 66 | 67 | if description: 68 | description = description.capitalize() 69 | 70 | if multiple: 71 | if not description.endswith("."): 72 | description += "." 73 | description += " Multiple usages of this option are allowed" 74 | 75 | param_decls = [x for x in param_decls if x not in unique_decls] 76 | if not param_decls: 77 | continue 78 | 79 | unique_decls += param_decls 80 | option = partial( 81 | click.Option, 82 | param_decls=param_decls, 83 | help=description, 84 | default=prpty_schema.get("default"), 85 | callback=validate_schema_callback, 86 | multiple=multiple, 87 | ) 88 | 89 | if choices: 90 | option = option(type=choices) 91 | elif click_type: 92 | option = option(type=click_type) 93 | else: 94 | option = option() 95 | 96 | params.append(option) 97 | return params 98 | 99 | 100 | def func_factory(p, method: str) -> callable: 101 | """ 102 | Dynamically generates callback commands to correlate to provider public methods 103 | 104 | :param p: A :class:`notifiers.core.Provider` object 105 | :param method: A string correlating to a provider method 106 | :return: A callback func 107 | """ 108 | 109 | def callback(pretty: bool = False): 110 | res = getattr(p, method) 111 | dump = partial(json.dumps, indent=4) if pretty else partial(json.dumps) 112 | click.echo(dump(res)) 113 | 114 | return callback 115 | 116 | 117 | def get_param_decals_from_name(option_name: str) -> str: 118 | """Converts a name to a param name""" 119 | name = option_name.replace("_", "-") 120 | return f"--{name}" 121 | 122 | 123 | def get_flag_param_decals_from_bool(option_name: str) -> str: 124 | """Return a '--do/not-do' style flag param""" 125 | name = option_name.replace("_", "-") 126 | return f"--{name}/--no-{name}" 127 | -------------------------------------------------------------------------------- /diffy_cli/utils/json_schema.py: -------------------------------------------------------------------------------- 1 | import click 2 | 3 | SCHEMA_BASE_MAP = { 4 | "string": click.STRING, 5 | "integer": click.INT, 6 | "number": click.FLOAT, 7 | "boolean": click.BOOL, 8 | } 9 | COMPLEX_TYPES = ["object", "array"] 10 | 11 | 12 | def handle_oneof(oneof_schema: list) -> tuple: 13 | """ 14 | Custom handle of `oneOf` JSON schema validator. Tried to match primitive type and see if it should be allowed 15 | to be passed multiple timns into a command 16 | 17 | :param oneof_schema: `oneOf` JSON schema 18 | :return: Tuple of :class:`click.ParamType`, ``multiple`` flag and ``description`` of option 19 | """ 20 | oneof_dict = {schema["type"]: schema for schema in oneof_schema} 21 | click_type = None 22 | multiple = False 23 | description = None 24 | for key, value in oneof_dict.items(): 25 | if key == "array": 26 | continue 27 | elif key in SCHEMA_BASE_MAP: 28 | if oneof_dict.get("array") and oneof_dict["array"]["items"]["type"] == key: 29 | multiple = True 30 | # Found a match to a primitive type 31 | click_type = SCHEMA_BASE_MAP[key] 32 | description = value.get("title") 33 | break 34 | return click_type, multiple, description 35 | 36 | 37 | def json_schema_to_click_type(schema: dict) -> tuple: 38 | """ 39 | A generic handler of a single property JSON schema to :class:`click.ParamType` converter 40 | 41 | :param schema: JSON schema property to operate on 42 | :return: Tuple of :class:`click.ParamType`, `description`` of option and optionally a :class:`click.Choice` 43 | if the allowed values are a closed list (JSON schema ``enum``) 44 | """ 45 | choices = None 46 | if isinstance(schema["type"], list): 47 | if "string" in schema["type"]: 48 | schema["type"] = "string" 49 | click_type = SCHEMA_BASE_MAP[schema["type"]] 50 | description = schema.get("title") 51 | if schema.get("enum"): 52 | choices = click.Choice(schema["enum"]) 53 | return click_type, description, choices 54 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | SPHINXPROJ = Diffy 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -------------------------------------------------------------------------------- /docs/changelog.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../CHANGELOG.rst -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Configuration file for the Sphinx documentation builder. 4 | # 5 | # This file does only contain a selection of the most common options. For a 6 | # full list see the documentation: 7 | # http://www.sphinx-doc.org/en/stable/config 8 | 9 | # -- Path setup -------------------------------------------------------------- 10 | 11 | # If extensions (or modules to document with autodoc) are in another directory, 12 | # add these directories to sys.path here. If the directory is relative to the 13 | # documentation root, use os.path.abspath to make it absolute, like shown here. 14 | # 15 | # import os 16 | # import sys 17 | # sys.path.insert(0, os.path.abspath('.')) 18 | from recommonmark.parser import CommonMarkParser 19 | 20 | # -- Project information ----------------------------------------------------- 21 | 22 | project = 'Diffy' 23 | copyright = '2018, Netflix' 24 | author = 'Netflix' 25 | 26 | # The short X.Y version 27 | version = '' 28 | # The full version, including alpha/beta/rc tags 29 | release = '' 30 | 31 | source_parsers = { 32 | '.md': CommonMarkParser, 33 | } 34 | 35 | 36 | # -- General configuration --------------------------------------------------- 37 | 38 | # If your documentation needs a minimal Sphinx version, state it here. 39 | # 40 | # needs_sphinx = '1.0' 41 | 42 | # Add any Sphinx extension module names here, as strings. They can be 43 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 44 | # ones. 45 | extensions = [ 46 | ] 47 | 48 | # Add any paths that contain templates here, relative to this directory. 49 | templates_path = ['_templates'] 50 | 51 | # The suffix(es) of source filenames. 52 | # You can specify multiple suffix as a list of string: 53 | # 54 | source_suffix = ['.rst', '.md'] 55 | 56 | # The master toctree document. 57 | master_doc = 'index' 58 | 59 | # The language for content autogenerated by Sphinx. Refer to documentation 60 | # for a list of supported languages. 61 | # 62 | # This is also used if you do content translation via gettext catalogs. 63 | # Usually you set "language" from the command line for these cases. 64 | language = None 65 | 66 | # List of patterns, relative to source directory, that match files and 67 | # directories to ignore when looking for source files. 68 | # This pattern also affects html_static_path and html_extra_path . 69 | exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] 70 | 71 | # The name of the Pygments (syntax highlighting) style to use. 72 | pygments_style = 'sphinx' 73 | 74 | 75 | # -- Options for HTML output ------------------------------------------------- 76 | 77 | # The theme to use for HTML and HTML Help pages. See the documentation for 78 | # a list of builtin themes. 79 | # 80 | html_theme = 'sphinx_rtd_theme' 81 | 82 | # Theme options are theme-specific and customize the look and feel of a theme 83 | # further. For a list of options available for each theme, see the 84 | # documentation. 85 | # 86 | # html_theme_options = {} 87 | 88 | # Add any paths that contain custom static files (such as style sheets) here, 89 | # relative to this directory. They are copied after the builtin static files, 90 | # so a file named "default.css" will overwrite the builtin "default.css". 91 | html_static_path = ['_static'] 92 | 93 | # Custom sidebar templates, must be a dictionary that maps document names 94 | # to template names. 95 | # 96 | # The default sidebars (for documents that don't match any pattern) are 97 | # defined by theme itself. Builtin themes are using these templates by 98 | # default: ``['localtoc.html', 'relations.html', 'sourcelink.html', 99 | # 'searchbox.html']``. 100 | # 101 | # html_sidebars = {} 102 | 103 | 104 | # -- Options for HTMLHelp output --------------------------------------------- 105 | 106 | # Output file base name for HTML help builder. 107 | htmlhelp_basename = 'Diffydoc' 108 | 109 | 110 | # -- Options for LaTeX output ------------------------------------------------ 111 | 112 | latex_elements = { 113 | # The paper size ('letterpaper' or 'a4paper'). 114 | # 115 | # 'papersize': 'letterpaper', 116 | 117 | # The font size ('10pt', '11pt' or '12pt'). 118 | # 119 | # 'pointsize': '10pt', 120 | 121 | # Additional stuff for the LaTeX preamble. 122 | # 123 | # 'preamble': '', 124 | 125 | # Latex figure (float) alignment 126 | # 127 | # 'figure_align': 'htbp', 128 | } 129 | 130 | # Grouping the document tree into LaTeX files. List of tuples 131 | # (source start file, target name, title, 132 | # author, documentclass [howto, manual, or own class]). 133 | latex_documents = [ 134 | (master_doc, 'Diffy.tex', 'Diffy Documentation', 135 | 'Netflix', 'manual'), 136 | ] 137 | 138 | 139 | # -- Options for manual page output ------------------------------------------ 140 | 141 | # One entry per manual page. List of tuples 142 | # (source start file, name, description, authors, manual section). 143 | man_pages = [ 144 | (master_doc, 'diffy', 'Diffy Documentation', 145 | [author], 1) 146 | ] 147 | 148 | 149 | # -- Options for Texinfo output ---------------------------------------------- 150 | 151 | # Grouping the document tree into Texinfo files. List of tuples 152 | # (source start file, target name, title, author, 153 | # dir menu entry, description, category) 154 | texinfo_documents = [ 155 | (master_doc, 'Diffy', 'Diffy Documentation', 156 | author, 'Diffy', 'One line description of project.', 157 | 'Miscellaneous'), 158 | ] 159 | 160 | -------------------------------------------------------------------------------- /docs/developer/index.rst: -------------------------------------------------------------------------------- 1 | Contributing 2 | ============ 3 | 4 | Want to contribute back to Diffy? This page describes the general development 5 | flow, our philosophy, the test suite, and issue tracking. 6 | 7 | Impostor Syndrome Disclaimer 8 | ---------------------------- 9 | 10 | Before we get into the details: **We want your help. No, really.** 11 | 12 | There may be a little voice inside your head that is telling you that you're 13 | not ready to be an open source contributor; that your skills aren't nearly good 14 | enough to contribute. What could you possibly offer a project like this one? 15 | 16 | We assure you -- the little voice in your head is wrong. If you can write code 17 | at all, you can contribute code to open source. Contributing to open source 18 | projects is a fantastic way to advance one's coding skills. Writing perfect 19 | code isn't the measure of a good developer (that would disqualify all of us!); 20 | it's trying to create something, making mistakes, and learning from those 21 | mistakes. That's how we all improve. 22 | 23 | We've provided some clear `Contribution Guidelines`_ that you can read below. 24 | The guidelines outline the process that you'll need to follow to get a patch 25 | merged. By making expectations and process explicit, we hope it will make it 26 | easier for you to contribute. 27 | 28 | And you don't just have to write code. You can help out by writing 29 | documentation, tests, or even by giving feedback about this work. (And yes, 30 | that includes giving feedback about the contribution guidelines.) 31 | 32 | (`Adrienne Friend`_ came up with this disclaimer language.) 33 | 34 | .. _Adrienne Friend: https://github.com/adriennefriend/imposter-syndrome-disclaimer 35 | 36 | Documentation 37 | ------------- 38 | 39 | If you're looking to help document Diffy, your first step is to get set up with 40 | Sphinx, our documentation tool. First you will want to make sure you have a few 41 | things on your local system: 42 | 43 | * python-dev (if you're on OS X, you already have this) 44 | * pip 45 | * virtualenvwrapper 46 | 47 | Once you've got all that, the rest is simple: 48 | 49 | :: 50 | 51 | # If you have a fork, you'll want to clone it instead 52 | git clone git://github.com/Netflix-Skunkworks/diffy.git 53 | 54 | # Create a python virtualenv 55 | mkvirtualenv diffy 56 | 57 | # Make the magic happen 58 | make dev-docs 59 | 60 | Running ``make dev-docs`` will install the basic requirements to get Sphinx 61 | running. 62 | 63 | 64 | Building Documentation 65 | ~~~~~~~~~~~~~~~~~~~~~~ 66 | 67 | Inside the ``docs`` directory, you can run ``make`` to build the documentation. 68 | See ``make help`` for available options and the `Sphinx Documentation 69 | `_ for more information. 70 | 71 | 72 | Developing Against HEAD 73 | ----------------------- 74 | 75 | We try to make it easy to get up and running in a development environment using 76 | a git checkout of Diffy. You'll want to make sure you have a few things on your 77 | local system first: 78 | 79 | * python-dev (if you're on OS X, you already have this) 80 | * pip 81 | * virtualenv (ideally virtualenvwrapper) 82 | * node.js (for npm and building css/javascript) 83 | * (Optional) PostgreSQL 84 | 85 | Once you've got all that, the rest is simple: 86 | 87 | :: 88 | 89 | # If you have a fork, you'll want to clone it instead 90 | git clone git://github.com/Netflix-Skunkworks/diffy.git 91 | 92 | # Create a python virtualenv 93 | mkvirtualenv diffy 94 | 95 | 96 | Coding Standards 97 | ---------------- 98 | 99 | Diffy follows the guidelines laid out in `pep8 100 | `_ with a little bit of flexibility 101 | on things like line length. We always give way for the `Zen of Python 102 | `_. We also use strict mode for 103 | JavaScript, enforced by jshint. 104 | 105 | You can run all linters with ``make lint``, or respectively ``lint-python`` or 106 | ``lint-js``. 107 | 108 | Spacing 109 | ~~~~~~~ 110 | 111 | Python: 112 | 4 Spaces 113 | 114 | JavaScript: 115 | 2 Spaces 116 | 117 | CSS: 118 | 2 Spaces 119 | 120 | HTML: 121 | 2 Spaces 122 | 123 | 124 | Running the Test Suite 125 | ---------------------- 126 | 127 | If you've setup your environment correctly, you can run the entire suite with 128 | the following command: 129 | 130 | :: 131 | 132 | pytest 133 | 134 | 135 | You'll notice that the test suite is structured based on where the code lives, 136 | and strongly encourages using the mock library to drive more accurate 137 | individual tests. 138 | 139 | .. note:: We use py.test for the Python test suite. 140 | 141 | 142 | Contribution Guidelines 143 | ======================= 144 | 145 | All patches should be sent as a pull request on GitHub, include tests, and 146 | documentation where needed. If you're fixing a bug or making a large change the 147 | patch **must** include test coverage. 148 | 149 | Uncertain about how to write tests? Take a look at some existing tests that are 150 | similar to the code you're changing, and go from there. 151 | 152 | You can see a list of open pull requests (pending changes) by visiting 153 | https://github.com/Netflix-Skunkworks/diffy/pulls 154 | 155 | Pull requests should be against **master** and pass all TravisCI checks. 156 | 157 | We use `pre-commit hooks`_ to help us all maintain a consistent standard for 158 | code. To get started, run: 159 | 160 | :: 161 | 162 | pre-commit install 163 | 164 | 165 | Before submitting code, run these: 166 | 167 | :: 168 | 169 | pre-commit run --all-files 170 | 171 | 172 | .. _pre-commit hooks: https://pre-commit.com/#usage 173 | 174 | Writing a Plugin 175 | ================ 176 | 177 | .. toctree:: 178 | :maxdepth: 2 179 | 180 | plugins/index 181 | 182 | 183 | Internals 184 | ========= 185 | 186 | .. toctree:: 187 | :maxdepth: 2 188 | 189 | internals/diffy 190 | -------------------------------------------------------------------------------- /docs/developer/plugins/index.rst: -------------------------------------------------------------------------------- 1 | Several interfaces exist for extending Diffy: 2 | 3 | * Analysis (diffy.plugins.bases.analysis) 4 | * Collection (diffy.plugins.bases.collection) 5 | * Payload (diffy.plugins.bases.payload) 6 | * Persistence (diffy.plugins.bases.persistence) 7 | * Target (diffy.plugins.bases.target) 8 | * Inventory (diffy.plugins.bases.inventory) 9 | 10 | Each interface has its own functions that will need to be defined in order for 11 | your plugin to work correctly. See :ref:`Plugin Interfaces ` 12 | for details. 13 | 14 | 15 | Structure 16 | --------- 17 | 18 | A plugins layout generally looks like the following:: 19 | 20 | setup.py 21 | diffy_pluginnae/ 22 | diffy_pluginname/__init__.py 23 | diffy_pluginname/plugin.py 24 | 25 | The ``__init__.py`` file should contain no plugin logic, and at most, a VERSION 26 | = 'x.x.x' line. For example, if you want to pull the version using 27 | pkg_resources (which is what we recommend), your file might contain:: 28 | 29 | try: 30 | VERSION = __import__('pkg_resources') \ 31 | .get_distribution(__name__).version 32 | except Exception as e: 33 | VERSION = 'unknown' 34 | 35 | Inside of ``plugin.py``, you'll declare your Plugin class, inheriting from the 36 | parent classes that establish your plugin's functionality:: 37 | 38 | import diffy_pluginname 39 | from diffy.plugins.bases import AnalysisPlugin, PersistencePlugin 40 | 41 | class PluginName(AnalysisPlugin): 42 | title = 'Plugin Name' 43 | slug = 'pluginname' 44 | description = 'My awesome plugin!' 45 | version = diffy_pluginname.VERSION 46 | 47 | author = 'Your Name' 48 | author_url = 'https://github.com/yourname/diffy_pluginname' 49 | 50 | def widget(self, request, group, **kwargs): 51 | return "

Absolutely useless widget

" 52 | 53 | And you'll register it via ``entry_points`` in your ``setup.py``:: 54 | 55 | setup( 56 | # ... 57 | entry_points={ 58 | 'diffy.plugins': [ 59 | 'pluginname = diffy_pluginname.analysis:PluginName' 60 | ], 61 | }, 62 | ) 63 | 64 | You can potentially package multiple plugin types in one package, say you want 65 | to create a source and destination plugins for the same third-party. To 66 | accomplish this simply alias the plugin in entry points to point at multiple 67 | plugins within your package:: 68 | 69 | setup( 70 | # ... 71 | entry_points={ 72 | 'diffy.plugins': [ 73 | 'pluginnamesource = diffy_pluginname.plugin:PluginNameSource', 74 | 'pluginnamedestination = diffy_pluginname.plugin:PluginNameDestination' 75 | ], 76 | }, 77 | ) 78 | 79 | Once your plugin files are in place and the ``setup.py`` file has been 80 | modified, you can load your plugin by reinstalling diffy:: 81 | 82 | (diffy)$ pip install -e . 83 | 84 | That's it! Users will be able to install your plugin via ``pip install ``. 86 | 87 | .. SeeAlso:: For more information about python packages see `Python Packaging `_ 88 | 89 | .. _PluginInterfaces: 90 | 91 | Plugin Interfaces 92 | ================= 93 | 94 | In order to use the interfaces all plugins are required to inherit and override 95 | unimplemented functions of the parent object. 96 | 97 | Analysis 98 | -------- 99 | 100 | Analysis plugins are used when you are trying to scope or evaluate information 101 | across a cluster. They can either process information locally or used an 102 | external system (i.e. for ML). 103 | 104 | 105 | The `AnalysisPlugin` exposes on function:: 106 | 107 | def run(self, items, **kwargs): 108 | # run analysis on items 109 | 110 | Diffy will pass all items collected it will additionally pass the optional 111 | `baseline` flag if the current configuration is deemed to be a baseline. 112 | 113 | Collection 114 | ---------- 115 | 116 | Collection plugins allow you to collect information from multiple hosts. This 117 | provides flexibility on how information is collected, depending on the 118 | infrastructure available to you. 119 | 120 | The CollectionPlugin requires only one function to be implemented:: 121 | 122 | def get(self, targets, incident, command, **kwargs) --> dict: 123 | """Queries system target. 124 | 125 | :returns command results as dict { 126 | 'command_id': [ 127 | { 128 | 'instance_id': 'i-123343243', 129 | 'status': 'success', 130 | 'collected_at' : 'datetime' 131 | 'stdout': {json osquery result} 132 | } 133 | ... 134 | {additional instances} 135 | ] 136 | } 137 | """ 138 | 139 | The `incident` string is intended to document a permanent identifier for your 140 | investigation. You may insert any unique ticketing system identifier (for 141 | example, `DFIR-21996`), or comment, here. 142 | 143 | Payload 144 | ------- 145 | 146 | Diffy includes the ability to modify the `payload` for any given command. In 147 | general this payload is the dynamic generation of commands sent to the target. 148 | For instance if you are simply running a `netstat` payload you may have to 149 | actually run a series of commands to generate a JSON output from the `netstat` 150 | command. 151 | 152 | Here again the incident is passed to be dynamically included into the commands 153 | if applicable. 154 | 155 | The PayloadPlugin requires only one function to be implemented:: 156 | 157 | def generate(self, incident, **kwargs) --> dict: 158 | # list of commands to be sent to the target 159 | 160 | 161 | Persistence 162 | ----------- 163 | 164 | Persistence plugins give Diffy to store the outputs of both collection and 165 | analysis to location other than memory. This is useful for baseline tasks or 166 | persisting data for external analysis tasks. 167 | 168 | The PersistencePlugin requires two functions to be implemented:: 169 | 170 | def get(self, key, **kwargs): 171 | # retrieve from location 172 | 173 | def save(self, key, item, **kwargs): 174 | # save to location 175 | 176 | Target 177 | ------ 178 | 179 | Target plugins give Diffy the ability to interact with external systems to resolve 180 | targets for commands. 181 | 182 | The TargetPlugin class requires one function to be implemented:: 183 | 184 | def get(self, key, **kwargs): 185 | # fetch targets based on key 186 | 187 | 188 | Inventory 189 | --------- 190 | 191 | Inventory plugins interact with asset inventory services to pull a list of 192 | targets for baselining and analysis. 193 | 194 | Inheriting from the InventoryPlugin class requires that you implement a ``process`` 195 | method:: 196 | 197 | def process(self, **kwargs): 198 | # Process a new set of targets from a desired source. 199 | # 200 | # This method should handle the interaction with your desired source, 201 | # and then send the results to :meth:`diffy_api.core.async_baseline`. 202 | # 203 | # If you poll the source regularly, ensure that you 204 | # only request recently deployed assets. 205 | 206 | 207 | Testing 208 | ======= 209 | 210 | Diffy provides a basic py.test-based testing framework for extensions. 211 | 212 | In a simple project, you'll need to do a few things to get it working: 213 | 214 | setup.py 215 | -------- 216 | 217 | Augment your setup.py to ensure at least the following: 218 | 219 | .. code-block:: python 220 | 221 | setup( 222 | # ... 223 | install_requires=[ 224 | 'diffy', 225 | ] 226 | ) 227 | 228 | 229 | conftest.py 230 | ----------- 231 | 232 | The ``conftest.py`` file is our main entry-point for py.test. We need to 233 | configure it to load the Diffy pytest configuration: 234 | 235 | .. code-block:: python 236 | 237 | from diffy.tests.conftest import * # noqa 238 | 239 | 240 | Running Tests 241 | ------------- 242 | 243 | Running tests follows the py.test standard. As long as your test files and 244 | methods are named appropriately (``test_filename.py`` and ``test_function()``) 245 | you can simply call out to py.test: 246 | 247 | :: 248 | 249 | $ py.test -v 250 | ============================== test session starts ============================== 251 | platform darwin -- Python 2.7.10, pytest-2.8.5, py-1.4.30, pluggy-0.3.1 252 | cachedir: .cache 253 | collected 346 items 254 | 255 | diffy/plugins/diffy_acme/tests/test_aws.py::test_ssm PASSED 256 | 257 | =========================== 1 passed in 0.35 seconds ============================ 258 | 259 | 260 | .. SeeAlso:: Diffy bundles several plugins that use the same interfaces mentioned above. 261 | -------------------------------------------------------------------------------- /docs/doing-a-release.rst: -------------------------------------------------------------------------------- 1 | Doing a release 2 | =============== 3 | 4 | Doing a release of ``diffy`` requires a few steps. 5 | 6 | Bumping the version number 7 | -------------------------- 8 | 9 | The next step in doing a release is bumping the version number in the 10 | software. 11 | 12 | * Update the version number in ``diffy/__about__.py``. 13 | * Set the release date in the :doc:`/changelog`. 14 | * Do a commit indicating this. 15 | * Send a pull request with this. 16 | * Wait for it to be merged. 17 | 18 | Performing the release 19 | ---------------------- 20 | 21 | The commit that merged the version number bump is now the official release 22 | commit for this release. You will need to have ``gpg`` installed and a ``gpg`` 23 | key in order to do a release. Once this has happened: 24 | 25 | * Run ``invoke release {version}``. 26 | 27 | The release should now be available on PyPI and a tag should be available in 28 | the repository. 29 | 30 | Verifying the release 31 | --------------------- 32 | 33 | You should verify that ``pip install diffy`` works correctly: 34 | 35 | .. code-block:: pycon 36 | 37 | >>> import diffy 38 | >>> diffy.__version__ 39 | '...' 40 | 41 | Verify that this is the version you just released. 42 | 43 | Post-release tasks 44 | ------------------ 45 | 46 | * Update the version number to the next major (e.g. ``0.5.dev1``) in 47 | ``diffy/__about__.py`` and 48 | * Add new :doc:`/changelog` entry with next version and note that it is under 49 | active development 50 | * Send a pull request with these items 51 | * Check for any outstanding code undergoing a deprecation cycle by looking in 52 | ``diffy.utils`` for ``DeprecatedIn**`` definitions. If any exist open 53 | a ticket to increment them for the next release. -------------------------------------------------------------------------------- /docs/faq.rst: -------------------------------------------------------------------------------- 1 | Frequently Asked Questions 2 | ========================== 3 | 4 | Common Problems 5 | --------------- 6 | 7 | 8 | How do I 9 | -------- 10 | 11 | -------------------------------------------------------------------------------- /docs/images/diffy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Netflix-Skunkworks/diffy/1d8c1093023832be40d04021906f8779b37e8521/docs/images/diffy.png -------------------------------------------------------------------------------- /docs/images/diffy_small.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Netflix-Skunkworks/diffy/1d8c1093023832be40d04021906f8779b37e8521/docs/images/diffy_small.png -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | Diffy (DEPRECATED) 2 | ===== 3 | 4 | **Diffy has been deprecated at Netflix.** This software is no longer supported, and will not receive maintenance. 5 | 6 | Diffy was a differencing engine for digital forensics and incident response 7 | (DFIR) in the cloud. Collect data across multiple virtual machines and use 8 | variations from a baseline, and/or clustering, to scope a incident. 9 | 10 | 11 | Installation 12 | ------------ 13 | 14 | .. toctree:: 15 | :maxdepth: 2 16 | 17 | quickstart/index 18 | production/index 19 | 20 | 21 | Developers 22 | ---------- 23 | 24 | .. toctree:: 25 | :maxdepth: 2 26 | 27 | developer/index 28 | 29 | Security 30 | -------- 31 | 32 | .. toctree:: 33 | :maxdepth: 2 34 | 35 | security 36 | 37 | Doing a Release 38 | --------------- 39 | 40 | .. toctree:: 41 | :maxdepth: 1 42 | 43 | doing-a-release 44 | 45 | FAQ 46 | --- 47 | 48 | .. toctree:: 49 | :maxdepth: 1 50 | 51 | faq 52 | 53 | Reference 54 | --------- 55 | 56 | .. toctree:: 57 | :maxdepth: 1 58 | 59 | changelog 60 | license/index 61 | -------------------------------------------------------------------------------- /docs/internals/index.rst.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Netflix-Skunkworks/diffy/1d8c1093023832be40d04021906f8779b37e8521/docs/internals/index.rst.py -------------------------------------------------------------------------------- /docs/license/index.rst: -------------------------------------------------------------------------------- 1 | License 2 | ======= 3 | 4 | Diffy is licensed under a three clause APACHE License. 5 | 6 | The full license text can be found below (:ref:`diffy-license`). 7 | 8 | Authors 9 | ------- 10 | 11 | Diffy was originally written and is maintained by Forest Monsen & Kevin Glisson. 12 | 13 | A list of additional contributors can be seen on `GitHub `_. 14 | 15 | .. _diffy-license: 16 | 17 | Diffy License 18 | ------------- 19 | 20 | .. include:: ../../LICENSE -------------------------------------------------------------------------------- /docs/production/index.rst: -------------------------------------------------------------------------------- 1 | Production 2 | ********** 3 | 4 | We haven't intended for folks to run Diffy in a production environment. 5 | However, if you'd like to do that, you'll have to do a few things first, to 6 | ensure that it will run reliably and securely. 7 | 8 | Basics 9 | ====== 10 | 11 | TODO 12 | -------------------------------------------------------------------------------- /docs/quickstart/index.rst: -------------------------------------------------------------------------------- 1 | Quickstart 2 | ========== 3 | 4 | This guide will step you through setting up a Python-based virtualenv, 5 | configuring it correctly, and running your first baseline and difference 6 | against an autoscaling group (ASG). This guide assumes you're operating on 7 | a freshly-installed `Ubuntu 16.04 instance`_. Commands may differ in your 8 | environment. 9 | 10 | .. _Ubuntu 16.04 instance: https://www.ubuntu.com/download 11 | 12 | Clone the repo:: 13 | 14 | $ git clone git@github.com:Netflix-Skunkworks/diffy.git && cd diffy 15 | 16 | Install a virtualenv there:: 17 | 18 | $ virtualenv venv 19 | 20 | Activate the virtualenv:: 21 | 22 | $ source venv/bin/activate 23 | 24 | Install the required "dev" packages into the virtualenv:: 25 | 26 | $ pip install -r dev-requirements.txt 27 | 28 | Install the local Diffy package:: 29 | 30 | $ pip install -e . 31 | 32 | Invoke the command line client with default options, to create a new functional 33 | baseline. In the command below, replace the ```` placeholder with the name of your 34 | `autoscaling group`_ (a concept particular to AWS):: 35 | 36 | $ diffy new baseline 37 | 38 | .. _`autoscaling group`: https://docs.aws.amazon.com/autoscaling/ec2/userguide/AutoScalingGroup.html 39 | 40 | You'll find a JSON file in your current directory. This file contains the 41 | observations collected as the baseline. 42 | 43 | Next, run an analysis across all members of that autoscaling group, to locate 44 | outliers:: 45 | 46 | $ diffy new analysis 47 | 48 | When done, deactivate your virtualenv:: 49 | 50 | $ deactivate 51 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Netflix-Skunkworks/diffy/1d8c1093023832be40d04021906f8779b37e8521/docs/requirements.txt -------------------------------------------------------------------------------- /docs/security.rst: -------------------------------------------------------------------------------- 1 | Security 2 | ======== 3 | 4 | We take the security of ``diffy`` seriously. The following are a set of 5 | policies we have adopted to ensure that security issues are addressed in a 6 | timely fashion. 7 | 8 | Reporting a security issue 9 | -------------------------- 10 | 11 | We ask that you do not report security issues to our normal GitHub issue 12 | tracker. 13 | 14 | If you believe you've identified a security issue with ``diffy``, please 15 | report it to ``cloudsecurity@netflix.com``. 16 | 17 | Once you've submitted an issue via email, you should receive an acknowledgment 18 | within 48 hours, and depending on the action to be taken, you may receive 19 | further follow-up emails. 20 | 21 | Supported Versions 22 | ------------------ 23 | 24 | At any given time, we will provide security support for the `master`_ branch 25 | as well as the 2 most recent releases. 26 | 27 | Disclosure Process 28 | ------------------ 29 | 30 | Our process for taking a security issue from private discussion to public 31 | disclosure involves multiple steps. 32 | 33 | Approximately one week before full public disclosure, we will send advance 34 | notification of the issue to a list of people and organizations, primarily 35 | composed of operating-system vendors and other distributors of 36 | ``diffy``. This notification will consist of an email message 37 | containing: 38 | 39 | * A full description of the issue and the affected versions of 40 | ``diffy``. 41 | * The steps we will be taking to remedy the issue. 42 | * The patches, if any, that will be applied to ``diffy``. 43 | * The date on which the ``diffy`` team will apply these patches, issue 44 | new releases, and publicly disclose the issue. 45 | 46 | Simultaneously, the reporter of the issue will receive notification of the date 47 | on which we plan to make the issue public. 48 | 49 | On the day of disclosure, we will take the following steps: 50 | 51 | * Apply the relevant patches to the ``diffy`` repository. The commit 52 | messages for these patches will indicate that they are for security issues, 53 | but will not describe the issue in any detail; instead, they will warn of 54 | upcoming disclosure. 55 | * Issue the relevant releases. 56 | 57 | If a reported issue is believed to be particularly time-sensitive – due to a 58 | known exploit in the wild, for example – the time between advance notification 59 | and public disclosure may be shortened considerably. 60 | 61 | The list of people and organizations who receives advanced notification of 62 | security issues is not, and will not, be made public. This list generally 63 | consists of high-profile downstream distributors and is entirely at the 64 | discretion of the ``diffy`` team. 65 | 66 | .. _`master`: https://github.com/Netflix/diffy -------------------------------------------------------------------------------- /localhost-localhost.json: -------------------------------------------------------------------------------- 1 | {"instance_id": "localhost", "status": "success", "collected_at": "2018-11-30 23:17:37", "stdout": []} -------------------------------------------------------------------------------- /requirements.in: -------------------------------------------------------------------------------- 1 | boto3 2 | click 3 | click-log 4 | rapidfuzz 5 | jsondiff 6 | jsonschema 7 | marshmallow-jsonschema 8 | python-dateutil 9 | PyYAML 10 | retrying 11 | swag-client 12 | tabulate 13 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile 3 | # To update, run: 4 | # 5 | # pip-compile --index-url=https://pypi.org/simple requirements.in 6 | # 7 | attrs==19.3.0 # via jsonschema 8 | boto3==1.11.17 9 | botocore==1.14.17 # via boto3, s3transfer 10 | click-log==0.3.2 11 | click==7.0 12 | decorator==4.4.1 # via dogpile.cache 13 | deepdiff==4.2.0 # via swag-client 14 | docutils==0.15.2 # via botocore 15 | dogpile.cache==0.9.0 # via swag-client 16 | fuzzywuzzy==0.18.0 17 | importlib-metadata==1.5.0 # via jsonschema 18 | jmespath==0.9.4 # via boto3, botocore, swag-client 19 | jsondiff==1.2.0 20 | jsonschema==3.2.0 21 | marshmallow-jsonschema==0.9.0 22 | marshmallow==2.20.5 # via marshmallow-jsonschema, swag-client 23 | ordered-set==3.1.1 # via deepdiff 24 | pyrsistent==0.15.7 # via jsonschema 25 | python-dateutil==2.8.1 26 | python-levenshtein==0.12.0 27 | pyyaml==5.3 28 | retrying==1.3.3 29 | s3transfer==0.3.3 # via boto3 30 | simplejson==3.17.0 # via swag-client 31 | six==1.14.0 # via jsonschema, pyrsistent, python-dateutil, retrying 32 | swag-client==0.4.7 33 | tabulate==0.8.6 34 | urllib3==1.25.8 # via botocore 35 | zipp==2.2.0 # via importlib-metadata -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [tool:pytest] 2 | python_files=test*.py 3 | addopts=--tb=native -p no:doctest 4 | norecursedirs=bin dist docs htmlcov script hooks node_modules .* {args} 5 | testpaths = tests 6 | 7 | [flake8] 8 | ignore = F999,E501,E128,E124,E402,W503,E731,F841,F405 9 | max-line-length = 100 10 | exclude = .tox,.git,docs/* 11 | 12 | [wheel] 13 | universal = 1 14 | 15 | [metadata] 16 | description-file = README.rst 17 | license-file = LICENSE 18 | 19 | [aliases] 20 | test = pytest 21 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | """ 2 | diffy 3 | ===== 4 | 5 | :copyright: (c) 2018 by Netflix, see AUTHORS for more 6 | :license: Apache, see LICENSE for more details. 7 | """ 8 | import pip 9 | 10 | import io 11 | from distutils.core import setup 12 | 13 | from setuptools import find_packages 14 | 15 | if tuple(map(int, pip.__version__.split("."))) >= (10, 0, 0): 16 | from pip._internal.download import PipSession 17 | from pip._internal.req import parse_requirements 18 | else: 19 | from pip.download import PipSession 20 | from pip.req import parse_requirements 21 | 22 | 23 | with io.open("README.rst", encoding="utf-8") as readme: 24 | long_description = readme.read() 25 | 26 | 27 | def load_requirements(filename): 28 | with io.open(filename, encoding="utf-8") as reqfile: 29 | return [line.split()[0] for line in reqfile if not line.startswith(("#", "-"))] 30 | 31 | 32 | def moto_broken(): 33 | """Until https://github.com/spulec/moto/pull/1589 is resolved. 34 | 35 | Then we will no longer need to fork moto, roll our own release, and rely either on 36 | this hack, or the dependency_links declaration. 37 | """ 38 | reqts = load_requirements("dev-requirements.txt") 39 | # return reqts.append('moto==1.3.5') 40 | return reqts 41 | 42 | 43 | # Populates __version__ without importing the package 44 | __version__ = None 45 | with io.open("diffy/_version.py", encoding="utf-8") as ver_file: 46 | exec(ver_file.read()) # nosec: config file safe 47 | if not __version__: 48 | print("Could not find __version__ from diffy/_version.py") 49 | exit(1) 50 | 51 | 52 | setup( 53 | name="diffy", 54 | version=__version__, 55 | author="Netflix", 56 | author_email="security@netflix.com", 57 | url="https://github.com/Netflix-Skunkworks/diffy", 58 | description="Forensic differentiator", 59 | long_description=long_description, 60 | packages=find_packages(exclude=["diffy.tests"]), 61 | include_package_data=True, 62 | # dependency_links=['git+https://github.com/forestmonster/moto.git@master#egg=moto-1.3.5'], 63 | install_requires=load_requirements("requirements.txt"), 64 | tests_require=["pytest"], 65 | extras_require={ 66 | "dev": load_requirements("dev-requirements.txt"), 67 | "web": load_requirements("web-requirements.txt"), 68 | }, 69 | entry_points={ 70 | "console_scripts": ["diffy = diffy_cli.core:entry_point"], 71 | "diffy.plugins": [ 72 | "aws_persistence_s3 = diffy.plugins.diffy_aws.plugin:S3PersistencePlugin", 73 | "aws_collection_ssm = diffy.plugins.diffy_aws.plugin:SSMCollectionPlugin", 74 | "aws_target_auto_scaling_group = diffy.plugins.diffy_aws.plugin:AutoScalingTargetPlugin", 75 | "local_analysis_simple = diffy.plugins.diffy_local.plugin:SimpleAnalysisPlugin", 76 | "local_analysis_cluster = diffy.plugins.diffy_local.plugin:ClusterAnalysisPlugin", 77 | "local_persistence_file = diffy.plugins.diffy_local.plugin:FilePersistencePlugin", 78 | "local_payload_command = diffy.plugins.diffy_local.plugin:CommandPayloadPlugin", 79 | "local_shell_collection = diffy.plugins.diffy_local.plugin:LocalShellCollectionPlugin", 80 | "local_target = diffy.plugins.diffy_local.plugin:LocalTargetPlugin", 81 | "osquery_payload = diffy.plugins.diffy_osquery.plugin:OSQueryPayloadPlugin", 82 | ], 83 | }, 84 | classifiers=[ 85 | "Intended Audience :: Developers", 86 | "Intended Audience :: System Administrators", 87 | "Operating System :: OS Independent", 88 | "Topic :: Software Development", 89 | "Programming Language :: Python", 90 | "Programming Language :: Python :: 3.6", 91 | "Natural Language :: English", 92 | "License :: OSI Approved :: Apache Software License", 93 | ], 94 | python_requires=">=3.6", 95 | ) 96 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Netflix-Skunkworks/diffy/1d8c1093023832be40d04021906f8779b37e8521/tests/__init__.py -------------------------------------------------------------------------------- /tests/conf.py: -------------------------------------------------------------------------------- 1 | # This is just Python which means you can inherit and tweak settings 2 | import os 3 | 4 | _basedir = os.path.abspath(os.path.dirname(__file__)) 5 | 6 | THREADS_PER_PAGE = 8 7 | 8 | # General 9 | # These will need to be set to `True` if you are developing locally 10 | CORS = False 11 | debug = False 12 | 13 | # this is the secret key used by flask session management 14 | SECRET_KEY = "I/dVhOZNSMZMqrFJa5tWli6VQccOGudKerq3eWPMSzQNmHHVhMAQfQ==" 15 | -------------------------------------------------------------------------------- /tests/config.yml: -------------------------------------------------------------------------------- 1 | # General Diffy configuration 2 | DIFFY_SWAG_ENABLED: True 3 | SWAG_BUCKET_NAME: example 4 | 5 | 6 | # Plugin specific options 7 | 8 | # AWS 9 | DIFFY_AWS_S3_BUCKET: bucket-blah 10 | DIFFY_AWS_SSM_IAM_ROLE: DiffyRole -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module: diffy.tests.conftest 3 | :platform: Unix 4 | :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more 5 | :license: Apache, see LICENSE for more details. 6 | .. author:: Kevin Glisson 7 | """ 8 | import os 9 | import pytest 10 | 11 | import boto3 12 | 13 | from moto import mock_ssm, mock_iam, mock_sts, mock_ec2, mock_s3, mock_autoscaling 14 | 15 | 16 | from diffy_api import create_app 17 | 18 | 19 | @pytest.yield_fixture(scope="session") 20 | def app(request): 21 | """ 22 | Creates a new Flask application for a test duration. 23 | Uses application factory `create_app`. 24 | """ 25 | _app = create_app(os.path.dirname(os.path.realpath(__file__)) + "/config.yml") 26 | ctx = _app.app_context() 27 | ctx.push() 28 | 29 | yield _app 30 | 31 | ctx.pop() 32 | 33 | 34 | @pytest.yield_fixture(scope="function") 35 | def client(app, client): 36 | yield client 37 | 38 | 39 | @pytest.fixture(scope="function") 40 | def s3(): 41 | with mock_s3(): 42 | yield boto3.client("s3", region_name="us-east-1") 43 | 44 | 45 | @pytest.fixture(scope="function") 46 | def ec2(): 47 | with mock_ec2(): 48 | yield boto3.client("ec2", region_name="us-east-1") 49 | 50 | 51 | @pytest.fixture(scope="function") 52 | def autoscaling(): 53 | with mock_autoscaling(): 54 | yield boto3.client("autoscaling", region_name="us-east-1") 55 | 56 | 57 | @pytest.fixture(scope="function") 58 | def ssm(): 59 | with mock_ssm(): 60 | yield boto3.client("ssm", region_name="us-east-1") 61 | 62 | 63 | @pytest.fixture(scope="function") 64 | def sts(): 65 | with mock_sts(): 66 | yield boto3.client("sts", region_name="us-east-1") 67 | 68 | 69 | @pytest.fixture(scope="function") 70 | def iam(): 71 | with mock_iam(): 72 | yield boto3.client("iam", region_name="us-east-1") 73 | 74 | 75 | @pytest.fixture(scope="function") 76 | def swag_accounts(s3): 77 | from swag_client.backend import SWAGManager 78 | from swag_client.util import parse_swag_config_options 79 | 80 | bucket_name = "SWAG" 81 | data_file = "accounts.json" 82 | region = "us-east-1" 83 | owner = "third-party" 84 | 85 | s3.create_bucket(Bucket=bucket_name) 86 | os.environ["SWAG_BUCKET"] = bucket_name 87 | os.environ["SWAG_DATA_FILE"] = data_file 88 | os.environ["SWAG_REGION"] = region 89 | os.environ["SWAG_OWNER"] = owner 90 | 91 | swag_opts = { 92 | "swag.type": "s3", 93 | "swag.bucket_name": bucket_name, 94 | "swag.data_file": data_file, 95 | "swag.region": region, 96 | "swag.cache_expires": 0, 97 | } 98 | 99 | swag = SWAGManager(**parse_swag_config_options(swag_opts)) 100 | 101 | account = { 102 | "aliases": ["test"], 103 | "contacts": ["admins@test.net"], 104 | "description": "LOL, Test account", 105 | "email": "testaccount@test.net", 106 | "environment": "test", 107 | "id": "012345678910", 108 | "name": "testaccount", 109 | "owner": "third-party", 110 | "provider": "aws", 111 | "sensitive": False, 112 | "services": [], 113 | } 114 | 115 | swag.create(account) 116 | 117 | 118 | @pytest.fixture(scope="function") 119 | def diffy_s3_bucket(s3): 120 | bucket_name = "test_bucket" 121 | os.environ["DIFFY_AWS_PERSISTENCE_BUCKET"] = bucket_name 122 | yield s3.create_bucket(Bucket=bucket_name) 123 | 124 | 125 | @pytest.fixture(scope="function") 126 | def diffy_autoscaling_group(ec2, autoscaling): 127 | vpc = ec2.create_vpc(CidrBlock="10.11.0.0/16") 128 | subnet1 = ec2.create_subnet( 129 | VpcId=vpc["Vpc"]["VpcId"], 130 | CidrBlock="10.11.1.0/24", 131 | AvailabilityZone="us-east-1a", 132 | ) 133 | 134 | _ = autoscaling.create_launch_configuration( 135 | LaunchConfigurationName="test_launch_configuration" 136 | ) 137 | 138 | autoscaling.create_auto_scaling_group( 139 | AutoScalingGroupName="test_asg", 140 | LaunchConfigurationName="test_launch_configuration", 141 | MinSize=0, 142 | MaxSize=4, 143 | DesiredCapacity=2, 144 | Tags=[ 145 | { 146 | "ResourceId": "test_asg", 147 | "ResourceType": "auto-scaling-group", 148 | "Key": "propogated-tag-key", 149 | "Value": "propogate-tag-value", 150 | "PropagateAtLaunch": True, 151 | } 152 | ], 153 | VPCZoneIdentifier=subnet1["Subnet"]["SubnetId"], 154 | ) 155 | 156 | instances_to_add = [ 157 | x["InstanceId"] 158 | for x in ec2.run_instances(ImageId="", MinCount=1, MaxCount=1)["Instances"] 159 | ] 160 | 161 | autoscaling.attach_instances( 162 | AutoScalingGroupName="test_asg", InstanceIds=instances_to_add 163 | ) 164 | 165 | yield ec2, autoscaling 166 | 167 | 168 | @pytest.fixture(scope="function") 169 | def diffy_role(iam): 170 | os.environ["DIFFY_ROLE"] = "Diffy" 171 | yield iam.create_role(RoleName="Diffy", AssumeRolePolicyDocument="{}") 172 | -------------------------------------------------------------------------------- /tests/test_analysis.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from diffy_api.analysis.views import * # noqa 3 | 4 | 5 | @pytest.mark.parametrize("token,status", [("", 200)]) 6 | def test_analysis_list_get(client, token, status): 7 | assert client.get(api.url_for(AnalysisList), headers=token).status_code == status 8 | 9 | 10 | @pytest.mark.skip("Fails while moto is broken") 11 | @pytest.mark.parametrize("token,status", [("", 400)]) 12 | def test_analysis_list_post(client, token, status, sts): 13 | assert ( 14 | client.post(api.url_for(AnalysisList), data={}, headers=token).status_code 15 | == status 16 | ) 17 | 18 | 19 | @pytest.mark.parametrize("token,status", [("", 405)]) 20 | def test_analysis_list_put(client, token, status): 21 | assert ( 22 | client.put(api.url_for(AnalysisList), data={}, headers=token).status_code 23 | == status 24 | ) 25 | 26 | 27 | @pytest.mark.parametrize("token,status", [("", 405)]) 28 | def test_analysis_list_delete(client, token, status): 29 | assert client.delete(api.url_for(AnalysisList), headers=token).status_code == status 30 | 31 | 32 | @pytest.mark.parametrize("token,status", [("", 405)]) 33 | def test_analysis_list_patch(client, token, status): 34 | assert ( 35 | client.patch(api.url_for(AnalysisList), data={}, headers=token).status_code 36 | == status 37 | ) 38 | 39 | 40 | @pytest.mark.parametrize("token,status", [("", 200)]) 41 | def test_analysis_get(client, token, status): 42 | assert ( 43 | client.get(api.url_for(Analysis, key="foo"), headers=token).status_code 44 | == status 45 | ) 46 | 47 | 48 | @pytest.mark.parametrize("token,status", [("", 405)]) 49 | def test_analysis_post(client, token, status): 50 | assert ( 51 | client.post( 52 | api.url_for(Analysis, key="foo"), data={}, headers=token 53 | ).status_code 54 | == status 55 | ) 56 | 57 | 58 | @pytest.mark.parametrize("token,status", [("", 405)]) 59 | def test_analysis_put(client, token, status): 60 | assert ( 61 | client.put(api.url_for(Analysis, key="foo"), data={}, headers=token).status_code 62 | == status 63 | ) 64 | 65 | 66 | @pytest.mark.parametrize("token,status", [("", 405)]) 67 | def test_analysis_delete(client, token, status): 68 | assert ( 69 | client.delete(api.url_for(Analysis, key="foo"), headers=token).status_code 70 | == status 71 | ) 72 | 73 | 74 | @pytest.mark.parametrize("token,status", [("", 405)]) 75 | def test_analysis_patch(client, token, status): 76 | assert ( 77 | client.patch( 78 | api.url_for(Analysis, key="foo"), data={}, headers=token 79 | ).status_code 80 | == status 81 | ) 82 | -------------------------------------------------------------------------------- /tests/test_baseline.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import json 3 | from diffy_api.baseline.views import * # noqa 4 | 5 | 6 | @pytest.mark.parametrize("token,status", [("", 200)]) 7 | def test_baseline_list_get(client, token, status): 8 | assert client.get(api.url_for(BaselineList), headers=token).status_code == status 9 | 10 | 11 | @pytest.mark.skip("Fails while moto is broken") 12 | @pytest.mark.parametrize("token,status", [("", 400)]) 13 | def test_baseline_list_post(client, token, status, sts): 14 | assert ( 15 | client.post(api.url_for(BaselineList), data={}, headers=token).status_code 16 | == status 17 | ) 18 | 19 | 20 | @pytest.mark.parametrize("token,status", [("", 405)]) 21 | def test_baseline_list_put(client, token, status): 22 | assert ( 23 | client.put(api.url_for(BaselineList), data={}, headers=token).status_code 24 | == status 25 | ) 26 | 27 | 28 | @pytest.mark.parametrize("token,status", [("", 405)]) 29 | def test_baseline_list_delete(client, token, status): 30 | assert client.delete(api.url_for(BaselineList), headers=token).status_code == status 31 | 32 | 33 | @pytest.mark.parametrize("token,status", [("", 405)]) 34 | def test_baseline_list_patch(client, token, status): 35 | assert ( 36 | client.patch(api.url_for(BaselineList), data={}, headers=token).status_code 37 | == status 38 | ) 39 | 40 | 41 | @pytest.mark.parametrize("token,status", [("", 200)]) 42 | def test_baseline_get(client, token, status): 43 | assert ( 44 | client.get(api.url_for(Baseline, key="foo"), headers=token).status_code 45 | == status 46 | ) 47 | 48 | 49 | @pytest.mark.parametrize("token,status", [("", 405)]) 50 | def test_baseline_post(client, token, status): 51 | assert ( 52 | client.post( 53 | api.url_for(Baseline, key="foo"), data={}, headers=token 54 | ).status_code 55 | == status 56 | ) 57 | 58 | 59 | @pytest.mark.parametrize("token,status", [("", 405)]) 60 | def test_baseline_put(client, token, status): 61 | assert ( 62 | client.put(api.url_for(Baseline, key="foo"), data={}, headers=token).status_code 63 | == status 64 | ) 65 | 66 | 67 | @pytest.mark.parametrize("token,status", [("", 405)]) 68 | def test_baseline_delete(client, token, status): 69 | assert ( 70 | client.delete(api.url_for(Baseline, key="foo"), headers=token).status_code 71 | == status 72 | ) 73 | 74 | 75 | @pytest.mark.parametrize("token,status", [("", 405)]) 76 | def test_baseline_patch(client, token, status): 77 | assert ( 78 | client.patch( 79 | api.url_for(Baseline, key="foo"), data={}, headers=token 80 | ).status_code 81 | == status 82 | ) 83 | -------------------------------------------------------------------------------- /web-requirements.in: -------------------------------------------------------------------------------- 1 | -r requirements.txt 2 | flask 3 | raven[flask] 4 | flask-restful 5 | gunicorn 6 | inflection 7 | Flask-RQ2 -------------------------------------------------------------------------------- /web-requirements.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile 3 | # To update, run: 4 | # 5 | # pip-compile --index-url=https://pypi.org/simple web-requirements.in 6 | # 7 | aniso8601==8.0.0 # via flask-restful 8 | attrs==19.3.0 9 | blinker==1.4 # via raven 10 | boto3==1.11.17 11 | botocore==1.14.17 12 | click-log==0.3.2 13 | click==7.0 14 | croniter==0.3.31 # via rq-scheduler 15 | decorator==4.4.1 16 | deepdiff==4.2.0 17 | docutils==0.15.2 18 | dogpile.cache==0.9.0 19 | flask-restful==0.3.8 20 | flask-rq2==18.3 21 | flask==1.1.1 22 | fuzzywuzzy==0.18.0 23 | gunicorn==20.0.4 24 | importlib-metadata==1.5.0 25 | inflection==0.3.1 26 | itsdangerous==1.1.0 # via flask 27 | jinja2==2.11.1 # via flask 28 | jmespath==0.9.4 29 | jsondiff==1.2.0 30 | jsonschema==3.2.0 31 | markupsafe==1.1.1 # via jinja2 32 | marshmallow-jsonschema==0.9.0 33 | marshmallow==2.20.5 34 | ordered-set==3.1.1 35 | pyrsistent==0.15.7 36 | python-dateutil==2.8.1 37 | python-levenshtein==0.12.0 38 | pytz==2019.3 # via flask-restful 39 | pyyaml==5.3 40 | raven[flask]==6.10.0 41 | redis==3.4.1 # via flask-rq2, rq 42 | retrying==1.3.3 43 | rq-scheduler==0.9.1 # via flask-rq2 44 | rq==1.2.2 # via flask-rq2, rq-scheduler 45 | s3transfer==0.3.3 46 | simplejson==3.17.0 47 | six==1.14.0 48 | swag-client==0.4.7 49 | tabulate==0.8.6 50 | urllib3==1.25.8 51 | werkzeug==1.0.0 # via flask 52 | zipp==2.2.0 --------------------------------------------------------------------------------