├── .dockerignore ├── .drone.yml ├── .env.example ├── .gitignore ├── ABOUT.md ├── CHANGELOG.MD ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── blueprint.apib ├── contrib ├── docker-compose.env ├── docker-compose.yml ├── r53_cleanup.py └── zinc.env ├── django_project ├── __init__.py ├── settings │ ├── __init__.py │ ├── base.py │ └── test.py ├── social_auth_pipeline.py ├── urls.py ├── vendors │ ├── __init__.py │ └── celery.py └── wsgi.py ├── docker-compose.env ├── docker-compose.yml ├── docker-entrypoint.sh ├── lattice_sync ├── __init__.py ├── apps.py ├── management │ ├── __init__.py │ └── commands │ │ ├── __init__.py │ │ └── ips_from_lattice.py ├── migrations │ └── __init__.py ├── models.py ├── sync.py ├── tasks.py └── tests │ └── test_sync.py ├── local_settings.py.example ├── manage.py ├── pyproject.toml ├── requirements.dev.txt ├── requirements.txt ├── setup.py ├── templates └── admin │ ├── filter.html │ └── login.html ├── tests ├── __init__.py ├── api │ ├── test_basic_api.py │ ├── test_policy.py │ ├── test_policy_record.py │ ├── test_zone.py │ └── test_zone_records.py ├── dns │ ├── __init__.py │ ├── test_health_checks.py │ ├── test_policy_record_tree.py │ └── test_zone.py ├── fixtures.py ├── test_models.py ├── test_ns_check.py └── utils.py └── zinc ├── __init__.py ├── admin ├── __init__.py ├── ip.py ├── policy.py ├── policy_record.py ├── soft_delete.py └── zone.py ├── management ├── __init__.py └── commands │ ├── __init__.py │ ├── cleanup_stale_zones.py │ ├── reconcile_healthchecks.py │ ├── reconcile_policy_records.py │ ├── reconcile_zones.py │ ├── seed.py │ └── update_ns_propagated.py ├── middleware.py ├── migrations ├── 0001_initial.py ├── 0002_auto_20170309_1144.py ├── 0003_zone_ns_propagated.py ├── 0004_zone_cached_ns_records.py ├── 0005_policymember_enabled.py ├── 0006_auto_20170414_0936.py ├── 0007_policy_routing.py ├── 0008_set_routing_policy.py ├── 0009_auto_20220228_1318.py ├── 0010_policy_ttl.py ├── 0011_alter_policy_name.py └── __init__.py ├── models.py ├── ns_check.py ├── pagination.py ├── route53 ├── __init__.py ├── client.py ├── health_check.py ├── policy.py ├── record.py └── zone.py ├── serializers ├── __init__.py ├── policy.py ├── record.py └── zone.py ├── tasks.py ├── urls.py ├── utils ├── __init__.py ├── generators.py └── validation.py ├── validators.py └── views.py /.dockerignore: -------------------------------------------------------------------------------- 1 | *.egg-info 2 | .coverage 3 | .tox 4 | build 5 | coverage-html 6 | docs/_site 7 | venv 8 | .tox 9 | **/*.pyc 10 | .cache 11 | local_settings.py 12 | 13 | Dockerfile 14 | README.md 15 | LICENSE 16 | docker-compose.yml 17 | Makefile 18 | Vagrantfile 19 | data/ -------------------------------------------------------------------------------- /.drone.yml: -------------------------------------------------------------------------------- 1 | --- 2 | kind: pipeline 3 | name: default 4 | 5 | platform: 6 | os: linux 7 | arch: amd64 8 | 9 | steps: 10 | - name: test 11 | pull: always 12 | image: python:3.11 13 | commands: 14 | - pip install -U -r requirements.dev.txt 15 | - make lint 16 | - make full-test 17 | environment: 18 | DJANGO_SETTINGS_MODULE: django_project.settings.test 19 | ZINC_AWS_KEY: 20 | from_secret: ZINC_AWS_KEY 21 | ZINC_AWS_SECRET: 22 | from_secret: ZINC_AWS_SECRET 23 | ZINC_SECRET_KEY: not-so-secret 24 | 25 | - name: publish-docker-image 26 | pull: if-not-exists 27 | image: plugins/docker 28 | settings: 29 | build_args: 30 | - release="${DRONE_COMMIT_SHA:0:7}" 31 | repo: presslabs/zinc 32 | tags: 33 | - ${DRONE_BRANCH/master/latest} 34 | - ${DRONE_COMMIT_SHA:0:7} 35 | username: presslabsbot 36 | password: 37 | from_secret: DOCKERHUB_TOKEN 38 | 39 | --- 40 | kind: signature 41 | hmac: 151963db3d5e1083b6031e207942b9377404e280f088752af9daabf9599b1b07 42 | 43 | ... 44 | -------------------------------------------------------------------------------- /.env.example: -------------------------------------------------------------------------------- 1 | # https://github.com/joke2k/django-environ 2 | # django configs 3 | ZINC_SECRET_KEY=this-must-be-secret 4 | ZINC_DEBUG=False 5 | 6 | # database 7 | ZINC_DB_URL=mysql://user:passwd@localhost:3306/db_name 8 | 9 | # aws config 10 | ZINC_AWS_KEY= 11 | ZINC_AWS_SECRET= -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | data/ 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *,cover 47 | .hypothesis/ 48 | 49 | # Translations 50 | *.mo 51 | *.pot 52 | 53 | # Django stuff: 54 | *.log 55 | local_settings.py 56 | 57 | # Flask stuff: 58 | instance/ 59 | .webassets-cache 60 | 61 | # Scrapy stuff: 62 | .scrapy 63 | 64 | # Sphinx documentation 65 | docs/_build/ 66 | 67 | # PyBuilder 68 | target/ 69 | 70 | # IPython Notebook 71 | .ipynb_checkpoints 72 | 73 | # pyenv 74 | .python-version 75 | 76 | # celery beat schedule file 77 | celerybeat-schedule 78 | 79 | # dotenv 80 | .env 81 | .env.* 82 | 83 | # virtualenv 84 | venv/ 85 | ENV/ 86 | 87 | # Spyder project settings 88 | .spyderproject 89 | 90 | # Rope project settings 91 | .ropeproject 92 | 93 | *.sqlite3 94 | .idea 95 | local_settings.py 96 | celerybeat-schedule.* 97 | 98 | # Vim temp files 99 | *.sw? 100 | secrets.yml 101 | 102 | # pylint 103 | .pylintrc 104 | 105 | # plaintext drone secrets 106 | .drone.sec.yml 107 | 108 | # celery stuff 109 | celerybeat.pid 110 | 111 | # django environ 112 | .env 113 | 114 | # cache 115 | .mypy_cache/ 116 | -------------------------------------------------------------------------------- /ABOUT.md: -------------------------------------------------------------------------------- 1 | # Zinc - Policy Records for Route53 2 | 3 | Zinc provides a simple REST API for your basic DNS needs and implements policy records for 4 | Route 53 using either 5 | [Weighted Routing](http://docs.aws.amazon.com/Route53/latest/DeveloperGuide/routing-policy.html#routing-policy-weighted) or 6 | [Latency-Based Routing](http://docs.aws.amazon.com/Route53/latest/DeveloperGuide/routing-policy.html#routing-policy-latency) 7 | 8 | ## Our use case 9 | 10 | We have hundreds of sites hosted on our geographically distributed fleet of front-end servers. We 11 | want to ensure availability through redundancy (no customer site should be served by a single 12 | server), and a fast experience for all our customers' visitors through latency-based routing (when 13 | someone tries to access a site, the server with the lowest latency should serve it for them). 14 | 15 | ## Why would one not use route53's policy based records? 16 | 17 | If you're like us and have several hundred policy routed records, the costs can be prohibitive. 18 | 19 | ## Does it support other DNS providers? 20 | 21 | Not yet, but one of the benefits of using Zinc is lower dependency on AWS. We figured in case we 22 | ever do decide to switch providers, adding support to Zinc should be easy and have the benefit of 23 | requiring only one system in our infrastructure to change. 24 | -------------------------------------------------------------------------------- /CHANGELOG.MD: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## Unrealeased changes 4 | - Bump dependencies to `Django==1.11.29`, `djangorestframework==3.11.2`, `gevent==1.4.0`, 5 | `requests==2.20.1`, `celery==4.4.7`, `gevent==1.4.0`, `python-redis-lock==3.7.0`. 6 | - Update AWS regions list 7 | 8 | ## 1.1.0 (2019-01-07) 9 | - Added a command to delete stale zones. 10 | - Return 429 error in API calls when being throttled by AWS. 11 | - Better error messaging when no member is in a policy. 12 | - Use python-redis-lock package insted of redis default lock. 13 | - Better integration with lattice. 14 | - Split overlength record values into chunks. 15 | - Bumped boto3 version to 1.5.29. 16 | - Use PyMySQL instead of mysqlclient. 17 | - Use gevent workers for API. 18 | - Fixed celery inside docker container. 19 | - Added django-environ along with a .env.example file to help with configuration. 20 | 21 | 22 | ## 1.0.1 (2017-06-28) 23 | - Added docker-compose.yml for starting zinc + services. 24 | - Updated the readme file. 25 | 26 | ## 1.0.0 (2017-06-28) 27 | _There is no changelog before this initial release._ 28 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.11-slim 2 | 3 | ARG release=git 4 | ENV ZINC_RELEASE "$release" 5 | 6 | ENV PYTHONUNBUFFERED=1 \ 7 | DJANGO_SETTINGS_MODULE=django_project.settings \ 8 | ZINC_WEB_ADDRESS=0.0.0.0:8080 \ 9 | CELERY_APP=django_project.vendors.celery 10 | 11 | COPY ./requirements.txt /requirements.txt 12 | RUN set -ex \ 13 | && addgroup zinc \ 14 | && adduser --system --disabled-password --ingroup zinc --shell /bin/bash --home /app zinc \ 15 | && pip install --no-cache-dir -r /requirements.txt 16 | 17 | COPY . /app 18 | WORKDIR /app 19 | USER zinc 20 | 21 | RUN set -ex \ 22 | && ZINC_SECRET_KEY="not-secure" /app/manage.py collectstatic --noinput 23 | 24 | 25 | ENTRYPOINT ["/app/docker-entrypoint.sh"] 26 | CMD ["web"] 27 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Copyright (c) 2017 Presslabs 3 | 4 | Version 2.0, January 2004 5 | http://www.apache.org/licenses/ 6 | 7 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 8 | 9 | 1. Definitions. 10 | 11 | "License" shall mean the terms and conditions for use, reproduction, 12 | and distribution as defined by Sections 1 through 9 of this document. 13 | 14 | "Licensor" shall mean the copyright owner or entity authorized by 15 | the copyright owner that is granting the License. 16 | 17 | "Legal Entity" shall mean the union of the acting entity and all 18 | other entities that control, are controlled by, or are under common 19 | control with that entity. For the purposes of this definition, 20 | "control" means (i) the power, direct or indirect, to cause the 21 | direction or management of such entity, whether by contract or 22 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 23 | outstanding shares, or (iii) beneficial ownership of such entity. 24 | 25 | "You" (or "Your") shall mean an individual or Legal Entity 26 | exercising permissions granted by this License. 27 | 28 | "Source" form shall mean the preferred form for making modifications, 29 | including but not limited to software source code, documentation 30 | source, and configuration files. 31 | 32 | "Object" form shall mean any form resulting from mechanical 33 | transformation or translation of a Source form, including but 34 | not limited to compiled object code, generated documentation, 35 | and conversions to other media types. 36 | 37 | "Work" shall mean the work of authorship, whether in Source or 38 | Object form, made available under the License, as indicated by a 39 | copyright notice that is included in or attached to the work 40 | (an example is provided in the Appendix below). 41 | 42 | "Derivative Works" shall mean any work, whether in Source or Object 43 | form, that is based on (or derived from) the Work and for which the 44 | editorial revisions, annotations, elaborations, or other modifications 45 | represent, as a whole, an original work of authorship. For the purposes 46 | of this License, Derivative Works shall not include works that remain 47 | separable from, or merely link (or bind by name) to the interfaces of, 48 | the Work and Derivative Works thereof. 49 | 50 | "Contribution" shall mean any work of authorship, including 51 | the original version of the Work and any modifications or additions 52 | to that Work or Derivative Works thereof, that is intentionally 53 | submitted to Licensor for inclusion in the Work by the copyright owner 54 | or by an individual or Legal Entity authorized to submit on behalf of 55 | the copyright owner. For the purposes of this definition, "submitted" 56 | means any form of electronic, verbal, or written communication sent 57 | to the Licensor or its representatives, including but not limited to 58 | communication on electronic mailing lists, source code control systems, 59 | and issue tracking systems that are managed by, or on behalf of, the 60 | Licensor for the purpose of discussing and improving the Work, but 61 | excluding communication that is conspicuously marked or otherwise 62 | designated in writing by the copyright owner as "Not a Contribution." 63 | 64 | "Contributor" shall mean Licensor and any individual or Legal Entity 65 | on behalf of whom a Contribution has been received by Licensor and 66 | subsequently incorporated within the Work. 67 | 68 | 2. Grant of Copyright License. Subject to the terms and conditions of 69 | this License, each Contributor hereby grants to You a perpetual, 70 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 71 | copyright license to reproduce, prepare Derivative Works of, 72 | publicly display, publicly perform, sublicense, and distribute the 73 | Work and such Derivative Works in Source or Object form. 74 | 75 | 3. Grant of Patent License. Subject to the terms and conditions of 76 | this License, each Contributor hereby grants to You a perpetual, 77 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 78 | (except as stated in this section) patent license to make, have made, 79 | use, offer to sell, sell, import, and otherwise transfer the Work, 80 | where such license applies only to those patent claims licensable 81 | by such Contributor that are necessarily infringed by their 82 | Contribution(s) alone or by combination of their Contribution(s) 83 | with the Work to which such Contribution(s) was submitted. If You 84 | institute patent litigation against any entity (including a 85 | cross-claim or counterclaim in a lawsuit) alleging that the Work 86 | or a Contribution incorporated within the Work constitutes direct 87 | or contributory patent infringement, then any patent licenses 88 | granted to You under this License for that Work shall terminate 89 | as of the date such litigation is filed. 90 | 91 | 4. Redistribution. You may reproduce and distribute copies of the 92 | Work or Derivative Works thereof in any medium, with or without 93 | modifications, and in Source or Object form, provided that You 94 | meet the following conditions: 95 | 96 | (a) You must give any other recipients of the Work or 97 | Derivative Works a copy of this License; and 98 | 99 | (b) You must cause any modified files to carry prominent notices 100 | stating that You changed the files; and 101 | 102 | (c) You must retain, in the Source form of any Derivative Works 103 | that You distribute, all copyright, patent, trademark, and 104 | attribution notices from the Source form of the Work, 105 | excluding those notices that do not pertain to any part of 106 | the Derivative Works; and 107 | 108 | (d) If the Work includes a "NOTICE" text file as part of its 109 | distribution, then any Derivative Works that You distribute must 110 | include a readable copy of the attribution notices contained 111 | within such NOTICE file, excluding those notices that do not 112 | pertain to any part of the Derivative Works, in at least one 113 | of the following places: within a NOTICE text file distributed 114 | as part of the Derivative Works; within the Source form or 115 | documentation, if provided along with the Derivative Works; or, 116 | within a display generated by the Derivative Works, if and 117 | wherever such third-party notices normally appear. The contents 118 | of the NOTICE file are for informational purposes only and 119 | do not modify the License. You may add Your own attribution 120 | notices within Derivative Works that You distribute, alongside 121 | or as an addendum to the NOTICE text from the Work, provided 122 | that such additional attribution notices cannot be construed 123 | as modifying the License. 124 | 125 | You may add Your own copyright statement to Your modifications and 126 | may provide additional or different license terms and conditions 127 | for use, reproduction, or distribution of Your modifications, or 128 | for any such Derivative Works as a whole, provided Your use, 129 | reproduction, and distribution of the Work otherwise complies with 130 | the conditions stated in this License. 131 | 132 | 5. Submission of Contributions. Unless You explicitly state otherwise, 133 | any Contribution intentionally submitted for inclusion in the Work 134 | by You to the Licensor shall be under the terms and conditions of 135 | this License, without any additional terms or conditions. 136 | Notwithstanding the above, nothing herein shall supersede or modify 137 | the terms of any separate license agreement you may have executed 138 | with Licensor regarding such Contributions. 139 | 140 | 6. Trademarks. This License does not grant permission to use the trade 141 | names, trademarks, service marks, or product names of the Licensor, 142 | except as required for reasonable and customary use in describing the 143 | origin of the Work and reproducing the content of the NOTICE file. 144 | 145 | 7. Disclaimer of Warranty. Unless required by applicable law or 146 | agreed to in writing, Licensor provides the Work (and each 147 | Contributor provides its Contributions) on an "AS IS" BASIS, 148 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 149 | implied, including, without limitation, any warranties or conditions 150 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 151 | PARTICULAR PURPOSE. You are solely responsible for determining the 152 | appropriateness of using or redistributing the Work and assume any 153 | risks associated with Your exercise of permissions under this License. 154 | 155 | 8. Limitation of Liability. In no event and under no legal theory, 156 | whether in tort (including negligence), contract, or otherwise, 157 | unless required by applicable law (such as deliberate and grossly 158 | negligent acts) or agreed to in writing, shall any Contributor be 159 | liable to You for damages, including any direct, indirect, special, 160 | incidental, or consequential damages of any character arising as a 161 | result of this License or out of the use or inability to use the 162 | Work (including but not limited to damages for loss of goodwill, 163 | work stoppage, computer failure or malfunction, or any and all 164 | other commercial damages or losses), even if such Contributor 165 | has been advised of the possibility of such damages. 166 | 167 | 9. Accepting Warranty or Additional Liability. While redistributing 168 | the Work or Derivative Works thereof, You may choose to offer, 169 | and charge a fee for, acceptance of support, warranty, indemnity, 170 | or other liability obligations and/or rights consistent with this 171 | License. However, in accepting such obligations, You may act only 172 | on Your own behalf and on Your sole responsibility, not on behalf 173 | of any other Contributor, and only if You agree to indemnify, 174 | defend, and hold each Contributor harmless for any liability 175 | incurred by, or claims asserted against, such Contributor by reason 176 | of your accepting any such warranty or additional liability. 177 | 178 | END OF TERMS AND CONDITIONS -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .DEFAULT_GOAL := build 2 | 3 | test: 4 | py.test -v -n auto --capture=no --no-migrations -k 'not with_aws' --ruff 5 | full-test: 6 | py.test -v --capture=no --color=yes 7 | lint: 8 | py.test -v -n auto --capture=no --color=yes --ruff -m 'ruff' 9 | run: 10 | @echo "#################################################################" 11 | @echo "# #" 12 | @echo "# Run 'make run-celery' in order to start the background worker #" 13 | @echo "# #" 14 | @echo "#################################################################" 15 | python ./manage.py runserver 16 | run-celery: 17 | celery worker -A django_project -B 18 | build: 19 | @echo "There is nothing to build for this project" 20 | seed: 21 | python ./manage.py migrate --no-input 22 | python ./manage.py seed 23 | .PHONY: test full-test run build lint seed 24 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # zinc 2 | [![Build Status](https://drone.presslabs.net/api/badges/PressLabs/zinc/status.svg)](https://drone.presslabs.net/PressLabs/zinc) 3 | 4 | # Welcome to Zinc 5 | 6 | Zinc is a Route 53 zone manager. 7 | 8 | Zinc was developed by the awesome engineering team at [Presslabs](https://www.presslabs.com/), 9 | a Managed WordPress Hosting provider. 10 | 11 | For more open-source projects, check [Presslabs Code](https://www.presslabs.org/). 12 | 13 | # Policy Records on the Cheap 14 | 15 | Q: Why would one use Zinc over AWS's Policy Records? 16 | 17 | A: Price. 50$ per Record adds up quickly. 18 | 19 | 20 | # Overview 21 | 22 | ## IPs, Policies and Policy Records 23 | 24 | At the end of the day your domain name `example.com` needs to resolve to one or more 25 | ip addresses. Here's how we go about it. 26 | 27 | ### IPs 28 | 29 | Should be self explanatory. An IP can be enabled or disabled. 30 | 31 | There is no explicit handling in zinc of multiple IPs belonging to one server. 32 | 33 | Enabling or disabling can be done from the admin or by implementing a django app (see 34 | lattice_sync for an example). 35 | 36 | **N.B.** If implementing your own app it's your responsibility to call 37 | `ip.mark_policy_records_dirty` if the IP changes, so that zinc's reconcile loop will 38 | actually pick up the changes. 39 | 40 | 41 | ### HealthChecks 42 | 43 | Zinc will create a Route53 Health Check for each IP. If Route53 deems the IP unavailable, 44 | it will stop routing traffic to it. 45 | 46 | Currently the Health Checks are hardcoded to expect all servers to accept requests with the 47 | same FQDN (defaults to node.presslabs.net, set `ZINC_HEALTH_CHECK_FQDN` to change). 48 | 49 | ### Policies 50 | 51 | A policy groups several IPs together. There are 2 types of policies: 52 | * Weighted 53 | * Latency 54 | 55 | Note that an IP can be a member of multiple Policies at the same time. A PolicyMember 56 | can has it's own enabled flag, so you can disable an IP for one Policy only, or you can 57 | disable the it for all Policies by setting the enabled flag on the IP model. 58 | 59 | #### Weighted 60 | 61 | Trafic will be routed to all IP's based on their weights. Bigger weight means more trafic. 62 | 63 | #### Latency 64 | 65 | Each IP you add to a Policy will have a region specified as well. The region must be an AWS 66 | region. IPs will still have weights, which will be used to balance the trafic within a 67 | region. When a cliend does a DNS lookup, they'll get directed to the region with the lowest 68 | latency, and then an IP will be picked based on weight. 69 | 70 | The resulting setup will be similar to the example described here: 71 | http://docs.aws.amazon.com/Route53/latest/DeveloperGuide/dns-failover-complex-configs.html 72 | 73 | ### Policy Records 74 | 75 | Your desired DNS record. In Route53 it will be an alias to the Latency or Weighted records 76 | that make up a Policy. 77 | 78 | ## Reconcile Loops and the Single Source of Truth 79 | 80 | For simple records in a zone (anything except a PolicyRecord) AWS is the Sigle Source of 81 | Truth. Zinc never stores those locally. 82 | 83 | For Zones, HealthChecks and PolicyRecords Zinc's database is the single source of truth. 84 | Zinc runs reconcile loops and attempts to update your AWS data to match the expected state 85 | in the DB. To minimize throttling by AWS, in most cases, Zinc only attempts to reconcile 86 | objects marked deemed dirty. This means it is possible to have a missmatch between what you 87 | have in AWS and Zinc's expected state if you make changes bypassing Zinc (using the AWS 88 | console, or the api). 89 | 90 | ## API 91 | 92 | You are encouraged to install django-rest-swagger, run zinc locally and explore the API at 93 | http://localhost:8080/swagger 94 | 95 | ### Policies 96 | 97 | Policies are read only trough the API. You can define them in the admin. 98 | 99 | #### Policy listing. 100 | `GET /policies` 101 | 102 | #### Policy detail. Example: 103 | `GET /policies/{id}` 104 | 105 | ``` 106 | GET /policies/344b7bee-da33-4234-b645-805cc26adab0 107 | { 108 | "id": "344b7bee-da33-4234-b645-805cc26adab0", 109 | "name": "policy-one", 110 | "members": [ 111 | { 112 | "id": "6bcb4e77-04dc-45f7-bebb-a2fcfadd7669", 113 | "region": "us-east-1", 114 | "ip": "192.0.2.11", 115 | "weight": 10, 116 | "enabled": true 117 | }, 118 | { 119 | "id": "4f83d47f-af0c-4fa7-80c8-710cb32e4928", 120 | "region": "us-west-1", 121 | "ip": "192.0.2.11", 122 | "weight": 10, 123 | "enabled": true 124 | } 125 | ], 126 | "url": "https://zinc.stage.presslabs.net/policies/344b7bee-da33-4234-b645-805cc26adab0" 127 | } 128 | ``` 129 | 130 | ### Zones 131 | 132 | #### Zone listing. 133 | `GET /zones/` 134 | 135 | #### Zone creation. 136 | `POST /zones/` 137 | 138 | Args: 139 | 140 | | argument | required | default | description | 141 | | --- | --- | --- | --- | 142 | | root | required | - | The domain name of this zone. Trailing dot is optional. | 143 | 144 | Returns the newly created zone object. 145 | 146 | #### Delete a zone. 147 | `DELETE /zones/{zone_id}/` 148 | 149 | #### Zone detail. 150 | `GET /zones/{zone_id}` 151 | 152 | Example: 153 | ``` 154 | GET /zones/102 155 | { 156 | "root": "zinc.example.presslabs.net.", 157 | "url": "https://zinc.stage.presslabs.net/zones/102", 158 | "records_url": "https://zinc.stage.presslabs.net/zones/102/records", 159 | "records": [ 160 | { 161 | "name": "@", 162 | "fqdn": "zinc.example.presslabs.net.", 163 | "type": "NS", 164 | "values": [ 165 | "ns-389.awsdns-48.com.", 166 | "ns-1596.awsdns-07.co.uk.", 167 | "ns-1008.awsdns-62.net.", 168 | "ns-1294.awsdns-33.org." 169 | ], 170 | "ttl": 172800, 171 | "dirty": false, 172 | "id": "Z6k504rwKzbamNZ9ZmY5lvkoOJGDW0", 173 | "url": "https://zinc.stage.presslabs.net/zones/102/records/Z6k504rwKzbamNZ9ZmY5lvkoOJGDW0", 174 | "managed": true 175 | }, 176 | { 177 | "name": "@", 178 | "fqdn": "zinc.example.presslabs.net.", 179 | "type": "SOA", 180 | "values": [ 181 | "ns-389.awsdns-48.com. awsdns-hostmaster.amazon.com. 1 7200 900 1209600 86400" 182 | ], 183 | "ttl": 900, 184 | "dirty": false, 185 | "id": "Z6k504rwKzbamNZ6Z7doJ0yg98j9zA", 186 | "url": "https://zinc.stage.presslabs.net/zones/102/records/Z6k504rwKzbamNZ6Z7doJ0yg98j9zA", 187 | "managed": true 188 | } 189 | ], 190 | "route53_id": "Z8QRF09VVGAC6", 191 | "dirty": false, 192 | "ns_propagated": false 193 | } 194 | ``` 195 | 196 | ### Records 197 | 198 | #### List records in a zone. 199 | `GET /zones/{zone_id}/records` 200 | 201 | Example: 202 | ``` 203 | GET /zones/102/records 204 | [ 205 | { 206 | "name": "@", 207 | "fqdn": "zinc.example.presslabs.net.", 208 | "type": "NS", 209 | "values": [ 210 | "ns-389.awsdns-48.com.", 211 | "ns-1596.awsdns-07.co.uk.", 212 | "ns-1008.awsdns-62.net.", 213 | "ns-1294.awsdns-33.org." 214 | ], 215 | "ttl": 172800, 216 | "dirty": false, 217 | "id": "Z6k504rwKzbamNZ9ZmY5lvkoOJGDW0", 218 | "url": "https://zinc.stage.presslabs.net/zones/102/records/Z6k504rwKzbamNZ9ZmY5lvkoOJGDW0", 219 | "managed": true 220 | }, 221 | { 222 | "name": "@", 223 | "fqdn": "zinc.example.presslabs.net.", 224 | "type": "SOA", 225 | "values": [ 226 | "ns-389.awsdns-48.com. awsdns-hostmaster.amazon.com. 1 7200 900 1209600 86400" 227 | ], 228 | "ttl": 900, 229 | "dirty": false, 230 | "id": "Z6k504rwKzbamNZ6Z7doJ0yg98j9zA", 231 | "url": "https://zinc.stage.presslabs.net/zones/102/records/Z6k504rwKzbamNZ6Z7doJ0yg98j9zA", 232 | "managed": true 233 | } 234 | ] 235 | ``` 236 | 237 | #### Create a record. 238 | `POST /zones/{zone_id}/records` 239 | 240 | Args: 241 | 242 | | argument | required | default | description | 243 | | --- | --- | --- | --- | 244 | | name | required | - | The domain name (without the zone root). | 245 | | type | required | - | The record type. Must be either POLICY\_ROUTED or a valid record type. | 246 | | values | required | - | List of values. Should be one IP for A, MX records, a policy id for POLICY_ROUTED, one or more domain names for NS records. | 247 | | ttl | optional | 300 | The TTL for DNS. | 248 | 249 | 250 | #### Delete a record. 251 | `DELETE /zones/{zone_id}/records/{record_id}` 252 | 253 | #### Record detail. 254 | `GET /zones/{zone_id}/records/{record_id}` 255 | 256 | Example: 257 | ``` 258 | GET /zones/102/records/Z6k504rwKzbamNZ1ZxLxRR4BKly04J 259 | { 260 | "name": "www", 261 | "fqdn": "www.zinc.example.presslabs.net.", 262 | "type": "POLICY_ROUTED", 263 | "values": [ 264 | "344b7bee-da33-4234-b645-805cc26adab0" 265 | ], 266 | "ttl": null, 267 | "dirty": false, 268 | "id": "Z6k504rwKzbamNZ1ZxLxRR4BKly04J", 269 | "url": "https://zinc.stage.presslabs.net/zones/102/records/Z6k504rwKzbamNZ1ZxLxRR4BKly04J", 270 | "managed": false 271 | } 272 | ``` 273 | 274 | #### Update an existing record. 275 | `PATCH /zones/{zone_id}/records/{record_id}` 276 | 277 | The type and name can't be changed. 278 | Missing attributes don't change. 279 | 280 | | argument | required | default | description | 281 | | --- | --- | --- | --- | 282 | | values | optional | - | List of values. Should be one IP for A, MX records, a policy id for POLICY_ROUTED, one or more domain names for NS records. | 283 | | ttl | optional | - | The TTL for DNS. | 284 | 285 | 286 | # Installing and Running 287 | 288 | The recomended way to get up and running is using our Docker container. 289 | 290 | ``` 291 | cd contrib/ 292 | docker-compose up 293 | ``` 294 | 295 | ## Config 296 | 297 | If you run the django project with default settings, you can configure zinc by setting 298 | environment variables. If you're using the provided docker-compose.yml you can set the 299 | environment in ./zinc.env 300 | 301 | The following are essential and required: 302 | ``` 303 | ZINC_AWS_KEY - AWS Key 304 | ZINC_AWS_SECRET - AWS Secret 305 | ZINC_SECRET_KEY - Django secret 306 | ``` 307 | 308 | You can also set the following: 309 | ``` 310 | ZINC_ALLOWED_HOSTS - Django Allowed Hosts 311 | ZINC_BROKER_URL - Celery Broker URL, defaults to ${REDIS_URL}/0 312 | ZINC_CELERY_RESULT_BACKEND - Celery Result Backend, defaults to ${REDIS_URL}/1 313 | ZINC_DATA_DIR - PROJECT_ROOT 314 | ZINC_DB_ENGINE - The django db engine to use. Defaults to 'django.db.backends.sqlite3' 315 | ZINC_DB_HOST - 316 | ZINC_DB_NAME - zinc 317 | ZINC_DB_PASSWORD - password 318 | ZINC_DB_PORT - 319 | ZINC_DB_USER - zinc 320 | ZINC_DEBUG - Django debug. Defaults to False. Set to the string "True" to turn on debugging. 321 | ZINC_DEFAULT_TTL - 300 322 | ZINC_ENV_NAME - The environment for sentry reporting. 323 | ZINC_GOOGLE_OAUTH2_KEY - For use with social-django. If you don't set this, social-django will be disabled. 324 | ZINC_GOOGLE_OAUTH2_SECRET - For use with social-django. 325 | ZINC_SOCIAL_AUTH_ADMIN_EMAILS - List of email addresses that will be automatically granted admin access. 326 | ZINC_SOCIAL_AUTH_GOOGLE_OAUTH2_WHITELISTED_DOMAINS - see http://python-social-auth.readthedocs.io/en/latest/configuration/settings.html?highlight=whitelisted#whitelists 327 | ZINC_HEALTH_CHECK_FQDN - Hostname to use in Health Checks. Defaults to 'node.presslabs.net.' 328 | ZINC_LOCK_SERVER_URL - Used with redis-lock. Defaults to ${REDIS_URL}/2. 329 | ZINC_LOG_LEVEL - Defaults to INFO 330 | ZINC_NS_CHECK_RESOLVERS - NameServers to use when checking zone propagation. Default: ['8.8.8.8'] 331 | ZINC_REDIS_URL - Defaults to 'redis://localhost:6379' 332 | ZINC_SECRET_KEY - The secret key used by the django app. 333 | ZINC_SENTRY_DSN - Set this to enable sentry error reporting. 334 | ZINC_STATIC_URL - Defaults to '/static/' 335 | ZINC_ZONE_OWNERSHIP_COMMENT - Set this comment on records, to Defaults to 'zinc' 336 | ``` 337 | 338 | # Development 339 | 340 | **Warning! Don't use production AWS credentials when developing or testing Zinc!** 341 | 342 | After you've cloned the code: 343 | ``` 344 | pip install -r requirements.dev.txt 345 | python setup.py develop 346 | cp local_settings.py.example local_settings.py 347 | # open local_settings.py in your favorite editor, and set AWS credentials 348 | ``` 349 | 350 | To run the tests: 351 | ``` 352 | # all tests 353 | py.test . 354 | 355 | # to skip tests that need AWS 356 | py.test -k 'not with_aws' . 357 | ``` 358 | -------------------------------------------------------------------------------- /blueprint.apib: -------------------------------------------------------------------------------- 1 | FORMAT: 1A 2 | HOST: https://api.zinc.presslabs.com 3 | 4 | # zinc API Specification 5 | 6 | 7 | # Group Overview 8 | ## Purpose 9 | zinc aims to provide a simple REST API for managing AWS Route53 hosted DNS zones and zone records. It also includes a `POLICY_ROUTED` custom DNS record type which is translated towards Route53 as an AWS ALIAS record. This has been implemented in order to benefit from records inside the same hosted zone pointing one to another, thus adding the possibility of using an in-house load balancer. 10 | ## HTTP Methods 11 | The zinc API makes use of the following HTTP methods: 12 | * `GET` - **Retrieve** a representation of the requested resource 13 | * `POST` - **Create** a new resource on an endpoint for all resources of this type. By POST-ing a new resource here we do not have any URI in mind for it, thus letting zinc assign one. The state of the new resource must be specified in the request body 14 | * `PUT` - **Update** or **Replace** a resource at a certain URI by providing the new state of the resource in the request body 15 | * `PATCH` - **Partially update** a resource by providing the parameters that need to be changed and their new values 16 | * `DELETE` - **Delete** a resource at a certain URI 17 | ## Possible responses 18 | Depending on the request being made, zinc will return one of the following status codes as response: 19 | * `200` - The request has been successfully performed 20 | * `204` - The request for deleting a resource has been successful 21 | * `400` - The request body contains attributes that zinc does not know how to handle 22 | * `401` - The request requires authentication 23 | * `404` - zinc can not find the requested resource 24 | ## Resources/Entities 25 | The entities that can be observed through the API endpoints are the following: 26 | * **Zone** - The representation for the Route53 hosted zones. They can be listed all at once, listed per resource and modified per resource. 27 | * **Record** - The representation for the Route53 resource record sets. Records can be listed within the zone endpoint and modified as a batch. 28 | * **Policy** - The representation for the zinc policies. Policies can be listed all at once or per resource and modified per resource. Policies can not be created or deleted using the API. 29 | ## Limitations 30 | Beside the limitations imposed by the [Amazon Route53 API](http://docs.aws.amazon.com/Route53/latest/DeveloperGuide/DNSLimitations.html) and the [boto3 client](https://boto3.readthedocs.io/en/latest/reference/services/route53.html#Route53.Client.change_resource_record_sets), zinc limits a zone to having a maximum of 50 records. This way it is assured that a bulk update on the whole DNS zone can be made through a single API call and atomicity is provided. 31 | ## Authentication 32 | Authentication will be required for any and every zinc API request. Authentication process `TBA`. 33 | 34 | 35 | # Group Zones 36 | The zones created/updated/deleted through the zinc API are further handled on AWS Route53. 37 | 38 | ## Zone List [/zones/] 39 | 40 | ### Retrieve Route53 hosted DNS zones [GET] 41 | + Response 200 (application/json) 42 | + Attributes (array[ZonesGet]) 43 | + Response 401 (application/json) 44 | + Attributes (AuthenticationRequired) 45 | + Response 404 (application/json) 46 | + Attributes (ResourceNotFound) 47 | 48 | ### Create new Route53 hosted DNS zone [POST] 49 | + Request (application/json) 50 | + Attributes (ZonesPost) 51 | + Response 200 (application/json) 52 | + Attributes (array[ZonesGet]) 53 | + Response 401 (application/json) 54 | + Attributes (AuthenticationRequired) 55 | 56 | 57 | ## Zone Resource [/zones/{id}/] 58 | + Parameters 59 | + id: `1` (number, required) - zinc zone ID 60 | 61 | ### Retrieve hosted DNS zone details [GET] 62 | + Response 200 (application/json) 63 | + Attributes (ZoneDetailGet) 64 | + Response 401 (application/json) 65 | + Attributes (AuthenticationRequired) 66 | + Response 404 (application/json) 67 | + Attributes (ResourceNotFound) 68 | 69 | ### Remove hosted DNS zone [DELETE] 70 | + Response 204 71 | + Response 401 (application/json) 72 | + Attributes (AuthenticationRequired) 73 | + Response 404 (application/json) 74 | + Attributes (ResourceNotFound) 75 | 76 | 77 | # Group Records 78 | 79 | ## Record List [/zones/{id}/records/] 80 | + Parameters 81 | + id: `1` (number, required) - zinc zone ID 82 | 83 | ### Create records resource [POST] 84 | + Request (application/json) 85 | + Attributes (ARecordPost) 86 | + Response 202 87 | + Attributes (ARecord) 88 | + Response 401 (application/json) 89 | + Attributes (AuthenticationRequired) 90 | + Response 404 (application/json) 91 | + Attributes (ResourceNotFound) 92 | 93 | ## Record Details [/zones/{id}/records/{record_id}/] 94 | + Parameters 95 | + id: `1` (number, required) - zinc zone ID 96 | + record_id: `Z3kBY37xQO1AX3Z1ZL72pb4wJ1zRXO` - zone record id 97 | 98 | ### Update record resource [PATCH] 99 | + Request (application/json) 100 | + Attributes(ARecordUpdate) 101 | + Response 200 102 | + Attributes (ARecordUpdateResult) 103 | 104 | ### Delete record resource [DELETE] 105 | + Request (application/json) 106 | + Response 204 107 | 108 | 109 | # Group Policies 110 | 111 | ## Policy List [/policies] 112 | 113 | ## Retrieve zinc policies [GET] 114 | + Response 200 (application/json) 115 | + Attributes (array[PoliciesGet]) 116 | + Response 401 (application/json) 117 | + Attributes (AuthenticationRequired) 118 | + Response 404 (application/json) 119 | + Attributes (ResourceNotFound) 120 | 121 | ## Policy Resource [/policies/{id}] 122 | + Parameters 123 | + id: `1` (number, required) - zinc policy ID 124 | 125 | ## Retrieve a specific policy [GET] 126 | + Response 200 (application/json) 127 | + Attributes (PoliciesGet) 128 | + Response 401 (application/json) 129 | + Attributes (AuthenticationRequired) 130 | + Response 404 (application/json) 131 | + Attributes (ResourceNotFound) 132 | 133 | ## Update a specific policy [PUT] 134 | A policy can be updated by specifying its new state in the request body. In case of an invalid attribute, a `Bad Request` response is returned. 135 | + Request (application/json) 136 | + Attributes (PolicyPut) 137 | + Response 200 (application/json) 138 | + Attributes (PolicyPut) 139 | + Response 400 (application/json) 140 | + Attributes (InvalidAttribute) 141 | + Response 401 (application/json) 142 | + Attributes (AuthenticationRequired) 143 | + Response 404 (application/json) 144 | + Attributes (ResourceNotFound) 145 | 146 | ## Partially update a specific policy [PATCH] 147 | A policy can be partially updated by specifying one or more of its attributes in the request body. In case of an invalid attribute, a `Bad Request` response is returned. 148 | + Request (application/json) 149 | + Attributes 150 | + members: `3`, `4`, `5` (array[number]) 151 | + Response 200 (application/json) 152 | + Attributes (PolicyPatch) 153 | + Response 400 (application/json) 154 | + Attributes (InvalidAttribute) 155 | + Response 401 (application/json) 156 | + Attributes (AuthenticationRequired) 157 | + Response 404 (application/json) 158 | + Attributes (ResourceNotFound) 159 | 160 | 161 | # Data Structures 162 | 163 | ## ARecord (object) 164 | - ttl: `300` (number) - Record TTL 165 | - type: `A` (string) - Record type 166 | - name: 'cdn' (string) - Record name 167 | - values: `127.0.0.1` (array[string]) - A list of values held by the record 168 | 169 | ## ARecordPost (object) 170 | - Include ARecord 171 | - ttl: `3600` (number) - Record TTL 172 | 173 | ## ARecordUpdate (object) 174 | - values: `1.2.3.4` (array[string]) - Fields that will be updated 175 | 176 | ## ARecordUpdateResult (object) 177 | - Include ARecord 178 | - values: `1.2.3.4` (array[string]) - Fields that will be updated 179 | 180 | ## NSRecord (object) 181 | - ttl: `300` (number) - Record TTL 182 | - type: `NS` (string) - Record type 183 | - name: `site.com.` (string) - Record name 184 | - values: `ns-333.foodns-22.com.`, `ns-123.bardns-32.co.uk.` (array[string]) - A list of values held by the record 185 | 186 | ## ZonesPost (object) 187 | - root: `site.com.` (string, required) - Root domain of the DNS zone 188 | 189 | ## ZonesGet (object) 190 | - id: `1` (number) - zinc zone ID 191 | - Include ZonesPost 192 | 193 | ## ZoneDetailGet (object) 194 | - Include ZonesGet 195 | - ns (NSRecord) - Nameserver record 196 | - records (array[ARecord]) - All other records 197 | 198 | ## ZoneDetailPatch (object) 199 | - id: `1` (number) 200 | - root: `site2.com.` (string) 201 | - ns (NSRecord) - Nameserver record 202 | - records (array[ARecord]) - All other records 203 | 204 | ## PoliciesGet (object) 205 | - name: `Policy1` (string) - Policy name 206 | - members: `1`, `2`, `3` (array[number]) - IDs of the policy members held by the policy 207 | 208 | ## PolicyGet (object) 209 | - Include PoliciesGet 210 | 211 | ## PolicyPut (object) 212 | - Include PolicyGet 213 | - members: `2`, `3` (array[number]) 214 | 215 | ## PolicyPatch (object) 216 | - Include PolicyGet 217 | - members: `3`, `4`, `5` (array[number]) 218 | 219 | ## AuthenticationRequired (object) 220 | - message: `Authentication is required for this operation` (string) 221 | 222 | ## InvalidAttribute (object) 223 | - message: `Invalid attributes` (string) 224 | 225 | ## ResourceNotFound (object) 226 | - message: `Requested resource not found` (string) 227 | -------------------------------------------------------------------------------- /contrib/docker-compose.env: -------------------------------------------------------------------------------- 1 | BROKER_URL=redis://redis:6379/0 2 | CELERY_RESULT_BACKEND=redis://redis:6379/1 3 | ONCE_REDIS_URL=redis://redis:6379/2 4 | -------------------------------------------------------------------------------- /contrib/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2' 2 | services: 3 | web: 4 | image: presslabs/zinc:latest 5 | command: web 6 | env_file: ./zinc.env 7 | restart: unless-stopped 8 | depends_on: 9 | - redis 10 | - mysql 11 | ports: 12 | - "127.0.0.1:18080:8080" 13 | volumes: 14 | - "./data/:/webroot" 15 | worker: 16 | image: presslabs/zinc:latest 17 | command: celery 18 | env_file: ./zinc.env 19 | restart: unless-stopped 20 | depends_on: 21 | - redis 22 | - mysql 23 | beat: 24 | image: presslabs/zinc:latest 25 | command: celerybeat 26 | env_file: ./zinc.env 27 | restart: unless-stopped 28 | depends_on: 29 | - redis 30 | - mysql 31 | redis: 32 | image: redis:3.2 33 | command: redis-server --save "" --appendonly no --maxmemory-policy allkeys-lru --maxmemory 256mb 34 | restart: unless-stopped 35 | mysql: 36 | image: percona:5.7 37 | command: "mysqld --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci" 38 | environment: 39 | MYSQL_DATABASE: "zinc" 40 | MYSQL_USER: "zinc" 41 | MYSQL_PASSWORD: "zinc_passwd" 42 | MYSQL_ROOT_PASSWORD: "zinc_root_passwd" 43 | restart: unless-stopped 44 | volumes: 45 | - "./data/mysql:/var/lib/mysql" 46 | -------------------------------------------------------------------------------- /contrib/r53_cleanup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import argparse 3 | import os 4 | 5 | import boto3 6 | 7 | client = None 8 | 9 | 10 | class Zone: 11 | def __init__(self, zone): 12 | self.zone = zone 13 | self.zone_id = zone['Id'].split('/')[2] 14 | 15 | @property 16 | def records(self): 17 | return client.list_resource_record_sets(HostedZoneId=self.zone_id) 18 | 19 | def destroy(self, dry_run=False): 20 | changes = [] 21 | for record in self.records['ResourceRecordSets']: 22 | if record['Name'] == self.zone['Name'] and record['Type'] in ['NS', 'SOA']: 23 | continue 24 | changes.append({ 25 | 'Action': 'DELETE', 26 | 'ResourceRecordSet': record 27 | }) 28 | 29 | print('{} {} ({})'.format('Deleting' if dry_run else 'Will delete', 30 | self.zone['Name'], self.zone_id)) 31 | if not dry_run: 32 | if changes: 33 | client.change_resource_record_sets(HostedZoneId=self.zone_id, 34 | ChangeBatch={ 35 | 'Changes': changes 36 | }) 37 | client.delete_hosted_zone(Id=self.zone_id) 38 | 39 | 40 | class Zones: 41 | def __init__(self, limit_comment=None): 42 | self.limit_comment = limit_comment 43 | 44 | def __iter__(self): 45 | next_marker = '' 46 | while next_marker is not None: 47 | kwargs = {} 48 | if next_marker: 49 | kwargs = {'Marker': next_marker} 50 | response = client.list_hosted_zones(**kwargs) 51 | for zone in response['HostedZones']: 52 | if (self.limit_comment is None or 53 | self.limit_comment == zone['Config'].get('Comment')): 54 | yield Zone(zone) 55 | next_marker = response['NextMarker'] if response['IsTruncated'] else None 56 | 57 | 58 | def parse_args(): 59 | parser = argparse.ArgumentParser(description='Ansible inventory for lattice.') 60 | parser.add_argument('--limit-comment', '-l', default='zinc', 61 | help='Remove only zones created using this comment. Defaults to \'zinc\'.') 62 | parser.add_argument('--dry-run', '-n', default=False, action='store_true', 63 | help='Do not actually delete zones, just print the actions.') 64 | parser.add_argument('--aws-key', default=os.getenv('AWS_KEY', '-'), 65 | help='AWS key to use. Defaults to AWS_KEY environment variable.') 66 | parser.add_argument('--aws-secret', default=os.getenv('AWS_SECRET', '-'), 67 | help='AWS secret to use. Defaults to AWS_SECRET environment variable.') 68 | return parser.parse_args() 69 | 70 | 71 | def main(): 72 | global client 73 | args = parse_args() 74 | client = boto3.client('route53', 75 | aws_access_key_id=args.aws_key, 76 | aws_secret_access_key=args.aws_secret) 77 | 78 | for zone in Zones(limit_comment=args.limit_comment): 79 | zone.destroy(dry_run=args.dry_run) 80 | 81 | 82 | if __name__ == '__main__': 83 | main() 84 | -------------------------------------------------------------------------------- /contrib/zinc.env: -------------------------------------------------------------------------------- 1 | ZINC_AWS_KEY=key 2 | ZINC_AWS_SECRET=secret 3 | ZINC_SECRET_KEY=secret 4 | ZINC_REDIS_URL=redis://redis:6379 5 | ZINC_MIGRATE=yes 6 | ZINC_DB_ENGINE=django.db.backends.mysql 7 | ZINC_DB_NAME=zinc 8 | ZINC_DB_USER=zinc 9 | ZINC_DB_PASSWORD=zinc_passwd 10 | ZINC_DB_HOST=mysql 11 | ZINC_ALLOWED_HOSTS=localhost 12 | ZINC_COLLECT_STATIC=yes 13 | ZINC_WEBROOT_DIR=/webroot 14 | ZINC_DEBUG=True 15 | ZINC_SERVE_STATIC=True -------------------------------------------------------------------------------- /django_project/__init__.py: -------------------------------------------------------------------------------- 1 | from .vendors.celery import app as celery_app 2 | -------------------------------------------------------------------------------- /django_project/settings/__init__.py: -------------------------------------------------------------------------------- 1 | # flake8: noqa 2 | try: 3 | from local_settings import * 4 | except ModuleNotFoundError as e: 5 | if e.name != 'local_settings': 6 | raise 7 | from .base import * 8 | 9 | -------------------------------------------------------------------------------- /django_project/settings/base.py: -------------------------------------------------------------------------------- 1 | """ 2 | Django settings for zinc project. 3 | 4 | Generated by 'django-admin startproject' using Django 1.9.8. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/1.9/topics/settings/ 8 | 9 | For the full list of settings and their values, see 10 | https://docs.djangoproject.com/en/1.9/ref/settings/ 11 | """ 12 | 13 | import os 14 | import environ 15 | import warnings 16 | from datetime import timedelta 17 | 18 | try: 19 | import pymysql 20 | pymysql.install_as_MySQLdb() 21 | except ImportError: 22 | pass 23 | 24 | root = environ.Path(__file__) - 3 # two folder back (/a/b/ - 2 = /) 25 | env = environ.Env(DEBUG=(bool, False)) # set default values and casting 26 | environ.Env.read_env() # reading .env file 27 | 28 | 29 | # Build paths inside the project like this: os.path.join(BASE_DIR, ...) 30 | # BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 31 | PROJECT_ROOT = root() 32 | DATA_DIR = env.str('ZINC_DATA_DIR', default=PROJECT_ROOT) 33 | WEBROOT_DIR = env.str('ZINC_WEBROOT_DIR', os.path.join(PROJECT_ROOT, 'webroot/')) 34 | 35 | # Quick-start development settings - unsuitable for production 36 | # See https://docs.djangoproject.com/en/1.9/howto/deployment/checklist/ 37 | 38 | # SECURITY WARNING: keep the secret key used in production secret! 39 | DEFAULT_SECRET_KEY = 'p@7-h3(%-ile((1fz2ei42)o^a-!cse@kp9jnhrx6x75)#1x(r' 40 | SECRET_KEY = env.str('ZINC_SECRET_KEY', default=DEFAULT_SECRET_KEY) 41 | if SECRET_KEY == DEFAULT_SECRET_KEY: 42 | warnings.warn("You are using the default secret key. Please set " 43 | "ZINC_SECRET_KEY in .env file") 44 | 45 | # SECURITY WARNING: don't run with debug turned on in production! 46 | DEBUG = env.bool('ZINC_DEBUG', True) 47 | SERVE_STATIC = env.bool('ZINC_SERVE_STATIC', False) 48 | 49 | ALLOWED_HOSTS = env.list('ZINC_ALLOWED_HOSTS', 50 | default=['localhost', '127.0.0.1', '0.0.0.0', 'zinc.lo']) 51 | SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https') 52 | 53 | if os.getenv('POD_IP'): 54 | ALLOWED_HOSTS.append(os.getenv('POD_IP')) 55 | 56 | SOCIAL_AUTH_GOOGLE_OAUTH2_KEY = env.str('ZINC_GOOGLE_OAUTH2_KEY', '') 57 | SOCIAL_AUTH_GOOGLE_OAUTH2_SECRET = env.str('ZINC_GOOGLE_OAUTH2_SECRET', '') 58 | 59 | # LATTICE 60 | 61 | LATTICE_URL = env.str('ZINC_LATTICE_URL', '') 62 | LATTICE_USER = env.str('ZINC_LATTICE_USER', '') 63 | LATTICE_PASSWORD = env.str('ZINC_LATTICE_PASSWORD', '') 64 | LATTICE_ROLES = env.list('ZINC_LATTICE_ROLES', default=['edge-node']) 65 | LATTICE_ENV = env.str('ZINC_LATTICE_ENV', 'production') 66 | 67 | 68 | # Application definition 69 | 70 | INSTALLED_APPS = [ 71 | 'django.contrib.admin', 72 | 'django.contrib.auth', 73 | 'django.contrib.contenttypes', 74 | 'django.contrib.sessions', 75 | 'django.contrib.messages', 76 | 'django.contrib.staticfiles', 77 | 'rest_framework', 78 | 'drf_yasg', 79 | 'zinc', 80 | ] 81 | 82 | if SOCIAL_AUTH_GOOGLE_OAUTH2_KEY: 83 | INSTALLED_APPS += ['social_django'] 84 | if LATTICE_URL: 85 | INSTALLED_APPS += ['lattice_sync'] 86 | 87 | AUTHENTICATION_BACKENDS = ( 88 | 'django.contrib.auth.backends.ModelBackend', 89 | ) 90 | if SOCIAL_AUTH_GOOGLE_OAUTH2_KEY: 91 | AUTHENTICATION_BACKENDS = ( 92 | 'social_core.backends.google.GoogleOAuth2', 93 | ) + AUTHENTICATION_BACKENDS 94 | LOGIN_URL = '/_auth/login/google-oauth2/' 95 | 96 | SOCIAL_AUTH_ADMIN_USER_SEARCH_FIELDS = ['username', 'first_name', 'email'] 97 | SOCIAL_AUTH_ADMIN_EMAILS = env.list("ZINC_SOCIAL_AUTH_ADMIN_EMAILS", default=[]) 98 | SOCIAL_AUTH_GOOGLE_OAUTH2_WHITELISTED_DOMAINS = env.list( 99 | "ZINC_SOCIAL_AUTH_GOOGLE_OAUTH2_WHITELISTED_DOMAINS", default=[]) 100 | SOCIAL_AUTH_GOOGLE_OAUTH2_SCOPE = [ 101 | 'profile', 102 | ] 103 | 104 | SOCIAL_AUTH_PIPELINE = ( 105 | # Get the information we can about the user and return it in a simple 106 | # format to create the user instance later. On some cases the details are 107 | # already part of the auth response from the provider, but sometimes this 108 | # could hit a provider API. 109 | 'social_core.pipeline.social_auth.social_details', 110 | 111 | # Get the social uid from whichever service we're authing thru. The uid is 112 | # the unique identifier of the given user in the provider. 113 | 'social_core.pipeline.social_auth.social_uid', 114 | 115 | # Verifies that the current auth process is valid within the current 116 | # project, this is where emails and domains whitelists are applied (if 117 | # defined). 118 | 'social_core.pipeline.social_auth.auth_allowed', 119 | 120 | # Checks if the current social-account is already associated in the site. 121 | 'social_core.pipeline.social_auth.social_user', 122 | 123 | # Make up a username for this person, appends a random string at the end if 124 | # there's any collision. 125 | 'social_core.pipeline.user.get_username', 126 | 127 | # Send a validation email to the user to verify its email address. 128 | # Disabled by default. 129 | # 'social_core.pipeline.mail.mail_validation', 130 | 131 | # Associates the current social details with another user account with 132 | # a similar email address. Disabled by default. 133 | # 'social_core.pipeline.social_auth.associate_by_email', 134 | 135 | # Create a user account if we haven't found one yet. 136 | 'social_core.pipeline.user.create_user', 137 | 138 | # Set superuser and is_staff 139 | 'django_project.social_auth_pipeline.set_user_perms', 140 | 141 | # Create the record that associates the social account with the user. 142 | 'social_core.pipeline.social_auth.associate_user', 143 | 144 | # Populate the extra_data field in the social record with the values 145 | # specified by settings (and the default ones like access_token, etc). 146 | 'social_core.pipeline.social_auth.load_extra_data', 147 | 148 | # Update the user record with any changed info from the auth service. 149 | 'social_core.pipeline.user.user_details', 150 | ) 151 | 152 | 153 | MIDDLEWARE = [ 154 | 'whitenoise.middleware.WhiteNoiseMiddleware', 155 | 'django.middleware.security.SecurityMiddleware', 156 | 'django.contrib.sessions.middleware.SessionMiddleware', 157 | 'django.middleware.common.CommonMiddleware', 158 | 'django.middleware.csrf.CsrfViewMiddleware', 159 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 160 | 'django.contrib.messages.middleware.MessageMiddleware', 161 | 'django.middleware.clickjacking.XFrameOptionsMiddleware', 162 | ] 163 | if SOCIAL_AUTH_GOOGLE_OAUTH2_KEY: 164 | MIDDLEWARE += [ 165 | 'social_django.middleware.SocialAuthExceptionMiddleware', 166 | ] 167 | 168 | ROOT_URLCONF = 'django_project.urls' 169 | 170 | TEMPLATES = [ 171 | { 172 | 'BACKEND': 'django.template.backends.django.DjangoTemplates', 173 | 'DIRS': [os.path.join(PROJECT_ROOT, 'templates/')], 174 | 'APP_DIRS': True, 175 | 'OPTIONS': { 176 | 'context_processors': [ 177 | 'django.template.context_processors.debug', 178 | 'django.template.context_processors.request', 179 | 'django.contrib.auth.context_processors.auth', 180 | 'django.contrib.messages.context_processors.messages', 181 | ], 182 | }, 183 | }, 184 | ] 185 | 186 | WSGI_APPLICATION = 'django_project.wsgi.application' 187 | 188 | DATABASES = { 189 | 'default': env.db('ZINC_DB_CONNECT_URL', 'sqlite:///%s' % os.path.join(DATA_DIR, 'db.sqlite3')) 190 | } 191 | 192 | # Password validation 193 | # https://docs.djangoproject.com/en/1.9/ref/settings/#auth-password-validators 194 | 195 | AUTH_PASSWORD_VALIDATORS = [ 196 | { 197 | 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', 198 | }, 199 | { 200 | 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', 201 | }, 202 | { 203 | 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', 204 | }, 205 | { 206 | 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', 207 | }, 208 | ] 209 | 210 | # Internationalization 211 | # https://docs.djangoproject.com/en/1.9/topics/i18n/ 212 | 213 | LANGUAGE_CODE = 'en-us' 214 | 215 | TIME_ZONE = 'UTC' 216 | 217 | USE_I18N = True 218 | 219 | USE_L10N = True 220 | 221 | USE_TZ = True 222 | 223 | 224 | # Static files (CSS, JavaScript, Images) 225 | # https://docs.djangoproject.com/en/1.9/howto/static-files/ 226 | 227 | STATIC_URL = env.str('ZINC_STATIC_URL', '/static/') 228 | STATIC_ROOT = os.path.join(WEBROOT_DIR, 'static/') 229 | 230 | # CELERY 231 | 232 | REDIS_URL = env.str('ZINC_REDIS_CONNECT_URL', 'redis://localhost:6379') 233 | BROKER_URL = env.str('ZINC_BROKER_URL', '{}/0'.format(REDIS_URL)) 234 | CELERY_RESULT_BACKEND = env.str('ZINC_CELERY_RESULT_BACKEND', 235 | '{}/1'.format(REDIS_URL)) 236 | CELERYBEAT_SCHEDULE = { 237 | 'reconcile_zones': { 238 | 'task': 'zinc.tasks.reconcile_zones', 239 | 'schedule': timedelta(seconds=10), 240 | }, 241 | 'update_ns_propagated': { 242 | 'task': 'zinc.tasks.update_ns_propagated', 243 | 'schedule': timedelta(minutes=10), 244 | }, 245 | 'check_clean_zones': { 246 | 'task': 'zinc.tasks.check_clean_zones', 247 | 'schedule': timedelta(minutes=15), 248 | }, 249 | } 250 | 251 | if LATTICE_URL: 252 | CELERYBEAT_SCHEDULE.update({ 253 | 'lattice_sync': { 254 | 'task': 'lattice_sync.tasks.lattice_sync', 255 | 'schedule': 30 256 | }, 257 | }) 258 | 259 | CELERY_ACCEPT_CONTENT = ['json'] 260 | CELERY_TASK_SERIALIZER = 'json' 261 | CELERY_RESULT_SERIALIZER = 'json' 262 | CELERYD_HIJACK_ROOT_LOGGER = False 263 | 264 | # Distributed lock server 265 | LOCK_SERVER_URL = env.str('ZINC_LOCK_SERVER_URL', default='{}/2'.format(REDIS_URL)) 266 | 267 | # HASHIDS 268 | HASHIDS_MIN_LENGTH = 0 269 | 270 | REST_FRAMEWORK = { 271 | 'PAGE_SIZE': 50, 272 | 'DEFAULT_PAGINATION_CLASS': 'zinc.pagination.LinkHeaderPagination', 273 | 'DEFAULT_PARSER_CLASSES': ( 274 | 'rest_framework.parsers.FormParser', 275 | 'rest_framework.parsers.MultiPartParser', 276 | 'rest_framework.parsers.JSONParser', 277 | ), 278 | 'DEFAULT_RENDERER_CLASSES': ( 279 | 'rest_framework.renderers.JSONRenderer', 280 | 'rest_framework.renderers.BrowsableAPIRenderer', 281 | ), 282 | 'DEFAULT_AUTHENTICATION_CLASSES': ( 283 | 'rest_framework.authentication.BasicAuthentication', 284 | 'rest_framework.authentication.SessionAuthentication', 285 | ), 286 | 'DEFAULT_PERMISSION_CLASSES': ( 287 | 'rest_framework.permissions.IsAuthenticated', 288 | ), 289 | 'EXCEPTION_HANDLER': 'zinc.middleware.custom_exception_handler' 290 | } 291 | 292 | HEALTH_CHECK_CONFIG = { 293 | 'Port': 80, 294 | 'Type': 'HTTP', 295 | 'ResourcePath': '/status', 296 | 'FullyQualifiedDomainName': env.str('ZINC_HEALTH_CHECK_FQDN', 'node.presslabs.net.'), 297 | } 298 | 299 | ZINC_DEFAULT_TTL = env.int('ZINC_DEFAULT_TTL', default=300) 300 | ZINC_NS_CHECK_RESOLVERS = env.list('ZINC_NS_CHECK_RESOLVERS', default=['8.8.8.8']) 301 | ZONE_OWNERSHIP_COMMENT = env.str('ZINC_ZONE_OWNERSHIP_COMMENT', 'zinc') 302 | 303 | AWS_KEY = env.str('ZINC_AWS_KEY', '') 304 | AWS_SECRET = env.str('ZINC_AWS_SECRET', '') 305 | 306 | # configure logging 307 | LOG_LEVEL = env.str('ZINC_LOG_LEVEL', 'INFO') 308 | 309 | LOGGING = { 310 | 'version': 1, 311 | 'disable_existing_loggers': False, 312 | 'formatters': { 313 | 'simple': { 314 | 'format': '%(asctime)s %(message)-80s logger=%(name)s level=%(levelname)s ' 315 | 'process=%(processName)s thread=%(threadName)s' 316 | }, 317 | }, 318 | 'handlers': { 319 | 'console': { 320 | 'class': 'logging.StreamHandler', 321 | 'formatter': 'simple', 322 | 'level': LOG_LEVEL, 323 | }, 324 | }, 325 | 'loggers': { 326 | 'celery.task': { 327 | 'handlers': ['console'], 328 | 'level': LOG_LEVEL, 329 | 'propagate': False 330 | }, 331 | 'zinc': { 332 | 'handlers': ['console'], 333 | 'level': LOG_LEVEL, 334 | }, 335 | 'celery': { 336 | 'handlers': ['console'], 337 | 'level': LOG_LEVEL, 338 | 'propagate': False 339 | }, 340 | 'django': { 341 | 'handlers': ['console'], 342 | 'level': LOG_LEVEL, 343 | 'propagate': False 344 | }, 345 | }, 346 | } 347 | 348 | # https://docs.djangoproject.com/en/3.2/releases/3.2/#customizing-type-of-auto-created-primary-keys 349 | DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' 350 | 351 | 352 | if env.str('ZINC_SENTRY_DSN', ''): 353 | import raven 354 | INSTALLED_APPS += ['raven.contrib.django.raven_compat'] 355 | release = env.str('ZINC_RELEASE', 'git') 356 | if release == 'git': 357 | try: 358 | release = raven.fetch_git_sha(os.path.dirname(os.pardir)), 359 | except Exception as exc: 360 | import traceback 361 | traceback.print_exc(exc) 362 | release = 'git+UNKNOWN' 363 | RAVEN_CONFIG = { 364 | 'dsn': env.str('ZINC_SENTRY_DSN', ''), 365 | # If you are using git, you can also automatically configure the 366 | # release based on the git info. 367 | 'release': release, 368 | 'environment': env.str('ZINC_ENV_NAME', ''), 369 | } 370 | 371 | # Sentry logging with celery is a real pain in the ass 372 | # https://github.com/getsentry/sentry/issues/4565 373 | LOGGING['handlers']['sentry'] = { 374 | 'level': 'ERROR', 375 | 'class': 'raven.contrib.django.raven_compat.handlers.SentryHandler' 376 | } 377 | for logger in LOGGING['loggers']: 378 | LOGGING['loggers'][logger]['handlers'].append('sentry') 379 | 380 | 381 | SWAGGER_ENABLED = env.bool('ZINC_SWAGGER_ENABLED', DEBUG) 382 | -------------------------------------------------------------------------------- /django_project/settings/test.py: -------------------------------------------------------------------------------- 1 | from django_project.settings import * # noqa 2 | 3 | SECRET_KEY = 'test-secret' 4 | CELERY_ALWAYS_EAGER = True 5 | CELERY_EAGER_PROPAGATES_EXCEPTIONS = True 6 | 7 | REST_FRAMEWORK.update({ # noqa: F405 8 | 'DEFAULT_RENDERER_CLASSES': ('rest_framework.renderers.JSONRenderer',) 9 | }) 10 | 11 | LOGGING['loggers']['zinc']['level'] = 'WARN' # noqa: F405 12 | 13 | 14 | ZONE_OWNERSHIP_COMMENT = 'zinc-pytest' 15 | 16 | SWAGGER_ENABLED = True 17 | -------------------------------------------------------------------------------- /django_project/social_auth_pipeline.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | 3 | 4 | def set_user_perms(details=None, user=None, is_new=False, **kwargs): 5 | if not details['email'].endswith('@presslabs.com'): 6 | return None 7 | if not is_new: 8 | return None 9 | if not user: 10 | return None 11 | 12 | if details['email'] in getattr(settings, 'SOCIAL_AUTH_ADMIN_EMAILS', []): 13 | user.is_staff = True 14 | user.is_superuser = True 15 | 16 | user.save() 17 | 18 | return None 19 | -------------------------------------------------------------------------------- /django_project/urls.py: -------------------------------------------------------------------------------- 1 | """zinc URL Configuration 2 | 3 | The `urlpatterns` list routes URLs to views. For more information please see: 4 | https://docs.djangoproject.com/en/1.9/topics/http/urls/ 5 | Examples: 6 | Function views 7 | 1. Add an import: from my_app import views 8 | 2. Add a URL to urlpatterns: url(r'^$', views.home, name='home') 9 | Class-based views 10 | 1. Add an import: from other_app.views import Home 11 | 2. Add a URL to urlpatterns: url(r'^$', Home.as_view(), name='home') 12 | Including another URLconf 13 | 1. Import the include() function: from django.conf.urls import url, include 14 | 2. Add a URL to urlpatterns: url(r'^blog/', include('blog.urls')) 15 | """ 16 | from django.conf import settings 17 | from django.urls import include, path 18 | from django.contrib import admin 19 | 20 | from zinc.views import HealtchCheck 21 | 22 | urlpatterns = [ 23 | path('admin/', admin.site.urls), 24 | path('', include('zinc.urls')), 25 | path('_health', HealtchCheck.as_view()) 26 | ] 27 | 28 | if settings.SOCIAL_AUTH_GOOGLE_OAUTH2_KEY: 29 | urlpatterns.append( 30 | path('_auth/', include('social_django.urls', namespace='social')) 31 | ) 32 | 33 | if settings.SERVE_STATIC: 34 | from django.conf.urls.static import static 35 | urlpatterns += static(settings.STATIC_URL, document_root=settings.STATIC_ROOT) 36 | 37 | 38 | if settings.SWAGGER_ENABLED: 39 | from rest_framework import permissions 40 | from drf_yasg.views import get_schema_view 41 | from drf_yasg import openapi 42 | schema_view = get_schema_view( 43 | openapi.Info( 44 | title="API", 45 | default_version='v1', 46 | description="Zinc API.", 47 | ), 48 | public=True, 49 | permission_classes=(permissions.IsAuthenticated,), 50 | ) 51 | 52 | urlpatterns.append( 53 | path('swagger/', schema_view.with_ui('swagger', cache_timeout=0), name='schema-swagger-ui') 54 | ) 55 | 56 | if settings.DEBUG: 57 | try: 58 | import debug_toolbar 59 | except ImportError: 60 | pass 61 | else: 62 | urlpatterns.insert(0, path(r'^__debug__/', include(debug_toolbar.urls))) 63 | -------------------------------------------------------------------------------- /django_project/vendors/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/presslabs/zinc/e912a1a39c3dea16cecece7486ea816f040ac358/django_project/vendors/__init__.py -------------------------------------------------------------------------------- /django_project/vendors/celery.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import celery 4 | from django.conf import settings 5 | 6 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "django_project.settings") 7 | 8 | 9 | class Celery(celery.Celery): 10 | def _configure_sentry(self, raven_config): 11 | import raven 12 | from raven.contrib.celery import (register_signal, 13 | register_logger_signal) 14 | client = raven.Client(**raven_config) 15 | 16 | # register a custom filter to filter out duplicate logs 17 | register_logger_signal(client) 18 | 19 | # hook into the Celery error handler 20 | register_signal(client) 21 | 22 | def on_configure(self): 23 | raven_config = getattr(settings, 'RAVEN_CONFIG', '') 24 | if raven_config: 25 | self._configure_sentry(raven_config) 26 | 27 | 28 | app = Celery(__name__) 29 | app.config_from_object('django.conf:settings') 30 | app.autodiscover_tasks() 31 | -------------------------------------------------------------------------------- /django_project/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for zinc project. 3 | 4 | It exposes the WSGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/1.9/howto/deployment/wsgi/ 8 | """ 9 | 10 | from django.core.wsgi import get_wsgi_application 11 | 12 | application = get_wsgi_application() 13 | -------------------------------------------------------------------------------- /docker-compose.env: -------------------------------------------------------------------------------- 1 | BROKER_URL=redis://redis:6379/0 2 | CELERY_RESULT_BACKEND=redis://redis:6379/1 3 | ONCE_REDIS_URL=redis://redis:6379/2 4 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2' 2 | services: 3 | redis: 4 | image: redis:3.2-alpine 5 | ports: 6 | - "127.0.0.1:6379:6379" 7 | -------------------------------------------------------------------------------- /docker-entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -eo pipefail 3 | 4 | exec_web(){ 5 | if [ "$ZINC_MIGRATE" == "yes" ] ; then 6 | /app/manage.py migrate --noinput 7 | fi 8 | 9 | if [ "$ZINC_LOAD_DEV_DATA" == "yes" ] ; then 10 | /app/manage.py seed 11 | fi 12 | 13 | exec gunicorn django_project.wsgi --bind "$ZINC_WEB_ADDRESS" -k gevent $@ 14 | } 15 | 16 | case "$1" in 17 | "web") shift 1; exec_web $@;; 18 | "celery") shift 1; exec celery worker $@;; 19 | "celerybeat") shift 1; exec celery beat $@;; 20 | esac 21 | 22 | exec "$@" 23 | -------------------------------------------------------------------------------- /lattice_sync/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/presslabs/zinc/e912a1a39c3dea16cecece7486ea816f040ac358/lattice_sync/__init__.py -------------------------------------------------------------------------------- /lattice_sync/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class LatticeSyncConfig(AppConfig): 5 | name = 'lattice_sync' 6 | -------------------------------------------------------------------------------- /lattice_sync/management/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/presslabs/zinc/e912a1a39c3dea16cecece7486ea816f040ac358/lattice_sync/management/__init__.py -------------------------------------------------------------------------------- /lattice_sync/management/commands/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/presslabs/zinc/e912a1a39c3dea16cecece7486ea816f040ac358/lattice_sync/management/commands/__init__.py -------------------------------------------------------------------------------- /lattice_sync/management/commands/ips_from_lattice.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from django.core.management.base import BaseCommand 4 | from django.conf import settings 5 | 6 | from lattice_sync import sync 7 | 8 | 9 | logger = logging.getLogger('zinc.cli') 10 | 11 | 12 | class Command(BaseCommand): 13 | help = 'Imports IPs from a lattice server' 14 | 15 | def add_arguments(self, parser): 16 | parser.add_argument('--url', default=settings.LATTICE_URL) 17 | parser.add_argument('--user', default=settings.LATTICE_USER) 18 | parser.add_argument('--password', default=settings.LATTICE_PASSWORD) 19 | 20 | def handle(self, *args, **options): 21 | lattice = sync.lattice_factory(options['url'], 22 | options['user'], 23 | options['password']) 24 | sync.sync(lattice) 25 | logger.info("done") 26 | -------------------------------------------------------------------------------- /lattice_sync/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/presslabs/zinc/e912a1a39c3dea16cecece7486ea816f040ac358/lattice_sync/migrations/__init__.py -------------------------------------------------------------------------------- /lattice_sync/models.py: -------------------------------------------------------------------------------- 1 | # This needs to exists so that django allows installing this app 2 | -------------------------------------------------------------------------------- /lattice_sync/sync.py: -------------------------------------------------------------------------------- 1 | from urllib.parse import urlparse 2 | from logging import getLogger 3 | 4 | from django.conf import settings 5 | from django.db import transaction 6 | from django.utils.ipv6 import clean_ipv6_address 7 | from django.core.exceptions import ValidationError 8 | from requests.auth import HTTPBasicAuth 9 | from zipa import lattice # pylint: disable=no-name-in-module 10 | 11 | from zinc import models 12 | 13 | 14 | logger = getLogger('zinc.' + __name__) 15 | 16 | 17 | def lattice_factory(url, user, password): 18 | parts = urlparse(url) 19 | 20 | if url.startswith('http://'): 21 | lattice.config.secure = False 22 | lattice.config.verify = False 23 | 24 | lattice.config.host = parts.netloc 25 | lattice.config.prefix = parts.path 26 | lattice.config.auth = HTTPBasicAuth(user, password) 27 | return lattice 28 | 29 | 30 | def handle_ip(ip_addr, server, locations): 31 | enabled = server['state'] == 'configured' 32 | datacenter_id = int( 33 | server['datacenter_url'].split('?')[0].split('/')[-1]) 34 | location = locations.get(datacenter_id, 'fake_location') 35 | 36 | friendly_name = '{} {}'.format(server['hostname'].split('.')[0], 37 | location) 38 | ip = models.IP.objects.filter( 39 | ip=ip_addr, 40 | ).first() 41 | changed = False 42 | if ip is None: # new record 43 | ip = models.IP(ip=ip_addr, enabled=enabled) 44 | ip.reconcile_healthcheck() 45 | changed = True 46 | elif ip.enabled != enabled: 47 | ip.enabled = enabled 48 | ip.mark_policy_records_dirty() 49 | changed = True 50 | if ip.hostname != server['hostname']: 51 | ip.hostname = server['hostname'] 52 | changed = True 53 | if ip.friendly_name != friendly_name: 54 | ip.friendly_name = friendly_name 55 | changed = True 56 | if changed: 57 | ip.save() 58 | return ip.pk 59 | 60 | 61 | def sync(lattice_client): 62 | roles = set(settings.LATTICE_ROLES) 63 | env = settings.LATTICE_ENV.lower() 64 | servers = [ 65 | server for server in lattice_client.servers 66 | if (set(server['roles']).intersection(roles) and 67 | server['environment'].lower() == env and 68 | server['state'].lower() not in ('unconfigured', 'decommissioned')) 69 | ] 70 | locations = {d['id']: d['location'] for d in lattice_client.datacenters} 71 | 72 | lattice_ip_pks = set() 73 | 74 | with transaction.atomic(): 75 | for server in servers: 76 | for ip in server.ips: 77 | # normalize IP in order to prevent having different values because in db 78 | # the IP is cleaned already 79 | ip_value = ip['ip'] 80 | if ':' in ip_value: 81 | try: 82 | ip_value = clean_ipv6_address(ip_value) 83 | except ValidationError: 84 | logger.error("Bad IPv6 address %s", ip_value) 85 | continue 86 | 87 | ip_pk = handle_ip(ip_value, server, locations) 88 | if ip_pk is not None: 89 | lattice_ip_pks.add(ip_pk) 90 | 91 | if not lattice_ip_pks: 92 | raise AssertionError("Refusing to delete all IPs!") 93 | 94 | ips_to_remove = set( 95 | models.IP.objects.values_list('pk', flat=True)) - lattice_ip_pks 96 | 97 | for ip in models.IP.objects.filter(pk__in=ips_to_remove): 98 | ip.soft_delete() 99 | -------------------------------------------------------------------------------- /lattice_sync/tasks.py: -------------------------------------------------------------------------------- 1 | from celery.utils.log import get_task_logger 2 | from celery import shared_task 3 | from django.conf import settings 4 | 5 | from lattice_sync import sync 6 | 7 | logger = get_task_logger(__name__) 8 | 9 | 10 | @shared_task(ignore_result=True, default_retry_delay=60) 11 | def lattice_sync(): 12 | lattice = sync.lattice_factory( 13 | settings.LATTICE_URL, settings.LATTICE_USER, settings.LATTICE_PASSWORD) 14 | sync.sync(lattice) 15 | -------------------------------------------------------------------------------- /lattice_sync/tests/test_sync.py: -------------------------------------------------------------------------------- 1 | import copy 2 | import json 3 | import mock 4 | 5 | import pytest 6 | import responses 7 | from django_dynamic_fixture import G 8 | 9 | from lattice_sync import sync 10 | from tests.fixtures import boto_client # noqa: F401 11 | from zinc.models import IP, PolicyMember 12 | 13 | lattice = sync.lattice_factory(url='http://lattice', user='user', password='password') 14 | 15 | 16 | @pytest.mark.django_db 17 | @responses.activate 18 | def test_sync_exception(boto_client): 19 | servers_payload = copy.deepcopy(default_servers_payload) 20 | servers_payload[0]["state"] = "maintenance" 21 | _mock_lattice_responses(servers_payload=servers_payload) 22 | 23 | ip = G(IP, ip=servers_payload[0]["ips"][0]["ip"]) 24 | G(PolicyMember, ip=ip) 25 | 26 | with mock.patch('zinc.models.IP.mark_policy_records_dirty') as m: 27 | m.side_effect = RuntimeError('MockedException') 28 | with pytest.raises(RuntimeError): 29 | sync.sync(lattice) 30 | 31 | ips = list(IP.objects.all()) 32 | assert ips == [ip] 33 | 34 | 35 | @pytest.mark.django_db 36 | @responses.activate 37 | def test_wont_delete_all_ips(boto_client): 38 | for url in ['http://lattice/servers', 'http://lattice/datacenters']: 39 | responses.add(responses.GET, url, 40 | body='[]', 41 | content_type='application/json') 42 | 43 | addr = '123.123.123.123' 44 | G(IP, ip=addr) 45 | 46 | assert list(IP.objects.all().values_list('ip', flat=True)) == [addr] 47 | with pytest.raises(AssertionError) as excp_info: 48 | sync.sync(lattice) 49 | assert excp_info.match("Refusing to delete all IPs") 50 | assert list(IP.objects.values_list('ip', flat=True)) == [addr] 51 | 52 | 53 | @pytest.mark.django_db 54 | @responses.activate 55 | def test_removes_ip(boto_client): 56 | _mock_lattice_responses() 57 | 58 | addr = '1.2.3.4' # not in the mock response 59 | G(IP, ip='1.2.3.4') 60 | 61 | assert list(IP.objects.all().values_list('ip', flat=True)) == [addr] 62 | sync.sync(lattice) 63 | assert not IP.objects.filter(ip=addr).exists() 64 | 65 | 66 | @pytest.mark.django_db 67 | @responses.activate 68 | def test_adds_only_ips_from_servers_in_specified_roles(boto_client): 69 | _mock_lattice_responses() 70 | 71 | sync.sync(lattice) 72 | 73 | assert IP.objects.count() == 2 74 | assert IP.objects.filter(ip__in=['123.123.123.123', '123.123.123.124']).count() == 2 75 | 76 | 77 | @pytest.mark.django_db 78 | @responses.activate 79 | def test_fields_on_written_ip(boto_client): 80 | _mock_lattice_responses() 81 | 82 | sync.sync(lattice) 83 | 84 | ip = IP.objects.get(ip='123.123.123.123') 85 | 86 | expected_fields = { 87 | 'ip': '123.123.123.123', 88 | 'friendly_name': 'a Amsterdam, NL', 89 | 'enabled': True, 90 | 'hostname': 'a.presslabs.net' 91 | } 92 | attributes = { 93 | field: getattr(ip, field) 94 | for field in expected_fields.keys() 95 | } 96 | assert attributes == expected_fields 97 | 98 | 99 | default_servers_payload = [ 100 | { 101 | "hostname": "a.presslabs.net", 102 | "state": "configured", 103 | "group": "", 104 | "environment": "production", 105 | "roles": [ 106 | "edge-node" 107 | ], 108 | "datacenter_name": "AMS1", 109 | "service_id": "", 110 | "ips": [ 111 | { 112 | "ip": "123.123.123.123", 113 | "netmask": "", 114 | "gateway": "", 115 | "url": "http://localhost:8001/servers/a/ips/123.123.123.123?format=json", 116 | "description": "" 117 | }, 118 | { 119 | "ip": "123.123.123.124", 120 | "netmask": "", 121 | "gateway": "", 122 | "url": "http://localhost:8001/servers/a/ips/123.123.123.124?format=json" 123 | }], 124 | "uplink_speed": None, 125 | "bandwidth": None, 126 | "memory": None, 127 | "cpu_model_name": "", 128 | "cpu_model_url": None, 129 | "datacenter_url": "http://localhost:8001/datacenters/5?format=json" 130 | }, 131 | { 132 | "hostname": "b", 133 | "state": "configured", 134 | "group": "", 135 | "environment": "production", 136 | "roles": [ 137 | "random-node" 138 | ], 139 | "datacenter_name": "AMS2", 140 | "service_id": "", 141 | "ips": [{ 142 | "ip": "123.123.123.125", 143 | "netmask": "", 144 | "gateway": "", 145 | "url": "http://localhost:8001/servers/b/ips/123.123.123.125?format=json", 146 | "description": "" 147 | }], 148 | "uplink_speed": None, 149 | "bandwidth": None, 150 | "memory": None, 151 | "cpu_model_name": "", 152 | "cpu_model_url": None, 153 | "datacenter_url": "http://localhost:8001/datacenters/6?format=json" 154 | }] 155 | 156 | default_datacenters_payload = [ 157 | { 158 | "name": "AMS1", 159 | "location": "Amsterdam, NL", 160 | "provider": "providers.providers.DigitalOcean", 161 | "latitude": None, 162 | "longitude": None, 163 | "id": 5 164 | }, 165 | { 166 | "name": "AMS2", 167 | "location": "Amsterdam, NL", 168 | "provider": "providers.providers.DigitalOcean", 169 | "latitude": None, 170 | "longitude": None, 171 | "id": 6 172 | }] 173 | 174 | 175 | def _mock_lattice_responses(servers_payload=None, datacenters_payload=None): 176 | responses.add(responses.GET, 'http://lattice/servers', 177 | body=json.dumps(servers_payload or default_servers_payload), 178 | content_type='application/json') 179 | responses.add(responses.GET, 'http://lattice/datacenters', 180 | body=json.dumps(datacenters_payload or default_datacenters_payload), 181 | content_type='application/json') 182 | -------------------------------------------------------------------------------- /local_settings.py.example: -------------------------------------------------------------------------------- 1 | # vim: set ft=python: 2 | from zinc.settings.base import * 3 | 4 | DEBUG=True 5 | SECRET_KEY='not-so-secret' 6 | 7 | AWS_KEY='ASK FOR A AWS KEY' 8 | AWS_SECRET='' 9 | 10 | CELERY_ALWAYS_EAGER = True 11 | CELERY_EAGER_PROPAGATES_EXCEPTIONS = True 12 | 13 | -------------------------------------------------------------------------------- /manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | 5 | if __name__ == "__main__": 6 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "django_project.settings") 7 | 8 | from django.core.management import execute_from_command_line 9 | 10 | execute_from_command_line(sys.argv) 11 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.pytest.ini_options] 2 | DJANGO_SETTINGS_MODULE = "django_project.settings.test" 3 | norecursedirs = "contrib" 4 | testpaths = [ "tests", ] 5 | 6 | [tool.ruff] 7 | line-length = 120 8 | exclude = [ "tests", ] 9 | 10 | 11 | [tool.ruff.per-file-ignores] 12 | "**/tests/*" = ["F811"] -------------------------------------------------------------------------------- /requirements.dev.txt: -------------------------------------------------------------------------------- 1 | -r requirements.txt 2 | 3 | pytest>=7.4,<8 4 | pytest-django>=4.5,<5 5 | pytest-runner>=6.0,<7 6 | pytest-xdist>=3.3,<4 7 | pytest-ruff 8 | responses 9 | 10 | -e .[test] 11 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | Django==4.1.10 2 | boto3==1.28.30 3 | celery==5.3.1 4 | drf-yasg>=1.21,<2 5 | djangorestframework==3.14.0 6 | dnspython==2.4.2 7 | gevent==23.7.0 8 | gunicorn==21.2.0 9 | hashids==1.3.1 10 | PyMySQL==1.1.0 11 | raven==6.10.0 12 | redis==5.0.1 13 | requests==2.31.0 14 | social-auth-app-django==5.2.0 15 | social-auth-core==4.4.2 16 | zipa==0.3.6 17 | django-dynamic-fixture==3.1.3 18 | whitenoise==6.5.0 19 | django-environ==0.10.0 20 | python-redis-lock==4.0.0 21 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | 3 | install_requires = [ 4 | 'boto3', 5 | 'celery', 6 | 'Django', 7 | 'djangorestframework', 8 | 'dnspython', 9 | 'hashids', 10 | 'redis', 11 | 'requests', 12 | 'zipa', 13 | ] 14 | tests_require = ['pytest', 'pytest-runner>=6.0,<7', 'pytest-ruff'] 15 | 16 | setup( 17 | name='zinc-dns', 18 | version='1.1.0', 19 | description="Route 53 zone manager", 20 | author="Presslabs", 21 | author_email="ping@presslabs.com", 22 | url="https://github.com/Presslabs/zinc", 23 | install_requires=install_requires, 24 | tests_require=tests_require, 25 | packages=find_packages(include=['zinc', 'zinc.*']), 26 | extras_require={ 27 | 'test': tests_require 28 | }, 29 | classifiers=[ 30 | 'Environment :: Web Environment', 31 | 'Framework :: Django', 32 | 'Intended Audience :: Developers', 33 | 'License :: OSI Approved :: BSD License', # example license 34 | 'Operating System :: OS Independent', 35 | 'Programming Language :: Python', 36 | 'Programming Language :: Python :: 3', 37 | 'Programming Language :: Python :: 3.11', 38 | ] 39 | ) 40 | -------------------------------------------------------------------------------- /templates/admin/filter.html: -------------------------------------------------------------------------------- 1 | {% load i18n %} 2 | 3 |

{% blocktrans with filter_title=title %} By {{ filter_title }} {% endblocktrans %}

4 | 24 | -------------------------------------------------------------------------------- /templates/admin/login.html: -------------------------------------------------------------------------------- 1 | {% extends "admin/login.html" %} 2 | 3 | {% block content %} 4 | {{ block.super }} 5 | 6 |
7 | {% url 'social:begin' 'google-oauth2' as social_login_url %} 8 | {% if social_login_url %} 9 |
10 | 11 | 12 |
13 | {% endif %} 14 |
15 | {% endblock %} -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/presslabs/zinc/e912a1a39c3dea16cecece7486ea816f040ac358/tests/__init__.py -------------------------------------------------------------------------------- /tests/api/test_basic_api.py: -------------------------------------------------------------------------------- 1 | # pylint: disable=no-member,protected-access,redefined-outer-name 2 | import re 3 | 4 | import pytest 5 | from django_dynamic_fixture import G 6 | 7 | from tests.fixtures import Moto, api_client, boto_client # noqa: F401 8 | from zinc import models as m 9 | 10 | 11 | @pytest.mark.django_db 12 | def test_header_pagination(api_client): 13 | for _ in range(2): 14 | G(m.Policy) 15 | page1 = api_client.get('/policies', {'page_size': 1, 'page': 1}) 16 | assert 'Link' in page1, 'Response is missing Link header' 17 | assert page1.status_code == 200, page1 18 | assert 'rel="next"' in page1['Link'], 'Response is missing rel="next" link header {}'.format(page1['Link']) # noqa 19 | 20 | match = re.search(r'<(http://.*)>; rel="next"', page1['Link']) 21 | assert match, "Invalid link header: {}".format(page1['Link']) 22 | 23 | page2 = api_client.get(match.group(1)) 24 | assert page2.status_code == 200, page2.request 25 | 26 | 27 | class ThrottledMoto: 28 | def __init__(self, throttle): 29 | self._call_countdown = dict() 30 | self._proxied_moto = Moto() 31 | for method_name, throttle_after_numcalls in throttle.items(): 32 | method = getattr(self._proxied_moto, method_name) 33 | assert method is not None, \ 34 | "No such method to throttle '{}'".format(method_name) 35 | assert callable(method),\ 36 | "Attribute '{}' is not a callable".format(method_name) 37 | self._call_countdown[method_name] = throttle_after_numcalls 38 | 39 | def __getattr__(self, attr_name): 40 | countdown = self._call_countdown.get(attr_name) 41 | if countdown is not None: 42 | if countdown == 0: 43 | raise self.exceptions.ThrottlingException( 44 | error_response={ 45 | 'Error': { 46 | 'Code': 'Throttled', 47 | 'Message': 'Hold your horses...', 48 | 'Type': 'Sender' 49 | }, 50 | }, 51 | operation_name=attr_name, 52 | ) 53 | else: 54 | self._call_countdown[attr_name] -= 1 55 | return getattr(self._proxied_moto, attr_name) 56 | 57 | def __hasattr__(self, attr_name): 58 | return hasattr(self._proxied_moto, attr_name) 59 | 60 | @classmethod 61 | def factory(cls, throttle): 62 | return lambda: cls(throttle) 63 | 64 | 65 | @pytest.mark.django_db 66 | @pytest.mark.parametrize( 67 | 'boto_client', [ 68 | ThrottledMoto.factory(throttle={'create_hosted_zone': 0}) 69 | ], indirect=True) 70 | def test_throttled(api_client, boto_client): 71 | root = 'example.com.presslabs.com.' 72 | resp = api_client.post( 73 | '/zones', 74 | data={ 75 | 'root': root, 76 | } 77 | ) 78 | assert resp.status_code == 429, resp.data 79 | -------------------------------------------------------------------------------- /tests/api/test_policy.py: -------------------------------------------------------------------------------- 1 | # pylint: disable=no-member,protected-access,redefined-outer-name 2 | import pytest 3 | from django_dynamic_fixture import G 4 | 5 | from tests.fixtures import api_client, boto_client, zone # noqa: F401 6 | from zinc import models as m 7 | 8 | 9 | def policy_member_to_dict(record): 10 | return { 11 | 'id': str(record.id), 12 | 'region': record.region, 13 | 'ip': record.ip.ip, 14 | 'weight': record.weight, 15 | 'enabled': record.enabled and record.ip.enabled 16 | } 17 | 18 | 19 | def policy_to_dict(policy): 20 | return { 21 | 'id': str(policy.id), 22 | 'name': policy.name, 23 | 'members': [policy_member_to_dict(member) for member in policy.members.all()], 24 | 'url': 'http://testserver/policies/{}'.format(policy.id) 25 | } 26 | 27 | 28 | @pytest.mark.django_db 29 | def test_policy_list(api_client): 30 | pol = G(m.Policy) 31 | resp = api_client.get( 32 | '/policies', 33 | format='json', 34 | ) 35 | assert resp.status_code == 200, resp 36 | assert [str(pol.id)] == [res['id'] for res in resp.data] 37 | 38 | 39 | @pytest.mark.django_db 40 | def test_policy_details(api_client): 41 | policy = G(m.Policy) 42 | response = api_client.get( 43 | '/policies/%s' % policy.id, 44 | format='json', 45 | ) 46 | assert response.status_code == 200, response 47 | assert response.data == policy_to_dict(policy) 48 | 49 | 50 | @pytest.mark.django_db 51 | def test_policy_with_records(api_client): 52 | policy = G(m.Policy) 53 | G(m.PolicyMember, policy=policy) 54 | G(m.PolicyMember, policy=policy) 55 | response = api_client.get( 56 | '/policies/%s' % policy.id, 57 | format='json', 58 | ) 59 | 60 | assert response.status_code == 200, response 61 | assert response.data == policy_to_dict(policy) 62 | -------------------------------------------------------------------------------- /tests/api/test_zone.py: -------------------------------------------------------------------------------- 1 | # pylint: disable=no-member,unused-argument,protected-access,redefined-outer-name 2 | import pytest 3 | 4 | from botocore.exceptions import ClientError 5 | from django_dynamic_fixture import G 6 | 7 | 8 | from tests.fixtures import api_client, boto_client, zone # noqa: F401 9 | from tests.utils import strip_ns_and_soa, get_test_record 10 | 11 | from zinc import models as m 12 | 13 | 14 | @pytest.mark.django_db 15 | def test_create_zone(api_client, boto_client): 16 | root = 'example.com.presslabs.com.' 17 | resp = api_client.post( 18 | '/zones', 19 | data={ 20 | 'root': root, 21 | } 22 | ) 23 | assert resp.status_code == 201, resp.data 24 | assert resp.data['root'] == root 25 | _id = resp.data['id'] 26 | assert list(m.Zone.objects.all().values_list('id', 'root')) == [(_id, root)] 27 | 28 | 29 | @pytest.mark.django_db 30 | def test_create_zone_passing_wrong_params(api_client, boto_client): 31 | resp = api_client.post( 32 | '/zones', 33 | data={ 34 | 'id': 'asd', 35 | 'root': 'asdasd' 36 | } 37 | ) 38 | assert resp.status_code == 400, resp.data 39 | assert resp.data['root'] == ['Invalid root domain'] 40 | 41 | 42 | @pytest.mark.django_db 43 | def test_list_zones(api_client, boto_client): 44 | zones = [G(m.Zone, root='1.test-zinc.com.', route53_id=None), 45 | G(m.Zone, root='2.test-zinc.com.', route53_id=None)] 46 | 47 | response = api_client.get('/zones') 48 | 49 | assert [result['url'] for result in response.data] == [ 50 | "http://testserver/zones/{}".format(zone.id) for zone in zones] 51 | assert ([(zone.id, zone.root, zone.dirty, zone.r53_zone.id) for zone in zones] == 52 | [(zone['id'], zone['root'], zone['dirty'], zone['route53_id']) 53 | for zone in response.data]) 54 | 55 | 56 | @pytest.mark.django_db 57 | def test_detail_zone(api_client, zone): 58 | response = api_client.get( 59 | '/zones/%s' % zone.id, 60 | ) 61 | assert strip_ns_and_soa(response.data['records']) == [ 62 | get_test_record(zone) 63 | ] 64 | assert response.data['route53_id'] == zone.route53_id 65 | assert response.data['dirty'] is False 66 | 67 | 68 | @pytest.mark.django_db 69 | def test_delete_a_zone(api_client, zone, boto_client): 70 | response = api_client.delete( 71 | '/zones/%s' % zone.id 72 | ) 73 | 74 | with pytest.raises(ClientError) as excp_info: 75 | boto_client.get_hosted_zone(Id=zone.route53_id) 76 | assert excp_info.value.response['Error']['Code'] == 'NoSuchHostedZone' 77 | assert m.Zone.objects.filter(pk=zone.pk).count() == 0 78 | assert not response.data 79 | 80 | 81 | @pytest.mark.django_db 82 | def test_policy_record_create_more_values(api_client, zone): 83 | response = api_client.post( 84 | '/zones/%s/records' % zone.id, 85 | data={ 86 | 'name': '@', 87 | 'type': 'CNAME', 88 | 'ttl': 300, 89 | 'values': ['test1.com', 'test2.com'] 90 | } 91 | ) 92 | assert response.status_code == 400 93 | assert response.data == { 94 | 'values': [ 95 | 'Only one value can be specified for CNAME records.' 96 | ] 97 | } 98 | 99 | 100 | @pytest.mark.django_db 101 | def test_create_zone_no_fqdn(api_client, boto_client): 102 | root = 'presslabs.com' 103 | resp = api_client.post( 104 | '/zones', 105 | data={ 106 | 'root': root, 107 | } 108 | ) 109 | root += '.' 110 | assert resp.status_code == 201, resp.data 111 | assert resp.data['root'] == root 112 | _id = resp.data['id'] 113 | assert list(m.Zone.objects.all().values_list('id', 'root')) == [(_id, root)] 114 | -------------------------------------------------------------------------------- /tests/dns/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/presslabs/zinc/e912a1a39c3dea16cecece7486ea816f040ac358/tests/dns/__init__.py -------------------------------------------------------------------------------- /tests/dns/test_health_checks.py: -------------------------------------------------------------------------------- 1 | # pylint: disable=no-member,protected-access,redefined-outer-name 2 | import pytest 3 | import botocore.exceptions 4 | 5 | from zinc import models as m 6 | from tests.fixtures import boto_client # noqa: F401, pylint: disable=unused-import 7 | 8 | 9 | @pytest.mark.django_db 10 | def test_health_check_create(boto_client): 11 | ip = m.IP.objects.create( 12 | ip='1.2.3.4', 13 | hostname='fe01-mordor.presslabs.net.', 14 | ) 15 | ip.reconcile_healthcheck() 16 | ip.refresh_from_db() 17 | expected_config = { 18 | 'IPAddress': ip.ip, 19 | 'Port': 80, 20 | 'Type': 'HTTP', 21 | 'ResourcePath': '/status', 22 | 'FullyQualifiedDomainName': 'node.presslabs.net.', 23 | } 24 | resp = boto_client.get_health_check(HealthCheckId=ip.healthcheck_id)['HealthCheck'] 25 | 26 | # {1:1, 2:2}.items() >= {2:2}.items() is True # first is a superset of the second, all is good 27 | # {1:1, 2:2}.items() <= {2:2}.items() is False # first is not a superset of the second 28 | assert resp['HealthCheckConfig'].items() >= expected_config.items() 29 | 30 | 31 | @pytest.mark.django_db 32 | def test_health_check_change(boto_client): 33 | ip = m.IP.objects.create( 34 | ip='1.2.3.4', 35 | hostname='fe01-mordor.presslabs.net.', 36 | ) 37 | ip.reconcile_healthcheck() 38 | ip.refresh_from_db() 39 | original_check_id = ip.healthcheck_id 40 | expected_config = { 41 | 'IPAddress': ip.ip, 42 | 'Port': 80, 43 | 'Type': 'HTTP', 44 | 'ResourcePath': '/status', 45 | 'FullyQualifiedDomainName': 'node.presslabs.net.', 46 | } 47 | resp = boto_client.get_health_check(HealthCheckId=ip.healthcheck_id)['HealthCheck'] 48 | assert resp['HealthCheckConfig'].items() >= expected_config.items() 49 | 50 | ip.ip = '1.1.1.1' # change the ip 51 | ip.save() 52 | ip.reconcile_healthcheck() 53 | ip.refresh_from_db() 54 | 55 | expected_config['IPAddress'] = ip.ip 56 | resp = boto_client.get_health_check(HealthCheckId=ip.healthcheck_id)['HealthCheck'] 57 | assert resp['HealthCheckConfig'].items() >= expected_config.items() 58 | # ensure the old healthcheck got deleted 59 | with pytest.raises(botocore.exceptions.ClientError) as excp_info: 60 | boto_client.get_health_check(HealthCheckId=original_check_id) 61 | assert excp_info.value.response['Error']['Code'] == 'NoSuchHealthCheck' 62 | 63 | 64 | @pytest.mark.django_db 65 | def test_health_check_reconcile(boto_client): 66 | ip = m.IP.objects.create( 67 | ip='1.2.3.4', 68 | hostname='fe01-mordor.presslabs.net.', 69 | ) 70 | ip.reconcile_healthcheck() 71 | ip.refresh_from_db() 72 | original_check_id = ip.healthcheck_id 73 | 74 | expected_config = { 75 | 'IPAddress': ip.ip, 76 | 'Port': 80, 77 | 'Type': 'HTTP', 78 | 'ResourcePath': '/status', 79 | 'FullyQualifiedDomainName': 'node.presslabs.net.', 80 | } 81 | resp = boto_client.get_health_check(HealthCheckId=ip.healthcheck_id)['HealthCheck'] 82 | assert resp['HealthCheckConfig'].items() >= expected_config.items() 83 | 84 | # simulate a failure during creation, so we have a HC in AWS but no HC id locally 85 | ip.healthcheck_id = None 86 | ip.save() 87 | # reconcile should preserve the caller reference and end up with the original id 88 | ip.reconcile_healthcheck() 89 | ip.refresh_from_db() 90 | assert ip.healthcheck_id == original_check_id 91 | resp = boto_client.get_health_check(HealthCheckId=ip.healthcheck_id)['HealthCheck'] 92 | assert resp['HealthCheckConfig'].items() >= expected_config.items() 93 | -------------------------------------------------------------------------------- /tests/dns/test_zone.py: -------------------------------------------------------------------------------- 1 | # pylint: disable=no-member,protected-access,redefined-outer-name 2 | import botocore.exceptions 3 | from django_dynamic_fixture import G 4 | 5 | import pytest 6 | from zinc import models, route53 7 | from tests.fixtures import boto_client, zone # noqa: F401 8 | from tests.utils import hash_test_record 9 | 10 | regions = route53.get_local_aws_regions() 11 | 12 | 13 | @pytest.mark.django_db 14 | def test_add_zone_record(zone): 15 | record = route53.Record( 16 | name='goo', 17 | type='CNAME', 18 | values=['google.com'], 19 | ttl=300, 20 | zone=zone.r53_zone, 21 | ) 22 | record.save() 23 | zone.r53_zone.commit() 24 | 25 | assert record.id in [r.id for r in zone.records] 26 | 27 | 28 | @pytest.mark.django_db 29 | def test_delete_zone_record(zone): 30 | record_hash = hash_test_record(zone) 31 | for r in zone.records: 32 | if r.id == record_hash: 33 | record = r 34 | 35 | zone.delete_record(record) 36 | zone.r53_zone.commit() 37 | 38 | assert record_hash not in [r.id for r in zone.records] 39 | 40 | 41 | @pytest.mark.django_db 42 | def test_delete_zone_record_by_hash(zone): 43 | record_hash = hash_test_record(zone) 44 | 45 | zone.delete_record_by_hash(record_hash) 46 | zone.r53_zone.commit() 47 | 48 | assert record_hash not in zone.records 49 | 50 | 51 | @pytest.mark.django_db 52 | def test_delete_zone_alias_record(zone): 53 | record = route53.Record( 54 | name='something', 55 | type='A', 56 | alias_target={ 57 | 'DNSName': 'test.%s' % zone.root, 58 | 'HostedZoneId': zone.r53_zone.id, 59 | 'EvaluateTargetHealth': False 60 | }, 61 | zone=zone.r53_zone, 62 | ) 63 | record.save() 64 | zone.commit() 65 | assert record.id in [r.id for r in zone.records] 66 | 67 | zone.delete_record(record) 68 | zone.r53_zone.commit() 69 | 70 | assert record.id not in [r.id for r in zone.records] 71 | 72 | 73 | @pytest.mark.django_db 74 | def test_delete_zone_alias_record_with_set_id(zone): 75 | record = route53.Record( 76 | name='_zn_something', 77 | type='A', 78 | alias_target={ 79 | 'DNSName': 'test.%s' % zone.root, 80 | 'HostedZoneId': zone.r53_zone.id, 81 | 'EvaluateTargetHealth': False 82 | }, 83 | set_identifier='set_id', 84 | region=regions[0], 85 | zone=zone.r53_zone, 86 | ) 87 | record.save() 88 | zone.r53_zone.commit() 89 | 90 | zone.delete_record(record) 91 | zone.r53_zone.commit() 92 | 93 | assert record.id not in zone.records 94 | 95 | 96 | @pytest.mark.django_db 97 | def test_zone_delete(zone, boto_client): 98 | zone_id = zone.r53_zone.id 99 | zone_name = 'test-zinc.net.' 100 | # make sure we have extra records in addition to the NS and SOA 101 | # to ensure zone.delete handles those as well 102 | boto_client.change_resource_record_sets( 103 | HostedZoneId=zone_id, 104 | ChangeBatch={ 105 | 'Comment': 'zinc-fixture', 106 | 'Changes': [ 107 | { 108 | 'Action': 'CREATE', 109 | 'ResourceRecordSet': { 110 | 'Name': 'some_ns.%s' % zone_name, 111 | 'Type': 'NS', 112 | 'TTL': 300, 113 | 'ResourceRecords': [ 114 | { 115 | 'Value': 'ns-1941.awszinc-50.co.uk.', 116 | } 117 | ] 118 | } 119 | }, 120 | { 121 | 'Action': 'CREATE', 122 | 'ResourceRecordSet': { 123 | 'Name': 'some_a.%s' % zone_name, 124 | 'Type': 'A', 125 | 'TTL': 300, 126 | 'ResourceRecords': [ 127 | { 128 | 'Value': '1.1.1.2', 129 | } 130 | ] 131 | } 132 | } 133 | ] 134 | } 135 | ) 136 | zone.r53_zone.delete() 137 | with pytest.raises(botocore.exceptions.ClientError) as excp: 138 | boto_client.get_hosted_zone(Id=zone_id) 139 | assert excp.value.response['Error']['Code'] == 'NoSuchHostedZone' 140 | 141 | 142 | def test_zone_exists_false(boto_client): 143 | db_zone = models.Zone(route53_id='Does/Not/Exist') 144 | zone = route53.Zone(db_zone) 145 | assert not zone.exists 146 | 147 | 148 | @pytest.mark.django_db 149 | def test_zone_reconcile_deleted_from_aws(zone, boto_client): 150 | original_id = zone.route53_id 151 | route53.Zone(zone)._delete_records() 152 | boto_client.delete_hosted_zone(Id=original_id) 153 | zone.r53_zone._clear_cache() 154 | zone.r53_zone.reconcile() 155 | assert zone.route53_id != original_id 156 | 157 | 158 | @pytest.mark.django_db 159 | def test_zone_exists_true(zone): 160 | assert route53.Zone(zone).exists 161 | 162 | 163 | @pytest.mark.django_db 164 | def test_delete_missing_zone(boto_client): 165 | """Test zone delete is idempotent 166 | If we have a zone marked deleted in the db, calling delete should be safe and 167 | remove the db record for good. 168 | """ 169 | db_zone = G(models.Zone, route53_id='Does/Not/Exist', deleted=True) 170 | route53.Zone(db_zone).delete() 171 | assert models.Zone.objects.filter(pk=db_zone.pk).count() == 0 172 | 173 | 174 | @pytest.mark.django_db 175 | def test_delete_zone_no_zone_id(boto_client): 176 | """Test zone delete works for zones that don't have a route53_id 177 | """ 178 | db_zone = G(models.Zone, route53_id=None, deleted=False) 179 | db_zone.soft_delete() 180 | assert not models.Zone.objects.filter(pk=db_zone.pk).exists() 181 | 182 | 183 | @pytest.mark.django_db 184 | def test_zone_need_reconciliation(zone): 185 | 186 | G(models.Zone, route53_id='fake/id/1', deleted=False) # ok zone 187 | no_id_zone = G(models.Zone, route53_id=None, deleted=False) 188 | soft_deleted_zone = G(models.Zone, route53_id='fake/id/2', deleted=True) 189 | G(models.PolicyRecord, zone=zone, dirty=True) 190 | expected_dirty = [no_id_zone, soft_deleted_zone, zone] 191 | expected = [(z.pk, z.root) for z in expected_dirty] 192 | assert sorted(expected) == sorted([(z.pk, z.root) for z in models.Zone.need_reconciliation()]) 193 | 194 | 195 | @pytest.mark.django_db 196 | def test_zone_get_clean_zones(zone): 197 | ok_zone = G(models.Zone, route53_id='fake/id/1', deleted=False) # ok zone 198 | G(models.Zone, route53_id=None, deleted=False) # no_id_zone 199 | G(models.Zone, route53_id='fake/id/2', deleted=True) # soft_deleted_zone 200 | G(models.PolicyRecord, zone=zone, dirty=True) 201 | expected_clean = [ok_zone] 202 | expected = [(z.pk, z.root) for z in expected_clean] 203 | assert sorted(expected) == sorted([(z.pk, z.root) for z in models.Zone.get_clean_zones()]) 204 | -------------------------------------------------------------------------------- /tests/test_models.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from django.core.exceptions import ValidationError 3 | 4 | from zinc.models import Policy 5 | 6 | 7 | @pytest.mark.django_db 8 | def test_policy_name_validation_not_unique_first_chars(): 9 | Policy(name="dev01").save() 10 | with pytest.raises(ValidationError): 11 | Policy(name="dev011").full_clean() 12 | 13 | Policy(name="dev022").save() 14 | with pytest.raises(ValidationError): 15 | Policy(name="dev02").full_clean() 16 | 17 | 18 | @pytest.mark.django_db 19 | def test_policy_name_validation_regex(): 20 | with pytest.raises(ValidationError): 21 | Policy(name="not-allowed-chars;").full_clean() 22 | 23 | with pytest.raises(ValidationError): 24 | Policy(name="UpperCaseName").full_clean() 25 | -------------------------------------------------------------------------------- /tests/test_ns_check.py: -------------------------------------------------------------------------------- 1 | import json 2 | from unittest import mock 3 | 4 | import pytest 5 | 6 | from zinc import ns_check, models 7 | from tests.fixtures import zone, boto_client, Moto # noqa: F401 8 | 9 | 10 | @pytest.mark.parametrize("boto_client", [Moto], ids=['fake_boto'], indirect=True) 11 | @pytest.mark.django_db 12 | def test_is_ns_propagated(zone, boto_client): 13 | resolver = mock.Mock() 14 | resolver.query.return_value = ["test_ns1.presslabs.net", "test_ns2.presslabs.net"] 15 | with mock.patch('zinc.ns_check.get_resolver', lambda: resolver): 16 | assert ns_check.is_ns_propagated(zone) 17 | 18 | 19 | @pytest.mark.parametrize("boto_client", [Moto], ids=['fake_boto'], indirect=True) 20 | @pytest.mark.django_db 21 | def test_is_ns_propagated_delegated_zone(zone, boto_client): 22 | """Ensure is_ns_propagated ignores NS records for delegated zones 23 | The root cause of https://github.com/PressLabs/zinc/issues/182 24 | """ 25 | boto_client.change_resource_record_sets( 26 | HostedZoneId=zone.route53_id, 27 | ChangeBatch={ 28 | 'Comment': 'zinc-fixture', 29 | 'Changes': [ 30 | { 31 | 'Action': 'CREATE', 32 | 'ResourceRecordSet': { 33 | 'Name': 'delegated.' + zone.root, 34 | 'Type': 'NS', 35 | 'TTL': 300, 36 | 'ResourceRecords': [ 37 | { 38 | 'Value': 'ns1.example.com', 39 | } 40 | ] 41 | } 42 | }, 43 | ] 44 | } 45 | ) 46 | resolver = mock.Mock() 47 | resolver.query.return_value = ["test_ns1.presslabs.net", "test_ns2.presslabs.net"] 48 | with mock.patch('zinc.ns_check.get_resolver', lambda: resolver): 49 | assert ns_check.is_ns_propagated(zone) 50 | 51 | 52 | @pytest.mark.parametrize("boto_client", [Moto], ids=['fake_boto'], indirect=True) 53 | @pytest.mark.django_db 54 | def test_is_ns_propagated_false(zone): 55 | resolver = mock.Mock() 56 | resolver.query.return_value = ["some_other_ns.example.com"] 57 | with mock.patch('zinc.ns_check.get_resolver', lambda: resolver): 58 | assert ns_check.is_ns_propagated(zone) is False 59 | 60 | 61 | @pytest.mark.parametrize("boto_client", [Moto], ids=['fake_boto'], indirect=True) 62 | @pytest.mark.django_db 63 | def test_update_ns_propagated(zone): 64 | assert zone.ns_propagated is False 65 | resolver = mock.Mock() 66 | resolver.query.return_value = ["test_ns1.presslabs.net", "test_ns2.presslabs.net"] 67 | with mock.patch('zinc.ns_check.get_resolver', lambda: resolver): 68 | models.Zone.update_ns_propagated() 69 | zone.refresh_from_db() 70 | assert zone.ns_propagated 71 | 72 | 73 | @pytest.mark.parametrize("boto_client", [Moto], ids=['fake_boto'], indirect=True) 74 | @pytest.mark.django_db 75 | def test_update_ns_propagated_false(zone): 76 | assert zone.ns_propagated is False 77 | resolver = mock.Mock() 78 | resolver.query.return_value = ["some_other_ns.example.com"] 79 | with mock.patch('zinc.ns_check.get_resolver', lambda: resolver): 80 | models.Zone.update_ns_propagated() 81 | zone.refresh_from_db() 82 | assert zone.ns_propagated is False 83 | 84 | 85 | @pytest.mark.parametrize("boto_client", [Moto], ids=['fake_boto'], indirect=True) 86 | @pytest.mark.django_db 87 | def test_update_ns_propagated_updates_cached_ns_records_empty_cache(zone): 88 | assert zone.cached_ns_records is None 89 | ns_records = ["test_ns1.presslabs.net", "test_ns2.presslabs.net"] 90 | resolver = mock.Mock() 91 | resolver.query.return_value = ["test_ns1.presslabs.net", "test_ns2.presslabs.net"] 92 | with mock.patch('zinc.ns_check.get_resolver', lambda: resolver): 93 | models.Zone.update_ns_propagated() 94 | zone.refresh_from_db() 95 | assert set(json.loads(zone.cached_ns_records)) == set(ns_records) 96 | assert zone.ns_propagated 97 | 98 | 99 | @pytest.mark.parametrize("boto_client", [Moto], ids=['fake_boto'], indirect=True) 100 | @pytest.mark.django_db 101 | def test_update_ns_propagated_updates_cached_ns_records(zone): 102 | zone.cached_ns_records = json.dumps(["ns1.example.com"]) 103 | zone.save() 104 | ns_records = ["test_ns1.presslabs.net", "test_ns2.presslabs.net"] 105 | resolver = mock.Mock() 106 | resolver.query.return_value = ["test_ns1.presslabs.net", "test_ns2.presslabs.net"] 107 | with mock.patch('zinc.ns_check.get_resolver', lambda: resolver): 108 | models.Zone.update_ns_propagated() 109 | zone.refresh_from_db() 110 | assert set(json.loads(zone.cached_ns_records)) == set(ns_records) 111 | assert zone.ns_propagated 112 | -------------------------------------------------------------------------------- /tests/utils.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from django_dynamic_fixture import G 4 | 5 | from zinc import models as m 6 | from zinc import route53 7 | from zinc.utils.generators import chunks 8 | 9 | 10 | def is_ns_or_soa(record): 11 | if isinstance(record, route53.record.BaseRecord): 12 | return (record.type in ('NS', 'SOA') and record.name == '@') 13 | else: 14 | return (record['type'] in ('NS', 'SOA') and record['name'] == '@') 15 | 16 | 17 | def strip_ns_and_soa(records): 18 | """The NS and SOA records are managed by AWS, so we won't care about them in tests""" 19 | return [dict(record) for record in records if not is_ns_or_soa(record)] 20 | 21 | 22 | def hash_test_record(zone): 23 | return route53.Record( 24 | name='test', 25 | type='A', 26 | zone=zone.r53_zone, 27 | ).id 28 | 29 | 30 | def hash_policy_record(policy_record): 31 | return policy_record.serialize().id 32 | 33 | 34 | def hash_record_dict(record, zone): 35 | return route53.Record(zone=zone.r53_zone, **record).id 36 | 37 | 38 | def aws_sort_key(record): 39 | return (record['Name'], record['Type'], record.get('SetIdentifier', None)) 40 | 41 | 42 | def aws_strip_ns_and_soa(records, zone_root): 43 | """The NS and SOA records are managed by AWS, so we won't care about them in tests""" 44 | return sorted([ 45 | record for record in records['ResourceRecordSets'] 46 | if not(record['Type'] == 'SOA' or (record['Type'] == 'NS' and record['Name'] == zone_root)) 47 | ], key=aws_sort_key) 48 | 49 | 50 | def get_test_record(zone): 51 | return { 52 | 'id': hash_test_record(zone), 53 | 'name': 'test', 54 | 'fqdn': 'test.%s' % zone.root, 55 | 'ttl': 300, 56 | 'type': 'A', 57 | 'values': ['1.1.1.1'], 58 | 'dirty': False, 59 | 'managed': False, 60 | 'url': 'http://testserver/zones/%s/records/%s' % (zone.id, hash_test_record(zone)) 61 | } 62 | 63 | 64 | def _split_overlength_value_into_chunks(value): 65 | # Only TXT records might require splitting their values, so it doesn't matter if we split 66 | # all record types values in tests 67 | 68 | max_length = 255 69 | 70 | if len(value) >= max_length: 71 | value = ' '.join('{}'.format(json.dumps(element)) 72 | for element in chunks(value, max_length)) 73 | 74 | return {'Value': value} 75 | 76 | 77 | def record_data_to_aws(record, zone_root): 78 | rrs = { 79 | 'Name': '{}.{}'.format(record['name'], zone_root), 80 | 'TTL': record['ttl'], 81 | 'Type': record['type'], 82 | 'ResourceRecords': [_split_overlength_value_into_chunks(value) 83 | for value in record['values']], 84 | } 85 | if record.get('SetIdentifier', None): 86 | rrs['SetIdentifier'] = record['SetIdentifier'] 87 | return rrs 88 | 89 | 90 | def create_ip_with_healthcheck(**kwargs): 91 | kwargs['healthcheck_id'] = None 92 | kwargs['healthcheck_caller_reference'] = None 93 | ip = G(m.IP, **kwargs) 94 | ip.reconcile_healthcheck() 95 | ip.refresh_from_db() 96 | return ip 97 | 98 | 99 | def record_data_to_response(record, zone, managed=False, dirty=False): 100 | record_hash = hash_record_dict(record, zone) 101 | keys = ['name', 'type', 'ttl', 'values'] 102 | return { 103 | **{key: value for key, value in record.items() if key in keys}, 104 | 'fqdn': '{}.{}'.format(record['name'], zone.root), 105 | 'id': record_hash, 106 | 'url': 'http://testserver/zones/%s/records/%s' % (zone.id, record_hash), 107 | 'managed': managed, 108 | 'dirty': dirty 109 | } 110 | 111 | 112 | def record_to_response(record, zone, managed=False, dirty=False): 113 | record_hash = record.id 114 | keys = ['name', 'type', 'ttl', 'values'] 115 | return { 116 | **{key: getattr(record, key) for key in keys}, 117 | 'fqdn': '{}.{}'.format(record.name, zone.root), 118 | 'id': record_hash, 119 | 'url': 'http://testserver/zones/%s/records/%s' % (zone.id, record_hash), 120 | 'managed': managed, 121 | 'dirty': dirty 122 | } 123 | 124 | 125 | def get_record_from_base(record, *a, **kwa): 126 | if isinstance(record, route53.Record): 127 | return record_to_response(record, *a, **kwa) 128 | else: 129 | return record_data_to_response(record, *a, **kwa) 130 | 131 | 132 | def meld(got, expected): 133 | if got == expected: 134 | return 135 | import inspect 136 | call_frame = inspect.getouterframes(inspect.currentframe(), 2) 137 | test_name = call_frame[1][3] 138 | from pprint import pformat 139 | import os 140 | from os import path 141 | os.makedirs(test_name, exist_ok=True) 142 | got_fn = path.join(test_name, 'got') 143 | expected_fn = path.join(test_name, 'expected') 144 | with open(got_fn, 'w') as got_f, open(expected_fn, 'w') as expected_f: 145 | got_f.write(pformat(got)) 146 | expected_f.write(pformat(expected)) 147 | import subprocess 148 | subprocess.run(['meld', got_fn, expected_fn]) 149 | -------------------------------------------------------------------------------- /zinc/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/presslabs/zinc/e912a1a39c3dea16cecece7486ea816f040ac358/zinc/__init__.py -------------------------------------------------------------------------------- /zinc/admin/__init__.py: -------------------------------------------------------------------------------- 1 | from .zone import ZoneAdmin 2 | from .ip import IPAdmin 3 | from .policy import PolicyAdmin 4 | from .policy_record import PolicyRecordAdmin 5 | -------------------------------------------------------------------------------- /zinc/admin/ip.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from django.contrib import admin 4 | from django.db import transaction 5 | from django.utils.html import format_html 6 | 7 | from zinc.models import IP 8 | 9 | from .soft_delete import SoftDeleteAdmin 10 | 11 | 12 | logger = logging.getLogger(__name__) 13 | 14 | 15 | @admin.register(IP) 16 | class IPAdmin(SoftDeleteAdmin): 17 | list_display = ('ip', 'hostname', 'enabled', 'healthcheck') 18 | list_filter = ('hostname', 'deleted') 19 | 20 | fields = ('ip', 'hostname', 'friendly_name', 'healthcheck_id', 21 | 'healthcheck_caller_reference', 'enabled', 'deleted') 22 | 23 | readonly_fields = ('deleted',) 24 | 25 | @transaction.atomic 26 | def save_model(self, request, obj, form, change): 27 | super().save_model(request, obj, form, change) 28 | obj.reconcile_healthcheck() 29 | obj.mark_policy_records_dirty() 30 | 31 | def healthcheck(self, obj): 32 | if obj.healthcheck_id: 33 | return format_html('AWS:{0}', obj.healthcheck_id) 35 | else: 36 | return "" 37 | -------------------------------------------------------------------------------- /zinc/admin/policy.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | from django.db import transaction 3 | from django.utils.html import mark_safe, format_html 4 | 5 | from zinc.models import Policy, PolicyMember 6 | 7 | 8 | class PolicyMemberInline(admin.TabularInline): 9 | readonly_fields = ('ip_enabled',) 10 | model = PolicyMember 11 | extra = 1 12 | verbose_name = 'member' 13 | verbose_name_plural = 'members' 14 | 15 | def ip_enabled(self, obj): 16 | return obj.ip.enabled 17 | ip_enabled.boolean = True 18 | 19 | 20 | @admin.register(Policy) 21 | class PolicyAdmin(admin.ModelAdmin): 22 | fields = ('name', 'routing', 'ttl') 23 | readonly_fields = () 24 | list_display = ('__str__', 'routing', 'regions', 'status') 25 | list_filter = ('routing', 'members__region') 26 | inlines = (PolicyMemberInline,) 27 | exclude = ('members',) 28 | 29 | def get_queryset(self, request): 30 | qs = super(PolicyAdmin, self).get_queryset(request) 31 | qs = qs.prefetch_related('members') 32 | return qs 33 | 34 | def regions(self, obj): 35 | # get_queryset prefetches related policy members so iterating over 36 | # objects is ok because we are iterating over already fetched data 37 | return ', '.join(sorted({m.region for m in obj.members.all()})) 38 | 39 | @transaction.atomic 40 | def save_model(self, request, obj, form, change): 41 | rv = super().save_model(request, obj, form, change) 42 | obj.change_trigger(form.changed_data) 43 | return rv 44 | 45 | def status(self, obj): 46 | warnings = [] 47 | if obj.routing == 'latency': 48 | members_by_region = {} 49 | for member in obj.members.all(): 50 | members_by_region.setdefault(member.region, []).append(member) 51 | if len(members_by_region) <= 1: 52 | warnings.append('✖ Latency routed policy should span multiple regions!') 53 | for region, members in members_by_region.items(): 54 | if len([m for m in members if m.weight > 0]) == 0: 55 | warnings.append( 56 | '✖ All members of region {} have weight zero!'.format(region)) 57 | elif obj.routing == 'weighted': 58 | active_members = [m for m in obj.members.all() if m.weight > 0] 59 | if len(active_members) == 0: 60 | warnings.append('✖ All members have weight zero!') 61 | if warnings: 62 | return format_html('{}', mark_safe("
".join(warnings))) 63 | else: 64 | return mark_safe("✔ ok") 65 | status.short_description = 'Status' 66 | -------------------------------------------------------------------------------- /zinc/admin/policy_record.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | from django.db import transaction 3 | from django.utils.html import mark_safe 4 | 5 | from zinc.models import PolicyRecord 6 | from zinc.admin.zone import aws_zone_link 7 | from zinc.admin.soft_delete import SoftDeleteAdmin 8 | 9 | 10 | def mark_dirty(modeladmin, request, queryset): 11 | for policy_record in queryset: 12 | policy_record.mark_dirty() 13 | mark_dirty.short_description = "Mark selected records dirty" # noqa: E305 14 | 15 | 16 | def mark_clean(modeladmin, request, queryset): 17 | for policy_record in queryset: 18 | policy_record.dirty = False 19 | policy_record.save() 20 | mark_clean.short_description = "Mark selected records as clean" # noqa: E305 21 | 22 | 23 | @admin.register(PolicyRecord) 24 | class PolicyRecordAdmin(SoftDeleteAdmin): 25 | list_display = ('__str__', 'record_type', 'policy', 'aws_link', 'synced', 'is_deleted') 26 | list_filter = ('zone', 'policy', 'dirty') 27 | 28 | fields = ('name', 'zone', 'policy', 'record_type', 'synced') 29 | readonly_fields = ('synced', ) 30 | actions = (mark_dirty, mark_clean, ) 31 | 32 | def get_readonly_fields(self, request, obj=None): 33 | if obj: # editing an existing object 34 | return self.readonly_fields + ('name',) 35 | return self.readonly_fields 36 | 37 | def synced(self, obj): 38 | return not obj.dirty 39 | synced.boolean = True 40 | synced.short_description = 'Synced' 41 | synced.admin_order_field = 'dirty' 42 | 43 | def aws_link(self, obj): 44 | return mark_safe(aws_zone_link(obj.zone.route53_id)) if obj.zone.route53_id else "" 45 | aws_link.short_description = 'Zone' 46 | aws_link.admin_order_field = 'zone' 47 | 48 | @transaction.atomic 49 | def save_model(self, request, obj, form, change): 50 | if form.changed_data: 51 | obj.dirty = True 52 | super().save_model(request, obj, form, change) 53 | -------------------------------------------------------------------------------- /zinc/admin/soft_delete.py: -------------------------------------------------------------------------------- 1 | from django.core.exceptions import PermissionDenied 2 | from django.contrib import admin 3 | from django.contrib.admin.actions import delete_selected as delete_selected_ 4 | 5 | 6 | def delete_selected(modeladmin, request, queryset): 7 | if not modeladmin.has_delete_permission(request): 8 | raise PermissionDenied 9 | if request.POST.get('post'): 10 | for obj in queryset: 11 | obj.soft_delete() 12 | else: 13 | return delete_selected_(modeladmin, request, queryset) 14 | delete_selected.short_description = "Delete selected" # noqa 15 | 16 | 17 | class SoftDeleteAdmin(admin.ModelAdmin): 18 | actions = (delete_selected,) 19 | 20 | def delete_model(self, request, obj): 21 | """Soft delete zone object""" 22 | obj.soft_delete() 23 | 24 | def is_deleted(self, obj): 25 | return "DELETED" if obj.deleted else "" 26 | -------------------------------------------------------------------------------- /zinc/admin/zone.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from botocore.exceptions import ClientError 4 | from django.contrib import admin 5 | from django.utils.html import mark_safe 6 | 7 | from zinc.models import Zone 8 | 9 | from .soft_delete import SoftDeleteAdmin 10 | 11 | logger = logging.getLogger('zinc.admin') 12 | 13 | 14 | def aws_zone_link(r53_zone_id): 15 | return ('AWS:{0}'.format(r53_zone_id)) 17 | 18 | 19 | @admin.register(Zone) 20 | class ZoneAdmin(SoftDeleteAdmin): 21 | list_filter = ('ns_propagated', 'deleted') 22 | list_display = ('root', 'aws_link', 'ns_propagated', 'is_deleted') 23 | fields = ('root', 'route53_id', 'caller_reference', 'ns_propagated') 24 | readonly_fields = ('route53_id', 'caller_reference', 'ns_propagated') 25 | search_fields = ('root', 'route53_id') 26 | 27 | def get_readonly_fields(self, request, obj=None): 28 | if obj: 29 | return self.readonly_fields + ('root',) 30 | return self.readonly_fields 31 | 32 | def save_model(self, request, obj, form, change): 33 | super().save_model(request, obj, form, change) 34 | try: 35 | obj.reconcile() 36 | except ClientError: 37 | logger.exception("Error while calling reconcile for hosted zone") 38 | 39 | def aws_link(self, obj): 40 | return mark_safe(aws_zone_link(obj.route53_id)) if obj.route53_id else "" 41 | aws_link.short_description = 'Zone' 42 | aws_link.admin_order_field = 'zone' 43 | -------------------------------------------------------------------------------- /zinc/management/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/presslabs/zinc/e912a1a39c3dea16cecece7486ea816f040ac358/zinc/management/__init__.py -------------------------------------------------------------------------------- /zinc/management/commands/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/presslabs/zinc/e912a1a39c3dea16cecece7486ea816f040ac358/zinc/management/commands/__init__.py -------------------------------------------------------------------------------- /zinc/management/commands/cleanup_stale_zones.py: -------------------------------------------------------------------------------- 1 | from django.core.management.base import BaseCommand 2 | 3 | from zinc import models 4 | from zinc import route53 5 | from zinc.route53.client import get_client 6 | 7 | 8 | class Command(BaseCommand): 9 | help = 'Delete unknown r53 zones' 10 | 11 | def add_arguments(self, parser): 12 | parser.add_argument( 13 | '--dry-run', 14 | action='store_true', 15 | dest='dry_run', 16 | default=False, 17 | help='Only print which zones would be deleted', 18 | ) 19 | 20 | def handle(self, *args, **options): 21 | dry_run = options['dry_run'] 22 | db_ids = set(models.Zone.objects.exclude(route53_id=None) 23 | .values_list('route53_id', flat=True)) 24 | client = get_client() 25 | paginator = client.get_paginator('list_hosted_zones') 26 | 27 | raw_zones = [] 28 | for page in paginator.paginate(): 29 | raw_zones.extend([ 30 | zone for zone in page['HostedZones'] 31 | if zone['Id'] not in db_ids]) 32 | 33 | for raw_zone in raw_zones: 34 | db_zone = models.Zone(route53_id=raw_zone['Id'], root=raw_zone['Name']) 35 | zone = route53.Zone(db_zone) 36 | print('deleting {}'.format(zone)) 37 | if not dry_run: 38 | zone.delete_from_r53() 39 | -------------------------------------------------------------------------------- /zinc/management/commands/reconcile_healthchecks.py: -------------------------------------------------------------------------------- 1 | from django.core.management.base import BaseCommand 2 | 3 | from zinc.models import IP 4 | from zinc.route53 import HealthCheck 5 | 6 | 7 | class Command(BaseCommand): 8 | help = 'Reconcile all ip healthchecks' 9 | 10 | def handle(self, *args, **options): 11 | HealthCheck.reconcile_for_ips(IP.objects.all()) 12 | -------------------------------------------------------------------------------- /zinc/management/commands/reconcile_policy_records.py: -------------------------------------------------------------------------------- 1 | from django.core.management.base import BaseCommand 2 | 3 | from zinc import tasks 4 | 5 | 6 | class Command(BaseCommand): 7 | help = 'Rebuild trees for any zone that has a dirty PolicyRecord' 8 | 9 | def handle(self, *args, **options): 10 | tasks.reconcile_policy_records.apply() 11 | -------------------------------------------------------------------------------- /zinc/management/commands/reconcile_zones.py: -------------------------------------------------------------------------------- 1 | from django.core.management.base import BaseCommand 2 | 3 | from zinc import models 4 | from zinc import route53 5 | 6 | 7 | class Command(BaseCommand): 8 | help = 'Reconcile all ip healthchecks' 9 | 10 | def handle(self, *args, **options): 11 | route53.Zone.reconcile_multiple( 12 | models.Zone.objects.all()) 13 | -------------------------------------------------------------------------------- /zinc/management/commands/seed.py: -------------------------------------------------------------------------------- 1 | from django.contrib.auth import get_user_model 2 | from django.contrib.auth.hashers import make_password 3 | from django.core.management.base import BaseCommand 4 | from django.db import transaction 5 | from django_dynamic_fixture import G 6 | 7 | from zinc import models 8 | 9 | 10 | class Command(BaseCommand): 11 | help = 'Seed the DB' 12 | 13 | @transaction.atomic 14 | def handle(self, *a, **kwa): 15 | admin = get_user_model()(**{ 16 | "pk": 1, 17 | "username": "admin", 18 | "password": make_password("admin"), 19 | "is_staff": True, 20 | "is_superuser": True 21 | }) 22 | admin.save() 23 | 24 | G(models.IP, **{ 25 | "ip": "138.68.67.220", 26 | "hostname": "test-ip-1", 27 | "friendly_name": "test-ip-1", 28 | "healthcheck_id": None, 29 | "enabled": True, 30 | "deleted": False 31 | }) 32 | 33 | G(models.IP, **{ 34 | "ip": "138.68.79.108", 35 | "hostname": "test-ip-2", 36 | "friendly_name": "test-ip-2", 37 | "healthcheck_id": None, 38 | "enabled": True, 39 | "deleted": False 40 | }) 41 | 42 | G(models.Policy, **{ 43 | "id": "aabb0610-36a4-4328-9879-338ab1813cfb", 44 | "name": "policy-one" 45 | }) 46 | 47 | G(models.PolicyMember, **{ 48 | "id": "5ae24136-7e2b-407b-9447-6fa70a7829df", 49 | "region": "us-east-1", 50 | "ip": "138.68.79.108", 51 | "policy": "aabb0610-36a4-4328-9879-338ab1813cfb", 52 | "weight": 10 53 | }) 54 | 55 | G(models.PolicyMember, **{ 56 | "id": "67999f8a-71f7-41f7-a73a-35a2c76895a6", 57 | "region": "eu-central-1", 58 | "ip": "138.68.67.220", 59 | "policy": "aabb0610-36a4-4328-9879-338ab1813cfb", 60 | "weight": 10 61 | }) 62 | -------------------------------------------------------------------------------- /zinc/management/commands/update_ns_propagated.py: -------------------------------------------------------------------------------- 1 | from django.core.management.base import BaseCommand 2 | 3 | from zinc.models import Zone 4 | 5 | 6 | class Command(BaseCommand): 7 | help = 'Update Zone.ns_propagated for all zones' 8 | 9 | def handle(self, *args, **options): 10 | Zone.update_ns_propagated() 11 | print("Done!") 12 | -------------------------------------------------------------------------------- /zinc/middleware.py: -------------------------------------------------------------------------------- 1 | from rest_framework.exceptions import Throttled 2 | from rest_framework.views import exception_handler 3 | 4 | from zinc.route53.client import get_client 5 | 6 | 7 | def custom_exception_handler(excp, context): 8 | # Call REST framework's default exception handler first, 9 | # to get the standard error response. 10 | if isinstance(excp, get_client().exceptions.ThrottlingException): 11 | excp = Throttled() 12 | return exception_handler(excp, context) 13 | -------------------------------------------------------------------------------- /zinc/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.10.6 on 2017-03-08 15:53 3 | from __future__ import unicode_literals 4 | 5 | import django.core.validators 6 | from django.db import migrations, models 7 | import django.db.models.deletion 8 | import uuid 9 | 10 | 11 | class Migration(migrations.Migration): 12 | 13 | initial = True 14 | 15 | dependencies = [ 16 | ] 17 | 18 | operations = [ 19 | migrations.CreateModel( 20 | name='IP', 21 | fields=[ 22 | ('ip', models.GenericIPAddressField(primary_key=True, protocol='IPv4', serialize=False, verbose_name='IP Address')), 23 | ('hostname', models.CharField(max_length=64, validators=[django.core.validators.RegexValidator(code='invalid_hostname', message='Invalid hostname', regex='^(?=[a-z0-9\\-\\.]{1,253}$)([a-z0-9](([a-z0-9\\-]){,61}[a-z0-9])?\\.)*([a-z0-9](([a-z0-9\\-]){,61}[a-z0-9])?)$')])), 24 | ('friendly_name', models.TextField(blank=True)), 25 | ('enabled', models.BooleanField(default=True)), 26 | ('healthcheck_id', models.CharField(blank=True, max_length=200, null=True)), 27 | ('healthcheck_caller_reference', models.UUIDField(blank=True, null=True)), 28 | ('deleted', models.BooleanField(default=False)), 29 | ], 30 | options={ 31 | 'verbose_name': 'IP', 32 | }, 33 | ), 34 | migrations.CreateModel( 35 | name='Policy', 36 | fields=[ 37 | ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), 38 | ('name', models.CharField(max_length=255, unique=True)), 39 | ], 40 | options={ 41 | 'verbose_name_plural': 'policies', 42 | }, 43 | ), 44 | migrations.CreateModel( 45 | name='PolicyMember', 46 | fields=[ 47 | ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), 48 | ('region', models.CharField(choices=[('us-east-1', 'us-east-1'), ('us-west-1', 'us-west-1'), ('us-west-2', 'us-west-2'), ('ap-northeast-1', 'ap-northeast-1'), ('ap-northeast-2', 'ap-northeast-2'), ('ap-south-1', 'ap-south-1'), ('ap-southeast-1', 'ap-southeast-1'), ('ap-southeast-2', 'ap-southeast-2'), ('sa-east-1', 'sa-east-1'), ('eu-west-1', 'eu-west-1'), ('eu-central-1', 'eu-central-1')], max_length=20)), 49 | ('weight', models.PositiveIntegerField(default=10)), 50 | ('ip', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='policy_members', to='zinc.IP')), 51 | ('policy', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='members', to='zinc.Policy')), 52 | ], 53 | ), 54 | migrations.CreateModel( 55 | name='PolicyRecord', 56 | fields=[ 57 | ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), 58 | ('name', models.CharField(max_length=255)), 59 | ('dirty', models.BooleanField(default=True, editable=False)), 60 | ('deleted', models.BooleanField(default=False)), 61 | ('policy', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='records', to='zinc.Policy')), 62 | ], 63 | ), 64 | migrations.CreateModel( 65 | name='Zone', 66 | fields=[ 67 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 68 | ('root', models.CharField(max_length=255, validators=[django.core.validators.RegexValidator(code='invalid_root_domain', message='Invalid root domain', regex='[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?(?:\\.(?!-)[a-z0-9-]{1,63}(? 1: 13 | policy.routing = 'latency' 14 | else: 15 | policy.routing = 'weighted' 16 | policy.save() 17 | 18 | 19 | class Migration(migrations.Migration): 20 | 21 | dependencies = [ 22 | ('zinc', '0007_policy_routing'), 23 | ] 24 | 25 | operations = [ 26 | migrations.RunPython(set_routing_policy) 27 | ] 28 | -------------------------------------------------------------------------------- /zinc/migrations/0009_auto_20220228_1318.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.29 on 2022-02-28 13:18 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('zinc', '0008_set_routing_policy'), 12 | ] 13 | 14 | operations = [ 15 | migrations.AddField( 16 | model_name='policyrecord', 17 | name='record_type', 18 | field=models.CharField(choices=[('A', 'A'), ('AAAA', 'AAAA')], default='A', max_length=10), 19 | ), 20 | migrations.AlterField( 21 | model_name='ip', 22 | name='ip', 23 | field=models.GenericIPAddressField(primary_key=True, serialize=False, verbose_name='IP Address'), 24 | ), 25 | migrations.AlterField( 26 | model_name='policymember', 27 | name='region', 28 | field=models.CharField(choices=[('af-south-1', 'Africa (Cape Town)'), ('ap-east-1', 'Asia Pacific (Hong Kong)'), ('ap-northeast-1', 'Asia Pacific (Tokyo)'), ('ap-northeast-2', 'Asia Pacific (Seoul)'), ('ap-northeast-3', 'Asia Pacific (Osaka-Local)'), ('ap-south-1', 'Asia Pacific (Mumbai)'), ('ap-southeast-1', 'Asia Pacific (Singapore)'), ('ap-southeast-2', 'Asia Pacific (Sydney)'), ('ap-southeast-3', 'Asia Pacific (Jakarta)'), ('ca-central-1', 'Canada (Central)'), ('cn-north-1', 'China (Beijing)'), ('cn-northwest-1', 'China (Ningxia)'), ('eu-central-1', 'Europe (Frankfurt)'), ('eu-north-1', 'Europe (Stockholm)'), ('eu-south-1', 'Europe (Milan)'), ('eu-west-1', 'Europe (Ireland)'), ('eu-west-2', 'Europe (London)'), ('eu-west-3', 'Europe (Paris)'), ('me-south-1', 'Middle East (Bahrain)'), ('sa-east-1', 'South America (São Paulo)'), ('us-east-1', 'US East (N. Virginia)'), ('us-east-2', 'US East (Ohio)'), ('us-west-1', 'US West (N. California)'), ('us-west-2', 'US West (Oregon)')], default='us-east-1', max_length=20), 29 | ), 30 | migrations.AlterUniqueTogether( 31 | name='policyrecord', 32 | unique_together=set([('name', 'record_type', 'zone')]), 33 | ), 34 | ] 35 | -------------------------------------------------------------------------------- /zinc/migrations/0010_policy_ttl.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.29 on 2022-02-28 14:01 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('zinc', '0009_auto_20220228_1318'), 12 | ] 13 | 14 | operations = [ 15 | migrations.AddField( 16 | model_name='policy', 17 | name='ttl', 18 | field=models.PositiveIntegerField(default=30), 19 | ), 20 | ] 21 | -------------------------------------------------------------------------------- /zinc/migrations/0011_alter_policy_name.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.1.10 on 2023-08-22 11:52 2 | 3 | import django.core.validators 4 | from django.db import migrations, models 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('zinc', '0010_policy_ttl'), 11 | ] 12 | 13 | operations = [ 14 | migrations.AlterField( 15 | model_name='policy', 16 | name='name', 17 | field=models.CharField(max_length=255, unique=True, validators=[django.core.validators.RegexValidator(code='invalid_policy_name', message='Policy name should contain only lowercase letters, numbers and hyphens', regex='^[a-z0-9-]+$')]), 18 | ), 19 | ] 20 | -------------------------------------------------------------------------------- /zinc/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/presslabs/zinc/e912a1a39c3dea16cecece7486ea816f040ac358/zinc/migrations/__init__.py -------------------------------------------------------------------------------- /zinc/models.py: -------------------------------------------------------------------------------- 1 | from collections import OrderedDict 2 | import collections.abc 3 | import contextlib 4 | import json 5 | import uuid 6 | from logging import getLogger 7 | 8 | from django.core.exceptions import ValidationError 9 | from django.db import models, transaction 10 | from django.db.models import Q 11 | 12 | from zinc import ns_check, route53, tasks 13 | from zinc.route53 import HealthCheck, get_local_aws_region_choices 14 | from zinc.route53.record import RECORD_PREFIX 15 | from zinc.validators import validate_domain, validate_hostname, validate_policy_name 16 | 17 | 18 | logger = getLogger(__name__) 19 | 20 | ROUTING_CHOICES = OrderedDict([ 21 | ("latency", "latency"), 22 | ("weighted", "weighted"), 23 | ]) 24 | 25 | 26 | class IP(models.Model): 27 | ip = models.GenericIPAddressField( 28 | primary_key=True, 29 | protocol='both', 30 | verbose_name='IP Address' 31 | ) 32 | hostname = models.CharField(max_length=64, validators=[validate_hostname]) 33 | friendly_name = models.TextField(blank=True) 34 | enabled = models.BooleanField(default=True) 35 | healthcheck_id = models.CharField(max_length=200, blank=True, null=True) 36 | healthcheck_caller_reference = models.UUIDField(null=True, blank=True) 37 | deleted = models.BooleanField(default=False) 38 | 39 | class Meta: 40 | verbose_name = 'IP' 41 | 42 | def mark_policy_records_dirty(self): 43 | # sadly this breaks sqlite 44 | # policies = [ 45 | # member.policy for member in 46 | # self.policy_members.order_by('policy_id').distinct('policy_id')] 47 | policies = set([ 48 | member.policy for member in 49 | self.policy_members.all()]) 50 | for policy in policies: 51 | policy.mark_policy_records_dirty() 52 | 53 | def soft_delete(self): 54 | self.deleted = True 55 | self.enabled = False 56 | self.save(update_fields=['deleted', 'enabled']) 57 | self.reconcile_healthcheck() 58 | 59 | def reconcile_healthcheck(self): 60 | HealthCheck(self).reconcile() 61 | 62 | def __str__(self): 63 | value = self.friendly_name or self.hostname.split(".", 1)[0] 64 | return '{} ({})'.format(self.ip, value) 65 | 66 | 67 | class Policy(models.Model): 68 | id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) 69 | name = models.CharField(max_length=255, validators=[validate_policy_name], 70 | unique=True, null=False) 71 | routing = models.CharField( 72 | max_length=255, choices=ROUTING_CHOICES.items(), default=ROUTING_CHOICES['latency']) 73 | 74 | ttl = models.PositiveIntegerField(default=30) 75 | 76 | dirty_trigger_fields = set(['name', 'ttl']) 77 | 78 | class Meta: 79 | verbose_name_plural = 'policies' 80 | ordering = ('name',) 81 | 82 | def __str__(self): 83 | return self.name 84 | 85 | def change_trigger(self, field_names): 86 | # if field_names is not a set-like object (eg. dict_keys) convert to set 87 | if not isinstance(field_names, collections.abc.Set): 88 | field_names = set(field_names) 89 | if field_names & self.dirty_trigger_fields: 90 | self.mark_policy_records_dirty() 91 | 92 | # atomic isn't strictly required since it's a single statement that would run 93 | # in a transaction in autocommit mode on innodb, but it's better to be explicit 94 | @transaction.atomic 95 | def mark_policy_records_dirty(self): 96 | self.records.update(dirty=True) 97 | 98 | def clean(self): 99 | # validate name to start with unique characters in order to prevent the tree builder 100 | # marching and removing from other policies with similar name. 101 | for policy in Policy.objects.exclude(id=self.id): 102 | min_len = min(len(policy.name), len(self.name)) 103 | if self.name[:min_len] == policy.name[:min_len]: 104 | raise ValidationError({ 105 | 'name': 'The name "{}" has first {} chars equal with policy "{}"'.format( 106 | self.name, min_len, policy.name 107 | ) 108 | }) 109 | 110 | 111 | class PolicyMember(models.Model): 112 | AWS_REGIONS = get_local_aws_region_choices() 113 | 114 | id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) 115 | region = models.CharField(choices=AWS_REGIONS, max_length=20, 116 | default='us-east-1') 117 | ip = models.ForeignKey(IP, on_delete=models.CASCADE, related_name='policy_members') 118 | policy = models.ForeignKey(Policy, on_delete=models.CASCADE, related_name='members') 119 | weight = models.PositiveIntegerField(default=10) 120 | enabled = models.BooleanField(default=True) 121 | 122 | class Meta: 123 | ordering = ('region', 'ip__hostname') 124 | 125 | def save(self, *args, **kwargs): 126 | self.policy.mark_policy_records_dirty() 127 | return super(PolicyMember, self).save(*args, **kwargs) 128 | 129 | def delete(self, *args, **kwargs): 130 | self.policy.mark_policy_records_dirty() 131 | return super(PolicyMember, self).delete(*args, **kwargs) 132 | 133 | def __str__(self): 134 | return '{} {} {}'.format(self.ip, self.region, self.weight) 135 | 136 | 137 | def validate_json(value): 138 | try: 139 | json.loads(value) 140 | except json.JSONDecodeError: 141 | raise ValidationError("Not valid json") 142 | 143 | 144 | class Zone(models.Model): 145 | root = models.CharField(max_length=255, validators=[validate_domain]) 146 | route53_id = models.CharField(max_length=32, unique=True, editable=False, 147 | null=True, default=None) 148 | caller_reference = models.UUIDField(editable=False, null=True) 149 | deleted = models.BooleanField(default=False) 150 | ns_propagated = models.BooleanField(default=False) 151 | cached_ns_records = models.TextField(validators=[validate_json], default=None, null=True) 152 | 153 | class Meta: 154 | ordering = ['root'] 155 | 156 | def __init__(self, *args, **kwargs): 157 | self._route53_instance = None 158 | super(Zone, self).__init__(*args, **kwargs) 159 | 160 | @property 161 | def dirty(self): 162 | dirty = False 163 | for policy_record in self.policy_records.all(): 164 | dirty |= policy_record.dirty 165 | 166 | return dirty 167 | 168 | def clean(self): 169 | # if the root is not a fqdn then add the dot at the end 170 | # this will be called from admin 171 | if not self.root.endswith('.'): 172 | self.root += '.' 173 | super().clean() 174 | 175 | def save(self, *args, **kwargs): 176 | if self.route53_id is not None: 177 | if self.route53_id.startswith('/hostedzone/'): 178 | self.route53_id = self.route53_id[len('/hostedzone/'):] 179 | return super(Zone, self).save(*args, **kwargs) 180 | 181 | def commit(self): 182 | self.r53_zone.commit() 183 | 184 | def delete_record_by_hash(self, record_hash): 185 | records = self.r53_zone.records() 186 | to_delete_record = records[record_hash] 187 | to_delete_record.deleted = True 188 | self.r53_zone.process_records([to_delete_record]) 189 | 190 | def delete_record(self, record): 191 | self.delete_record_by_hash(record.id) 192 | 193 | def get_policy_records(self): 194 | # return a list with Policy records 195 | records = [] 196 | for policy_record in self.policy_records.all(): 197 | records.append(policy_record.serialize()) 198 | 199 | return records 200 | 201 | @property 202 | def r53_zone(self): 203 | if not self._route53_instance: 204 | self._route53_instance = route53.Zone(self) 205 | return self._route53_instance 206 | 207 | def soft_delete(self): 208 | self.deleted = True 209 | self.save(update_fields=['deleted']) 210 | tasks.aws_delete_zone.delay(self.pk) 211 | 212 | @property 213 | def records(self): 214 | records = self.r53_zone.records() 215 | filtered_records = [] 216 | policy_records = self.get_policy_records() 217 | 218 | for record in records.values(): 219 | if record.is_hidden: 220 | continue 221 | if record.is_alias and any(((record.name == pr.name) for pr in policy_records)): 222 | continue 223 | filtered_records.append(record) 224 | 225 | # Add policy records. 226 | for record in policy_records: 227 | filtered_records.append(record) 228 | 229 | return filtered_records 230 | 231 | def update_records(self, records): 232 | self.r53_zone.process_records(records) 233 | 234 | def __str__(self): 235 | return '{} ({})'.format(self.root, self.route53_id) 236 | 237 | @transaction.atomic 238 | def reconcile(self): 239 | self.r53_zone.reconcile() 240 | 241 | @contextlib.contextmanager 242 | @transaction.atomic 243 | def lock_dirty_policy_records(self): 244 | policy_records = self.policy_records.select_for_update() \ 245 | .select_related('policy').filter(dirty=True) 246 | yield policy_records 247 | 248 | def _delete_orphaned_managed_records(self): 249 | """Delete any managed record not belonging to one of the zone's policies""" 250 | policies = set([pr.policy for pr in self.policy_records.select_related('policy')]) 251 | pol_names = ['{}_{}'.format(RECORD_PREFIX, policy.name) for policy in policies] 252 | for record in self.r53_zone.records().values(): 253 | name = record.name 254 | if name.startswith(RECORD_PREFIX): 255 | for pol_name in pol_names: 256 | if name.startswith(pol_name): 257 | break 258 | else: 259 | self.delete_record(record) 260 | 261 | @classmethod 262 | def update_ns_propagated(cls, delay=0): 263 | resolver = ns_check.get_resolver() 264 | # the order matters because we want unpropagated zones to be checked first 265 | # to minimize the delay in tarnsitioning to propagated state 266 | for zone in cls.objects.order_by('ns_propagated').all(): 267 | try: 268 | zone.ns_propagated = ns_check.is_ns_propagated( 269 | zone, resolver=resolver, delay=delay) 270 | except ns_check.CouldNotResolve: 271 | logger.warn('Failed to resolve nameservers for %s', zone.root) 272 | else: 273 | if not zone.ns_propagated: 274 | logger.info('ns_propagated %-5s %s', zone.ns_propagated, zone.root) 275 | zone.save() 276 | 277 | @classmethod 278 | def _dirty_query(cls): 279 | return Q(deleted=True) | Q(route53_id=None) | Q(policy_records__dirty=True) 280 | 281 | @classmethod 282 | def need_reconciliation(cls): 283 | return cls.objects.filter( 284 | cls._dirty_query() 285 | ) 286 | 287 | @classmethod 288 | def get_clean_zones(cls): 289 | return cls.objects.filter( 290 | ~cls._dirty_query() 291 | ) 292 | 293 | 294 | class PolicyRecord(models.Model): 295 | RECORD_TYPES = [ 296 | ('A', 'A'), 297 | ('AAAA', 'AAAA') 298 | ] 299 | id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) 300 | name = models.CharField(max_length=255) 301 | record_type = models.CharField(max_length=10, choices=RECORD_TYPES, default='A') 302 | policy = models.ForeignKey(Policy, related_name='records', on_delete=models.CASCADE) 303 | dirty = models.BooleanField(default=True, editable=False) 304 | zone = models.ForeignKey(Zone, related_name='policy_records', on_delete=models.CASCADE) 305 | deleted = models.BooleanField(default=False) 306 | 307 | class Meta: 308 | unique_together = ('name', 'record_type', 'zone') 309 | 310 | def __init__(self, *a, **kwa): 311 | super().__init__(*a, **kwa) 312 | self._r53_policy_record = None 313 | 314 | def __str__(self): 315 | return '{}.{}'.format(self.name, self.zone.root) 316 | 317 | def serialize(self): 318 | assert self.zone is not None 319 | record = route53.PolicyRecord(policy_record=self, zone=self.zone.r53_zone) 320 | record.dirty = self.dirty 321 | record.managed = False 322 | record.deleted = self.deleted 323 | return record 324 | 325 | def soft_delete(self): 326 | self.deleted = True 327 | self.dirty = True 328 | self.save(update_fields=['deleted', 'dirty']) 329 | 330 | def mark_dirty(self): 331 | self.dirty = True 332 | self.save(update_fields=['dirty']) 333 | 334 | def clean(self): 335 | zone_records = self.zone.r53_zone.records() 336 | # guard against PolicyRecords/CNAME name clashes 337 | if not self.deleted: 338 | # don't do the check unless the PR is deleted 339 | for record in zone_records.values(): 340 | if record.name == self.name and record.type == 'CNAME': 341 | raise ValidationError( 342 | {'name': "A CNAME record of the same name already exists."}) 343 | 344 | super().clean() 345 | 346 | @property 347 | def r53_policy_record(self): 348 | if self._r53_policy_record is None: 349 | self._r53_policy_record = route53.PolicyRecord( 350 | policy_record=self, zone=self.zone.r53_zone) 351 | return self._r53_policy_record 352 | 353 | @transaction.atomic 354 | def apply_record(self): 355 | # build the tree for this policy record. 356 | if self.deleted: 357 | # if the zone is marked as deleted don't try to build the tree. 358 | self.delete_record() 359 | self.delete() 360 | return 361 | 362 | self.zone.r53_zone.process_records([self.r53_policy_record]) 363 | 364 | self.dirty = False # mark as clean 365 | self.save() 366 | 367 | @classmethod 368 | def new_or_deleted(cls, name, record_type, zone): 369 | # if the record hasn't been reconciled yet (still exists in the DB), we want to reuse it 370 | # to avoid violating the unique together constraint on name and zone 371 | # TODO: if we add deleted to that constraint and make it null-able, we can keep the DB 372 | # sane and simplify the system. Reusing the record like this opens up the possibility 373 | # of running into concurrency issues. 374 | try: 375 | model = cls.objects.get(deleted=True, name=name, record_type=record_type, zone=zone) 376 | model.deleted = False 377 | return model 378 | except cls.DoesNotExist: 379 | return cls(name=name, record_type=record_type, zone=zone) 380 | -------------------------------------------------------------------------------- /zinc/ns_check.py: -------------------------------------------------------------------------------- 1 | import json 2 | import time 3 | 4 | from django.conf import settings 5 | 6 | from dns.resolver import Resolver 7 | from dns.exception import DNSException 8 | 9 | 10 | class CouldNotResolve(Exception): 11 | pass 12 | 13 | 14 | def get_resolver(): 15 | resolver = Resolver() 16 | resolver.nameservers = settings.ZINC_NS_CHECK_RESOLVERS 17 | return resolver 18 | 19 | 20 | def is_ns_propagated(zone, resolver=None, delay=0): 21 | if not zone.r53_zone.exists: 22 | return False 23 | if resolver is None: 24 | resolver = get_resolver() 25 | try: 26 | name_servers = sorted([str(ns) for ns in resolver.query(zone.root, 'NS')]) 27 | except DNSException as e: 28 | raise CouldNotResolve(e) 29 | if zone.cached_ns_records: 30 | r53_name_servers = json.loads(zone.cached_ns_records) 31 | if delay: 32 | time.sleep(delay) 33 | if r53_name_servers == name_servers: 34 | return True 35 | # in case the nameservers don't match we update the cached_ns_records and 36 | # compare again 37 | r53_name_servers = sorted(zone.r53_zone.ns.values) 38 | zone.cached_ns_records = json.dumps(r53_name_servers) 39 | zone.save(update_fields=['cached_ns_records']) 40 | return r53_name_servers == name_servers 41 | -------------------------------------------------------------------------------- /zinc/pagination.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2015 Presslabs SRL 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | from rest_framework.settings import api_settings 17 | from rest_framework.pagination import PageNumberPagination 18 | from rest_framework.response import Response 19 | from rest_framework.utils.urls import replace_query_param, remove_query_param 20 | 21 | 22 | class LinkHeaderPagination(PageNumberPagination): 23 | page_size = api_settings.PAGE_SIZE or 30 24 | page_size_query_param = 'page_size' 25 | max_page_size = 100 26 | 27 | def get_last_link(self): 28 | url = self.request.build_absolute_uri() 29 | page_number = self.page.paginator.num_pages 30 | return replace_query_param(url, self.page_query_param, page_number) 31 | 32 | def get_first_link(self, display_page_query_param=True): 33 | url = self.request.build_absolute_uri() 34 | if display_page_query_param: 35 | page_number = self.page.paginator.validate_number(1) 36 | return replace_query_param(url, self.page_query_param, page_number) 37 | else: 38 | return remove_query_param(url, self.page_query_param) 39 | 40 | def get_paginated_response(self, data): 41 | next_url = self.get_next_link() 42 | previous_url = self.get_previous_link() 43 | first_url = self.get_first_link() 44 | last_url = self.get_last_link() 45 | 46 | if next_url is not None and previous_url is not None: 47 | link = '<{next_url}>; rel="next", <{previous_url}>; rel="prev"' 48 | elif next_url is not None: 49 | link = '<{next_url}>; rel="next"' 50 | elif previous_url is not None: 51 | link = '<{previous_url}>; rel="prev"' 52 | else: 53 | link = '' 54 | 55 | if link: 56 | link += ', ' 57 | 58 | link += '<{first_url}>; rel="first", <{last_url}>; rel="last"' 59 | 60 | link = link.format(next_url=next_url, previous_url=previous_url, 61 | first_url=first_url, last_url=last_url) 62 | headers = {'Link': link} if link else {} 63 | 64 | return Response(data, headers=headers) 65 | -------------------------------------------------------------------------------- /zinc/route53/__init__.py: -------------------------------------------------------------------------------- 1 | from boto3.session import Session 2 | 3 | from .record import Record, PolicyRecord, record_factory # noqa: F401 4 | from .policy import Policy # noqa: F401 5 | from .zone import Zone # noqa: F401 6 | from .health_check import HealthCheck # noqa: F401 7 | 8 | 9 | def _get_aws_regions(): 10 | """Retrieve a list of region tuples available in AWS EC2.""" 11 | return Session().get_available_regions('ec2') 12 | 13 | 14 | def get_local_aws_region_choices(): 15 | return ( 16 | ('af-south-1', 'Africa (Cape Town)'), 17 | ('ap-east-1', 'Asia Pacific (Hong Kong)'), 18 | ('ap-northeast-1', 'Asia Pacific (Tokyo)'), 19 | ('ap-northeast-2', 'Asia Pacific (Seoul)'), 20 | ('ap-northeast-3', 'Asia Pacific (Osaka-Local)'), 21 | ('ap-south-1', 'Asia Pacific (Mumbai)'), 22 | ('ap-southeast-1', 'Asia Pacific (Singapore)'), 23 | ('ap-southeast-2', 'Asia Pacific (Sydney)'), 24 | ('ap-southeast-3', 'Asia Pacific (Jakarta)'), 25 | ('ca-central-1', 'Canada (Central)'), 26 | ('cn-north-1', 'China (Beijing)'), 27 | ('cn-northwest-1', 'China (Ningxia)'), 28 | ('eu-central-1', 'Europe (Frankfurt)'), 29 | ('eu-north-1', 'Europe (Stockholm)'), 30 | ('eu-south-1', 'Europe (Milan)'), 31 | ('eu-west-1', 'Europe (Ireland)'), 32 | ('eu-west-2', 'Europe (London)'), 33 | ('eu-west-3', 'Europe (Paris)'), 34 | ('il-central-1', 'Israel (Tel Aviv)'), 35 | ('me-south-1', 'Middle East (Bahrain)'), 36 | ('sa-east-1', 'South America (São Paulo)'), 37 | ('us-east-1', 'US East (N. Virginia)'), 38 | ('us-east-2', 'US East (Ohio)'), 39 | ('us-west-1', 'US West (N. California)'), 40 | ('us-west-2', 'US West (Oregon)'), 41 | ) 42 | 43 | 44 | def get_local_aws_regions(): 45 | return [region[0] for region in get_local_aws_region_choices()] 46 | -------------------------------------------------------------------------------- /zinc/route53/client.py: -------------------------------------------------------------------------------- 1 | import random 2 | 3 | import boto3 4 | import botocore.retryhandler 5 | from django.conf import settings 6 | 7 | 8 | def delay_exponential(base, *a, **kwa): 9 | """ 10 | Override botocore's delay_exponential retry strategy, to ensure min delay is non-zero. 11 | We want to use a random base between 0.2 and 0.8. Final progressions are: 12 | min: [0.2, 0.4, 0.8, 1.6, 3.2] - 6.2 s 13 | max: [0.8, 1.6, 3.2, 6.4, 12.8] - 24.8 s 14 | """ 15 | if base == 'rand': 16 | # 1 / 1.(6) == 0.6 17 | base = 0.2 + random.random() / 1.666666666666666666 18 | return botocore.retryhandler._orig_delay_exponential(base, *a, **kwa) 19 | 20 | 21 | # we monkeypatch the retry handler because the original logic in botocore is to optimistic 22 | # in the case of a random backoff (they pick a base in the 0-1.0 second interval) 23 | botocore.retryhandler._orig_delay_exponential = botocore.retryhandler.delay_exponential 24 | botocore.retryhandler.delay_exponential = delay_exponential 25 | 26 | AWS_KEY = getattr(settings, 'AWS_KEY', '') 27 | AWS_SECRET = getattr(settings, 'AWS_SECRET', '') 28 | 29 | # Pass '-' as if AWS_KEY from settings is empty 30 | # because boto will look into '~/.aws/config' file if 31 | # AWS_KEY or AWS_SECRET are not defined, which is the default 32 | # and can mistaknely use production keys 33 | 34 | _client = boto3.client( 35 | service_name='route53', 36 | aws_access_key_id=AWS_KEY or '-', 37 | aws_secret_access_key=AWS_SECRET or '-', 38 | ) 39 | 40 | 41 | def get_client(): 42 | return _client 43 | -------------------------------------------------------------------------------- /zinc/route53/health_check.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | import logging 3 | 4 | from botocore.exceptions import ClientError 5 | from django.conf import settings 6 | 7 | from .client import get_client 8 | 9 | logger = logging.getLogger('zinc.route53') 10 | 11 | 12 | def generate_caller_ref(): 13 | return 'zinc {}'.format(uuid.uuid4()) 14 | 15 | 16 | class HealthCheck: 17 | def __init__(self, ip): 18 | self.ip = ip 19 | self._aws_data = None 20 | self._client = get_client() 21 | 22 | @property 23 | def exists(self): 24 | self._load() 25 | return self._aws_data is not None 26 | 27 | @property 28 | def id(self): 29 | self._load() 30 | return self._aws_data.get('Id') 31 | 32 | def _load(self): 33 | if self._aws_data is not None: 34 | return 35 | if self.ip.healthcheck_id is not None: 36 | try: 37 | health_check = self._client.get_health_check(HealthCheckId=self.ip.healthcheck_id) 38 | self._aws_data = health_check.get('HealthCheck') 39 | except self._client.exceptions.NoSuchHealthCheck: 40 | pass 41 | 42 | @property 43 | def desired_config(self): 44 | config = { 45 | 'IPAddress': self.ip.ip, 46 | } 47 | config.update(settings.HEALTH_CHECK_CONFIG) 48 | return config 49 | 50 | @property 51 | def config(self): 52 | self._load() 53 | return self._aws_data.get('HealthCheckConfig') 54 | 55 | def create(self): 56 | if self.ip.healthcheck_caller_reference is None: 57 | self.ip.healthcheck_caller_reference = uuid.uuid4() 58 | logger.info("%-15s new caller_reference %s", 59 | self.ip.ip, self.ip.healthcheck_caller_reference) 60 | self.ip.save() 61 | resp = self._client.create_health_check( 62 | CallerReference=str(self.ip.healthcheck_caller_reference), 63 | HealthCheckConfig=self.desired_config 64 | ) 65 | self.ip.healthcheck_id = resp['HealthCheck']['Id'] 66 | logger.info("%-15s created hc: %s", self.ip.ip, self.ip.healthcheck_id) 67 | self.ip.save() 68 | 69 | def delete(self): 70 | if self.exists: 71 | logger.info("%-15s delete hc: %s", self.ip.ip, self.ip.healthcheck_id) 72 | self._client.delete_health_check(HealthCheckId=self.id) 73 | self.ip.healthcheck_caller_reference = None 74 | self.ip.save(update_fields=['healthcheck_caller_reference']) 75 | 76 | def reconcile(self): 77 | if self.ip.deleted: 78 | self.delete() 79 | self.ip.delete() 80 | elif self.exists: 81 | # if the desired config is not a subset of the current config 82 | if not self.desired_config.items() <= self.config.items(): 83 | self.delete() 84 | self.create() 85 | else: 86 | logger.info("%-15s nothing to do", self.ip.ip) 87 | else: 88 | try: 89 | self.create() 90 | except self._client.exceptions.HealthCheckAlreadyExists: 91 | self.ip.healthcheck_caller_reference = None 92 | self.ip.save() 93 | self.create() 94 | 95 | @classmethod 96 | def reconcile_for_ips(cls, ips): 97 | checks = [cls(ip) for ip in ips] 98 | for check in checks: 99 | try: 100 | check.reconcile() 101 | except ClientError: 102 | logger.exception("Error while handling %s", check.ip.friendly_name) 103 | -------------------------------------------------------------------------------- /zinc/route53/policy.py: -------------------------------------------------------------------------------- 1 | import copy 2 | from collections import OrderedDict 3 | 4 | import zinc.route53 5 | from zinc.utils import memoized_property 6 | from .record import Record, RECORD_PREFIX 7 | 8 | 9 | class Policy: 10 | def __init__(self, zone, policy): 11 | assert isinstance(zone, zinc.route53.Zone) 12 | self.zone = zone 13 | self.db_policy = policy 14 | 15 | @property 16 | def name(self): 17 | return self.db_policy.name 18 | 19 | @property 20 | def id(self): 21 | return self.db_policy.id 22 | 23 | @property 24 | def routing(self): 25 | return self.db_policy.routing 26 | 27 | @memoized_property 28 | def aws_records(self): 29 | """What we have in AWS""" 30 | return dict([ 31 | (r_id, record) for (r_id, record) in self.zone.records().items() 32 | if record.is_member_of(self) 33 | ]) 34 | 35 | @memoized_property 36 | def desired_records(self): 37 | """The records we should have (the desired state of the world)""" 38 | return OrderedDict([(record.id, record) for record in self._build_tree()]) 39 | 40 | def _build_weighted_tree(self, policy_members, region_suffixed=True): 41 | # Build simple tree 42 | records = [] 43 | for policy_member in policy_members: 44 | record_type = 'A' 45 | if ':' in policy_member.ip.ip: 46 | record_type = 'AAAA' 47 | 48 | health_check_kwa = {} 49 | if policy_member.ip.healthcheck_id: 50 | health_check_kwa['health_check_id'] = str(policy_member.ip.healthcheck_id) 51 | record = Record( 52 | ttl=self.db_policy.ttl, 53 | type=record_type, 54 | values=[policy_member.ip.ip], 55 | set_identifier='{}-{}'.format(str(policy_member.id), policy_member.region), 56 | weight=policy_member.weight, 57 | zone=self.zone, 58 | **health_check_kwa, 59 | ) 60 | # TODO: maybe we should have a specialized subclass for PolicyRecords 61 | # and this logic should be moved there 62 | if region_suffixed: 63 | record.name = '{}_{}_{}'.format(RECORD_PREFIX, self.name, policy_member.region) 64 | else: 65 | record.name = '{}_{}'.format(RECORD_PREFIX, self.name) 66 | records.append(record) 67 | 68 | return records 69 | 70 | def _build_lbr_tree(self, policy_members, regions): 71 | # Build latency based routed tree 72 | records = self._build_weighted_tree(policy_members) 73 | for region in regions: 74 | record = Record( 75 | name='{}_{}'.format(RECORD_PREFIX, self.name), 76 | type='A', 77 | alias_target={ 78 | 'HostedZoneId': self.zone.id, 79 | 'DNSName': '{}_{}_{}.{}'.format( 80 | RECORD_PREFIX, self.name, region, self.zone.root), 81 | 'EvaluateTargetHealth': True # len(regions) > 1 82 | }, 83 | region=region, 84 | set_identifier=region, 85 | zone=self.zone, 86 | ) 87 | if self._has_ipv4_records_in_region(policy_members, region): 88 | records.append(record) 89 | 90 | # create a similar AAAA record if there exists IPv6 ips in this region. 91 | if self._has_ipv6_records_in_region(policy_members, region): 92 | record = copy.copy(record) 93 | record.type = 'AAAA' 94 | records.append(record) 95 | 96 | return records 97 | 98 | def _build_tree(self): 99 | policy_members = self.db_policy.members.exclude(enabled=False).exclude(ip__enabled=False) 100 | # ensure we always build region subtrees in alphabetical order; makes tests simpler 101 | regions = sorted(set([pm.region for pm in policy_members])) 102 | if len(regions) == 0: 103 | raise Exception( 104 | "Policy can't be applied for zone '{}'; " 105 | "There is no member in the '{}' policy.".format( 106 | self.zone, self 107 | ) 108 | ) 109 | if self.routing == 'latency': 110 | # Here is the case where are multiple regions 111 | records = self._build_lbr_tree(policy_members, regions=regions) 112 | # elif len(regions) == 1: 113 | elif self.routing == 'weighted': 114 | # Case with a single region 115 | records = self._build_weighted_tree( 116 | policy_members, region_suffixed=False) 117 | else: 118 | raise AssertionError('invalid routing {} for policy {}'.format( 119 | self.routing, self.db_policy)) 120 | return records 121 | 122 | def reconcile(self): 123 | aws_record_ids = self.aws_records.keys() 124 | desired_record_ids = self.desired_records.keys() 125 | to_delete = [] 126 | for obsolete_rec_id in aws_record_ids - desired_record_ids: 127 | record = self.aws_records[obsolete_rec_id] 128 | record.deleted = True 129 | to_delete.append(record) 130 | self.zone.process_records(to_delete) 131 | to_upsert = [] 132 | for rec_id, desired_record in self.desired_records.items(): 133 | existing_record = self.aws_records.get(rec_id) 134 | if existing_record is None: 135 | to_upsert.append(desired_record) 136 | else: 137 | # if desired is a subset of existing 138 | if not desired_record.to_aws().items() <= existing_record.to_aws().items(): 139 | to_upsert.append(desired_record) 140 | self.zone.process_records(to_upsert) 141 | 142 | def remove(self): 143 | records = list(self.aws_records.values()) 144 | for record in records: 145 | record.deleted = True 146 | self.zone.process_records(records) 147 | 148 | def _has_ipv6_records_in_region(self, policy_members, region): 149 | has_ipv6 = False 150 | for pm in policy_members: 151 | if region and pm.region != region: 152 | continue 153 | 154 | if ':' in pm.ip.ip: 155 | has_ipv6 = True 156 | 157 | return has_ipv6 158 | 159 | def _has_ipv4_records_in_region(self, policy_members, region): 160 | has_ipv4 = False 161 | for pm in policy_members: 162 | if region and pm.region != region: 163 | continue 164 | 165 | if '.' in pm.ip.ip: 166 | has_ipv4 = True 167 | 168 | return has_ipv4 169 | -------------------------------------------------------------------------------- /zinc/route53/record.py: -------------------------------------------------------------------------------- 1 | import json 2 | import hashlib 3 | 4 | from hashids import Hashids 5 | from django.conf import settings 6 | from django.core.exceptions import SuspiciousOperation, ValidationError 7 | 8 | from zinc import models, route53 9 | from zinc.utils import memoized_property 10 | from zinc.utils.generators import chunks 11 | 12 | 13 | HASHIDS_SALT = getattr(settings, 'SECRET_KEY', '') 14 | HASHIDS_MIN_LENGTH = getattr(settings, 'HASHIDS_MIN_LENGTH', 7) 15 | HASHIDS_ALPHABET = getattr(settings, 'HASHIDS_ALPHABET', 16 | 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXY1234567890') 17 | hashids = Hashids(salt=HASHIDS_SALT, 18 | alphabet=HASHIDS_ALPHABET) 19 | 20 | RECORD_PREFIX = '_zn' 21 | 22 | POLICY_ROUTED = 'POLICY_ROUTED' 23 | POLICY_ROUTED_IPv6 = 'POLICY_ROUTED_IPv6' 24 | 25 | POLICY_ROUTED_TO_DNS = { 26 | POLICY_ROUTED: 'A', 27 | POLICY_ROUTED_IPv6: 'AAAA', 28 | } 29 | 30 | ZINC_CUSTOM_RECORD_TYPES = [ 31 | POLICY_ROUTED, POLICY_ROUTED_IPv6 32 | ] 33 | 34 | RECORD_TYPES = [ 35 | 'A', 'AAAA', 'CNAME', 'MX', 'TXT', 'SOA', 36 | 'SPF', 'SRV', 'NS', 'CAA', 37 | ] + ZINC_CUSTOM_RECORD_TYPES 38 | 39 | ALLOWED_RECORD_TYPES = set(RECORD_TYPES) 40 | ALLOWED_RECORD_TYPES.remove('SOA') 41 | 42 | ZINC_RECORD_TYPES = [(rtype, rtype) for rtype in RECORD_TYPES] 43 | 44 | ZINC_RECORD_TYPES_MAP = {i: RECORD_TYPES[i] for i in range(0, len(RECORD_TYPES))} 45 | ZINC_RECORD_TYPES_MAP_REV = {rtype: i for i, rtype in ZINC_RECORD_TYPES_MAP.items()} 46 | 47 | 48 | def get_record_type(rtype): 49 | if type(rtype) is int: 50 | return ZINC_RECORD_TYPES_MAP[rtype] 51 | else: 52 | return ZINC_RECORD_TYPES_MAP_REV[rtype] 53 | 54 | 55 | def _encode(*args): 56 | _set_id = ':'.join([str(arg) for arg in args]) 57 | _set_id = int(hashlib.sha256(_set_id.encode('utf-8')).hexdigest()[:16], base=16) 58 | return hashids.encode(_set_id) 59 | 60 | 61 | class BaseRecord: 62 | _obj_to_r53 = dict([ 63 | ('name', 'Name'), 64 | ('type', 'Type'), 65 | ('managed', 'Managed'), 66 | ('ttl', 'ttl'), 67 | ('alias_target', 'AliasTarget'), 68 | ('values', 'Values'), 69 | ('weight', 'Weight'), 70 | ('region', 'Region'), 71 | ('set_identifier', 'SetIdentifier'), 72 | ('health_check_id', 'HealthCheckId'), 73 | ('traffic_policy_instance_id', 'TrafficPolicyInstanceId'), 74 | ]) 75 | _r53_to_obj = {v: k for k, v in _obj_to_r53.items()} 76 | 77 | def __init__(self, name=None, alias_target=None, created=False, deleted=False, dirty=False, 78 | health_check_id=None, managed=False, region=None, set_identifier=None, 79 | traffic_policy_instance_id=None, ttl=None, values=None, weight=None, 80 | zone=None): 81 | self.name = name 82 | self.alias_target = alias_target 83 | self.created = created 84 | assert alias_target is None or ttl is None 85 | self.ttl = ttl 86 | self._values = values 87 | self.weight = weight 88 | self.region = region 89 | self.set_identifier = set_identifier 90 | self.health_check_id = health_check_id 91 | self.traffic_policy_instance_id = traffic_policy_instance_id 92 | self.zone = zone 93 | self.zone_id = zone.id 94 | self.zone_root = zone.root 95 | assert self.zone_id is not None 96 | assert self.zone_root is not None 97 | self.deleted = deleted 98 | self.dirty = dirty 99 | self.managed = managed 100 | 101 | def __repr__(self): 102 | return "<{} id={} {}:{} {}>".format( 103 | type(self).__name__, self.id, self.type, self.name, self.values) 104 | 105 | @property 106 | def values(self): 107 | if self.is_alias: 108 | if 'DNSName' in self.alias_target: 109 | return ['ALIAS {}'.format(self.alias_target['DNSName'])] 110 | else: 111 | return self._values 112 | 113 | @values.setter 114 | def values(self, value): 115 | assert not self.is_alias 116 | self._values = value 117 | 118 | @staticmethod 119 | def _strip_root(name, root): 120 | return '@' if name == root else name.replace('.' + root, '') 121 | 122 | @staticmethod 123 | def _add_root(name, root): 124 | return root if name == '@' else '{}.{}'.format(name, root) 125 | 126 | @classmethod 127 | def unpack_txt_value(cls, value): 128 | if value.startswith('"') and value.endswith('"'): 129 | value = value[1:-1] 130 | 131 | return ''.join(json.loads('"%s"' % chunk) for chunk in value.split('" "')) 132 | 133 | @classmethod 134 | def from_aws_record(cls, record, zone): 135 | # Determine if a R53 DNS record is of type ALIAS 136 | def is_alias_record(record): 137 | return 'AliasTarget' in record.keys() 138 | 139 | # Determine if a record is the NS or SOA record of the root domain 140 | def root_ns_soa(record, root): 141 | return record['Name'] == root and record['Type'] in ['NS', 'SOA'] 142 | 143 | kwargs = {} 144 | for attr_name in ['weight', 'region', 'set_identifier', 'health_check_id', 145 | 'traffic_policy_instance_id']: 146 | kwargs[attr_name] = record.get(cls._obj_to_r53[attr_name], None) 147 | 148 | new = cls(zone=zone, **kwargs) 149 | new.name = cls._strip_root(record['Name'], zone.root) 150 | new.type = record['Type'] 151 | new.managed = ((record.get('SetIdentifier', False)) or 152 | root_ns_soa(record, zone.root) or (is_alias_record(record))) 153 | 154 | new.ttl = record.get('TTL') 155 | if is_alias_record(record): 156 | new.alias_target = { 157 | 'DNSName': record['AliasTarget']['DNSName'], 158 | 'EvaluateTargetHealth': record['AliasTarget']['EvaluateTargetHealth'], 159 | 'HostedZoneId': record['AliasTarget']['HostedZoneId'] 160 | } 161 | elif record['Type'] == 'TXT': 162 | # Decode json escaped strings 163 | new.values = [cls.unpack_txt_value(value['Value']) 164 | for value in record.get('ResourceRecords', [])] 165 | else: 166 | new.values = [value['Value'] for value in 167 | record.get('ResourceRecords', [])] 168 | return new 169 | 170 | @property 171 | def id(self): 172 | zone_hash = _encode(self.zone_id) 173 | record_hash = _encode(self.name, self.type, self.set_identifier) 174 | return 'Z{zone}Z{type}Z{id}'.format( 175 | zone=zone_hash, type=get_record_type(self.type), id=record_hash) 176 | 177 | @classmethod 178 | def pack_txt_value(cls, value): 179 | max_length = 255 180 | 181 | if len(value) < max_length: 182 | value = json.dumps(value) 183 | else: 184 | value = ' '.join('{}'.format(json.dumps(element)) 185 | for element in chunks(value, max_length)) 186 | 187 | return {'Value': value} 188 | 189 | def to_aws(self): 190 | encoded_record = { 191 | 'Name': self._add_root(self.name, self.zone_root), 192 | 'Type': self.type, 193 | } 194 | if not self.is_alias: 195 | if self.type == 'TXT': 196 | # Encode json escape. 197 | encoded_record['ResourceRecords'] = [self.pack_txt_value(value) 198 | for value in self.values] 199 | else: 200 | encoded_record['ResourceRecords'] = [{'Value': value} for value in self.values] 201 | else: 202 | encoded_record['AliasTarget'] = { 203 | 'DNSName': self.alias_target['DNSName'], 204 | 'EvaluateTargetHealth': self.alias_target['EvaluateTargetHealth'], 205 | 'HostedZoneId': self.alias_target['HostedZoneId'], 206 | } 207 | if self.ttl is not None: 208 | encoded_record['TTL'] = self.ttl 209 | 210 | for attr_name in ['Weight', 'Region', 'SetIdentifier', 211 | 'HealthCheckId', 'TrafficPolicyInstanceId']: 212 | value = getattr(self, self._r53_to_obj[attr_name]) 213 | if value is not None: 214 | encoded_record[attr_name] = value 215 | 216 | return encoded_record 217 | 218 | @property 219 | def is_alias(self): 220 | return self.alias_target is not None 221 | 222 | @property 223 | def is_hidden(self): 224 | return self.name.startswith(RECORD_PREFIX) 225 | 226 | def is_member_of(self, policy): 227 | return self.name.startswith('{}_{}'.format(RECORD_PREFIX, policy.name)) 228 | 229 | def save(self): 230 | self.zone.process_records([self]) 231 | 232 | def is_subset(self, other): 233 | return self.to_aws().items() <= other.to_aws().items() 234 | 235 | def validate_unique(self): 236 | """You're not allowed to have a CNAME clash with any other type of record""" 237 | if self.deleted: 238 | # allow deleting any conflicting record 239 | return 240 | if self.type == 'CNAME': 241 | clashing = tuple((self.name, r_type) for r_type in RECORD_TYPES) 242 | else: 243 | clashing = ((self.name, 'CNAME'), ) 244 | for record in self.zone.db_zone.records: 245 | for other in clashing: 246 | if (record.name, record.type) == other and record.id != self.id: 247 | raise ValidationError( 248 | {'name': "A {} record of the same name already exists.".format(other[1])}) 249 | 250 | def clean(self): 251 | pass 252 | 253 | def clean_fields(self): 254 | pass 255 | 256 | def full_clean(self): 257 | self.clean_fields() 258 | self.clean() 259 | self.validate_unique() 260 | 261 | 262 | class Record(BaseRecord): 263 | def __init__(self, type=None, **kwa): 264 | super().__init__(**kwa) 265 | self.type = type 266 | 267 | 268 | class PolicyRecord(BaseRecord): 269 | def __init__(self, zone, policy_record=None, policy=None, dirty=None, 270 | deleted=None, created=None): 271 | if policy is None: 272 | policy = policy_record.policy 273 | if dirty is None: 274 | dirty = policy_record.dirty 275 | if deleted is None: 276 | deleted = policy_record.deleted 277 | 278 | self.db_policy_record = policy_record 279 | self._policy = None 280 | self.policy = policy 281 | self.zone = zone 282 | self.record_type = self.db_policy_record.record_type 283 | 284 | super().__init__( 285 | name=self.db_policy_record.name, 286 | zone=zone, 287 | alias_target={ 288 | 'HostedZoneId': zone.id, 289 | 'DNSName': '{}_{}.{}'.format(RECORD_PREFIX, self.policy.name, zone.root), 290 | 'EvaluateTargetHealth': False 291 | }, 292 | deleted=deleted, 293 | dirty=dirty, 294 | created=created, 295 | ) 296 | 297 | def save(self): 298 | if self.deleted: 299 | # The record will be deleted 300 | self.db_policy_record.deleted = True 301 | self.db_policy_record.dirty = True 302 | else: 303 | # Update policy for this record. 304 | self.db_policy_record.policy_id = self.policy.id 305 | self.db_policy_record.deleted = False # clear deleted flag 306 | self.db_policy_record.dirty = True 307 | self.db_policy_record.full_clean() 308 | self.db_policy_record.save() 309 | 310 | def reconcile(self): 311 | # upsert or delete the top level alias 312 | if self.deleted: 313 | if self._top_level_record.id in self.zone.records(): 314 | self.zone.process_records([self]) 315 | self.db_policy_record.delete() 316 | else: 317 | existing_alias = self._existing_alias 318 | if (existing_alias is None or not self._top_level_record.is_subset(existing_alias)): 319 | self.zone.process_records([self]) 320 | self.db_policy_record.dirty = False # mark as clean 321 | self.db_policy_record.save() 322 | 323 | @memoized_property 324 | def _top_level_record(self): 325 | return Record( 326 | name=self.name, 327 | type=self.record_type, 328 | alias_target={ 329 | 'HostedZoneId': self.zone.id, 330 | 'DNSName': '{}_{}.{}'.format(RECORD_PREFIX, self.policy.name, self.zone.root), 331 | 'EvaluateTargetHealth': False 332 | }, 333 | zone=self.zone, 334 | ) 335 | 336 | @memoized_property 337 | def _existing_alias(self): 338 | return self.zone.records().get(self.id) 339 | 340 | def to_aws(self): 341 | return self._top_level_record.to_aws() 342 | 343 | @property 344 | def id(self): 345 | return self._top_level_record.id 346 | 347 | @property 348 | def values(self): 349 | return [str(self.policy.id)] 350 | 351 | @values.setter 352 | def values(self, values): 353 | (pol_id, ) = values 354 | policy = route53.Policy(policy=models.Policy.objects.get(id=pol_id), zone=self.zone) 355 | self.policy = policy 356 | 357 | @property 358 | def type(self): 359 | if self.record_type == 'AAAA': 360 | return POLICY_ROUTED_IPv6 361 | 362 | return POLICY_ROUTED 363 | 364 | @property 365 | def policy(self): 366 | return self._policy 367 | 368 | @policy.setter 369 | def policy(self, value): 370 | if value is None: 371 | self.db_policy_record.policy = None 372 | else: 373 | self.db_policy_record.policy_id = value.id 374 | self._policy = value 375 | 376 | 377 | def record_factory(zone, created=None, **validated_data): 378 | record_type = validated_data.pop('type') 379 | if record_type in ZINC_CUSTOM_RECORD_TYPES: 380 | assert len(validated_data['values']) == 1 381 | policy_id = validated_data['values'][0] 382 | try: 383 | policy = models.Policy.objects.get(id=policy_id) 384 | except models.Policy.DoesNotExist: 385 | raise SuspiciousOperation("Policy {} does not exists.".format( 386 | policy_id)) 387 | rtype = POLICY_ROUTED_TO_DNS[record_type] 388 | record_model = models.PolicyRecord.new_or_deleted( 389 | name=validated_data['name'], record_type=rtype, zone=zone 390 | ) 391 | obj = PolicyRecord( 392 | policy_record=record_model, 393 | zone=zone.r53_zone, 394 | policy=policy, 395 | dirty=True, 396 | created=created, 397 | ) 398 | else: 399 | obj = Record(zone=zone.r53_zone, type=record_type, created=created, **validated_data) 400 | return obj 401 | -------------------------------------------------------------------------------- /zinc/route53/zone.py: -------------------------------------------------------------------------------- 1 | from collections import OrderedDict 2 | import uuid 3 | import logging 4 | 5 | from botocore.exceptions import ClientError 6 | from django.db import transaction 7 | from django.conf import settings 8 | 9 | from .record import Record 10 | from .policy import Policy 11 | from .client import get_client 12 | 13 | 14 | logger = logging.getLogger(__name__) 15 | 16 | 17 | class Zone(object): 18 | 19 | def __init__(self, db_zone): 20 | self.db_zone = db_zone 21 | self._aws_records = None 22 | self._exists = None 23 | self._change_batch = [] 24 | self._client = get_client() 25 | 26 | def __repr__(self): 27 | return "".format(self, id(self)) 28 | 29 | def __str__(self): 30 | return "{}:{}".format(self.id, self.root) 31 | 32 | @property 33 | def id(self): 34 | return self.db_zone.route53_id 35 | 36 | @property 37 | def root(self): 38 | return self.db_zone.root 39 | 40 | def process_records(self, records): 41 | for record in records: 42 | self._add_record_changes(record) 43 | 44 | def _add_record_changes(self, record): 45 | if record.deleted: 46 | action = 'DELETE' 47 | else: 48 | if record.created is True: 49 | action = 'CREATE' 50 | else: 51 | action = 'UPSERT' 52 | 53 | self._change_batch.append({ 54 | 'Action': action, 55 | 'ResourceRecordSet': record.to_aws() 56 | }) 57 | 58 | def _reset_change_batch(self): 59 | self._change_batch = [] 60 | 61 | def commit(self, preserve_cache=False): 62 | if not preserve_cache: 63 | self._clear_cache() 64 | if not self._change_batch: 65 | return 66 | 67 | try: 68 | self._client.change_resource_record_sets( 69 | HostedZoneId=self.id, 70 | ChangeBatch={'Changes': self._change_batch} 71 | ) 72 | except self._client.exceptions.InvalidChangeBatch: 73 | logger.warning("failed to process batch %r", self._change_batch) 74 | raise 75 | self._reset_change_batch() 76 | 77 | def records(self): 78 | self._cache_aws_records() 79 | entries = OrderedDict() 80 | for aws_record in self._aws_records or []: 81 | record = Record.from_aws_record(aws_record, zone=self) 82 | if record: 83 | entries[record.id] = record 84 | return entries 85 | 86 | @property 87 | def exists(self): 88 | self._cache_aws_records() 89 | return self._exists 90 | 91 | @property 92 | def ns(self): 93 | if not self.exists: 94 | return None 95 | ns = [record for record in self.records().values() 96 | if record.type == 'NS' and record.name == '@'] 97 | assert len(ns) == 1 98 | return ns[0] 99 | 100 | def _cache_aws_records(self): 101 | if self._aws_records is not None: 102 | return 103 | if not self.id: 104 | return 105 | paginator = self._client.get_paginator('list_resource_record_sets') 106 | records = [] 107 | try: 108 | for page in paginator.paginate(HostedZoneId=self.id): 109 | records.extend(page['ResourceRecordSets']) 110 | except self._client.exceptions.NoSuchHostedZone: 111 | self._clear_cache() 112 | else: 113 | self._aws_records = records 114 | self._exists = True 115 | 116 | def _clear_cache(self): 117 | self._aws_records = None 118 | self._exists = None 119 | 120 | def delete_from_r53(self): 121 | self._delete_records() 122 | self._client.delete_hosted_zone(Id=self.id) 123 | 124 | def delete(self): 125 | if self.exists: 126 | self.delete_from_r53() 127 | self.db_zone.delete() 128 | 129 | def _delete_records(self): 130 | self._cache_aws_records() 131 | zone_root = self.root 132 | 133 | to_delete = [] 134 | for record in self._aws_records: 135 | if record['Type'] in ['NS', 'SOA'] and record['Name'] == zone_root: 136 | continue 137 | 138 | to_delete.append({ 139 | 'Action': 'DELETE', 140 | 'ResourceRecordSet': record 141 | }) 142 | 143 | if to_delete: 144 | self._client.change_resource_record_sets( 145 | HostedZoneId=self.id, 146 | ChangeBatch={ 147 | 'Changes': to_delete 148 | }) 149 | 150 | def create(self): 151 | if self.db_zone.caller_reference is None: 152 | self.db_zone.caller_reference = uuid.uuid4() 153 | self.db_zone.save() 154 | zone = self._client.create_hosted_zone( 155 | Name=self.root, 156 | CallerReference=str(self.db_zone.caller_reference), 157 | HostedZoneConfig={ 158 | 'Comment': getattr(settings, 'ZONE_OWNERSHIP_COMMENT', 'zinc') 159 | } 160 | ) 161 | self.db_zone.route53_id = zone['HostedZone']['Id'] 162 | self.db_zone.save() 163 | 164 | def _reconcile_zone(self): 165 | """ 166 | Handles zone creation/deletion. 167 | """ 168 | if self.db_zone.deleted: 169 | self.delete() 170 | elif self.db_zone.route53_id is None: 171 | self.create() 172 | elif not self.exists: 173 | try: 174 | self.create() 175 | except self._client.exceptions.HostedZoneAlreadyExists: 176 | # This can happen if a zone was manually deleted from AWS. 177 | # Create will fail because we re-use the caller_reference 178 | self.db_zone.caller_reference = None 179 | self.db_zone.save() 180 | self.create() 181 | 182 | def check_policy_trees(self): 183 | clean_policy_records = self.db_zone.policy_records.filter(dirty=False) 184 | clean_policies = set([policy_record.policy for policy_record in clean_policy_records]) 185 | assert self._change_batch == [] 186 | for policy in clean_policies: 187 | r53_policy = Policy(policy=policy, zone=self) 188 | r53_policy.reconcile() 189 | if self._change_batch: 190 | logger.error("Glitch in the matrix for %s %s", self.root, policy.name) 191 | self._change_batch = [] 192 | 193 | def _reconcile_policy_records(self): 194 | """ 195 | Reconcile policy records for this zone. 196 | """ 197 | with self.db_zone.lock_dirty_policy_records() as dirty_policy_records: 198 | dirty_policies = set() 199 | for policy_record in dirty_policy_records: 200 | if not policy_record.deleted: 201 | dirty_policies.add(policy_record.policy) 202 | for policy in dirty_policies: 203 | r53_policy = Policy(policy=policy, zone=self) 204 | r53_policy.reconcile() 205 | self.commit(preserve_cache=True) 206 | for policy_record in dirty_policy_records: 207 | try: 208 | with transaction.atomic(): 209 | policy_record.r53_policy_record.reconcile() 210 | self.commit(preserve_cache=True) 211 | except ClientError: 212 | logger.exception("failed to reconcile record %r", policy_record) 213 | self._reset_change_batch() 214 | self._delete_orphaned_managed_records() 215 | self.commit() 216 | 217 | def _delete_orphaned_managed_records(self): 218 | """Delete any managed record not belonging to one of the zone's policies""" 219 | active_policy_records = self.db_zone.policy_records.select_related('policy') \ 220 | .exclude(deleted=True) 221 | policies = set([pr.policy for pr in active_policy_records]) 222 | for record in self.records().values(): 223 | if record.is_hidden: 224 | for policy in policies: 225 | if record.is_member_of(policy): 226 | break 227 | else: 228 | record.deleted = True 229 | self.process_records([record]) 230 | 231 | def reconcile(self): 232 | self._reconcile_zone() 233 | self._reconcile_policy_records() 234 | 235 | @classmethod 236 | def reconcile_multiple(cls, zones): 237 | for db_zone in zones: 238 | zone = cls(db_zone) 239 | try: 240 | zone.reconcile() 241 | except ClientError: 242 | logger.exception("Error while handling %s", db_zone.name) 243 | -------------------------------------------------------------------------------- /zinc/serializers/__init__.py: -------------------------------------------------------------------------------- 1 | from zinc.serializers.policy import PolicySerializer, PolicyMemberSerializer 2 | from zinc.serializers.record import RecordListSerializer, RecordSerializer 3 | from zinc.serializers.zone import ZoneListSerializer, ZoneDetailSerializer 4 | -------------------------------------------------------------------------------- /zinc/serializers/policy.py: -------------------------------------------------------------------------------- 1 | from rest_framework import serializers 2 | from zinc.models import Policy, PolicyMember 3 | 4 | 5 | class PolicyMemberSerializer(serializers.ModelSerializer): 6 | id = serializers.CharField(read_only=True) 7 | enabled = serializers.SerializerMethodField(read_only=True) 8 | 9 | def get_enabled(self, obj): 10 | return obj.enabled and obj.ip.enabled 11 | 12 | class Meta: 13 | model = PolicyMember 14 | fields = ['id', 'region', 'ip', 'weight', 'enabled'] 15 | 16 | 17 | class PolicySerializer(serializers.HyperlinkedModelSerializer): 18 | id = serializers.CharField(read_only=True) 19 | members = PolicyMemberSerializer(many=True) 20 | 21 | class Meta: 22 | model = Policy 23 | fields = ['id', 'name', 'members', 'url'] 24 | -------------------------------------------------------------------------------- /zinc/serializers/record.py: -------------------------------------------------------------------------------- 1 | import json 2 | from contextlib import contextmanager 3 | 4 | from botocore.exceptions import ClientError 5 | from rest_framework import fields 6 | from rest_framework import serializers 7 | from rest_framework.exceptions import ValidationError 8 | 9 | from django.core.exceptions import ValidationError as DjangoValidationError 10 | from django.conf import settings 11 | 12 | from zinc import route53 13 | from zinc.models import RECORD_PREFIX 14 | from zinc.route53.record import ZINC_RECORD_TYPES, ALLOWED_RECORD_TYPES, ZINC_CUSTOM_RECORD_TYPES 15 | 16 | 17 | @contextmanager 18 | def interpret_client_error(): 19 | try: 20 | yield 21 | except ClientError as error: 22 | if 'ARRDATAIllegalIPv4Address' in error.response['Error']['Message']: 23 | raise ValidationError({'values': ["Value is not a valid IPv4 address."]}) 24 | elif 'AAAARRDATAIllegalIPv6Address' in error.response['Error']['Message']: 25 | raise ValidationError({'values': ["Value is not a valid IPv6 address."]}) 26 | error = error.response['Error']['Message'] 27 | try: 28 | error = json.loads(error) 29 | except TypeError: 30 | pass 31 | except json.JSONDecodeError: 32 | # boto returns a badly formatted error 33 | if error[0] == "[" and error[1] != "\"": 34 | error = error[1:-1] 35 | if not isinstance(error, list): 36 | error = [error] 37 | 38 | raise ValidationError({'non_field_error': error}) 39 | except DjangoValidationError as error: 40 | raise ValidationError(error.message_dict) 41 | 42 | 43 | class RecordListSerializer(serializers.ListSerializer): 44 | # This is used for list the records in Zone serializer 45 | # by using many=True and passing the entier zone as object 46 | 47 | def to_representation(self, zone): 48 | # pass to RecordSerializer zone in the context. 49 | self.context['zone'] = zone 50 | 51 | return super(RecordListSerializer, self).to_representation(zone.records) 52 | 53 | def update(self, instance, validated_data): 54 | raise NotImplementedError('Can not update records this way. Use records/ endpoint.') 55 | 56 | 57 | class RecordSerializer(serializers.Serializer): 58 | name = fields.CharField(max_length=255) 59 | fqdn = fields.SerializerMethodField(required=False) 60 | type = fields.ChoiceField(choices=ZINC_RECORD_TYPES) 61 | values = fields.ListField(child=fields.CharField()) 62 | ttl = fields.IntegerField(allow_null=True, min_value=1, required=False) 63 | dirty = fields.SerializerMethodField(required=False) 64 | id = fields.SerializerMethodField(required=False) 65 | url = fields.SerializerMethodField(required=False) 66 | managed = fields.SerializerMethodField(required=False) 67 | 68 | class Meta: 69 | list_serializer_class = RecordListSerializer 70 | 71 | def get_fqdn(self, obj): 72 | zone = self.context['zone'] 73 | if obj.name == '@': 74 | return zone.root 75 | return '{}.{}'.format(obj.name, zone.root) 76 | 77 | def get_id(self, obj): 78 | return obj.id 79 | 80 | def get_url(self, obj): 81 | # compute the url for record 82 | zone = self.context['zone'] 83 | request = self.context['request'] 84 | record_id = self.get_id(obj) 85 | return request.build_absolute_uri('/zones/%s/records/%s' % (zone.id, record_id)) 86 | 87 | def get_managed(self, obj): 88 | return obj.managed 89 | 90 | def get_dirty(self, obj): 91 | return obj.dirty 92 | 93 | def to_representation(self, obj): 94 | assert obj.values if obj.is_alias else True 95 | rv = super().to_representation(obj) 96 | return rv 97 | 98 | def create(self, validated_data): 99 | zone = self.context['zone'] 100 | obj = route53.record_factory(zone=zone, created=True, **validated_data) 101 | with interpret_client_error(): 102 | obj.full_clean() 103 | obj.save() 104 | zone.r53_zone.commit() 105 | return obj 106 | 107 | def update(self, obj, validated_data): 108 | zone = self.context['zone'] 109 | if obj.managed: 110 | raise ValidationError("Can't change a managed record.") 111 | for attr, value in validated_data.items(): 112 | setattr(obj, attr, value) 113 | obj.full_clean() 114 | obj.save() 115 | with interpret_client_error(): 116 | zone.commit() 117 | return obj 118 | 119 | def validate_type(self, value): 120 | if value not in ALLOWED_RECORD_TYPES: 121 | raise ValidationError("Type '{}' is not allowed.".format(value)) 122 | return value 123 | 124 | def validate_name(self, value): 125 | # record name should not start with reserved prefix. 126 | if value.startswith(RECORD_PREFIX): 127 | raise ValidationError( 128 | ('Record {} can\'t start with {}. ' 129 | 'It\'s a reserved prefix.').format(value, RECORD_PREFIX) 130 | ) 131 | return value 132 | 133 | def validate(self, data): 134 | errors = {} 135 | # TODO: this stinks! we need a cleaner approach here 136 | # if is a delete then the data should be {'deleted': True} 137 | if self.context['request'].method == 'DELETE': 138 | return {'deleted': True} 139 | 140 | # for PATCH type and name field can't be modified. 141 | if self.context['request'].method == 'PATCH': 142 | if 'type' in data or 'name' in data: 143 | errors.update({'non_field_errors': ["Can't update 'name' and 'type' fields. "]}) 144 | else: 145 | # POST method 146 | # for POLICY_ROUTED the values should contain just one value 147 | if data['type'] in ['CNAME'] + ZINC_CUSTOM_RECORD_TYPES: 148 | if not len(data['values']) == 1: 149 | errors.update({ 150 | 'values': ('Only one value can be ' 151 | 'specified for {} records.'.format(data['type'])) 152 | }) 153 | else: 154 | data.setdefault('ttl', settings.ZINC_DEFAULT_TTL) 155 | # for normal records values is required. 156 | if not data.get('values', False): 157 | errors.update({'values': 'This field is required.'}) 158 | 159 | if errors: 160 | raise ValidationError(errors) 161 | 162 | return data 163 | -------------------------------------------------------------------------------- /zinc/serializers/zone.py: -------------------------------------------------------------------------------- 1 | from django.db import transaction 2 | from rest_framework import serializers 3 | from rest_framework.reverse import reverse 4 | 5 | from zinc.models import Zone 6 | from zinc.serializers import RecordSerializer 7 | 8 | 9 | class ZoneListSerializer(serializers.HyperlinkedModelSerializer): 10 | 11 | class Meta: 12 | model = Zone 13 | fields = ['root', 'url', 'id', 'route53_id', 'dirty', 'ns_propagated'] 14 | read_only_fields = ['dirty', 'ns_propagated'] 15 | 16 | @transaction.atomic 17 | def create(self, validated_data): 18 | zone = Zone.objects.create(**validated_data) 19 | zone.r53_zone.create() 20 | return zone 21 | 22 | def validate_root(self, value): 23 | if not value.endswith('.'): 24 | value += '.' 25 | return value 26 | 27 | 28 | class ZoneDetailSerializer(serializers.HyperlinkedModelSerializer): 29 | records = RecordSerializer(many=True, source='*') 30 | records_url = serializers.SerializerMethodField() 31 | 32 | def get_records_url(self, obj): 33 | request = self.context.get('request') 34 | return reverse('record-create', request=request, 35 | kwargs={ 36 | 'zone_id': obj.pk 37 | }) 38 | 39 | class Meta: 40 | model = Zone 41 | fields = ['root', 'url', 'records_url', 'records', 'route53_id', 'dirty', 'ns_propagated'] 42 | read_only_fields = ['root', 'url', 'route53_id', 'dirty', 'ns_propagated'] 43 | 44 | def __init__(self, *args, **kwargs): 45 | super(ZoneDetailSerializer, self).__init__(*args, **kwargs) 46 | self.partial = False 47 | -------------------------------------------------------------------------------- /zinc/tasks.py: -------------------------------------------------------------------------------- 1 | import redis 2 | import redis_lock 3 | 4 | from celery import shared_task 5 | from celery.exceptions import MaxRetriesExceededError 6 | from celery.utils.log import get_task_logger 7 | from django.conf import settings 8 | 9 | from zinc import models, route53 10 | 11 | logger = get_task_logger(__name__) 12 | 13 | 14 | @shared_task(bind=True, ignore_result=True, default_retry_delay=60) 15 | def aws_delete_zone(self, pk): 16 | zone = models.Zone.objects.get(pk=pk) 17 | assert zone.deleted 18 | aws_zone = zone.r53_zone 19 | 20 | try: 21 | aws_zone.delete() 22 | except Exception as e: 23 | logger.exception(e) 24 | try: 25 | self.retry() 26 | except MaxRetriesExceededError: 27 | logger.error('Failed to remove zone %s', zone.id) 28 | 29 | 30 | @shared_task(bind=True, ignore_result=True) 31 | def reconcile_zones(bind=True): 32 | """ 33 | Periodic task that reconciles everything zone-related (zone deletion, policy record updates) 34 | """ 35 | redis_client = redis.from_url(settings.LOCK_SERVER_URL) 36 | lock = redis_lock.Lock(redis_client, 'recouncile_zones', expire=60) 37 | 38 | if not lock.acquire(blocking=False): 39 | logger.info('Cannot aquire task lock. Probaly another task is running. Bailing out.') 40 | return 41 | 42 | try: 43 | for zone in models.Zone.need_reconciliation(): 44 | try: 45 | zone.reconcile() 46 | lock.extend(5) # extend the lease each time we rebuild a tree 47 | except Exception: 48 | logger.exception( 49 | "reconcile failed for Zone %s.%s", zone, zone.root 50 | ) 51 | finally: 52 | lock.release() 53 | 54 | 55 | @shared_task(bind=True, ignore_result=True) 56 | def check_clean_zones(bind=True): 57 | for zone in models.Zone.get_clean_zones(): 58 | zone.r53_zone.check_policy_trees() 59 | 60 | 61 | @shared_task(bind=True, ignore_result=True) 62 | def reconcile_healthchecks(bind=True): 63 | route53.HealthCheck.reconcile_for_ips(models.IP.objects.all()) 64 | 65 | 66 | @shared_task(bind=True, ignore_result=True) 67 | def update_ns_propagated(bind=True): 68 | redis_client = redis.from_url(settings.LOCK_SERVER_URL) 69 | 70 | # make this lock timeout big enough to cover updating about 1000 zones 71 | # ns_propagated flag and small enough to update the flag in an acceptable 72 | # time frame. 5 minutes sound good at the moment. 73 | lock = redis_lock.Lock(redis_client, 'update_ns_propagated', expire=300) 74 | 75 | if not lock.acquire(blocking=False): 76 | logger.info('Cannot aquire task lock. Probaly another task is running. Bailing out.') 77 | return 78 | try: 79 | models.Zone.update_ns_propagated(delay=getattr(settings, 'ZINC_NS_UPDATE_DELAY', 0.3)) 80 | except Exception: 81 | logger.exception("Could not update ns_propagated flag") 82 | finally: 83 | lock.release() 84 | -------------------------------------------------------------------------------- /zinc/urls.py: -------------------------------------------------------------------------------- 1 | from django.urls import path 2 | from rest_framework import routers 3 | 4 | from zinc import views 5 | 6 | 7 | router = routers.DefaultRouter(trailing_slash=False) 8 | router.register('policies', views.PolicyViewset, 'policy') 9 | router.register('zones', views.ZoneViewset, 'zone') 10 | 11 | urlpatterns = router.urls + [ 12 | path('zones//records/', 13 | views.RecordDetail.as_view(), name='record-detail'), 14 | path('zones//records', 15 | views.RecordCreate.as_view(), name='record-create'), 16 | ] 17 | -------------------------------------------------------------------------------- /zinc/utils/__init__.py: -------------------------------------------------------------------------------- 1 | from functools import wraps 2 | 3 | 4 | def memoized_property(method): 5 | """ 6 | Caches a method's return value on the instance. 7 | """ 8 | @property 9 | @wraps(method) 10 | def caching_wrapper(self): 11 | cache_key = "__cached_" + method.__name__ 12 | if not hasattr(self, cache_key): 13 | return_value = method(self) 14 | setattr(self, cache_key, return_value) 15 | return getattr(self, cache_key) 16 | return caching_wrapper 17 | -------------------------------------------------------------------------------- /zinc/utils/generators.py: -------------------------------------------------------------------------------- 1 | def chunks(lst, n): 2 | """Yield successive n-sized chunks from lst.""" 3 | for i in range(0, len(lst), n): 4 | yield lst[i:i + n] 5 | -------------------------------------------------------------------------------- /zinc/utils/validation.py: -------------------------------------------------------------------------------- 1 | import ipaddress 2 | 3 | 4 | def is_ipv6(ip_addr): 5 | try: 6 | ipaddress.IPv6Address(ip_addr) 7 | return True 8 | except ipaddress.AddressValueError: 9 | return False 10 | -------------------------------------------------------------------------------- /zinc/validators.py: -------------------------------------------------------------------------------- 1 | from django.core.validators import RegexValidator 2 | 3 | 4 | validate_hostname = RegexValidator( 5 | regex=(r'^(?=[a-z0-9\-\.]{1,253}$)([a-z0-9](([a-z0-9\-]){,61}[a-z0-9])?\.)' 6 | r'*([a-z0-9](([a-z0-9\-]){,61}[a-z0-9])?)$'), 7 | message=u'Invalid hostname', 8 | code='invalid_hostname' 9 | ) 10 | 11 | # Regex inspired from django.core.validators.URLValidator 12 | validate_domain = RegexValidator( 13 | regex=(r'[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?(?:\.(?!-)[a-z0-9-]{1,63}(?