├── .dockerignore ├── .gitignore ├── Dockerfile ├── LICENSE ├── README.md ├── bin └── manage.py ├── docker-compose.yml ├── etc └── containerpilot.json5 └── local-compose.yml /.dockerignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | **/.DS_Store 3 | _env* 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # build outputs 2 | *.pyc 3 | 4 | # credentials 5 | _env* 6 | manta 7 | manta.pub 8 | 9 | # temp 10 | python-manta/ 11 | 12 | # macos frustration 13 | .DS_Store 14 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM mongo:3.4 2 | 3 | RUN apt-get update \ 4 | && apt-get install -y \ 5 | python \ 6 | python-dev \ 7 | gcc \ 8 | curl \ 9 | libffi-dev \ 10 | libssl-dev \ 11 | unzip \ 12 | && rm -rf /var/lib/apt/lists/* 13 | 14 | # get Python drivers MongoDB, Consul, and Manta 15 | RUN curl -Ls -o get-pip.py https://bootstrap.pypa.io/get-pip.py && \ 16 | python get-pip.py && \ 17 | pip install \ 18 | PyMongo==3.4.0 \ 19 | python-Consul==0.7.0 \ 20 | manta==2.5.0 \ 21 | mock==2.0.0 22 | 23 | # Add consul agent 24 | RUN export CONSUL_VERSION=1.0.6 \ 25 | && export CONSUL_CHECKSUM=bcc504f658cef2944d1cd703eda90045e084a15752d23c038400cf98c716ea01 \ 26 | && curl --retry 7 --fail -vo /tmp/consul.zip "https://releases.hashicorp.com/consul/${CONSUL_VERSION}/consul_${CONSUL_VERSION}_linux_amd64.zip" \ 27 | && echo "${CONSUL_CHECKSUM} /tmp/consul.zip" | sha256sum -c \ 28 | && unzip /tmp/consul -d /usr/local/bin \ 29 | && rm /tmp/consul.zip \ 30 | && mkdir -p /opt/consul/config 31 | 32 | # Add ContainerPilot and set its configuration file path 33 | ENV CONTAINERPILOT_VER 3.7.0 34 | ENV CONTAINERPILOT /etc/containerpilot.json5 35 | RUN export CONTAINERPILOT_CHECKSUM=b10b30851de1ae1c095d5f253d12ce8fe8e7be17 \ 36 | && curl -Lso /tmp/containerpilot.tar.gz \ 37 | "https://github.com/joyent/containerpilot/releases/download/${CONTAINERPILOT_VER}/containerpilot-${CONTAINERPILOT_VER}.tar.gz" \ 38 | && echo "${CONTAINERPILOT_CHECKSUM} /tmp/containerpilot.tar.gz" | sha1sum -c \ 39 | && tar zxf /tmp/containerpilot.tar.gz -C /usr/local/bin \ 40 | && rm /tmp/containerpilot.tar.gz 41 | 42 | # add stopping timeouts for MongoDB 43 | ENV MONGO_SECONDARY_CATCHUP_PERIOD 8 44 | ENV MONGO_STEPDOWN_TIME 60 45 | ENV MONGO_ELECTION_TIMEOUT 30 46 | 47 | # Configure ContainerPilot and Mongo 48 | COPY etc/* /etc/ 49 | COPY bin/* /usr/local/bin/ 50 | 51 | ENTRYPOINT ["containerpilot"] 52 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Mozilla Public License Version 2.0 2 | ================================== 3 | 4 | 1. Definitions 5 | -------------- 6 | 7 | 1.1. "Contributor" 8 | means each individual or legal entity that creates, contributes to 9 | the creation of, or owns Covered Software. 10 | 11 | 1.2. "Contributor Version" 12 | means the combination of the Contributions of others (if any) used 13 | by a Contributor and that particular Contributor's Contribution. 14 | 15 | 1.3. "Contribution" 16 | means Covered Software of a particular Contributor. 17 | 18 | 1.4. "Covered Software" 19 | means Source Code Form to which the initial Contributor has attached 20 | the notice in Exhibit A, the Executable Form of such Source Code 21 | Form, and Modifications of such Source Code Form, in each case 22 | including portions thereof. 23 | 24 | 1.5. "Incompatible With Secondary Licenses" 25 | means 26 | 27 | (a) that the initial Contributor has attached the notice described 28 | in Exhibit B to the Covered Software; or 29 | 30 | (b) that the Covered Software was made available under the terms of 31 | version 1.1 or earlier of the License, but not also under the 32 | terms of a Secondary License. 33 | 34 | 1.6. "Executable Form" 35 | means any form of the work other than Source Code Form. 36 | 37 | 1.7. "Larger Work" 38 | means a work that combines Covered Software with other material, in 39 | a separate file or files, that is not Covered Software. 40 | 41 | 1.8. "License" 42 | means this document. 43 | 44 | 1.9. "Licensable" 45 | means having the right to grant, to the maximum extent possible, 46 | whether at the time of the initial grant or subsequently, any and 47 | all of the rights conveyed by this License. 48 | 49 | 1.10. "Modifications" 50 | means any of the following: 51 | 52 | (a) any file in Source Code Form that results from an addition to, 53 | deletion from, or modification of the contents of Covered 54 | Software; or 55 | 56 | (b) any new file in Source Code Form that contains any Covered 57 | Software. 58 | 59 | 1.11. "Patent Claims" of a Contributor 60 | means any patent claim(s), including without limitation, method, 61 | process, and apparatus claims, in any patent Licensable by such 62 | Contributor that would be infringed, but for the grant of the 63 | License, by the making, using, selling, offering for sale, having 64 | made, import, or transfer of either its Contributions or its 65 | Contributor Version. 66 | 67 | 1.12. "Secondary License" 68 | means either the GNU General Public License, Version 2.0, the GNU 69 | Lesser General Public License, Version 2.1, the GNU Affero General 70 | Public License, Version 3.0, or any later versions of those 71 | licenses. 72 | 73 | 1.13. "Source Code Form" 74 | means the form of the work preferred for making modifications. 75 | 76 | 1.14. "You" (or "Your") 77 | means an individual or a legal entity exercising rights under this 78 | License. For legal entities, "You" includes any entity that 79 | controls, is controlled by, or is under common control with You. For 80 | purposes of this definition, "control" means (a) the power, direct 81 | or indirect, to cause the direction or management of such entity, 82 | whether by contract or otherwise, or (b) ownership of more than 83 | fifty percent (50%) of the outstanding shares or beneficial 84 | ownership of such entity. 85 | 86 | 2. License Grants and Conditions 87 | -------------------------------- 88 | 89 | 2.1. Grants 90 | 91 | Each Contributor hereby grants You a world-wide, royalty-free, 92 | non-exclusive license: 93 | 94 | (a) under intellectual property rights (other than patent or trademark) 95 | Licensable by such Contributor to use, reproduce, make available, 96 | modify, display, perform, distribute, and otherwise exploit its 97 | Contributions, either on an unmodified basis, with Modifications, or 98 | as part of a Larger Work; and 99 | 100 | (b) under Patent Claims of such Contributor to make, use, sell, offer 101 | for sale, have made, import, and otherwise transfer either its 102 | Contributions or its Contributor Version. 103 | 104 | 2.2. Effective Date 105 | 106 | The licenses granted in Section 2.1 with respect to any Contribution 107 | become effective for each Contribution on the date the Contributor first 108 | distributes such Contribution. 109 | 110 | 2.3. Limitations on Grant Scope 111 | 112 | The licenses granted in this Section 2 are the only rights granted under 113 | this License. No additional rights or licenses will be implied from the 114 | distribution or licensing of Covered Software under this License. 115 | Notwithstanding Section 2.1(b) above, no patent license is granted by a 116 | Contributor: 117 | 118 | (a) for any code that a Contributor has removed from Covered Software; 119 | or 120 | 121 | (b) for infringements caused by: (i) Your and any other third party's 122 | modifications of Covered Software, or (ii) the combination of its 123 | Contributions with other software (except as part of its Contributor 124 | Version); or 125 | 126 | (c) under Patent Claims infringed by Covered Software in the absence of 127 | its Contributions. 128 | 129 | This License does not grant any rights in the trademarks, service marks, 130 | or logos of any Contributor (except as may be necessary to comply with 131 | the notice requirements in Section 3.4). 132 | 133 | 2.4. Subsequent Licenses 134 | 135 | No Contributor makes additional grants as a result of Your choice to 136 | distribute the Covered Software under a subsequent version of this 137 | License (see Section 10.2) or under the terms of a Secondary License (if 138 | permitted under the terms of Section 3.3). 139 | 140 | 2.5. Representation 141 | 142 | Each Contributor represents that the Contributor believes its 143 | Contributions are its original creation(s) or it has sufficient rights 144 | to grant the rights to its Contributions conveyed by this License. 145 | 146 | 2.6. Fair Use 147 | 148 | This License is not intended to limit any rights You have under 149 | applicable copyright doctrines of fair use, fair dealing, or other 150 | equivalents. 151 | 152 | 2.7. Conditions 153 | 154 | Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted 155 | in Section 2.1. 156 | 157 | 3. Responsibilities 158 | ------------------- 159 | 160 | 3.1. Distribution of Source Form 161 | 162 | All distribution of Covered Software in Source Code Form, including any 163 | Modifications that You create or to which You contribute, must be under 164 | the terms of this License. You must inform recipients that the Source 165 | Code Form of the Covered Software is governed by the terms of this 166 | License, and how they can obtain a copy of this License. You may not 167 | attempt to alter or restrict the recipients' rights in the Source Code 168 | Form. 169 | 170 | 3.2. Distribution of Executable Form 171 | 172 | If You distribute Covered Software in Executable Form then: 173 | 174 | (a) such Covered Software must also be made available in Source Code 175 | Form, as described in Section 3.1, and You must inform recipients of 176 | the Executable Form how they can obtain a copy of such Source Code 177 | Form by reasonable means in a timely manner, at a charge no more 178 | than the cost of distribution to the recipient; and 179 | 180 | (b) You may distribute such Executable Form under the terms of this 181 | License, or sublicense it under different terms, provided that the 182 | license for the Executable Form does not attempt to limit or alter 183 | the recipients' rights in the Source Code Form under this License. 184 | 185 | 3.3. Distribution of a Larger Work 186 | 187 | You may create and distribute a Larger Work under terms of Your choice, 188 | provided that You also comply with the requirements of this License for 189 | the Covered Software. If the Larger Work is a combination of Covered 190 | Software with a work governed by one or more Secondary Licenses, and the 191 | Covered Software is not Incompatible With Secondary Licenses, this 192 | License permits You to additionally distribute such Covered Software 193 | under the terms of such Secondary License(s), so that the recipient of 194 | the Larger Work may, at their option, further distribute the Covered 195 | Software under the terms of either this License or such Secondary 196 | License(s). 197 | 198 | 3.4. Notices 199 | 200 | You may not remove or alter the substance of any license notices 201 | (including copyright notices, patent notices, disclaimers of warranty, 202 | or limitations of liability) contained within the Source Code Form of 203 | the Covered Software, except that You may alter any license notices to 204 | the extent required to remedy known factual inaccuracies. 205 | 206 | 3.5. Application of Additional Terms 207 | 208 | You may choose to offer, and to charge a fee for, warranty, support, 209 | indemnity or liability obligations to one or more recipients of Covered 210 | Software. However, You may do so only on Your own behalf, and not on 211 | behalf of any Contributor. You must make it absolutely clear that any 212 | such warranty, support, indemnity, or liability obligation is offered by 213 | You alone, and You hereby agree to indemnify every Contributor for any 214 | liability incurred by such Contributor as a result of warranty, support, 215 | indemnity or liability terms You offer. You may include additional 216 | disclaimers of warranty and limitations of liability specific to any 217 | jurisdiction. 218 | 219 | 4. Inability to Comply Due to Statute or Regulation 220 | --------------------------------------------------- 221 | 222 | If it is impossible for You to comply with any of the terms of this 223 | License with respect to some or all of the Covered Software due to 224 | statute, judicial order, or regulation then You must: (a) comply with 225 | the terms of this License to the maximum extent possible; and (b) 226 | describe the limitations and the code they affect. Such description must 227 | be placed in a text file included with all distributions of the Covered 228 | Software under this License. Except to the extent prohibited by statute 229 | or regulation, such description must be sufficiently detailed for a 230 | recipient of ordinary skill to be able to understand it. 231 | 232 | 5. Termination 233 | -------------- 234 | 235 | 5.1. The rights granted under this License will terminate automatically 236 | if You fail to comply with any of its terms. However, if You become 237 | compliant, then the rights granted under this License from a particular 238 | Contributor are reinstated (a) provisionally, unless and until such 239 | Contributor explicitly and finally terminates Your grants, and (b) on an 240 | ongoing basis, if such Contributor fails to notify You of the 241 | non-compliance by some reasonable means prior to 60 days after You have 242 | come back into compliance. Moreover, Your grants from a particular 243 | Contributor are reinstated on an ongoing basis if such Contributor 244 | notifies You of the non-compliance by some reasonable means, this is the 245 | first time You have received notice of non-compliance with this License 246 | from such Contributor, and You become compliant prior to 30 days after 247 | Your receipt of the notice. 248 | 249 | 5.2. If You initiate litigation against any entity by asserting a patent 250 | infringement claim (excluding declaratory judgment actions, 251 | counter-claims, and cross-claims) alleging that a Contributor Version 252 | directly or indirectly infringes any patent, then the rights granted to 253 | You by any and all Contributors for the Covered Software under Section 254 | 2.1 of this License shall terminate. 255 | 256 | 5.3. In the event of termination under Sections 5.1 or 5.2 above, all 257 | end user license agreements (excluding distributors and resellers) which 258 | have been validly granted by You or Your distributors under this License 259 | prior to termination shall survive termination. 260 | 261 | ************************************************************************ 262 | * * 263 | * 6. Disclaimer of Warranty * 264 | * ------------------------- * 265 | * * 266 | * Covered Software is provided under this License on an "as is" * 267 | * basis, without warranty of any kind, either expressed, implied, or * 268 | * statutory, including, without limitation, warranties that the * 269 | * Covered Software is free of defects, merchantable, fit for a * 270 | * particular purpose or non-infringing. The entire risk as to the * 271 | * quality and performance of the Covered Software is with You. * 272 | * Should any Covered Software prove defective in any respect, You * 273 | * (not any Contributor) assume the cost of any necessary servicing, * 274 | * repair, or correction. This disclaimer of warranty constitutes an * 275 | * essential part of this License. No use of any Covered Software is * 276 | * authorized under this License except under this disclaimer. * 277 | * * 278 | ************************************************************************ 279 | 280 | ************************************************************************ 281 | * * 282 | * 7. Limitation of Liability * 283 | * -------------------------- * 284 | * * 285 | * Under no circumstances and under no legal theory, whether tort * 286 | * (including negligence), contract, or otherwise, shall any * 287 | * Contributor, or anyone who distributes Covered Software as * 288 | * permitted above, be liable to You for any direct, indirect, * 289 | * special, incidental, or consequential damages of any character * 290 | * including, without limitation, damages for lost profits, loss of * 291 | * goodwill, work stoppage, computer failure or malfunction, or any * 292 | * and all other commercial damages or losses, even if such party * 293 | * shall have been informed of the possibility of such damages. This * 294 | * limitation of liability shall not apply to liability for death or * 295 | * personal injury resulting from such party's negligence to the * 296 | * extent applicable law prohibits such limitation. Some * 297 | * jurisdictions do not allow the exclusion or limitation of * 298 | * incidental or consequential damages, so this exclusion and * 299 | * limitation may not apply to You. * 300 | * * 301 | ************************************************************************ 302 | 303 | 8. Litigation 304 | ------------- 305 | 306 | Any litigation relating to this License may be brought only in the 307 | courts of a jurisdiction where the defendant maintains its principal 308 | place of business and such litigation shall be governed by laws of that 309 | jurisdiction, without reference to its conflict-of-law provisions. 310 | Nothing in this Section shall prevent a party's ability to bring 311 | cross-claims or counter-claims. 312 | 313 | 9. Miscellaneous 314 | ---------------- 315 | 316 | This License represents the complete agreement concerning the subject 317 | matter hereof. If any provision of this License is held to be 318 | unenforceable, such provision shall be reformed only to the extent 319 | necessary to make it enforceable. Any law or regulation which provides 320 | that the language of a contract shall be construed against the drafter 321 | shall not be used to construe this License against a Contributor. 322 | 323 | 10. Versions of the License 324 | --------------------------- 325 | 326 | 10.1. New Versions 327 | 328 | Mozilla Foundation is the license steward. Except as provided in Section 329 | 10.3, no one other than the license steward has the right to modify or 330 | publish new versions of this License. Each version will be given a 331 | distinguishing version number. 332 | 333 | 10.2. Effect of New Versions 334 | 335 | You may distribute the Covered Software under the terms of the version 336 | of the License under which You originally received the Covered Software, 337 | or under the terms of any subsequent version published by the license 338 | steward. 339 | 340 | 10.3. Modified Versions 341 | 342 | If you create software not governed by this License, and you want to 343 | create a new license for such software, you may create and use a 344 | modified version of this License if you rename the license and remove 345 | any references to the name of the license steward (except to note that 346 | such modified license differs from this License). 347 | 348 | 10.4. Distributing Source Code Form that is Incompatible With Secondary 349 | Licenses 350 | 351 | If You choose to distribute Source Code Form that is Incompatible With 352 | Secondary Licenses under the terms of this version of the License, the 353 | notice described in Exhibit B of this License must be attached. 354 | 355 | Exhibit A - Source Code Form License Notice 356 | ------------------------------------------- 357 | 358 | This Source Code Form is subject to the terms of the Mozilla Public 359 | License, v. 2.0. If a copy of the MPL was not distributed with this 360 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 361 | 362 | If it is not possible or desirable to put the notice in a particular 363 | file, then You may include the notice in a location (such as a LICENSE 364 | file in a relevant directory) where a recipient would be likely to look 365 | for such a notice. 366 | 367 | You may add additional accurate notices of copyright ownership. 368 | 369 | Exhibit B - "Incompatible With Secondary Licenses" Notice 370 | --------------------------------------------------------- 371 | 372 | This Source Code Form is "Incompatible With Secondary Licenses", as 373 | defined by the Mozilla Public License, v. 2.0. 374 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # AutoPilot Pattern MongoDB 2 | 3 | *A robust and highly-scalable implementation of MongoDB in Docker using the Autopilot Pattern* 4 | 5 | ## Architecture 6 | 7 | A running cluster includes the following components: 8 | - [ContainerPilot](https://www.joyent.com/containerpilot): included in our MongoDB containers to orchestrate bootstrap behavior and coordinate replica joining using keys and checks stored in Consul in the `health`, and `onChange` handlers 9 | - [MongoDB](https://www.mongodb.com/community): we're using MongoDB 3.4 and setting up a [replica set](https://docs.mongodb.com/manual/replication/) 10 | - [Consul](https://www.consul.io/): used to coordinate replication and failover 11 | 12 | ## Running the cluster 13 | 14 | Starting a new cluster is easy once you have [your `_env` file set with the configuration details](#configuration) 15 | 16 | - for Triton, just run `docker-compose up -d` 17 | - for non-Triton, just run `docker-compose -f local-compose.yml up -d` 18 | 19 | In a few moments you'll have a running MongoDB ready for a replica set. Both the master and replicas are described as a single `docker-compose` service. During startup, [ContainerPilot](http://containerpilot.io) will ask Consul if an existing master has been created. If not, the node will initialize as a new MongoDB replica set and all future nodes will be added to the replica set by the current master. All master election is handled by [MongoDB itself](https://docs.mongodb.com/manual/core/replica-set-elections/) and the result is cached in Consul. 20 | 21 | **Run `docker-compose -f local-compose.yml scale mongodb=2` to add a replica (or more than one!)**. The replicas will automatically be added to the replica set on the master and will register themselves in Consul as replicas once they're ready. 22 | 23 | ### Configuration 24 | 25 | Pass these variables via an `_env` file. 26 | 27 | - `LOG_LEVEL`: control the amount of logging from ContainerPilot 28 | - when the primary node is sent a `SIGTERM` it will [step down](https://docs.mongodb.com/manual/reference/command/replSetStepDown/) as primary; the following control those timeouts 29 | - `MONGO_SECONDARY_CATCHUP_PERIOD`: the number of seconds that the mongod will wait for an electable secondary to catch up to the primary 30 | - `MONGO_STEPDOWN_TIME`: the number of seconds to step down the primary, during which time the stepdown member is ineligible for becoming primary 31 | - `MONGO_ELECTION_TIMEOUT`: after the primary steps down, the amount a tries to check that a new primary has been elected before the node shuts down 32 | - `CONSUL` (optional): when using `local-compose.yml`, this will default to `consul` (and thus use the DNS provided by Docker), but for deploying on Triton via `docker-compose.yml`, this should be set to [the CNS path of the `consul` service (`consul.svc.XXX...`)](https://docs.joyent.com/public-cloud/network/cns) 33 | 34 | Not yet implemented: 35 | - `MANTA_URL`: the full Manta endpoint URL. (ex. `https://us-east.manta.joyent.com`) 36 | - `MANTA_USER`: the Manta account name. 37 | - `MANTA_SUBUSER`: the Manta subuser account name, if any. 38 | - `MANTA_ROLE`: the Manta role name, if any. 39 | - `MANTA_KEY_ID`: the MD5-format ssh key id for the Manta account/subuser (ex. `1a:b8:30:2e:57:ce:59:1d:16:f6:19:97:f2:60:2b:3d`); the included `setup.sh` will encode this automatically 40 | - `MANTA_PRIVATE_KEY`: the private ssh key for the Manta account/subuser; the included `setup.sh` will encode this automatically 41 | - `MANTA_BUCKET`: the path on Manta where backups will be stored. (ex. `/myaccount/stor/triton-mysql`); the bucket must already exist and be writeable by the `MANTA_USER`/`MANTA_PRIVATE_KEY` 42 | 43 | ### Sponsors 44 | 45 | Initial development of this project was sponsored by [Joyent](https://www.joyent.com). 46 | -------------------------------------------------------------------------------- /bin/manage.py: -------------------------------------------------------------------------------- 1 | import fcntl 2 | import inspect 3 | import json 4 | import logging 5 | import os 6 | import socket 7 | import signal 8 | import struct 9 | import sys 10 | import time 11 | import datetime 12 | import subprocess 13 | import manta 14 | 15 | from functools import wraps 16 | 17 | import consul as pyconsul 18 | #import manta 19 | from pymongo import MongoClient 20 | from pymongo.errors import * 21 | 22 | CONSUL_AGENT = os.getenv('CONSUL_AGENT', False) 23 | CONSUL_HOST = ('localhost' if CONSUL_AGENT else os.getenv('CONSUL', 'consul')) 24 | 25 | consul = pyconsul.Consul(host=CONSUL_HOST) 26 | logging.basicConfig(format='%(asctime)s %(levelname)s %(name)s %(message)s', 27 | stream=sys.stdout, 28 | level=logging.getLevelName( 29 | os.environ.get('LOG_LEVEL', 'INFO'))) 30 | 31 | log = logging.getLogger('manage.py') 32 | 33 | def debug(fn): 34 | """ 35 | Function/method decorator to trace calls via debug logging. 36 | Is a pass-thru if we're not at LOG_LEVEL=DEBUG. Normally this 37 | would have a lot of perf impact but this application doesn't 38 | have significant throughput. 39 | """ 40 | @wraps(fn) 41 | def wrapper(*args, **kwargs): 42 | try: 43 | # because we have concurrent processes running we want 44 | # to tag each stack with an identifier for that process 45 | arg = "[{}]".format(sys.argv[1]) 46 | except IndexError: 47 | arg = "[pre_start]" 48 | name = '{}{}{}'.format(arg, (len(inspect.stack()) * " "), fn.__name__) 49 | log.debug('%s' % name) 50 | out = apply(fn, args, kwargs) 51 | log.debug('%s: %s', name, out) 52 | return out 53 | return wrapper 54 | 55 | def get_environ(key, default): 56 | """ 57 | Gets an environment variable and trims away comments and whitespace. 58 | """ 59 | val = os.environ.get(key, default) 60 | try: 61 | val = val.split('#')[0] 62 | val = val.strip() 63 | val = os.path.expandvars(val) 64 | finally: 65 | # just swallow AttributeErrors for non-strings 66 | return val 67 | 68 | # --------------------------------------------------------- 69 | 70 | SESSION_CACHE_FILE = get_environ('SESSION_CACHE_FILE', '/tmp/mongodb-session') 71 | SESSION_NAME = get_environ('SESSION_NAME', 'mongodb-replica-set-lock') 72 | SESSION_TTL = int(get_environ('SESSION_TTL', 60)) 73 | 74 | # consts for node state 75 | PRIMARY = 'mongodb-replicaset' 76 | #SECONDARY = 'mongodb-secondary' 77 | 78 | # key where primary will be stored in consul 79 | PRIMARY_KEY = get_environ('PRIMARY_KEY', 'mongodb-primary') 80 | 81 | # how many time to retry connecting to mongo 82 | # be aware that the health check may timeout before this is reached 83 | MONGO_RETRY_TIMES=int(get_environ('MONGO_RETRY_TIMES', 10)) 84 | 85 | # timeouts when the primary node gets a SIGTERM and needs to step down as primary 86 | MONGO_STEPDOWN_TIME=int(get_environ('MONGO_STEPDOWN_TIME', 60)) 87 | MONGO_SECONDARY_CATCHUP_PERIOD=int(get_environ('MONGO_SECONDARY_CATCHUP_PERIOD', 8)) 88 | MONGO_ELECTION_TIMEOUT=int(get_environ('MONGO_ELECTION_TIMEOUT', 30)) 89 | 90 | # Manta related information 91 | MANTA_URL=get_environ('MANTA_URL','https://us-east.manta.joyent.com') 92 | MANTA_USER=get_environ('MANTA_USER','') 93 | MANTA_KEY_ID=get_environ('MANTA_KEY_ID','') 94 | MANTA_TLS_INSECURE=bool(get_environ('MANTA_TLS_INSECURE',True)) 95 | 96 | # --------------------------------------------------------- 97 | # Top-level functions called by ContainerPilot or forked by this program 98 | 99 | @debug 100 | def pre_start(): 101 | """ 102 | MongoDB must be running in order to execute most of our setup behavior 103 | """ 104 | # TODO is there anything that needs to be done before starting mongo? 105 | sys.exit(0) 106 | 107 | @debug 108 | def pre_stop(): 109 | """ 110 | If we are the primary in the MongoDB replica set, we need to step down 111 | because we are about to be shut down 112 | """ 113 | 114 | ip = get_ip() 115 | local_mongo = MongoClient(ip, connect=False) 116 | 117 | # since we are shutting down, it is ok to stop if mongo is already non-responsive 118 | if not is_mongo_up(local_mongo): 119 | return True 120 | 121 | try: 122 | repl_status = local_mongo.admin.command('replSetGetStatus') 123 | is_mongo_primary = repl_status['myState'] == 1 124 | # ref https://docs.mongodb.com/manual/reference/replica-states/ 125 | except Exception as e: 126 | log.error(e, 'unable to get primary status while shuting down') 127 | return True 128 | 129 | if is_mongo_primary: 130 | # the primary will wait up to X seconds for a secondary member 131 | # to catch up and sets itself as ineligible to be primary again for 60 seconds 132 | # https://docs.mongodb.com/manual/reference/command/replSetStepDown/ 133 | # this is set to 8 so that we timeout before `docker stop` would send a sigkill 134 | try: 135 | local_mongo.admin.command('replSetStepDown', MONGO_STEPDOWN_TIME, secondaryCatchUpPeriodSecs=MONGO_SECONDARY_CATCHUP_PERIOD) 136 | except ConnectionFailure: 137 | # this means mongo closed all connections and this node is no longer primary 138 | wait_for_election = True 139 | except ExecutionTimeout as e: 140 | # stepdown fails, ie no secondary that is caught up 141 | log.debug(e) 142 | try: 143 | # force 144 | local_mongo.admin.command('replSetStepDown', MONGO_STEPDOWN_TIME, force=True) 145 | except ConnectionFailure: 146 | # this means mongo closed all connections and this node is no longer primary 147 | wait_for_election = True 148 | 149 | if wait_for_election: 150 | timeout = 0 151 | while True: 152 | if timeout >= MONGO_ELECTION_TIMEOUT: 153 | log.error('did not see mongodb election results of new primary after %i times' % timeout) 154 | return False 155 | timeout += 1 156 | # use a replica client so that we get "primary" data 157 | mongo_client = MongoClient(ip, connect=False, replicaset=repl_status['set'], serverSelectionTimeoutMS=500) 158 | # is_mongo_up will sleep on failure, so we don't need a "time.sleep(1)" 159 | if is_mongo_up(mongo_client, 1): 160 | primary = mongo_client.primary 161 | if primary != None: 162 | log.debug('primary elected: {0!s}'.format(primary)) 163 | return True 164 | 165 | return True 166 | 167 | @debug 168 | def upload(filename): 169 | try: 170 | log.debug(filename) 171 | client = manta.MantaClient(url=MANTA_URL, 172 | account=MANTA_USER, 173 | signer=manta.SSHAgentSigner([MANTA_KEY_ID]), 174 | disable_ssl_certificate_validation=MANTA_TLS_INSECURE 175 | ) 176 | client.put_object('/%s/stor/%s' % (MANTA_USER, filename), path='/tmp/%s' % filename) 177 | except Exception as e: 178 | log.debug(e) 179 | return False 180 | return True 181 | 182 | @debug 183 | def backup(): 184 | """ 185 | Run periodic mongodump and save backup to MANTA 186 | """ 187 | 188 | hostname = socket.gethostname() 189 | ip = get_ip() 190 | local_mongo = MongoClient(ip, connect=False) 191 | 192 | timestamp = datetime.datetime.utcnow().isoformat() 193 | dump = "dump-%s.gz" % timestamp 194 | 195 | try: 196 | backup_output = subprocess.check_output([ 'mongodump', '--archive=/tmp/%s' % dump, '--gzip' ]) 197 | log.debug(backup_output) 198 | 199 | # Upload dump onto Manta 200 | #TODO: do not manage to upload to Manta 201 | # upload(dump) 202 | 203 | # Delete local dump once uploaded 204 | # os.remove('/tmp/%s' % dump) 205 | except Exception as e: 206 | log.debug('Error during backup process'); 207 | log.debug(e); 208 | return False 209 | 210 | return True 211 | 212 | @debug 213 | def health(): 214 | """ 215 | Run a simple health check. Also acts as a check for whether the 216 | ContainerPilot configuration needs to be reloaded (if it's been 217 | changed externally). 218 | """ 219 | 220 | hostname = socket.gethostname() 221 | ip = get_ip() 222 | local_mongo = MongoClient(ip, connect=False) 223 | 224 | # check that mongo is responsive 225 | if not is_mongo_up(local_mongo): 226 | return False 227 | 228 | # make sure this node has a valid consul session to work with 229 | get_session() 230 | 231 | try: 232 | repl_status = local_mongo.admin.command('replSetGetStatus') 233 | # TODO handle non-exceptional states 234 | # if repl_status['myState'] == 1: 235 | # # ref https://docs.mongodb.com/manual/reference/replica-states/ 236 | # state = PRIMARY 237 | # # mongo_update_replset_config is not required in the health check 238 | # # but may speed up adding of new members 239 | # # dropping to keep consul traffic minimal 240 | # #mongo_update_replset_config(local_mongo, hostname) 241 | # elif repl_status['myState'] in (2, 3, 5): 242 | # # mongo states of: SECONDARY or RECOVERING or STARTUP2 243 | # state = SECONDARY 244 | except OperationFailure as e: 245 | # happens when replica set is not initialized 246 | log.debug(e) 247 | consul_primary = get_primary_node_from_consul() 248 | if not consul_primary: 249 | # this should only happen at the beginning when there is no replica set 250 | # so the first node to get the lock in consul will initialize the set 251 | # by setting self as primary in consul and then rs.init() 252 | mark_as_primary(hostname) 253 | #state = PRIMARY 254 | local_mongo.admin.command('replSetInitiate') 255 | else: 256 | # this happens when the primary node is still initializing 257 | # wait for it to finish so that it can add this node to the replica set 258 | # while waiting, we are a "healthy" node, since mongo is responsive 259 | # TODO maybe make this a second state of "recovering/initializing" node? 260 | return True 261 | 262 | return True 263 | 264 | @debug 265 | def on_change(): 266 | ''' 267 | called when there is a change in the list of IPs and ports for this backend 268 | ''' 269 | hostname = socket.gethostname() 270 | ip = get_ip() 271 | local_mongo = MongoClient(ip, connect=False) 272 | 273 | try: 274 | repl_status = local_mongo.admin.command('replSetGetStatus') 275 | is_mongo_primary = repl_status['myState'] == 1 276 | # ref https://docs.mongodb.com/manual/reference/replica-states/ 277 | except Exception as e: 278 | log.error(e, 'unable to get primary status') 279 | return False 280 | 281 | if is_mongo_primary: 282 | return mongo_update_replset_config(local_mongo, ip) 283 | else: 284 | return True 285 | 286 | # --------------------------------------------------------- 287 | 288 | def is_mongo_up(local_mongo, max_timeout=MONGO_RETRY_TIMES): 289 | ''' 290 | check to see if mongo is up yet, retying the given number of times 291 | ''' 292 | timeout = 0 293 | ip = get_ip() 294 | while True: 295 | if timeout >= max_timeout: 296 | if timeout != 1: 297 | log.error('unable to connect to mongodb or replica set after %i times' % timeout) 298 | return False 299 | timeout += 1 300 | 301 | try: 302 | # check that mongo is up 303 | server_info = local_mongo.server_info() 304 | if not server_info['ok']: 305 | log.info('Mongo response not "ok" on %s; retrying...' % ip) 306 | time.sleep(1) 307 | continue 308 | break 309 | except (AutoReconnect, ServerSelectionTimeoutError) as e: 310 | log.info('Mongo or specified replica set not yet available on %s; retrying...' % ip) 311 | time.sleep(1) 312 | continue 313 | except (ConnectionFailure, NetworkTimeout, NotMasterError) as e: 314 | # TODO retry like AutoReconnect error above? 315 | log.info(e) 316 | time.sleep(1) 317 | return False 318 | except Exception as e: 319 | # just bail on unexpected exceptions when trying to connect 320 | log.error(e) 321 | return False 322 | return True 323 | 324 | @debug 325 | def mongo_update_replset_config(local_mongo, hostname): 326 | ''' 327 | called from the primary node to update the replica config in mongo 328 | using the current set of healthy mongo containers listed in consul 329 | ''' 330 | try: 331 | # get current replica config from mongodb 332 | repl_config = local_mongo.admin.command('replSetGetConfig') 333 | if not repl_config['ok']: 334 | raise Exception('could not get replica config: %s' % repl_config['errmsg']) 335 | repl_config = repl_config['config'] 336 | 337 | # use consul health to get our healthy services 338 | index, consul_services = consul.health.service(PRIMARY) 339 | 340 | # translate the name stored by consul to be the "host" name stored 341 | # in mongo config, skipping any non-mongo services 342 | mongos_in_consul = [consul_to_mongo_hostname(svc['Service']) for svc in consul_services] 343 | mongos_in_consul = [svc for svc in mongos_in_consul if svc] 344 | # empty list from consul means we have nothing to compare against 345 | if not mongos_in_consul: 346 | return 347 | # if the master node is not in the consul services list we need to 348 | # wait a little longer before configuring mongo 349 | if not hostname + ':27017' in mongos_in_consul: 350 | return 351 | 352 | members = repl_config['members'] 353 | existing_hosts, ids = zip(*[(member['host'], member['_id']) for member in members]) 354 | ids = list(ids) 355 | existing = set(existing_hosts) 356 | current = set(mongos_in_consul) 357 | 358 | new_mongos = current - existing 359 | stale_mongos = existing - current 360 | 361 | if not new_mongos and not stale_mongos: 362 | return # no change 363 | 364 | # don't keep mongo replica members that are not listed in consul 365 | for member in members: 366 | if member['host'] in stale_mongos: 367 | members.remove(member) 368 | for new_mongo in new_mongos: 369 | new_id = max(ids) + 1 370 | ids.append(new_id) 371 | members.append({'_id': new_id, 'host': new_mongo}) 372 | 373 | # TODO voting membership 374 | # https://docs.mongodb.com/manual/core/replica-set-architectures/#maximum-number-of-voting-members 375 | # ERROR manage.py Replica set configuration contains 10 voting members, but must be at least 1 and no more than 7 376 | # it should also be odd for tie breaking 377 | # also limit number of nodes to 50, since that is all a replica set can have 378 | 379 | repl_config['members'] = members 380 | repl_config['version'] += 1 381 | local_mongo.admin.command('replSetReconfig', repl_config) 382 | 383 | log.info('updating replica config in mongo from consul info') 384 | return repl_config 385 | 386 | except Exception as e: 387 | log.exception(e) 388 | sys.exit(1) 389 | 390 | def consul_to_mongo_hostname(service): 391 | # if name.startswith(SECONDARY + '-'): 392 | # prefix = SECONDARY + '-' 393 | if service['ID'].startswith(PRIMARY + '-'): 394 | return service['Address'] + ':' + str(service['Port']) 395 | else: 396 | return None 397 | 398 | # --------------------------------------------------------- 399 | 400 | @debug 401 | def get_primary_node_from_consul(timeout=10): 402 | while timeout > 0: 403 | try: 404 | result = consul.kv.get(PRIMARY_KEY) 405 | if result[1]: 406 | if result[1].get('Session', False): 407 | return result[1]['Value'] 408 | # either there is no primary or the session has expired 409 | return None 410 | except Exception as e: 411 | timeout = timeout - 1 412 | time.sleep(1) 413 | raise e 414 | 415 | @debug 416 | def mark_as_primary(hostname): 417 | """ Write flag to Consul to mark this node as primary """ 418 | session_id = get_session() 419 | if not mark_with_session(PRIMARY_KEY, hostname, session_id): 420 | log.error('Tried to mark node primary but primary exists, ' 421 | 'exiting for retry on next check.') 422 | sys.exit(1) 423 | 424 | @debug 425 | def mark_with_session(key, val, session_id, timeout=10): 426 | while timeout > 0: 427 | try: 428 | return consul.kv.put(key, val, acquire=session_id) 429 | except Exception as e: 430 | log.debug(e) 431 | timeout = timeout - 1 432 | time.sleep(1) 433 | raise e 434 | 435 | def get_session(no_cache=False): 436 | """ 437 | Gets a Consul session ID from the on-disk cache or calls into 438 | `create_session` to generate and cache a new one. 439 | Also, renews the session TTL of on-disk key, to ensure it is valid 440 | in Consul 441 | """ 442 | if no_cache: 443 | return create_session() 444 | 445 | try: 446 | with open(SESSION_CACHE_FILE, 'r') as f: 447 | session_id = f.read() 448 | 449 | # ensure the session_id is valid and refresh it 450 | consul.session.renew(session_id) 451 | except (IOError, pyconsul.base.NotFound) as e: 452 | # this means we have don't have a key locally, or 453 | # it is expired, so get a new one 454 | session_id = create_session() 455 | 456 | return session_id 457 | 458 | def create_session(ttl=None): 459 | """ 460 | We can't rely on storing Consul session IDs in memory because 461 | `health` and `onChange` handler calls happen in a subsequent 462 | process. Here we creates a session on Consul and cache the 463 | session ID to disk. Returns the session ID. 464 | """ 465 | session_id = consul.session.create(name=SESSION_NAME, 466 | behavior='release', 467 | ttl=ttl) 468 | with open(SESSION_CACHE_FILE, 'w') as f: 469 | f.write(session_id) 470 | return session_id 471 | 472 | # --------------------------------------------------------- 473 | # utility functions 474 | 475 | def get_ip(iface='eth0'): 476 | """ 477 | Use Linux SIOCGIFADDR ioctl to get the IP for the interface. 478 | ref http://code.activestate.com/recipes/439094-get-the-ip-address-associated-with-a-network-inter/ 479 | """ 480 | sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 481 | return socket.inet_ntoa(fcntl.ioctl( 482 | sock.fileno(), 483 | 0x8915, # SIOCGIFADDR 484 | struct.pack('256s', iface[:15]) 485 | )[20:24]) 486 | 487 | # --------------------------------------------------------- 488 | 489 | if __name__ == '__main__': 490 | 491 | # manta_config = Manta() 492 | 493 | if len(sys.argv) > 1: 494 | func = sys.argv[1] 495 | try: 496 | if not locals()[func](): 497 | log.info('Function failed %s' % func) 498 | sys.exit(1) 499 | except KeyError: 500 | log.error('Invalid command %s', func) 501 | sys.exit(1) 502 | else: 503 | # default behavior will be to start mysqld, running the 504 | # initialization if required 505 | pre_start() 506 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2.1' 2 | services: 3 | mongodb: 4 | extends: 5 | file: local-compose.yml 6 | service: mongodb 7 | mem_limit: 4g 8 | labels: 9 | - triton.cns.services=mongodb 10 | links: 11 | - consul:consul 12 | network_mode: bridge 13 | 14 | consul: 15 | extends: 16 | file: local-compose.yml 17 | service: consul 18 | labels: 19 | - triton.cns.services=consul 20 | network_mode: bridge 21 | dns: 22 | - 127.0.0.1 23 | -------------------------------------------------------------------------------- /etc/containerpilot.json5: -------------------------------------------------------------------------------- 1 | { 2 | consul: "{{ if .CONSUL_AGENT }}localhost{{ else }}{{ if .CONSUL }}{{ .CONSUL }}{{ else }}consul{{ end }}{{ end }}:8500", 3 | logging: { 4 | level: "DEBUG", 5 | format: "text" 6 | }, 7 | jobs: [ 8 | { 9 | name: "preStart", 10 | exec: "python /usr/local/bin/manage.py" 11 | }, 12 | { 13 | name: "preStop", 14 | exec: "python /usr/local/bin/manage.py pre_stop" 15 | }, 16 | {{ if .CONSUL_AGENT }} 17 | { 18 | name: "consul-agent", 19 | exec: ["/usr/local/bin/consul", "agent", 20 | "-data-dir=/opt/consul/data", 21 | "-config-dir=/opt/consul/config", 22 | "-rejoin", 23 | "-retry-join", "{{ if .CONSUL }}{{ .CONSUL }}{{ else }}consul{{ end }}", 24 | "-retry-max", "10", 25 | "-retry-interval", "10s"], 26 | restarts: "unlimited" 27 | }, 28 | {{ end }} 29 | { 30 | name: "mongodb-replicaset", 31 | port: 27017, 32 | exec: "mongod --replSet={{ if .REPLICASET }}{{ .REPLICASET }}{{ else }}joyent{{ end }}", 33 | when: { 34 | source: "preStart", 35 | once: "exitSuccess" 36 | }, 37 | health: { 38 | exec: "python /usr/local/bin/manage.py health", 39 | interval: 10, 40 | ttl: 25 41 | }, 42 | }, 43 | { 44 | name: "onchange-mongo", 45 | exec: "python /usr/local/bin/manage.py on_change", 46 | when: { 47 | source: "watch.mongodb-replicaset", 48 | each: "changed" 49 | } 50 | }, 51 | ], 52 | watches: [ 53 | { 54 | name: "mongodb-replicaset", 55 | interval: 2 56 | } 57 | ] 58 | } 59 | -------------------------------------------------------------------------------- /local-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2.1' 2 | services: 3 | mongodb: 4 | image: autopilotpattern/mongodb 5 | command: --replSet=rs0 6 | restart: always 7 | mem_limit: 512m 8 | build: . 9 | ports: 10 | - 27017 11 | 12 | consul: 13 | image: consul:0.7.5 14 | command: agent -server -client=0.0.0.0 -bootstrap -ui 15 | ports: 16 | - "8500:8500" 17 | restart: always 18 | --------------------------------------------------------------------------------