├── .evergreen └── config.yml ├── .github └── workflows │ ├── release-python.yml │ └── test.yml ├── .gitignore ├── LICENSE ├── README.rst ├── mongo-orchestration.config ├── mongo_orchestration ├── __init__.py ├── _version.py ├── apps │ ├── __init__.py │ ├── links.py │ ├── replica_sets.py │ ├── servers.py │ └── sharded_clusters.py ├── common.py ├── compat.py ├── configurations │ ├── replica_sets │ │ ├── allengines.json │ │ ├── arbiter.json │ │ ├── auth.json │ │ ├── basic.json │ │ ├── clean.json │ │ ├── mmapv1.json │ │ ├── ssl.json │ │ ├── ssl_auth.json │ │ └── wiredtiger.json │ ├── servers │ │ ├── auth.json │ │ ├── basic.json │ │ ├── clean.json │ │ ├── mmapv1.json │ │ ├── ssl.json │ │ ├── ssl_auth.json │ │ └── wiredtiger.json │ └── sharded_clusters │ │ ├── auth.json │ │ ├── basic.json │ │ ├── clean.json │ │ ├── mmapv1.json │ │ ├── ssl.json │ │ ├── ssl_auth.json │ │ └── wiredtiger.json ├── container.py ├── daemon.py ├── errors.py ├── launch.py ├── lib │ ├── ca.pem │ ├── client.pem │ └── server.pem ├── process.py ├── replica_sets.py ├── server.py ├── servers.py ├── sharded_clusters.py └── singleton.py ├── pyproject.toml ├── scripts ├── mo ├── mongo-orchestration-setup.ps1 ├── mongo-orchestration-start.ps1 ├── mongo-orchestration-stop.ps1 ├── server └── setup-configuration ├── setup.py └── tests ├── __init__.py ├── test_container.py ├── test_launch.py ├── test_process.py ├── test_replica_set.py ├── test_replica_sets.py ├── test_servers.py ├── test_sharded_clusters.py └── test_singleton.py /.evergreen/config.yml: -------------------------------------------------------------------------------- 1 | ######################################## 2 | # Evergreen Template for MongoDB Drivers 3 | ######################################## 4 | 5 | # When a task that used to pass starts to fail 6 | # Go through all versions that may have been skipped to detect 7 | # when the task started failing 8 | stepback: true 9 | 10 | # Mark a failure as a system/bootstrap failure (purple box) rather then a task 11 | # failure by default. 12 | # Actual testing tasks are marked with `type: test` 13 | command_type: system 14 | 15 | # Protect ourself against rogue test case, or curl gone wild, that runs forever 16 | # Good rule of thumb: the averageish length a task takes, times 5 17 | # That roughly accounts for variable system performance for various buildvariants 18 | exec_timeout_secs: 1800 # 30 minutes is the longest we'll ever run 19 | 20 | # What to do when evergreen hits the timeout (`post:` tasks are run automatically) 21 | timeout: 22 | - command: shell.exec 23 | params: 24 | script: | 25 | ls -la 26 | 27 | functions: 28 | "fetch source": 29 | # Executes git clone and applies the submitted patch, if any 30 | - command: git.get_project 31 | params: 32 | directory: "src" 33 | # Applies the submitted patch, if any 34 | # Deprecated. Should be removed. But still needed for certain agents (ZAP) 35 | - command: git.apply_patch 36 | # Make an evergreen exapanstion file with dynamic values 37 | - command: shell.exec 38 | params: 39 | working_dir: "src" 40 | script: | 41 | # Get the current unique version of this checkout 42 | if [ "${is_patch}" = "true" ]; then 43 | CURRENT_VERSION=$(git describe)-patch-${version_id} 44 | else 45 | CURRENT_VERSION=latest 46 | fi 47 | 48 | # Install MongoDB 49 | wget -qO - https://www.mongodb.org/static/pgp/server-6.0.asc | sudo apt-key add - 50 | echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu jammy/mongodb-org/6.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-6.0.list 51 | sudo apt-get update 52 | sudo apt-get install -y mongodb-org 53 | 54 | export UPLOAD_BUCKET="${project}" 55 | 56 | cat < expansion.yml 57 | CURRENT_VERSION: "$CURRENT_VERSION" 58 | UPLOAD_BUCKET: "$UPLOAD_BUCKET" 59 | PREPARE_SHELL: | 60 | set -o errexit 61 | export UPLOAD_BUCKET="$UPLOAD_BUCKET" 62 | export TMPDIR="/tmp/db" 63 | export PROJECT="${project}" 64 | EOT 65 | # See what we've done 66 | cat expansion.yml 67 | 68 | # Load the expansion file to make an evergreen variable with the current unique version 69 | - command: expansions.update 70 | params: 71 | file: src/expansion.yml 72 | 73 | "upload mongo artifacts": 74 | - command: shell.exec 75 | params: 76 | script: | 77 | ${PREPARE_SHELL} 78 | find /tmp/db -name \*.log -exec sh -c 'x="{}"; mv $x $PWD/out_dir/$(basename $(dirname $x))_$(basename $x)' \; 79 | tar zcvf mongodb-logs.tar.gz -C out_dir/ . 80 | rm -rf out_dir 81 | - command: s3.put 82 | params: 83 | aws_key: ${aws_key} 84 | aws_secret: ${aws_secret} 85 | local_file: mongodb-logs.tar.gz 86 | remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/logs/${task_id}-${execution}-mongodb-logs.tar.gz 87 | bucket: mciuploads 88 | permissions: public-read 89 | content_type: ${content_type|application/x-gzip} 90 | display_name: "mongodb-logs.tar.gz" 91 | 92 | "run tests": 93 | - command: shell.exec 94 | type: test 95 | params: 96 | working_dir: "src" 97 | shell: "bash" 98 | script: | 99 | ${PREPARE_SHELL} 100 | . .venv/bin/activate 101 | pytest ${PYTEST_ARGS} 102 | 103 | "install dependencies": 104 | - command: shell.exec 105 | params: 106 | working_dir: "src" 107 | script: | 108 | ${PREPARE_SHELL} 109 | python3 -m venv .venv 110 | .venv/bin/python -m pip install -e ".[test]" 111 | 112 | pre: 113 | - func: "fetch source" 114 | - func: "install dependencies" 115 | 116 | post: 117 | - func: "upload mongo artifacts" 118 | 119 | tasks: 120 | - name: "test-other" 121 | commands: 122 | - func: "run tests" 123 | vars: 124 | PYTEST_ARGS: "--ignore=tests/test_replica_set.py --ignore=tests/test_replica_sets.py --ignore=tests/test_sharded_clusters.py" 125 | 126 | - name: "test-replica_set" 127 | commands: 128 | - func: "run tests" 129 | vars: 130 | PYTEST_ARGS: "tests/test_replica_set.py" 131 | 132 | - name: "test-replica_sets" 133 | commands: 134 | - func: "run tests" 135 | vars: 136 | PYTEST_ARGS: "tests/test_replica_sets.py" 137 | 138 | - name: "test-sharded_cluster" 139 | commands: 140 | - func: "run tests" 141 | vars: 142 | PYTEST_ARGS: "tests/test_sharded_clusters.py" 143 | 144 | 145 | buildvariants: 146 | 147 | - name: "tests-all" 148 | display_name: "All Tests" 149 | run_on: 150 | - ubuntu2204-small 151 | tasks: 152 | - name: "test-other" 153 | - name: "test-replica_set" 154 | - name: "test-replica_sets" 155 | - name: "test-sharded_cluster" 156 | -------------------------------------------------------------------------------- /.github/workflows/release-python.yml: -------------------------------------------------------------------------------- 1 | name: Python Wheels 2 | 3 | on: 4 | push: 5 | branches: ["master"] 6 | tags: 7 | - "**" 8 | pull_request: 9 | workflow_dispatch: 10 | 11 | concurrency: 12 | group: wheels-${{ github.ref }} 13 | cancel-in-progress: true 14 | 15 | defaults: 16 | run: 17 | shell: bash -eux {0} 18 | 19 | jobs: 20 | 21 | build_dist: 22 | name: Build Distribution Files 23 | runs-on: ubuntu-latest 24 | steps: 25 | - uses: actions/checkout@v4 26 | with: 27 | fetch-depth: 0 28 | persist-credentials: false 29 | 30 | - uses: actions/setup-python@v5 31 | with: 32 | # Build sdist on lowest supported Python 33 | python-version: '3.9' 34 | 35 | - name: Install build 36 | run: | 37 | python -m pip install build 38 | 39 | - name: build the dist files 40 | run: | 41 | python -m build . 42 | 43 | - name: Upload the dist files 44 | uses: actions/upload-artifact@v4 45 | with: 46 | name: dist-${{ github.run_id }} 47 | path: ./dist/*.* 48 | 49 | publish: 50 | # https://packaging.python.org/en/latest/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/#publishing-the-distribution-to-pypi 51 | needs: [build_dist] 52 | if: startsWith(github.ref, 'refs/tags/') 53 | runs-on: ubuntu-latest 54 | environment: release 55 | permissions: 56 | id-token: write 57 | steps: 58 | - name: Download the dists 59 | uses: actions/download-artifact@v4 60 | with: 61 | name: dist-${{ github.run_id }} 62 | path: dist/ 63 | - name: Publish distribution 📦 to PyPI 64 | uses: pypa/gh-action-pypi-publish@release/v1 65 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Tests 2 | 3 | on: 4 | push: 5 | branches: ['master'] 6 | pull_request: 7 | 8 | concurrency: 9 | group: tests-${{ github.ref }} 10 | cancel-in-progress: true 11 | 12 | defaults: 13 | run: 14 | shell: bash -eux {0} 15 | 16 | jobs: 17 | test: 18 | runs-on: ubuntu-22.04 19 | name: Run ${{ matrix.options[0] }} Tests 20 | strategy: 21 | fail-fast: false 22 | matrix: 23 | # Github Actions doesn't support pairing matrix values together, let's improvise 24 | # https://github.com/github/feedback/discussions/7835#discussioncomment-1769026 25 | options: 26 | - ["Other", "--ignore=tests/test_replica_set.py --ignore=tests/test_replica_sets.py --ignore=tests/test_sharded_clusters.py", "3.9"] 27 | - ["Replica Set", "tests/test_replica_set.py", "3.10"] 28 | - ["Replica Sets", "tests/test_replica_sets.py", "3.11"] 29 | - ["Sharded", "tests/test_sharded_clusters.py", "3.13"] 30 | steps: 31 | - uses: actions/checkout@v4 32 | - name: Setup Python 33 | uses: actions/setup-python@v5 34 | with: 35 | python-version: ${{ matrix.options[2] }} 36 | - name: Install MongoDB 37 | run: | 38 | wget -qO - https://www.mongodb.org/static/pgp/server-6.0.asc | sudo apt-key add - 39 | echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu jammy/mongodb-org/6.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-6.0.list 40 | sudo apt-get update 41 | sudo apt-get install -y mongodb-org 42 | - name: Install Dependencies 43 | run: | 44 | pip install -e ".[test]" 45 | - name: Run Tests 46 | run: | 47 | pytest -raXs -v --durations 10 --color=yes ${{ matrix.options[1] }} 48 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled source # 2 | ################### 3 | *.com 4 | *.class 5 | *.dll 6 | *.exe 7 | *.o 8 | *.so 9 | *.pyc 10 | 11 | # Packages # 12 | ############ 13 | # it's better to unpack these files and commit the raw source 14 | # git has its own built in compression methods 15 | *.7z 16 | *.dmg 17 | *.gz 18 | *.iso 19 | *.jar 20 | *.rar 21 | *.tar 22 | *.zip 23 | 24 | # Logs and databases # 25 | ###################### 26 | *.log 27 | *.sql 28 | *.sqlite 29 | 30 | # OS generated files # 31 | ###################### 32 | .DS_Store* 33 | ehthumbs.db 34 | Icon? 35 | Thumbs.db 36 | 37 | # Other # 38 | ###################### 39 | target 40 | work 41 | *.iml 42 | *.idea 43 | *.iws 44 | *.tmp* 45 | 46 | # Pid # 47 | ###################### 48 | *.pid 49 | 50 | # Install Garbage # 51 | ################### 52 | 53 | build 54 | dist 55 | mongo_orchestration.egg-info 56 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ------------------- 2 | Mongo Orchestration 3 | ------------------- 4 | 5 | See the `wiki `__ 6 | for documentation. 7 | 8 | Mongo Orchestration is an HTTP server that provides a REST API for 9 | creating and managing MongoDB configurations on a single host. 10 | 11 | **THIS PROJECT IS FOR TESTING OF MONGODB DRIVERS.** 12 | 13 | Features 14 | -------- 15 | 16 | - Start and stop mongod servers, replica sets, and sharded clusters on the host running mongo-orchestration. 17 | - Add and remove replica set members. 18 | - Add and remove shards and mongos routers. 19 | - Reset replica sets and clusters to restart all members that were 20 | stopped. 21 | - Freeze secondary members of replica sets. 22 | - Retrieve information about MongoDB resources. 23 | - Interaction all through REST interface. 24 | - Launch simple local servers using ``mongo-launch`` CLI tool. 25 | 26 | Requires 27 | -------- 28 | 29 | - `Python >=3.8 `__ 30 | - `bottle>=0.12.7 `__ 31 | - `pymongo>=3.0.2,<4 `__ 32 | - `cheroot>=5.11 `__ 33 | 34 | Installation 35 | ------------ 36 | 37 | The easiest way to install Mongo Orchestration is with `pip `__: 38 | 39 | :: 40 | 41 | pip install mongo-orchestration 42 | 43 | You can also install the development version of Mongo Orchestration 44 | manually: 45 | 46 | :: 47 | 48 | git clone https://github.com/10gen/mongo-orchestration.git 49 | cd mongo-orchestration 50 | pip install . 51 | 52 | Cloning the `repository `__ this way will also give you access to the tests for Mongo Orchestration as well as the ``mo`` script. Note that you may 53 | have to run the above commands with ``sudo``, depending on where you're 54 | installing Mongo Orchestration and what privileges you have. 55 | Installation will place a ``mongo-orchestration`` script on your path. 56 | 57 | Usage 58 | ----- 59 | 60 | :: 61 | 62 | mongo-orchestration [-h] [-f CONFIG] [-e ENV] [--no-fork] [-b BIND IP="localhost"] [-p PORT] 63 | [-s {auto,cheroot,wsgiref}] [--socket-timeout-ms MILLIS] 64 | [--pidfile PIDFILE] [--enable-majority-read-concern] {start,stop,restart} 65 | 66 | 67 | Arguments: 68 | 69 | - **-h** - show help 70 | - **-f, --config** - path to config file 71 | - **-e, --env** - default release to use, as specified in the config 72 | file 73 | - **--no-fork** - run server in foreground 74 | - **-b, --bind** - host on which Mongo Orchestration and subordinate mongo processes should listen for requests. Defaults to "localhost". 75 | - **-s, --server** - HTTP backend to use: one of `auto`, `cheroot`, or `wsgiref`. `auto` 76 | configures bottle to automatically choose an available backend. 77 | - **-p** - port number (8889 by default) 78 | - **--socket-timeout-ms** - socket timeout when connecting to MongoDB servers 79 | - **--pidfile** - location where mongo-orchestration should place its pid file 80 | - **--enable-majority-read-concern** - enable "majority" read concern on server versions that support it. 81 | - **start/stop/restart**: start, stop, or restart the server, 82 | respectively 83 | 84 | In addition, Mongo Orchestration can be influenced by the following environment variables: 85 | 86 | - ``MONGO_ORCHESTRATION_HOME`` - informs the 87 | server where to find the "configurations" directory for presets as well 88 | as where to put the log and pid files. 89 | - ``MONGO_ORCHESTRATION_TMP`` - the temporary folder root location. 90 | - ``MO_HOST`` - the server host (``localhost`` by default) 91 | - ``MO_PORT`` - the server port (8889 by default) 92 | - ``MONGO_ORCHESTRATION_CLIENT_CERT`` - set the client certificate file 93 | to be used by ``mongo-orchestration``. 94 | 95 | Examples 96 | ~~~~~~~~ 97 | 98 | ``mongo-orchestration start`` 99 | 100 | Starts Mongo Orchestration as service on port 8889. 101 | 102 | ``mongo-orchestration stop`` 103 | 104 | Stop the server. 105 | 106 | ``mongo-orchestration -f mongo-orchestration.config -e 30-release -p 8888 --no-fork start`` 107 | 108 | Starts Mongo Orchestration on port 8888 using ``30-release`` defined in 109 | ``mongo-orchestration.config``. Stops with *Ctrl+C*. 110 | 111 | If you have installed mongo-orchestration but you're still getting 112 | ``command not found: mongo-orchestration`` this means that the script was 113 | installed to a directory that is not on your ``PATH``. As an alternative use: 114 | 115 | ``python -m mongo_orchestration.server start`` 116 | 117 | Configuration File 118 | ~~~~~~~~~~~~~~~~~~ 119 | 120 | Mongo Orchestration may be given a JSON configuration file with the 121 | ``--config`` option specifying where to find MongoDB binaries. See 122 | `mongo-orchestration.config `__ 123 | for an example. When no configuration file is provided, Mongo 124 | Orchestration uses whatever binaries are on the user's PATH. 125 | 126 | Predefined Configurations 127 | ------------------------- 128 | 129 | Mongo Orchestration has a set of predefined 130 | `configurations `__ 131 | that can be used to start, restart, or stop MongoDB processes. You can 132 | use a tool like ``curl`` to send these files directly to the Mongo 133 | Orchestration server, or use the ``mo`` script in the ``scripts`` 134 | directory (in the `repository `__ only). Some examples: 135 | 136 | - Start a single node without SSL or auth: 137 | 138 | :: 139 | 140 | mo configurations/servers/clean.json start 141 | 142 | - Get the status of a single node without SSL or auth: 143 | 144 | :: 145 | 146 | mo configurations/servers/clean.json status 147 | 148 | - Stop a single node without SSL or auth: 149 | 150 | :: 151 | 152 | mo configurations/servers/clean.json stop 153 | 154 | - Start a replica set with ssl and auth: 155 | 156 | :: 157 | 158 | mo configurations/replica_sets/ssl_auth.json start 159 | 160 | - Use ``curl`` to create a basic sharded cluster with the id 161 | "myCluster": 162 | 163 | :: 164 | 165 | curl -XPUT http://localhost:8889/v1/sharded_clusters/myCluster \ 166 | -d@configurations/sharded_clusters/basic.json 167 | 168 | Note that in order to run the ``mo`` script, you need to be in the same 169 | directory as "configurations". 170 | 171 | **Helpful hint**: You can prettify JSON responses from the server by 172 | piping the response into ``python -m json.tool``, e.g.: 173 | 174 | :: 175 | 176 | $ curl http://localhost:8889/v1/servers/myServer | python -m json.tool 177 | 178 | { 179 | "id": "myServer", 180 | "mongodb_uri": "mongodb://localhost:1025", 181 | "orchestration": "servers", 182 | "procInfo": { 183 | "alive": true, 184 | "name": "mongod", 185 | "optfile": "/var/folders/v9/spc2j6cx3db71l/T/mongo-KHUACD", 186 | "params": { 187 | "dbpath": "/var/folders/v9/spc2j6cx3db71l/T/mongo-vAgYaQ", 188 | "ipv6": true, 189 | "journal": true, 190 | "logappend": true, 191 | "oplogSize": 100, 192 | "port": 1025 193 | }, 194 | "pid": 51320 195 | }, 196 | // etc. 197 | } 198 | 199 | Mongo Launch 200 | ------------ 201 | 202 | The ``mongo-launch`` CLI tool allows you to spin up servers locally 203 | with minimal configuration. 204 | 205 | .. 206 | 207 | mongo-launch --help 208 | Usage: launch.py [single|replica|shard] [ssl] [auth] 209 | 210 | .. 211 | 212 | mongo-orchestration start 213 | mongo-launch replica ssl auth 214 | 215 | Tests 216 | ----- 217 | 218 | In order to run the tests, you should first clone the `repository `__. 219 | 220 | Run all tests 221 | ~~~~~~~~~~~~~ 222 | 223 | ``python -m unittest`` 224 | 225 | Run a test module 226 | ~~~~~~~~~~~~~~~~~ 227 | 228 | ``python -m unittest tests.test_servers`` 229 | 230 | Run a single test case 231 | ~~~~~~~~~~~~~~~~~~~~~~ 232 | 233 | ``python -m unittest tests.test_servers.ServerSSLTestCase`` 234 | 235 | Run a single test method 236 | ~~~~~~~~~~~~~~~~~~~~~~~~ 237 | 238 | ``python -m unittest tests.test_servers.ServerSSLTestCase.test_ssl_auth`` 239 | 240 | Run a single test example for debugging with verbose and immediate stdout output 241 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 242 | 243 | ``python -m unittest -v tests.test_servers.ServerSSLTestCase`` 244 | 245 | Changelog 246 | --------- 247 | 248 | Changes in Version 0.11.0 (2024-12-30) 249 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 250 | 251 | - Allow server daemon to be run as a library in addition to as a cli. 252 | - Add support for ``MONGO_ORCHESTRATION_CLIENT_CERT`` environment variable to set the client certificate file 253 | to be used by ``mongo-orchestration``. 254 | 255 | Changes in Version 0.10.0 (2024-11-21) 256 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 257 | 258 | - Add support for requireApiVersion for standalone clusters and replica sets. 259 | - Drop support for Python 3.8 and add support for Python 3.13. 260 | 261 | Changes in Version 0.9.0 (2043-09-04) 262 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 263 | 264 | - Fix handling of ``enableMajorityReadConcern``. 265 | - Remove 'journal' options for newer mongod ``(>=6.1)``. 266 | - Switch to Hatch build backend. 267 | 268 | Changes in Version 0.8.0 (2023-05-16) 269 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 270 | 271 | - Add ``mongo-launch`` CLI tool. 272 | - Upgrade to PyMongo 4.x and set up GitHub Actions testing. 273 | - Remove support for managing MongoDB 3.4 or earlier servers. 274 | - Remove support for Python 3.5 or earlier. 275 | - Replaced dependency on CherryPy with cheroot. `-s auto` is the new default 276 | and `-s cherrypy` is no longer supported. 277 | - Remove transactionLifetimeLimitSeconds default. 278 | 279 | Changes in Version 0.7.0 (2021-04-06) 280 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 281 | 282 | - Remove support for managing MongoDB 2.4 servers. 283 | - Add support for Python 3.8 and 3.9. 284 | - Add support for MongoDB 4.2 and 4.4. 285 | - Upgrade from pymongo 3.5.1 to 3.X latest. (#284). 286 | - Ensure createUser succeeds on all replica set members. (#282) 287 | - Create admin user with both SCRAM-SHA-256 and SCRAM-SHA-1. (#281) 288 | - Wait for mongo-orchestration server to fully terminate in "stop". (#276) 289 | - Allow starting clusters with enableTestCommands=0. (#269) 290 | - Decrease transactionLifetimeLimitSeconds on 4.2+ by default. (#267) 291 | - Increase maxTransactionLockRequestTimeoutMillis by default. (#270) 292 | - Reduce periodicNoopIntervalSecs for faster driver change stream testing. (#283) 293 | - Enable ztsd compression by default on 4.2+ (#263) 294 | 295 | Changes in Version 0.6.12 (2018-12-14) 296 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 297 | 298 | - Allow running the mongo-orchestration server over IPv6 localhost. (#237) 299 | - Increase default mongodb server logging verbosity. (#255) 300 | - Fixed a bug when shutting down clusters where mongo-orchestration would 301 | hang forever if the server had already exited. (#253) 302 | -------------------------------------------------------------------------------- /mongo-orchestration.config: -------------------------------------------------------------------------------- 1 | { 2 | "releases": { 3 | "27-nightly": "/mnt/jenkins/mongodb/master-nightly/master-nightly-release/bin", 4 | "26-release": "/mnt/jenkins/mongodb/26/26-release/bin", 5 | "24-release": "/mnt/jenkins/mongodb/24/24-release/bin", 6 | "22-release": "/mnt/jenkins/mongodb/22/22-release/bin", 7 | "20-release": "/mnt/jenkins/mongodb/20/20-release/bin" 8 | }, 9 | "last_updated": "2014-08-29 20:57:00.000000" 10 | } 11 | -------------------------------------------------------------------------------- /mongo_orchestration/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # coding=utf-8 3 | # Copyright 2012-2023 MongoDB, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import sys 18 | 19 | from mongo_orchestration._version import __version__ 20 | from mongo_orchestration.servers import Servers 21 | from mongo_orchestration.replica_sets import ReplicaSets 22 | from mongo_orchestration.sharded_clusters import ShardedClusters 23 | 24 | 25 | def set_releases(releases=None, default_release=None): 26 | Servers().set_settings(releases, default_release) 27 | ReplicaSets().set_settings(releases, default_release) 28 | ShardedClusters().set_settings(releases, default_release) 29 | 30 | 31 | def cleanup_storage(*args): 32 | """Clean up processes after SIGTERM or SIGINT is received.""" 33 | ShardedClusters().cleanup() 34 | ReplicaSets().cleanup() 35 | Servers().cleanup() 36 | sys.exit(0) 37 | -------------------------------------------------------------------------------- /mongo_orchestration/_version.py: -------------------------------------------------------------------------------- 1 | __version__ = "0.12.0.dev0" -------------------------------------------------------------------------------- /mongo_orchestration/apps/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2012-2014 MongoDB, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import json 16 | import logging 17 | import sys 18 | import traceback 19 | 20 | from bson import json_util 21 | 22 | from collections import namedtuple 23 | 24 | from bottle import route, response 25 | 26 | sys.path.insert(0, '..') 27 | 28 | from mongo_orchestration.compat import reraise, PY3 29 | from mongo_orchestration.errors import RequestError 30 | 31 | if PY3: 32 | unicode = str 33 | 34 | logger = logging.getLogger(__name__) 35 | 36 | Route = namedtuple('Route', ('path', 'method')) 37 | 38 | 39 | def setup_versioned_routes(routes, version=None): 40 | """Set up routes with a version prefix.""" 41 | prefix = '/' + version if version else "" 42 | for r in routes: 43 | path, method = r 44 | route(prefix + path, method, routes[r]) 45 | 46 | 47 | def send_result(code, result=None): 48 | response.content_type = None 49 | if result is not None and 200 <= code < 300: 50 | # Use json_util.dumps in case the result contains non JSON 51 | # serializable BSON types. 52 | result = json_util.dumps(result) 53 | response.content_type = "application/json" 54 | 55 | logger.debug("send_result({code})".format(**locals())) 56 | response.status = code 57 | return result 58 | 59 | 60 | def error_wrap(f): 61 | def wrap(*arg, **kwd): 62 | f_name = f.__name__ 63 | logger.debug("{f_name}({arg}, {kwd})".format( 64 | f_name=f_name, arg=arg, kwd=kwd)) 65 | try: 66 | return f(*arg, **kwd) 67 | except Exception: 68 | logger.exception(str(f)) 69 | err_message = ''.join(traceback.format_exception(*sys.exc_info())) 70 | return send_result(500, err_message) 71 | 72 | return wrap 73 | 74 | 75 | def get_json(req_body): 76 | try: 77 | str_body = req_body.read() 78 | if str_body: 79 | str_body = str_body.decode('utf-8') 80 | return json.loads(str_body) 81 | return {} 82 | except ValueError: 83 | exc_type, exc_value, exc_tb = sys.exc_info() 84 | message = "Could not parse the JSON sent to the server." 85 | reraise(RequestError, message, exc_tb) 86 | -------------------------------------------------------------------------------- /mongo_orchestration/apps/links.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # coding=utf-8 3 | # Copyright 2014 MongoDB, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | """Utilities for building links for discoverable API.""" 17 | 18 | 19 | _BASE_LINKS = { 20 | 'get-releases': {'rel': 'get-releases', 'href': '/v1/releases', 21 | 'method': 'GET'}, 22 | 'service': {'rel': 'service', 'href': '/v1', 'method': 'GET'} 23 | } 24 | _SERVER_LINKS = { 25 | 'get-servers': {'method': 'GET', 'href': '{servers_href}'}, 26 | 'add-server': {'method': 'POST', 'href': '{servers_href}'}, 27 | 'add-server-by-id': {'method': 'PUT', 'href': '{servers_href}/{server_id}'}, 28 | 'delete-server': {'method': 'DELETE', 'href': '{servers_href}/{server_id}'}, 29 | 'get-server-info': {'method': 'GET', 'href': '{servers_href}/{server_id}'}, 30 | 'server-command': {'method': 'POST', 'href': '{servers_href}/{server_id}', 31 | 'template': {'action': ""}, 32 | 'actions': ['start', 'stop', 'restart', 'freeze', 33 | 'stepdown', 'reset']} 34 | } 35 | _REPLICA_SET_LINKS = { 36 | 'get-replica-set-info': { 37 | 'method': 'GET', 'href': '{repls_href}/{repl_id}'}, 38 | 'get-replica-sets': {'method': 'GET', 'href': '{repls_href}'}, 39 | 'add-replica-set': {'method': 'POST', 'href': '{repls_href}'}, 40 | 'add-replica-set-by-id': { 41 | 'method': 'PUT', 'href': '{repls_href}/{repl_id}'}, 42 | 'delete-replica-set': { 43 | 'href': '{repls_href}/{repl_id}', 'method': 'DELETE'}, 44 | 'replica-set-command': { 45 | 'href': '{repls_href}/{repl_id}', 'method': 'POST', 46 | 'actions': ['reset'], 'template': {'action': ''}}, 47 | 'get-replica-set-members': { 48 | 'method': 'GET', 'href': '{repls_href}/{repl_id}/members'}, 49 | 'add-replica-set-member': { 50 | 'method': 'POST', 'href': '{repls_href}/{repl_id}/members'}, 51 | 'delete-replica-set-member': { 52 | 'method': 'DELETE', 53 | 'href': '{repls_href}/{repl_id}/members/{member_id}'}, 54 | 'update-replica-set-member-config': { 55 | 'method': 'PATCH', 56 | 'href': '{repls_href}/{repl_id}/members/{member_id}'}, 57 | 'get-replica-set-member-info': { 58 | 'method': 'GET', 'href': '{repls_href}/{repl_id}/members/{member_id}'}, 59 | 'get-replica-set-secondaries': { 60 | 'method': 'GET', 'href': '{repls_href}/{repl_id}/secondaries'}, 61 | 'get-replica-set-arbiters': { 62 | 'method': 'GET', 'href': '{repls_href}/{repl_id}/arbiters'}, 63 | 'get-replica-set-hidden-members': { 64 | 'method': 'GET', 'href': '{repls_href}/{repl_id}/hidden'}, 65 | 'get-replica-set-passive-members': { 66 | 'method': 'GET', 'href': '{repls_href}/{repl_id}/passives'}, 67 | 'get-replica-set-servers': { 68 | 'method': 'GET', 'href': '{repls_href}/{repl_id}/servers'}, 69 | 'get-replica-set-primary': { 70 | 'method': 'GET', 'href': '{repls_href}/{repl_id}/primary'}, 71 | } 72 | _SHARDED_CLUSTER_LINKS = { 73 | 'add-sharded-cluster': {'method': 'POST', 'href': '{clusters_href}'}, 74 | 'get-sharded-clusters': {'method': 'GET', 'href': '{clusters_href}'}, 75 | 'get-sharded-cluster-info': { 76 | 'method': 'GET', 'href': '{clusters_href}/{cluster_id}'}, 77 | 'sharded-cluster-command': { 78 | 'method': 'POST', 'href': '{clusters_href}/{cluster_id}'}, 79 | 'add-sharded-cluster-by-id': { 80 | 'method': 'PUT', 'href': '{clusters_href}/{cluster_id}'}, 81 | 'delete-sharded-cluster': { 82 | 'method': 'DELETE', 'href': '{clusters_href}/{cluster_id}'}, 83 | 'add-shard': { 84 | 'method': 'POST', 'href': '{clusters_href}/{cluster_id}/shards'}, 85 | 'get-shards': { 86 | 'method': 'GET', 'href': '{clusters_href}/{cluster_id}/shards'}, 87 | 'get-configsvrs': { 88 | 'method': 'GET', 'href': '{clusters_href}/{cluster_id}/configsvrs'}, 89 | 'get-routers': { 90 | 'method': 'GET', 'href': '{clusters_href}/{cluster_id}/routers'}, 91 | 'add-router': { 92 | 'method': 'POST', 'href': '{clusters_href}/{cluster_id}/routers'}, 93 | 'delete-router': { 94 | 'method': 'DELETE', 95 | 'href': '{clusters_href}/{cluster_id}/routers/{router_id}'}, 96 | 'get-shard-info': { 97 | 'method': 'GET', 98 | 'href': '{clusters_href}/{cluster_id}/shards/{shard_id}'}, 99 | 'delete-shard': { 100 | 'method': 'DELETE', 101 | 'href': '{clusters_href}/{cluster_id}/shards/{shard_id}'} 102 | } 103 | 104 | 105 | def base_link(rel, self_rel=False): 106 | """Helper for getting a link document under the API root, given a rel.""" 107 | link = _BASE_LINKS[rel].copy() 108 | link['rel'] = 'self' if self_rel else rel 109 | return link 110 | 111 | 112 | def all_base_links(rel_to=None): 113 | """Get a list of all links to be included to base (/) API requests.""" 114 | links = [ 115 | base_link('get-releases'), 116 | base_link('service'), 117 | server_link('get-servers'), 118 | server_link('add-server'), 119 | replica_set_link('add-replica-set'), 120 | replica_set_link('get-replica-sets'), 121 | sharded_cluster_link('add-sharded-cluster'), 122 | sharded_cluster_link('get-sharded-clusters') 123 | ] 124 | for link in links: 125 | if link['rel'] == rel_to: 126 | link['rel'] = 'self' 127 | return links 128 | 129 | 130 | def server_link(rel, server_id=None, self_rel=False): 131 | """Helper for getting a Server link document, given a rel.""" 132 | servers_href = '/v1/servers' 133 | link = _SERVER_LINKS[rel].copy() 134 | link['href'] = link['href'].format(**locals()) 135 | link['rel'] = 'self' if self_rel else rel 136 | return link 137 | 138 | 139 | def all_server_links(server_id, rel_to=None): 140 | """Get a list of all links to be included with Servers.""" 141 | return [ 142 | server_link(rel, server_id, self_rel=(rel == rel_to)) 143 | for rel in ('delete-server', 'get-server-info', 'server-command') 144 | ] 145 | 146 | 147 | def replica_set_link(rel, repl_id=None, member_id=None, self_rel=False): 148 | """Helper for getting a ReplicaSet link document, given a rel.""" 149 | repls_href = '/v1/replica_sets' 150 | link = _REPLICA_SET_LINKS[rel].copy() 151 | link['href'] = link['href'].format(**locals()) 152 | link['rel'] = 'self' if self_rel else rel 153 | return link 154 | 155 | 156 | def all_replica_set_links(rs_id, rel_to=None): 157 | """Get a list of all links to be included with replica sets.""" 158 | return [ 159 | replica_set_link(rel, rs_id, self_rel=(rel == rel_to)) 160 | for rel in ( 161 | 'get-replica-set-info', 162 | 'delete-replica-set', 'replica-set-command', 163 | 'get-replica-set-members', 'add-replica-set-member', 164 | 'get-replica-set-secondaries', 'get-replica-set-primary', 165 | 'get-replica-set-arbiters', 'get-replica-set-hidden-members', 166 | 'get-replica-set-passive-members', 'get-replica-set-servers' 167 | ) 168 | ] 169 | 170 | 171 | def sharded_cluster_link(rel, cluster_id=None, 172 | shard_id=None, router_id=None, self_rel=False): 173 | """Helper for getting a ShardedCluster link document, given a rel.""" 174 | clusters_href = '/v1/sharded_clusters' 175 | link = _SHARDED_CLUSTER_LINKS[rel].copy() 176 | link['href'] = link['href'].format(**locals()) 177 | link['rel'] = 'self' if self_rel else rel 178 | return link 179 | 180 | 181 | def all_sharded_cluster_links(cluster_id, shard_id=None, 182 | router_id=None, rel_to=None): 183 | """Get a list of all links to be included with ShardedClusters.""" 184 | return [ 185 | sharded_cluster_link(rel, cluster_id, shard_id, router_id, 186 | self_rel=(rel == rel_to)) 187 | for rel in ( 188 | 'get-sharded-clusters', 'get-sharded-cluster-info', 189 | 'sharded-cluster-command', 'delete-sharded-cluster', 190 | 'add-shard', 'get-shards', 'get-configsvrs', 191 | 'get-routers', 'add-router' 192 | ) 193 | ] 194 | -------------------------------------------------------------------------------- /mongo_orchestration/apps/replica_sets.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # coding=utf-8 3 | # Copyright 2012-2014 MongoDB, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import logging 18 | import sys 19 | 20 | from bottle import request, run 21 | 22 | sys.path.insert(0, '..') 23 | 24 | from mongo_orchestration.apps import (error_wrap, get_json, Route, 25 | send_result, setup_versioned_routes) 26 | from mongo_orchestration.apps.links import ( 27 | replica_set_link, server_link, all_replica_set_links, 28 | sharded_cluster_link, base_link) 29 | from mongo_orchestration.common import * 30 | from mongo_orchestration.errors import RequestError 31 | from mongo_orchestration.replica_sets import ReplicaSets 32 | 33 | logger = logging.getLogger(__name__) 34 | 35 | 36 | def _rs_create(params): 37 | rs_id = ReplicaSets().create(params) 38 | result = ReplicaSets().info(rs_id) 39 | result['links'] = all_replica_set_links(rs_id) 40 | # Add GET link to corresponding Server resource. 41 | for member in result['members']: 42 | member['links'] = [ 43 | server_link('get-server-info', server_id=member['server_id']), 44 | replica_set_link( 45 | 'get-replica-set-member-info', rs_id, member['_id']) 46 | ] 47 | return result 48 | 49 | 50 | def _build_member_links(rs_id, member_doc): 51 | server_id = member_doc['server_id'] 52 | member_id = member_doc['_id'] 53 | member_links = [ 54 | replica_set_link('get-replica-set-member-info', rs_id, member_id), 55 | replica_set_link('delete-replica-set-member', rs_id, member_id), 56 | replica_set_link('update-replica-set-member-config', 57 | rs_id, member_id), 58 | server_link('get-server-info', server_id=server_id) 59 | ] 60 | return member_links 61 | 62 | 63 | def _build_member_parent_links(rs_id, rel_self=None): 64 | return [ 65 | replica_set_link(rel, rs_id, self_rel=(rel == rel_self)) 66 | for rel in ( 67 | 'get-replica-set-primary', 'get-replica-set-info', 68 | 'get-replica-set-members', 'get-replica-set-secondaries', 69 | 'get-replica-set-arbiters', 'get-replica-set-hidden-members', 70 | 'get-replica-set-passive-members', 'get-replica-set-servers' 71 | ) 72 | ] 73 | 74 | 75 | @error_wrap 76 | def rs_create(): 77 | logger.debug("rs_create()") 78 | data = get_json(request.body) 79 | data = preset_merge(data, 'replica_sets') 80 | result = _rs_create(data) 81 | result['links'].extend([ 82 | base_link('service'), 83 | base_link('get-releases'), 84 | sharded_cluster_link('get-sharded-clusters'), 85 | replica_set_link('get-replica-sets'), 86 | replica_set_link('add-replica-set', self_rel=True), 87 | server_link('get-servers') 88 | ]) 89 | return send_result(200, result) 90 | 91 | 92 | @error_wrap 93 | def rs_list(): 94 | logger.debug("rs_list()") 95 | replica_sets = [] 96 | for rs_id in ReplicaSets(): 97 | repl_info = {'id': rs_id} 98 | repl_info['links'] = all_replica_set_links(rs_id, 'get-replica-sets') 99 | replica_sets.append(repl_info) 100 | response = {'links': [ 101 | base_link('service'), 102 | base_link('get-releases'), 103 | sharded_cluster_link('get-sharded-clusters'), 104 | replica_set_link('get-replica-sets', self_rel=True), 105 | replica_set_link('add-replica-set'), 106 | server_link('get-servers') 107 | ]} 108 | response['replica_sets'] = replica_sets 109 | return send_result(200, response) 110 | 111 | 112 | @error_wrap 113 | def rs_info(rs_id): 114 | logger.debug("rs_info({rs_id})".format(**locals())) 115 | if rs_id not in ReplicaSets(): 116 | return send_result(404) 117 | result = ReplicaSets().info(rs_id) 118 | result['links'] = all_replica_set_links(rs_id, 'get-replica-set-info') 119 | for member in result['members']: 120 | member['links'] = [ 121 | server_link('get-server-info', server_id=member['server_id']), 122 | replica_set_link( 123 | 'get-replica-set-member-info', rs_id, member['_id']) 124 | ] 125 | return send_result(200, result) 126 | 127 | 128 | @error_wrap 129 | def rs_command(rs_id): 130 | logger.debug("rs_command({rs_id})".format(**locals())) 131 | if rs_id not in ReplicaSets(): 132 | return send_result(404) 133 | command = get_json(request.body).get('action') 134 | if command is None: 135 | raise RequestError('Expected body with an {"action": ...}.') 136 | result = { 137 | 'command_result': ReplicaSets().command(rs_id, command), 138 | 'links': all_replica_set_links(rs_id, 'replica-set-command') 139 | } 140 | result['links'].append( 141 | replica_set_link('replica-set-command', self_rel=True)) 142 | return send_result(200, result) 143 | 144 | 145 | @error_wrap 146 | def rs_create_by_id(rs_id): 147 | logger.debug("rs_create_by_id()") 148 | data = get_json(request.body) 149 | data = preset_merge(data, 'replica_sets') 150 | data['id'] = rs_id 151 | result = _rs_create(data) 152 | result['links'].extend([ 153 | base_link('service'), 154 | base_link('get-releases'), 155 | sharded_cluster_link('get-sharded-clusters'), 156 | replica_set_link('get-replica-sets'), 157 | replica_set_link('add-replica-set'), 158 | replica_set_link('add-replica-set-by-id', rs_id, self_rel=True), 159 | server_link('get-servers') 160 | ]) 161 | return send_result(200, result) 162 | 163 | 164 | @error_wrap 165 | def rs_del(rs_id): 166 | logger.debug("rs_del({rs_id})".format(**locals())) 167 | if rs_id not in ReplicaSets(): 168 | return send_result(404) 169 | result = ReplicaSets().remove(rs_id) 170 | return send_result(204, result) 171 | 172 | 173 | @error_wrap 174 | def member_add(rs_id): 175 | logger.debug("member_add({rs_id})".format(**locals())) 176 | if rs_id not in ReplicaSets(): 177 | return send_result(404) 178 | data = get_json(request.body) 179 | member_id = ReplicaSets().member_add(rs_id, data) 180 | result = ReplicaSets().member_info(rs_id, member_id) 181 | result['links'] = _build_member_links(rs_id, result) 182 | result['links'].extend( 183 | _build_member_parent_links(rs_id, 'add-replica-set-member')) 184 | result['links'].append( 185 | replica_set_link('add-replica-set-member', self_rel=True)) 186 | return send_result(200, result) 187 | 188 | 189 | @error_wrap 190 | def members(rs_id): 191 | logger.debug("members({rs_id})".format(**locals())) 192 | if rs_id not in ReplicaSets(): 193 | return send_result(404) 194 | member_docs = [] 195 | for member_info in ReplicaSets().members(rs_id): 196 | member_info['links'] = _build_member_links(rs_id, member_info) 197 | member_docs.append(member_info) 198 | result = { 199 | 'members': member_docs, 200 | 'links': _build_member_parent_links(rs_id, 'get-replica-set-members') 201 | } 202 | return send_result(200, result) 203 | 204 | 205 | @error_wrap 206 | def secondaries(rs_id): 207 | logger.debug("secondaries({rs_id})".format(**locals())) 208 | if rs_id not in ReplicaSets(): 209 | return send_result(404) 210 | secondary_docs = [] 211 | for secondary_info in ReplicaSets().secondaries(rs_id): 212 | secondary_info['links'] = _build_member_links(rs_id, secondary_info) 213 | secondary_docs.append(secondary_info) 214 | result = { 215 | 'secondaries': secondary_docs, 216 | 'links': _build_member_parent_links( 217 | rs_id, 'get-replica-set-secondaries') 218 | } 219 | return send_result(200, result) 220 | 221 | 222 | @error_wrap 223 | def arbiters(rs_id): 224 | logger.debug("arbiters({rs_id})".format(**locals())) 225 | if rs_id not in ReplicaSets(): 226 | return send_result(404) 227 | arbiter_docs = [] 228 | for arbiter_info in ReplicaSets().arbiters(rs_id): 229 | arbiter_info['links'] = _build_member_links(rs_id, arbiter_info) 230 | arbiter_docs.append(arbiter_info) 231 | result = { 232 | 'arbiters': arbiter_docs, 233 | 'links': _build_member_parent_links(rs_id, 'get-replica-set-arbiters') 234 | } 235 | return send_result(200, result) 236 | 237 | 238 | @error_wrap 239 | def hidden(rs_id): 240 | logger.debug("hidden({rs_id})".format(**locals())) 241 | if rs_id not in ReplicaSets(): 242 | return send_result(404) 243 | hidden_docs = [] 244 | for hidden_info in ReplicaSets().hidden(rs_id): 245 | hidden_info['links'] = _build_member_links(rs_id, hidden_info) 246 | hidden_docs.append(hidden_info) 247 | result = { 248 | 'hidden': hidden_docs, 249 | 'links': _build_member_parent_links( 250 | rs_id, 'get-replica-set-hidden-members') 251 | } 252 | return send_result(200, result) 253 | 254 | 255 | @error_wrap 256 | def passives(rs_id): 257 | logger.debug("passives({rs_id})".format(**locals())) 258 | if rs_id not in ReplicaSets(): 259 | return send_result(404) 260 | passive_docs = [] 261 | for passive_info in ReplicaSets().passives(rs_id): 262 | passive_info['links'] = _build_member_links(rs_id, passive_info) 263 | passive_docs.append(passive_info) 264 | result = { 265 | 'passives': passive_docs, 266 | 'links': _build_member_parent_links( 267 | rs_id, 'get-replica-set-passive-members') 268 | } 269 | return send_result(200, result) 270 | 271 | 272 | @error_wrap 273 | def servers(rs_id): 274 | logger.debug("hosts({rs_id})".format(**locals())) 275 | if rs_id not in ReplicaSets(): 276 | return send_result(404) 277 | server_docs = [] 278 | for server_info in ReplicaSets().servers(rs_id): 279 | server_info['links'] = _build_member_links(rs_id, server_info) 280 | server_docs.append(server_info) 281 | result = { 282 | 'servers': server_docs, 283 | 'links': _build_member_parent_links(rs_id, 'get-replica-set-servers') 284 | } 285 | return send_result(200, result) 286 | 287 | 288 | @error_wrap 289 | def rs_member_primary(rs_id): 290 | logger.debug("rs_member_primary({rs_id})".format(**locals())) 291 | if rs_id not in ReplicaSets(): 292 | return send_result(404) 293 | result = ReplicaSets().primary(rs_id) 294 | result['links'] = _build_member_links(rs_id, result) 295 | result['links'].extend( 296 | _build_member_parent_links(rs_id, 'get-replica-set-primary')) 297 | return send_result(200, result) 298 | 299 | 300 | @error_wrap 301 | def member_info(rs_id, member_id): 302 | logger.debug("member_info({rs_id}, {member_id})".format(**locals())) 303 | member_id = int(member_id) 304 | if rs_id not in ReplicaSets(): 305 | return send_result(404) 306 | result = ReplicaSets().member_info(rs_id, member_id) 307 | result['links'] = _build_member_links(rs_id, result) 308 | result['links'].extend(_build_member_parent_links(rs_id)) 309 | return send_result(200, result) 310 | 311 | 312 | @error_wrap 313 | def member_del(rs_id, member_id): 314 | logger.debug("member_del({rs_id}), {member_id}".format(**locals())) 315 | member_id = int(member_id) 316 | if rs_id not in ReplicaSets(): 317 | return send_result(404) 318 | result = ReplicaSets().member_del(rs_id, member_id) 319 | return send_result(204, result) 320 | 321 | 322 | @error_wrap 323 | def member_update(rs_id, member_id): 324 | logger.debug("member_update({rs_id}, {member_id})".format(**locals())) 325 | member_id = int(member_id) 326 | if rs_id not in ReplicaSets(): 327 | return send_result(404) 328 | data = get_json(request.body) 329 | ReplicaSets().member_update(rs_id, member_id, data) 330 | result = ReplicaSets().member_info(rs_id, member_id) 331 | result['links'] = _build_member_links(rs_id, result) 332 | result['links'].extend(_build_member_parent_links(rs_id)) 333 | result['links'].append(replica_set_link( 334 | 'update-replica-set-member-config', rs_id, member_id, self_rel=True)) 335 | return send_result(200, result) 336 | 337 | 338 | ROUTES = { 339 | Route('/replica_sets', method='POST'): rs_create, 340 | Route('/replica_sets', method='GET'): rs_list, 341 | Route('/replica_sets/', method='GET'): rs_info, 342 | Route('/replica_sets/', method='POST'): rs_command, 343 | Route('/replica_sets/', method='PUT'): rs_create_by_id, 344 | Route('/replica_sets/', method='DELETE'): rs_del, 345 | Route('/replica_sets//members', method='POST'): member_add, 346 | Route('/replica_sets//members', method='GET'): members, 347 | Route('/replica_sets//secondaries', method='GET'): secondaries, 348 | Route('/replica_sets//arbiters', method='GET'): arbiters, 349 | Route('/replica_sets//hidden', method='GET'): hidden, 350 | Route('/replica_sets//passives', method='GET'): passives, 351 | Route('/replica_sets//servers', method='GET'): servers, 352 | Route('/replica_sets//primary', method='GET'): rs_member_primary, 353 | Route('/replica_sets//members/', 354 | method='GET'): member_info, 355 | Route('/replica_sets//members/', 356 | method='DELETE'): member_del, 357 | Route('/replica_sets//members/', 358 | method='PATCH'): member_update 359 | } 360 | 361 | setup_versioned_routes(ROUTES, version='v1') 362 | # Assume v1 if no version is specified. 363 | setup_versioned_routes(ROUTES) 364 | 365 | 366 | if __name__ == '__main__': 367 | rs = ReplicaSets() 368 | rs.set_settings() 369 | run(host='localhost', port=8889, debug=True, reloader=False) 370 | -------------------------------------------------------------------------------- /mongo_orchestration/apps/servers.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # coding=utf-8 3 | # Copyright 2012-2014 MongoDB, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import logging 18 | import sys 19 | 20 | from bottle import request, run 21 | 22 | sys.path.insert(0, '..') 23 | 24 | from mongo_orchestration.apps import (error_wrap, get_json, Route, 25 | send_result, setup_versioned_routes) 26 | from mongo_orchestration.apps.links import ( 27 | base_link, server_link, all_server_links, all_base_links, 28 | sharded_cluster_link, replica_set_link) 29 | from mongo_orchestration.common import * 30 | from mongo_orchestration.errors import RequestError 31 | from mongo_orchestration.servers import Servers 32 | 33 | 34 | logger = logging.getLogger(__name__) 35 | 36 | 37 | __version__ = '0.9' 38 | 39 | 40 | def _host_create(params): 41 | host_id = params.get('id') 42 | host_id = Servers().create(params['name'], 43 | params.get('procParams', {}), 44 | params.get('sslParams', {}), 45 | params.get('auth_key', ''), 46 | params.get('login', ''), 47 | params.get('password', ''), 48 | params.get('authSource', 'admin'), 49 | params.get('timeout', 300), 50 | params.get('autostart', True), 51 | host_id, 52 | params.get('version', ''), 53 | params.get('requireApiVersion', '')) 54 | result = Servers().info(host_id) 55 | server_id = result['id'] 56 | result['links'] = all_server_links(server_id) 57 | return result 58 | 59 | 60 | @error_wrap 61 | def base_uri(): 62 | logger.debug("base_uri()") 63 | data = { 64 | "service": "mongo-orchestration", 65 | "version": __version__, 66 | "links": all_base_links(rel_to='service') 67 | } 68 | return send_result(200, data) 69 | 70 | 71 | @error_wrap 72 | def releases_list(): 73 | response = { 74 | 'releases': Servers().releases, 75 | 'links': all_base_links(rel_to='get-releases') 76 | } 77 | return send_result(200, response) 78 | 79 | 80 | @error_wrap 81 | def host_create(): 82 | data = get_json(request.body) 83 | data = preset_merge(data, 'servers') 84 | result = _host_create(data) 85 | result['links'].extend([ 86 | base_link('service'), 87 | base_link('get-releases'), 88 | server_link('get-servers'), 89 | server_link('add-server', self_rel=True), 90 | replica_set_link('get-replica-sets'), 91 | sharded_cluster_link('get-sharded-clusters') 92 | ]) 93 | return send_result(200, result) 94 | 95 | 96 | @error_wrap 97 | def host_list(): 98 | logger.debug("host_list()") 99 | servers = [] 100 | for server_id in Servers(): 101 | server_info = {'id': server_id} 102 | server_info['links'] = all_server_links( 103 | server_id, rel_to='get-servers') 104 | servers.append(server_info) 105 | response = {'links': [ 106 | base_link('service'), 107 | base_link('get-releases'), 108 | server_link('get-servers', self_rel=True), 109 | server_link('add-server'), 110 | replica_set_link('get-replica-sets'), 111 | sharded_cluster_link('get-sharded-clusters') 112 | ]} 113 | response['servers'] = servers 114 | return send_result(200, response) 115 | 116 | 117 | @error_wrap 118 | def host_info(host_id): 119 | logger.debug("host_info({host_id})".format(**locals())) 120 | if host_id not in Servers(): 121 | return send_result(404) 122 | result = Servers().info(host_id) 123 | result['links'] = all_server_links(host_id, rel_to='get-server-info') 124 | result['links'].append(server_link('get-servers')) 125 | return send_result(200, result) 126 | 127 | 128 | @error_wrap 129 | def host_create_by_id(host_id): 130 | data = get_json(request.body) 131 | data = preset_merge(data, 'servers') 132 | data['id'] = host_id 133 | result = _host_create(data) 134 | result['links'].extend([ 135 | base_link('service'), 136 | base_link('get-releases'), 137 | server_link('get-servers'), 138 | server_link('add-server'), 139 | server_link('add-server-by-id', host_id, self_rel=True), 140 | replica_set_link('get-replica-sets'), 141 | sharded_cluster_link('get-sharded-clusters') 142 | ]) 143 | return send_result(200, result) 144 | 145 | 146 | @error_wrap 147 | def host_del(host_id): 148 | logger.debug("host_del({host_id})") 149 | if host_id not in Servers(): 150 | return send_result(404) 151 | Servers().remove(host_id) 152 | return send_result(204) 153 | 154 | 155 | @error_wrap 156 | def host_command(host_id): 157 | logger.debug("host_command({host_id})".format(**locals())) 158 | if host_id not in Servers(): 159 | return send_result(404) 160 | command = get_json(request.body).get('action') 161 | if command is None: 162 | raise RequestError('Expected body with an {"action": ...}.') 163 | result = { 164 | 'command_result': Servers().command(host_id, command), 165 | 'links': all_server_links(host_id, rel_to='server-command') 166 | } 167 | result['links'].append(server_link('get-servers')) 168 | return send_result(200, result) 169 | 170 | 171 | ROUTES = { 172 | Route('/', method='GET'): base_uri, 173 | Route('/releases', method='GET'): releases_list, 174 | Route('/servers', method='POST'): host_create, 175 | Route('/servers', method='GET'): host_list, 176 | Route('/servers/', method='GET'): host_info, 177 | Route('/servers/', method='PUT'): host_create_by_id, 178 | Route('/servers/', method='DELETE'): host_del, 179 | Route('/servers/', method='POST'): host_command 180 | } 181 | 182 | setup_versioned_routes(ROUTES, version='v1') 183 | # Assume v1 if no version is specified. 184 | setup_versioned_routes(ROUTES) 185 | 186 | if __name__ == '__main__': 187 | hs = Servers() 188 | hs.set_settings() 189 | run(host='localhost', port=8889, debug=True, reloader=False) 190 | -------------------------------------------------------------------------------- /mongo_orchestration/apps/sharded_clusters.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # coding=utf-8 3 | # Copyright 2012-2014 MongoDB, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import logging 18 | import sys 19 | 20 | from bottle import request, run 21 | 22 | sys.path.insert(0, '..') 23 | 24 | from mongo_orchestration.apps import (error_wrap, get_json, Route, 25 | send_result, setup_versioned_routes) 26 | from mongo_orchestration.apps.links import ( 27 | sharded_cluster_link, all_sharded_cluster_links, base_link, 28 | server_link, replica_set_link) 29 | from mongo_orchestration.common import * 30 | from mongo_orchestration.errors import RequestError 31 | from mongo_orchestration.sharded_clusters import ShardedClusters 32 | 33 | logger = logging.getLogger(__name__) 34 | 35 | 36 | def _server_or_rs_link(shard_doc): 37 | resource_id = shard_doc['_id'] 38 | if shard_doc.get('isReplicaSet'): 39 | return replica_set_link('get-replica-set-info', resource_id) 40 | return server_link('get-server-info', resource_id) 41 | 42 | 43 | def _sh_create(params): 44 | cluster_id = ShardedClusters().create(params) 45 | result = ShardedClusters().info(cluster_id) 46 | result['links'] = all_sharded_cluster_links(cluster_id) 47 | for router in result['routers']: 48 | router['links'] = [ 49 | server_link('get-server-info', server_id=router['id']) 50 | ] 51 | for cfg in result['configsvrs']: 52 | cfg['links'] = [ 53 | server_link('get-server-info', server_id=cfg['id']) 54 | ] 55 | for sh in result['shards']: 56 | sh['links'] = [ 57 | sharded_cluster_link('get-shard-info', cluster_id, sh['id']), 58 | _server_or_rs_link(sh) 59 | ] 60 | return result 61 | 62 | 63 | @error_wrap 64 | def sh_create(): 65 | logger.debug("sh_create()") 66 | data = get_json(request.body) 67 | data = preset_merge(data, 'sharded_clusters') 68 | result = _sh_create(data) 69 | result['links'].extend([ 70 | base_link('service'), 71 | base_link('get-releases'), 72 | sharded_cluster_link('get-sharded-clusters'), 73 | sharded_cluster_link('add-sharded-cluster', self_rel=True), 74 | replica_set_link('get-replica-sets'), 75 | server_link('get-servers') 76 | ]) 77 | return send_result(200, result) 78 | 79 | 80 | @error_wrap 81 | def sh_list(): 82 | logger.debug("sh_list()") 83 | sharded_clusters = [] 84 | for cluster_id in ShardedClusters(): 85 | cluster_info = {'id': cluster_id} 86 | cluster_info['links'] = all_sharded_cluster_links( 87 | cluster_id, rel_to='get-sharded-clusters') 88 | sharded_clusters.append(cluster_info) 89 | response = {'links': [ 90 | base_link('service'), 91 | base_link('get-releases'), 92 | sharded_cluster_link('get-sharded-clusters', self_rel=True), 93 | sharded_cluster_link('add-sharded-cluster'), 94 | replica_set_link('get-replica-sets'), 95 | server_link('get-servers') 96 | ]} 97 | response['sharded_clusters'] = sharded_clusters 98 | return send_result(200, response) 99 | 100 | 101 | @error_wrap 102 | def info(cluster_id): 103 | logger.debug("info({cluster_id})".format(**locals())) 104 | if cluster_id not in ShardedClusters(): 105 | return send_result(404) 106 | result = ShardedClusters().info(cluster_id) 107 | result['links'] = all_sharded_cluster_links( 108 | cluster_id, rel_to='get-sharded-cluster-info') 109 | for router in result['routers']: 110 | router['links'] = [ 111 | server_link('get-server-info', server_id=router['id']) 112 | ] 113 | for cfg in result['configsvrs']: 114 | cfg['links'] = [ 115 | server_link('get-server-info', server_id=cfg['id']) 116 | ] 117 | for sh in result['shards']: 118 | sh['links'] = [ 119 | sharded_cluster_link('get-shard-info', cluster_id, sh['id']), 120 | _server_or_rs_link(sh) 121 | ] 122 | return send_result(200, result) 123 | 124 | 125 | @error_wrap 126 | def sh_command(cluster_id): 127 | logger.debug("sh_command({cluster_id})".format(**locals())) 128 | if cluster_id not in ShardedClusters(): 129 | return send_result(404) 130 | command = get_json(request.body).get('action') 131 | if command is None: 132 | raise RequestError('Expected body with an {"action": ...}.') 133 | result = { 134 | 'command_result': ShardedClusters().command(cluster_id, command), 135 | 'links': all_sharded_cluster_links(cluster_id, 136 | rel_to='sharded-cluster-command') 137 | } 138 | return send_result(200, result) 139 | 140 | 141 | @error_wrap 142 | def sh_create_by_id(cluster_id): 143 | logger.debug("sh_create()") 144 | data = get_json(request.body) 145 | data = preset_merge(data, 'sharded_clusters') 146 | data['id'] = cluster_id 147 | result = _sh_create(data) 148 | result['links'].extend([ 149 | sharded_cluster_link('add-sharded-cluster-by-id', 150 | cluster_id, self_rel=True), 151 | base_link('service'), 152 | base_link('get-releases'), 153 | sharded_cluster_link('get-sharded-clusters'), 154 | sharded_cluster_link('add-sharded-cluster'), 155 | replica_set_link('get-replica-sets'), 156 | server_link('get-servers') 157 | ]) 158 | return send_result(200, result) 159 | 160 | 161 | @error_wrap 162 | def sh_del(cluster_id): 163 | logger.debug("sh_del({cluster_id})".format(**locals())) 164 | if cluster_id not in ShardedClusters(): 165 | return send_result(404) 166 | result = ShardedClusters().remove(cluster_id) 167 | return send_result(204, result) 168 | 169 | 170 | @error_wrap 171 | def shard_add(cluster_id): 172 | logger.debug("shard_add({cluster_id})".format(**locals())) 173 | if cluster_id not in ShardedClusters(): 174 | return send_result(404) 175 | data = get_json(request.body) 176 | result = ShardedClusters().member_add(cluster_id, data) 177 | resource_id = result['_id'] 178 | shard_id = result['id'] 179 | result['links'] = [ 180 | sharded_cluster_link('get-shard-info', cluster_id, shard_id), 181 | sharded_cluster_link('delete-shard', cluster_id, shard_id), 182 | sharded_cluster_link('add-shard', cluster_id, self_rel=True), 183 | sharded_cluster_link('get-sharded-cluster-info', cluster_id), 184 | sharded_cluster_link('get-shards', cluster_id) 185 | ] 186 | result['links'].append(_server_or_rs_link(result)) 187 | return send_result(200, result) 188 | 189 | 190 | @error_wrap 191 | def shards(cluster_id): 192 | logger.debug("shards({cluster_id})".format(**locals())) 193 | if cluster_id not in ShardedClusters(): 194 | return send_result(404) 195 | shard_docs = [] 196 | for shard_info in ShardedClusters().members(cluster_id): 197 | shard_id = shard_info['id'] 198 | resource_id = shard_info['_id'] 199 | shard_info['links'] = [ 200 | sharded_cluster_link('get-shard-info', cluster_id, shard_id), 201 | sharded_cluster_link('delete-shard', cluster_id, shard_id), 202 | sharded_cluster_link('get-sharded-cluster-info', cluster_id), 203 | ] 204 | shard_info['links'].append(_server_or_rs_link(shard_info)) 205 | shard_docs.append(shard_info) 206 | result = { 207 | 'shards': shard_docs, 208 | 'links': [ 209 | sharded_cluster_link('get-sharded-cluster-info', cluster_id), 210 | sharded_cluster_link('get-shards', cluster_id, self_rel=True), 211 | sharded_cluster_link('get-configsvrs', cluster_id), 212 | sharded_cluster_link('get-routers', cluster_id) 213 | ] 214 | } 215 | return send_result(200, result) 216 | 217 | 218 | @error_wrap 219 | def configsvrs(cluster_id): 220 | logger.debug("configsvrs({cluster_id})".format(**locals())) 221 | if cluster_id not in ShardedClusters(): 222 | return send_result(404) 223 | config_docs = [] 224 | for config_info in ShardedClusters().configsvrs(cluster_id): 225 | server_id = config_info['id'] 226 | config_info['links'] = [ 227 | server_link('get-server-info', server_id) 228 | ] 229 | config_docs.append(config_info) 230 | result = { 231 | 'configsvrs': config_docs, 232 | 'links': [ 233 | sharded_cluster_link('get-sharded-cluster-info', cluster_id), 234 | sharded_cluster_link('get-shards', cluster_id), 235 | sharded_cluster_link('get-configsvrs', cluster_id, self_rel=True), 236 | sharded_cluster_link('get-routers', cluster_id) 237 | ] 238 | } 239 | return send_result(200, result) 240 | 241 | 242 | @error_wrap 243 | def routers(cluster_id): 244 | logger.debug("routers({cluster_id})".format(**locals())) 245 | if cluster_id not in ShardedClusters(): 246 | return send_result(404) 247 | router_docs = [] 248 | for router_info in ShardedClusters().routers(cluster_id): 249 | # Server id is the same as router id. 250 | server_id = router_info['id'] 251 | links = [ 252 | sharded_cluster_link('delete-router', 253 | cluster_id, router_id=server_id), 254 | server_link('get-server-info', server_id) 255 | ] 256 | router_info['links'] = links 257 | router_docs.append(router_info) 258 | result = { 259 | 'routers': router_docs, 260 | 'links': [ 261 | sharded_cluster_link('get-sharded-cluster-info', cluster_id), 262 | sharded_cluster_link('get-shards', cluster_id), 263 | sharded_cluster_link('get-configsvrs', cluster_id), 264 | sharded_cluster_link('get-routers', cluster_id, self_rel=True) 265 | ] 266 | } 267 | return send_result(200, result) 268 | 269 | 270 | @error_wrap 271 | def router_add(cluster_id): 272 | logger.debug("router_add({cluster_id})".format(**locals())) 273 | if cluster_id not in ShardedClusters(): 274 | return send_result(404) 275 | data = get_json(request.body) 276 | result = ShardedClusters().router_add(cluster_id, data) 277 | router_id = result['id'] 278 | result['links'] = [ 279 | server_link('get-server-info', router_id), 280 | sharded_cluster_link('add-router', cluster_id, self_rel=True), 281 | sharded_cluster_link('delete-router', cluster_id, router_id=router_id), 282 | sharded_cluster_link('get-sharded-cluster-info', cluster_id), 283 | sharded_cluster_link('get-routers', cluster_id), 284 | ] 285 | return send_result(200, result) 286 | 287 | 288 | @error_wrap 289 | def router_del(cluster_id, router_id): 290 | logger.debug("router_del({cluster_id}), {router_id}".format(**locals())) 291 | if cluster_id not in ShardedClusters(): 292 | return send_result(404) 293 | result = ShardedClusters().router_del(cluster_id, router_id) 294 | return send_result(204, result) 295 | 296 | 297 | @error_wrap 298 | def shard_info(cluster_id, shard_id): 299 | logger.debug("shard_info({cluster_id}, {shard_id})".format(**locals())) 300 | if cluster_id not in ShardedClusters(): 301 | return send_result(404) 302 | result = ShardedClusters().member_info(cluster_id, shard_id) 303 | resource_id = result['_id'] 304 | shard_id = result['id'] 305 | result['links'] = [ 306 | sharded_cluster_link( 307 | 'get-shard-info', cluster_id, shard_id, self_rel=True), 308 | sharded_cluster_link('delete-shard', cluster_id, shard_id), 309 | sharded_cluster_link('add-shard', cluster_id), 310 | sharded_cluster_link('get-sharded-cluster-info', cluster_id), 311 | sharded_cluster_link('get-shards', cluster_id) 312 | ] 313 | result['links'].append(_server_or_rs_link(result)) 314 | return send_result(200, result) 315 | 316 | 317 | @error_wrap 318 | def shard_del(cluster_id, shard_id): 319 | logger.debug("member_del({cluster_id}), {shard_id}".format(**locals())) 320 | if cluster_id not in ShardedClusters(): 321 | return send_result(404) 322 | result = ShardedClusters().member_del(cluster_id, shard_id) 323 | return send_result(204, result) 324 | 325 | 326 | ROUTES = { 327 | Route('/sharded_clusters', method='POST'): sh_create, 328 | Route('/sharded_clusters', method='GET'): sh_list, 329 | Route('/sharded_clusters/', method='GET'): info, 330 | Route('/sharded_clusters/', method='POST'): sh_command, 331 | Route('/sharded_clusters/', method='PUT'): sh_create_by_id, 332 | Route('/sharded_clusters/', method='DELETE'): sh_del, 333 | Route('/sharded_clusters//shards', method='POST'): shard_add, 334 | Route('/sharded_clusters//shards', method='GET'): shards, 335 | Route('/sharded_clusters//configsvrs', 336 | method='GET'): configsvrs, 337 | Route('/sharded_clusters//routers', method='GET'): routers, 338 | Route('/sharded_clusters//routers', method='POST'): router_add, 339 | Route('/sharded_clusters//routers/', 340 | method='DELETE'): router_del, 341 | Route('/sharded_clusters//shards/', 342 | method='GET'): shard_info, 343 | Route('/sharded_clusters//shards/', 344 | method='DELETE'): shard_del 345 | } 346 | 347 | setup_versioned_routes(ROUTES, version='v1') 348 | # Assume v1 if no version is specified. 349 | setup_versioned_routes(ROUTES) 350 | 351 | if __name__ == '__main__': 352 | rs = ShardedClusters() 353 | rs.set_settings() 354 | run(host='localhost', port=8889, debug=True, reloader=False) 355 | -------------------------------------------------------------------------------- /mongo_orchestration/common.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # coding=utf-8 3 | # Copyright 2014-2023 MongoDB, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import collections 18 | import copy 19 | import json 20 | import os 21 | import ssl 22 | import stat 23 | import tempfile 24 | 25 | WORK_DIR = os.environ.get('MONGO_ORCHESTRATION_HOME', os.getcwd()) 26 | PID_FILE = os.path.join(WORK_DIR, 'server.pid') 27 | LOG_FILE = os.path.join(WORK_DIR, 'server.log') 28 | TMP_DIR = os.environ.get('MONGO_ORCHESTRATION_TMP') 29 | 30 | LOGGING_FORMAT = '%(asctime)s [%(levelname)s] %(name)s:%(lineno)d - %(message)s' 31 | 32 | DEFAULT_BIND = os.environ.get('MO_HOST', 'localhost') 33 | DEFAULT_PORT = int(os.environ.get('MO_PORT', '8889')) 34 | DEFAULT_SERVER = 'auto' 35 | DEFAULT_SOCKET_TIMEOUT = 20000 # 20 seconds. 36 | 37 | # Username for included client x509 certificate. 38 | DEFAULT_SUBJECT = ( 39 | 'C=US,ST=New York,L=New York City,O=MongoDB,OU=KernelUser,' 40 | 'CN=mongo_orchestration' 41 | ) 42 | DEFAULT_CLIENT_CERT = os.path.join( 43 | os.environ.get( 44 | 'MONGO_ORCHESTRATION_HOME', os.path.dirname(__file__)), 45 | 'lib', 46 | 'client.pem' 47 | ) 48 | DEFAULT_SSL_OPTIONS = { 49 | 'ssl': True, 50 | 'tlsCertificateKeyFile': os.environ.get('MONGO_ORCHESTRATION_CLIENT_CERT', DEFAULT_CLIENT_CERT), 51 | 'tlsAllowInvalidCertificates': True 52 | } 53 | 54 | 55 | class BaseModel(object): 56 | """Base object for Server, ReplicaSet, and ShardedCluster.""" 57 | 58 | _user_role_documents = [ 59 | {'role': 'userAdminAnyDatabase', 'db': 'admin'}, 60 | {'role': 'clusterAdmin', 'db': 'admin'}, 61 | {'role': 'dbAdminAnyDatabase', 'db': 'admin'}, 62 | {'role': 'readWriteAnyDatabase', 'db': 'admin'}, 63 | {'role': 'restore', 'db': 'admin'}, 64 | {'role': 'backup', 'db': 'admin'} 65 | ] 66 | socket_timeout = DEFAULT_SOCKET_TIMEOUT 67 | 68 | @property 69 | def key_file(self): 70 | """Get the path to the key file containig our auth key, or None.""" 71 | if self.auth_key: 72 | key_file_path = os.path.join(orchestration_mkdtemp(), 'key') 73 | with open(key_file_path, 'w') as fd: 74 | fd.write(self.auth_key) 75 | os.chmod(key_file_path, stat.S_IRUSR) 76 | return key_file_path 77 | 78 | def _strip_auth(self, proc_params): 79 | """Remove options from parameters that cause auth to be enabled.""" 80 | params = proc_params.copy() 81 | params.pop("auth", None) 82 | params.pop("clusterAuthMode", None) 83 | return params 84 | 85 | def mongodb_auth_uri(self, hosts): 86 | """Get a connection string with all info necessary to authenticate.""" 87 | parts = ['mongodb://'] 88 | if self.login: 89 | parts.append(self.login) 90 | if self.password: 91 | parts.append(':' + self.password) 92 | parts.append('@') 93 | parts.append(hosts + '/') 94 | if self.login: 95 | parts.append('?authSource=' + self.auth_source) 96 | if self.x509_extra_user: 97 | parts.append('&authMechanism=MONGODB-X509') 98 | return ''.join(parts) 99 | 100 | def _get_server_version(self, client): 101 | return tuple(client.admin.command('buildinfo')['versionArray']) 102 | 103 | def _user_roles(self, client): 104 | server_version_tuple = self._get_server_version(client) 105 | if server_version_tuple < (2, 6): 106 | # MongoDB 2.4 roles are an array of strs like ['clusterAdmin', ...]. 107 | return [role['role'] for role in self._user_role_documents] 108 | return self._user_role_documents 109 | 110 | def _add_users(self, db, mongo_version): 111 | """Add given user, and extra x509 user if necessary.""" 112 | roles = self._user_roles(db.client) 113 | if self.x509_extra_user: 114 | db.command('createUser', DEFAULT_SUBJECT, roles=roles, 115 | writeConcern=db.write_concern.document) 116 | # Fix kwargs to MongoClient. 117 | self.kwargs['tlsCertificateKeyFile'] = DEFAULT_CLIENT_CERT 118 | 119 | # Add secondary user given from request. 120 | create_user(db, mongo_version, self.login, self.password, roles) 121 | 122 | 123 | def create_user(db, mongo_version, user, password, roles): 124 | db.command('createUser', user, pwd=password, roles=roles, 125 | writeConcern=db.write_concern.document) 126 | 127 | 128 | def connected(client): 129 | # Await connection in PyMongo. 130 | client.admin.command('isMaster') 131 | return client 132 | 133 | 134 | def update(d, u): 135 | for k, v in u.items(): 136 | if isinstance(v, collections.Mapping): 137 | r = update(d.get(k, {}), v) 138 | d[k] = r 139 | else: 140 | d[k] = u[k] 141 | return d 142 | 143 | 144 | def preset_merge(data, cluster_type): 145 | preset = data.get('preset', None) 146 | if preset is not None: 147 | base_path = os.environ.get("MONGO_ORCHESTRATION_HOME", 148 | os.path.dirname(__file__)) 149 | path = os.path.join(base_path, 'configurations', cluster_type, preset) 150 | preset_data = {} 151 | with open(path, "r") as preset_file: 152 | preset_data = json.loads(preset_file.read()) 153 | data = update(copy.deepcopy(preset_data), data) 154 | return data 155 | 156 | 157 | def orchestration_mkdtemp(prefix=None): 158 | if TMP_DIR and not os.path.exists(TMP_DIR): 159 | os.makedirs(TMP_DIR) 160 | 161 | kwargs = {} 162 | if prefix is not None: 163 | kwargs['prefix'] = prefix 164 | if TMP_DIR is not None: 165 | kwargs['dir'] = TMP_DIR 166 | 167 | return tempfile.mkdtemp(**kwargs) 168 | 169 | 170 | def ipv6_enabled_single(params): 171 | return params.get('ipv6') 172 | 173 | 174 | def ipv6_enabled_repl(params): 175 | members = params.get('members', []) 176 | return any(m.get('procParams', {}).get('ipv6') for m in members) 177 | 178 | 179 | def ipv6_enabled_repl_single(params): 180 | if 'members' in params: 181 | return ipv6_enabled_repl(params) 182 | else: 183 | # Standalone mongod or mongos 184 | return ipv6_enabled_single(params) 185 | 186 | 187 | def ipv6_enabled_sharded(params): 188 | configs = params.get('configsvrs', []) 189 | routers = params.get('routers', []) 190 | shards = params.get('shards', []) 191 | return (any(ipv6_enabled_repl_single(p) for p in configs) or 192 | any(ipv6_enabled_single(p) for p in routers) or 193 | any(ipv6_enabled_repl_single(p) for p in shards)) 194 | 195 | 196 | def enable_ipv6_single(proc_params): 197 | proc_params.setdefault('ipv6', True) 198 | proc_params.setdefault('bind_ip', '127.0.0.1,::1') 199 | 200 | 201 | def enable_ipv6_repl(params): 202 | if 'members' in params: 203 | members = params['members'] 204 | for m in members: 205 | enable_ipv6_single(m.setdefault('procParams', {})) 206 | else: 207 | # Standalone mongod or mongos 208 | enable_ipv6_single(params.setdefault('procParams', {})) 209 | -------------------------------------------------------------------------------- /mongo_orchestration/compat.py: -------------------------------------------------------------------------------- 1 | # Copyright 2014 MongoDB, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import sys 16 | 17 | PY3 = (sys.version_info[0] == 3) 18 | 19 | if PY3: 20 | def reraise(exctype, value, trace=None): 21 | raise exctype(str(value)).with_traceback(trace) 22 | else: 23 | exec("""def reraise(exctype, value, trace=None): 24 | raise exctype, str(value), trace""") 25 | -------------------------------------------------------------------------------- /mongo_orchestration/configurations/replica_sets/allengines.json: -------------------------------------------------------------------------------- 1 | { 2 | "members": [ 3 | { 4 | "procParams": { 5 | "ipv6": true, 6 | "journal": true, 7 | "storageEngine": "mmapv1", 8 | "noprealloc": true, 9 | "nssize": 1, 10 | "smallfiles": true 11 | }, 12 | "rsParams": { 13 | "priority": 99, 14 | "tags": { 15 | "ordinal": "one", 16 | "dc": "ny", 17 | "engine": "mmapv1" 18 | } 19 | } 20 | }, 21 | { 22 | "procParams": { 23 | "ipv6": true, 24 | "journal": true, 25 | "storageEngine": "wiredTiger" 26 | }, 27 | "rsParams": { 28 | "priority": 1.1, 29 | "tags": { 30 | "ordinal": "two", 31 | "dc": "pa", 32 | "engine": "wiredTiger" 33 | } 34 | } 35 | }, 36 | { 37 | "procParams": { 38 | "ipv6": true, 39 | "journal": true, 40 | "storageEngine": "wiredTiger" 41 | }, 42 | "rsParams": { 43 | "arbiterOnly": true 44 | } 45 | } 46 | ] 47 | } 48 | -------------------------------------------------------------------------------- /mongo_orchestration/configurations/replica_sets/arbiter.json: -------------------------------------------------------------------------------- 1 | { 2 | "members": [ 3 | { 4 | "procParams": { 5 | "ipv6": true, 6 | "journal": true 7 | }, 8 | "rsParams": { 9 | "priority": 99, 10 | "tags": { 11 | "ordinal": "one", 12 | "dc": "ny" 13 | } 14 | } 15 | }, 16 | { 17 | "procParams": { 18 | "ipv6": true, 19 | "journal": true 20 | }, 21 | "rsParams": { 22 | "priority": 1.1, 23 | "tags": { 24 | "ordinal": "two", 25 | "dc": "pa" 26 | } 27 | } 28 | }, 29 | { 30 | "procParams": { 31 | "ipv6": true, 32 | "journal": true 33 | }, 34 | "rsParams": { 35 | "arbiterOnly": true 36 | } 37 | } 38 | ] 39 | } 40 | -------------------------------------------------------------------------------- /mongo_orchestration/configurations/replica_sets/auth.json: -------------------------------------------------------------------------------- 1 | { 2 | "auth_key": "secret", 3 | "id": "repl0", 4 | "login": "bob", 5 | "members": [ 6 | { 7 | "procParams": { 8 | "dbpath": "$DBPATH/db27017", 9 | "ipv6": true, 10 | "logpath": "$LOGPATH/db27017.log", 11 | "journal": true, 12 | "port": 27017 13 | }, 14 | "rsParams": { 15 | "priority": 99, 16 | "tags": { 17 | "ordinal": "one", 18 | "dc": "ny" 19 | } 20 | } 21 | }, 22 | { 23 | "procParams": { 24 | "dbpath": "$DBPATH/db27018", 25 | "ipv6": true, 26 | "logpath": "$LOGPATH/db27018.log", 27 | "journal": true, 28 | "port": 27018 29 | }, 30 | "rsParams": { 31 | "priority": 1.1, 32 | "tags": { 33 | "ordinal": "two", 34 | "dc": "pa" 35 | } 36 | } 37 | }, 38 | { 39 | "procParams": { 40 | "dbpath": "$DBPATH/db27019", 41 | "ipv6": true, 42 | "logpath": "$LOGPATH/27019.log", 43 | "journal": true, 44 | "port": 27019 45 | }, 46 | "rsParams": { 47 | "arbiterOnly": true 48 | } 49 | } 50 | ], 51 | "password": "pwd123" 52 | } 53 | -------------------------------------------------------------------------------- /mongo_orchestration/configurations/replica_sets/basic.json: -------------------------------------------------------------------------------- 1 | { 2 | "members": [ 3 | { 4 | "procParams": { 5 | "ipv6": true, 6 | "journal": true 7 | }, 8 | "rsParams": { 9 | "priority": 99, 10 | "tags": { 11 | "ordinal": "one", 12 | "dc": "ny" 13 | } 14 | } 15 | }, 16 | { 17 | "procParams": { 18 | "ipv6": true, 19 | "journal": true 20 | }, 21 | "rsParams": { 22 | "priority": 1.1, 23 | "tags": { 24 | "ordinal": "two", 25 | "dc": "pa" 26 | } 27 | } 28 | }, 29 | { 30 | "procParams": { 31 | "ipv6": true, 32 | "journal": true 33 | }, 34 | "rsParams": { 35 | "arbiterOnly": true 36 | } 37 | } 38 | ] 39 | } 40 | -------------------------------------------------------------------------------- /mongo_orchestration/configurations/replica_sets/clean.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "repl0", 3 | "members": [ 4 | { 5 | "procParams": { 6 | "dbpath": "$DBPATH/db27017", 7 | "ipv6": true, 8 | "logpath": "$LOGPATH/db27017.log", 9 | "journal": true, 10 | "port": 27017 11 | }, 12 | "rsParams": { 13 | "priority": 99, 14 | "tags": { 15 | "ordinal": "one", 16 | "dc": "ny" 17 | } 18 | } 19 | }, 20 | { 21 | "procParams": { 22 | "dbpath": "$DBPATH/db27018", 23 | "ipv6": true, 24 | "logpath": "$LOGPATH/db27018.log", 25 | "journal": true, 26 | "port": 27018 27 | }, 28 | "rsParams": { 29 | "priority": 1.1, 30 | "tags": { 31 | "ordinal": "two", 32 | "dc": "pa" 33 | } 34 | } 35 | }, 36 | { 37 | "procParams": { 38 | "dbpath": "$DBPATH/db27019", 39 | "ipv6": true, 40 | "logpath": "$LOGPATH/27019.log", 41 | "journal": true, 42 | "port": 27019 43 | }, 44 | "rsParams": { 45 | "arbiterOnly": true 46 | } 47 | } 48 | ] 49 | } 50 | -------------------------------------------------------------------------------- /mongo_orchestration/configurations/replica_sets/mmapv1.json: -------------------------------------------------------------------------------- 1 | { 2 | "members": [ 3 | { 4 | "procParams": { 5 | "ipv6": true, 6 | "journal": true, 7 | "storageEngine": "mmapv1", 8 | "noprealloc": true, 9 | "nssize": 1, 10 | "smallfiles": true 11 | }, 12 | "rsParams": { 13 | "priority": 99, 14 | "tags": { 15 | "ordinal": "one", 16 | "dc": "ny" 17 | } 18 | } 19 | }, 20 | { 21 | "procParams": { 22 | "ipv6": true, 23 | "journal": true, 24 | "storageEngine": "mmapv1", 25 | "noprealloc": true, 26 | "nssize": 1, 27 | "smallfiles": true 28 | }, 29 | "rsParams": { 30 | "priority": 1.1, 31 | "tags": { 32 | "ordinal": "two", 33 | "dc": "pa" 34 | } 35 | } 36 | }, 37 | { 38 | "procParams": { 39 | "ipv6": true, 40 | "journal": true, 41 | "storageEngine": "mmapv1", 42 | "noprealloc": true, 43 | "nssize": 1, 44 | "smallfiles": true 45 | }, 46 | "rsParams": { 47 | "arbiterOnly": true 48 | } 49 | } 50 | ] 51 | } 52 | -------------------------------------------------------------------------------- /mongo_orchestration/configurations/replica_sets/ssl.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "repl0", 3 | "members": [ 4 | { 5 | "procParams": { 6 | "dbpath": "$DBPATH/db27017", 7 | "ipv6": true, 8 | "logpath": "$LOGPATH/db27017.log", 9 | "journal": true, 10 | "port": 27017 11 | }, 12 | "rsParams": { 13 | "priority": 99, 14 | "tags": { 15 | "ordinal": "one", 16 | "dc": "ny" 17 | } 18 | } 19 | }, 20 | { 21 | "procParams": { 22 | "dbpath": "$DBPATH/db27018", 23 | "ipv6": true, 24 | "logpath": "$LOGPATH/db27018.log", 25 | "journal": true, 26 | "port": 27018 27 | }, 28 | "rsParams": { 29 | "priority": 1.1, 30 | "tags": { 31 | "ordinal": "two", 32 | "dc": "pa" 33 | } 34 | } 35 | }, 36 | { 37 | "procParams": { 38 | "dbpath": "$DBPATH/db27019", 39 | "ipv6": true, 40 | "logpath": "$LOGPATH/27019.log", 41 | "journal": true, 42 | "port": 27019 43 | }, 44 | "rsParams": { 45 | "arbiterOnly": true 46 | } 47 | } 48 | ], 49 | "sslParams": { 50 | "sslCAFile": "$SSL_FILES/ca.pem", 51 | "sslOnNormalPorts": true, 52 | "sslPEMKeyFile": "$SSL_FILES/server.pem", 53 | "sslWeakCertificateValidation": true 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /mongo_orchestration/configurations/replica_sets/ssl_auth.json: -------------------------------------------------------------------------------- 1 | { 2 | "auth_key": "secret", 3 | "id": "repl0", 4 | "login": "bob", 5 | "members": [ 6 | { 7 | "procParams": { 8 | "dbpath": "$DBPATH/db27017", 9 | "ipv6": true, 10 | "logpath": "$LOGPATH/db27017.log", 11 | "journal": true, 12 | "port": 27017 13 | }, 14 | "rsParams": { 15 | "priority": 99, 16 | "tags": { 17 | "ordinal": "one", 18 | "dc": "ny" 19 | } 20 | } 21 | }, 22 | { 23 | "procParams": { 24 | "dbpath": "$DBPATH/db27018", 25 | "ipv6": true, 26 | "logpath": "$LOGPATH/db27018.log", 27 | "journal": true, 28 | "port": 27018 29 | }, 30 | "rsParams": { 31 | "priority": 1.1, 32 | "tags": { 33 | "ordinal": "two", 34 | "dc": "pa" 35 | } 36 | } 37 | }, 38 | { 39 | "procParams": { 40 | "dbpath": "$DBPATH/db27019", 41 | "ipv6": true, 42 | "logpath": "$LOGPATH/27019.log", 43 | "journal": true, 44 | "port": 27019 45 | }, 46 | "rsParams": { 47 | "arbiterOnly": true 48 | } 49 | } 50 | ], 51 | "password": "pwd123", 52 | "sslParams": { 53 | "sslCAFile": "$SSL_FILES/ca.pem", 54 | "sslOnNormalPorts": true, 55 | "sslPEMKeyFile": "$SSL_FILES/server.pem", 56 | "sslWeakCertificateValidation": true 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /mongo_orchestration/configurations/replica_sets/wiredtiger.json: -------------------------------------------------------------------------------- 1 | { 2 | "members": [ 3 | { 4 | "procParams": { 5 | "ipv6": true, 6 | "journal": true, 7 | "storageEngine": "wiredTiger" 8 | }, 9 | "rsParams": { 10 | "priority": 99, 11 | "tags": { 12 | "ordinal": "one", 13 | "dc": "ny" 14 | } 15 | } 16 | }, 17 | { 18 | "procParams": { 19 | "ipv6": true, 20 | "journal": true, 21 | "storageEngine": "wiredTiger" 22 | }, 23 | "rsParams": { 24 | "priority": 1.1, 25 | "tags": { 26 | "ordinal": "two", 27 | "dc": "pa" 28 | } 29 | } 30 | }, 31 | { 32 | "procParams": { 33 | "ipv6": true, 34 | "journal": true, 35 | "storageEngine": "wiredTiger" 36 | }, 37 | "rsParams": { 38 | "arbiterOnly": true 39 | } 40 | } 41 | ] 42 | } 43 | -------------------------------------------------------------------------------- /mongo_orchestration/configurations/servers/auth.json: -------------------------------------------------------------------------------- 1 | { 2 | "id" : "standalone", 3 | "auth_key": "secret", 4 | "login": "bob", 5 | "name": "mongod", 6 | "password": "pwd123", 7 | "procParams": { 8 | "dbpath": "$DBPATH", 9 | "ipv6": true, 10 | "logappend": true, 11 | "logpath": "$LOGPATH/mongod.log", 12 | "journal": true, 13 | "port": 27017 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /mongo_orchestration/configurations/servers/basic.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "mongod", 3 | "procParams": { 4 | "ipv6": true, 5 | "logappend": true, 6 | "journal": true 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /mongo_orchestration/configurations/servers/clean.json: -------------------------------------------------------------------------------- 1 | { 2 | "id" : "standalone", 3 | "name": "mongod", 4 | "procParams": { 5 | "dbpath": "$DBPATH", 6 | "ipv6": true, 7 | "logappend": true, 8 | "logpath": "$LOGPATH/mongod.log", 9 | "journal": true, 10 | "port": 27017 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /mongo_orchestration/configurations/servers/mmapv1.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "mongod", 3 | "procParams": { 4 | "ipv6": true, 5 | "logappend": true, 6 | "journal": true, 7 | "storageEngine": "mmapv1" 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /mongo_orchestration/configurations/servers/ssl.json: -------------------------------------------------------------------------------- 1 | { 2 | "id" : "standalone", 3 | "name": "mongod", 4 | "procParams": { 5 | "dbpath": "$DBPATH", 6 | "ipv6": true, 7 | "logappend": true, 8 | "logpath": "$LOGPATH/mongod.log", 9 | "journal": true, 10 | "port": 27017 11 | }, 12 | "sslParams": { 13 | "sslCAFile": "$SSL_FILES/ca.pem", 14 | "sslOnNormalPorts": true, 15 | "sslPEMKeyFile": "$SSL_FILES/server.pem", 16 | "sslWeakCertificateValidation": true 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /mongo_orchestration/configurations/servers/ssl_auth.json: -------------------------------------------------------------------------------- 1 | { 2 | "id" : "standalone", 3 | "auth_key": "secret", 4 | "login": "bob", 5 | "name": "mongod", 6 | "password": "pwd123", 7 | "procParams": { 8 | "dbpath": "$DBPATH", 9 | "ipv6": true, 10 | "logappend": true, 11 | "logpath": "$LOGPATH/mongod.log", 12 | "journal": true, 13 | "port": 27017 14 | }, 15 | "sslParams": { 16 | "sslCAFile": "$SSL_FILES/ca.pem", 17 | "sslOnNormalPorts": true, 18 | "sslPEMKeyFile": "$SSL_FILES/server.pem", 19 | "sslWeakCertificateValidation": true 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /mongo_orchestration/configurations/servers/wiredtiger.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "mongod", 3 | "procParams": { 4 | "ipv6": true, 5 | "logappend": true, 6 | "journal": true, 7 | "storageEngine": "wiredTiger" 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /mongo_orchestration/configurations/sharded_clusters/auth.json: -------------------------------------------------------------------------------- 1 | { 2 | "auth_key": "secret", 3 | "configsvrs": [ 4 | { 5 | "dbpath": "$DBPATH/db27117", 6 | "logpath": "$LOGPATH/configsvr27117.log", 7 | "port": 27117 8 | } 9 | ], 10 | "id": "shard_cluster_1", 11 | "login": "bob", 12 | "shards": [ 13 | { 14 | "id": "sh01", 15 | "shardParams": { 16 | "procParams": { 17 | "dbpath": "$DBPATH/db27217", 18 | "logpath": "$LOGPATH/sh01.log", 19 | "port": 27217 20 | } 21 | } 22 | }, 23 | { 24 | "id": "sh02", 25 | "shardParams": { 26 | "procParams": { 27 | "dbpath": "$DBPATH/db27218", 28 | "logpath": "$LOGPATH/sh02.log", 29 | "port": 27218 30 | } 31 | } 32 | } 33 | ], 34 | "password": "pwd123", 35 | "routers": [ 36 | { 37 | "logpath": "$LOGPATH/router27017.log", 38 | "port": 27017 39 | }, 40 | { 41 | "logpath": "$LOGPATH/router27018.log", 42 | "port": 27018 43 | } 44 | ] 45 | } 46 | -------------------------------------------------------------------------------- /mongo_orchestration/configurations/sharded_clusters/basic.json: -------------------------------------------------------------------------------- 1 | { 2 | "configsvrs": [{}], 3 | "shards": [ 4 | {"id": "sh01"}, 5 | {"id": "sh02"} 6 | ], 7 | "routers": [{}, {}] 8 | } 9 | -------------------------------------------------------------------------------- /mongo_orchestration/configurations/sharded_clusters/clean.json: -------------------------------------------------------------------------------- 1 | { 2 | "configsvrs": [ 3 | { 4 | "dbpath": "$DBPATH/db27117", 5 | "logpath": "$LOGPATH/configsvr27117.log", 6 | "port": 27117 7 | } 8 | ], 9 | "id": "shard_cluster_1", 10 | "shards": [ 11 | { 12 | "id": "sh01", 13 | "shardParams": { 14 | "procParams": { 15 | "dbpath": "$DBPATH/db27217", 16 | "logpath": "$LOGPATH/sh01.log", 17 | "port": 27217 18 | } 19 | } 20 | }, 21 | { 22 | "id": "sh02", 23 | "shardParams": { 24 | "procParams": { 25 | "dbpath": "$DBPATH/db27218", 26 | "logpath": "$LOGPATH/sh02.log", 27 | "port": 27218 28 | } 29 | } 30 | } 31 | ], 32 | "routers": [ 33 | { 34 | "logpath": "$LOGPATH/router27017.log", 35 | "port": 27017 36 | }, 37 | { 38 | "logpath": "$LOGPATH/router27018.log", 39 | "port": 27018 40 | } 41 | ] 42 | } 43 | -------------------------------------------------------------------------------- /mongo_orchestration/configurations/sharded_clusters/mmapv1.json: -------------------------------------------------------------------------------- 1 | { 2 | "configsvrs": [{}], 3 | "shards": [ 4 | { 5 | "id": "sh01", 6 | "shardParams": { 7 | "procParams": { 8 | "storageEngine": "mmapv1" 9 | } 10 | } 11 | }, 12 | {"id": "sh02"} 13 | ], 14 | "routers": [{}, {}] 15 | } 16 | -------------------------------------------------------------------------------- /mongo_orchestration/configurations/sharded_clusters/ssl.json: -------------------------------------------------------------------------------- 1 | { 2 | "configsvrs": [ 3 | { 4 | "dbpath": "$DBPATH/db27117", 5 | "logpath": "$LOGPATH/configsvr27117.log", 6 | "port": 27117 7 | } 8 | ], 9 | "id": "shard_cluster_1", 10 | "shards": [ 11 | { 12 | "id": "sh01", 13 | "shardParams": { 14 | "procParams": { 15 | "dbpath": "$DBPATH/db27217", 16 | "logpath": "$LOGPATH/sh01.log", 17 | "port": 27217 18 | } 19 | } 20 | }, 21 | { 22 | "id": "sh02", 23 | "shardParams": { 24 | "procParams": { 25 | "dbpath": "$DBPATH/db27218", 26 | "logpath": "$LOGPATH/sh02.log", 27 | "port": 27218 28 | } 29 | } 30 | } 31 | ], 32 | "routers": [ 33 | { 34 | "logpath": "$LOGPATH/router27017.log", 35 | "port": 27017 36 | }, 37 | { 38 | "logpath": "$LOGPATH/router27018.log", 39 | "port": 27018 40 | } 41 | ], 42 | "sslParams": { 43 | "sslCAFile": "$SSL_FILES/ca.pem", 44 | "sslOnNormalPorts": true, 45 | "sslPEMKeyFile": "$SSL_FILES/server.pem", 46 | "sslWeakCertificateValidation": true 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /mongo_orchestration/configurations/sharded_clusters/ssl_auth.json: -------------------------------------------------------------------------------- 1 | { 2 | "auth_key": "secret", 3 | "configsvrs": [ 4 | { 5 | "dbpath": "$DBPATH/db27117", 6 | "logpath": "$LOGPATH/configsvr27117.log", 7 | "port": 27117 8 | } 9 | ], 10 | "id": "shard_cluster_1", 11 | "login": "bob", 12 | "shards": [ 13 | { 14 | "id": "sh01", 15 | "shardParams": { 16 | "procParams": { 17 | "dbpath": "$DBPATH/db27217", 18 | "logpath": "$LOGPATH/sh01.log", 19 | "port": 27217 20 | } 21 | } 22 | }, 23 | { 24 | "id": "sh02", 25 | "shardParams": { 26 | "procParams": { 27 | "dbpath": "$DBPATH/db27218", 28 | "logpath": "$LOGPATH/sh02.log", 29 | "port": 27218 30 | } 31 | } 32 | } 33 | ], 34 | "password": "pwd123", 35 | "routers": [ 36 | { 37 | "logpath": "$LOGPATH/router27017.log", 38 | "port": 27017 39 | }, 40 | { 41 | "logpath": "$LOGPATH/router27018.log", 42 | "port": 27018 43 | } 44 | ], 45 | "sslParams": { 46 | "sslCAFile": "$SSL_FILES/ca.pem", 47 | "sslOnNormalPorts": true, 48 | "sslPEMKeyFile": "$SSL_FILES/server.pem", 49 | "sslWeakCertificateValidation": true 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /mongo_orchestration/configurations/sharded_clusters/wiredtiger.json: -------------------------------------------------------------------------------- 1 | { 2 | "configsvrs": [{}], 3 | "shards": [ 4 | { 5 | "id": "sh01", 6 | "shardParams": { 7 | "procParams": { 8 | "storageEngine": "wiredTiger" 9 | } 10 | } 11 | }, 12 | {"id": "sh02"} 13 | ], 14 | "routers": [{}, {}] 15 | } 16 | -------------------------------------------------------------------------------- /mongo_orchestration/container.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # coding=utf-8 3 | # Copyright 2012-2014 MongoDB, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import logging 18 | 19 | from mongo_orchestration.errors import MongoOrchestrationError 20 | 21 | logger = logging.getLogger(__name__) 22 | 23 | 24 | class Container(object): 25 | """ Container is a dict-like collection for objects""" 26 | _storage = {} 27 | _name = 'container' 28 | _obj_type = object 29 | 30 | def set_settings(self, releases=None, default_release=None): 31 | """set path to storage""" 32 | if (self._storage is None or 33 | getattr(self, 'releases', {}) != releases or 34 | getattr(self, 'default_release', '') != default_release): 35 | self._storage = {} 36 | self.releases = releases or {} 37 | self.default_release = default_release 38 | 39 | def bin_path(self, release=None): 40 | """Get the bin path for a particular release.""" 41 | if release: 42 | for r in self.releases: 43 | if release in r: 44 | return self.releases[r] 45 | raise MongoOrchestrationError("No such release '%s' in %r" 46 | % (release, self.releases)) 47 | if self.default_release: 48 | return self.releases[self.default_release] 49 | if self.releases: 50 | return list(self.releases.values())[0] 51 | return '' 52 | 53 | def __getitem__(self, key): 54 | return self._storage[key] 55 | 56 | def __setitem__(self, key, value): 57 | if isinstance(value, self._obj_type): 58 | self._storage[key] = value 59 | else: 60 | raise ValueError("Can only store objects of type %s, not %s" 61 | % (self._obj_type, type(value))) 62 | 63 | def __delitem__(self, key): 64 | return self._storage.pop(key) 65 | 66 | def __del__(self): 67 | self.cleanup() 68 | 69 | def __contains__(self, item): 70 | return item in self._storage 71 | 72 | def __iter__(self): 73 | # Iterate over a copy of storage's keys 74 | return iter(list(self._storage)) 75 | 76 | def __len__(self): 77 | return len(self._storage) 78 | 79 | def __nonzero__(self): 80 | return bool(len(self)) 81 | 82 | def __bool__(self): 83 | # Python 3 compatibility 84 | return self.__nonzero__() # pragma: no cover 85 | 86 | def cleanup(self): 87 | self._storage.clear() 88 | 89 | def create(self): 90 | raise NotImplementedError("Please Implement this method") 91 | 92 | def remove(self): 93 | raise NotImplementedError("Please Implement this method") 94 | 95 | def info(self): 96 | raise NotImplementedError("Please Implement this method") 97 | -------------------------------------------------------------------------------- /mongo_orchestration/daemon.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | # Copyright 2013-2014 MongoDB, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import atexit 18 | import errno 19 | import logging 20 | import os 21 | import subprocess 22 | import sys 23 | import time 24 | 25 | from signal import SIGTERM 26 | 27 | DEVNULL = open(os.devnull, 'r+b') 28 | 29 | logger = logging.getLogger(__name__) 30 | logger.setLevel(logging.DEBUG) 31 | 32 | 33 | class Daemon(object): 34 | """ 35 | A generic daemon class. 36 | 37 | Usage: subclass the Daemon class and override the run() method 38 | 39 | source: http://www.jejik.com/articles/2007/02/a_simple_unix_linux_daemon_in_python/ 40 | """ 41 | def __init__(self, pidfile, 42 | stdin=DEVNULL, stdout=DEVNULL, stderr=DEVNULL, 43 | timeout=0): 44 | self.stdin = stdin 45 | self.stdout = stdout 46 | self.stderr = stderr 47 | self.pidfile = pidfile 48 | self.timeout = timeout # sleep before exit from parent 49 | 50 | def daemonize(self): 51 | if os.name == 'nt': 52 | return self.daemonize_win32() 53 | else: 54 | return self.daemonize_posix() 55 | 56 | def daemonize_win32(self): 57 | logger.info('daemonize_win32: %r' % (sys.argv, )) 58 | DETACHED_PROCESS = 0x00000008 59 | pid = subprocess.Popen(sys.argv + ["--no-fork"], 60 | creationflags=DETACHED_PROCESS, shell=True, 61 | stderr=sys.stderr, stdout=sys.stdout).pid 62 | 63 | try: 64 | with open(self.pidfile, 'w+') as fd: 65 | fd.write("%s\n" % pid) 66 | except: 67 | logger.exception('write pidfile %r' % self.pidfile) 68 | raise 69 | 70 | return pid 71 | 72 | def daemonize_posix(self): 73 | """ 74 | do the UNIX double-fork magic, see Stevens' "Advanced 75 | Programming in the UNIX Environment" for details (ISBN 0201563177) 76 | http://www.erlenstar.demon.co.uk/unix/faq_2.html#SEC16 77 | """ 78 | logger.info('daemonize_posix') 79 | try: 80 | pid = os.fork() 81 | if pid > 0: 82 | logger.debug('forked first child, pid = %d' % (pid,)) 83 | return pid 84 | logger.debug('in child after first fork, pid = %d' % (pid, )) 85 | except OSError as error: 86 | logger.exception('fork #1') 87 | sys.stderr.write("fork #1 failed: %d (%s)\n" % (error.errno, error.strerror)) 88 | sys.exit(1) 89 | 90 | # decouple from parent environment 91 | os.chdir("/") 92 | os.setsid() 93 | os.umask(0) 94 | 95 | # do second fork 96 | try: 97 | pid = os.fork() 98 | if pid > 0: 99 | # exit from second parent 100 | logger.debug('forked second child, pid = %d, exiting' % (pid,)) 101 | sys.exit(0) 102 | except OSError as error: 103 | logger.exception('fork #2') 104 | sys.stderr.write("fork #2 failed: %d (%s)\n" % (error.errno, error.strerror)) 105 | sys.exit(1) 106 | 107 | # redirect standard file descriptors 108 | logger.info('daemonized, pid = %d' % (pid, )) 109 | sys.stdin.flush() 110 | sys.stdout.flush() 111 | sys.stderr.flush() 112 | 113 | os.dup2(self.stdin.fileno(), sys.stdin.fileno()) 114 | os.dup2(self.stdout.fileno(), sys.stdout.fileno()) 115 | os.dup2(self.stderr.fileno(), sys.stderr.fileno()) 116 | 117 | # write pidfile 118 | atexit.register(self.delpid) 119 | pid = str(os.getpid()) 120 | with open(self.pidfile, 'w+') as fd: 121 | fd.write("%s\n" % pid) 122 | 123 | def delpid(self): 124 | """remove pidfile""" 125 | os.remove(self.pidfile) 126 | 127 | def start(self): 128 | """ 129 | Start the daemon 130 | """ 131 | # Check for a pidfile to see if the daemon already runs 132 | logger.info('Starting daemon') 133 | try: 134 | with open(self.pidfile, 'r') as fd: 135 | pid = int(fd.read().strip()) 136 | except IOError: 137 | pid = None 138 | 139 | if pid: 140 | message = "pidfile %s already exist. Daemon already running?\n" 141 | sys.stderr.write(message % self.pidfile) 142 | sys.exit(1) 143 | 144 | # Start the daemon 145 | pid = self.daemonize() 146 | if pid: 147 | return pid 148 | self.run() 149 | 150 | def stop(self): 151 | """ 152 | Stop the daemon 153 | """ 154 | # Get the pid from the pidfile 155 | logger.debug("reading %s" % (self.pidfile,)) 156 | try: 157 | with open(self.pidfile, 'r') as fd: 158 | pid = int(fd.read().strip()) 159 | except IOError: 160 | logger.exception("reading %s" % (self.pidfile, )) 161 | pid = None 162 | 163 | if not pid: 164 | message = "pidfile %s does not exist. Daemon not running?\n" 165 | sys.stderr.write(message % self.pidfile) 166 | return # not an error in a restart 167 | 168 | if os.name == "nt": 169 | subprocess.call(["taskkill", "/f", "/t", "/pid", str(pid)]) 170 | 171 | if os.path.exists(self.pidfile): 172 | os.remove(self.pidfile) 173 | else: 174 | # Try killing the daemon process 175 | try: 176 | os.kill(pid, SIGTERM) 177 | while is_unix_process_running(pid): 178 | time.sleep(0.25) 179 | except OSError as err: 180 | if err.errno == errno.ESRCH: 181 | if os.path.exists(self.pidfile): 182 | os.remove(self.pidfile) 183 | else: 184 | raise 185 | 186 | def restart(self): 187 | """ 188 | Restart the daemon 189 | """ 190 | self.stop() 191 | self.start() 192 | 193 | def run(self): 194 | """ 195 | You should override this method when you subclass Daemon. It will be called after the process has been 196 | daemonized by start() or restart(). 197 | """ 198 | 199 | def is_unix_process_running(pid): 200 | """ 201 | Helper function used to determine if a given pid corresponds to a running process. 202 | This does NOT work on Windows 203 | """ 204 | try: 205 | os.kill(pid, 0) 206 | except OSError as err: 207 | if err.errno == errno.ESRCH: 208 | return False 209 | else: 210 | raise err 211 | return True 212 | 213 | -------------------------------------------------------------------------------- /mongo_orchestration/errors.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # coding=utf-8 3 | # Copyright 2012-2014 MongoDB, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | class MongoOrchestrationError(Exception): 19 | """Base class for all mongo-orchestration exceptions.""" 20 | 21 | 22 | class RequestError(MongoOrchestrationError): 23 | """Raised when a bad request is made to the web interface.""" 24 | 25 | 26 | class ServersError(MongoOrchestrationError): 27 | """Base class for all Server exceptions.""" 28 | 29 | 30 | class ReplicaSetError(MongoOrchestrationError): 31 | """Base class for all ReplicaSet exceptions.""" 32 | 33 | 34 | class ShardedClusterError(MongoOrchestrationError): 35 | """Base class for all ShardedCluster exceptions.""" 36 | 37 | 38 | class OperationFailure(MongoOrchestrationError): 39 | """Raised when an operation fails.""" 40 | 41 | def __init__(self, error, code=None): 42 | self.code = code # pragma: no cover 43 | MongoOrchestrationError.__init__(self, error) # pragma: no cover 44 | 45 | 46 | class TimeoutError(OperationFailure): 47 | """Raised when an operation times out.""" 48 | -------------------------------------------------------------------------------- /mongo_orchestration/launch.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023-Present MongoDB, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import atexit 16 | import copy 17 | import itertools 18 | import time 19 | import os 20 | import sys 21 | 22 | import pymongo 23 | import requests 24 | 25 | # Configurable hosts and ports used in the tests 26 | db_user = str(os.environ.get("DB_USER", "")) 27 | db_password = str(os.environ.get("DB_PASSWORD", "")) 28 | 29 | # Document count for stress tests 30 | STRESS_COUNT = 100 31 | 32 | # Test namespace, timestamp arguments 33 | TESTARGS = ('test.test', 1) 34 | 35 | _mo_address = os.environ.get("MO_ADDRESS", "localhost:8889") 36 | _mongo_start_port = int(os.environ.get("MONGO_PORT", 27017)) 37 | _free_port = itertools.count(_mongo_start_port) 38 | 39 | DEFAULT_OPTIONS = { 40 | 'logappend': True, 41 | 'ipv6': True, 42 | 'bind_ip': '127.0.0.1,::1', 43 | # 'storageEngine': 'mmapv1', 44 | # 'networkMessageCompressors': 'disabled', 45 | # 'vvvvv': '', 46 | 'setParameter': {'enableTestCommands': 1}, # 'logicalSessionRefreshMillis': 1000000}, 47 | } 48 | 49 | 50 | _post_request_template = {} 51 | if db_user and db_password: 52 | _post_request_template = {'login': db_user, 'password': db_password} 53 | 54 | 55 | def _mo_url(resource, *args): 56 | return 'http://' + '/'.join([_mo_address, resource] + list(args)) 57 | 58 | 59 | @atexit.register 60 | def kill_all(): 61 | try: 62 | clusters = requests.get(_mo_url('sharded_clusters')).json() 63 | except requests.ConnectionError as e: 64 | return 65 | repl_sets = requests.get(_mo_url('replica_sets')).json() 66 | servers = requests.get(_mo_url('servers')).json() 67 | for cluster in clusters['sharded_clusters']: 68 | requests.delete(_mo_url('sharded_clusters', cluster['id'])) 69 | for rs in repl_sets['replica_sets']: 70 | requests.delete(_mo_url('relica_sets', rs['id'])) 71 | for server in servers['servers']: 72 | requests.delete(_mo_url('servers', server['id'])) 73 | 74 | 75 | class MCTestObject(object): 76 | 77 | def proc_params(self): 78 | params = copy.deepcopy(DEFAULT_OPTIONS) 79 | params.update(self._proc_params) 80 | params["port"] = next(_free_port) 81 | return params 82 | 83 | def get_config(self): 84 | raise NotImplementedError 85 | 86 | def _make_post_request(self): 87 | config = _post_request_template.copy() 88 | config.update(self.get_config()) 89 | import pprint 90 | pprint.pprint(config) 91 | ret = requests.post( 92 | _mo_url(self._resource), timeout=None, json=config)#.json() 93 | 94 | if not ret.ok: 95 | raise RuntimeError( 96 | "Error sending POST to cluster: %s" % (ret.text,)) 97 | 98 | ret = ret.json() 99 | if type(ret) == list: # Will return a list if an error occurred. 100 | raise RuntimeError("Error sending POST to cluster: %s" % (ret,)) 101 | pprint.pprint(ret) 102 | return ret 103 | 104 | def _make_get_request(self): 105 | ret = requests.get(_mo_url(self._resource, self.id), timeout=None) 106 | 107 | if not ret.ok: 108 | raise RuntimeError( 109 | "Error sending GET to cluster: %s" % (ret.text,)) 110 | 111 | ret = ret.json() 112 | if type(ret) == list: # Will return a list if an error occurred. 113 | raise RuntimeError("Error sending GET to cluster: %s" % (ret,)) 114 | return ret 115 | 116 | def client(self, **kwargs): 117 | kwargs = kwargs.copy() 118 | if db_user: 119 | kwargs['username'] = db_user 120 | kwargs['password'] = db_password 121 | client = pymongo.MongoClient(self.uri, **kwargs) 122 | return client 123 | 124 | def stop(self): 125 | requests.delete(_mo_url(self._resource, self.id)) 126 | 127 | 128 | class Server(MCTestObject): 129 | 130 | _resource = 'servers' 131 | 132 | def __init__(self, id=None, uri=None, **kwargs): 133 | self.id = id 134 | self.uri = uri 135 | self._proc_params = kwargs 136 | 137 | def get_config(self): 138 | return { 139 | 'name': 'mongod', 140 | 'procParams': self.proc_params()} 141 | 142 | def start(self): 143 | if self.id is None: 144 | try: 145 | response = self._make_post_request() 146 | except requests.ConnectionError as e: 147 | print('Please start mongo-orchestration!') 148 | sys.exit(1) 149 | self.id = response['id'] 150 | self.uri = response.get('mongodb_auth_uri', 151 | response['mongodb_uri']) 152 | else: 153 | requests.post( 154 | _mo_url('servers', self.id), timeout=None, 155 | json={'action': 'start'} 156 | ) 157 | return self 158 | 159 | def stop(self, destroy=True): 160 | if destroy: 161 | super(Server, self).stop() 162 | else: 163 | requests.post(_mo_url('servers', self.id), timeout=None, 164 | json={'action': 'stop'}) 165 | 166 | 167 | class ReplicaSet(MCTestObject): 168 | 169 | _resource = 'replica_sets' 170 | 171 | def __init__(self, id=None, uri=None, primary=None, secondary=None, 172 | single=False, **kwargs): 173 | self.single = single 174 | self.id = id 175 | self.uri = uri 176 | self.primary = primary 177 | self.secondary = secondary 178 | self._proc_params = kwargs 179 | self.members = [] 180 | 181 | def proc_params(self): 182 | params = super(ReplicaSet, self).proc_params() 183 | # params.setdefault('setParameter', {}).setdefault('transactionLifetimeLimitSeconds', 3) 184 | # params.setdefault('setParameter', {}).setdefault('periodicNoopIntervalSecs', 1) 185 | return params 186 | 187 | def get_config(self): 188 | members = [{'procParams': self.proc_params()}] 189 | if not self.single: 190 | members.extend([ 191 | {'procParams': self.proc_params()}, 192 | {#'rsParams': {'arbiterOnly': True}, 193 | 'procParams': self.proc_params()} 194 | ]) 195 | return {'members': members} 196 | 197 | def _init_from_response(self, response): 198 | self.id = response['id'] 199 | self.uri = response.get('mongodb_auth_uri', response['mongodb_uri']) 200 | for member in response['members']: 201 | m = Server(member['server_id'], member['host']) 202 | self.members.append(m) 203 | if member['state'] == 1: 204 | self.primary = m 205 | elif member['state'] == 2: 206 | self.secondary = m 207 | return self 208 | 209 | def start(self): 210 | # We never need to restart a replica set, only start new ones. 211 | return self._init_from_response(self._make_post_request()) 212 | 213 | def restart_primary(self): 214 | self.primary.stop(destroy=False) 215 | time.sleep(5) 216 | self.primary.start() 217 | time.sleep(1) 218 | self._init_from_response(self._make_get_request()) 219 | print('New primary: %s' % self.primary.uri) 220 | 221 | 222 | class ReplicaSetSingle(ReplicaSet): 223 | 224 | def get_config(self): 225 | return { 226 | 'members': [ 227 | {'procParams': self.proc_params()} 228 | ] 229 | } 230 | 231 | 232 | class ShardedCluster(MCTestObject): 233 | 234 | _resource = 'sharded_clusters' 235 | _shard_type = ReplicaSet 236 | 237 | def __init__(self, **kwargs): 238 | self.id = None 239 | self.uri = None 240 | self.shards = [] 241 | self._proc_params = kwargs 242 | 243 | def get_config(self): 244 | return { 245 | # 'configsvrs': [{'members': [DEFAULT_OPTIONS.copy()]}], 246 | 'routers': [self.proc_params(), self.proc_params()], 247 | 'shards': [ 248 | {'id': 'demo-set-0', 'shardParams': 249 | self._shard_type().get_config()}, 250 | # {'id': 'demo-set-1', 'shardParams': 251 | # self._shard_type().get_config()} 252 | ] 253 | } 254 | 255 | def start(self): 256 | # We never need to restart a sharded cluster, only start new ones. 257 | response = self._make_post_request() 258 | for shard in response['shards']: 259 | shard_resp = requests.get(_mo_url('replica_sets', shard['_id'])) 260 | shard_json = shard_resp.json() 261 | self.shards.append(self._shard_type()._init_from_response(shard_json)) 262 | self.id = response['id'] 263 | self.uri = response.get('mongodb_auth_uri', response['mongodb_uri']) 264 | return self 265 | 266 | 267 | class ShardedClusterSingle(ShardedCluster): 268 | _shard_type = ReplicaSetSingle 269 | 270 | 271 | def argv_has(string): 272 | return any(string in arg for arg in sys.argv[1:]) 273 | 274 | 275 | DEFAULT_CERTS = os.path.join( 276 | os.environ.get( 277 | 'MONGO_ORCHESTRATION_HOME', os.path.dirname(__file__)), 278 | 'lib' 279 | ) 280 | CERTS = os.environ.get('MONGO_ORCHESTRATION_CERTS', DEFAULT_CERTS) 281 | 282 | 283 | def main(): 284 | for arg in sys.argv[1:]: 285 | try: 286 | port = int(arg) 287 | _free_port = itertools.count(port) 288 | except: 289 | pass 290 | for version in ['3.6', '4.0', '4.2', '4.4', '5.0', '6.0', '7.0', 'latest']: 291 | if argv_has(version): 292 | _post_request_template['version'] = version 293 | break 294 | 295 | if argv_has('ssl') or argv_has('tls'): 296 | _post_request_template['sslParams'] = { 297 | "sslOnNormalPorts": True, 298 | "sslPEMKeyFile": os.path.join(CERTS, "server.pem"), 299 | "sslCAFile": os.path.join(CERTS, "ca.pem"), 300 | "sslWeakCertificateValidation" : True 301 | } 302 | if argv_has('auth'): 303 | _post_request_template['login'] = db_user or 'user' 304 | _post_request_template['password'] = db_password or 'password' 305 | _post_request_template['auth_key'] = 'secret' 306 | 307 | single = argv_has('single') or argv_has('standalone') or argv_has('mongod') 308 | msg = 'Type "q" to quit: ' 309 | if argv_has('repl'): 310 | # DEFAULT_OPTIONS['enableMajorityReadConcern'] = '' 311 | cluster = ReplicaSet(single=single) 312 | msg = 'Type "q" to quit, "r" to shutdown and restart the primary": ' 313 | elif argv_has('shard') or argv_has('mongos'): 314 | cluster = ShardedClusterSingle() 315 | elif single: 316 | cluster = Server() 317 | else: 318 | exit('Usage: %s [single|replica|shard] [ssl] [auth]' % (__file__,)) 319 | 320 | cluster.start() 321 | 322 | try: 323 | while True: 324 | data = input(msg) 325 | if data == 'q': 326 | break 327 | if data == 'r' and argv_has('repl'): 328 | cluster.restart_primary() 329 | finally: 330 | cluster.stop() 331 | 332 | 333 | # Requires mongo-orchestration running on port 8889. 334 | # 335 | # Usage: 336 | # mongo-launch 337 | # 338 | # Examples (standalone node): 339 | # mongo-launch single 340 | # mongo-launch single auth 341 | # mongo-launch single auth ssl 342 | # 343 | # Sharded clusters: 344 | # mongo-launch shard 345 | # mongo-launch shard auth 346 | # mongo-launch shard auth ssl 347 | # 348 | # Replica sets: 349 | # mongo-launch repl 350 | # mongo-launch repl single 351 | # mongo-launch repl single auth 352 | if __name__ == '__main__': 353 | main() -------------------------------------------------------------------------------- /mongo_orchestration/lib/ca.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDfzCCAmegAwIBAgIDB1MGMA0GCSqGSIb3DQEBCwUAMHkxGzAZBgNVBAMTEkRy 3 | aXZlcnMgVGVzdGluZyBDQTEQMA4GA1UECxMHRHJpdmVyczEQMA4GA1UEChMHTW9u 4 | Z29EQjEWMBQGA1UEBxMNTmV3IFlvcmsgQ2l0eTERMA8GA1UECBMITmV3IFlvcmsx 5 | CzAJBgNVBAYTAlVTMB4XDTE5MDUyMjIwMjMxMVoXDTM5MDUyMjIwMjMxMVoweTEb 6 | MBkGA1UEAxMSRHJpdmVycyBUZXN0aW5nIENBMRAwDgYDVQQLEwdEcml2ZXJzMRAw 7 | DgYDVQQKEwdNb25nb0RCMRYwFAYDVQQHEw1OZXcgWW9yayBDaXR5MREwDwYDVQQI 8 | EwhOZXcgWW9yazELMAkGA1UEBhMCVVMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw 9 | ggEKAoIBAQCl7VN+WsQfHlwapcOpTLZVoeMAl1LTbWTFuXSAavIyy0W1Ytky1UP/ 10 | bxCSW0mSWwCgqoJ5aXbAvrNRp6ArWu3LsTQIEcD3pEdrFIVQhYzWUs9fXqPyI9k+ 11 | QNNQ+MRFKeGteTPYwF2eVEtPzUHU5ws3+OKp1m6MCLkwAG3RBFUAfddUnLvGoZiT 12 | pd8/eNabhgHvdrCw+tYFCWvSjz7SluEVievpQehrSEPKe8DxJq/IM3tSl3tdylzT 13 | zeiKNO7c7LuQrgjAfrZl7n2SriHIlNmqiDR/kdd8+TxBuxjFlcf2WyHCO3lIcIgH 14 | KXTlhUCg50KfHaxHu05Qw0x8869yIzqbAgMBAAGjEDAOMAwGA1UdEwQFMAMBAf8w 15 | DQYJKoZIhvcNAQELBQADggEBAEHuhTL8KQZcKCTSJbYA9MgZj7U32arMGBbc1hiq 16 | VBREwvdVz4+9tIyWMzN9R/YCKmUTnCq8z3wTlC8kBtxYn/l4Tj8nJYcgLJjQ0Fwe 17 | gT564CmvkUat8uXPz6olOCdwkMpJ9Sj62i0mpgXJdBfxKQ6TZ9yGz6m3jannjZpN 18 | LchB7xSAEWtqUgvNusq0dApJsf4n7jZ+oBZVaQw2+tzaMfaLqHgMwcu1FzA8UKCD 19 | sxCgIsZUs8DdxaD418Ot6nPfheOTqe24n+TTa+Z6O0W0QtnofJBx7tmAo1aEc57i 20 | 77s89pfwIJetpIlhzNSMKurCAocFCJMJLAASJFuu6dyDvPo= 21 | -----END CERTIFICATE----- -------------------------------------------------------------------------------- /mongo_orchestration/lib/client.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIEpAIBAAKCAQEAsNS8UEuin7/K29jXfIOLpIoh1jEyWVqxiie2Onx7uJJKcoKo 3 | khA3XeUnVN0k6X5MwYWcN52xcns7LYtyt06nRpTG2/emoV44w9uKTuHsvUbiOwSV 4 | m/ToKQQ4FUFZoqorXH+ZmJuIpJNfoW+3CkE1vEDCIecIq6BNg5ySsPtvSuSJHGjp 5 | mc7/5ZUDvFE2aJ8QbJU3Ws0HXiEb6ymi048LlzEL2VKX3w6mqqh+7dcZGAy7qYk2 6 | 5FZ9ktKvCeQau7mTyU1hsPrKFiKtMN8Q2ZAItX13asw5/IeSTq2LgLFHlbj5Kpq4 7 | GmLdNCshzH5X7Ew3IYM8EHmsX8dmD6mhv7vpVwIDAQABAoIBABOdpb4qhcG+3twA 8 | c/cGCKmaASLnljQ/UU6IFTjrsjXJVKTbRaPeVKX/05sgZQXZ0t3s2mV5AsQ2U1w8 9 | Cd+3w+qaemzQThW8hAOGCROzEDX29QWi/o2sX0ydgTMqaq0Wv3SlWv6I0mGfT45y 10 | /BURIsrdTCvCmz2erLqa1dL4MWJXRFjT9UTs5twlecIOM2IHKoGGagFhymRK4kDe 11 | wTRC9fpfoAgyfus3pCO/wi/F8yKGPDEwY+zgkhrJQ+kSeki7oKdGD1H540vB8gRt 12 | EIqssE0Y6rEYf97WssQlxJgvoJBDSftOijS6mwvoasDUwfFqyyPiirawXWWhHXkc 13 | DjIi/XECgYEA5xfjilw9YyM2UGQNESbNNunPcj7gDZbN347xJwmYmi9AUdPLt9xN 14 | 3XaMqqR22k1DUOxC/5hH0uiXir7mDfqmC+XS/ic/VOsa3CDWejkEnyGLiwSHY502 15 | wD/xWgHwUiGVAG9HY64vnDGm6L3KGXA2oqxanL4V0+0+Ht49pZ16i8sCgYEAw+Ox 16 | CHGtpkzjCP/z8xr+1VTSdpc/4CP2HONnYopcn48KfQnf7Nale69/1kZpypJlvQSG 17 | eeA3jMGigNJEkb8/kaVoRLCisXcwLc0XIfCTeiK6FS0Ka30D/84Qm8UsHxRdpGkM 18 | kYITAa2r64tgRL8as4/ukeXBKE+oOhX43LeEfyUCgYBkf7IX2Ndlhsm3GlvIarxy 19 | NipeP9PGdR/hKlPbq0OvQf9R1q7QrcE7H7Q6/b0mYNV2mtjkOQB7S2WkFDMOP0P5 20 | BqDEoKLdNkV/F9TOYH+PCNKbyYNrodJOt0Ap6Y/u1+Xpw3sjcXwJDFrO+sKqX2+T 21 | PStG4S+y84jBedsLbDoAEwKBgQCTz7/KC11o2yOFqv09N+WKvBKDgeWlD/2qFr3w 22 | UU9K5viXGVhqshz0k5z25vL09Drowf1nAZVpFMO2SPOMtq8VC6b+Dfr1xmYIaXVH 23 | Gu1tf77CM9Zk/VSDNc66e7GrUgbHBK2DLo+A+Ld9aRIfTcSsMbNnS+LQtCrQibvb 24 | cG7+MQKBgQCY11oMT2dUekoZEyW4no7W5D74lR8ztMjp/fWWTDo/AZGPBY6cZoZF 25 | IICrzYtDT/5BzB0Jh1f4O9ZQkm5+OvlFbmoZoSbMzHL3oJCBOY5K0/kdGXL46WWh 26 | IRJSYakNU6VIS7SjDpKgm9D8befQqZeoSggSjIIULIiAtYgS80vmGA== 27 | -----END RSA PRIVATE KEY----- 28 | -----BEGIN CERTIFICATE----- 29 | MIIDgzCCAmugAwIBAgIDAxOUMA0GCSqGSIb3DQEBCwUAMHkxGzAZBgNVBAMTEkRy 30 | aXZlcnMgVGVzdGluZyBDQTEQMA4GA1UECxMHRHJpdmVyczEQMA4GA1UEChMHTW9u 31 | Z29EQjEWMBQGA1UEBxMNTmV3IFlvcmsgQ2l0eTERMA8GA1UECBMITmV3IFlvcmsx 32 | CzAJBgNVBAYTAlVTMB4XDTE5MDUyMjIzNTU1NFoXDTM5MDUyMjIzNTU1NFowaTEP 33 | MA0GA1UEAxMGY2xpZW50MRAwDgYDVQQLEwdEcml2ZXJzMQwwCgYDVQQKEwNNREIx 34 | FjAUBgNVBAcTDU5ldyBZb3JrIENpdHkxETAPBgNVBAgTCE5ldyBZb3JrMQswCQYD 35 | VQQGEwJVUzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALDUvFBLop+/ 36 | ytvY13yDi6SKIdYxMllasYontjp8e7iSSnKCqJIQN13lJ1TdJOl+TMGFnDedsXJ7 37 | Oy2LcrdOp0aUxtv3pqFeOMPbik7h7L1G4jsElZv06CkEOBVBWaKqK1x/mZibiKST 38 | X6FvtwpBNbxAwiHnCKugTYOckrD7b0rkiRxo6ZnO/+WVA7xRNmifEGyVN1rNB14h 39 | G+spotOPC5cxC9lSl98Opqqofu3XGRgMu6mJNuRWfZLSrwnkGru5k8lNYbD6yhYi 40 | rTDfENmQCLV9d2rMOfyHkk6ti4CxR5W4+SqauBpi3TQrIcx+V+xMNyGDPBB5rF/H 41 | Zg+pob+76VcCAwEAAaMkMCIwCwYDVR0PBAQDAgeAMBMGA1UdJQQMMAoGCCsGAQUF 42 | BwMCMA0GCSqGSIb3DQEBCwUAA4IBAQAqRcLAGvYMaGYOV4HJTzNotT2qE0I9THNQ 43 | wOV1fBg69x6SrUQTQLjJEptpOA288Wue6Jt3H+p5qAGV5GbXjzN/yjCoItggSKxG 44 | Xg7279nz6/C5faoIKRjpS9R+MsJGlttP9nUzdSxrHvvqm62OuSVFjjETxD39DupE 45 | YPFQoHOxdFTtBQlc/zIKxVdd20rs1xJeeU2/L7jtRBSPuR/Sk8zot7G2/dQHX49y 46 | kHrq8qz12kj1T6XDXf8KZawFywXaz0/Ur+fUYKmkVk1T0JZaNtF4sKqDeNE4zcns 47 | p3xLVDSl1Q5Gwj7bgph9o4Hxs9izPwiqjmNaSjPimGYZ399zcurY 48 | -----END CERTIFICATE----- -------------------------------------------------------------------------------- /mongo_orchestration/lib/server.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIEogIBAAKCAQEAhNrB0E6GY/kFSd8/vNpu/t952tbnOsD5drV0XPvmuy7SgKDY 3 | a/S+xb/jPnlZKKehdBnH7qP/gYbv34ZykzcDFZscjPLiGc2cRGP+NQCSFK0d2/7d 4 | y15zSD3zhj14G8+MkpAejTU+0/qFNZMc5neDvGanTe0+8aWa0DXssM0MuTxIv7j6 5 | CtsMWeqLLofN7a1Kw2UvmieCHfHMuA/08pJwRnV/+5T9WONBPJja2ZQRrG1BjpI4 6 | 81zSPUZesIqi8yDlExdvgNaRZIEHi/njREqwVgJOZomUY57zmKypiMzbz48dDTsV 7 | gUStxrEqbaP+BEjQYPX5+QQk4GdMjkLf52LR6QIDAQABAoIBAHSs+hHLJNOf2zkp 8 | S3y8CUblVMsQeTpsR6otaehPgi9Zy50TpX4KD5D0GMrBH8BIl86y5Zd7h+VlcDzK 9 | gs0vPxI2izhuBovKuzaE6rf5rFFkSBjxGDCG3o/PeJOoYFdsS3RcBbjVzju0hFCs 10 | xnDQ/Wz0anJRrTnjyraY5SnQqx/xuhLXkj/lwWoWjP2bUqDprnuLOj16soNu60Um 11 | JziWbmWx9ty0wohkI/8DPBl9FjSniEEUi9pnZXPElFN6kwPkgdfT5rY/TkMH4lsu 12 | ozOUc5xgwlkT6kVjXHcs3fleuT/mOfVXLPgNms85JKLucfd6KiV7jYZkT/bXIjQ+ 13 | 7CZEn0ECgYEA5QiKZgsfJjWvZpt21V/i7dPje2xdwHtZ8F9NjX7ZUFA7mUPxUlwe 14 | GiXxmy6RGzNdnLOto4SF0/7ebuF3koO77oLup5a2etL+y/AnNAufbu4S5D72sbiz 15 | wdLzr3d5JQ12xeaEH6kQNk2SD5/ShctdS6GmTgQPiJIgH0MIdi9F3v0CgYEAlH84 16 | hMWcC+5b4hHUEexeNkT8kCXwHVcUjGRaYFdSHgovvWllApZDHSWZ+vRcMBdlhNPu 17 | 09Btxo99cjOZwGYJyt20QQLGc/ZyiOF4ximQzabTeFgLkTH3Ox6Mh2Rx9yIruYoX 18 | nE3UfMDkYELanEJUv0zenKpZHw7tTt5yXXSlEF0CgYBSsEOvVcKYO/eoluZPYQAA 19 | F2jgzZ4HeUFebDoGpM52lZD+463Dq2hezmYtPaG77U6V3bUJ/TWH9VN/Or290vvN 20 | v83ECcC2FWlSXdD5lFyqYx/E8gqE3YdgqfW62uqM+xBvoKsA9zvYLydVpsEN9v8m 21 | 6CSvs/2btA4O21e5u5WBTQKBgGtAb6vFpe0gHRDs24SOeYUs0lWycPhf+qFjobrP 22 | lqnHpa9iPeheat7UV6BfeW3qmBIVl/s4IPE2ld4z0qqZiB0Tf6ssu/TpXNPsNXS6 23 | dLFz+myC+ufFdNEoQUtQitd5wKbjTCZCOGRaVRgJcSdG6Tq55Fa22mOKPm+mTmed 24 | ZdKpAoGAFsTYBAHPxs8nzkCJCl7KLa4/zgbgywO6EcQgA7tfelB8bc8vcAMG5o+8 25 | YqAfwxrzhVSVbJx0fibTARXROmbh2pn010l2wj3+qUajM8NiskCPFbSjGy7HSUze 26 | P8Kt1uMDJdj55gATzn44au31QBioZY2zXleorxF21cr+BZCJgfA= 27 | -----END RSA PRIVATE KEY----- 28 | -----BEGIN CERTIFICATE----- 29 | MIIDlTCCAn2gAwIBAgICdxUwDQYJKoZIhvcNAQELBQAweTEbMBkGA1UEAxMSRHJp 30 | dmVycyBUZXN0aW5nIENBMRAwDgYDVQQLEwdEcml2ZXJzMRAwDgYDVQQKEwdNb25n 31 | b0RCMRYwFAYDVQQHEw1OZXcgWW9yayBDaXR5MREwDwYDVQQIEwhOZXcgWW9yazEL 32 | MAkGA1UEBhMCVVMwHhcNMTkwNTIyMjIzMjU2WhcNMzkwNTIyMjIzMjU2WjBwMRIw 33 | EAYDVQQDEwlsb2NhbGhvc3QxEDAOBgNVBAsTB0RyaXZlcnMxEDAOBgNVBAoTB01v 34 | bmdvREIxFjAUBgNVBAcTDU5ldyBZb3JrIENpdHkxETAPBgNVBAgTCE5ldyBZb3Jr 35 | MQswCQYDVQQGEwJVUzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAITa 36 | wdBOhmP5BUnfP7zabv7fedrW5zrA+Xa1dFz75rsu0oCg2Gv0vsW/4z55WSinoXQZ 37 | x+6j/4GG79+GcpM3AxWbHIzy4hnNnERj/jUAkhStHdv+3ctec0g984Y9eBvPjJKQ 38 | Ho01PtP6hTWTHOZ3g7xmp03tPvGlmtA17LDNDLk8SL+4+grbDFnqiy6Hze2tSsNl 39 | L5ongh3xzLgP9PKScEZ1f/uU/VjjQTyY2tmUEaxtQY6SOPNc0j1GXrCKovMg5RMX 40 | b4DWkWSBB4v540RKsFYCTmaJlGOe85isqYjM28+PHQ07FYFErcaxKm2j/gRI0GD1 41 | +fkEJOBnTI5C3+di0ekCAwEAAaMwMC4wLAYDVR0RBCUwI4IJbG9jYWxob3N0hwR/ 42 | AAABhxAAAAAAAAAAAAAAAAAAAAABMA0GCSqGSIb3DQEBCwUAA4IBAQBol8+YH7MA 43 | HwnIh7KcJ8h87GkCWsjOJCDJWiYBJArQ0MmgDO0qdx+QEtvLMn3XNtP05ZfK0WyX 44 | or4cWllAkMFYaFbyB2hYazlD1UAAG+22Rku0UP6pJMLbWe6pnqzx+RL68FYdbZhN 45 | fCW2xiiKsdPoo2VEY7eeZKrNr/0RFE5EKXgzmobpTBQT1Dl3Ve4aWLoTy9INlQ/g 46 | z40qS7oq1PjjPLgxINhf4ncJqfmRXugYTOnyFiVXLZTys5Pb9SMKdToGl3NTYWLL 47 | 2AZdjr6bKtT+WtXyHqO0cQ8CkAW0M6VOlMluACllcJxfrtdlQS2S4lUIj76QKBdZ 48 | khBHXq/b8MFX 49 | -----END CERTIFICATE----- -------------------------------------------------------------------------------- /mongo_orchestration/process.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # coding=utf-8 3 | # Copyright 2012-2014 MongoDB, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import errno 18 | import json 19 | import logging 20 | import os 21 | import platform 22 | import shutil 23 | import stat 24 | import socket 25 | import subprocess 26 | import time 27 | import tempfile 28 | 29 | try: 30 | from subprocess import DEVNULL 31 | except ImportError: 32 | DEVNULL = open(os.devnull, 'wb') 33 | 34 | from mongo_orchestration.common import DEFAULT_BIND, LOG_FILE 35 | from mongo_orchestration.compat import reraise, PY3 36 | from mongo_orchestration.errors import TimeoutError, RequestError 37 | from mongo_orchestration.singleton import Singleton 38 | 39 | logger = logging.getLogger(__name__) 40 | logger.setLevel(logging.DEBUG) 41 | 42 | 43 | class PortPool(Singleton): 44 | 45 | __ports = set() 46 | __closed = set() 47 | __id = None 48 | 49 | def __init__(self, min_port=1025, max_port=2000, port_sequence=None): 50 | """ 51 | Args: 52 | min_port - min port number (ignoring if 'port_sequence' is not None) 53 | max_port - max port number (ignoring if 'port_sequence' is not None) 54 | port_sequence - iterate sequence which contains numbers of ports 55 | """ 56 | if not self.__id: # singleton checker 57 | self.__id = id(self) 58 | self.__init_range(min_port, max_port, port_sequence) 59 | 60 | def __init_range(self, min_port=1025, max_port=2000, port_sequence=None): 61 | if port_sequence: 62 | self.__ports = set(port_sequence) 63 | else: 64 | self.__ports = set(range(min_port, max_port + 1)) 65 | self.__closed = set() 66 | self.refresh() 67 | 68 | def __check_port(self, port): 69 | """check port status 70 | return True if port is free, False else 71 | """ 72 | s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 73 | try: 74 | s.bind((DEFAULT_BIND, port)) 75 | return True 76 | except socket.error: 77 | return False 78 | finally: 79 | s.close() 80 | 81 | def release_port(self, port): 82 | """release port""" 83 | if port in self.__closed: 84 | self.__closed.remove(port) 85 | self.__ports.add(port) 86 | 87 | def port(self, check=False): 88 | """return next opened port 89 | Args: 90 | check - check is port realy free 91 | """ 92 | if not self.__ports: # refresh ports if sequence is empty 93 | self.refresh() 94 | 95 | try: 96 | port = self.__ports.pop() 97 | if check: 98 | while not self.__check_port(port): 99 | self.release_port(port) 100 | port = self.__ports.pop() 101 | except (IndexError, KeyError): 102 | raise IndexError("Could not find a free port,\nclosed ports: {closed}".format(closed=self.__closed)) 103 | self.__closed.add(port) 104 | return port 105 | 106 | def refresh(self, only_closed=False): 107 | """refresh ports status 108 | Args: 109 | only_closed - check status only for closed ports 110 | """ 111 | if only_closed: 112 | opened = filter(self.__check_port, self.__closed) 113 | self.__closed = self.__closed.difference(opened) 114 | self.__ports = self.__ports.union(opened) 115 | else: 116 | ports = self.__closed.union(self.__ports) 117 | self.__ports = set(filter(self.__check_port, ports)) 118 | self.__closed = ports.difference(self.__ports) 119 | 120 | def change_range(self, min_port=1025, max_port=2000, port_sequence=None): 121 | """change Pool port range""" 122 | self.__init_range(min_port, max_port, port_sequence) 123 | 124 | 125 | def connect_port(port): 126 | """waits while process starts. 127 | Args: 128 | proc - Popen object 129 | port_num - port number 130 | timeout - specify how long, in seconds, a command can take before times out. 131 | return True if process started, return False if not 132 | """ 133 | s = None 134 | try: 135 | s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 136 | s.connect((DEFAULT_BIND, port)) 137 | s.close() 138 | return True 139 | except (IOError, socket.error): 140 | if s: 141 | s.close() 142 | return False 143 | 144 | 145 | def wait_for(proc, port_num, timeout): 146 | """waits while process starts. 147 | Args: 148 | proc - Popen object 149 | port_num - port number 150 | timeout - specify how long, in seconds, a command can take before times out. 151 | return True if process started, return False if not 152 | """ 153 | logger.debug("wait for {port_num}".format(**locals())) 154 | t_start = time.time() 155 | sleeps = 0.1 156 | while time.time() - t_start < timeout: 157 | if proc.poll() is not None: 158 | logger.debug("process is not alive") 159 | raise OSError("Process started, but died immediately") 160 | if connect_port(port_num): 161 | return True 162 | time.sleep(sleeps) 163 | return False 164 | 165 | 166 | def repair_mongo(name, dbpath): 167 | """repair mongodb after usafe shutdown""" 168 | log_file = os.path.join(dbpath, 'mongod.log') 169 | cmd = [name, "--dbpath", dbpath, "--logpath", log_file, "--logappend", 170 | "--repair"] 171 | proc = subprocess.Popen( 172 | cmd, universal_newlines=True, 173 | stdout=subprocess.PIPE, stderr=subprocess.STDOUT) 174 | timeout = 45 175 | t_start = time.time() 176 | while time.time() - t_start < timeout: 177 | line = str(proc.stdout.readline()) 178 | logger.info("repair output: %s" % (line,)) 179 | return_code = proc.poll() 180 | if return_code is not None: 181 | if return_code: 182 | raise Exception("mongod --repair failed with exit code %s, " 183 | "check log file: %s" % (return_code, log_file)) 184 | # Success when poll() returns 0 185 | return 186 | time.sleep(1) 187 | proc.terminate() 188 | raise Exception("mongod --repair failed to exit after %s seconds, " 189 | "check log file: %s" % (timeout, log_file)) 190 | 191 | 192 | def mprocess(name, config_path, port=None, timeout=180): 193 | """start 'name' process with params from config_path. 194 | Args: 195 | name - process name or path 196 | config_path - path to file where should be stored configuration 197 | port - process's port 198 | timeout - specify how long, in seconds, a command can take before times out. 199 | if timeout <=0 - doesn't wait for complete start process 200 | return tuple (Popen object, host) if process started, return (None, None) if not 201 | """ 202 | 203 | logger.debug( 204 | "mprocess(name={name!r}, config_path={config_path!r}, port={port!r}, " 205 | "timeout={timeout!r})".format(**locals())) 206 | if not (config_path and isinstance(config_path, str) and os.path.exists(config_path)): 207 | raise OSError("can't find config file {config_path}".format(**locals())) 208 | 209 | cfg = read_config(config_path) 210 | cmd = [name, "--config", config_path] 211 | 212 | if cfg.get('port', None) is None or port: 213 | port = port or PortPool().port(check=True) 214 | cmd.extend(['--port', str(port)]) 215 | host = "{host}:{port}".format(host=DEFAULT_BIND, port=port) 216 | try: 217 | logger.debug("execute process: %s", ' '.join(cmd)) 218 | # Redirect server startup errors (written to stdout/stderr) to our log 219 | with open(LOG_FILE, 'a+') as outfile: 220 | proc = subprocess.Popen( 221 | cmd, 222 | stdout=outfile, 223 | stderr=subprocess.STDOUT) 224 | 225 | if proc.poll() is not None: 226 | logger.debug("process is not alive") 227 | raise OSError("Process started, but died immediately.") 228 | except (OSError, TypeError) as err: 229 | message = "exception while executing process: {err}".format(err=err) 230 | logger.debug(message) 231 | raise OSError(message) 232 | if timeout > 0: 233 | if wait_for(proc, port, timeout): 234 | logger.debug("process '{name}' has started: pid={proc.pid}," 235 | " host={host}".format(**locals())) 236 | return (proc, host) 237 | else: 238 | logger.debug("hasn't connected to pid={proc.pid} with host={host}" 239 | " during timeout {timeout} ".format(**locals())) 240 | logger.debug("terminate process with" 241 | " pid={proc.pid}".format(**locals())) 242 | kill_mprocess(proc) 243 | proc_alive(proc) and time.sleep(3) # wait while process stoped 244 | message = ("Could not connect to process during " 245 | "{timeout} seconds".format(timeout=timeout)) 246 | raise TimeoutError(message, errno.ETIMEDOUT) 247 | return (proc, host) 248 | 249 | 250 | def wait_mprocess(process, timeout): 251 | """Compatibility function for waiting on a process with a timeout. 252 | 253 | Raises TimeoutError when the timeout is reached. 254 | """ 255 | if PY3: 256 | try: 257 | return process.wait(timeout=timeout) 258 | except subprocess.TimeoutExpired as exc: 259 | raise TimeoutError(str(exc)) 260 | 261 | # On Python 2, simulate the timeout parameter and raise TimeoutError. 262 | start = time.time() 263 | while True: 264 | exit_code = process.poll() 265 | if exit_code is not None: 266 | return exit_code 267 | if time.time() - start > timeout: 268 | raise TimeoutError("Process %s timed out after %s seconds" % 269 | (process.pid, timeout)) 270 | time.sleep(0.05) 271 | 272 | 273 | def kill_mprocess(process): 274 | """kill process 275 | Args: 276 | process - Popen object for process 277 | """ 278 | if process and proc_alive(process): 279 | process.terminate() 280 | process.communicate() 281 | return not proc_alive(process) 282 | 283 | 284 | def cleanup_mprocess(config_path, cfg): 285 | """remove all process's stuff 286 | Args: 287 | config_path - process's options file 288 | cfg - process's config 289 | """ 290 | for key in ('keyFile', 'dbpath'): 291 | remove_path(cfg.get(key, None)) 292 | isinstance(config_path, str) and os.path.exists(config_path) and remove_path(config_path) 293 | 294 | 295 | def remove_path(path): 296 | """remove path from file system 297 | If path is None - do nothing""" 298 | if path is None or not os.path.exists(path): 299 | return 300 | if platform.system() == 'Windows': 301 | # Need to have write permission before deleting the file. 302 | os.chmod(path, stat.S_IWRITE) 303 | try: 304 | if os.path.isdir(path): 305 | shutil.rmtree(path) 306 | elif os.path.isfile(path): 307 | shutil.os.remove(path) 308 | except OSError: 309 | logger.exception("Could not remove path: %s" % path) 310 | 311 | 312 | def write_config(params, config_path=None): 313 | """write mongo*'s config file 314 | Args: 315 | params - options wich file contains 316 | config_path - path to the config_file, will create if None 317 | Return config_path 318 | where config_path - path to mongo*'s options file 319 | """ 320 | if config_path is None: 321 | config_path = tempfile.mktemp(prefix="mongo-") 322 | 323 | cfg = params.copy() 324 | if 'setParameter' in cfg: 325 | set_parameters = cfg.pop('setParameter') 326 | try: 327 | for key, value in set_parameters.items(): 328 | cfg['setParameter = ' + key] = value 329 | except AttributeError: 330 | reraise(RequestError, 331 | 'Not a valid value for setParameter: %r ' 332 | 'Expected "setParameter": { : value, ...}' 333 | % set_parameters) 334 | 335 | # fix boolean value 336 | for key, value in cfg.items(): 337 | if isinstance(value, bool): 338 | cfg[key] = json.dumps(value) 339 | 340 | with open(config_path, 'w') as fd: 341 | data = '\n'.join('%s=%s' % (key, item) for key, item in cfg.items()) 342 | fd.write(data) 343 | return config_path 344 | 345 | 346 | def read_config(config_path): 347 | """read config_path and return options as dictionary""" 348 | result = {} 349 | with open(config_path, 'r') as fd: 350 | for line in fd.readlines(): 351 | if '=' in line: 352 | key, value = line.split('=', 1) 353 | try: 354 | result[key] = json.loads(value) 355 | except ValueError: 356 | result[key] = value.rstrip('\n') 357 | return result 358 | 359 | 360 | def proc_alive(process): 361 | """Check if process is alive. Return True or False.""" 362 | return process.poll() is None if process else False 363 | -------------------------------------------------------------------------------- /mongo_orchestration/server.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | import argparse 3 | import json 4 | import logging 5 | import os.path 6 | import signal 7 | import socket 8 | import sys 9 | import time 10 | import traceback 11 | 12 | from bson import SON 13 | 14 | from mongo_orchestration import __version__ 15 | from mongo_orchestration.common import ( 16 | BaseModel, 17 | DEFAULT_BIND, DEFAULT_PORT, DEFAULT_SERVER, DEFAULT_SOCKET_TIMEOUT, 18 | PID_FILE, LOG_FILE, LOGGING_FORMAT) 19 | from mongo_orchestration.daemon import Daemon 20 | from mongo_orchestration.servers import Server 21 | 22 | # How many times to attempt connecting to mongo-orchestration server. 23 | CONNECT_ATTEMPTS = 5 24 | # How many seconds to wait before timing out a connection attempt to the 25 | # mongo-orchestration server. 26 | CONNECT_TIMEOUT = 5 27 | 28 | 29 | def read_env(argv=None): 30 | """return command-line arguments""" 31 | parser = argparse.ArgumentParser(description='mongo-orchestration server') 32 | parser.add_argument('-f', '--config', 33 | action='store', default=None, type=str, dest='config') 34 | parser.add_argument('-e', '--env', 35 | action='store', type=str, dest='env', default=None) 36 | parser.add_argument(action='store', type=str, dest='command', 37 | default='start', choices=('start', 'stop', 'restart')) 38 | parser.add_argument('--no-fork', 39 | action='store_true', dest='no_fork', default=False) 40 | parser.add_argument('-b', '--bind', 41 | action='store', dest='bind', type=str, 42 | default=DEFAULT_BIND) 43 | parser.add_argument('-p', '--port', 44 | action='store', dest='port', type=int, 45 | default=DEFAULT_PORT) 46 | parser.add_argument('--enable-majority-read-concern', action='store_true', 47 | default=False) 48 | parser.add_argument('-s', '--server', 49 | action='store', dest='server', type=str, 50 | default=DEFAULT_SERVER, choices=('auto', 'cheroot', 'wsgiref')) 51 | parser.add_argument('--version', action='version', 52 | version='Mongo Orchestration v' + __version__) 53 | parser.add_argument('--socket-timeout-ms', action='store', 54 | dest='socket_timeout', 55 | type=int, default=DEFAULT_SOCKET_TIMEOUT) 56 | parser.add_argument('--pidfile', action='store', type=str, dest='pidfile', 57 | default=PID_FILE) 58 | 59 | cli_args = parser.parse_args(argv) 60 | 61 | if cli_args.env and not cli_args.config: 62 | print("Specified release '%s' without a config file" % cli_args.env) 63 | sys.exit(1) 64 | if cli_args.command == 'stop' or not cli_args.config: 65 | return cli_args 66 | try: 67 | # read config 68 | with open(cli_args.config, 'r') as fd: 69 | config = json.loads(fd.read(), object_pairs_hook=SON) 70 | if not 'releases' in config: 71 | print("No releases defined in %s" % cli_args.config) 72 | sys.exit(1) 73 | releases = config['releases'] 74 | if cli_args.env is not None and cli_args.env not in releases: 75 | print("Release '%s' is not defined in %s" 76 | % (cli_args.env, cli_args.config)) 77 | sys.exit(1) 78 | cli_args.releases = releases 79 | return cli_args 80 | except (IOError): 81 | print("config file not found") 82 | sys.exit(1) 83 | except (ValueError): 84 | print("config file is corrupted") 85 | sys.exit(1) 86 | 87 | 88 | def setup(releases, default_release): 89 | """setup storages""" 90 | from mongo_orchestration import set_releases, cleanup_storage 91 | set_releases(releases, default_release) 92 | signal.signal(signal.SIGTERM, cleanup_storage) 93 | signal.signal(signal.SIGINT, cleanup_storage) 94 | 95 | 96 | def get_app(): 97 | """return bottle app that includes all sub-apps""" 98 | from bottle import default_app 99 | default_app.push() 100 | for module in ("mongo_orchestration.apps.servers", 101 | "mongo_orchestration.apps.replica_sets", 102 | "mongo_orchestration.apps.sharded_clusters"): 103 | __import__(module) 104 | app = default_app.pop() 105 | return app 106 | 107 | 108 | class MyDaemon(Daemon): 109 | """class uses to run server as daemon""" 110 | 111 | def __init__(self, *args, **kwd): 112 | super(MyDaemon, self).__init__(*args, **kwd) 113 | 114 | def run(self): 115 | log = logging.getLogger(__name__) 116 | 117 | from bottle import run 118 | setup(getattr(self.args, 'releases', {}), self.args.env) 119 | BaseModel.socket_timeout = self.args.socket_timeout 120 | if self.args.command in ('start', 'restart'): 121 | print("Starting Mongo Orchestration on port %d..." % self.args.port) 122 | try: 123 | log.debug('Starting HTTP server on host: %s; port: %d', 124 | self.args.bind, self.args.port) 125 | run(get_app(), host=self.args.bind, port=self.args.port, 126 | debug=False, reloader=False, quiet=not self.args.no_fork, 127 | server=self.args.server) 128 | except Exception: 129 | traceback.print_exc(file=sys.stdout) 130 | log.exception('Could not start a new server.') 131 | raise 132 | 133 | def set_args(self, args): 134 | self.args = args 135 | 136 | 137 | def await_connection(host, port): 138 | """Wait for the mongo-orchestration server to accept connections.""" 139 | for i in range(CONNECT_ATTEMPTS): 140 | try: 141 | conn = socket.create_connection((host, port), CONNECT_TIMEOUT) 142 | conn.close() 143 | return True 144 | except (IOError, socket.error): 145 | time.sleep(1) 146 | return False 147 | 148 | 149 | def main(argv=None): 150 | args = read_env(argv) 151 | Server.enable_majority_read_concern = args.enable_majority_read_concern 152 | # Log both to STDOUT and the log file. 153 | logging.basicConfig(level=logging.DEBUG, filename=LOG_FILE, 154 | format=LOGGING_FORMAT) 155 | log = logging.getLogger(__name__) 156 | 157 | daemon = MyDaemon(os.path.abspath(args.pidfile), timeout=5, 158 | stdout=sys.stdout) 159 | daemon.set_args(args) 160 | # Set default bind ip for mongo processes using argument from --bind. 161 | Server.mongod_default['bind_ip'] = args.bind 162 | if args.command == 'stop': 163 | daemon.stop() 164 | if args.command == 'start' and not args.no_fork: 165 | print('Preparing to start mongo-orchestration daemon') 166 | pid = daemon.start() 167 | print('Daemon process started with pid: %d' % pid) 168 | if not await_connection(host=args.bind, port=args.port): 169 | print( 170 | 'Could not connect to daemon running on %s:%d (pid: %d) ' 171 | 'within %d attempts.' 172 | % (args.bind, args.port, pid, CONNECT_ATTEMPTS)) 173 | daemon.stop() 174 | if args.command == 'start' and args.no_fork: 175 | log.debug('Starting mongo-orchestration in the foreground') 176 | daemon.run() 177 | if args.command == 'restart': 178 | daemon.restart() 179 | 180 | 181 | if __name__ == "__main__": 182 | main() 183 | -------------------------------------------------------------------------------- /mongo_orchestration/singleton.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # coding=utf-8 3 | # Copyright 2012-2014 MongoDB, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | class Singleton(object): 19 | _instances = {} 20 | 21 | def __new__(class_, *args, **kwargs): 22 | if class_ not in class_._instances: 23 | class_._instances[class_] = super(Singleton, class_).__new__(class_, *args, **kwargs) 24 | return class_._instances[class_] 25 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["hatchling>=1.24"] 3 | build-backend = "hatchling.build" 4 | 5 | [project] 6 | name = "mongo-orchestration" 7 | dynamic = ["version"] 8 | description = "Restful service for managing MongoDB servers" 9 | readme = "README.rst" 10 | license = {file="LICENSE"} 11 | requires-python = ">=3.9" 12 | authors = [ 13 | { name = "The MongoDB Python Team" }, 14 | ] 15 | keywords = [ 16 | "mongo", 17 | "mongo-orchestration", 18 | "mongodb", 19 | "rest", 20 | "testing", 21 | ] 22 | classifiers = [ 23 | "Development Status :: 4 - Beta", 24 | "Intended Audience :: Developers", 25 | "License :: OSI Approved :: Apache Software License", 26 | "Operating System :: MacOS :: MacOS X", 27 | "Operating System :: Microsoft :: Windows", 28 | "Operating System :: POSIX", 29 | "Programming Language :: Python :: 3", 30 | "Programming Language :: Python :: 3.9", 31 | "Programming Language :: Python :: Implementation :: CPython", 32 | "Programming Language :: Python :: 3.10", 33 | "Programming Language :: Python :: 3.11", 34 | "Programming Language :: Python :: 3.12", 35 | "Programming Language :: Python :: 3.13", 36 | ] 37 | dependencies = [ 38 | "bottle>=0.12.7", 39 | "cheroot>=5.11", 40 | "pymongo>=4,<5", 41 | "requests", 42 | ] 43 | 44 | [project.optional-dependencies] 45 | test = [ 46 | "coverage>=3.5", 47 | "pexpect", 48 | "pytest", 49 | ] 50 | 51 | [project.scripts] 52 | mongo-launch = "mongo_orchestration.launch:main" 53 | mongo-orchestration = "mongo_orchestration.server:main" 54 | 55 | [project.urls] 56 | Homepage = "https://github.com/10gen/mongo-orchestration" 57 | 58 | [tool.hatch.version] 59 | path = "mongo_orchestration/_version.py" 60 | 61 | [tool.hatch.build.targets.sdist] 62 | include = [ 63 | "/mongo_orchestration", 64 | ] 65 | 66 | [tool.coverage.run] 67 | include = "mongo_orchestration/*" 68 | 69 | [tool.coverage.report] 70 | fail_under = 84 71 | 72 | [tool.pytest.ini_options] 73 | addopts = ["-raXs", "-v", "--durations", "10", "--color=yes"] -------------------------------------------------------------------------------- /scripts/mo: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2012-2014 MongoDB, Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | function eval_params { 17 | local params=$(sed -e 's|["]|\\\"|g' $1) 18 | echo $(eval echo \"$params\") 19 | } 20 | 21 | function r { 22 | echo $1| cut -d'/' -f 2 23 | } 24 | 25 | function a { 26 | echo $(cd $(dirname $1); pwd)/$(basename $1) 27 | } 28 | 29 | function id { 30 | local id_line=$(grep id $1 | head -n 1) 31 | echo $(expr "$id_line" : '.*: *"\(.*\)" *,*') 32 | } 33 | 34 | function get { 35 | echo "GET $1 $(curl --header 'Accept: application/json' --include --silent --request GET $1)" 36 | } 37 | 38 | function post { 39 | echo "POST $1 $(curl --header 'Accept: application/json' --include --silent --request POST --data "$2" $1)" 40 | } 41 | 42 | function delete { 43 | echo "DELETE $1 $(curl --header 'Accept: application/json' --include --silent --request DELETE $1)" 44 | } 45 | 46 | function code { 47 | expr "$1" : '.*HTTP/1.[01] \([0-9]*\)' 48 | } 49 | 50 | function usage { 51 | echo "usage: $0 configurations/cluster/file.json action" 52 | echo "cluster: servers|replica_sets|sharded_clusters" 53 | echo "action: start|status|stop" 54 | exit 1 55 | } 56 | 57 | SSL_FILES=$(a ./ssl-files) 58 | BASE_URL=${MONGO_ORCHESTRATION:-'http://localhost:8889'} 59 | 60 | if [ $# -ne 2 ]; then usage; fi 61 | if [ ! -f "$1" ]; then echo "configuration file '$1' not found"; exit 1; fi 62 | 63 | ID=$(id $1) 64 | if [ ! "$ID" ]; then echo "id field not found in configuration file '$1'"; exit 1; fi 65 | R=$(r $1) 66 | 67 | GET=$(get $BASE_URL/$R/$ID) 68 | HTTP_CODE=$(code "$GET") 69 | EXIT_CODE=0 70 | 71 | case $2 in 72 | start) 73 | if [ "$HTTP_CODE" != "200" ] 74 | then 75 | WORKSPACE=~/tmp/orchestrations 76 | rm -fr $WORKSPACE 77 | mkdir $WORKSPACE 78 | LOGPATH=$WORKSPACE 79 | DBPATH=$WORKSPACE 80 | POST_DATA=$(eval_params $1) 81 | echo "DBPATH=$DBPATH" 82 | echo "LOGPATH=$LOGPATH" 83 | echo "POST_DATA='$POST_DATA'" 84 | echo 85 | POST=$(post $BASE_URL/$R "$POST_DATA") 86 | echo "$POST" 87 | HTTP_CODE=$(code "$POST") 88 | if [ "$HTTP_CODE" != 200 ]; then EXIT_CODE=1; fi 89 | else 90 | echo "$GET" 91 | fi 92 | ;; 93 | stop) 94 | if [ "$HTTP_CODE" == "200" ] 95 | then 96 | DELETE=$(delete $BASE_URL/$R/$ID) 97 | echo "$DELETE" 98 | HTTP_CODE=$(code "$DELETE") 99 | if [ "$HTTP_CODE" != 204 ]; then EXIT_CODE=1; fi 100 | else 101 | echo "$GET" 102 | fi 103 | ;; 104 | status) 105 | if [ "$HTTP_CODE" == "200" ] 106 | then 107 | echo "$GET" 108 | else 109 | echo "$GET" 110 | EXIT_CODE=1 111 | fi 112 | ;; 113 | *) 114 | usage 115 | ;; 116 | esac 117 | exit $EXIT_CODE 118 | 119 | -------------------------------------------------------------------------------- /scripts/mongo-orchestration-setup.ps1: -------------------------------------------------------------------------------- 1 | # Copyright 2014 MongoDB, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | param([string]$server, [string]$configuration, [string]$authentication="noauth", [string]$ssl="nossl") 16 | 17 | echo "-------------------------------------------------------" 18 | echo "Server: $server" 19 | echo "Configuration: $configuration" 20 | echo "Authentication: $authentication" 21 | echo "SSL: $ssl" 22 | echo "-------------------------------------------------------" 23 | 24 | $BASEPATH_DOUBLEBACK=$env:BASEPATH -replace '\\','\\' 25 | 26 | # Note: backslashes must be escaped in the following string: 27 | $DATAPATH="$BASEPATH_DOUBLEBACK\\data" 28 | # Note: backslashes must be escaped in the following string: 29 | $SSL_FILES_ROOT="C:\\test-lib\\ssl-files" 30 | # This environment variable is injected by Jenkins. 31 | # Uncomment the following line to use this script outside of Jenkins: 32 | # $WORKSPACE="C:\\mongo" 33 | # Note: backslashes must be escaped in the following string: 34 | $LOGPATH="$BASEPATH_DOUBLEBACK\\logs" 35 | 36 | # Clean up files 37 | $ErrorActionPreference = 'SilentlyContinue' 38 | del -Recurse -Force $DATAPATH 39 | del -Recurse -Force $LOGPATH 40 | $ErrorActionPreference = 'Continue' 41 | 42 | md "$($DATAPATH)\db27016" 43 | md "$($DATAPATH)\db27017" 44 | md "$($DATAPATH)\db27018" 45 | md "$($DATAPATH)\db27019" 46 | md "$LOGPATH" 47 | 48 | if (($server -eq "22-release") -Or ($server -eq "20-release")) { 49 | $TEST_PARAMS='"vv" : true, ' 50 | } elseif ($server -eq "24-release") { 51 | $TEST_PARAMS='"setParameter" : "textSearchEnabled=true", "vv" : true, ' 52 | } else { 53 | $TEST_PARAMS='"setParameter":"enableTestCommands=1", "vv" : true, ' 54 | } 55 | 56 | if ($authentication -eq "auth") { 57 | $AUTH_PARAMS='"login":"bob", "password": "pwd123", "auth_key": "secret",' 58 | } 59 | 60 | if ($ssl -eq "ssl") { 61 | echo "Using SSL" 62 | $SSL_PARAMS="`"sslParams`": {`"tlsMode`": `"requireTLS`", `"tlsAllowInvalidCertificates`" : true, `"tlsCertificateKeyFile`":`"$($SSL_FILES_ROOT)\\server.pem`", `"tlsCAFile`": `"$($SSL_FILES_ROOT)\\ca.pem`", `"sslWeakCertificateValidation`" : true}," 63 | } 64 | 65 | echo "TEST_PARAMS=$TEST_PARAMS" 66 | echo "AUTH_PARAMS=$AUTH_PARAMS" 67 | echo "SSL_PARAMS=$SSL_PARAMS" 68 | 69 | echo "-------------------------------------------------------" 70 | echo "MongoDB Configuration: $configuration" 71 | echo "-------------------------------------------------------" 72 | 73 | $http_request = New-Object -ComObject Msxml2.XMLHTTP 74 | if ($configuration -eq "single_server") { 75 | $post_url = "http://localhost:8889/servers" 76 | $get_url = "http://localhost:8889/servers" 77 | $request_body="{$AUTH_PARAMS $SSL_PARAMS `"name`": `"mongod`", `"procParams`": {$TEST_PARAMS `"port`": 27017, `"dbpath`": `"$DATAPATH`", `"logpath`":`"$($LOGPATH)\\mongo.log`", `"ipv6`":true, `"logappend`":true, `"nojournal`":true}}" 78 | } elseif ($configuration -eq "replica_set") { 79 | $post_url = "http://localhost:8889/replica_sets" 80 | $get_url = "http://localhost:8889/replica_sets/repl0" 81 | $request_body="{$AUTH_PARAMS $SSL_PARAMS `"id`": `"repl0`", `"members`":[{`"rsParams`":{`"priority`": 99}, `"procParams`": {$TEST_PARAMS `"dbpath`":`"$($DATAPATH)\\db27017`", `"port`": 27017, `"logpath`":`"$($LOGPATH)\\db27017.log`", `"nojournal`":false, `"oplogSize`": 150, `"ipv6`": true}}, {`"rsParams`": {`"priority`": 1.1}, `"procParams`":{$TEST_PARAMS `"dbpath`":`"$($DATAPATH)\\db27018`", `"port`": 27018, `"logpath`":`"$($LOGPATH)\\db27018.log`", `"nojournal`":false, `"oplogSize`": 150, `"ipv6`": true}}, {`"procParams`":{`"dbpath`":`"$($DATAPATH)\\db27019`", `"port`": 27019, `"logpath`":`"$($LOGPATH)\\27019.log`", `"nojournal`":false, `"oplogSize`": 150, `"ipv6`": true}}]}" 82 | } elseif ($configuration -eq "sharded") { 83 | $post_url = "http://localhost:8889/sharded_clusters" 84 | $get_url = "http://localhost:8889/sharded_clusters/shard_cluster_1" 85 | $request_body = "{$AUTH_PARAMS $SSL_PARAMS `"routers`": [{$TEST_PARAMS `"port`": 27017, `"logpath`": `"$LOGPATH\\router27017.log`"}, {$TEST_PARAMS `"port`": 27018, `"logpath`": `"$LOGPATH\\router27018.log`"}], `"configsvrs`": [{`"port`": 27016, `"dbpath`": `"$DATAPATH\\db27016`", `"logpath`": `"$LOGPATH\\configsvr27016.log`"}], `"id`": `"shard_cluster_1`", `"members`": [{`"id`": `"sh01`", `"shardParams`": {`"procParams`": {$TEST_PARAMS `"port`": 27020, `"dbpath`": `"$DATAPATH\\db27020`", `"logpath`":`"$LOGPATH\\db27020.log`", `"ipv6`":true, `"logappend`":true, `"nojournal`":false}}}]}" 86 | } else{ 87 | echo "Unrecognized configuration: $configuration" 88 | exit 1 89 | } 90 | echo "Sending $request_body to $post_url" 91 | $http_request.open('POST', $post_url, $false) 92 | $http_request.setRequestHeader("Content-Type", "application/json") 93 | $http_request.setRequestHeader("Accept", "application/json") 94 | $http_request.send($request_body) 95 | $response = $http_request.statusText 96 | 97 | $get_request = New-Object -ComObject Msxml2.XMLHTTP 98 | $get_request.open('GET', $get_url, $false) 99 | $get_request.setRequestHeader("Accept", "application/json") 100 | $get_request.send("") 101 | echo $get_request.statusText 102 | -------------------------------------------------------------------------------- /scripts/mongo-orchestration-start.ps1: -------------------------------------------------------------------------------- 1 | # Copyright 2014 MongoDB, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | param([string]$configuration_file="mongo-orchestration-windows.config", [string]$server, [string]$python_bin="C:\\Python27\\python.exe") 16 | 17 | echo "====== CLEANUP ======" 18 | echo "*** Killing any existing MongoDB Processes which may not have shut down on a prior job." 19 | 20 | # Cleanup code may raise errors if there are no such processes running, directories to remove, etc. 21 | # Don't show these in the build log. 22 | $ErrorActionPreference = 'SilentlyContinue' 23 | 24 | $mongods = (Get-Process mongod) 25 | echo "Found existing mongod Processes: $mongods" 26 | $mongoss = (Get-Process mongos) 27 | echo "Found existing mongos Processes: $mongoss" 28 | Stop-Process -InputObject $mongods 29 | Stop-Process -InputObject $mongoss 30 | 31 | $pythons = (Get-Process python) 32 | foreach ($python in $pythons) { 33 | $procid = $python.id 34 | $wmi = (Get-WmiObject Win32_Process -Filter "Handle = '$procid'") 35 | if ($wmi.CommandLine -like "*server.py*") { 36 | Stop-Process -id $wmi.Handle 37 | } 38 | } 39 | 40 | echo "remove old files from $env:BASEPATH" 41 | del -Recurse -Force "$($env:BASEPATH)\data" 42 | del -Recurse -Force "$($env:BASEPATH)\logs" 43 | 44 | # Start caring about errors messages again. 45 | $ErrorActionPreference = 'Continue' 46 | 47 | echo "====== END CLEANUP ======" 48 | 49 | echo "Copying mongo-orchestration to $env:WORKSPACE\mongo-orchestration" 50 | copy-item -recurse D:\jenkins\mongo-orchestration $env:WORKSPACE\mongo-orchestration 51 | cd $env:WORKSPACE\mongo-orchestration 52 | 53 | echo "Start-Process -FilePath $python_bin -ArgumentList server.py,start,-f,$configuration_file,-e,$server,--no-fork" 54 | Start-Process -FilePath $python_bin -ArgumentList server.py,start,-f,$($configuration_file),-e,$($server),--no-fork 55 | 56 | $connected = $False 57 | $ErrorActionPreference = 'SilentlyContinue' 58 | echo "Waiting for Mongo Orchestration to become available..." 59 | for ( $attempts = 0; $attempts -lt 1000 -and ! $connected; $attempts++ ) { 60 | $s = New-Object Net.Sockets.TcpClient 61 | $s.Connect("localhost", 8889) 62 | if ($s.Connected) { 63 | $connected = $True 64 | } else { 65 | Start-Sleep -m 100 66 | } 67 | } 68 | if (! $connected) { 69 | throw ("Could not connect to Mongo Orchestration.") 70 | } 71 | $ErrorActionPreference = 'Continue' 72 | -------------------------------------------------------------------------------- /scripts/mongo-orchestration-stop.ps1: -------------------------------------------------------------------------------- 1 | # Copyright 2014 MongoDB, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | param([string]$python_bin="C:\\Python27\\python.exe") 16 | 17 | cd $env:WORKSPACE\mongo-orchestration 18 | & $python_bin server.py stop 19 | 20 | echo "====== CLEANUP ======" 21 | 22 | # Cleanup code may raise errors if there are no such processes running, directories to remove, etc. 23 | # Don't show these in the build log. 24 | $ErrorActionPreference = 'SilentlyContinue' 25 | 26 | echo "*** Killing any existing MongoDB Processes which may not have shut down on a prior job." 27 | $mongods = (Get-Process mongod) 28 | echo "Found existing mongod Processes: $mongods" 29 | $mongoss = (Get-Process mongos) 30 | echo "Found existing mongos Processes: $mongoss" 31 | Stop-Process -InputObject $mongods 32 | Stop-Process -InputObject $mongoss 33 | 34 | $pythons = (Get-Process python) 35 | foreach ($python in $pythons) { 36 | $procid = $python.id 37 | $wmi = (Get-WmiObject Win32_Process -Filter "Handle = '$procid'") 38 | if ($wmi.CommandLine -like "*server.py*") { 39 | Stop-Process -id $wmi.Handle 40 | } 41 | } 42 | 43 | echo "remove old files from $env:BASEPATH" 44 | del -Recurse -Force $env:BASEPATH\data 45 | del -Recurse -Force $env:BASEPATH\logs 46 | echo "====== END CLEANUP ======" 47 | 48 | # Start caring about errors messages again. 49 | $ErrorActionPreference = 'Continue' 50 | -------------------------------------------------------------------------------- /scripts/server: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2012-2014 MongoDB, Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | MONGO_ORCHESTRATION_DIR=~/mongo-orchestration 17 | PY=python 18 | 19 | DEFAULT_CONFIG=$MONGO_ORCHESTRATION_DIR/mongo-orchestration.config 20 | DEFAULT_RELEASE=stable-release 21 | DEFAULT_PORT=8889 22 | DEFAULT_MODE= 23 | 24 | 25 | printHelp() { 26 | echo "usage: server-restart [-h] [-f CONFIG] [-e ENV] [--no-fork] [-p PORT] {start|stop|restart}" 27 | 28 | echo "commands: start/stop/restart - default start" 29 | echo "" 30 | echo "optional arguments:" 31 | echo " -h, show this help message and exit" 32 | echo " -f CONFIG path to config file" 33 | echo " -e ENV mongo release" 34 | echo " --no-fork server mode" 35 | echo " -p PORT port number" 36 | 37 | } 38 | 39 | ARG_HELP=false 40 | ARG_MODE= 41 | ARG_COMMAND=start 42 | CONFIG=$DEFAULT_CONFIG 43 | RELEASE=$DEFAULT_RELEASE 44 | PORT=$DEFAULT_PORT 45 | 46 | shopt -s extglob 47 | 48 | while [[ $# -ge 1 ]]; do 49 | case "$1" in 50 | -h) 51 | ARG_HELP=true 52 | ;; 53 | --help) 54 | ARG_HELP=true 55 | ;; 56 | -f) 57 | CONFIG=${2#} 58 | ;; 59 | -f=+([[:alpha:]_-.])) 60 | CONFIG=${1#--foo=} 61 | ;; 62 | -e) 63 | RELEASE=${2#} 64 | ;; 65 | -e=+([[:alpha:]_-.])) 66 | RELEASE=${1#--other=} 67 | ;; 68 | -p) 69 | PORT=${2#} 70 | ;; 71 | -p=+([[:alpha:]_-.])) 72 | PORT=${1#--other=} 73 | ;; 74 | --no-fork) 75 | ARG_MODE=--no-fork 76 | ;; 77 | start) 78 | ARG_COMMAND=start 79 | ;; 80 | restart) 81 | ARG_COMMAND=restart 82 | ;; 83 | stop) 84 | ARG_COMMAND=stop 85 | ;; 86 | *) 87 | # ... invalid argument? 88 | ;; 89 | esac 90 | shift 91 | done 92 | 93 | if [ $ARG_HELP == true ]; then 94 | printHelp 95 | exit 96 | fi 97 | 98 | echo $PY $MONGO_ORCHESTRATION_DIR/server.py $ARG_COMMAND -f $CONFIG -e $RELEASE -p $PORT $ARG_MODE 99 | $PY $MONGO_ORCHESTRATION_DIR/server.py $ARG_COMMAND -f $CONFIG -e $RELEASE -p $PORT $ARG_MODE 100 | -------------------------------------------------------------------------------- /scripts/setup-configuration: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2012-2014 MongoDB, Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | WORKSPACE=/tmp/mongo-orchestration 17 | LOGPATH=$WORKSPACE/log 18 | DBPATH=$WORKSPACE/db 19 | 20 | mkdir -p "$LOGPATH" 21 | mkdir -p "$DBPATH" 22 | 23 | BASE_URL=http://localhost:8889 24 | 25 | cleanupAll() { 26 | rm -rf $DBPATH/* 27 | rm -rf $LOGPATH/* 28 | rm -rf $WORKSPACE/* 29 | } 30 | 31 | cleanupSingleServer() { 32 | rm -rf $DBPATH/db27017 33 | rm $LOGPATH/mongod27017.log 34 | } 35 | 36 | cleanupReplicaSet() { 37 | rm -rf $DBPATH/db27017 38 | rm -rf $DBPATH/db27018 39 | rm -rf $DBPATH/db27019 40 | rm $LOGPATH/mongod27017.log $LOGPATH/mongod27018.log $LOGPATH/mongod27019.log 41 | } 42 | 43 | cleanupShard() { 44 | rm -rf $DBPATH/db27017 45 | rm -rf $DBPATH/db27018 46 | rm -rf $DBPATH/db27019 47 | rm -rf $DBPATH/db27020 48 | rm $LOGPATH/mongod27017.log $LOGPATH/mongod27018.log $LOGPATH/mongod27019.log $LOGPATH/mongod27019.log 49 | } 50 | 51 | 52 | if [ $# -ne 1 ]; then 53 | echo "usage: setup-configuration config " 54 | exit 1 55 | fi 56 | 57 | date 58 | echo "" 59 | if [ "$1" == "single_server" ]; then 60 | cleanupSingleServer 61 | params="{\"name\": \"mongod\", \"params\": {\"port\": 27017, \"dbpath\": \"$DBPATH/db27017\", \"logpath\":\"$LOGPATH/mongod27017.log\", \"ipv6\":true, \"verbose\":\"vvvvv\", \"logappend\":true, \"nojournal\":true}}" 62 | echo curl -i -H "Accept: application/json" -X POST -d "$params" $BASE_URL/hosts 63 | echo "" 64 | curl -i -H "Accept: application/json" -X POST -d "$params" $BASE_URL/hosts 65 | echo "" 66 | echo curl -i -H "Accept: application/json" -X GET $BASE_URL/hosts 67 | echo "" 68 | curl -i -H "Accept: application/json" -X GET $BASE_URL/hosts 69 | 70 | elif [ "$1" == "replica_set" ]; then 71 | cleanupReplicaSet 72 | params="{\"id\": \"repl0\", \"members\":[{\"rsParams\": {\"priority\": 1.1}, \"procParams\":{\"dbpath\":\"$DBPATH/db27018\", \"port\": 27018, \"logpath\":\"$LOGPATH/db27018.log\", \"nojournal\":true, \"oplogSize\": 150, \"verbose\": \"vvvvv\", \"ipv6\": true}}, {\"rsParams\":{\"priority\": 99}, \"procParams\": {\"dbpath\":\"$DBPATH/db27017\", \"port\": 27017, \"logpath\":\"$LOGPATH/db27017.log\", \"nojournal\":true, \"oplogSize\": 150, \"verbose\": \"vvvvv\", \"ipv6\": true}}, {\"procParams\":{\"dbpath\":\"$DBPATH/db27019\", \"port\": 27019, \"logpath\":\"$LOGPATH/27019.log\", \"nojournal\":true, \"oplogSize\": 150, \"verbose\": \"vvvvv\", \"ipv6\": true}}]}" 73 | echo curl -i -H "Accept: application/json" -X POST -d "$params" $BASE_URL/rs 74 | echo 75 | curl -i -H "Accept: application/json" -X POST -d "$params" $BASE_URL/rs 76 | echo 77 | echo curl -i -H "Accept: application/json" -X GET $BASE_URL/rs/repl0/members 78 | echo 79 | curl -i -H "Accept: application/json" -X GET $BASE_URL/rs/repl0/members 80 | 81 | elif [ "$1" == "shard" ]; then 82 | cleanupShard 83 | params="{\"routers\": [{\"port\": 27017}], \"configsvrs\": [{\"port\": 27020, \"dbpath\": \"$DBPATH/db27020\"}], \"id\": \"shard_cluster_1\", \"members\": [{\"id\": \"sh01\", \"shardParams\": {\"port\": 27018, \"dbpath\": \"$DBPATH/db27018\"}}, {\"shardParams\": {\"port\": 27019, \"dbpath\":\"$DBPATH/db27019\"}, \"id\": \"sh02\"}, {\"shardParams\": {\"id\": \"rs1\", \"members\": [{}, {}]}, \"id\": \"default\"}]}" 84 | echo curl -i -H "Accept: application/json" -X POST -d "$params" $BASE_URL/sh 85 | echo "" 86 | curl -i -H "Accept: application/json" -X POST -d "$params" $BASE_URL/sh 87 | echo "" 88 | echo curl -i -H "Accept: application/json" -X GET $BASE_URL/sh/shard_cluster_1 89 | echo "" 90 | curl -i -H "Accept: application/json" -X GET $BASE_URL/sh/shard_cluster_1 91 | 92 | else 93 | echo "unknown configuration" 94 | exit 1 95 | fi 96 | echo "" 97 | date 98 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | from setuptools import setup 3 | setup() -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2014 MongoDB, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import os 16 | import re 17 | import time 18 | import unittest 19 | from unittest import SkipTest 20 | 21 | from mongo_orchestration import set_releases 22 | from mongo_orchestration.servers import Servers 23 | 24 | PORT = int(os.environ.get('MO_PORT', '8889')) 25 | HOSTNAME = os.environ.get('MO_HOST', 'localhost') 26 | TEST_SUBJECT = ( 27 | 'C=US,ST=New York,L=New York City,O=MongoDB,OU=KernelUser,CN=client_revoked' 28 | ) 29 | TEST_RELEASES = ( 30 | {'default-release': os.environ.get('MONGOBIN', '')}, 31 | 'default-release') 32 | 33 | # Set up the default mongo binaries to use from MONGOBIN. 34 | set_releases(*TEST_RELEASES) 35 | 36 | SSL_ENABLED = False 37 | SERVER_VERSION = (2, 6) 38 | __server_id = Servers().create(name='mongod', procParams={}) 39 | try: 40 | # Server version 41 | info = Servers().info(__server_id)['serverInfo'] 42 | version_str = re.search(r'((\d+\.)+\d+)', info['version']).group(0) 43 | SERVER_VERSION = tuple(map(int, version_str.split('.'))) 44 | # Do we have SSL support? 45 | SSL_ENABLED = bool(info.get('OpenSSLVersion')) 46 | finally: 47 | Servers().cleanup() 48 | 49 | 50 | class SSLTestCase(unittest.TestCase): 51 | 52 | @classmethod 53 | def setUpClass(cls): 54 | if not SSL_ENABLED: 55 | raise SkipTest("SSL is not enabled on this server.") 56 | 57 | 58 | def certificate(cert_name): 59 | """Return the path to the PEM file with the given name.""" 60 | from mongo_orchestration import __path__ 61 | mo_path = list(__path__)[0] 62 | return os.path.join(mo_path, 'lib', cert_name) 63 | 64 | 65 | def assert_eventually(condition, message=None, max_tries=60): 66 | for i in range(max_tries): 67 | if condition(): 68 | break 69 | time.sleep(1) 70 | else: 71 | raise AssertionError(message or "Failed after %d attempts." % max_tries) 72 | -------------------------------------------------------------------------------- /tests/test_container.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # coding=utf-8 3 | # Copyright 2012-2014 MongoDB, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import os 18 | import operator 19 | import sys 20 | 21 | from bson import SON 22 | 23 | sys.path.insert(0, '../') 24 | 25 | from mongo_orchestration.container import Container 26 | from mongo_orchestration.errors import MongoOrchestrationError 27 | from tests import unittest 28 | 29 | 30 | class ContainerTestCase(unittest.TestCase): 31 | def setUp(self): 32 | self.container = Container() 33 | self.container.set_settings() 34 | 35 | def tearDown(self): 36 | self.container.cleanup() 37 | 38 | def test_set_settings(self): 39 | default_release = 'old-release' 40 | releases = {default_release: os.path.join(os.getcwd(), 'bin')} 41 | orig_releases = self.container.releases 42 | orig_default_release = self.container.default_release 43 | try: 44 | self.container.set_settings(releases, default_release) 45 | self.assertEqual(releases, self.container.releases) 46 | self.assertEqual(default_release, self.container.default_release) 47 | finally: 48 | self.container.set_settings(orig_releases, orig_default_release) 49 | 50 | def test_bin_path(self): 51 | releases = SON([('20-release', '/path/to/20/release'), 52 | ('24.9-release', '/path/to/24.9/release'), 53 | ('24-release', '/path/to/24/release'), 54 | ('26-release', '/path/to/26/release')]) 55 | default_release = '26-release' 56 | self.container.set_settings(releases, default_release) 57 | self.assertRaises(MongoOrchestrationError, 58 | self.container.bin_path, '27') 59 | self.assertEqual(self.container.bin_path('20'), 60 | releases['20-release']) 61 | self.assertEqual(self.container.bin_path('24'), 62 | releases['24.9-release']) 63 | self.assertEqual(self.container.bin_path(), releases[default_release]) 64 | # Clear default release. 65 | self.container.set_settings(releases) 66 | self.assertEqual(self.container.bin_path(), releases['20-release']) 67 | # Clear all releases. 68 | self.container.set_settings({}) 69 | self.assertEqual(self.container.bin_path(), '') 70 | 71 | def test_getitem(self): 72 | self.container['key'] = 'value' 73 | self.assertEqual('value', self.container['key']) 74 | self.assertRaises(KeyError, operator.getitem, self.container, 'error-key') 75 | 76 | def test_setitem(self): 77 | self.assertEqual(None, operator.setitem(self.container, 'key', 'value')) 78 | self.container._obj_type = int 79 | self.assertEqual(None, operator.setitem(self.container, 'key2', 15)) 80 | self.assertRaises(ValueError, operator.setitem, self.container, 'key3', 'value') 81 | 82 | def test_delitem(self): 83 | self.assertEqual(0, len(self.container)) 84 | self.container['key'] = 'value' 85 | self.assertEqual(1, len(self.container)) 86 | self.assertEqual(None, operator.delitem(self.container, 'key')) 87 | self.assertEqual(0, len(self.container)) 88 | 89 | def test_operations(self): 90 | self.assertEqual(0, len(self.container)) 91 | keys = ('key1', 'key2', 'key3') 92 | values = ('value1', 'value2', 'value3') 93 | for key, value in zip(keys, values): 94 | self.container[key] = value 95 | self.assertEqual(len(keys), len(self.container)) 96 | # test contains 97 | for key in keys: 98 | self.assertTrue(key in self.container) 99 | # test iteration 100 | for key in self.container: 101 | self.assertTrue(key in keys) 102 | self.assertTrue(self.container[key] in values) 103 | 104 | # test cleanup 105 | self.container.cleanup() 106 | self.assertEqual(0, len(self.container)) 107 | 108 | def test_bool(self): 109 | self.assertEqual(False, bool(self.container)) 110 | self.container['key'] = 'value' 111 | self.assertTrue(True, bool(self.container)) 112 | 113 | def test_notimplemented(self): 114 | self.assertRaises(NotImplementedError, self.container.create) 115 | self.assertRaises(NotImplementedError, self.container.remove) 116 | self.assertRaises(NotImplementedError, self.container.info) 117 | 118 | if __name__ == '__main__': 119 | unittest.main() 120 | -------------------------------------------------------------------------------- /tests/test_launch.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # coding=utf-8 3 | # Copyright 2023 MongoDB, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | import os 17 | import shlex 18 | import pexpect 19 | import subprocess 20 | import unittest 21 | 22 | def run(cmd, **kwargs): 23 | proc = subprocess.run(shlex.split(cmd), **kwargs) 24 | if proc.returncode != 0: 25 | raise RuntimeError('Process failed!') 26 | 27 | 28 | class TestLaunch(unittest.TestCase): 29 | 30 | def test_launch_single(self): 31 | if os.name != 'posix': 32 | raise unittest.SkipTest('Only works on posix!') 33 | run('mongo-orchestration start') 34 | proc = pexpect.spawn('mongo-launch', ['single']) 35 | proc.expect('Type "q" to quit:') 36 | proc.send('q\n') 37 | proc.wait() 38 | self.assertEqual(proc.exitstatus, 0) 39 | run('mongo-orchestration stop') 40 | 41 | def test_launch_replica_set(self): 42 | if os.name != 'posix': 43 | raise unittest.SkipTest('Only works on posix!') 44 | run('mongo-orchestration start') 45 | proc = pexpect.spawn('mongo-launch', ['replicaset', 'ssl']) 46 | proc.expect('"r" to shutdown and restart the primary') 47 | proc.send('q\n') 48 | proc.wait() 49 | self.assertEqual(proc.exitstatus, 0) 50 | run('mongo-orchestration stop') 51 | 52 | def test_launch_sharded(self): 53 | if os.name != 'posix': 54 | raise unittest.SkipTest('Only works on posix!') 55 | run('mongo-orchestration start') 56 | proc = pexpect.spawn('mongo-launch', ['shard', 'auth']) 57 | proc.expect('Type "q" to quit:') 58 | proc.send('q\n') 59 | proc.wait() 60 | self.assertEqual(proc.exitstatus, 0) 61 | run('mongo-orchestration stop') -------------------------------------------------------------------------------- /tests/test_process.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # coding=utf-8 3 | # Copyright 2012-2014 MongoDB, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import os 18 | import platform 19 | import random 20 | import socket 21 | import subprocess 22 | import sys 23 | import tempfile 24 | 25 | sys.path.insert(0, '../') 26 | 27 | import mongo_orchestration.process as process 28 | 29 | from mongo_orchestration.errors import TimeoutError 30 | 31 | from tests import unittest, SkipTest, HOSTNAME 32 | 33 | 34 | class PortPoolTestCase(unittest.TestCase): 35 | 36 | def setUp(self): 37 | self.hostname = HOSTNAME 38 | self.pp = process.PortPool() 39 | self.pp.change_range(min_port=1025, max_port=1080) 40 | self.sockets = {} 41 | 42 | def tearDown(self): 43 | for s in self.sockets: 44 | self.sockets[s].close() 45 | 46 | def listen_port(self, port, max_connection=0): 47 | if self.sockets.get(port, None): 48 | self.sockets[port].close() 49 | 50 | s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 51 | s.bind((HOSTNAME, port)) 52 | s.listen(max_connection) 53 | self.sockets[port] = s 54 | 55 | def test_singleton(self): 56 | pp2 = process.PortPool(min_port=1025, max_port=1038) 57 | self.assertEqual(id(self.pp), id(pp2)) 58 | 59 | def test_port_sequence(self): 60 | ports = set([1025, 1026, 1027, 1028, 30, 28, 22, 45]) 61 | self.pp.change_range(port_sequence=ports) 62 | _ports = self.pp._PortPool__closed.union(self.pp._PortPool__ports) 63 | self.assertEqual(ports, _ports) 64 | 65 | def test_find_port(self): 66 | port = self.pp.port() 67 | self.pp.change_range(port, port) 68 | port = self.pp.port() 69 | self.assertTrue(port > 0) 70 | self.listen_port(port) 71 | self.assertRaises(IndexError, self.pp.port) 72 | 73 | def test_port_with_check(self): 74 | self.pp.change_range(min_port=1100, max_port=1200) 75 | port1, port2 = self.pp.port(check=True), self.pp.port(check=True) 76 | self.pp.change_range(port_sequence=[port1, port2]) 77 | self.listen_port(port1, 0) 78 | self.assertTrue(port2 == self.pp.port(check=True)) 79 | 80 | def test_check_port(self): 81 | port = self.pp.port(check=True) 82 | self.assertTrue(self.pp._PortPool__check_port(port)) 83 | self.listen_port(port) 84 | self.assertFalse(self.pp._PortPool__check_port(port)) 85 | 86 | def test_release_port(self): 87 | port = self.pp.port(check=True) 88 | self.assertTrue(port in self.pp._PortPool__closed) 89 | self.pp.release_port(port) 90 | self.assertFalse(port in self.pp._PortPool__closed) 91 | 92 | def test_refresh(self): 93 | ports = set([random.randint(1025, 2000) for i in range(15)]) 94 | self.pp.change_range(port_sequence=ports) 95 | ports_opened = self.pp._PortPool__ports.copy() 96 | test_port = ports_opened.pop() 97 | self.assertTrue(test_port in self.pp._PortPool__ports) 98 | self.assertTrue(len(self.pp._PortPool__ports) > 1) 99 | for port in ports: 100 | if port != test_port: 101 | try: 102 | self.listen_port(port) 103 | except (socket.error): 104 | pass 105 | 106 | self.pp.refresh() 107 | self.assertTrue(len(self.pp._PortPool__ports) == 1) 108 | 109 | def test_refresh_only_closed(self): 110 | ports = set([random.randint(1025, 2000) for _ in range(15)]) 111 | self.pp.change_range(port_sequence=ports) 112 | closed_num = len(self.pp._PortPool__closed) 113 | self.pp.port(), self.pp.port() 114 | self.assertTrue(closed_num + 2 == len(self.pp._PortPool__closed)) 115 | 116 | ports_opened = self.pp._PortPool__ports.copy() 117 | test_port = ports_opened.pop() 118 | self.listen_port(test_port) 119 | self.pp.refresh(only_closed=True) 120 | self.assertTrue(closed_num == len(self.pp._PortPool__closed)) 121 | 122 | self.pp.refresh() 123 | self.assertTrue(closed_num + 1 == len(self.pp._PortPool__closed)) 124 | 125 | def test_change_range(self): 126 | self.pp.change_range(min_port=1025, max_port=1033) 127 | ports = self.pp._PortPool__closed.union(self.pp._PortPool__ports) 128 | self.assertTrue(ports == set(range(1025, 1033 + 1))) 129 | 130 | random_ports = set([random.randint(1025, 2000) for i in range(15)]) 131 | self.pp.change_range(port_sequence=random_ports) 132 | ports = self.pp._PortPool__closed.union(self.pp._PortPool__ports) 133 | self.assertTrue(ports == random_ports) 134 | 135 | 136 | class ProcessTestCase(unittest.TestCase): 137 | def setUp(self): 138 | self.hostname = HOSTNAME 139 | self.s = None 140 | self.executable = sys.executable 141 | self.pp = process.PortPool(min_port=1025, max_port=2000) 142 | self.sockets = {} 143 | self.tmp_files = list() 144 | self.bin_path = os.path.join(os.environ.get('MONGOBIN', ''), 'mongod') 145 | self.db_path = tempfile.mkdtemp() 146 | self.cfg = {"oplogSize": 10, 'dbpath': self.db_path} 147 | 148 | def tearDown(self): 149 | for s in self.sockets: 150 | self.sockets[s].close() 151 | if self.cfg: 152 | process.cleanup_mprocess('', self.cfg) 153 | for item in self.tmp_files: 154 | if os.path.exists(item): 155 | os.remove(item) 156 | 157 | def listen_port(self, port, max_connection=0): 158 | if self.sockets.get(port, None): 159 | self.sockets[port].close() 160 | 161 | s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 162 | s.bind((HOSTNAME, port)) 163 | s.listen(max_connection) 164 | self.sockets[port] = s 165 | 166 | def test_repair(self): 167 | port = self.pp.port(check=True) 168 | # Assume we're testing on 64-bit machines. 169 | self.cfg['nojournal'] = True 170 | lock_file = os.path.join(self.cfg['dbpath'], 'mongod.lock') 171 | config_path = process.write_config(self.cfg) 172 | self.tmp_files.append(config_path) 173 | proc, host = process.mprocess(self.bin_path, config_path, port=port, timeout=60) 174 | self.assertTrue(os.path.exists(lock_file)) 175 | if platform.system() == 'Windows': 176 | # mongod.lock cannot be read by any external process on Windows. 177 | with self.assertRaises(IOError): 178 | open(lock_file, 'r') 179 | else: 180 | with open(lock_file, 'r') as fd: 181 | self.assertGreater(len(fd.read()), 0) 182 | proc.terminate() 183 | proc.communicate() 184 | process.repair_mongo(self.bin_path, self.cfg['dbpath']) 185 | with open(lock_file, 'r') as fd: 186 | contents = fd.read() 187 | self.assertEqual(len(contents), 0, 188 | "lock_file contains: " + contents) 189 | 190 | def test_mprocess_fail(self): 191 | fd_cfg, config_path = tempfile.mkstemp() 192 | os.close(fd_cfg) 193 | self.tmp_files.append(config_path) 194 | self.assertRaises(OSError, process.mprocess, 195 | 'fake-process_', config_path, None, 30) 196 | process.write_config({"fake": True}, config_path) 197 | self.assertRaises(OSError, process.mprocess, 198 | self.bin_path, config_path, None, 30) 199 | 200 | def test_mprocess(self): 201 | port = self.pp.port(check=True) 202 | config_path = process.write_config(self.cfg) 203 | self.tmp_files.append(config_path) 204 | result = process.mprocess(self.bin_path, config_path, port=port) 205 | self.assertTrue(isinstance(result, tuple)) 206 | proc, host = result 207 | self.assertTrue(isinstance(proc, subprocess.Popen)) 208 | self.assertTrue(isinstance(host, str)) 209 | process.kill_mprocess(proc) 210 | 211 | def test_mprocess_busy_port(self): 212 | config_path = process.write_config(self.cfg) 213 | self.tmp_files.append(config_path) 214 | port = self.pp.port() 215 | self.listen_port(port, max_connection=0) 216 | proc, host = process.mprocess(self.executable, config_path, 217 | port=port, timeout=2) 218 | self.assertTrue(proc.pid > 0) 219 | self.assertEqual(host, self.hostname + ':' + str(port)) 220 | self.sockets.pop(port).close() 221 | self.assertRaises(OSError, process.mprocess, 222 | self.executable, '', port, 1) 223 | 224 | def test_kill_mprocess(self): 225 | p = subprocess.Popen([self.executable]) 226 | self.assertTrue(process.proc_alive(p)) 227 | process.kill_mprocess(p) 228 | self.assertFalse(process.proc_alive(p)) 229 | 230 | def test_cleanup_process(self): 231 | fd_cfg, config_path = tempfile.mkstemp() 232 | fd_key, key_file = tempfile.mkstemp() 233 | fd_log, log_path = tempfile.mkstemp() 234 | db_path = tempfile.mkdtemp() 235 | self.assertTrue(os.path.exists(config_path)) 236 | self.assertTrue(os.path.exists(key_file)) 237 | self.assertTrue(os.path.exists(log_path)) 238 | self.assertTrue(os.path.exists(db_path)) 239 | with os.fdopen(fd_cfg, 'w') as fd: 240 | fd.write('keyFile={key_file}\n' 241 | 'logPath={log_path}\n' 242 | 'dbpath={db_path}'.format(**locals())) 243 | for fd in (fd_cfg, fd_key, fd_log): 244 | try: 245 | os.close(fd) 246 | except OSError: 247 | # fd_cfg may be closed already if fdopen() didn't raise 248 | pass 249 | cfg = {'keyFile': key_file, 'logpath': log_path, 'dbpath': db_path} 250 | process.cleanup_mprocess(config_path, cfg) 251 | self.assertFalse(os.path.exists(config_path)) 252 | self.assertFalse(os.path.exists(key_file)) 253 | self.assertTrue(os.path.exists(log_path)) 254 | self.assertFalse(os.path.exists(db_path)) 255 | process.remove_path(log_path) 256 | self.assertFalse(os.path.exists(log_path)) 257 | 258 | def test_remove_path(self): 259 | fd, file_path = tempfile.mkstemp() 260 | os.close(fd) 261 | self.assertTrue(os.path.exists(file_path)) 262 | process.remove_path(file_path) 263 | self.assertFalse(os.path.exists(file_path)) 264 | 265 | dir_path = tempfile.mkdtemp() 266 | fd, file_path = tempfile.mkstemp(dir=dir_path) 267 | os.close(fd) 268 | process.remove_path(dir_path) 269 | self.assertFalse(os.path.exists(file_path)) 270 | self.assertFalse(os.path.exists(dir_path)) 271 | 272 | def test_write_config(self): 273 | cfg = {'port': 27017, 'objcheck': 'true'} 274 | config_path = process.write_config(cfg) 275 | self.assertTrue(os.path.exists(config_path)) 276 | with open(config_path, 'r') as fd: 277 | config_data = fd.read() 278 | self.assertTrue('port=27017' in config_data) 279 | self.assertTrue('objcheck=true' in config_data) 280 | process.cleanup_mprocess(config_path, cfg) 281 | 282 | def test_write_config_with_specify_config_path(self): 283 | cfg = {'port': 27017, 'objcheck': 'true'} 284 | fd_key, file_path = tempfile.mkstemp() 285 | os.close(fd_key) 286 | config_path = process.write_config(cfg, file_path) 287 | self.assertEqual(file_path, config_path) 288 | process.cleanup_mprocess(config_path, cfg) 289 | 290 | def test_proc_alive(self): 291 | p = subprocess.Popen([self.executable]) 292 | self.assertTrue(process.proc_alive(p)) 293 | p.terminate() 294 | p.wait() 295 | self.assertFalse(process.proc_alive(p)) 296 | self.assertFalse(process.proc_alive(None)) 297 | 298 | def test_read_config(self): 299 | cfg = {"oplogSize": 10, "other": "some string"} 300 | config_path = process.write_config(cfg) 301 | self.tmp_files.append(config_path) 302 | self.assertEqual(process.read_config(config_path), cfg) 303 | 304 | 305 | if __name__ == '__main__': 306 | unittest.main() 307 | -------------------------------------------------------------------------------- /tests/test_singleton.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # coding=utf-8 3 | # Copyright 2012-2014 MongoDB, Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import sys 18 | 19 | sys.path.insert(0, '../') 20 | 21 | from mongo_orchestration.singleton import Singleton 22 | from tests import unittest 23 | 24 | 25 | class SingletonTestCase(unittest.TestCase): 26 | 27 | def test_singleton(self): 28 | a = Singleton() 29 | b = Singleton() 30 | self.assertEqual(id(a), id(b)) 31 | c = Singleton() 32 | self.assertEqual(id(c), id(b)) 33 | 34 | 35 | if __name__ == '__main__': 36 | unittest.main() 37 | --------------------------------------------------------------------------------