├── .gitignore ├── .idea ├── crmngr-project.iml ├── encodings.xml ├── inspectionProfiles │ └── profiles_settings.xml ├── misc.xml ├── modules.xml └── vcs.xml ├── CHANGELOG.rst ├── LICENSE ├── README.rst ├── crmngr ├── __init__.py ├── __main__.py ├── cache.py ├── cli.py ├── config.py ├── controlrepository.py ├── cprint.py ├── forgeapi.py ├── git.py ├── puppetfile.py ├── utils.py └── version.py ├── requirements.in ├── requirements.txt ├── setup.cfg ├── setup.py └── tests └── test_crmngr.py /.gitignore: -------------------------------------------------------------------------------- 1 | # pycharm 2 | /.idea/tasks.xml 3 | /.idea/workspace.xml 4 | 5 | # cache 6 | *.py[cod] 7 | __pycache__/ 8 | 9 | # virtualenv 10 | /pyvenv/ 11 | 12 | # packaging 13 | /crmngr.egg-info/ 14 | /dist/ 15 | /build/ 16 | 17 | # testing 18 | /.cache/ 19 | /pytestdebug.log 20 | -------------------------------------------------------------------------------- /.idea/crmngr-project.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 14 | -------------------------------------------------------------------------------- /.idea/encodings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /.idea/inspectionProfiles/profiles_settings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 7 | -------------------------------------------------------------------------------- /.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /CHANGELOG.rst: -------------------------------------------------------------------------------- 1 | Change Log 2 | ========== 3 | 4 | All notable changes to this project will be documented in this file. 5 | 6 | `Unreleased`_ 7 | ------------- 8 | 9 | Fixed 10 | ~~~~~ 11 | 12 | - Simplify version lookup, make it compatible with python 3.4 again. 13 | 14 | 15 | `2.0.3`_ - 2018-01-18 16 | --------------------- 17 | 18 | Changed 19 | ~~~~~~~ 20 | 21 | - New way of handling version resolution during the build process. 22 | 23 | 24 | `2.0.2`_ - 2018-01-09 25 | --------------------- 26 | 27 | Fixed 28 | ~~~~~ 29 | 30 | - Title in environments command did not reflect the active profile correctly, 31 | it would always report the environment list would be for the default 32 | profile. 33 | 34 | 35 | `2.0.1`_ - 2017-02-13 36 | --------------------- 37 | 38 | Fixed 39 | ~~~~~ 40 | 41 | - Fix handling of forge modules without a current_release. (`#13`_) 42 | 43 | 44 | `2.0.0`_ - 2017-01-13 45 | --------------------- 46 | 47 | Changed 48 | ~~~~~~~ 49 | 50 | - Clarify install procedures in README. 51 | 52 | 53 | `2.0.0rc1`_ - 2017-01-05 54 | ------------------------ 55 | 56 | Added 57 | ~~~~~ 58 | 59 | - Added `--reference`/`-r` option to the update command. This allows updating 60 | modules to the same versions as in the reference environment. Can be combined 61 | with `--add`/`--update`. 62 | - Added `--compare`/`-c` option to the report command. This mode will show only 63 | the differences between two or more environments. I.e. modules that are 64 | deployed in different versions, or modules missing from some environments. 65 | - The match algorithm used for environments (-e) and modules (-m) does now. 66 | support negation. When the first pattern to match is `!`, environments and 67 | modules that do **NOT** match any of the following patterns will be 68 | processed. 69 | - Added `--wrap`/`--no-wrap` options to report mode. This allows to disable 70 | automatic wrapping of long-lines. Default is to wrap, unless overriden by 71 | the new `wrap` option in the `prefs` file. 72 | - crmngr will emit a warning and exit with an exit code 1, if the requested 73 | operation does not affect any environment. 74 | - Added command `environments` to list environments. 75 | - Added command `create` to create a new environment. This supports creating 76 | either empty environments or environments as clones of existing ones. 77 | - Added command `delete` to delete an environment. 78 | 79 | Changed 80 | ~~~~~~~ 81 | 82 | - When updating existing git modules, it is no longer necessary to specify the 83 | URL. 84 | - In update mode, when not specifying update or version updates, all modules 85 | matched by the filter options will be updated to the latest available version. 86 | For forge modules this will be the latest forge version, for git modules it 87 | will be the latest tag (if any) or HEAD of the default branch. 88 | - In update mode, when working on forge module you have to explicitly specify 89 | the new paramter `--forge`. This makes the CLI more consistent between forge 90 | and git modules. 91 | - `--version-check`/`--no-version-check` are new options of the report 92 | command rather than global crmngr options. 93 | - `--cache`/`--no-cache` options have been replaced by a `--cache-ttl` option. 94 | This allows more granuality. Setting `--cache-ttl 0` will yield the same 95 | behaviour than `--no-cache` would have in previous versions. In the `prefs` 96 | file a new option `cache_ttl` has been introduced to set the default value 97 | for the `--cache_ttl` cli option. The `cache` `prefs` option has been removed. 98 | - crmngr no longer updates the local clone of the control repository before 99 | updating Puppetfiles. This means if someone else pushes changes to the 100 | control repository during your crmngr run, crmngr might fail to update the 101 | control repository and exit. Previously changes would silently be reverted. 102 | - In report mode, environments are now displayed space-separated rather 103 | than comma-separated. This now matches how environments need to be 104 | specified on the CLI. 105 | - In the report, versions are now sorted with a natural sorting algorithm. 106 | This means that f.e. 1.10.x will correctly show as newer than 1.2.x which was 107 | not the case before. 108 | - Cache handling and internal data structures have been vastly improved. 109 | update operations do not need a populated cache anymore and only 110 | information for modules and environments currently working on are being 111 | processed. This is a major performance improvement over the previous 112 | release. 113 | - Wherever possible crmngr now uses git shallow clones to save bandwidth and 114 | increase performance. 115 | - crmngr now depends on the 3rd-party libraries `natsort`_ and `requests`_ 116 | 117 | Removed 118 | ~~~~~~~ 119 | 120 | - support for console_clear_command has been removed. 121 | - support for Puppetfile module categorization has been removed. 122 | - option `--report-unused` has been removed from the report command. A similar 123 | functionality is provided by the new `--compare`/`-c` option. 124 | 125 | 126 | 127 | 1.0.0 - 2015-12-04 128 | ------------------ 129 | 130 | Added 131 | ~~~~~ 132 | 133 | - initial public release 134 | 135 | .. _Unreleased: https://github.com/vshn/crmngr/compare/v2.0.3...HEAD 136 | .. _2.0.3: https://github.com/vshn/crmngr/compare/v2.0.2...v2.0.3 137 | .. _2.0.2: https://github.com/vshn/crmngr/compare/v2.0.1...v2.0.2 138 | .. _2.0.1: https://github.com/vshn/crmngr/compare/v2.0.0...v2.0.1 139 | .. _2.0.0: https://github.com/vshn/crmngr/compare/v2.0.0rc1...v2.0.0 140 | .. _2.0.0rc1: https://github.com/vshn/crmngr/compare/v1.0.0...v2.0.0rc1 141 | .. _#13: https://github.com/vshn/crmngr/issues/13 142 | .. _natsort: https://pypi.python.org/pypi/natsort 143 | .. _requests: https://pypi.python.org/pypi/requests 144 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2015-2017, VSHN AG, info@vshn.ch 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions 6 | are met: 7 | 8 | 1. Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | 2. Redistributions in binary form must reproduce the above copyright 11 | notice, this list of conditions and the following disclaimer in the 12 | documentation and/or other materials provided with the distribution. 13 | 3. Neither the name of the copyright holder nor the names of its 14 | contributors may be used to endorse or promote products derived from 15 | this software without specific prior written permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A 20 | PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 | HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 23 | TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 24 | PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 25 | LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 26 | NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ###### 2 | crmngr 3 | ###### 4 | 5 | .. contents:: Table of Contents 6 | 7 | 8 | ******** 9 | Overview 10 | ******** 11 | 12 | crmngr (Control Repository Manager) is a tool to aid with the management of a 13 | r10k-style control repository for puppet 14 | 15 | about r10k 16 | ========== 17 | 18 | from `r10k's github page `_: 19 | R10k provides a general purpose toolset for deploying Puppet environments 20 | and modules. It implements the `Puppetfile`_ format and provides a native 21 | implementation of Puppet dynamic environments. 22 | 23 | r10k is a tool that creates a puppet environment for each branch in a git 24 | repository (the "control repository") on the puppetmaster. Each branch contains 25 | a `Puppetfile` that declares which puppet modules in which versions from which 26 | source (puppetforge or a git URL) should be installed in the corresponding 27 | environment. 28 | When working with more than a handful environments it is hard to keep track of 29 | the modules spread over all these environments. Here is where crmngr comes to 30 | the rescue. 31 | 32 | crmngr 33 | ====== 34 | 35 | crmngr (Control Repository Manager) can generate reports and help with adding, 36 | updating or removing modules in Puppetfiles spread over multiple branches. 37 | 38 | See usage section of this document for details. 39 | 40 | CAVEATS: 41 | crmngr will not parse/validate metadata.json and thus will not check if all 42 | dependencies are satisfied between the modules in a certain environment ! 43 | 44 | It is similar to https://github.com/camptocamp/puppetfile-updater and was 45 | developed independently during the same timeframe. 46 | 47 | 48 | ************ 49 | Dependencies 50 | ************ 51 | 52 | crmngr supports python >=3.4 and has the following 3rd-party dependencies 53 | - `natsort `_ (>= 4.0.0) 54 | - `requests `_ (>= 2.1) 55 | 56 | crmngr further relies on git >=1.8 57 | 58 | 59 | ************ 60 | Installation 61 | ************ 62 | 63 | crmngr can be installed using different methods, see below. 64 | 65 | Additionally the git binary needs to be in the invoking users PATH, and the 66 | access to the r10k control repository needs to be password-less. (f.e. using 67 | SSH pubkey authentication). This also applies to git repositories used in 68 | Puppetfiles. 69 | 70 | 71 | pip 72 | === 73 | 74 | .. code-block:: text 75 | 76 | pip3 install crmngr 77 | 78 | 79 | ArchLinux 80 | ========= 81 | 82 | crmngr is available in the `AUR`_ 83 | 84 | 85 | Ubuntu 86 | ====== 87 | 88 | crmngr is available in the `PPA`_, package is called `python3-crmngr` 89 | 90 | 91 | ************* 92 | Configuration 93 | ************* 94 | 95 | crmngr is looking for its configuration files in `~/.crmngr/` and will create 96 | them if run for the first time. 97 | 98 | profiles 99 | ======== 100 | 101 | crmngr supports multiple profiles. Each profile represents an r10k-style control 102 | repository. 103 | 104 | Profiles are read from `~/.crmngr/profiles` 105 | 106 | The only mandatory setting is `repository` in the `default` section defining the 107 | git url of the r10k-style control repository of the default profile. 108 | 109 | .. code-block:: ini 110 | 111 | [default] 112 | repository = git@git.example.org:user/control.git 113 | 114 | If started without configuration file, crmngr will offer to create one. 115 | 116 | Additional sections can be added to support multiple profiles 117 | 118 | .. code-block:: ini 119 | 120 | [profile2] 121 | repository = git@git2.example.org:anotheruser/control.git 122 | 123 | Run crmngr with option `--profile` to use a profile other than `default`. 124 | 125 | 126 | prefs 127 | ===== 128 | 129 | the default behaviour of crmngr can be adjusted in the `~/.crmngr/prefs` file. 130 | 131 | Defaults (i.e. behaviour if no prefs file is present): 132 | 133 | .. code-block:: ini 134 | 135 | [crmngr] 136 | cache_ttl = 86400 137 | version_check = yes 138 | wrap = yes 139 | 140 | Supported settings: 141 | 142 | * *cache_ttl*: yes/no 143 | Whether or not to read version info from cache. This sets the default value 144 | of the `--cache-ttl` cli argument. 145 | 146 | * *version_check*: yes/no 147 | Whether or not to check for latest version in report mode . This influences 148 | the default behaviour of `--version-check` / `--no-version-check` cli 149 | arguments 150 | 151 | * *wrap*: yes/no 152 | Whether or not to wrap long lines in report mode. This influences the 153 | default behaviour of `--wrap` / `--no-wrap` cli arguments. 154 | 155 | 156 | ***** 157 | Usage 158 | ***** 159 | 160 | .. code-block:: text 161 | 162 | usage: crmngr [-h] [-v] [--cache-ttl TTL] [-d] [-p PROFILE] 163 | {clean,create,delete,environments,profiles,report,update} ... 164 | 165 | manage a r10k-style control repository 166 | 167 | optional arguments: 168 | -h, --help show this help message and exit 169 | -v, --version show program's version number and exit 170 | --cache-ttl TTL time-to-live in seconds for version cache entries 171 | (default: 86400) 172 | -d, --debug enable debug output (default: False) 173 | -p PROFILE, 174 | --profile PROFILE 175 | crmngr configuration profile (default: default) 176 | 177 | commands: 178 | valid commands. Use -h/--help on command for usage details 179 | 180 | {clean,create,delete,environments,profiles,report,update} 181 | clean clean version cache 182 | create create a new environment 183 | delete delete an environment 184 | environments list all environments of the selected profile 185 | profiles list available configuration profiles 186 | report generate a report about modules and versions 187 | update update puppet environment 188 | 189 | 190 | clean 191 | ===== 192 | 193 | The clean command clears the cache used by crmngr. 194 | 195 | .. code-block:: text 196 | 197 | usage: crmngr clean [-h] 198 | 199 | Clean version cache. 200 | 201 | This will delete the cache directory (~/.crmngr/cache). 202 | 203 | optional arguments: 204 | -h, --help show this help message and exit 205 | 206 | 207 | create 208 | ====== 209 | 210 | .. code-block:: text 211 | 212 | usage: crmngr create [-h] [-t ENVIRONMENT] [--no-report] 213 | [--version-check | --no-version-check] 214 | [--wrap | --no-wrap] 215 | environment 216 | 217 | Create a new environment. 218 | 219 | Unless --template/-t is specified, this command will create a new 220 | environment containing the following files (tldr. an environemnt without 221 | any modules): 222 | 223 | Puppetfile 224 | --- 225 | forge 'http://forge.puppetlabs.com' 226 | 227 | --- 228 | 229 | manifests/site.pp 230 | --- 231 | hiera_include('classes') 232 | --- 233 | 234 | If --template/-t is specified, the command will clone the existing 235 | ENVIRONMENT including all files and directories it contains. 236 | 237 | positional arguments: 238 | environment name of the new environment 239 | 240 | optional arguments: 241 | -h, --help show this help message and exit 242 | -t ENVIRONMENT, --template ENVIRONMENT 243 | name of an existing environment to clone the new 244 | environment from 245 | 246 | report options: 247 | --no-report disable printing a report for the new 248 | environment (default: False) 249 | --version-check enable check for latest version (forge modules) 250 | or latest git tag (git modules). (default: True) 251 | --no-version-check disable check for latest version (forge modules) 252 | or latest git tag (git modules). (default: 253 | False) 254 | --wrap enable wrapping of long lines. (default: True) 255 | --no-wrap disable wrapping long lines. (default: False) 256 | 257 | 258 | delete 259 | ====== 260 | 261 | .. code-block:: text 262 | 263 | usage: crmngr delete [-h] environment 264 | 265 | Delete an environment. 266 | 267 | The command will ask for confirmation. 268 | 269 | positional arguments: 270 | environment name of the environment to delete 271 | 272 | optional arguments: 273 | -h, --help show this help message and exit 274 | 275 | 276 | environments 277 | ============ 278 | 279 | .. code-block:: text 280 | 281 | usage: crmngr environments [-h] 282 | 283 | List all environments in the control-repository of the currently 284 | selected profile. 285 | 286 | optional arguments: 287 | -h, --help show this help message and exit 288 | 289 | 290 | profiles 291 | ======== 292 | 293 | .. code-block:: text 294 | 295 | usage: crmngr profiles [-h] 296 | 297 | List all available configuration profiles. 298 | 299 | To add a new configuration profile, open ~/.crmngr/profiles and add a 300 | new section: 301 | 302 | [new_profile_name] 303 | repository = control-repo-url 304 | 305 | Ensure there is always a default section! 306 | 307 | optional arguments: 308 | -h, --help show this help message and exit 309 | 310 | 311 | report 312 | ====== 313 | 314 | The report command is used to generate reports about module versions used in 315 | the various branches of a control repository. 316 | 317 | The report is aggregated by module, listing all module version, which branch 318 | they use and what would be the latest installable version. (Version for 319 | forge.puppetlabs.com modules, Tag for modules installed from git) 320 | 321 | **NOTE**: 322 | The report command will output colorized text. When using a pager, 323 | make sure the pager understands these colors. For less use option -r: 324 | 325 | .. code-block:: text 326 | 327 | crmngr report | less -r 328 | 329 | # or if the output shall be preserved in a file 330 | crmngr report > report.out 331 | less -r report.out 332 | 333 | # or if you want to strip color codes all together 334 | crmngr report | perl -pe 's/\e\[?.*?[\@-~]//g' 335 | 336 | 337 | .. code-block:: text 338 | 339 | usage: crmngr report [-h] [-e [PATTERN [PATTERN ...]]] 340 | [-m [MODULES [MODULES ...]]] [-c] 341 | [--version-check | --no-version-check] 342 | [--wrap | --no-wrap] 343 | 344 | Generate a report about modules and versions deployed in the puppet 345 | environments. 346 | 347 | optional arguments: 348 | -h, --help show this help message and exit 349 | 350 | filter options: 351 | -e [PATTERN [PATTERN ...]], 352 | --env [PATTERN [PATTERN ...]], 353 | --environment [PATTERN [PATTERN ...]], 354 | --environments [PATTERN [PATTERN ...]] 355 | only report modules in environments matching any 356 | PATTERN. If the first supplied PATTERN is !, only 357 | report modules in environments NOT matching any 358 | PATTERN. PATTERN is a case-sensitive glob(7)-style 359 | pattern. 360 | -m [MODULES [MODULES ...]], 361 | --mod [MODULES [MODULES ...]], 362 | --module [MODULES [MODULES ...]], 363 | --modules [MODULES [MODULES ...]] 364 | only report modules matching any PATTERN. If the 365 | first supplied PATTERN is !, only report modules NOT 366 | matching any PATTERN. PATTERN is a case-sensitive 367 | glob(7)-style pattern. 368 | 369 | display options: 370 | -c, --compare compare mode will only show modules that differ 371 | between environments. 372 | --version-check disable check for latest version (forge modules) or 373 | latest git tag (git modules). The information is 374 | cached for subsequent runs. (default: True) 375 | --no-version-check disable check for latest version (forge modules) or 376 | latest git tag (git modules). (default: False) 377 | --wrap enable wrapping of long lines. (default: True) 378 | --no-wrap disable wrapping of long lines. (default: False) 379 | 380 | Examples 381 | -------- 382 | 383 | Gather a report of all module versions, in all branches: 384 | 385 | .. code-block:: text 386 | 387 | crmngr report 388 | 389 | 390 | Gather a report of all modules in branches ending with Production: 391 | 392 | .. code-block:: text 393 | 394 | crmngr report --environments "*Production" 395 | 396 | 397 | Gather a report of all modules that contain profile in their name: 398 | 399 | .. code-block:: text 400 | 401 | crmngr report --modules "*profile*" 402 | 403 | 404 | Gather a report of modules apache, php and mysql in environments starting with 405 | Cust: 406 | 407 | .. code-block:: text 408 | 409 | crmngr report --environments "Cust*" --modules apache php mysql 410 | 411 | Gather a report of all modules in environments CustProd, CustStage and CustDev. 412 | Only show the differences. 413 | 414 | .. code-block:: text 415 | 416 | crmngr report --environments CustProd CustStage CustDev --compare 417 | 418 | update 419 | ====== 420 | 421 | The update command updates, adds or removes modules from environments. 422 | 423 | The update command will display a diff for every affected environment and will 424 | ask you to confirm the changes. 425 | 426 | **NOTE**: 427 | The author part of a module name is *only* used to find the correct module 428 | on forge. If you run update on --module puppetlabs/stdlib, this will also 429 | affect all other stdlib modules that might be in a environment (i.e. 430 | otherauthor/stdlib or stdlib installed from git will be replaced by 431 | puppetlabs/stdlib). 432 | 433 | 434 | .. code-block:: text 435 | 436 | 437 | 438 | usage: crmngr update [-h] [-e [PATTERN [PATTERN ...]]] 439 | [-m [PATTERN [PATTERN ...]]] [--add] [--remove] 440 | [-r ENVIRONMENT] [-n | --non-interactive] 441 | [--forge | --git [URL]] [--version [FORGE_VERSION] | 442 | --tag [GIT_TAG] | --commit GIT_COMMIT | --branch 443 | GIT_BRANCH] 444 | 445 | Update puppet environment. 446 | 447 | This command allows to update module version, and addition or removal of 448 | puppet modules in one or more environment. 449 | 450 | If the update command is run without update or version options all modules 451 | matching the filter options will be updated to the latest available version. 452 | (forge version for forge module, git tag (if available) or HEAD for git 453 | modules). 454 | 455 | optional arguments: 456 | -h, --help show this help message and exit 457 | 458 | filter options: 459 | -e [PATTERN [PATTERN ...]], 460 | --env [PATTERN [PATTERN ...]], 461 | --environment [PATTERN [PATTERN ...]], 462 | --environments [PATTERN [PATTERN ...]] 463 | only update modules in environments matching any 464 | PATTERN. If the first supplied PATTERN is !, only 465 | update modules in environments NOT matching any 466 | PATTERN. PATTERN is a case-sensitive glob(7)-style 467 | pattern. 468 | -m [PATTERN [PATTERN ...]], 469 | --mod [PATTERN [PATTERN ...]], 470 | --module [PATTERN [PATTERN ...]], 471 | --modules [PATTERN [PATTERN ...]] 472 | only update modules matching any PATTERN. If the 473 | first supplied PATTERN is !, only update modules NOT 474 | matching any PATTERN. PATTERN is a case-sensitive 475 | glob(7)-style pattern unless a version option is 476 | specified. If a version option is specified, PATTERN 477 | needs to be a single module name. If updating a 478 | forge module (--forge) this needs to be in 479 | author/module format. 480 | 481 | update options: 482 | --add add modules (-m) if not already in environment. 483 | Default behaviour is to only update modules (-m) in 484 | environments they are already deployed in. 485 | --remove remove module from Puppetfile. version options 486 | (--version, --tag, --commit, --branch) are 487 | irrelevant. All modules matching a module filter 488 | pattern (-m) will be removed. This also applies if a 489 | module pattern includes an author (forge module). 490 | Only the module name is relevant. 491 | -r ENVIRONMENT, --reference ENVIRONMENT 492 | use ENVIRONMENT as reference. All modules will be 493 | updated to the version deployed in the reference 494 | ENVIRONMENT. If combined with --add, modules not yet 495 | in the environments (-e) are added. If combined with 496 | --remove, modules not in reference will be removed 497 | from the environments (-e). 498 | 499 | interactivity options: 500 | -n, --dry-run, --diff-only 501 | display diffs of what would be changed 502 | --non-interactive in non-interactive mode, crmngr will neither ask for 503 | confirmation before commit or push, nor will it show 504 | diffs of what will be changed. Use with care! 505 | 506 | version options: 507 | these options are only applicable if operating on a single module. 508 | 509 | --forge source module from puppet forge. 510 | --git [URL] source module from git URL. If specified without URL 511 | the existing URL will be used. URL is mandatory if 512 | invoked with --add. 513 | --version [FORGE_VERSION] 514 | pin module to forge version. If parameter is 515 | specified without VERSION, latest available version 516 | from forge will be used instead 517 | --tag [GIT_TAG] pin a module to a git tag. If parameter is specified 518 | without TAG, latest tag available in git repository 519 | is used instead 520 | --commit GIT_COMMIT pin module to a git commit 521 | --branch GIT_BRANCH pin module to a git branch 522 | 523 | 524 | Examples 525 | -------- 526 | 527 | Sanitize Puppetfiles of all branches: 528 | 529 | .. code-block:: text 530 | 531 | crmngr update 532 | 533 | 534 | Update stdlib module in all branches to latest forge version. 535 | 536 | 537 | .. code-block:: text 538 | 539 | crmngr update --module puppetlabs/stdlib --forge --version 540 | 541 | 542 | Update stdlib module in all branches to latest forge version. Additionally add 543 | the module to branches that currently lack the stdlib module 544 | 545 | .. code-block:: text 546 | 547 | crmngr update --module puppetlabs/stdlib --forge --version --add 548 | 549 | 550 | Remove icinga modules from control repository branches that end with Vagrant. 551 | 552 | .. code-block:: text 553 | 554 | crmngr update --remove --module icinga --environments "*Vagrant" 555 | 556 | 557 | Update apache module to git branch 2.0.x in control repository branch Devel 558 | 559 | .. code-block:: text 560 | 561 | crmngr update --environments Devel \ 562 | --module apache \ 563 | --git git@github.com:puppetlabs/puppetlabs-apache.git \ 564 | --branch 2.0.x 565 | 566 | 567 | profiles 568 | ======== 569 | 570 | The profile command lists available configuration profiles. 571 | 572 | .. code-block:: bash 573 | 574 | usage: crmngr profiles 575 | 576 | 577 | *********** 578 | Development 579 | *********** 580 | 581 | run development version 582 | ======================= 583 | 584 | .. code-block:: bash 585 | 586 | git clone https://github.com/vshn/crmngr crmngr-project 587 | cd crmngr-project 588 | python -m venv pyvenv 589 | . pyvenv/bin/activate 590 | pip install -r requirements.txt 591 | 592 | python -m crmngr 593 | 594 | 595 | 596 | .. _AUR: https://aur.archlinux.org/packages/crmngr/ 597 | .. _PPA: https://launchpad.net/~vshn/+archive/ubuntu/crmngr 598 | .. _github-r10k: https://github.com/puppetlabs/r10k 599 | .. _Puppetfile: 600 | https://github.com/puppetlabs/r10k/blob/master/doc/puppetfile.mkd 601 | -------------------------------------------------------------------------------- /crmngr/__init__.py: -------------------------------------------------------------------------------- 1 | """manage a r10k-style control repository""" 2 | 3 | # stdlib 4 | from configparser import NoSectionError 5 | import logging 6 | import sys 7 | 8 | # 3rd-party 9 | from crmngr import cprint 10 | from crmngr.cache import JsonCache 11 | from crmngr.cli import parse_cli_args 12 | from crmngr.config import CrmngrConfig 13 | from crmngr.config import setup_logging 14 | from crmngr.controlrepository import ControlRepository 15 | from crmngr.controlrepository import NoEnvironmentError 16 | from crmngr.utils import query_yes_no 17 | 18 | LOG = logging.getLogger(__name__) 19 | 20 | 21 | def main(): 22 | """main entrypoint""" 23 | try: 24 | configuration = CrmngrConfig() 25 | except NoSectionError: 26 | print("No valid profile file found!") 27 | print("Enter git url of control repositoriy to create one.") 28 | print("Leave empty to abort") 29 | print() 30 | print("Control repository url: ", end="") 31 | default_profile_url = input().strip() 32 | if default_profile_url: 33 | configuration = CrmngrConfig.create_default_configuration( 34 | default_profile_url 35 | ) 36 | else: 37 | sys.exit() 38 | 39 | cli_args = parse_cli_args(configuration) 40 | # now that the profile is known, reload the configuration using the 41 | # correct profile 42 | try: 43 | configuration = CrmngrConfig(profile=cli_args.profile) 44 | except NoSectionError: 45 | cprint.red('No configuration for profile {profile}'.format( 46 | profile=cli_args.profile 47 | )) 48 | sys.exit(1) 49 | 50 | setup_logging(cli_args.debug) 51 | 52 | version_cache = JsonCache(configuration.cache_dir, ttl=cli_args.cache_ttl) 53 | 54 | commands = { 55 | 'clean': command_clean, 56 | 'create': command_create, 57 | 'delete': command_delete, 58 | 'environments': command_environments, 59 | 'profiles': command_profiles, 60 | 'report': command_report, 61 | 'update': command_update, 62 | } 63 | try: 64 | commands[cli_args.command]( 65 | configuration=configuration, 66 | cli_args=cli_args, 67 | version_cache=version_cache) 68 | except NoEnvironmentError: 69 | cprint.yellow_bold('no environment is affected by your command. typo?') 70 | except KeyboardInterrupt: 71 | cprint.red_bold('crmngr has been aborted.') 72 | 73 | 74 | def command_create(*, configuration, cli_args, version_cache, 75 | **kwargs): # pylint: disable=unused-argument 76 | """run create command""" 77 | control_repo = ControlRepository( 78 | clone_url=configuration.control_repo_url, 79 | ) 80 | environments = [environment.name 81 | for environment in control_repo.environments] 82 | 83 | if cli_args.environment in environments: 84 | cprint.red( 85 | 'Template environment {environment} already exists in ' 86 | 'control repository {profile} ({url})'.format( 87 | environment=cli_args.environment, 88 | profile=cli_args.profile, 89 | url=control_repo.url, 90 | )) 91 | sys.exit(1) 92 | if cli_args.template: 93 | if cli_args.template.strip() not in environments: 94 | cprint.red( 95 | 'Template environment {template} does not exist in ' 96 | 'control repository {profile} ({url})'.format( 97 | template=cli_args.template, 98 | profile=cli_args.profile, 99 | url=control_repo.url, 100 | )) 101 | sys.exit(1) 102 | control_repo.clone_environment( 103 | cli_args.template, 104 | cli_args.environment, 105 | report=cli_args.report, 106 | ) 107 | if cli_args.report: 108 | control_repo.report( 109 | version_cache=version_cache, 110 | version_check=cli_args.version_check, 111 | wrap=cli_args.wrap, 112 | ) 113 | else: 114 | control_repo.new_environment( 115 | cli_args.environment, 116 | report=cli_args.report 117 | ) 118 | cprint.green('Created new empty environment %s' % cli_args.environment) 119 | 120 | 121 | def command_delete(*, configuration, cli_args, 122 | **kwargs): # pylint: disable=unused-argument 123 | """run delete command""" 124 | control_repo = ControlRepository( 125 | clone_url=configuration.control_repo_url, 126 | environments=[cli_args.environment, ] 127 | ) 128 | environment = sorted(control_repo.environments)[0] 129 | if query_yes_no("Really delete environment {}? This is a irreversible " 130 | "operation!".format(environment), default='no'): 131 | control_repo.delete_environment(environment) 132 | cprint.green('Deleted environment {}'.format(environment)) 133 | 134 | 135 | def command_report(*, configuration, cli_args, version_cache, 136 | **kwargs): # pylint: disable=unused-argument 137 | """run report command""" 138 | control_repo = ControlRepository( 139 | clone_url=configuration.control_repo_url, 140 | environments=cli_args.environments, 141 | modules=cli_args.modules, 142 | ) 143 | 144 | if cli_args.compare and not len(control_repo.environments) >= 2: 145 | cprint.yellow_bold( 146 | 'At least two environments required in compare mode. Only matched ' 147 | 'environment: {}'.format( 148 | ', '.join([environment.name 149 | for environment in control_repo.environments]) 150 | ) 151 | ) 152 | sys.exit(1) 153 | 154 | control_repo.report( 155 | compare=cli_args.compare, 156 | version_cache=version_cache, 157 | version_check=cli_args.version_check, 158 | wrap=cli_args.wrap, 159 | ) 160 | 161 | 162 | def command_environments(*, configuration, 163 | **kwargs): # pylint: disable=unused-argument 164 | """run environments command""" 165 | control_repo = ControlRepository( 166 | clone_url=configuration.control_repo_url, 167 | ) 168 | cprint.white_bold('Environments in profile %s' % configuration.profile) 169 | for environment in sorted(control_repo.environments): 170 | cprint.white(' - {}'.format(environment.name)) 171 | 172 | 173 | def command_update(*, configuration, cli_args, 174 | **kwargs): # pylint: disable=unused-argument 175 | """run report command""" 176 | 177 | if cli_args.reference: 178 | environments = cli_args.environments + [cli_args.reference] 179 | else: 180 | environments = cli_args.environments 181 | 182 | control_repo = ControlRepository( 183 | clone_url=configuration.control_repo_url, 184 | environments=environments, 185 | ) 186 | control_repo.update_puppetfiles( 187 | cli_args=cli_args, 188 | ) 189 | 190 | 191 | def command_clean(*, version_cache, 192 | **kwargs): # pylint: disable=unused-argument 193 | """run clean command""" 194 | if query_yes_no("Really clear cache directory?"): 195 | return version_cache.clear() 196 | 197 | 198 | def command_profiles(*, configuration, 199 | **kwargs): # pylint: disable=unused-argument 200 | """run profiles command""" 201 | cprint.white_bold('Available profiles:') 202 | for profile in configuration.profiles: 203 | cprint.white(" - %s: %s" % (profile.name, profile.repository)) 204 | -------------------------------------------------------------------------------- /crmngr/__main__.py: -------------------------------------------------------------------------------- 1 | """ crmngr helper to run an uninstalled version """ 2 | 3 | import crmngr 4 | crmngr.main() 5 | -------------------------------------------------------------------------------- /crmngr/cache.py: -------------------------------------------------------------------------------- 1 | """ crmngr cache module """ 2 | 3 | import json 4 | import logging 5 | import os 6 | import shutil 7 | import time 8 | 9 | LOG = logging.getLogger(__name__) 10 | 11 | 12 | class CacheError(Exception): 13 | """custom exception for cache related errors""" 14 | pass 15 | 16 | 17 | class JsonCache: 18 | """json file based cache""" 19 | 20 | def __init__(self, directory, ttl=86400, fail_silently=True): 21 | """constructor, takes directory as argument""" 22 | LOG.debug("initialize JsonCache in %s", directory) 23 | self._directory = directory 24 | self._default_ttl = ttl 25 | self._fail_silently = fail_silently 26 | 27 | def clear(self): 28 | """delete cache directory""" 29 | shutil.rmtree(self._directory) 30 | LOG.debug("deleted cache directory %s", self._directory) 31 | 32 | def read(self, key, ttl=None): 33 | """read json dict from file""" 34 | if ttl is None: 35 | ttl = self._default_ttl 36 | try: 37 | LOG.debug("attempt to read %s from cache", key) 38 | with open(os.path.join(self._directory, key)) as cache_fd: 39 | cache = json.load(cache_fd) 40 | LOG.debug("received %s from cache", cache) 41 | if cache.get('updated', 0) + ttl >= int(time.time()): 42 | LOG.debug("cache entry is valid, return it") 43 | return cache 44 | LOG.debug("cache expired, returning empty response") 45 | return {} 46 | except (AttributeError, KeyError, OSError, ValueError) as exc: 47 | LOG.debug( 48 | "cache lookup for %s failed. fail silently.", key 49 | ) 50 | if self._fail_silently: 51 | return {} 52 | else: 53 | raise CacheError('could not read from cache') from exc 54 | 55 | def write(self, key, jsondict): 56 | """write json dict to file""" 57 | try: 58 | LOG.debug( 59 | "attempt to write %s to cache using key %s", jsondict, key 60 | ) 61 | with open(os.path.join(self._directory, key), 'w') as cache_fd: 62 | localdict = jsondict.copy() 63 | localdict.update( 64 | { 65 | 'updated': int(time.time()), 66 | } 67 | ) 68 | json.dump(localdict, cache_fd) 69 | LOG.debug('wrote %s to cache using key %s', localdict, key) 70 | except (AttributeError, KeyError, OSError, ValueError) as exc: 71 | if not self._fail_silently: 72 | raise CacheError('could not write to cache') from exc 73 | LOG.debug('failed to write to cache. fail silently.') 74 | -------------------------------------------------------------------------------- /crmngr/cli.py: -------------------------------------------------------------------------------- 1 | """ crmngr cli module """ 2 | 3 | # stdlib 4 | import argparse 5 | import sys 6 | import textwrap 7 | 8 | # crmngr 9 | from crmngr.version import __version__ 10 | 11 | 12 | def parse_cli_args(configuration): 13 | """parse CLI args""" 14 | 15 | # global cli options 16 | parser = argparse.ArgumentParser( 17 | description='manage a r10k-style control repository', 18 | formatter_class=argparse.ArgumentDefaultsHelpFormatter, 19 | prog='crmngr', 20 | ) 21 | parser.add_argument( 22 | '-v', '--version', 23 | action='version', version='%(prog)s ' + __version__ 24 | ) 25 | parser.add_argument( 26 | '--cache-ttl', 27 | dest='cache_ttl', type=int, metavar='TTL', 28 | help='time-to-live in seconds for version cache entries', 29 | ) 30 | parser.add_argument( 31 | '-d', '--debug', 32 | dest='debug', action='store_true', default=False, 33 | help='enable debug output' 34 | ) 35 | parser.add_argument( 36 | '-p', '--profile', 37 | dest='profile', default='default', 38 | help='crmngr configuration profile' 39 | ) 40 | 41 | # set defaults for global options 42 | parser.set_defaults( 43 | cache_ttl=configuration.cache_ttl, 44 | ) 45 | 46 | # define command parsers 47 | command_parser = parser.add_subparsers( 48 | title='commands', 49 | dest='command', 50 | description=('valid commands. Use -h/--help on command for usage ' 51 | 'details'), 52 | ) 53 | command_parser.required = True 54 | command_parser_defaults = { 55 | 'parent_parser': command_parser, 56 | 'configuration': configuration, 57 | } 58 | clean_command_parser(**command_parser_defaults) 59 | create_command_parser(**command_parser_defaults) 60 | delete_command_parser(**command_parser_defaults) 61 | environments_command_parser(**command_parser_defaults) 62 | profiles_command_parser(**command_parser_defaults) 63 | report_command_parser(**command_parser_defaults) 64 | update_parser = update_command_parser(**command_parser_defaults) 65 | 66 | args = parser.parse_args() 67 | try: 68 | verify_update_args(args) 69 | except CliError as exc: 70 | update_parser.print_help() 71 | sys.exit("error: %s" % exc) 72 | return args 73 | 74 | 75 | def _ensure_single_module(args): 76 | """ensure that we only operate on a single module.""" 77 | if args.modules is None or len(args.modules) < 1: 78 | raise CliError('it is not supported to specify --git/--forge ' 79 | 'without specifying a module (-m).') 80 | if args.modules is not None and len(args.modules) > 1: 81 | raise CliError('cannot operate on multiple modules when version ' 82 | 'options are set.') 83 | if args.reference is not None: 84 | raise CliError('it is not supported to specify -r/--reference ' 85 | 'in combination with version options.') 86 | if args.add and args.remove: 87 | raise CliError('it is not supported to specify both --add/--remove ' 88 | 'when working on a single module.') 89 | 90 | 91 | def _reject_git_version_parameters(args): 92 | """reject git version parameters.""" 93 | if args.git_branch or args.git_commit or args.git_tag: 94 | raise CliError('it is not supported to specify --branch/--commit/' 95 | '--tag without --git.') 96 | 97 | 98 | def _reject_forge_version_parameter(args): 99 | """reject forge version parameter.""" 100 | if args.forge_version: 101 | raise CliError('it is not supported to specify --version without ' 102 | '--forge.') 103 | 104 | 105 | def verify_update_args(args): 106 | """perform some extended validation of update parser cli arguments""" 107 | if args.command != 'update': 108 | return 109 | 110 | if args.git_url: 111 | _ensure_single_module(args) 112 | _reject_forge_version_parameter(args) 113 | if args.forge_version: 114 | raise CliError('--version is not supported for git modules.') 115 | elif args.forge: 116 | _ensure_single_module(args) 117 | _reject_git_version_parameters(args) 118 | if not args.remove: 119 | if '/' not in args.modules[0]: 120 | raise CliError('when adding or updating forge modules, -m ' 121 | 'has to be in author/module format') 122 | else: 123 | _reject_forge_version_parameter(args) 124 | _reject_git_version_parameters(args) 125 | if args.add: 126 | raise CliError('--add is not supported for bulk updates. Combine ' 127 | '--add with version options.') 128 | if args.remove: 129 | if args.modules is None or len(args.modules) < 1: 130 | raise CliError('it is not supported to specify --remove ' 131 | 'without specifying a module filter (-m).') 132 | 133 | 134 | def clean_command_parser(parent_parser, 135 | **kwargs): # pylint: disable=unused-argument 136 | """sets up the argument parser for the clean command""" 137 | parser = parent_parser.add_parser( 138 | 'clean', 139 | description=( 140 | 'Clean version cache.\n' 141 | '\n' 142 | 'This will delete the cache directory (~/.crmngr/cache).' 143 | ), 144 | formatter_class=KeepNewlineDescriptionHelpFormatter, 145 | help='clean version cache', 146 | ) 147 | return parser 148 | 149 | 150 | def create_command_parser(parent_parser, configuration, 151 | **kwargs): # pylint: disable=unused-argument 152 | """sets up the argument parser for the create command""" 153 | parser = parent_parser.add_parser( 154 | 'create', 155 | description=( 156 | 'Create a new environment.\n' 157 | '\n' 158 | 'Unless --template/-t is specified, this command will create a new ' 159 | 'environment containing the following files (tldr. an environemnt ' 160 | 'without any modules):\n' 161 | '\n' 162 | 'Puppetfile\n' 163 | '---\n' 164 | "forge 'http://forge.puppetlabs.com'\n" 165 | '\n' 166 | '---\n' 167 | '\n' 168 | 'manifests/site.pp\n' 169 | '---\n' 170 | "hiera_include('classes')\n" 171 | '---\n' 172 | '\n' 173 | 'If --template/-t is specified, the command will clone the ' 174 | 'existing ENVIRONMENT including all files and directories it ' 175 | 'contains.' 176 | ), 177 | formatter_class=KeepNewlineDescriptionHelpFormatter, 178 | help='create a new environment', 179 | ) 180 | parser.add_argument( 181 | dest='environment', type=str, 182 | help='name of the new environment' 183 | ) 184 | parser.add_argument( 185 | '-t', '--template', 186 | type=str, dest='template', metavar='ENVIRONMENT', 187 | help='name of an existing environment to clone the new environment from' 188 | ) 189 | report_group = parser.add_argument_group('report options') 190 | report_group.add_argument( 191 | '--no-report', 192 | dest='report', 193 | action='store_false', 194 | help=('disable printing a report for the new environment ' 195 | '(default: False)'), 196 | ) 197 | 198 | version_check_group = report_group.add_mutually_exclusive_group() 199 | version_check_group.add_argument( 200 | '--version-check', 201 | dest='version_check', 202 | action='store_true', 203 | help=('enable check for latest version (forge modules) or latest git ' 204 | 'tag (git modules). ' 205 | '(default: %s)' % str(configuration.version_check)) 206 | ) 207 | version_check_group.add_argument( 208 | '--no-version-check', 209 | dest='version_check', 210 | action='store_false', 211 | help=('disable check for latest version (forge modules) or latest ' 212 | 'git tag (git modules). ' 213 | '(default: %s)' % str(not configuration.version_check)) 214 | ) 215 | wrap_group = report_group.add_mutually_exclusive_group() 216 | wrap_group.add_argument( 217 | '--wrap', 218 | dest='wrap', 219 | action='store_true', 220 | help=('enable wrapping of long lines. ' 221 | '(default: %s)' % str(configuration.wrap)), 222 | ) 223 | wrap_group.add_argument( 224 | '--no-wrap', 225 | dest='wrap', 226 | action='store_false', 227 | help=('disable wrapping long lines. ' 228 | '(default: %s)' % str(not configuration.wrap)), 229 | ) 230 | parser.set_defaults( 231 | version_check=configuration.version_check, 232 | wrap=configuration.wrap, 233 | ) 234 | return parser 235 | 236 | 237 | def delete_command_parser(parent_parser, 238 | **kwargs): # pylint: disable=unused-argument 239 | """sets up the argument parser for the delete command""" 240 | parser = parent_parser.add_parser( 241 | 'delete', 242 | description=( 243 | 'Delete an environment.\n' 244 | '\n' 245 | 'The command will ask for confirmation.' 246 | ), 247 | formatter_class=KeepNewlineDescriptionHelpFormatter, 248 | help='delete an environment', 249 | ) 250 | parser.add_argument( 251 | dest='environment', type=str, 252 | help='name of the environment to delete' 253 | ) 254 | return parser 255 | 256 | 257 | def environments_command_parser(parent_parser, 258 | **kwargs): # pylint: disable=unused-argument 259 | """sets up the argument parser for the environments command""" 260 | parser = parent_parser.add_parser( 261 | 'environments', 262 | description=('List all environments in the control-repository of the ' 263 | 'currently selected profile.'), 264 | help='list all environments of the selected profile', 265 | ) 266 | return parser 267 | 268 | 269 | def profiles_command_parser(parent_parser, 270 | **kwargs): # pylint: disable=unused-argument 271 | """sets up the argument parser for the profiles command""" 272 | parser = parent_parser.add_parser( 273 | 'profiles', 274 | description=( 275 | 'List all available configuration profiles.\n' 276 | '\n' 277 | 'To add a new configuration profile, open ~/.crmngr/profiles and ' 278 | 'add a new section:\n' 279 | '\n' 280 | '[new_profile_name]\n' 281 | 'repository = control-repo-url\n' 282 | '\n' 283 | 'Ensure there is always a default section!' 284 | ), 285 | formatter_class=KeepNewlineDescriptionHelpFormatter, 286 | help='list available configuration profiles', 287 | ) 288 | return parser 289 | 290 | 291 | def report_command_parser(parent_parser, configuration, 292 | **kwargs): # pylint: disable=unused-argument 293 | """sets up the argument parser for the report command""" 294 | parser = parent_parser.add_parser( 295 | 'report', 296 | description=( 297 | 'Generate a report about modules and versions deployed in the ' 298 | 'puppet environments.' 299 | ), 300 | help='generate a report about modules and versions', 301 | ) 302 | filter_group = parser.add_argument_group('filter options') 303 | filter_group.add_argument( 304 | '-e', '--env', '--environment', '--environments', 305 | nargs='*', type=str, dest='environments', metavar='PATTERN', 306 | help=('only report modules in environments matching any PATTERN. ' 307 | 'If the first supplied PATTERN is !, only report modules ' 308 | 'in environments NOT matching any PATTERN. ' 309 | 'PATTERN is a case-sensitive glob(7)-style pattern.') 310 | ) 311 | filter_group.add_argument( 312 | '-m', '--mod', '--module', '--modules', 313 | nargs='*', type=str, dest='modules', 314 | help=('only report modules matching any PATTERN. If the first ' 315 | 'supplied PATTERN is !, only report modules NOT matching any ' 316 | 'PATTERN. PATTERN is a case-sensitive glob(7)-style pattern.') 317 | ) 318 | display_group = parser.add_argument_group('display options') 319 | display_group.add_argument( 320 | '-c', '--compare', 321 | dest='compare', action='store_true', 322 | help=('compare mode will only show modules that differ between ' 323 | 'environments.'), 324 | ) 325 | version_check_group = display_group.add_mutually_exclusive_group() 326 | version_check_group.add_argument( 327 | '--version-check', 328 | dest='version_check', 329 | action='store_true', 330 | help=('disable check for latest version (forge modules) or latest git ' 331 | 'tag (git modules). The information is cached for subsequent' 332 | ' runs. (default: %s)' % str(configuration.version_check) 333 | ) 334 | ) 335 | version_check_group.add_argument( 336 | '--no-version-check', 337 | dest='version_check', 338 | action='store_false', 339 | help=('disable check for latest version (forge modules) or latest ' 340 | 'git tag (git modules). ' 341 | '(default: %s)' % str(not configuration.version_check)) 342 | ) 343 | wrap_group = display_group.add_mutually_exclusive_group() 344 | wrap_group.add_argument( 345 | '--wrap', 346 | dest='wrap', 347 | action='store_true', 348 | help=('enable wrapping of long lines. ' 349 | '(default: %s)' % str(configuration.wrap)), 350 | ) 351 | wrap_group.add_argument( 352 | '--no-wrap', 353 | dest='wrap', 354 | action='store_false', 355 | help=('disable wrapping of long lines. ' 356 | '(default: %s)' % str(not configuration.wrap)), 357 | ) 358 | parser.set_defaults( 359 | version_check=configuration.version_check, 360 | wrap=configuration.wrap, 361 | ) 362 | return parser 363 | 364 | 365 | def update_command_parser(parent_parser, 366 | **kwargs): # pylint: disable=unused-argument 367 | """sets up the argument parser for the report command""" 368 | parser = parent_parser.add_parser( 369 | 'update', 370 | description=( 371 | 'Update puppet environment.\n' 372 | '\n' 373 | 'This command allows to update module version, and addition or ' 374 | 'removal of puppet modules in one or more environment.\n' 375 | '\n' 376 | 'If the update command is run without update or version options ' 377 | 'all modules matching the filter options will be updated to the ' 378 | 'latest available version. (forge version for forge module, git ' 379 | 'tag (if available) or HEAD for git modules).' 380 | ), 381 | formatter_class=KeepNewlineDescriptionHelpFormatter, 382 | help='update puppet environment' 383 | ) 384 | filter_group = parser.add_argument_group('filter options') 385 | filter_group.add_argument( 386 | '-e', '--env', '--environment', '--environments', 387 | nargs='*', type=str, dest='environments', metavar='PATTERN', 388 | help=('only update modules in environments matching any PATTERN. ' 389 | 'If the first supplied PATTERN is !, only update modules ' 390 | 'in environments NOT matching any PATTERN. ' 391 | 'PATTERN is a case-sensitive glob(7)-style pattern.') 392 | ) 393 | filter_group.add_argument( 394 | '-m', '--mod', '--module', '--modules', 395 | nargs='*', type=str, dest='modules', metavar='PATTERN', 396 | help=('only update modules matching any PATTERN. If the first ' 397 | 'supplied PATTERN is !, only update modules NOT matching any ' 398 | 'PATTERN. PATTERN is a case-sensitive glob(7)-style pattern ' 399 | 'unless a version option is specified. If a version option is ' 400 | 'specified, PATTERN needs to be a single module name. If ' 401 | 'updating a forge module (--forge) this needs to be in ' 402 | 'author/module format.') 403 | ) 404 | update_options = parser.add_argument_group('update options') 405 | update_options.add_argument( 406 | '--add', 407 | default=False, action='store_true', 408 | help=('add modules (-m) if not already in environment. Default ' 409 | 'behaviour is to only update modules (-m) in environments they ' 410 | 'are already deployed in.') 411 | ) 412 | update_options.add_argument( 413 | '--remove', 414 | default=False, action='store_true', 415 | help=('remove module from Puppetfile. version options (--version, ' 416 | '--tag, --commit, --branch) are irrelevant. All modules matching ' 417 | 'a module filter pattern (-m) will be removed. This also applies ' 418 | 'if a module pattern includes an author (forge module). Only the ' 419 | 'module name is relevant.') 420 | ) 421 | update_options.add_argument( 422 | '-r', '--reference', metavar='ENVIRONMENT', type=str, 423 | help=('use ENVIRONMENT as reference. All modules will be updated to ' 424 | 'the version deployed in the reference ENVIRONMENT. If combined ' 425 | 'with --add, modules not yet in the environments (-e) are added. ' 426 | 'If combined with --remove, modules not in reference will be ' 427 | 'removed from the environments (-e).') 428 | ) 429 | interactivity_group = parser.add_argument_group('interactivity options') 430 | interactivity_mutex = interactivity_group.add_mutually_exclusive_group() 431 | interactivity_mutex.add_argument( 432 | '-n', '--dry-run', '--diff-only', 433 | default=False, action='store_true', dest='diffonly', 434 | help='display diffs of what would be changed', 435 | ) 436 | interactivity_mutex.add_argument( 437 | '--non-interactive', 438 | default=False, action='store_true', dest='noninteractive', 439 | help=('in non-interactive mode, crmngr will neither ask for ' 440 | 'confirmation before commit or push, nor will it show diffs ' 441 | 'of what will be changed. Use with care!') 442 | ) 443 | version_group = parser.add_argument_group( 444 | 'version options', 445 | description=('these options are only applicable if operating on a ' 446 | 'single module.'), 447 | ) 448 | source_mutex = version_group.add_mutually_exclusive_group() 449 | source_mutex.add_argument( 450 | '--forge', 451 | action='store_true', dest='forge', 452 | help='source module from puppet forge.', 453 | ) 454 | source_mutex.add_argument( 455 | '--git', 456 | nargs='?', type=str, metavar="URL", dest='git_url', 457 | const='USE_EXISTING_URL', 458 | help=('source module from git URL. If specified without URL the ' 459 | 'existing URL will be used. URL is mandatory if invoked with ' 460 | '--add.'), 461 | ) 462 | version_mutex = version_group.add_mutually_exclusive_group() 463 | version_mutex.add_argument( 464 | '--version', 465 | nargs='?', const="LATEST_FORGE_VERSION", type=str, dest='forge_version', 466 | help=('pin module to forge version. If parameter is specified without ' 467 | 'VERSION, latest available version from forge will be used ' 468 | 'instead') 469 | ) 470 | version_mutex.add_argument( 471 | '--tag', 472 | nargs='?', const="LATEST_GIT_TAG", type=str, dest='git_tag', 473 | help=('pin a module to a git tag. If parameter is specified without ' 474 | 'TAG, latest tag available in git repository is used instead') 475 | ) 476 | version_mutex.add_argument( 477 | '--commit', 478 | type=str, dest='git_commit', 479 | help='pin module to a git commit' 480 | ) 481 | version_mutex.add_argument( 482 | '--branch', 483 | type=str, dest='git_branch', 484 | help='pin module to a git branch' 485 | ) 486 | return parser 487 | 488 | 489 | class CliError(Exception): 490 | """exception raised when invalid cli arguments are supplied""" 491 | 492 | 493 | class KeepNewlineDescriptionHelpFormatter(argparse.HelpFormatter): 494 | """argparse helpformatter that keeps newlines in the description""" 495 | 496 | def _fill_text(self, text, width, indent): 497 | return '\n'.join( 498 | ['\n'.join(textwrap.wrap( 499 | line, 500 | width, 501 | initial_indent=indent, 502 | subsequent_indent=indent, 503 | break_long_words=False, 504 | replace_whitespace=False 505 | )) for line in text.splitlines()]) 506 | -------------------------------------------------------------------------------- /crmngr/config.py: -------------------------------------------------------------------------------- 1 | """ crmngr configuration module """ 2 | 3 | # stdlib 4 | from collections import namedtuple 5 | from configparser import ConfigParser 6 | import logging 7 | import logging.config 8 | import os 9 | 10 | LOG = logging.getLogger(__name__) 11 | 12 | 13 | def setup_logging(debug): 14 | """setup logging configuration""" 15 | if debug: 16 | logging.config.dictConfig({ 17 | 'version': 1, 18 | 'disable_existing_loggers': False, 19 | 'formatters': { 20 | 'standard': { 21 | 'format': '%(asctime)s - %(levelname)s - %(message)s' 22 | }, 23 | }, 24 | 'handlers': { 25 | 'default': { 26 | 'formatter': 'standard', 27 | 'class': 'logging.StreamHandler', 28 | }, 29 | }, 30 | 'loggers': { 31 | '': { 32 | 'handlers': ['default'], 33 | 'level': 'DEBUG', 34 | 'propagate': True 35 | }, 36 | } 37 | }) 38 | 39 | 40 | class CrmngrConfig: 41 | """crmngr configuration and profile handling""" 42 | 43 | def __init__(self, profile='default'): 44 | """initialize crmngr configuration for the specified profile""" 45 | self._cache_dir = None 46 | self._config_dir = self._ensure_configuration_directory() 47 | self._profile = profile 48 | 49 | # initialize preferences 50 | self._config = ConfigParser( 51 | defaults={ 52 | 'cache_ttl': '86400', 53 | 'version_check': 'yes', 54 | 'wrap': 'yes' 55 | } 56 | ) 57 | self._config.read(os.path.join(self._config_dir, 'prefs')) 58 | if not self._config.has_section('crmngr'): 59 | self._config.add_section('crmngr') 60 | 61 | # initialize profiles 62 | self._profiles = ConfigParser() 63 | self._profiles.read(os.path.join(self._config_dir, 'profiles')) 64 | self._control_repo_url = self._profiles.get(self.profile, 'repository') 65 | 66 | @classmethod 67 | def create_default_configuration(cls, default_profile_url): 68 | """ensure a default profile is configured.""" 69 | config_directory = cls._ensure_configuration_directory() 70 | config = ConfigParser() 71 | config.add_section('default') 72 | config.set('default', 'repository', default_profile_url) 73 | with open(os.path.join(config_directory, 'profiles'), 74 | 'w') as profiles_file: 75 | config.write(profiles_file) 76 | return cls() 77 | 78 | @property 79 | def cache_dir(self): 80 | """returns the cache directory""" 81 | self._cache_dir = os.path.join(self._config_dir, 'cache') 82 | os.makedirs(self._cache_dir, exist_ok=True) 83 | return self._cache_dir 84 | 85 | @property 86 | def control_repo_url(self): 87 | """returns control repo url""" 88 | return self._control_repo_url 89 | 90 | @property 91 | def profile(self): 92 | """returns active configuration profile""" 93 | return self._profile 94 | 95 | @property 96 | def profiles(self): 97 | """returns a list of valid profiles""" 98 | profiles = [] 99 | Profile = namedtuple( # pylint: disable=invalid-name 100 | 'Profile', ['name', 'repository'] 101 | ) 102 | # make sure, we list the default profile first 103 | profiles.append( 104 | Profile( 105 | name='default', 106 | repository=self._profiles.get('default', 107 | 'repository') 108 | ) 109 | ) 110 | for profile in sorted(self._profiles.sections()): 111 | # default profile already in result list, ignore it. 112 | if profile == 'default': 113 | continue 114 | profiles.append( 115 | Profile( 116 | name=profile, 117 | repository=self._profiles.get(profile, 118 | 'repository', 119 | fallback='unconfigured'), 120 | ) 121 | ) 122 | return profiles 123 | 124 | @property 125 | def version_check(self): 126 | """returns version_check config setting as bool""" 127 | return self._config.getboolean('crmngr', 'version_check') 128 | 129 | @property 130 | def cache_ttl(self): 131 | """returns cache_ttl config setting as int""" 132 | return self._config.getint('crmngr', 'cache_ttl') 133 | 134 | @property 135 | def wrap(self): 136 | """returns wrap config setting as bool""" 137 | return self._config.getboolean('crmngr', 'wrap') 138 | 139 | @staticmethod 140 | def _ensure_configuration_directory(directory=None): 141 | """ensures the crmngr configuration directory exists""" 142 | if directory is None: 143 | directory = os.path.join(os.path.expanduser('~'), '.crmngr') 144 | os.makedirs(directory, exist_ok=True, mode=0o750) 145 | LOG.debug("Configuration directory is %s", directory) 146 | return directory 147 | -------------------------------------------------------------------------------- /crmngr/controlrepository.py: -------------------------------------------------------------------------------- 1 | """ crmngr controlrepository module """ 2 | 3 | # stdlib 4 | from collections import defaultdict 5 | from collections import OrderedDict 6 | from itertools import chain 7 | import logging 8 | import os 9 | import re 10 | import sys 11 | from tempfile import TemporaryDirectory 12 | from textwrap import TextWrapper 13 | 14 | # crmgnr 15 | from crmngr.cache import JsonCache 16 | from crmngr.forgeapi import ForgeApi, ForgeError 17 | from crmngr.git import Repository 18 | from crmngr.git import GitError 19 | from crmngr.puppetfile import Forge 20 | from crmngr.puppetfile import ForgeModule 21 | from crmngr.puppetfile import GitBranch 22 | from crmngr.puppetfile import GitCommit 23 | from crmngr.puppetfile import GitModule 24 | from crmngr.puppetfile import GitTag 25 | from crmngr.puppetfile import PuppetModule 26 | from crmngr import cprint 27 | from crmngr.utils import fnlistmatch 28 | from crmngr.utils import query_yes_no 29 | 30 | # 3rd-party 31 | from natsort import natsorted 32 | 33 | LOG = logging.getLogger(__name__) 34 | 35 | 36 | class NoEnvironmentError(Exception): 37 | """exception raised when no environment is matched""" 38 | 39 | 40 | class PuppetEnvironment: 41 | """r10k puppet environment""" 42 | 43 | def __init__(self, name, modules=None): 44 | """initialize puppet environment""" 45 | self._name = name 46 | self._modules = modules or OrderedDict() 47 | LOG.debug('initialize PuppetEnvironment(%s)', self._name) 48 | 49 | def __repr__(self): 50 | return self._name 51 | 52 | def __iter__(self): 53 | return iter(self._modules.items()) 54 | 55 | def __delitem__(self, key): 56 | del self._modules[key] 57 | 58 | def __getitem__(self, item): 59 | return self._modules[item] 60 | 61 | def __setitem__(self, key, value): 62 | self._modules[key] = value 63 | LOG.debug('added %s(%s) for PuppetEnvironment(%s)', 64 | key, 65 | vars(value), 66 | self.name) 67 | 68 | def __lt__(self, other): 69 | return str(other) > str(self) 70 | 71 | @property 72 | def modules(self): 73 | """Returns a list of modules in the puppet environment""" 74 | return self._modules 75 | 76 | @property 77 | def name(self): 78 | """Returns the name of the current puppet environment""" 79 | return self._name 80 | 81 | @property 82 | def puppetfile(self): 83 | """Returns the puppetfile reprensentation as iterator""" 84 | for _, module in sorted(self._modules.items()): 85 | yield module.puppetfile 86 | 87 | 88 | class ControlRepository(Repository): 89 | """r10k-style control repository""" 90 | 91 | def __init__(self, clone_url, environments=None, modules=None): 92 | """clone control repository and parse the puppetfiles it contains.""" 93 | super().__init__(clone_url) 94 | 95 | self._environments = [] 96 | self._parse_puppetfiles( 97 | puppetfiles=self._collect_puppetfiles(environments), 98 | puppetmodules=modules, 99 | ) 100 | 101 | @property 102 | def environments(self): 103 | """returns a list of all environment objects""" 104 | return self._environments 105 | 106 | @property 107 | def environment_names(self): 108 | """return a set of all environment names""" 109 | return set([environment.name for environment in self._environments]) 110 | 111 | def get_environment(self, environment): 112 | """return a specifc environment""" 113 | return next(env 114 | for env in self._environments if env.name == environment) 115 | 116 | def clone_environment(self, base_env, new_env, *, report=True): 117 | """creates a new environment as a clone of an existing one.""" 118 | self.git(['checkout', base_env]) 119 | self.git(['checkout', '--orphan', new_env]) 120 | self.git(['commit', '-m', 'Initialize new environment.']) 121 | self.git(['push', 'origin', new_env]) 122 | 123 | if report: 124 | # reread control repository with only new environment 125 | self._environments = [] 126 | self._parse_puppetfiles( 127 | self._collect_puppetfiles([new_env, ]) 128 | ) 129 | 130 | def delete_environment(self, environment): 131 | """deletes an existing environment.""" 132 | self.git(['push', 'origin', ':%s' % environment]) 133 | 134 | def new_environment(self, new_env, *, report=True): 135 | """creates a new empty environment.""" 136 | self.git(['checkout', '--orphan', new_env]) 137 | self.git(['reset', '--hard']) 138 | 139 | with open(os.path.join(self._workdir, 'Puppetfile'), 'w') as puppetfile: 140 | puppetfile.write("forge 'http://forge.puppetlabs.com'\n\n") 141 | self.git(['add', 'Puppetfile']) 142 | os.mkdir(os.path.join(self._workdir, 'manifests')) 143 | with open(os.path.join(self._workdir, 'manifests', 'site.pp'), 144 | 'w') as site_pp: 145 | site_pp.write("hiera_include('classes')") 146 | self.git(['add', os.path.join('manifests', 'site.pp')]) 147 | self.git(['commit', '-m', 'Initialize new environment.']) 148 | self.git(['push', 'origin', new_env]) 149 | if report: 150 | # reread control repository with only new environment 151 | self._environments = [] 152 | self._parse_puppetfiles( 153 | self._collect_puppetfiles([new_env, ]) 154 | ) 155 | 156 | def _collect_puppetfiles(self, environments=None): 157 | """collect Puppetfile from all control repository branches. 158 | 159 | This will return a dictionary with environments as keys and a list of 160 | every Puppetfile mod line as value. 161 | 162 | Mulitiline mod lines are collapsed, empty and comment lines ignored. 163 | """ 164 | 165 | puppetfiles = {} 166 | 167 | for branch in self.branches: 168 | if environments is not None: 169 | if environments[0] == '!': 170 | if fnlistmatch(branch, patterns=environments[1:]): 171 | LOG.debug( 172 | ('branch %s does match an exclude pattern ' 173 | '%s. Skipping.'), 174 | branch, 175 | environments[1:] 176 | ) 177 | continue 178 | else: 179 | if not fnlistmatch(branch, patterns=environments): 180 | LOG.debug( 181 | ('branch %s does not match any include pattern ' 182 | '%s. Skipping.'), 183 | branch, 184 | environments 185 | ) 186 | continue 187 | 188 | self.git(['checkout', branch]) 189 | with open(os.path.join(self._workdir, 'Puppetfile')) as puppetfile: 190 | puppetfiles[branch] = self._collapse_puppetfile( 191 | puppetfile.readlines() 192 | ) 193 | 194 | if puppetfiles: 195 | return puppetfiles 196 | else: 197 | raise NoEnvironmentError 198 | 199 | @staticmethod 200 | def _collapse_puppetfile(lines): 201 | """remove whitespace and comments from puppetfile lines""" 202 | puppetfile_lines = [] 203 | re_comment = re.compile(r'^\s*#') 204 | re_mod = re.compile(r'^\s*mod') 205 | line_buffer = None 206 | for line in lines: 207 | stripped_line = line.strip() 208 | # skip empty line or comment 209 | if stripped_line == '' or re_comment.match(stripped_line): 210 | continue 211 | # processing multiline mod-block 212 | if line_buffer: 213 | # append current line to buffer 214 | line_buffer += stripped_line 215 | # if line does not end with a comma, we are done 216 | if not stripped_line.endswith(','): 217 | puppetfile_lines.append(line_buffer) 218 | line_buffer = None 219 | continue 220 | # we are not processing a multiline block, only mod lines 221 | # are interesting 222 | if re_mod.match(stripped_line): 223 | # start processing of multiline mod-block 224 | if stripped_line.endswith(','): 225 | line_buffer = stripped_line 226 | else: 227 | puppetfile_lines.append(stripped_line) 228 | return puppetfile_lines 229 | 230 | def _parse_puppetfiles(self, puppetfiles, puppetmodules=None): 231 | """extract module information from puppetfiles""" 232 | for environment, modulelines in puppetfiles.items(): 233 | puppetenvironment = PuppetEnvironment( 234 | environment 235 | ) 236 | for moduleline in modulelines: 237 | LOG.debug('processing module %s in environment %s', 238 | moduleline, 239 | environment) 240 | module_object = PuppetModule.from_moduleline( 241 | moduleline 242 | ) 243 | if puppetmodules is not None: 244 | if puppetmodules[0] == '!': 245 | if fnlistmatch(module_object.name, 246 | patterns=puppetmodules[1:]): 247 | LOG.debug( 248 | ('module %s does match an exclude pattern %s. ' 249 | 'Skipping.'), 250 | module_object.name, 251 | puppetmodules[1:] 252 | ) 253 | continue 254 | else: 255 | if not fnlistmatch(module_object.name, 256 | patterns=puppetmodules): 257 | LOG.debug( 258 | ('module %s does not match any include pattern ' 259 | '%s. Skipping.'), 260 | module_object.name, 261 | puppetmodules 262 | ) 263 | continue 264 | puppetenvironment[module_object.name] = module_object 265 | self._environments.append(puppetenvironment) 266 | 267 | @staticmethod 268 | def _bulk_update(environment, *, modules, cache): 269 | """updates modules in environment to latest version.""" 270 | cprint.white_bold('Bulk update environment {}'.format(environment.name)) 271 | for _, module in environment: 272 | if modules is not None: 273 | if modules[0] == '!': 274 | if fnlistmatch(module.name, patterns=modules[1:]): 275 | LOG.debug('module %s does match an exclude pattern %s. ' 276 | 'Skipping.', module.name, modules[1:]) 277 | continue 278 | else: 279 | if not fnlistmatch(module.name, patterns=modules): 280 | LOG.debug('module %s does not match any include ' 281 | 'pattern %s. Skipping.', module.name, modules) 282 | continue 283 | try: 284 | cprint.white('Get latest version for module {}'.format( 285 | module.name 286 | )) 287 | module.version = module.get_latest_version(cache) 288 | except TypeError: 289 | LOG.debug('Could not determine latest module version for ' 290 | 'module %s. Setting to None.', module.name) 291 | cprint.yellow_bold('Could not determine latest module version ' 292 | 'for module {}!'.format(module.name)) 293 | module.version = None 294 | return environment 295 | 296 | @staticmethod 297 | def _bulk_remove(environment, *, modules): 298 | """bulk remove modules from an environment.""" 299 | cprint.white_bold( 300 | 'Bulk remove modules from environment {}'.format(environment.name) 301 | ) 302 | for _, module in environment.modules.copy().items(): 303 | if modules is not None: 304 | if modules[0] == '!': 305 | if fnlistmatch(module.name, patterns=modules[1:]): 306 | LOG.debug('module %s does match an exclude pattern %s. ' 307 | 'Skipping.', module.name, modules[1:]) 308 | continue 309 | else: 310 | if not fnlistmatch(module.name, patterns=modules): 311 | LOG.debug('module %s does not match any include ' 312 | 'pattern %s. Skipping.', module.name, modules) 313 | continue 314 | LOG.debug('remove module %s from environment %s', 315 | module.name, environment.name) 316 | del environment[module.name] 317 | return environment 318 | 319 | @staticmethod 320 | def _reference_update(environment, *, reference, add=False, remove=True): 321 | """updates an environment based on a reference branch.""" 322 | if add and remove: 323 | # if we set both add and remove, we basically replace 324 | # the environment with the template 325 | environment = PuppetEnvironment(environment.name, 326 | reference.modules) 327 | LOG.debug('replaced environment %s with %s', 328 | environment.name, 329 | reference.name) 330 | else: 331 | for _, module in environment.modules.copy().items(): 332 | if module.name in reference.modules: 333 | environment[module.name] = reference[module.name] 334 | LOG.debug('module %s in environemnt %s has been replaced ' 335 | 'with version from reference environment %s', 336 | module.name, environment.name, reference.name) 337 | if remove and (module.name not in reference.modules): 338 | del environment[module.name] 339 | LOG.debug('removed module %s from environment %s as it is ' 340 | 'not present in the reference environment %s', 341 | module.name, environment.name, reference.name) 342 | if add: 343 | # add modules missing in environment branch (but present in 344 | # reference branch) 345 | for module in (set(reference.modules) - 346 | set(environment.modules)): 347 | environment[module] = reference[module] 348 | return environment 349 | 350 | @staticmethod 351 | def _update_forge_module(module_string, version): 352 | """return new version of a single forge module""" 353 | module = ForgeModule(*PuppetModule.parse_module_name( 354 | module_string 355 | )) 356 | forge_api = ForgeApi(name=module.name, author=module.author) 357 | if version is None: 358 | module.version = None 359 | elif version == 'LATEST_FORGE_VERSION': 360 | try: 361 | module.version = Forge(forge_api.current_version['version']) 362 | except ForgeError as exc: 363 | cprint.red( 364 | 'Could not determine latest version of forge ' 365 | 'module {author}/{module}: {error}'.format( 366 | author=module.author, 367 | module=module.name, 368 | error=exc, 369 | ) 370 | ) 371 | sys.exit(1) 372 | else: 373 | try: 374 | if not forge_api.has_version(version): 375 | cprint.red( 376 | '{version} is not a valid version for module ' 377 | '{author}/{module}'.format( 378 | version=version, 379 | author=module.author, 380 | module=module.name, 381 | ) 382 | ) 383 | sys.exit(1) 384 | except ForgeError as exc: 385 | cprint.red( 386 | 'Could not verify version {version} of forge ' 387 | 'module {author}/{module}: {error}'.format( 388 | version=version, 389 | author=module.author, 390 | module=module.name, 391 | error=exc, 392 | ) 393 | ) 394 | sys.exit(1) 395 | module.version = Forge(version) 396 | return module 397 | 398 | def _update_git_module(self, module_string, *, url, branch, commit, tag): 399 | """return new version of a single git module""" 400 | # pylint: disable=too-many-branches 401 | module_name = PuppetModule.parse_module_name(module_string).module 402 | 403 | if url == 'USE_EXISTING_URL': 404 | git_urls = {version.url: environments 405 | for version, environments in 406 | self.modules[module_name].items() 407 | if isinstance(version, GitModule)} 408 | if len(git_urls) > 1: 409 | cprint.red('Multiple URLs for {module} found across specified ' 410 | 'environments:\n - {urls}'.format( 411 | module=module_name, 412 | urls='\n - '.join([ 413 | "{url} ({environments})".format( 414 | url=url, 415 | environments=', '.join( 416 | sorted(environments) 417 | ) 418 | ) 419 | for url, environments in sorted( 420 | git_urls.items() 421 | ) 422 | ]) 423 | )) 424 | cprint.red('Specify an URL using --git option or restrict the ' 425 | 'update to environments using the same URL for ' 426 | '{}'.format(module_name)) 427 | sys.exit(1) 428 | elif len(git_urls) < 1: 429 | cprint.red('Git module {module} not in any of the specified ' 430 | 'environments. To switch from forge to git for ' 431 | '{module}, pass an URL to --git.'.format( 432 | module=module_name 433 | )) 434 | sys.exit(1) 435 | url = list(git_urls)[0] 436 | try: 437 | module_repository = Repository(url) 438 | except GitError as exc: 439 | cprint.red( 440 | '{url} is not a valid git repository: {error}'.format( 441 | url=url, 442 | error=exc, 443 | ) 444 | ) 445 | sys.exit(1) 446 | module = GitModule(module_name, url=url) 447 | if branch is not None: 448 | try: 449 | module_repository.git(['fetch', '--unshallow']) 450 | module_repository.validate_branch(branch) 451 | module.version = GitBranch(branch) # pylint: disable=R0204 452 | except GitError as exc: 453 | cprint.red('Could not verify branch {branch} for {module}: ' 454 | '{error}'.format( 455 | branch=branch, 456 | module=module.name, 457 | error=exc 458 | )) 459 | sys.exit(1) 460 | elif commit is not None: 461 | try: 462 | module_repository.git(['fetch', '--unshallow']) 463 | module_repository.validate_commit(commit) 464 | module.version = GitCommit(commit) # pylint: disable=R0204 465 | except GitError as exc: 466 | cprint.red('Could not verify commit {commit} for {module}: ' 467 | '{error}'.format( 468 | commit=commit, 469 | module=module.name, 470 | error=exc 471 | )) 472 | sys.exit(1) 473 | elif tag is not None: 474 | if tag == 'LATEST_GIT_TAG': 475 | try: 476 | module.version = GitTag( # pylint: disable=R0204 477 | module_repository.latest_tag.name 478 | ) 479 | except GitError as exc: 480 | cprint.red('Could not determine latest tag for git module ' 481 | '{module}: {error}'.format( 482 | module=module.name, 483 | error=exc, 484 | )) 485 | sys.exit(1) 486 | else: 487 | try: 488 | module_repository.git(['fetch', 'origin', '--tags']) 489 | module_repository.validate_tag(tag) 490 | module.version = GitTag(tag) # pylint: disable=R0204 491 | except GitError as exc: 492 | cprint.red('Could not verify tag {tag} for {module}: ' 493 | '{error}'.format( 494 | tag=tag, 495 | module=module.name, 496 | error=exc 497 | )) 498 | sys.exit(1) 499 | return module 500 | 501 | def update_puppetfiles(self, *, cli_args): 502 | """update puppetfiles""" 503 | with TemporaryDirectory(prefix='crmngr_update_cache') as cache_dir: 504 | reference = None 505 | module = None 506 | 507 | update_cache = JsonCache(cache_dir) 508 | if cli_args.reference: 509 | try: 510 | reference = self.get_environment(cli_args.reference) 511 | except StopIteration: 512 | cprint.red('%s specified as reference environment does not ' 513 | 'exist' % cli_args.reference) 514 | sys.exit(1) 515 | elif cli_args.git_url and not cli_args.remove: 516 | module = self._update_git_module( # pylint: disable=R0204 517 | module_string=cli_args.modules[0], 518 | url=cli_args.git_url, 519 | branch=cli_args.git_branch, 520 | commit=cli_args.git_commit, 521 | tag=cli_args.git_tag, 522 | ) 523 | elif cli_args.forge and not cli_args.remove: 524 | module = self._update_forge_module( # pylint: disable=R0204 525 | module_string=cli_args.modules[0], 526 | version=cli_args.forge_version, 527 | ) 528 | for environment in sorted(self._environments): 529 | # reference update mode 530 | commit_message = 'Update Environment' 531 | if reference: 532 | if environment == reference: 533 | # when having a reference environment, it will be in the 534 | # control repository. So we skip it. 535 | continue 536 | environment = self._reference_update( 537 | environment, 538 | reference=reference, 539 | add=cli_args.add, 540 | remove=cli_args.remove, 541 | ) 542 | commit_message = 'Update {} based on {}.'.format( 543 | environment.name, 544 | reference.name, 545 | ) 546 | elif module is not None: 547 | if cli_args.add or module.name in environment.modules: 548 | environment[module.name] = module 549 | commit_message = module.update_commit_message 550 | elif cli_args.remove: 551 | environment = self._bulk_remove( 552 | environment, 553 | modules=cli_args.modules, 554 | ) 555 | commit_message = 'Bulk update {}.'.format(environment.name) 556 | # bulk update mode (i.e. no version / update options specified) 557 | else: 558 | environment = self._bulk_update( 559 | environment, 560 | modules=cli_args.modules, 561 | cache=update_cache, 562 | ) 563 | commit_message = 'Bulk update {}.'.format(environment.name) 564 | self.write_puppetfile( 565 | commit_message=commit_message, 566 | diff_only=cli_args.diffonly, 567 | environment=environment, 568 | non_interactive=cli_args.noninteractive, 569 | ) 570 | 571 | def write_puppetfile(self, environment, *, 572 | commit_message='Update Puppetfile', diff_only=False, 573 | non_interactive=False): 574 | """write a PuppetEnvironment to a Puppetfile""" 575 | self.git(['checkout', environment.name]) 576 | with open(os.path.join(self._workdir, 'Puppetfile'), 577 | 'w') as puppetfile: 578 | LOG.debug('write new version of Puppetfile in environment %s', 579 | environment.name) 580 | # write file header 581 | puppetfile.write("forge 'http://forge.puppetlabs.com'\n\n") 582 | # write module lines 583 | puppetfile.writelines( 584 | ('{}\n'.format(line) 585 | for line in chain.from_iterable(environment.puppetfile)) 586 | ) 587 | # ask git for a diff 588 | diff = self.git(['diff']) 589 | if diff: 590 | if not non_interactive: 591 | cprint.white_bold( 592 | 'Diff for environment %s:' % environment.name 593 | ) 594 | cprint.diff(diff) 595 | if diff_only: 596 | # revert pending changes 597 | self.git(['checkout', '--', 'Puppetfile']) 598 | else: 599 | if non_interactive or query_yes_no( 600 | 'Update (commit and push) Puppetfile for ' 601 | 'environment {}'.format(environment.name)): 602 | # commit and push pending changes 603 | self.git( 604 | ['commit', '-m', commit_message, 'Puppetfile'] 605 | ) 606 | try: 607 | self.git(['push', 'origin', environment.name]) 608 | cprint.green('Updated environment {}'.format( 609 | environment.name 610 | )) 611 | except GitError as exc: 612 | cprint.red( 613 | 'Could not update environment {environment}. ' 614 | 'Maybe somebody else pushed changes to ' 615 | '{environment} during current crmngr run. ' 616 | 'Full git error: {error}'.format( 617 | environment=environment.name, 618 | error=exc, 619 | )) 620 | sys.exit(1) 621 | else: 622 | # revert pending changes 623 | self.git(['checkout', '--', 'Puppetfile']) 624 | else: 625 | LOG.debug('Puppetfile for environment %s unchanged', environment.name) 626 | 627 | @property 628 | def modules(self): 629 | """returns modules and module versions. 630 | 631 | This will return a dict containing all modules with their versions and 632 | environments they are deployed in. 633 | """ 634 | modules = defaultdict(lambda: defaultdict(set)) 635 | for environment in self._environments: 636 | for module, module_object in environment: 637 | modules[module][module_object].add(environment.name) 638 | return modules 639 | 640 | def report(self, wrap=True, version_check=True, version_cache=None, 641 | compare=True): 642 | """print control repository report""" 643 | 644 | for module, versions in sorted(self.modules.items()): 645 | 646 | # in compare mode, skip modules that are identical in all processed 647 | # environments. 648 | if compare and len(versions) == 1 and \ 649 | len(list(versions.values())[0]) == len(self._environments): 650 | continue 651 | 652 | cprint.white_bold('Module: %s' % module) 653 | for version, environments in natsorted( 654 | versions.items(), 655 | reverse=True, 656 | key=str 657 | ): 658 | version.print_version_information( 659 | version_check, 660 | version_cache 661 | ) 662 | if len(self._environments) > 1: 663 | cprint.white('Used by:', lpad=4, rpad=4, end='') 664 | if wrap: 665 | used_by = TextWrapper( 666 | subsequent_indent=' ' * 16 667 | ) 668 | for line in used_by.wrap( 669 | ' '.join(sorted(environments)) 670 | ): 671 | cprint.cyan(line) 672 | else: 673 | cprint.cyan(' '.join(sorted(environments))) 674 | print() 675 | 676 | if compare: 677 | # check for modules that are in not in all (but in at least one) 678 | # processed environments 679 | missing = set.union(*list(versions.values())) ^ set( 680 | [environment.name for environment in self._environments] 681 | ) 682 | if missing: 683 | cprint.yellow_bold('Missing from:', lpad=2) 684 | if wrap: 685 | not_in = TextWrapper( 686 | initial_indent=' ' * 16, 687 | subsequent_indent=' ' * 16 688 | ) 689 | for line in not_in.wrap( 690 | ' '.join(sorted(missing)) 691 | ): 692 | cprint.yellow(line) 693 | else: 694 | print(' ' * 16, end='') 695 | cprint.yellow(' '.join(sorted(missing))) 696 | print() 697 | -------------------------------------------------------------------------------- /crmngr/cprint.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | """function for colored terminal output""" 4 | 5 | TERM_BLUE = '\033[0;34m' 6 | TERM_BLUE_BOLD = '\033[1;34m' 7 | TERM_CYAN = '\033[0;36m' 8 | TERM_CYAN_BOLD = '\033[1;36m' 9 | TERM_GREEN = '\033[0;32m' 10 | TERM_GREEN_BOLD = '\033[1;32m' 11 | TERM_MAGENTA = '\033[0;35m' 12 | TERM_MAGENTA_BOLD = '\033[1;35m' 13 | TERM_RED = '\033[0;31m' 14 | TERM_RED_BOLD = '\033[1;31m' 15 | TERM_WHITE = '\033[0;37m' 16 | TERM_WHITE_BOLD = '\033[1;37m' 17 | TERM_YELLOW = '\033[0;33m' 18 | TERM_YELLOW_BOLD = '\033[1;33m' 19 | 20 | TERM_NONE = '\033[0;m' 21 | 22 | 23 | def _cprint(color, text, **kwargs): 24 | """helper function to print colored text""" 25 | prefix = kwargs.pop('prefix', '') 26 | suffix = kwargs.pop('suffix', '') 27 | lpad = kwargs.pop('lpad', 0) 28 | rpad = kwargs.pop('rpad', 0) 29 | sep = kwargs.pop('sep', '') 30 | 31 | print( 32 | ' '*lpad, 33 | prefix, 34 | color, 35 | text, 36 | suffix, 37 | ' '*rpad, 38 | TERM_NONE, 39 | sep=sep, 40 | **kwargs 41 | ) 42 | 43 | 44 | def blue(text, **kwargs): 45 | """print blue text""" 46 | _cprint(TERM_BLUE, text, **kwargs) 47 | 48 | 49 | def blue_bold(text, **kwargs): 50 | """print bold blue text""" 51 | _cprint(TERM_BLUE_BOLD, text, **kwargs) 52 | 53 | 54 | def cyan(text, **kwargs): 55 | """print cyan text""" 56 | _cprint(TERM_CYAN, text, **kwargs) 57 | 58 | 59 | def cyan_bold(text, **kwargs): 60 | """print bold cyan text""" 61 | _cprint(TERM_CYAN_BOLD, text, **kwargs) 62 | 63 | 64 | def green(text, **kwargs): 65 | """print green text""" 66 | _cprint(TERM_GREEN, text, **kwargs) 67 | 68 | 69 | def green_bold(text, **kwargs): 70 | """print bold green text""" 71 | _cprint(TERM_GREEN_BOLD, text, **kwargs) 72 | 73 | 74 | def magenta(text, **kwargs): 75 | """print magenta text""" 76 | _cprint(TERM_MAGENTA, text, **kwargs) 77 | 78 | 79 | def magenta_bold(text, **kwargs): 80 | """print bold magenta text""" 81 | _cprint(TERM_MAGENTA_BOLD, text, **kwargs) 82 | 83 | 84 | def red(text, **kwargs): 85 | """print red text""" 86 | _cprint(TERM_RED, text, **kwargs) 87 | 88 | 89 | def red_bold(text, **kwargs): 90 | """print bold red text""" 91 | _cprint(TERM_RED_BOLD, text, **kwargs) 92 | 93 | 94 | def white(text, **kwargs): 95 | """print white text""" 96 | _cprint(TERM_WHITE, text, **kwargs) 97 | 98 | 99 | def white_bold(text, **kwargs): 100 | """print bold white text""" 101 | _cprint(TERM_WHITE_BOLD, text, **kwargs) 102 | 103 | 104 | def yellow(text, **kwargs): 105 | """print yellow text""" 106 | _cprint(TERM_YELLOW, text, **kwargs) 107 | 108 | 109 | def yellow_bold(text, **kwargs): 110 | """print bold yellow text""" 111 | _cprint(TERM_YELLOW_BOLD, text, **kwargs) 112 | 113 | 114 | def diff(text): 115 | """print colorized diff""" 116 | for line in text.split('\n'): 117 | if line.startswith('-'): 118 | red(line) 119 | elif line.startswith('+'): 120 | green(line) 121 | else: 122 | white(line) 123 | -------------------------------------------------------------------------------- /crmngr/forgeapi.py: -------------------------------------------------------------------------------- 1 | """ crmngr forgeapi module """ 2 | 3 | # stdlib 4 | from datetime import datetime 5 | import logging 6 | 7 | # 3rd-party 8 | import requests 9 | from requests.exceptions import RequestException 10 | 11 | # crmngr 12 | from crmngr.utils import truncate 13 | 14 | LOG = logging.getLogger(__name__) 15 | 16 | 17 | class ForgeError(Exception): 18 | """exception raised when a forge connection/parse error occurs.""" 19 | 20 | 21 | class ForgeApi: 22 | """puppetforge module api""" 23 | 24 | def __init__(self, *, name, author): 25 | """initialize module api""" 26 | self._name = name 27 | self._author = author 28 | self._url = '{forgeapi}/{author}-{module}'.format( 29 | forgeapi='https://forgeapi.puppetlabs.com/v3/modules', 30 | author=self._author, 31 | module=self._name 32 | ) 33 | 34 | @property 35 | def current_version(self): 36 | """get version for current release""" 37 | try: 38 | LOG.debug('request info from %s', self._url) 39 | api = requests.get(self._url) 40 | api_info = api.json()['current_release'] 41 | LOG.debug('received module info from API: %s', truncate(api_info)) 42 | except (RequestException, KeyError) as exc: 43 | LOG.debug('could not read from api: %s', exc) 44 | raise ForgeError('could not read from api: %s' % exc) from None 45 | 46 | try: 47 | return { 48 | 'version': api_info['version'], 49 | 'date': datetime.strptime( 50 | api_info['updated_at'], '%Y-%m-%d %H:%M:%S %z' 51 | ).strftime('%Y-%m-%d'), 52 | } 53 | except (AttributeError, KeyError, TypeError, ValueError) as exc: 54 | LOG.debug('could not parse api response: %s', exc) 55 | raise ForgeError('could not parse api response: %s' % exc) from None 56 | 57 | def has_version(self, version): 58 | """verify wheter a release with requested version exists.""" 59 | try: 60 | LOG.debug('request info from %s', self._url) 61 | api = requests.get(self._url) 62 | api_info = api.json()['releases'] 63 | LOG.debug( 64 | 'received module info from API: %s', 65 | truncate(api_info), 66 | ) 67 | except (RequestException, KeyError) as exc: 68 | LOG.debug('could not read from api: %s', exc) 69 | raise ForgeError('could not read from api: %s' % exc) from None 70 | 71 | return bool([release['version'] 72 | for release in api_info if release['version'] == version]) 73 | -------------------------------------------------------------------------------- /crmngr/git.py: -------------------------------------------------------------------------------- 1 | """ crmngr git module """ 2 | 3 | # stdlib 4 | from collections import namedtuple 5 | from datetime import datetime 6 | import logging 7 | import os 8 | import re 9 | import subprocess 10 | from tempfile import TemporaryDirectory 11 | 12 | LOG = logging.getLogger(__name__) 13 | 14 | 15 | class GitError(Exception): 16 | """exception raised when a git command fails""" 17 | 18 | 19 | class Repository: 20 | """a git repository""" 21 | 22 | def __init__(self, clone_url): 23 | """clone a remote repository""" 24 | self._url = clone_url 25 | self._tmpdir = TemporaryDirectory(prefix='crmngr_repository_') 26 | self.git([ 27 | 'clone', 28 | '--depth=1', 29 | '--quiet', 30 | '--no-single-branch', 31 | self._url, 32 | 'git' 33 | ], cwd=self._tmpdir.name) 34 | self._workdir = os.path.join(self._tmpdir.name, 'git') 35 | LOG.debug('cloned %s into %s', self._url, self._workdir) 36 | 37 | def __enter__(self): 38 | return self 39 | 40 | def __exit__(self, *args): 41 | self._tmpdir.cleanup() 42 | 43 | def git(self, cmds, cwd=None, **kwargs): 44 | """execute a git command""" 45 | cmds = ['git'] + cmds 46 | if cwd is None: 47 | cwd = self._workdir 48 | 49 | try: 50 | rval = subprocess.check_output( 51 | cmds, 52 | stderr=subprocess.STDOUT, 53 | cwd=cwd, 54 | universal_newlines=True, 55 | **kwargs 56 | ) 57 | LOG.debug( 58 | 'command "%s" completed with exit code "0" and output: "%s"', 59 | ' '.join(cmds), 60 | rval.replace('\n', '; ').strip('; '), 61 | ) 62 | except subprocess.CalledProcessError as exc: 63 | raise GitError( 64 | 'command "%s" failed with exit code "%s" and output: "%s"' % ( 65 | ' '.join(cmds), 66 | exc.returncode, 67 | exc.output.replace('\n', '; ').strip('; '), 68 | ) 69 | ) from None 70 | return rval 71 | 72 | def validate_branch(self, branch): 73 | """verify if repository has a specific branch""" 74 | if not self.git(['branch', '--list', '--all', 'origin/%s' % branch]): 75 | raise GitError( 76 | "Branch {branch} not found for repository {url}".format( 77 | branch=branch, 78 | url=self._url 79 | ) 80 | ) 81 | 82 | def validate_tag(self, tag): 83 | """verify if repository has a specific tag""" 84 | if not self.git(['tag', '--list', tag]): 85 | raise GitError( 86 | "Tag {tag} not found for repository {url}".format( 87 | tag=tag, 88 | url=self._url 89 | ) 90 | ) 91 | 92 | def validate_commit(self, commit): 93 | """verify if repository has a specific commit""" 94 | output = self.git(['cat-file', '-t', commit]).strip() 95 | 96 | if output != 'commit': 97 | raise GitError( 98 | "Commit {commit} not found for repository {url}".format( 99 | commit=commit, 100 | url=self._url 101 | ) 102 | ) 103 | 104 | @property 105 | def branches(self): 106 | """returns all """ 107 | gitbranches = self.git(['branch', '--list', '--all']).split('\n') 108 | 109 | re_branch = re.compile(r'\s*remotes/origin/(?P[^\s]+$)') 110 | 111 | for gitbranch in gitbranches: 112 | try: 113 | yield re_branch.match(gitbranch).groupdict()['branch'] 114 | except AttributeError: 115 | continue 116 | 117 | @property 118 | def latest_tag(self): 119 | """returns a namedtuple of (name, date) for the newest tag""" 120 | Tag = namedtuple( # pylint: disable=invalid-name 121 | 'GitTagDate', ['name', 'date'] 122 | ) 123 | 124 | try: 125 | self.git(['fetch', '--tags']) 126 | # get sha1 for latest tag 127 | cid = self.git(['rev-list', '--tags', '--max-count=1']).strip() 128 | # get tag name from sha1 129 | tag_name = self.git(['describe', '--tags', cid]).strip() 130 | # get date for tag 131 | date = datetime.strptime(self.git( 132 | ['show', '-s', '--format=%ci', '%s^{commit}' % tag_name], 133 | ).strip(), '%Y-%m-%d %H:%M:%S %z') 134 | except GitError as exc: 135 | LOG.debug('could not determine latest tag in repository %s: %s', 136 | self._url, exc) 137 | raise 138 | 139 | return Tag(name=tag_name, date=date) 140 | 141 | @property 142 | def url(self): 143 | """returns repository url""" 144 | return self._url 145 | -------------------------------------------------------------------------------- /crmngr/puppetfile.py: -------------------------------------------------------------------------------- 1 | """ crmngr puppetmodule module """ 2 | 3 | # stdlib 4 | from collections import namedtuple 5 | import hashlib 6 | import logging 7 | from datetime import datetime 8 | 9 | # crmngr 10 | from crmngr import cprint 11 | from crmngr.forgeapi import ForgeApi 12 | from crmngr.forgeapi import ForgeError 13 | from crmngr.git import GitError 14 | from crmngr.git import Repository 15 | 16 | LOG = logging.getLogger(__name__) 17 | 18 | 19 | class PuppetModule: 20 | """Base class for puppet modules""" 21 | 22 | def __init__(self, name): 23 | """Initialize puppet module""" 24 | self._name = name 25 | self._version = None 26 | 27 | @staticmethod 28 | def parse_module_name(string): 29 | """parse module file name into author/name""" 30 | ModuleName = namedtuple( # pylint: disable=invalid-name 31 | 'ModuleName', ['module', 'author'] 32 | ) 33 | module_name = string.strip(' \'"').rsplit('/', 1) 34 | try: 35 | module = ModuleName(module=module_name[1], author=module_name[0]) 36 | except IndexError: 37 | module = ModuleName(module=module_name[0], author=None) 38 | LOG.debug("%s parsed into %s", string, module) 39 | return module 40 | 41 | @classmethod 42 | def from_moduleline(cls, moduleline): 43 | """returns a crmngr module object based on a puppetfile module line""" 44 | # split module line into comma-separated parts (starting after 45 | # 'mod ') 46 | line_parts = moduleline[4:].split(',') 47 | 48 | module_name = cls.parse_module_name(line_parts[0]) 49 | 50 | # parse additional parts of mod line 51 | module_info = {} 52 | for fragment in line_parts[1:]: 53 | clean = fragment.strip(' \'"') 54 | # if part not start with a colon, it is git module 55 | if clean.startswith(':'): 56 | if clean.startswith(':git'): 57 | module_info['url'] = clean.rsplit('>', 1)[1].strip(' \'"') 58 | elif clean.startswith(':commit'): 59 | module_info['version'] = GitCommit( 60 | clean.rsplit('>', 1)[1].strip(' \'"') 61 | ) 62 | elif clean.startswith(':ref'): 63 | module_info['version'] = GitRef( 64 | clean.rsplit('>', 1)[1].strip(' \'"') 65 | ) 66 | elif clean.startswith(':tag'): 67 | module_info['version'] = GitTag( 68 | clean.rsplit('>', 1)[1].strip(' \'"') 69 | ) 70 | elif clean.startswith(':branch'): 71 | module_info['version'] = GitBranch( 72 | clean.rsplit('>', 1)[1].strip(' \'"') 73 | ) 74 | # forge module 75 | else: 76 | module_info['version'] = Forge(clean) 77 | LOG.debug("%s parsed into %s", ','.join(line_parts[1:]), module_info) 78 | 79 | # forge module 80 | if module_name.author is not None and 'url' not in module_info: 81 | return ForgeModule( 82 | author=module_name.author, 83 | name=module_name.module, 84 | version=module_info.get('version'), 85 | ) 86 | # git module 87 | else: 88 | return GitModule( 89 | name=module_name.module, 90 | url=module_info['url'], 91 | version=module_info.get('version'), 92 | ) 93 | 94 | @property 95 | def name(self): 96 | """Name of this module""" 97 | return self._name 98 | 99 | @property 100 | def version(self): 101 | """Return this modules version""" 102 | raise NotImplementedError 103 | 104 | @version.setter 105 | def version(self, value): 106 | """Set this modules version""" 107 | raise NotImplementedError 108 | 109 | @property 110 | def update_commit_message(self): 111 | """returns commit message for updating module""" 112 | try: 113 | commit_message = 'Update {} module ({})'.format( 114 | self.name, 115 | self.version.commit_message, 116 | ) 117 | except AttributeError: 118 | commit_message = 'Update {} module'.format(self.name) 119 | return commit_message 120 | 121 | def __hash__(self): 122 | return hash(self.__repr__()) 123 | 124 | def __repr__(self): 125 | return "%s" % self.name 126 | 127 | def __eq__(self, other): 128 | return str(self) >= str(other) >= str(self) 129 | 130 | def __lt__(self, other): 131 | return str(other) > str(self) 132 | 133 | 134 | class GitModule(PuppetModule): 135 | """Puppet mdoule hosted on git""" 136 | 137 | def __init__(self, name, url, version=None): 138 | """Initialize git module 139 | :argument name Name of module 140 | :argument url Repository URL of module 141 | """ 142 | super().__init__(name) 143 | self._url = url 144 | self.version = version 145 | 146 | @property 147 | def version(self): 148 | """Return this modules version""" 149 | return self._version 150 | 151 | @version.setter 152 | def version(self, value): 153 | """Set version of this puppet module""" 154 | if isinstance(value, (GitBranch, GitCommit, GitRef, GitTag)): 155 | self._version = value 156 | else: 157 | if value is None: 158 | self._version = None 159 | else: 160 | raise TypeError('Unsupported type %s for value' % type(value)) 161 | 162 | @property 163 | def url(self): 164 | """URL to git repository for this puppet module""" 165 | return self._url 166 | 167 | def __repr__(self): 168 | """Return unique string representation""" 169 | representation = "%s:git:%s" % (self.name, self.url) 170 | if self.version: 171 | representation += ":%s" % self.version 172 | return representation 173 | 174 | @property 175 | def puppetfile(self): 176 | """Return puppetfile representation of module""" 177 | lines = [ 178 | "mod '%s'," % self.name, 179 | ] 180 | git_line = " :git => '%s'" % self.url 181 | if self.version: 182 | git_line += "," 183 | lines.append(git_line) 184 | 185 | if isinstance(self.version, BaseVersion): 186 | lines.append(self.version.puppetfile) 187 | return lines 188 | 189 | def print_version_information(self, version_check=True, version_cache=None): 190 | """Print out version information""" 191 | if version_check: 192 | latest_version = self.get_latest_version(version_cache) 193 | else: 194 | latest_version = Unknown() 195 | 196 | cprint.magenta_bold('Version:', lpad=2) 197 | cprint.white('Git:', lpad=4, rpad=8, end='') 198 | cprint.white(self.url) 199 | if isinstance(self.version, GitBranch): 200 | cprint.blue_bold('Branch: ', lpad=16, end='') 201 | cprint.yellow_bold(self.version.version, end='') 202 | elif isinstance(self.version, GitCommit): 203 | cprint.red('Commit: ', lpad=16, end='') 204 | cprint.red_bold(self.version.version[:7], end='') 205 | elif isinstance(self.version, GitRef): 206 | cprint.red('Ref: ', lpad=16, end='') 207 | cprint.red_bold(self.version.version, end='') 208 | elif isinstance(self.version, GitTag): 209 | cprint.blue_bold('Tag: ', lpad=16, end='') 210 | if latest_version.version == self.version.version: 211 | cprint.green_bold(self.version.version, end='') 212 | else: 213 | cprint.yellow_bold(self.version.version, end='') 214 | else: 215 | cprint.red_bold('UNSPECIFIED', lpad=16, end='') 216 | 217 | if version_check: 218 | cprint.white('[Latest: %s]' % ( 219 | latest_version.report 220 | ), lpad=1) 221 | else: 222 | cprint.white('') 223 | 224 | def get_latest_version(self, version_cache=None): 225 | """return a dict with version, date of newest tag in repository""" 226 | if version_cache is not None: 227 | local_info = version_cache.read(self.cachename) 228 | else: 229 | local_info = {} 230 | 231 | if not local_info: 232 | with Repository(self.url) as repository: 233 | try: 234 | latest_tag = repository.latest_tag 235 | except GitError: 236 | local_info = {} 237 | else: 238 | local_info = { 239 | 'version': latest_tag.name, 240 | 'date': latest_tag.date.strftime('%Y-%m-%d'), 241 | } 242 | if version_cache is not None: 243 | version_cache.write(self.cachename, local_info) 244 | 245 | try: 246 | version = GitTag( 247 | version=local_info['version'], 248 | date=datetime.strptime( 249 | local_info['date'], '%Y-%m-%d' 250 | ).date() 251 | ) 252 | except KeyError: 253 | version = Unknown() # pylint: disable=redefined-variable-type 254 | LOG.debug("latest version for %s is %s", self.name, version) 255 | return version 256 | 257 | @property 258 | def cachename(self): 259 | """returns cache lookup key""" 260 | return hashlib.sha256(self.url.encode('utf-8')).hexdigest() 261 | 262 | 263 | class ForgeModule(PuppetModule): 264 | """Puppet module hosted on forge""" 265 | 266 | def __init__(self, name, author, version=None): 267 | """Initialize forge module 268 | :argument name Name of module 269 | :argument author Author/Namespace of the module 270 | """ 271 | super().__init__(name) 272 | self._author = author 273 | self.version = version 274 | 275 | @property 276 | def version(self): 277 | """Return this modules version""" 278 | return self._version 279 | 280 | @version.setter 281 | def version(self, value): 282 | """Set version of this puppet module""" 283 | if isinstance(value, Forge): 284 | self._version = value 285 | else: 286 | if value is None: 287 | self._version = None 288 | else: 289 | raise TypeError('Unsupported type %s for value' % type(value)) 290 | 291 | @property 292 | def author(self): 293 | """Puppet module author / namespace""" 294 | return self._author 295 | 296 | @property 297 | def forgename(self): 298 | """Return module name as used on forge""" 299 | return "%s/%s" % (self._author, self._name) 300 | 301 | @property 302 | def cachename(self): 303 | """returns cache lookup key""" 304 | return hashlib.sha256(self.forgename.encode('utf-8')).hexdigest() 305 | 306 | def __repr__(self): 307 | """Return unique string representation""" 308 | representation = "%s:forge:%s" % (self.name, self.author) 309 | if self.version: 310 | representation += ":%s" % self.version 311 | return representation 312 | 313 | def print_version_information(self, version_check=True, version_cache=None): 314 | """Print out version information""" 315 | if version_check: 316 | latest_version = self.get_latest_version(version_cache) 317 | else: 318 | latest_version = Unknown() 319 | 320 | cprint.magenta_bold('Version:', lpad=2) 321 | cprint.white('Forge:', lpad=4, rpad=6, end='') 322 | cprint.white(self.forgename, suffix=':') 323 | if not self.version: 324 | cprint.red_bold('UNSPECIFIED', lpad=16, end='') 325 | else: 326 | if latest_version.version == self.version.version: 327 | cprint.green_bold(self.version.version, lpad=16, end='') 328 | else: 329 | cprint.yellow_bold(self.version.version, lpad=16, end='') 330 | 331 | if version_check: 332 | cprint.white(' [Latest: %s]' % latest_version.report) 333 | else: 334 | cprint.white('') 335 | 336 | def get_latest_version(self, version_cache=None): 337 | """returns dict with version and date of the newest version on forge""" 338 | if version_cache is not None: 339 | local_info = version_cache.read(self.cachename) 340 | else: 341 | local_info = {} 342 | 343 | if not local_info: 344 | try: 345 | local_info = ForgeApi( 346 | name=self.name, 347 | author=self.author 348 | ).current_version 349 | except ForgeError: 350 | return Unknown() 351 | 352 | if version_cache is not None: 353 | version_cache.write(self.cachename, local_info) 354 | 355 | try: 356 | return Forge( 357 | version=local_info['version'], 358 | date=datetime.strptime( 359 | local_info['date'], '%Y-%m-%d' 360 | ).date() 361 | ) 362 | except KeyError: 363 | return Unknown() 364 | 365 | @property 366 | def puppetfile(self): 367 | """Return puppetfile representation of module""" 368 | line = "mod '%s/%s'" % (self.author, self.name) 369 | if self.version: 370 | line += ", %s" % self.version.puppetfile 371 | return [line, ] 372 | 373 | 374 | class BaseVersion: 375 | """Base class for version objects""" 376 | 377 | def __init__(self, version, date=None): 378 | """Initialize Version 379 | :argument version Version(-string) for this module. 380 | """ 381 | self._date = date 382 | self._version = version 383 | 384 | def __hash__(self): 385 | return hash(self._version) 386 | 387 | def __repr__(self): 388 | return "%s(%s)" % (type(self).__name__, str(self._version)) 389 | 390 | @property 391 | def version(self): 392 | """Return Version(-string)""" 393 | return self._version 394 | 395 | @property 396 | def date(self): 397 | """Return Date of Version""" 398 | return self._date 399 | 400 | @property 401 | def report(self): 402 | """Return version in suitable format for crmngr report""" 403 | if self._date is None: 404 | return "%s" % self._version 405 | else: 406 | return "%s (%s)" % (self._version, self._date) 407 | 408 | @property 409 | def commit_message(self): 410 | """Return version in suitable format for commit message""" 411 | return "%s" % self.version 412 | 413 | 414 | class Unknown(BaseVersion): 415 | """Object to represent and unknown Version""" 416 | def __init__(self, version=None, date=None): 417 | super().__init__(version, date) 418 | 419 | def __repr__(self): 420 | return "%s()" % type(self).__name__ 421 | 422 | @property 423 | def report(self): 424 | """Return version in suitable format for crmngr report""" 425 | return 'unknown' 426 | 427 | 428 | class Forge(BaseVersion): 429 | """Puppet Forge Version""" 430 | 431 | @property 432 | def puppetfile(self): 433 | """Return version in suitable format for puppetfile""" 434 | return "'%s'" % self.version 435 | 436 | 437 | class GitBranch(BaseVersion): 438 | """Git Branch""" 439 | 440 | @property 441 | def puppetfile(self): 442 | """Return version in suitable format for puppetfile""" 443 | return " :branch => '%s'" % self.version 444 | 445 | @property 446 | def commit_message(self): 447 | """Return version in suitable format for commit message""" 448 | return "branch [%s]" % self.version 449 | 450 | 451 | class GitCommit(BaseVersion): 452 | """Git Commit""" 453 | 454 | @property 455 | def puppetfile(self): 456 | """Return version in suitable format for puppetfile""" 457 | return " :commit => '%s'" % self.version 458 | 459 | @property 460 | def commit_message(self): 461 | """Return version in suitable format for commit message""" 462 | return "commit [%s]" % self.version 463 | 464 | 465 | class GitRef(BaseVersion): 466 | """Git Ref""" 467 | 468 | @property 469 | def puppetfile(self): 470 | """Return version in suitable format for puppetfile""" 471 | return " :ref => '%s'" % self.version 472 | 473 | 474 | class GitTag(BaseVersion): 475 | """ Git Tag""" 476 | 477 | @property 478 | def puppetfile(self): 479 | """Return version in suitable format for puppetfile""" 480 | return " :tag => '%s'" % self.version 481 | 482 | @property 483 | def commit_message(self): 484 | """Return version in suitable format for commit message""" 485 | return "tag [%s]" % self.version 486 | -------------------------------------------------------------------------------- /crmngr/utils.py: -------------------------------------------------------------------------------- 1 | """ crmngr utility module """ 2 | 3 | # stdlib 4 | from fnmatch import fnmatchcase 5 | import sys 6 | 7 | 8 | def truncate(string, max_len=1000): 9 | """returns a truncated to max_len version of a string (or str(string))""" 10 | string = str(string) 11 | if len(string) > max_len - 12: 12 | return string[:max_len] + '...TRUNCATED' 13 | return string 14 | 15 | 16 | def fnlistmatch(value, patterns): 17 | """match a value against a list of fnmatch patterns. 18 | 19 | returns True if any pattern matches. 20 | """ 21 | for pattern in patterns: 22 | if fnmatchcase(value, pattern): 23 | return True 24 | return False 25 | 26 | 27 | def query_yes_no(question, default="yes"): 28 | """Asks a yes/no question via and returns the answer as bool.""" 29 | valid = {"yes": True, "y": True, "ye": True, "j": True, 30 | "no": False, "n": False} 31 | if default is None: 32 | prompt = " [y/n] " 33 | elif default == "yes": 34 | prompt = " [Y/n] " 35 | elif default == "no": 36 | prompt = " [y/N] " 37 | else: 38 | raise ValueError("invalid default answer: '%s'" % default) 39 | 40 | while True: 41 | sys.stdout.write(question + prompt) 42 | choice = input().lower() 43 | if default is not None and choice == '': 44 | return valid[default] 45 | elif choice in valid: 46 | return valid[choice] 47 | else: 48 | print("Please respond with 'yes' or 'no' (or 'y' or 'n').") 49 | -------------------------------------------------------------------------------- /crmngr/version.py: -------------------------------------------------------------------------------- 1 | """ crmngr version """ 2 | __version__ = '2.0.3' 3 | -------------------------------------------------------------------------------- /requirements.in: -------------------------------------------------------------------------------- 1 | natsort 2 | pytest 3 | pytest-runner 4 | requests 5 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile 3 | # To update, run: 4 | # 5 | # pip-compile -U 6 | # 7 | atomicwrites==1.3.0 # via pytest 8 | attrs==19.1.0 # via pytest 9 | certifi==2019.3.9 # via requests 10 | chardet==3.0.4 # via requests 11 | idna==2.8 # via requests 12 | more-itertools==7.0.0 # via pytest 13 | natsort==6.0.0 14 | pluggy==0.9.0 # via pytest 15 | py==1.8.0 # via pytest 16 | pytest-runner==4.4 17 | pytest==4.4.1 18 | requests==2.21.0 19 | six==1.12.0 # via pytest 20 | urllib3==1.24.3 # via requests 21 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [aliases] 2 | test=pytest 3 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | """ crmngr setup module. """ 2 | 3 | from pathlib import Path 4 | import re 5 | from setuptools import setup, find_packages 6 | 7 | 8 | def find_version(source_file): 9 | """read __version__ from source file""" 10 | with open(source_file) as version_file: 11 | version_match = re.search(r"^__version__\s*=\s* ['\"]([^'\"]*)['\"]", 12 | version_file.read(), re.M) 13 | if version_match: 14 | return version_match.group(1) 15 | raise RuntimeError('Unable to find package version') 16 | 17 | 18 | setup( 19 | name='crmngr', 20 | author='Andre Keller', 21 | author_email='andre.keller@vshn.ch', 22 | classifiers=[ 23 | 'Development Status :: 4 - Beta', 24 | 'Environment :: Console', 25 | 'Intended Audience :: System Administrators', 26 | 'License :: OSI Approved :: BSD License', 27 | 'Operating System :: Unix', 28 | 'Programming Language :: Python :: 3 :: Only', 29 | 'Topic :: System :: Systems Administration', 30 | ], 31 | description='manage a r10k-style control repository', 32 | entry_points={ 33 | 'console_scripts': [ 34 | 'crmngr = crmngr:main' 35 | ] 36 | }, 37 | install_requires=[ 38 | 'natsort>=4.0.0', 39 | 'requests>=2.1.0', 40 | ], 41 | setup_requires=[ 42 | 'pytest-runner', 43 | ], 44 | tests_require=[ 45 | 'pytest', 46 | ], 47 | python_requires='>=3.4', 48 | # BSD 3-Clause License: 49 | # - http://opensource.org/licenses/BSD-3-Clause 50 | license='BSD', 51 | packages=find_packages(), 52 | url='https://github.com/vshn/crmngr', 53 | version=find_version(str(Path('./crmngr/version.py'))), 54 | ) 55 | -------------------------------------------------------------------------------- /tests/test_crmngr.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | from pathlib import Path 3 | from tempfile import TemporaryDirectory 4 | 5 | import pytest 6 | 7 | from crmngr import ControlRepository 8 | from crmngr.puppetfile import GitTag 9 | 10 | 11 | @pytest.fixture() 12 | def control_repo(): 13 | git_dir = TemporaryDirectory(prefix='crmngr_test_') 14 | bare_dir = Path(git_dir.name, 'bare') 15 | work_dir = Path(git_dir.name, 'work') 16 | subprocess.run(['git', 'init', '--bare', str(bare_dir)]) 17 | subprocess.run(['git', 'clone', bare_dir, str(work_dir)]) 18 | subprocess.run(['git', 'checkout', '-b', 'production'], cwd=str(work_dir)) 19 | with open(str(Path(work_dir, 'Puppetfile')), 'w') as puppetfile: 20 | puppetfile.write("\n".join([ 21 | "forge 'http://forge.puppetlabs.com'", 22 | "", 23 | "mod 'firewall',", 24 | " :git => 'https://github.com/puppetlabs/puppetlabs-firewall.git',", 25 | " :tag => '1.11.0'", 26 | "mod 'puppetlabs/stdlib', '4.20.0'" 27 | "" 28 | ])) 29 | subprocess.run(['git', 'add', str(Path(work_dir, 'Puppetfile'))], cwd=str(work_dir)) 30 | subprocess.run(['git', 'commit', '-m', 'Initial commit', 'Puppetfile'], cwd=str(work_dir)) 31 | subprocess.run(['git', 'push', 'origin', 'production'], cwd=str(work_dir)) 32 | subprocess.run(['git', 'checkout', '--orphan', 'staging'], cwd=str(work_dir)) 33 | with open(str(Path(work_dir, 'Puppetfile')), 'w') as puppetfile: 34 | puppetfile.write("\n".join([ 35 | "forge 'http://forge.puppetlabs.com'", 36 | "", 37 | "mod 'puppetlabs/firewall', '1.10.0", 38 | "mod 'puppetlabs/stdlib', '4.23.0'" 39 | "" 40 | ])) 41 | subprocess.run(['git', 'add', str(Path(work_dir, 'Puppetfile'))], cwd=str(work_dir)) 42 | subprocess.run(['git', 'commit', '-m', 'Initial commit', 'Puppetfile'], cwd=str(work_dir)) 43 | subprocess.run(['git', 'push', 'origin', 'staging'], cwd=str(work_dir)) 44 | yield ControlRepository(clone_url="file://{}".format(bare_dir)) 45 | git_dir.cleanup() 46 | 47 | 48 | class TestCrmngr: 49 | 50 | def test_environments(self, control_repo): 51 | assert sorted(control_repo.branches) == ['production', 'staging'] 52 | 53 | def test_stdlib_staging(self, control_repo): 54 | staging = control_repo.get_environment('staging') 55 | assert str(staging['firewall']) == 'firewall:forge:puppetlabs:Forge(1.10.0)' 56 | 57 | def test_stdlib_production(self, control_repo): 58 | production = control_repo.get_environment('production') 59 | assert str(production['firewall']) == 'firewall:git:https://github.com/puppetlabs/puppetlabs-firewall.git:GitTag(1.11.0)' 60 | --------------------------------------------------------------------------------