├── .bundle └── config ├── .gitignore ├── .rubocop.yml ├── .travis.yml ├── Gemfile ├── Gemfile.lock ├── LICENSE.txt ├── README.md ├── Rakefile ├── bin └── pupistry ├── exe └── pupistry ├── lib ├── pupistry.rb └── pupistry │ ├── agent.rb │ ├── artifact.rb │ ├── bootstrap.rb │ ├── config.rb │ ├── gpg.rb │ ├── hieracrypt.rb │ ├── packer.rb │ ├── storage_aws.rb │ └── version.rb ├── pupistry.gemspec ├── resources ├── aws │ ├── README_AWS.md │ └── cfn_pupistry_bucket_and_iam.template ├── bootstrap │ ├── BOOTSTRAP_NOTES.md │ ├── amazon-any.erb │ ├── centos-7.erb │ ├── debian-8.erb │ ├── debian-9.erb │ ├── fedora-any.erb │ ├── freebsd-10.erb │ ├── openbsd-6.0.erb │ ├── ubuntu-14.04.erb │ ├── ubuntu-16.04-puppet4.erb │ └── ubuntu-16.04.erb └── packer │ ├── PACKER_NOTES.md │ ├── aws_amazon-any.json.erb │ ├── aws_freebsd-10.json.erb │ └── aws_ubuntu-14.04.json.erb ├── settings.example.yaml └── test ├── data ├── empty.yaml └── nonyaml.txt ├── minitest_helper.rb ├── test_config.rb └── test_pupistry.rb /.bundle/config: -------------------------------------------------------------------------------- 1 | --- 2 | BUNDLE_PATH: "vendor" 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | vendor/ 2 | coverage/ 3 | bin/ 4 | *.gem 5 | *.swp 6 | settings.yaml 7 | -------------------------------------------------------------------------------- /.rubocop.yml: -------------------------------------------------------------------------------- 1 | AllCops: 2 | DisplayCopNames: true 3 | 4 | Style/Documentation: 5 | Enabled: false 6 | 7 | 8 | # Generally all lines are under 80 chars, but sometimes it just makes sense to 9 | # go beyond it for clariy or for output/log entries and it gets annoying and 10 | # ugly having exclusions all over the place. 11 | 12 | Metrics/LineLength: 13 | Max: 200 14 | 15 | 16 | # We should work on cutting these down over time, however if we just run with 17 | # the defaults right now, it creates far too much noise to ever get anything 18 | # useful out of Rubocop. 19 | # 20 | # TODO: Lower levels and address some of the length/complexity issues. 21 | Metrics/AbcSize: 22 | Max: 100 23 | 24 | Metrics/MethodLength: 25 | Max: 200 26 | 27 | Metrics/ClassLength: 28 | Max: 1000 29 | 30 | Metrics/CyclomaticComplexity: 31 | Max: 20 32 | 33 | Metrics/PerceivedComplexity: 34 | Max: 20 35 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | # Note: You can discover list of supported Ruby versions at: 2 | # http://rubies.travis-ci.org/ 3 | 4 | language: ruby 5 | rvm: 6 | - "2.1.0" 7 | - "2.2.2" 8 | - "2.3.1" 9 | - "2.4.4" 10 | - "2.5.1" 11 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | # Specify your gem's dependencies in pupistry.gemspec 4 | gemspec 5 | 6 | # See comments in gemspec around Puppet - we need it to run via the bundle 7 | # so we include it here, but we don't want it as a dependency of the gem 8 | # itself. 9 | gem "puppet" 10 | 11 | -------------------------------------------------------------------------------- /Gemfile.lock: -------------------------------------------------------------------------------- 1 | PATH 2 | remote: . 3 | specs: 4 | pupistry (2.0.0) 5 | aws-sdk-v1 6 | erubis 7 | r10k 8 | rufus-scheduler (~> 3) 9 | safe_yaml 10 | thor 11 | whichr 12 | 13 | GEM 14 | remote: https://rubygems.org/ 15 | specs: 16 | ast (2.4.0) 17 | aws-sdk-v1 (1.67.0) 18 | json (~> 1.4) 19 | nokogiri (~> 1) 20 | colored (1.2) 21 | cri (2.6.1) 22 | colored (~> 1.2) 23 | docile (1.3.1) 24 | erubis (2.7.0) 25 | et-orbi (1.1.2) 26 | tzinfo 27 | facter (2.5.1) 28 | faraday (0.13.1) 29 | multipart-post (>= 1.2, < 3) 30 | faraday_middleware (0.12.2) 31 | faraday (>= 0.7.4, < 1.0) 32 | fast_gettext (1.1.2) 33 | fugit (1.1.1) 34 | et-orbi (~> 1.1, >= 1.1.1) 35 | raabro (~> 1.1) 36 | gettext (3.2.9) 37 | locale (>= 2.0.5) 38 | text (>= 1.3.0) 39 | gettext-setup (0.30) 40 | fast_gettext (~> 1.1.0) 41 | gettext (>= 3.0.2) 42 | locale 43 | hiera (3.4.3) 44 | json (1.8.6) 45 | locale (2.1.2) 46 | log4r (1.1.10) 47 | mini_portile2 (2.3.0) 48 | minitar (0.6.1) 49 | minitest (5.11.3) 50 | multi_json (1.13.1) 51 | multipart-post (2.0.0) 52 | nokogiri (1.8.2) 53 | mini_portile2 (~> 2.3.0) 54 | os (0.9.6) 55 | parallel (1.12.1) 56 | parser (2.5.1.0) 57 | ast (~> 2.4.0) 58 | powerpack (0.1.1) 59 | puppet (5.5.1) 60 | facter (> 2.0.1, < 4) 61 | fast_gettext (~> 1.1.2) 62 | hiera (>= 3.2.1, < 4) 63 | locale (~> 2.1) 64 | multi_json (~> 1.10) 65 | puppet_forge (2.2.9) 66 | faraday (>= 0.9.0, < 0.14.0) 67 | faraday_middleware (>= 0.9.0, < 0.13.0) 68 | gettext-setup (~> 0.11) 69 | minitar 70 | semantic_puppet (~> 1.0) 71 | r10k (2.6.2) 72 | colored (= 1.2) 73 | cri (~> 2.6.1) 74 | gettext-setup (~> 0.5) 75 | log4r (= 1.1.10) 76 | multi_json (~> 1.10) 77 | puppet_forge (~> 2.2.8) 78 | raabro (1.1.5) 79 | rainbow (3.0.0) 80 | rake (10.5.0) 81 | rubocop (0.56.0) 82 | parallel (~> 1.10) 83 | parser (>= 2.5) 84 | powerpack (~> 0.1) 85 | rainbow (>= 2.2.2, < 4.0) 86 | ruby-progressbar (~> 1.7) 87 | unicode-display_width (~> 1.0, >= 1.0.1) 88 | ruby-progressbar (1.9.0) 89 | rufus-scheduler (3.5.0) 90 | fugit (~> 1.1, >= 1.1.1) 91 | safe_yaml (1.0.4) 92 | sane (0.25.8) 93 | os (~> 0) 94 | semantic_puppet (1.0.2) 95 | simplecov (0.16.1) 96 | docile (~> 1.1) 97 | json (>= 1.8, < 3) 98 | simplecov-html (~> 0.10.0) 99 | simplecov-html (0.10.2) 100 | text (1.3.1) 101 | thor (0.20.0) 102 | thread_safe (0.3.6) 103 | tzinfo (1.2.5) 104 | thread_safe (~> 0.1) 105 | unicode-display_width (1.3.3) 106 | whichr (0.3.6) 107 | sane 108 | 109 | PLATFORMS 110 | ruby 111 | 112 | DEPENDENCIES 113 | bundler (~> 1.9) 114 | minitest (~> 5.6) 115 | pupistry! 116 | puppet 117 | rake (~> 10.0) 118 | rubocop 119 | simplecov (~> 0.10) 120 | 121 | BUNDLED WITH 122 | 1.16.2 123 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # pupistry 2 | 3 | [![Build Status](https://travis-ci.org/jethrocarr/pupistry.svg)](https://travis-ci.org/jethrocarr/pupistry) 4 | 5 | Pupistry (puppet + artistry) is a solution for implementing reliable and secure 6 | masterless puppet deployments by taking Puppet modules assembled by `r10k` and 7 | generating compressed and signed archives for distribution to the masterless 8 | servers. 9 | 10 | Pupistry builds on the functionality offered by the `r10k` workflow but rather 11 | than requiring the implementing of site-specific custom bootstrap and custom 12 | workflow mechanisms, Pupistry executes `r10k`, assembles the combined modules 13 | and then generates a compressed artifact file. It then optionally signs the 14 | artifact with GPG and finally uploads it into an Amazon S3 bucket along with a 15 | manifest file. 16 | 17 | The masterless Puppet machines then just run a Pupistry job which checks for a 18 | new version of the manifest file. If there is, it downloads the new artifact 19 | and does an optional GPG validation before applying it and running Puppet. To 20 | make life even easier, Pupistry will even spit out bootstrap files for your 21 | platform which sets up each server from scratch to pull and run the artifacts. 22 | 23 | Essentially Pupistry is intended to be a robust solution for masterless Puppet 24 | deployments and makes it trivial for beginners to get started with Puppet. 25 | 26 | 27 | # Why Pupistry? 28 | 29 | Masterless Puppet is a great solution for anyone wanting to avoid scaling issues 30 | and risk of centralised failure due to a central Puppet master, but it does bring 31 | a number of issues with it. 32 | 33 | 1. Having to setup deployer keys to every Git repo used is a maintainance headache. Pupistry means only your workstation needs access, which presumably will have access to most/all repos already. 34 | 2. Your system build success is dependent on all the Git repos you've used, including any third parties that could vanish. A single missing or broken repo could prevent autoscaling or new machine builds at a critical time. Pupistry's use of artifact files prevents surprises - if you can hit S3, you're sorted. 35 | 3. It is easy for malicious code in the third party repos to slip in without noticing. Even if the author themselves is honest, not all repos have proper security like two-factor. Pupistry prevents surprise updates of modules and also has an easy diff feature to see what changed since you last generated an artifact. 36 | 4. Puppet masterless tends to be implemented in many different ways using everyone's own hacky scripts. Pupistry's goal is to create a singular standard approach to masterless, in the same way that `r10k` created a standard approach to Git-based Puppet workflows. And this makes things easy - install Pupistry, add the companion Puppet module and run the bootstrap script. Easy! 37 | 5. No dodgy cronjobs running `r10k` and Puppet in weird ways. A simple clean agent with daemon or run-once functionality. 38 | 6. Performance - Go from 30+ seconds `r10k` update checks to 2 second Pupistry update checks. And when there is a change, it's a fast efficent compressed file download from S3 rather than pulling numerious Git repos. 39 | 40 | 41 | 42 | # Usage 43 | 44 | ## Building new artifacts 45 | 46 | Build a new artifact: 47 | 48 | $ pupistry build 49 | I, [2015-04-08T22:19:30.419392 #52534] INFO -- : Using r10k utility to fetch the latest Puppet code 50 | [R10K::Action::Deploy::Environment - INFO] Deploying environment /Users/jethro/.pupistry/cache/puppetcode/master 51 | [R10K::Action::Deploy::Environment - INFO] Deploying module /Users/jethro/.pupistry/cache/puppetcode/master/modules/stdlib 52 | [R10K::Action::Deploy::Environment - INFO] Deploying module /Users/jethro/.pupistry/cache/puppetcode/master/modules/ruby 53 | [R10K::Action::Deploy::Environment - INFO] Deploying module /Users/jethro/.pupistry/cache/puppetcode/master/modules/gcc 54 | [R10K::Action::Deploy::Environment - INFO] Deploying module /Users/jethro/.pupistry/cache/puppetcode/master/modules/inifile 55 | [R10K::Action::Deploy::Environment - INFO] Deploying module /Users/jethro/.pupistry/cache/puppetcode/master/modules/vcsrepo 56 | [R10K::Action::Deploy::Environment - INFO] Deploying module /Users/jethro/.pupistry/cache/puppetcode/master/modules/git 57 | [R10K::Action::Deploy::Environment - INFO] Deploying module /Users/jethro/.pupistry/cache/puppetcode/master/modules/ntp 58 | [R10K::Action::Deploy::Environment - INFO] Deploying module /Users/jethro/.pupistry/cache/puppetcode/master/modules/firewall 59 | [R10K::Action::Deploy::Environment - INFO] Deploying module /Users/jethro/.pupistry/cache/puppetcode/master/modules/soe 60 | I, [2015-04-08T22:21:21.705315 #52534] INFO -- : r10k run completed 61 | I, [2015-04-08T22:21:21.706023 #52534] INFO -- : Creating artifact... 62 | I, [2015-04-08T22:21:21.999753 #52534] INFO -- : Compressing artifact... 63 | I, [2015-04-08T22:21:22.103131 #52534] INFO -- : Building manifest information for artifact... 64 | I, [2015-04-08T22:21:22.107012 #52534] INFO -- : New artifact version 3f29c324aab076cd81667f9031a675e7 ready for pushing 65 | -- 66 | Tip: Run pupistry diff to see what changed since the last artifact version 67 | 68 | 69 | Note that artifact builds are done from the upstream Git repos, so if you 70 | have made changes, remember to `git push` first before generating. The tool will 71 | remind you if it detects nothing has changed since the last run. 72 | 73 | Once your artifact is built, you can double check what has changed in the 74 | Puppet modules since the last run with: 75 | 76 | $ pupistry diff 77 | diff -Nuar unpacked.3f29c324aab076cd81667f9031a675e7/puppetcode/master/README.md unpacked.4a522dd22c0453e1e3ec3d17dfed151b/puppetcode/master/README.md 78 | --- unpacked.3f29c324aab076cd81667f9031a675e7/puppetcode/master/README.md 2015-04-08 22:19:42.000000000 +1200 79 | +++ unpacked.4a522dd22c0453e1e3ec3d17dfed151b/puppetcode/master/README.md 2015-04-08 23:01:14.000000000 +1200 80 | @@ -1 +1,4 @@ 81 | Personal Puppet Repo 82 | + 83 | +Example of a changed file in a module somewhere, nice and visible for all to see. 84 | + 85 | -- 86 | Tip: Run pupistry push to GPG sign & upload if happy to go live 87 | 88 | 89 | Finally when you're happy, push it to S3 to be delivered to all your servers. 90 | If you have gpg signing enabled, it will ask you to sign here... or tell you 91 | off if you have it disabled. :-) 92 | 93 | $ pupistry push 94 | I, [2015-04-08T22:52:01.020865 #53037] INFO -- : Uploading artifact version latest (3f29c324aab076cd81667f9031a675e7) 95 | W, [2015-04-08T22:52:01.888356 #53037] WARN -- : You have GPG signing *disabled*, whilst not critical it does weaken your security. 96 | W, [2015-04-08T22:52:01.888418 #53037] WARN -- : Skipping signing step... 97 | I, [2015-04-08T22:52:03.043886 #53037] INFO -- : Upload of artifact version 3f29c324aab076cd81667f9031a675e7 completed and is now latest 98 | 99 | 100 | 101 | ## Bootstrapping nodes 102 | 103 | New machines need to be bootstrapped in order to install Pupistry, configure it 104 | and be able to download configuration. Generally this is a step done differently 105 | site-by-site (and you can still do it that way if you want), but if you want a 106 | nice easy life, Pupistry can generate you a bootstrap script for your platform. 107 | 108 | $ pupistry bootstrap 109 | - centos-7 110 | - ubuntu-14.04 111 | 112 | $ pupistry boostrap --template centos-7 113 | # Compatible with RHEL 7, CentOS 7 and maybe other variations. 114 | 115 | rpm -ivh https://yum.puppetlabs.com/puppetlabs-release-el-7.noarch.rpm 116 | 117 | yum update --assumeyes 118 | yum install --assumeyes puppet ruby-devel rubygems gcc zlib-devel libxml2-devel patch 119 | 120 | gem install pupistry 121 | mkdir /etc/pupistry 122 | cat > /etc/pupistry/settings.yaml << "EOF" 123 | general: 124 | app_cache: ~/.pupistry/cache 125 | s3_bucket: example 126 | s3_prefix: 127 | gpg_disable: true 128 | gpg_signing_key: XYZXYZ 129 | agent: 130 | puppetcode: /etc/puppet/environments/ 131 | access_key_id: 132 | secret_access_key: 133 | region: ap-southeast-2 134 | proxy_uri: 135 | EOF 136 | pupistry apply --verbose 137 | 138 | You generally can run this on a new non-Puppetised machine, or paste into the 139 | user data field of most cloud providers like AWS or Digital Ocean. If using CFN 140 | with AWS, you can make it part of the stack itself. 141 | 142 | These bootstraps aren't mandatory, if you prefer a different approach you can 143 | use these as an example and write your own - generally the essential bit is to 144 | get puppet installed, get pupistry (and deps to build its gems) installed and 145 | write the config before finally executing your first Pupistry/Puppet run. 146 | 147 | If using AWS and IAM Roles feature, it is acceptable for access_key_id and 148 | secret_access_key to be blank, if not you will need to have these set to an 149 | account with read-only access to the configured S3 bucket! 150 | 151 | 152 | ## Running Puppet on target nodes 153 | 154 | Pupistry replaces the need to call Puppet directly. Instead, call Pupistry and 155 | it will handle getting the artifact and then executing Puppet for you. It 156 | respects some parameters like --environment and --noop for easy testing of new 157 | manifests and modules. 158 | 159 | At its simplest, to apply the current Puppet manifests: 160 | 161 | $ pupistry apply 162 | I, [2015-04-10T00:44:40.623101 #6726] INFO -- : Pulling latest artifact.... 163 | I, [2015-04-10T00:44:42.700540 #6726] INFO -- : Executing Puppet... 164 | Notice: Compiled catalog for testhost1 in environment master in 2.21 seconds 165 | Notice: Finished catalog run in 3.07 seconds 166 | 167 | 168 | Check what is going to be applied (Puppet in --noop mode) 169 | 170 | pupistry apply --noop 171 | 172 | Specify an alternative environment: 173 | 174 | pupistry apply --environment staging 175 | 176 | Run pupistry as a system daemon. When you use the companion Puppet module, a 177 | system init file gets installed that sets this daemon up for you automatically. 178 | 179 | pupistry apply --daemon 180 | 181 | Note that the daemon runs & logs to the foreground if you run it like the above, 182 | the init script handles the syslog & backgrounding for you (why code what the 183 | init system can do for us?). 184 | 185 | Alternatively, if you don't wish to use Pupistry to run the nodes, you don't 186 | have to. You can use Pupistry to build the artifacts and then pull them down 187 | and unpack via any means you find appropiate. It's just standard S3 + tar with 188 | some YAML and optional GPG signing. 189 | 190 | 191 | # Installation 192 | 193 | ## 1. Application 194 | 195 | First install Pupistry onto your workstation. You can make pupistry generate 196 | you a config file if you've never used it before 197 | 198 | gem install pupistry 199 | pupistry setup 200 | 201 | Alternatively if you like living on the edge, download this repository and run: 202 | 203 | gembuild pupistry.gemspec 204 | gem install pupistry-VERSION.gem 205 | pupistry setup 206 | 207 | Pupistry will write an example config file into `~/.pupistry/settings.yaml` for 208 | you, you will need to edit it with your preferred editor. 209 | 210 | 211 | ## 2. S3 Bucket 212 | 213 | Pupistry uses S3 for storing and pulling the artifact files. You need to 214 | configure the following: 215 | 216 | * A *private* S3 bucket (you'll get this by default). 217 | * An IAM account with access to write that bucket (for your build workstation) 218 | * An IAM account with access to read that bucket (for your servers) 219 | 220 | If you're not already using IAM with your AWS account you want to be - your 221 | servers should only ever have read access to the bucket and only your build 222 | workstation should be permitted to write new artifacts. IE, don't share your 223 | AWS root account around the place. :-) 224 | 225 | Note that if you're running EC2 instances and using IAM roles, you can avoid 226 | needing to create explicit IAM credentials for the agents/servers, as long as 227 | you include read access to the Pupistry S3 bucket in the IAM roles for all 228 | servers that will be running it. 229 | 230 | 231 | If you're new to AWS, we've made your life easy - there's an AWS CloudFormation 232 | template included with Pupistry that will build an S3 bucket and two IAM user 233 | accounts for you with sensible default policies. 234 | 235 | Just make sure you have a working `aws` command - that's the Python CLI issued 236 | by AWS themselves setup instructions can be found at: 237 | http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-set-up.html 238 | 239 | Provided that you've setup `aws` correctly and have full permissions to your 240 | account, you can now build your S3 bucket and IAM users with: 241 | 242 | wget https://raw.githubusercontent.com/jethrocarr/pupistry/master/resources/aws/cfn_pupistry_bucket_and_iam.template 243 | 244 | aws cloudformation create-stack \ 245 | --capabilities CAPABILITY_IAM \ 246 | --template-body file://cfn_pupistry_bucket_and_iam.template \ 247 | --stack-name pupistry-resources-changeme 248 | 249 | It is *very important* that you change the stack name to something globally 250 | unique, or the stack will fail to build. 251 | 252 | It may take 30 seconds or so to build, you can check for completion (or for an 253 | error) with: 254 | 255 | aws cloudformation describe-stacks --query "Stacks[*].StackStatus" --stack-name pupistry-resources-changeme 256 | 257 | Once status is CREATE_COMPLETE, you can get all the outputs from the stack with: 258 | 259 | aws cloudformation describe-stacks --query "Stacks[*].Outputs[*]" --stack-name pupistry-resources-changeme 260 | 261 | You now need to edit `~/.pupistry/settings.yaml` and enter in the equivalent 262 | OutputValue for the following labels: 263 | 264 | general: 265 | s3_bucket: S3Bucket 266 | ... 267 | agent: 268 | access_key_id: AgentAccessKeyId 269 | secret_access_key: AgentSecretKeyID 270 | region: S3Region 271 | ... 272 | build: 273 | access_key_id: BuildAccessKeyId 274 | secret_access_key: BuildSecretKeyID 275 | region: S3Region 276 | ... 277 | 278 | 279 | 280 | ## 3. Puppet Manifests & Configuration 281 | 282 | ### Puppet Code Structure 283 | 284 | The following is the expected minimum structure of the Puppetcode repository to 285 | enable it to work with Pupistry: 286 | 287 | /Puppetfile 288 | /hiera.yaml 289 | /manifests/site.pp 290 | 291 | `Puppetfile` is standard `r10k` and `site.pp` is standard Puppet. The Hiera config 292 | is generally normal, but you do need to define a datadir to tell Puppet to look 293 | where the puppet code gets unpacked to. Generally the following sample Hiera 294 | will do the trick: 295 | 296 | --- 297 | :backends: yaml 298 | :yaml: 299 | :datadir: "%{::settings::confdir}/%{::environment}/hieradata" 300 | :hierarchy: 301 | - "environments/%{::environment}" 302 | - "nodes/%{::hostname}" 303 | - common 304 | 305 | Pupistry will override the default `%{::settings::confdir}` value with wherever 306 | the Pupistry agent has been configured to write to (by default this will be the 307 | Puppet-4 style `/etc/puppetlabs/code/environments` path) so this Hiera config 308 | should work fine for any Pupistry deployed setups. 309 | 310 | If you're using a hybrid of Pupistry and masterful Puppet, you may need to adjust 311 | the `datadir` parameter in Hiera to a fixed path and the `puppetcode` parameter 312 | in Pupistry to be the exact same value, since `%{::settings::confdir}` will 313 | differ between Pupistry and masterful Puppet. 314 | 315 | Pupistry will default to applying the `master` branch if one is not listed, if 316 | you are doing branch-based environments, you can specifiy when bootstrapping 317 | and override on a per-execution basis with `--environment`. 318 | 319 | You'll notice pretty quickly if something is broken when doing `pupistry apply` 320 | 321 | Confused? No worried, check out the sample repo that shows a very simple setup. 322 | You can copy this and start your own Puppet adventure, just add in your modules 323 | to `Puppetfile` and add them to the relevant machines in `manifests/site.pp`. 324 | 325 | https://github.com/jethrocarr/pupistry-samplepuppet 326 | 327 | TODO: Longer term intend to add support for various popular structures, but 328 | for now it is what it is. It's not hard, check out bin/puppistry and send 329 | pull requests. 330 | 331 | 332 | ### Helper Module 333 | 334 | Whilst you can use Pupistry to roll out any particular design of Puppet 335 | manifests, you will save yourself a lot of pain by also including the Pupistry 336 | companion Puppet module in your manifests. 337 | 338 | The companion Puppet module will configure Pupistry for you, including setting 339 | up the system service and configuring Puppet and Hiera correctly for masterless 340 | operation. 341 | 342 | You can fetch the module from: 343 | https://github.com/jethrocarr/puppet-pupistry 344 | 345 | If you're doing `r10k` and Puppet masterless from scratch, this is probably 346 | something you want to make life easy. With `r10k`, just add the following to your 347 | `Puppetfile`: 348 | 349 | # Install the Pupistry companion module 350 | mod 'jethrocarr/pupistry' 351 | 352 | # Dependencies for Pupistry companion if not already defined 353 | mod 'puppetlabs/stdlib' 354 | mod 'jethrocarr/initfact' 355 | 356 | And include the pupistry module in all your systems: 357 | 358 | node default { 359 | include pupistry 360 | ... 361 | } 362 | 363 | 364 | ## 4. Building your first node (Bootstrapping) 365 | 366 | No need for manual configuration of your servers/nodes, you just need to build 367 | your first artifact with Pupistry (`pupistry build && pupistry push`) and then 368 | generate a bootstrap script for your particular OS with `pupistry bootstrap` 369 | 370 | The bootstrap script will: 371 | 372 | 1. Install Puppet and Pupistry for the particular OS. 373 | 2. Download the latest artifact 374 | 3. Trigger a Puppet run to build your server. 375 | 376 | These bootstrap scripts can be generated for you. Refer to "Bootstrapping" under 377 | "Usage" instructions above for details. 378 | 379 | The bootstrap script goal is to get you from stock OS to running Pupistry and 380 | doing your first Puppet run. After that - it's up to you and your Puppet 381 | skills to make your node actually do something useful. :-) 382 | 383 | 384 | ## 5. (optional) Baking an image with Packer 385 | 386 | Note that the node initialisation process is still susceptible to weaknesses 387 | such as a bug in a new version of Puppet or Pupistry, or changes to the OS 388 | packages. If this is a concern/issue for you and you want complete reliability, 389 | then use the user data to build a host pre-loaded with Puppet and Pupistry and 390 | then create an image of it using a tool like [Packer](http://www.packer.io/). Doing 391 | this, you can make it possible to build all the way to Puppet execution with no 392 | dependencies on any third parties other than your VM provider and AWS S3. 393 | 394 | Pupistry includes support for generating some Packer examples that you can 395 | either use as-is or built upon to meet your own needs. You can list all the 396 | available Packer templates with: 397 | 398 | pupistry packer 399 | 400 | You can select a template and generate a Packer file by specifying the template 401 | and the output file on the command line: 402 | 403 | pupistry packer --template aws_amazon-any --file packer.json 404 | 405 | Once the file has been generated, you can build your packer environment with 406 | the `packer build` command. Note that some templates will require additional 407 | variables to be passed to them at run time, for example the AWS template 408 | requires a VPC ID and subnet ID specific to your account. 409 | 410 | packer build \ 411 | -var 'aws_vpc_id=vpc-example' \ 412 | -var 'aws_subnet_id=subnet-example' \ 413 | output.json 414 | 415 | By default any Packer machines are built with the hostname of "packer" which 416 | allows you to specifically target them with your manifests. If you don't do any 417 | targetting, the default manifests will be applied. 418 | 419 | Templates tend to have other customisable variables, check the available 420 | options and their defaults with `packer inspect output.json`. 421 | 422 | 423 | # Tutorials 424 | 425 | If you're looking for a more complete introduction to doing masterless Puppet 426 | and want to use Pupistry, check out a tutorial by the author: 427 | 428 | https://www.jethrocarr.com/2015/05/10/setting-up-and-using-pupistry 429 | 430 | By following this tutorial you can go from nothing, to having a complete up 431 | and running masterless Puppet environment using Pupistry. It covers the very 432 | basics of setting up your `r10k` environment. 433 | 434 | 435 | # GPG Notes 436 | 437 | GPG can be a bit of a beast to setup and get used to. Pupistry tries to make 438 | the signing and key sharing process as simple as possible, but setting up GPG 439 | on your platform and creating your key is beyond the scope of this document. 440 | 441 | If you are being asked for your GPG password for every `pupistry push` even in 442 | rapid succession, then you may need to setup gpg-agent so it can keep you 443 | logged in for short durations to get a better balance of security vs usability. 444 | 445 | Currently Pupistry supports a 1:1 approach, where the key used to sign the 446 | artifact is the key used to verify it. Pull requests to add support for signing 447 | and verifying against a keyring list would be welcome to make it easier for 448 | teams to use GPG without having everyone with a single master key. 449 | 450 | Note that GPG isn't vital for security - you still have end-to-end transport 451 | security between your build machine and your servers via HTTPS/TLS to and from 452 | the S3 bucket, all that GPG does is prevent anyone who managed to break into 453 | your S3 bucket from pushing their own Puppet manifests out. 454 | 455 | Generally S3 is secure (assuming no bugs in AWS itself), any likely exploit 456 | would be from you accidentally sharing your IAM credentials in the wrong place, 457 | or an exploited build server. 458 | 459 | 460 | # Securing Hiera with HieraCrypt 461 | 462 | In a standard Puppet master situation, the Puppet master parses the Hiera data 463 | and then passes only the values that apply to a particular host to it. But with 464 | masterless Puppet, all machines get a full copy of Hiera data, which could be a 465 | major issue if one box gets expoited and the contents leaked. Generally it goes 466 | against good practise and damanges the isolation ability of VMs if you give all 467 | the VMs enough information to do some serious damage to themselves. 468 | 469 | By default an out-of-the-box Pupistry installation suffers this limitation like 470 | most master-less Puppet solutions. However, there is an optional feature built 471 | into Pupistry called "HieraCrypt" which can be used to encrypt data and prevent 472 | excessive exposure of information to nodes. 473 | 474 | The solutions works, by generating a cert on each node you use with the 475 | `pupistry hieracrypt --generate` parameter and saving the output into your 476 | puppetcode repository at `hieracrypt/nodes/HOSTNAME`. This output includes a 477 | x509 cert made against the host's SSH RSA host key and a JSON array of all 478 | the facter facts on that host that correlate to values inside the hiera.yaml 479 | file. 480 | 481 | When you run Pupistry on your build workstation, it parses the hiera.yaml file 482 | for each environment and generates a match of files per-node. It then encrypts 483 | these files and creates an encrypted package for each node that only they can 484 | decrypt. 485 | 486 | For example, if your hiera.yaml file looks like: 487 | 488 | :hierarchy: 489 | - "environments/%{::environment}" 490 | - "nodes/%{::hostname}" 491 | - common 492 | 493 | And your hieradata directory looks like: 494 | 495 | hieradata/ 496 | hieradata/common.yaml 497 | hieradata/environments 498 | hieradata/nodes 499 | hieradata/nodes/testhost.yaml 500 | 501 | When Pupistry builds the artifact, it will include the `common.yaml` file for 502 | all nodes, however the `testhost.yaml` file will only be included for the 503 | server with that hostname. 504 | 505 | All servers still get the encrypted data for all the other nodes as they're 506 | shipped as part of the artifact, but nodes can only decrypt the data signed 507 | against their key. 508 | 509 | 510 | # Caveats & Future Plans 511 | 512 | ## Use r10k 513 | 514 | Currently only an `r10k` workflow is supported. Pull requests for others (eg 515 | Librarian Puppet) are welcome, but it's not a priority for this author as r10k 516 | is working nicely. 517 | 518 | 519 | ## Bootstrap Functionality 520 | 521 | Currently Pupistry only supports generation of bootstrap for select popular 522 | distributions and platforms. Other distributions will be added, but it may take 523 | time to get to your particular favourite distribution. 524 | 525 | Note that it isn't a show stopper if support for your platform of choice 526 | doesn't yet exist - you can use pupistry with pretty much any nix platform, 527 | you'll just not have the handy advantage of automatically generated bootstrap 528 | for your servers. And in many cases, one of the existing ones can easily be 529 | adapted to your platform of choice. 530 | 531 | If you do customise it for a different platform, pull requests are VERY 532 | welcome, I'll add pretty much any OS if you write a decent bootstrap template 533 | for it. 534 | 535 | Please see resources/bootstrap/BOOTSTRAP_NOTES.md for more details on how to 536 | write and debug bootstrap templates. 537 | 538 | 539 | ## Continuous Deployment 540 | 541 | A lot of what Pupistry does can also be accomplished by various home-grown 542 | Continious Deployment (CD) solutions using platforms like Jenkins or Bamboo. CD 543 | is an excellent approach for larger organisations, but Pupistry has been 544 | designed for both large and small users so does not mandate it. 545 | 546 | It would be possible to use Pupistry as part of your CD process and if you 547 | decide to do so, a pull request to better support CD systems out-of-the-box 548 | would be welcome. 549 | 550 | 551 | ## PuppetDB 552 | 553 | There's nothing stopping you from using PuppetDB other than Pupistry has no 554 | automatic setup hooks in the bootstrap config. Pull requests to support 555 | PuppetDB for masterless machines are welcome, although masterless users tend 556 | to want to avoid dependencies on a central point. 557 | 558 | 559 | ## Windows 560 | 561 | No idea whether this works under Windows, or what would be required to make it 562 | do so. Again, pull requests always welcome but it's not a priority for the 563 | author. 564 | 565 | 566 | 567 | # Developing 568 | 569 | When developing Pupistry, you can run the Git repo copy with: 570 | 571 | gem install bundler 572 | bundle install 573 | bundle exec pupistry 574 | 575 | By default Pupistry will try to load a settings.yaml file in the current 576 | working directory, before then trying `~/.pupistry/settings.yaml` and then 577 | finally `/etc/pupistry/settings.yaml`. You can also override with `--config`. 578 | 579 | Add `--verbose` for additional debugging information. If you have a bug this 580 | is the first thing you should run to get more context for reports. 581 | 582 | Whilst Pupistry has few tests, we would like to improve this. Please feel free 583 | to contribute any additional tests and aim to write tests for new features and 584 | definetly for any bug fixes. Once you have written tests, check the output of 585 | the tests and Rubocop with: 586 | 587 | bundle exec rake 588 | 589 | 590 | 591 | # Contributions 592 | 593 | Pull requests are very welcome. Pupistry is a very young app and there is 594 | plenty of work that can be done to improve it's code quality, enhance existing 595 | features and add handy new features. Constructive feedback/requests via the 596 | issue tracker is fine, but pull requests speak louder than words. :-) 597 | 598 | If you find a bug or need support, please use the issue tracker rather than 599 | personal emails to the author. 600 | 601 | Feel free to grep the source for "TODO" comments on various tasks that 602 | need doing, or check out the issuer tracker for interesting issues to 603 | tackle. 604 | 605 | 606 | 607 | # Author 608 | 609 | Pupistry is developed by Jethro Carr. Blog posts about Pupistry and new 610 | features can be found at http://www.jethrocarr.com/tag/pupistry 611 | 612 | Beer welcome. 613 | 614 | 615 | # License 616 | 617 | Pupistry is licensed under the Apache License, Version 2.0 (the "License"). 618 | See the `LICENSE.txt` or http://www.apache.org/licenses/LICENSE-2.0 619 | 620 | Unless required by applicable law or agreed to in writing, software 621 | distributed under the License is distributed on an "AS IS" BASIS, 622 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 623 | See the License for the specific language governing permissions and 624 | limitations under the License. 625 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require 'bundler/setup' 2 | require 'bundler/gem_tasks' 3 | require 'rake/testtask' 4 | require 'rake/clean' 5 | require 'rubocop/rake_task' 6 | 7 | RuboCop::RakeTask.new(:rubocop) do |r| 8 | r.patterns = ['lib/**/*.rb', 'exe/*.rb'] 9 | 10 | # Rubocop is important, but it shouldn't be a reason for build failure 11 | r.fail_on_error = false 12 | end 13 | 14 | Rake::TestTask.new do |t| 15 | t.pattern = 'test/test_*.rb' 16 | end 17 | 18 | task 'default' => [:test, :rubocop] 19 | -------------------------------------------------------------------------------- /bin/pupistry: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # 3 | # This file was generated by Bundler. 4 | # 5 | # The application 'pupistry' is installed as part of a gem, and 6 | # this file is here to facilitate running it. 7 | # 8 | 9 | require 'pathname' 10 | ENV['BUNDLE_GEMFILE'] ||= File.expand_path("../../Gemfile", 11 | Pathname.new(__FILE__).realpath) 12 | 13 | require 'rubygems' 14 | require 'bundler/setup' 15 | 16 | load Gem.bin_path('pupistry', 'pupistry') 17 | -------------------------------------------------------------------------------- /exe/pupistry: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # Lancher for Pupistry CLI 3 | 4 | require 'rubygems' 5 | require 'thor' 6 | require 'logger' 7 | require 'fileutils' 8 | require 'pupistry' 9 | 10 | # Ensure all output is real time - this is a long running process with 11 | # continual output, we want it to sync ASAP 12 | STDOUT.sync = true 13 | 14 | # Logging - STDOUT only 15 | $logger = Logger.new(STDOUT) 16 | 17 | # Thor is a toolkit for producing command line applications, see http://whatisthor.com/ 18 | class CLI < Thor 19 | class_option :verbose, type: :boolean 20 | class_option :config, type: :string 21 | 22 | ## Agent Commands 23 | 24 | desc 'apply', 'Apply the latest Puppet artifact' 25 | method_option :noop, type: :boolean, desc: 'No changes mode (note: does change checked out artifact, but no Puppet changes)' 26 | method_option :force, type: :boolean, desc: 'Ignore existing versions, re-apply every time' 27 | method_option :minimal, type: :boolean, desc: "Don't run Puppet unless the artifact has changed" 28 | method_option :daemon, type: :boolean, desc: 'Run as a system daemon' 29 | method_option :environment, type: :string, desc: 'Specifiy which environment to deploy (default: master)' 30 | def apply 31 | # Thor seems to force class options to be defined repeatedly? :-/ 32 | if options[:verbose] 33 | $logger.level = Logger::DEBUG 34 | else 35 | $logger.level = Logger::INFO 36 | end 37 | 38 | if options[:config] 39 | Pupistry::Config.load(options[:config]) 40 | else 41 | Pupistry::Config.find_and_load 42 | end 43 | 44 | # Muppet Check 45 | $logger.warn 'A daemon running in noop will do nothing except log what changes it could apply' if options[:noop] and options[:daemon] 46 | 47 | $logger.warn 'A daemon running with force will be very wasteful of system resources! NOT RECOMMENDED.' if options[:force] and options[:daemon] 48 | 49 | if options[:daemon] 50 | # Run as a daemon service 51 | Pupistry::Agent.daemon options 52 | else 53 | # Single-run Agent Execution 54 | Pupistry::Agent.apply options 55 | end 56 | end 57 | 58 | ## Workstation Commands 59 | 60 | desc 'build', 'Build a new archive file' 61 | def build 62 | # Thor seems to force class options to be defined repeatedly? :-/ 63 | if options[:verbose] 64 | $logger.level = Logger::DEBUG 65 | else 66 | $logger.level = Logger::INFO 67 | end 68 | 69 | if options[:config] 70 | Pupistry::Config.load(options[:config]) 71 | else 72 | Pupistry::Config.find_and_load 73 | end 74 | 75 | begin 76 | # Fetch the latest data with r10k 77 | artifact = Pupistry::Artifact.new 78 | 79 | artifact.fetch_r10k 80 | artifact.hieracrypt_encrypt 81 | artifact.build_artifact 82 | 83 | puts '--' 84 | puts 'Tip: Run pupistry diff to see what changed since the last artifact version' 85 | 86 | rescue StandardError => e 87 | $logger.fatal 'An unexpected error occured when trying to generate the new artifact file' 88 | raise e 89 | end 90 | end 91 | 92 | desc 'diff', 'Show what has changed between now and the current live artifact' 93 | def diff 94 | # Thor seems to force class options to be defined repeatedly? :-/ 95 | if options[:verbose] 96 | $logger.level = Logger::DEBUG 97 | else 98 | $logger.level = Logger::INFO 99 | end 100 | 101 | if options[:config] 102 | Pupistry::Config.load(options[:config]) 103 | else 104 | Pupistry::Config.find_and_load 105 | end 106 | 107 | # Fetch the latest artifact 108 | artifact_upstream = Pupistry::Artifact.new 109 | artifact_upstream.checksum = artifact_upstream.fetch_latest 110 | 111 | unless artifact_upstream.checksum 112 | $logger.error 'There is no upstream artifact to compare to.' 113 | exit 0 114 | end 115 | 116 | artifact_upstream.fetch_artifact 117 | 118 | # Fetch the current artifact 119 | artifact_current = Pupistry::Artifact.new 120 | artifact_current.checksum = artifact_current.fetch_current 121 | 122 | unless artifact_current.checksum 123 | $logger.error 'There is no current artifact to compare to, run "pupistry build" first to generate one with current changes' 124 | exit 0 125 | end 126 | 127 | artifact_current.fetch_artifact 128 | 129 | # Are they the same version? 130 | $logger.info 'Current version and upstream version are the same, no diff' if artifact_current.checksum == artifact_upstream.checksum 131 | 132 | # Unpack the archives 133 | artifact_current.unpack 134 | artifact_upstream.unpack 135 | 136 | # Diff the contents. This is actually bit of a pain, there's no native way 137 | # of diffing an entire directory and a lot of the gems out there that promise 138 | # to do diffing a) can't handle dirs and b) generally exec out to native diff 139 | # anyway. :-( 140 | # 141 | # So given this, we might as well go native and just rely on the system 142 | # diff command to do the job. 143 | # 144 | # TODO: We need smarts here to handle git branching, so a branch doens't 145 | # produce a new mega diff, we want only the real changes to be 146 | # easily visible, or the diff function loses value to people. 147 | # Pull requests welcome :-) xoxo 148 | 149 | Dir.chdir("#{$config['general']['app_cache']}/artifacts/") do 150 | unless system "diff -Nuar --exclude hieracrypt unpacked.#{artifact_upstream.checksum} unpacked.#{artifact_current.checksum}" 151 | end 152 | end 153 | 154 | # Cleanup 155 | artifact_current.clean_unpack 156 | artifact_upstream.clean_unpack 157 | 158 | puts '--' 159 | puts 'Tip: Run pupistry push to GPG sign & upload if happy to go live' 160 | end 161 | 162 | desc 'push', 'Sign & Upload a new artifact version' 163 | def push 164 | # Thor seems to force class options to be defined repeatedly? :-/ 165 | if options[:verbose] 166 | $logger.level = Logger::DEBUG 167 | else 168 | $logger.level = Logger::INFO 169 | end 170 | 171 | if options[:config] 172 | Pupistry::Config.load(options[:config]) 173 | else 174 | Pupistry::Config.find_and_load 175 | end 176 | 177 | # Push the artifact to S3 178 | artifact = Pupistry::Artifact.new 179 | artifact.push_artifact 180 | end 181 | 182 | desc 'bootstrap', 'Generate a user-data bootstrap script for a node' 183 | method_option :template, type: :string, desc: 'The template you want to generate' 184 | method_option :base64, type: :boolean, desc: 'Output in base64 format' 185 | method_option :environment, type: :string, desc: 'Environment to run puppet in' 186 | def bootstrap 187 | # Thor seems to force class options to be defined repeatedly? :-/ 188 | if options[:verbose] 189 | $logger.level = Logger::DEBUG 190 | else 191 | $logger.level = Logger::INFO 192 | end 193 | 194 | if options[:config] 195 | Pupistry::Config.load(options[:config]) 196 | else 197 | Pupistry::Config.find_and_load 198 | end 199 | 200 | if options[:template] 201 | $logger.debug "Generating bootstrap template #{options[:template]}" 202 | 203 | environment = options[:environment] || 'master' 204 | templates = Pupistry::Bootstrap.new 205 | templates.build options[:template], environment 206 | 207 | if options[:base64] 208 | templates.output_base64 209 | else 210 | templates.output_plain 211 | end 212 | else 213 | templates = Pupistry::Bootstrap.new 214 | templates.list 215 | 216 | puts '--' 217 | puts 'Tip: Run `pupistry bootstrap --template example` to generate a specific template' 218 | end 219 | end 220 | 221 | desc 'packer', 'Generate a packer template for a particular provider with OS bootstrap script included' 222 | method_option :template, type: :string, desc: 'The template you want to generate' 223 | method_option :file, type: :string, desc: 'File to write the generated packer template into' 224 | def packer 225 | # Thor seems to force class options to be defined repeatedly? :-/ 226 | if options[:verbose] 227 | $logger.level = Logger::DEBUG 228 | else 229 | $logger.level = Logger::INFO 230 | end 231 | 232 | if options[:config] 233 | Pupistry::Config.load(options[:config]) 234 | else 235 | Pupistry::Config.find_and_load 236 | end 237 | 238 | if options[:template] 239 | $logger.info "Generating packer template #{options[:template]}" 240 | 241 | templates = Pupistry::Packer.new 242 | templates.build options[:template] 243 | 244 | if options[:file] 245 | templates.output_file options[:file] 246 | else 247 | templates.output_plain 248 | end 249 | else 250 | templates = Pupistry::Packer.new 251 | templates.list 252 | 253 | puts '--' 254 | puts 'Tip: Run `pupistry packer --template example` to generate a specific template' 255 | end 256 | end 257 | 258 | ## Hieracrypt Feature 259 | 260 | desc 'hieracrypt', 'Manage the encryption of Hiera data to securely restrict access from nodes' 261 | method_option :generate, type: :boolean, desc: 'Generate an export of public cert and facts for Hieracrypt usage.' 262 | def hieracrypt 263 | # Thor seems to force class options to be defined repeatedly? :-/ 264 | if options[:verbose] 265 | $logger.level = Logger::DEBUG 266 | else 267 | $logger.level = Logger::INFO 268 | end 269 | 270 | if options[:config] 271 | Pupistry::Config.load(options[:config]) 272 | else 273 | Pupistry::Config.find_and_load 274 | end 275 | 276 | # TODO 277 | if options[:generate] 278 | Pupistry::HieraCrypt.generate_nodedata 279 | else 280 | puts "Run `pupistry hieracrypt --generate` on each node to get back a data file to be saved into puppetcode" 281 | end 282 | end 283 | 284 | ## Other Commands 285 | 286 | desc 'setup', 'Write a template configuration file' 287 | method_option :force, type: :boolean, desc: 'Replace an existing config file' 288 | def setup 289 | # Thor seems to force class options to be defined repeatedly? :-/ 290 | if options[:verbose] 291 | $logger.level = Logger::DEBUG 292 | else 293 | $logger.level = Logger::INFO 294 | end 295 | 296 | # Generally we should put the Pupistry configuration into the home dir, a 297 | # developer who wants it elsewhere will be capable of figuring out how to 298 | # install themselves. 299 | config_dest = '~/.pupistry/settings.yaml' 300 | 301 | # If the HOME environmental hasn't been set, dump the config into CWD. 302 | unless ENV['HOME'] 303 | config_dest = "#{Dir.pwd}/settings.yaml" 304 | $logger.warn "HOME is not set, so writing configuration file into #{config_dest}" 305 | end 306 | 307 | config_dest = File.expand_path config_dest 308 | 309 | # Make sure the directory exists 310 | FileUtils.mkdir_p(File.dirname(config_dest)) unless Dir.exist?(File.dirname(config_dest)) 311 | 312 | # Does a local template exist? 313 | if File.exist?("#{Dir.pwd}/settings.example.yaml") 314 | config_source = "#{Dir.pwd}/settings.example.yaml" 315 | else 316 | # Check for GEM installed location 317 | begin 318 | config_source = Gem::Specification.find_by_name('pupistry').gem_dir 319 | config_source = "#{config_source}/settings.example.yaml" 320 | rescue Gem::LoadError 321 | # Yeah I dunno what you're doing... 322 | $logger.error 'Unable to find settings.example.yaml, seems we are not running as a Gem nor in the CWD of the app source' 323 | exit 0 324 | end 325 | end 326 | 327 | unless File.exist?(config_source) 328 | $logger.error "Template configuration should exist in #{config_source} but no file found/readable!" 329 | exit 0 330 | end 331 | 332 | # Prevent Overwrite 333 | if File.exist?(config_dest) 334 | if options[:force] 335 | $logger.warn "Overwriting #{config_dest}..." 336 | else 337 | $logger.error "Configuration file #{config_dest} already exists, if you wish to replace it please call with --force" 338 | exit 0 339 | end 340 | end 341 | 342 | # Copy the template configuration file to destination 343 | begin 344 | FileUtils.cp config_source, config_dest 345 | 346 | $logger.info "Successfully installed configuration file into #{config_dest}" 347 | rescue StandardError => e 348 | $logger.error "An unexpected error occured when copying #{config_source} to #{config_dest}" 349 | raise e 350 | end 351 | 352 | # TODO: This is where I'd like to do things more cleverly. Currently we 353 | # just tell the user to edit the configuration file, but would be really 354 | # cool to write a setup wizard that helps them complete the configuration 355 | # file and validates the input (eg AWS keys). 356 | 357 | if ENV['EDITOR'] 358 | $logger.info "Now open the config file with `#{ENV['EDITOR']} #{config_dest}` and set your configuration values before running Pupistry." 359 | else 360 | $logger.info "You now need to edit #{config_dest} with your configuration values before running Pupistry." 361 | end 362 | end 363 | end 364 | 365 | CLI.start(ARGV) 366 | 367 | # vim:shiftwidth=2:tabstop=2:softtabstop=2:expandtab:smartindent 368 | -------------------------------------------------------------------------------- /lib/pupistry.rb: -------------------------------------------------------------------------------- 1 | require 'pupistry/version' 2 | require 'pupistry/agent' 3 | require 'pupistry/artifact' 4 | require 'pupistry/bootstrap' 5 | require 'pupistry/config' 6 | require 'pupistry/gpg' 7 | require 'pupistry/hieracrypt' 8 | require 'pupistry/packer' 9 | require 'pupistry/storage_aws' 10 | 11 | # vim:shiftwidth=2:tabstop=2:softtabstop=2:expandtab:smartindent 12 | -------------------------------------------------------------------------------- /lib/pupistry/agent.rb: -------------------------------------------------------------------------------- 1 | # rubocop:disable Style/GlobalVars 2 | require 'rubygems' 3 | require 'fileutils' 4 | require 'rufus-scheduler' 5 | 6 | # Pupistry::Agent 7 | module Pupistry 8 | # Functions for running the Pupistry agent aka "apply mode" to actually 9 | # download and run Puppet against the contents of the artifact. 10 | class Agent 11 | ## Run as a daemon 12 | def self.daemon(options) 13 | # Since options comes from Thor, it can't be modified, so we need to 14 | # copy the options and then we can edit it. 15 | 16 | options_new = options.inject({}) do |new, (name, value)| 17 | new[name] = value 18 | new 19 | end 20 | 21 | # If the minimal mode has been enabled in config, respect. 22 | options_new[:minimal] = true if $config['agent']['daemon_minimal'] 23 | 24 | # If no frequency supplied, use 300 seconds safe default. 25 | $config['agent']['daemon_frequency'] = 300 unless $config['agent']['daemon_frequency'] 26 | 27 | # Use rufus-scheduler to run our apply job as a regularly scheduled job 28 | # but with build in locking handling. 29 | 30 | $logger.info "Launching daemon... frequency of #{$config['agent']['daemon_frequency']} seconds." 31 | 32 | begin 33 | 34 | scheduler = Rufus::Scheduler.new 35 | 36 | scheduler.every "#{$config['agent']['daemon_frequency']}s", overlap: false, timeout: '1d', first_at: Time.now + 1 do 37 | $logger.info "Triggering another Pupistry run (#{$config['agent']['daemon_frequency']}s)" 38 | apply options_new 39 | end 40 | 41 | scheduler.join 42 | 43 | rescue Rufus::Scheduler::TimeoutError 44 | $logger.error 'A run of Pupistry timed out after 1 day as a safety measure. There may be a bug or a Puppet action causing it to get stuck' 45 | 46 | rescue SignalException 47 | # Clean shutdown signal (eg SIGTERM) 48 | $logger.info 'Clean shutdown of Pupistry daemon requests' 49 | exit 0 50 | 51 | rescue StandardError => e 52 | raise e 53 | end 54 | end 55 | 56 | def self.apply(options) 57 | ## Download and apply the latest artifact (if any) 58 | 59 | # Fetch artifact versions 60 | $logger.info 'Checking version of artifact available...' 61 | 62 | artifact = Pupistry::Artifact.new 63 | artifact.checksum = artifact.fetch_latest 64 | 65 | unless artifact.checksum 66 | $logger.error 'There is no current artifact available for download, no steps can be taken.' 67 | return false 68 | end 69 | 70 | artifact_installed = Pupistry::Artifact.new 71 | artifact_installed.checksum = artifact_installed.fetch_installed 72 | 73 | if artifact_installed.checksum 74 | $logger.debug "Currently on #{artifact_installed.checksum}" 75 | else 76 | $logger.debug 'No currently installed artifact - blank slate!' 77 | end 78 | 79 | # Download the new artifact if one has changed. If we already have this 80 | # version, then we should skip downloading and go straight to running 81 | # Puppet - unless the user runs with --force (eg to fix a corrupted 82 | # artifact). 83 | 84 | if artifact.checksum != artifact_installed.checksum || options[:force] 85 | $logger.warn 'Forcing download of latest artifact regardless of current one.' if options[:force] 86 | 87 | # Install the artifact 88 | $logger.info "Downloading latest artifact (#{artifact.checksum})..." 89 | 90 | artifact.fetch_artifact 91 | artifact.unpack 92 | artifact.hieracrypt_decrypt 93 | 94 | unless artifact.install 95 | $logger.fatal 'An unexpected error happened when installing the latest artifact, cancelling Puppet run' 96 | return false 97 | end 98 | 99 | # Remove temporary unpacked files 100 | artifact.clean_unpack 101 | else 102 | $logger.info 'Already have latest artifact applied.' 103 | 104 | # By default we run Puppet even if we have the latest artifact. There's 105 | # some grounds for debate about whether this is the right thing - in some 106 | # ways it is often a waste of CPU, since if the artifact hasn't changed, 107 | # then it's unlikley anything else has changed. 108 | # 109 | # But that's not always 100% true - Puppet will undo local changes or 110 | # upgrade package versions (ensure => latest) if appropiate, so we should 111 | # act like the standard command and attempt to apply whatever we can. 112 | # 113 | # To provide users with options, we provide the --lazy parameter to avoid 114 | # running Puppet except when the artifact changes. By default, Puppet 115 | # runs every thing to avoid surprise. 116 | 117 | if options[:minimal] 118 | $logger.info 'Running with minimal effort mode enabled, not running Puppet since artifact version already applied' 119 | return false 120 | end 121 | 122 | end 123 | 124 | # If the environment has been specified, use it. 125 | environment = $config['agent']['environment'] || 'master' 126 | # override if environment is supplied on CLI 127 | environment = options["environment"] || environment 128 | 129 | unless Dir.exist?("#{$config['agent']['puppetcode']}/#{environment}") 130 | $logger.fatal "The requested branch/environment of #{environment} does not exist, unable to run Puppet" 131 | return false 132 | end 133 | 134 | # Execute Puppet. 135 | puppet_cmd = 'puppet apply' 136 | 137 | puppet_cmd += ' --noop' if options[:noop] 138 | puppet_cmd += ' --show_diff' if options[:verbose] 139 | 140 | puppet_cmd += " --environment #{environment}" 141 | puppet_cmd += " --confdir #{$config['agent']['puppetcode']}" 142 | puppet_cmd += " --environmentpath #{$config['agent']['puppetcode']}" 143 | puppet_cmd += " --modulepath #{build_modulepath(environment)}" 144 | puppet_cmd += " --hiera_config #{$config['agent']['puppetcode']}/#{environment}/hiera.yaml" 145 | puppet_cmd += " #{$config['agent']['puppetcode']}/#{environment}/manifests/" 146 | 147 | $logger.info 'Executing Puppet...' 148 | $logger.debug "With: #{puppet_cmd}" 149 | 150 | $logger.error 'An unexpected issue occured when running puppet' unless system puppet_cmd 151 | end 152 | 153 | def self.build_modulepath(environment) 154 | 155 | environment_path = "#{$config['agent']['puppetcode']}/#{environment}" 156 | environment_conf = "#{environment_path}/environment.conf" 157 | 158 | configured_paths = [] 159 | 160 | if File.exist?(environment_conf) 161 | $logger.debug "Adding modulepath config from '#{environment_path}'" 162 | 163 | File.open(environment_conf, 'r').readlines.each do |line| 164 | if line !~ /^\s*#/ && /^(.*)=(.*)/ =~ line 165 | key, val = $1.strip, $2.strip 166 | configured_paths = val.split(':') if key == 'modulepath' 167 | end 168 | end 169 | end 170 | 171 | modulepaths = configured_paths.map { |path| File.expand_path(path, environment_path) } 172 | 173 | # Ensure '/modules' in modulepath. 174 | ensure_path = File.expand_path('modules', environment_path) 175 | modulepaths.insert(0, ensure_path) unless modulepaths.include? ensure_path 176 | 177 | modulepaths.join(File::PATH_SEPARATOR) 178 | end 179 | end 180 | end 181 | 182 | # vim:shiftwidth=2:tabstop=2:softtabstop=2:expandtab:smartindent 183 | -------------------------------------------------------------------------------- /lib/pupistry/artifact.rb: -------------------------------------------------------------------------------- 1 | # rubocop:disable Style/Documentation, Style/GlobalVars 2 | require 'rubygems' 3 | require 'yaml' 4 | require 'safe_yaml' 5 | require 'time' 6 | require 'digest' 7 | require 'fileutils' 8 | require 'base64' 9 | 10 | module Pupistry 11 | # Pupistry::Artifact 12 | 13 | class Artifact 14 | # All the functions needed for manipulating the artifats 15 | attr_accessor :checksum 16 | 17 | def fetch_r10k 18 | $logger.info 'Using r10k utility to fetch the latest Puppet code' 19 | 20 | unless defined? $config['build']['puppetcode'] 21 | $logger.fatal 'You must configure the build:puppetcode config option in settings.yaml' 22 | fail 'Invalid Configuration' 23 | end 24 | 25 | # https://github.com/puppetlabs/r10k 26 | # 27 | # r10k does a fantastic job with all the git stuff and we want to use it 28 | # to download the Puppet code from all the git modules (based on following 29 | # the master one provided), then we can steal the Puppet code from the 30 | # artifact generated. 31 | # 32 | # TODO: We should re-write this to hook directly into r10k's libraries, 33 | # given that both Pupistry and r10k are Ruby, presumably it should be 34 | # doable and much more polished approach. For now the MVP is to just run 35 | # it via system, pull requests/patches to fix very welcome! 36 | 37 | # Build the r10k config to instruct it to use our cache path for storing 38 | # it's data and exporting the finished result. 39 | $logger.debug 'Generating an r10k configuration file...' 40 | r10k_config = { 41 | 'cachedir' => "#{$config['general']['app_cache']}/r10kcache", 42 | 'sources' => { 43 | 'puppet' => { 44 | 'remote' => $config['build']['puppetcode'], 45 | 'basedir' => $config['general']['app_cache'] + '/puppetcode' 46 | } 47 | } 48 | } 49 | 50 | begin 51 | File.open("#{$config['general']['app_cache']}/r10kconfig.yaml", 'w') do |fh| 52 | fh.write YAML.dump(r10k_config) 53 | end 54 | rescue StandardError => e 55 | $logger.fatal 'Unexpected error when trying to write the r10k configuration file' 56 | raise e 57 | end 58 | 59 | # Execute R10k with the provided configuration 60 | $logger.debug 'Executing r10k' 61 | 62 | if system "r10k deploy environment -c #{$config['general']['app_cache']}/r10kconfig.yaml -pv debug" 63 | $logger.info 'r10k run completed' 64 | else 65 | $logger.error 'r10k run failed, unable to generate artifact' 66 | fail 'r10k run did not complete, unable to generate artifact' 67 | end 68 | end 69 | 70 | def fetch_latest 71 | # Fetch the latest S3 YAML file and check the version metadata without writing 72 | # it to disk. Returns the version. Useful for quickly checking for updates :-) 73 | 74 | $logger.debug 'Checking latest artifact version...' 75 | 76 | s3 = Pupistry::StorageAWS.new 'agent' 77 | contents = s3.download 'manifest.latest.yaml' 78 | 79 | if contents 80 | manifest = YAML.load(contents, safe: true, raise_on_unknown_tag: true) 81 | 82 | if defined? manifest['version'] 83 | # We have a manifest version supplied, however since the manifest 84 | # isn't signed, there's risk of an exploited S3 bucket replacing 85 | # the version with injections designed to attack the shell commands 86 | # we call from Pupistry. 87 | # 88 | # Therefore we need to make sure the manifest version matches a 89 | # regex suitable for a checksum. 90 | 91 | if /^[A-Za-z0-9]{32}$/.match(manifest['version']) 92 | return manifest['version'] 93 | else 94 | $logger.error 'Manifest version returned from S3 manifest.latest.yaml did not match expected regex of MD5.' 95 | $logger.error 'Possible bug or security incident, investigate with care!' 96 | $logger.error "Returned version string was: \"#{manifest['version']}\"" 97 | exit 0 98 | end 99 | else 100 | return false 101 | end 102 | 103 | else 104 | # download did not work 105 | return false 106 | end 107 | end 108 | 109 | def fetch_current 110 | # Fetch the latest on-disk YAML file and check the version metadata, used 111 | # to determine the latest artifact that has not yet been pushed to S3. 112 | # Returns the version. 113 | 114 | # Read the symlink information to get the latest version 115 | if File.exist?($config['general']['app_cache'] + '/artifacts/manifest.latest.yaml') 116 | manifest = YAML.load(File.open($config['general']['app_cache'] + '/artifacts/manifest.latest.yaml'), safe: true, raise_on_unknown_tag: true) 117 | @checksum = manifest['version'] 118 | else 119 | $logger.error 'No artifact has been built yet. You need to run pupistry build first?' 120 | return false 121 | end 122 | end 123 | 124 | def fetch_installed 125 | # Fetch the current version that is installed. 126 | 127 | # Make sure the Puppetcode install directory exists 128 | unless Dir.exist?($config['agent']['puppetcode']) 129 | $logger.warn "The destination path of #{$config['agent']['puppetcode']} does not appear to exist or is not readable" 130 | return false 131 | end 132 | 133 | # Look for a manifest file in the directory and read the version from it. 134 | if File.exist?($config['agent']['puppetcode'] + '/manifest.pupistry.yaml') 135 | manifest = YAML.load(File.open($config['agent']['puppetcode'] + '/manifest.pupistry.yaml'), safe: true, raise_on_unknown_tag: true) 136 | 137 | return manifest['version'] 138 | else 139 | $logger.warn 'No current version installed' 140 | return false 141 | end 142 | end 143 | 144 | def fetch_artifact 145 | # Figure out which version to fetch (if not explicitly defined) 146 | if defined? @checksum 147 | $logger.debug "Downloading artifact version #{@checksum}" 148 | else 149 | @checksum = fetch_latest 150 | 151 | if defined? @checksum 152 | $logger.debug "Downloading latest artifact (#{@checksum})" 153 | else 154 | $logger.error 'There is not current artifact that can be fetched' 155 | return false 156 | end 157 | 158 | end 159 | 160 | # Make sure the download dir/cache exists 161 | FileUtils.mkdir_p $config['general']['app_cache'] + '/artifacts/' unless Dir.exist?($config['general']['app_cache'] + '/artifacts/') 162 | 163 | # Download files if they don't already exist 164 | if File.exist?($config['general']['app_cache'] + "/artifacts/manifest.#{@checksum}.yaml") && 165 | File.exist?($config['general']['app_cache'] + "/artifacts/artifact.#{@checksum}.tar.gz") 166 | $logger.debug 'This artifact is already present, no download required.' 167 | else 168 | s3 = Pupistry::StorageAWS.new 'agent' 169 | s3.download "manifest.#{@checksum}.yaml", $config['general']['app_cache'] + "/artifacts/manifest.#{@checksum}.yaml" 170 | s3.download "artifact.#{@checksum}.tar.gz", $config['general']['app_cache'] + "/artifacts/artifact.#{@checksum}.tar.gz" 171 | end 172 | end 173 | 174 | def hieracrypt_encrypt 175 | # Stub function, since HieraCrypt has no association with the actual 176 | # artifact file, but rather the post-r10k checked data, it could be 177 | # invoked directly. However it's worth wrapping here incase we ever 178 | # do change this behavior. 179 | 180 | Pupistry::HieraCrypt.encrypt_hieradata 181 | 182 | end 183 | 184 | def hieracrypt_decrypt 185 | # Decrypt any encrypted Hieradata inside the currently unpacked artifact 186 | # before it gets copied to the installation location. 187 | 188 | if defined? @checksum 189 | Pupistry::HieraCrypt.decrypt_hieradata $config['general']['app_cache'] + "/artifacts/unpacked.#{@checksum}/puppetcode" 190 | else 191 | $logger.warn "Tried to request hieracrypt_decrypt on no artifact." 192 | end 193 | 194 | end 195 | def push_artifact 196 | # The push step involves 2 steps: 197 | # 1. GPG sign the artifact and write it into the manifest file 198 | # 2. Upload the manifest and archive files to S3. 199 | # 3. Upload a copy as the "latest" manifest file which will be hit by clients. 200 | 201 | # Determine which version we are uploading. Either one specifically 202 | # selected, otherwise find the latest one to push 203 | 204 | if defined? @checksum 205 | $logger.info "Uploading artifact version #{@checksum}." 206 | else 207 | @checksum = fetch_current 208 | 209 | if @checksum 210 | $logger.info "Uploading artifact version latest (#{@checksum})" 211 | else 212 | # If there is no current version, we can't do much.... 213 | exit 0 214 | end 215 | end 216 | 217 | # Do we even need to upload? If nothing has changed.... 218 | if @checksum == fetch_latest 219 | $logger.error "You've already pushed this artifact version, nothing to do." 220 | exit 0 221 | end 222 | 223 | # Make sure the files actually exist... 224 | unless File.exist?($config['general']['app_cache'] + "/artifacts/manifest.#{@checksum}.yaml") 225 | $logger.error "The files expected for #{@checksum} do not appear to exist or are not readable" 226 | fail 'Fatal unexpected error' 227 | end 228 | 229 | unless File.exist?($config['general']['app_cache'] + "/artifacts/artifact.#{@checksum}.tar.gz") 230 | $logger.error "The files expected for #{@checksum} do not appear to exist or are not readable" 231 | fail 'Fatal unexpected error' 232 | end 233 | 234 | # GPG sign the files 235 | if $config['general']['gpg_disable'] == true 236 | $logger.warn 'You have GPG signing *disabled*, whilst not critical it does weaken your security.' 237 | $logger.warn 'Skipping signing step...' 238 | else 239 | 240 | gpgsig = Pupistry::GPG.new @checksum 241 | 242 | # Sign the artifact 243 | unless gpgsig.artifact_sign 244 | $logger.fatal 'Unable to proceed with an unsigned artifact' 245 | exit 0 246 | end 247 | 248 | # Verify the signature - we want to make sure what we've just signed 249 | # can actually be validated properly :-) 250 | unless gpgsig.artifact_verify 251 | $logger.fatal 'Whilst a signature was generated, it was unable to be validated. This would suggest a bug of some kind.' 252 | exit 0 253 | end 254 | 255 | # Save the signature to the manifest 256 | unless gpgsig.signature_save 257 | $logger.fatal 'Unable to write the signature into the manifest file for the artifact.' 258 | exit 0 259 | end 260 | 261 | end 262 | 263 | # Upload the artifact & manifests to S3. We also make an additional copy 264 | # as the "latest" file which will be downloaded by all the agents checking 265 | # for new updates. 266 | 267 | s3 = Pupistry::StorageAWS.new 'build' 268 | s3.upload $config['general']['app_cache'] + "/artifacts/artifact.#{@checksum}.tar.gz", "artifact.#{@checksum}.tar.gz" 269 | s3.upload $config['general']['app_cache'] + "/artifacts/manifest.#{@checksum}.yaml", "manifest.#{@checksum}.yaml" 270 | s3.upload $config['general']['app_cache'] + "/artifacts/manifest.#{@checksum}.yaml", 'manifest.latest.yaml' 271 | 272 | # Test a read of the manifest, we do this to make sure the S3 ACLs setup 273 | # allow downloading of the uploaded files - helps avoid user headaches if 274 | # they misconfigure and then blindly trust their bootstrap config. 275 | # 276 | # Only worth doing this step if they've explicitly set their AWS IAM credentials 277 | # for the agent, which should be everyone except for IAM role users. 278 | 279 | if $config['agent']['access_key_id'] 280 | fetch_artifact 281 | else 282 | $logger.warn "The agent's AWS credentials are unset on this machine, unable to do download test to check permissions for you." 283 | $logger.warn "Assuming you know what you're doing, please set if unsure." 284 | end 285 | 286 | $logger.info "Upload of artifact version #{@checksum} completed and is now latest" 287 | end 288 | 289 | def build_artifact 290 | # r10k has done all the heavy lifting for us, we just need to generate a 291 | # tarball from the app_cache /puppetcode directory. There are some Ruby 292 | # native libraries, but really we might as well just use the native tools 293 | # since we don't want to do anything clever like in-memory assembly of 294 | # the file. Like r10k, if you want to convert to a nicely polished native 295 | # Ruby solution, patches welcome. 296 | 297 | $logger.info 'Creating artifact...' 298 | 299 | Dir.chdir($config['general']['app_cache']) do 300 | # Make sure there is a directory to write artifacts into 301 | FileUtils.mkdir_p('artifacts') 302 | 303 | # Build the tar file - we delibertly don't compress in a single step 304 | # so that we can grab the checksum, since checksum will always differ 305 | # post-compression. 306 | 307 | tar = Pupistry::Config.which_tar 308 | $logger.debug "Using tar at #{tar}" 309 | 310 | tar += " -c" 311 | tar += " --exclude '.git'" 312 | if Pupistry::HieraCrypt.is_enabled? 313 | # We want to exclude unencrypted hieradata (duh security) and also the node files (which aren't needed) 314 | tar += " --exclude 'hieradata'" 315 | tar += " --exclude 'hieracrypt/nodes'" 316 | else 317 | # Hieracrypt is disable, exclude any old out of date encrypted files 318 | tar += " --exclude 'hieracrypt/encrypted'" 319 | end 320 | tar += " -f artifacts/artifact.temp.tar puppetcode/*" 321 | 322 | unless system tar 323 | $logger.error 'Unable to create tarball' 324 | fail 'An unexpected error occured when executing tar' 325 | end 326 | 327 | # The checksum is important, we use it as our version for each artifact 328 | # so we can tell them apart in a unique way. 329 | @checksum = Digest::MD5.file($config['general']['app_cache'] + '/artifacts/artifact.temp.tar').hexdigest 330 | 331 | # Now we have the checksum, check if it's the same as any existing 332 | # artifacts. If so, drop out here, good to give feedback to the user 333 | # if nothing has changed since it's easy to forget to git push a single 334 | # module/change. 335 | 336 | if File.exist?($config['general']['app_cache'] + "/artifacts/manifest.#{@checksum}.yaml") 337 | $logger.error "This artifact version (#{@checksum}) has already been built, nothing todo." 338 | $logger.error "Did you remember to \"git push\" your module changes?" 339 | 340 | # TODO: Unfortunatly Hieracrypt breaks this, since the encrypted Hieradata is different 341 | # on every run, which results in the checksum always being different even if nothing in 342 | # the repo itself has changed. We need a proper fix for this at some stage, for now it's 343 | # covered in the readme notes for HieraCrypt as a flaw. 344 | 345 | # Cleanup temp file 346 | FileUtils.rm($config['general']['app_cache'] + '/artifacts/artifact.temp.tar') 347 | exit 0 348 | end 349 | 350 | # Compress the artifact now that we have taken it's checksum 351 | $logger.info 'Compressing artifact...' 352 | 353 | if system 'gzip artifacts/artifact.temp.tar' 354 | else 355 | $logger.error 'An unexpected error occured during compression of the artifact' 356 | fail 'An unexpected error occured during compression of the artifact' 357 | end 358 | end 359 | 360 | # We have the checksum, so we can now rename the artifact file 361 | FileUtils.mv($config['general']['app_cache'] + '/artifacts/artifact.temp.tar.gz', 362 | $config['general']['app_cache'] + "/artifacts/artifact.#{@checksum}.tar.gz") 363 | 364 | $logger.info 'Building manifest information for artifact...' 365 | 366 | # Create the manifest file, this is used by clients for pulling details about 367 | # the latest artifacts. We don't GPG sign here, but we do put in a placeholder. 368 | manifest = { 369 | 'version' => @checksum, 370 | 'date' => Time.new.inspect, 371 | 'builduser' => ENV['USER'] || 'unlabled', 372 | 'gpgsig' => 'unsigned' 373 | } 374 | 375 | begin 376 | File.open("#{$config['general']['app_cache']}/artifacts/manifest.#{@checksum}.yaml", 'w') do |fh| 377 | fh.write YAML.dump(manifest) 378 | end 379 | rescue StandardError => e 380 | $logger.fatal 'Unexpected error when trying to write the manifest file' 381 | raise e 382 | end 383 | 384 | # This is the latest artifact, create some symlinks pointing the latest to it 385 | begin 386 | FileUtils.ln_s("manifest.#{@checksum}.yaml", 387 | "#{$config['general']['app_cache']}/artifacts/manifest.latest.yaml", 388 | force: true) 389 | FileUtils.ln_s("artifact.#{@checksum}.tar.gz", 390 | "#{$config['general']['app_cache']}/artifacts/artifact.latest.tar.gz", 391 | force: true) 392 | rescue StandardError => e 393 | $logger.fatal 'Something weird went really wrong trying to symlink the latest artifacts' 394 | raise e 395 | end 396 | 397 | $logger.info "New artifact version #{@checksum} ready for pushing" 398 | end 399 | 400 | def unpack 401 | # Unpack the currently selected artifact to the archives directory. 402 | 403 | # An application version must be specified 404 | fail 'Application bug, trying to unpack no artifact' unless defined? @checksum 405 | 406 | # Make sure the files actually exist... 407 | unless File.exist?($config['general']['app_cache'] + "/artifacts/manifest.#{@checksum}.yaml") 408 | $logger.error "The files expected for #{@checksum} do not appear to exist or are not readable" 409 | fail 'Fatal unexpected error' 410 | end 411 | 412 | unless File.exist?($config['general']['app_cache'] + "/artifacts/artifact.#{@checksum}.tar.gz") 413 | $logger.error "The files expected for #{@checksum} do not appear to exist or are not readable" 414 | fail 'Fatal unexpected error' 415 | end 416 | 417 | # Clean up an existing unpacked copy - in *theory* it should be same, but 418 | # a mistake like running out of disk could have left it in an unclean state 419 | # so let's make sure it's gone 420 | clean_unpack 421 | 422 | # Unpack the archive file 423 | FileUtils.mkdir_p($config['general']['app_cache'] + "/artifacts/unpacked.#{@checksum}") 424 | Dir.chdir($config['general']['app_cache'] + "/artifacts/unpacked.#{@checksum}") do 425 | tar = Pupistry::Config.which_tar 426 | $logger.debug "Using tar at #{tar}" 427 | 428 | if system "#{tar} -xzf ../artifact.#{@checksum}.tar.gz" 429 | $logger.debug "Successfully unpacked artifact #{@checksum}" 430 | else 431 | $logger.error "Unable to unpack artifact files to #{Dir.pwd}" 432 | fail 'An unexpected error occured when executing tar' 433 | end 434 | end 435 | end 436 | 437 | def install 438 | # Copy the unpacked artifact into the agent's configured location. Generally all the 439 | # heavy lifting is done by fetch_latest and unpack methods. 440 | 441 | # An application version must be specified 442 | fail 'Application bug, trying to install no artifact' unless defined? @checksum 443 | 444 | # Validate the artifact if GPG is enabled. 445 | if $config['general']['gpg_disable'] == true 446 | $logger.warn 'You have GPG validation *disabled*, whilst not critical it does weaken your security.' 447 | $logger.warn 'Skipping validation step...' 448 | else 449 | 450 | gpgsig = Pupistry::GPG.new @checksum 451 | 452 | unless gpgsig.artifact_verify 453 | $logger.fatal 'The GPG signature could not be validated for the artifact. This could be a bug, a file corruption or a POSSIBLE SECURITY ISSUE such as maliciously modified content.' 454 | fail 'Fatal unexpected error' 455 | end 456 | 457 | end 458 | 459 | # Make sure the artifact has been unpacked 460 | unless Dir.exist?($config['general']['app_cache'] + "/artifacts/unpacked.#{@checksum}") 461 | $logger.error "The unpacked directory expected for #{@checksum} does not appear to exist or is not readable" 462 | fail 'Fatal unexpected error' 463 | end 464 | 465 | # Purge any currently installed files in the directory. See clean_install 466 | # TODO: notes for how this could be improved. 467 | $logger.error 'Installation not proceeding due to issues cleaning/prepping destination dir' unless clean_install 468 | 469 | # Make sure the destination directory exists 470 | unless Dir.exist?($config['agent']['puppetcode']) 471 | $logger.error "The destination path of #{$config['agent']['puppetcode']} does not appear to exist or is not readable" 472 | fail 'Fatal unexpected error' 473 | end 474 | 475 | # Clone unpacked contents to the installation directory 476 | begin 477 | FileUtils.cp_r $config['general']['app_cache'] + "/artifacts/unpacked.#{@checksum}/puppetcode/.", $config['agent']['puppetcode'] 478 | FileUtils.cp $config['general']['app_cache'] + "/artifacts/manifest.#{@checksum}.yaml", $config['agent']['puppetcode'] + '/manifest.pupistry.yaml' 479 | return true 480 | rescue 481 | $logger.fatal "An unexpected error occured when copying the unpacked artifact to #{$config['agent']['puppetcode']}" 482 | raise e 483 | end 484 | end 485 | 486 | def clean_install 487 | # Cleanup the destination installation directory before we unpack the artifact 488 | # into it, otherwise long term we will end up with old deprecated files hanging 489 | # around. 490 | # 491 | # TODO: Do this smarter, we should track what files we drop in, and then remove 492 | # any that weren't touched. Need to avoid rsync and stick with native to make 493 | # support easier for weird/minimilistic distributions. 494 | 495 | if defined? $config['agent']['puppetcode'] # rubocop:disable Style/GuardClause 496 | if $config['agent']['puppetcode'].empty? 497 | $logger.error "You must configure a location for the agent's Puppet code to be deployed to" 498 | return false 499 | else 500 | $logger.debug "Cleaning up #{$config['agent']['puppetcode']} directory" 501 | 502 | if Dir.exist?($config['agent']['puppetcode']) 503 | FileUtils.rm_r Dir.glob($config['agent']['puppetcode'] + '/*'), secure: true 504 | else 505 | FileUtils.mkdir_p $config['agent']['puppetcode'] 506 | FileUtils.chmod(0700, $config['agent']['puppetcode']) 507 | end 508 | 509 | return true 510 | end 511 | end 512 | end 513 | 514 | def clean_unpack 515 | # Cleanup/remove any unpacked archive directories. Requires that the 516 | # checksum be set to the version to be purged. 517 | 518 | fail 'Application bug, trying to unpack no artifact' unless defined? @checksum 519 | 520 | if Dir.exist?($config['general']['app_cache'] + "/artifacts/unpacked.#{@checksum}/") 521 | $logger.debug "Cleaning up #{$config['general']['app_cache']}/artifacts/unpacked.#{@checksum}..." 522 | FileUtils.rm_r $config['general']['app_cache'] + "/artifacts/unpacked.#{@checksum}", secure: true 523 | return true 524 | else 525 | $logger.debug 'Nothing to cleanup (selected artifact is not currently unpacked)' 526 | return true 527 | end 528 | 529 | false 530 | end 531 | end 532 | end 533 | 534 | # vim:shiftwidth=2:tabstop=2:softtabstop=2:expandtab:smartindent 535 | -------------------------------------------------------------------------------- /lib/pupistry/bootstrap.rb: -------------------------------------------------------------------------------- 1 | # rubocop:disable Style/Documentation, Style/GlobalVars 2 | require 'rubygems' 3 | require 'base64' 4 | require 'erubis' 5 | 6 | module Pupistry 7 | # Pupistry::Bootstrap 8 | 9 | class Bootstrap 10 | attr_accessor :template_dir 11 | attr_accessor :contents 12 | 13 | def initialize 14 | # We need to find where the templates are located - either it should be 15 | # in the current working directory, or if we are an installed gem, we 16 | # can try the gem's installed path. 17 | 18 | if Dir.exist?('resources/bootstrap/') 19 | # Use local PWD version first if possible 20 | @template_dir = Dir.pwd 21 | else 22 | # Check for GEM installed location 23 | begin 24 | @template_dir = Gem::Specification.find_by_name('pupistry').gem_dir 25 | rescue Gem::LoadError 26 | $logger.error "Unable to find templates/ directory, doesn't appear we are running from project dir nor as a Gem" 27 | return false 28 | end 29 | end 30 | 31 | @template_dir = @template_dir.chomp('/') + '/resources/bootstrap/' 32 | 33 | if Dir.exist?(@template_dir) 34 | $logger.debug "Using directory #{@template_dir} for bootstrap templates" 35 | else 36 | $logger.error "Unable to find templates dir at #{@template_dir}, unable to proceed." 37 | return false 38 | end 39 | end 40 | 41 | def list 42 | # Simply glob the templates directory and list their names. 43 | $logger.debug 'Finding all available templates' 44 | 45 | Dir.glob("#{@template_dir}/*.erb").each do |file| 46 | puts "- #{File.basename(file, '.erb')}" 47 | end 48 | end 49 | 50 | def build(template, environment) 51 | # Build a template with the configured parameters already to go and save 52 | # into the object, so it can be outputted in the desired format. 53 | 54 | $logger.info "Generating a bootstrap script for #{template} with environment #{environment}" 55 | 56 | unless File.exist?("#{@template_dir}/#{template}.erb") 57 | $logger.error 'The requested template does not exist, unable to build' 58 | return 0 59 | end 60 | 61 | # Assume values we care about 62 | template_values = { 63 | s3_bucket: $config['general']['s3_bucket'], 64 | s3_prefix: $config['general']['s3_prefix'], 65 | gpg_disable: $config['general']['gpg_disable'], 66 | gpg_signing_key: $config['general']['gpg_signing_key'], 67 | puppetcode: $config['agent']['puppetcode'], 68 | access_key_id: $config['agent']['access_key_id'], 69 | secret_access_key: $config['agent']['secret_access_key'], 70 | region: $config['agent']['region'], 71 | proxy_uri: $config['agent']['proxy_uri'], 72 | daemon_frequency: $config['agent']['daemon_frequency'], 73 | daemon_minimal: $config['agent']['daemon_minimal'], 74 | environment: environment 75 | } 76 | 77 | # Generate template using ERB 78 | begin 79 | @contents = Erubis::Eruby.new(File.read("#{@template_dir}/#{template}.erb")).result(template_values) 80 | rescue StandardError => e 81 | $logger.error 'An unexpected error occured when trying to generate the bootstrap template' 82 | raise e 83 | end 84 | end 85 | 86 | def output_array 87 | # Return the output as an array of lines. Useful by other internal 88 | # methods such as Packer templates. 89 | @contents.split(/\n/) 90 | end 91 | 92 | def output_plain 93 | # Do nothing clever, just output the template data. 94 | puts '-- Bootstrap Start --' 95 | puts @contents 96 | puts '-- Bootstrap End --' 97 | end 98 | 99 | def output_base64 100 | # Some providers like AWS can accept the data in Base64 version which is 101 | # smaller and less likely to get messed up by copy and paste or weird 102 | # formatting issues. 103 | puts '-- Bootstrap Start --' 104 | puts Base64.encode64(@contents) 105 | puts '-- Bootstrap End --' 106 | end 107 | end 108 | end 109 | 110 | # vim:shiftwidth=2:tabstop=2:softtabstop=2:expandtab:smartindent 111 | -------------------------------------------------------------------------------- /lib/pupistry/config.rb: -------------------------------------------------------------------------------- 1 | # rubocop:disable Style/Documentation, Style/GlobalVars 2 | require 'rubygems' 3 | require 'fileutils' 4 | require 'tempfile' 5 | require 'yaml' 6 | require 'safe_yaml' 7 | require 'whichr' 8 | 9 | module Pupistry 10 | # Pupistry::Config 11 | # 12 | # Provides loading of configuration. 13 | # 14 | 15 | class Config 16 | def self.load(file) 17 | $logger.debug "Loading configuration file #{file}" 18 | 19 | # Load YAML file with minimum safety/basic checks 20 | unless File.exist?(file) 21 | $logger.fatal 'The configuration file provided does not exist, or cannot be accessed' 22 | exit 0 23 | end 24 | 25 | begin 26 | $config = YAML.load(File.open(file), safe: true, raise_on_unknown_tag: true) 27 | rescue => ex 28 | $logger.fatal 'The supplied file is not a valid YAML configuration file' 29 | $logger.debug ex.message 30 | exit 0 31 | end 32 | 33 | 34 | # Run checks for minimum configuration parameters 35 | # TODO: Is there a smarter way of doing this? Maybe a better config parser? 36 | begin 37 | fail 'Missing general:app_cache' unless defined? $config['general']['app_cache'] 38 | fail 'Missing general:s3_bucket' unless defined? $config['general']['s3_bucket'] 39 | fail 'Missing general:gpg_disable' unless defined? $config['general']['gpg_disable'] 40 | fail 'Missing agent:puppetcode' unless defined? $config['agent']['puppetcode'] 41 | rescue => ex 42 | $logger.fatal 'The supplied configuration files doesn\'t include the minimum expect configuration parameters' 43 | $logger.debug ex.message 44 | exit 0 45 | end 46 | 47 | 48 | 49 | # Make sure cache directory exists, create it otherwise 50 | $config['general']['app_cache'] = File.expand_path($config['general']['app_cache']).chomp('/') 51 | 52 | unless Dir.exist?($config['general']['app_cache']) 53 | begin 54 | FileUtils.mkdir_p($config['general']['app_cache']) 55 | FileUtils.chmod(0700, $config['general']['app_cache']) # Generally only the user running Pupistry should have access 56 | rescue StandardError => e 57 | $logger.fatal "Unable to create cache directory at \"#{$config['general']['app_cache']}\"." 58 | raise e 59 | end 60 | end 61 | 62 | # Write test file to confirm writability 63 | begin 64 | FileUtils.touch($config['general']['app_cache'] + '/testfile') 65 | FileUtils.rm($config['general']['app_cache'] + '/testfile') 66 | rescue StandardError => e 67 | $logger.fatal "Unexpected exception when creating testfile in cache directory at \"#{$config['general']['app_cache']}\", is the directory writable?" 68 | raise e 69 | end 70 | 71 | 72 | # Check if Puppet is available 73 | unless system('puppet --version 2>&1 > /dev/null') 74 | $logger.fatal "Unable to find an installation of Puppet - please make sure Puppet is installed from either OS package or Gem" 75 | exit 0 76 | end 77 | 78 | end 79 | 80 | def self.find_and_load 81 | $logger.debug 'Looking for configuration file in common locations' 82 | 83 | # If the HOME environmental hasn't been set (which can happen when 84 | # running via some cloud user-data/init systems) the app will die 85 | # horribly, we should set a HOME path default. 86 | unless ENV['HOME'] 87 | $logger.warn 'No HOME environmental set, defaulting to /tmp' 88 | ENV['HOME'] = '/tmp' 89 | end 90 | 91 | # Locations in order of preference: 92 | # settings.yaml (current dir) 93 | # ~/.pupistry/settings.yaml 94 | # /etc/pupistry/settings.yaml 95 | 96 | config = '' 97 | local_dir = Dir.pwd 98 | 99 | if File.exist?("#{local_dir}/settings.yaml") 100 | config = "#{local_dir}/settings.yaml" 101 | 102 | elsif File.exist?(File.expand_path '~/.pupistry/settings.yaml') 103 | config = File.expand_path '~/.pupistry/settings.yaml' 104 | 105 | elsif File.exist?('/usr/local/etc/pupistry/settings.yaml') 106 | config = '/usr/local/etc/pupistry/settings.yaml' 107 | 108 | elsif File.exist?('/etc/pupistry/settings.yaml') 109 | config = '/etc/pupistry/settings.yaml' 110 | 111 | else 112 | $logger.error 'No configuration file provided.' 113 | $logger.error 'See pupistry help for information on configuration' 114 | exit 0 115 | end 116 | 117 | load(config) 118 | end 119 | 120 | # Return which tar binary to use. 121 | def self.which_tar 122 | # Try to use GNU tar if present to work around weird issues with some 123 | # versions of BSD tar when using the tar files with GNU tar subsequently. 124 | tar = RubyWhich.new.which('gtar').first || RubyWhich.new.which('gnutar').first || 'tar' 125 | 126 | return tar 127 | end 128 | 129 | end 130 | end 131 | # vim:shiftwidth=2:tabstop=2:softtabstop=2:expandtab:smartindent 132 | -------------------------------------------------------------------------------- /lib/pupistry/gpg.rb: -------------------------------------------------------------------------------- 1 | # rubocop:disable Style/Documentation, Style/GlobalVars 2 | require 'rubygems' 3 | require 'yaml' 4 | require 'safe_yaml' 5 | require 'fileutils' 6 | require 'base64' 7 | 8 | module Pupistry 9 | # Pupistry::GPG 10 | 11 | class GPG 12 | # All the functions needed for manipulating the GPG signatures 13 | attr_accessor :checksum 14 | attr_accessor :signature 15 | 16 | def initialize(checksum) 17 | # Need a checksum to do signing for 18 | if checksum 19 | @checksum = checksum 20 | else 21 | $logger.fatal 'Probable bug, need a checksum provided with GPG validation' 22 | exit 0 23 | end 24 | 25 | # Make sure that we have GPG available 26 | unless system('gpg --version >> /dev/null 2>&1') # rubocop:disable Style/GuardClause 27 | $logger.fatal "'gpg' command is not available, unable to do any signature creation or verification." 28 | exit 0 29 | end 30 | end 31 | 32 | # Sign the artifact and return the signature. Does not validation of the signature. 33 | # 34 | # false Failure 35 | # base64 Encoded signature 36 | # 37 | def artifact_sign 38 | @signature = 'unsigned' 39 | 40 | # Clean up the existing signature file 41 | signature_cleanup 42 | 43 | Dir.chdir("#{$config['general']['app_cache']}/artifacts/") do 44 | # Generate the signature file and pick up the signature data 45 | unless system "gpg --use-agent --detach-sign artifact.#{@checksum}.tar.gz" 46 | $logger.error 'Unable to sign the artifact, an unexpected failure occured. No file uploaded.' 47 | return false 48 | end 49 | 50 | if File.exist?("artifact.#{@checksum}.tar.gz.sig") 51 | $logger.info 'A signature file was successfully generated.' 52 | else 53 | $logger.error 'A signature file was NOT generated.' 54 | return false 55 | end 56 | 57 | # Convert the signature into base64. It's easier to bundle all the 58 | # metadata into a single file and extracting it out when needed, than 59 | # having to keep track of yet-another-file. Because we encode into 60 | # ASCII here, no need to call GPG with --armor either. 61 | 62 | @signature = Base64.encode64(File.read("artifact.#{@checksum}.tar.gz.sig")) 63 | 64 | unless @signature 65 | $logger.error 'An unexpected issue occured and no signature was generated' 66 | return false 67 | end 68 | end 69 | 70 | # Make sure the public key has been uploaded if it hasn't already 71 | pubkey_upload 72 | 73 | @signature 74 | end 75 | 76 | # Verify the signature for a particular artifact. 77 | # 78 | # true Signature is legit 79 | # false Signature is invalid (security issue!) 80 | # 81 | def artifact_verify 82 | Dir.chdir("#{$config['general']['app_cache']}/artifacts/") do 83 | if File.exist?("artifact.#{@checksum}.tar.gz.sig") 84 | $logger.debug 'Signature already extracted on disk, running verify....' 85 | else 86 | $logger.debug 'Extracting signature from manifest data...' 87 | signature_extract 88 | end 89 | 90 | # Verify the signature 91 | pubkey_install unless pubkey_exist? 92 | 93 | output_verify = `gpg --quiet --status-fd 1 --verify artifact.#{@checksum}.tar.gz.sig 2>&1` 94 | 95 | # Cleanup on disk file 96 | signature_cleanup 97 | 98 | # Was it valid? 99 | output_verify.each_line do |line| 100 | if /\[GNUPG:\]\sGOODSIG\s[A-Z0-9]*#{$config["general"]["gpg_signing_key"]}\s/.match(line) 101 | $logger.info "Artifact #{@checksum} has a valid signature belonging to #{$config['general']['gpg_signing_key']}" 102 | return true 103 | end 104 | 105 | if /\[GNUPG:\]\sBADSIG\s/.match(line) 106 | $logger.fatal "Artifact #{@checksum} has AN INVALID GPG SECURITY SIGNATURE and could be CORRUPT or TAMPERED with." 107 | exit 0 108 | end 109 | end 110 | 111 | # Unexpected error 112 | $logger.error 'An unexpected validation issue occured, see below debug information:' 113 | 114 | output_verify.each_line do |line| 115 | $logger.error "GPG: #{line}" 116 | end 117 | end 118 | 119 | # Something went wrong 120 | $logger.fatal "Artifact #{@checksum} COULD NOT BE GPG VALIDATED and could be CORRUPT or TAMPERED with." 121 | exit 0 122 | end 123 | 124 | # Generally we should clean up old signature files before and after using them 125 | # 126 | def signature_cleanup 127 | FileUtils.rm("#{$config['general']['app_cache']}/artifacts/artifact.#{@checksum}.tar.gz.sig", force: true) 128 | end 129 | 130 | # Extract the signature from the manifest file and write it to file in native binary format. 131 | # 132 | # false Unable to extract 133 | # unsigned Manifest shows that the artifact is not signed 134 | # base64 Encoded signature 135 | # 136 | def signature_extract 137 | manifest = YAML.load(File.open($config['general']['app_cache'] + "/artifacts/manifest.#{@checksum}.yaml"), safe: true, raise_on_unknown_tag: true) 138 | 139 | if manifest['gpgsig'] 140 | # We have the base64 version 141 | @signature = manifest['gpgsig'] 142 | 143 | # Decode the base64 and write the signature file 144 | File.write("#{$config['general']['app_cache']}/artifacts/artifact.#{@checksum}.tar.gz.sig", Base64.decode64(@signature)) 145 | 146 | return @signature 147 | else 148 | return false 149 | end 150 | 151 | rescue StandardError => e 152 | $logger.error 'Something unexpected occured when reading the manifest file' 153 | raise e 154 | end 155 | 156 | # Save the signature into the manifest file 157 | # 158 | def signature_save 159 | manifest = YAML.load(File.open($config['general']['app_cache'] + "/artifacts/manifest.#{@checksum}.yaml"), safe: true, raise_on_unknown_tag: true) 160 | manifest['gpgsig'] = @signature 161 | 162 | File.open("#{$config['general']['app_cache']}/artifacts/manifest.#{@checksum}.yaml", 'w') do |fh| 163 | fh.write YAML.dump(manifest) 164 | end 165 | 166 | return true 167 | 168 | rescue StandardError 169 | $logger.error 'Something unexpected occured when updating the manifest file with GPG signature' 170 | return false 171 | end 172 | 173 | # Check if the public key is installed on this machine? 174 | # 175 | def pubkey_exist? 176 | # We prefix with 0x to avoid matching on strings in key names 177 | if system "gpg --status-fd a --list-keys 0x#{$config['general']['gpg_signing_key']} 2>&1 >> /dev/null" 178 | $logger.debug 'Public key exists on this system' 179 | return true 180 | else 181 | $logger.debug 'Public key does not exist on this system' 182 | return false 183 | end 184 | end 185 | 186 | # Extract & upload the public key to the s3 bucket for other users 187 | # 188 | def pubkey_upload 189 | unless File.exist?("#{$config['general']['app_cache']}/artifacts/#{$config['general']['gpg_signing_key']}.publickey") 190 | 191 | # GPG key does not exist locally, we therefore assume it's not in the S3 192 | # bucket either, so we should export out and upload. Technically this may 193 | # result in a few extra uploads (once for any new machine using Pupistry) 194 | # but it doesn't cause any issue and saves me writing more code ;-) 195 | 196 | $logger.info "Exporting GPG key #{$config['general']['gpg_signing_key']} and uploading to S3 bucket..." 197 | 198 | # If it doesn't exist on this machine, then we're a bit stuck! 199 | unless pubkey_exist? 200 | $logger.error "The public key #{$config['general']['gpg_signing_key']} does not exist on this system, so unable to export it out" 201 | return false 202 | end 203 | 204 | # Export out key 205 | unless system "gpg --export --armour 0x#{$config['general']['gpg_signing_key']} > #{$config['general']['app_cache']}/artifacts/#{$config['general']['gpg_signing_key']}.publickey" 206 | $logger.error 'A fault occured when trying to export the GPG key' 207 | return false 208 | end 209 | 210 | # Upload 211 | s3 = Pupistry::StorageAWS.new 'build' 212 | 213 | unless s3.upload "#{$config['general']['app_cache']}/artifacts/#{$config['general']['gpg_signing_key']}.publickey", "#{$config['general']['gpg_signing_key']}.publickey" 214 | $logger.error 'Unable to upload GPG key to S3 bucket' 215 | return false 216 | end 217 | 218 | end 219 | end 220 | 221 | # Install the public key. This is a potential avenue for exploit, if a 222 | # machine is being built for the first time, it has no existing trust of 223 | # the GPG key, other than transit encryption to the S3 bucket. To protect 224 | # against attacks at the bootstrap time, you should pre-load your machine 225 | # images with the public GPG key. 226 | # 227 | # For those users who trade off some security for convienence, we install 228 | # the GPG public key for them direct from the S3 repo. 229 | # 230 | def pubkey_install 231 | $logger.warn "Installing GPG key #{$config['general']['gpg_signing_key']}..." 232 | 233 | s3 = Pupistry::StorageAWS.new 'agent' 234 | 235 | unless s3.download "#{$config['general']['gpg_signing_key']}.publickey", "#{$config['general']['app_cache']}/artifacts/#{$config['general']['gpg_signing_key']}.publickey" 236 | $logger.error 'Unable to download GPG key from S3 bucket, this will prevent validation of signature' 237 | return false 238 | end 239 | 240 | unless system "gpg --import < #{$config['general']['app_cache']}/artifacts/#{$config['general']['gpg_signing_key']}.publickey > /dev/null 2>&1" 241 | $logger.error 'A fault occured when trying to import the GPG key' 242 | return false 243 | end 244 | 245 | rescue StandardError 246 | $logger.error 'Something unexpected occured when installing the GPG public key' 247 | return false 248 | end 249 | end 250 | end 251 | 252 | # vim:shiftwidth=2:tabstop=2:softtabstop=2:expandtab:smartindent 253 | -------------------------------------------------------------------------------- /lib/pupistry/hieracrypt.rb: -------------------------------------------------------------------------------- 1 | # rubocop:disable Style/Documentation, Style/GlobalVars 2 | require 'rubygems' 3 | require 'yaml' 4 | require 'json' 5 | require 'safe_yaml' 6 | require 'fileutils' 7 | require 'base64' 8 | 9 | module Pupistry 10 | # Pupistry::HieraCrypt 11 | 12 | class HieraCrypt 13 | 14 | # As HieraCrypt is an optional extension, we should provide calling code 15 | # an easy way to determine if we're enabled or not. 16 | def self.is_enabled? 17 | begin 18 | if $config['build']['hieracrypt'] == true 19 | $logger.debug 'Hieracrypt is enabled.' 20 | return true 21 | end 22 | rescue => ex 23 | # Nothing todo, fall back. 24 | end 25 | 26 | $logger.debug 'Hieracrypt is disabled.' 27 | return false 28 | end 29 | 30 | 31 | # To encrypt the Hieradata against the certs we have, there's a few things 32 | # that we need to do. 33 | # 34 | # 1. Firstly we need to iterate through all the available environments in 35 | # the app_cache/puppetcode directory and for each one, load the Hiera 36 | # rules. 37 | # 38 | # 2. Secondly (assuming HieraCrypt is even enabled) we must find all the 39 | # node files that contain the cert & fact data. 40 | # 41 | # 3. Apply the rules to the host and determine which files should go into 42 | # the encrypted hieradata file for that host and copy to a dir. 43 | # 44 | # 4. Generate the encrypted HieraCrypt file with the files in it, one per 45 | # each node we have. 46 | # 47 | # 5. Purge the unencrypted hieradata and the working files. 48 | # 49 | # Run after fetch_r10k and before build_artifact 50 | # 51 | def self.encrypt_hieradata 52 | unless is_enabled? 53 | return false 54 | end 55 | 56 | $logger.info "Encrypting Hieradata (HieraCrypt Feature)..." 57 | 58 | 59 | # Key paths to remember inside puppetcode / BRANCH: 60 | # 61 | # hieracrypt/nodes/ Where the various per-host files live. 62 | # hieradata/hiera.yaml The Hiera rules 63 | # hieradata/* Any/all Hiera data. 64 | # 65 | puppetcode = $config['general']['app_cache'] + '/puppetcode' 66 | 67 | 68 | # Run through each environment. 69 | for env in Dir.glob(puppetcode +'/*') 70 | env = File.basename(env) 71 | 72 | if Dir.exists?(puppetcode + '/' + env) 73 | $logger.debug "Processing branch: #{env}" 74 | 75 | Dir.chdir(puppetcode + '/' + env) do 76 | # Directory env exists, check inside it for a hiera.yaml 77 | if File.exists?('hiera.yaml') 78 | $logger.debug 'Found hiera file '+ puppetcode + '/' + env + '/hiera.yaml' 79 | else 80 | $logger.warn "No hiera.yaml could be found for branch #{env}, no logic to encrypt on" 81 | return false 82 | end 83 | 84 | 85 | # Iterate through each node in the environment 86 | unless Dir.exists?('hieradata') 87 | $logger.warn "No hieradata found for branch #{env}, so nothing to encrypt. Skipping." 88 | break 89 | end 90 | 91 | if Dir.exists?('hieracrypt') 92 | $logger.debug 'Found hieracrypt directory' 93 | else 94 | $logger.warn "No hieracrypt/ directory could be found for branch #{env}, no encryption can take place there." 95 | break 96 | end 97 | 98 | unless Dir.exists?('hieracrypt/nodes') 99 | $logger.warn "No hieracrypt/nodes directory could be found for branch #{env}, no encryption can take place there." 100 | break 101 | end 102 | 103 | unless Dir.exists?('hieracrypt/encrypted') 104 | # We place the encrypted data files in here. 105 | Dir.mkdir('hieracrypt/encrypted') 106 | end 107 | 108 | nodes = Dir.glob('hieracrypt/nodes/*') 109 | 110 | if nodes 111 | # Track if we end up with facts referenced in hiera.yaml that are 112 | # not in the Hieracrypt data for nodes. 113 | missing_facts = 0 114 | 115 | for node in nodes 116 | node = File.basename(node) 117 | 118 | $logger.debug "Found node #{node} for environment #{env}, processing now..." 119 | 120 | begin 121 | # We need to load the JSON-based facts that are appended to the 122 | # cert file. However the JSON parser loses it's shit since it 123 | # doesn't like the header of the cert contents, so we need to 124 | # seek past that ourselves. 125 | json_raw = "" 126 | 127 | IO.readlines("hieracrypt/nodes/#{node}").each do |line| 128 | unless json_raw.empty? 129 | # Subsequent Lines 130 | json_raw += line 131 | end 132 | 133 | if /{/.match(line) 134 | # We have found the first {, must be a valid JSON line 135 | json_raw += line 136 | end 137 | end 138 | 139 | # Extract the facts from the json 140 | puppet_facts = JSON.load(json_raw) 141 | 142 | rescue Exception => ex 143 | $logger.fatal "Unable to parse the JSON data for host/node #{node}" 144 | fail 'A fatal error occurred when processing HieraCrypt node data' 145 | end 146 | 147 | 148 | # It's common to use the 'environment' fact in Hiera, however 149 | # it's going to have been exported as null, since it wouldn't 150 | # have been set at time of generation. Hence, if it is there 151 | # and it is null, we should set it to the current environment 152 | # since we know exactly what it will be because we're inside 153 | # the environment :-) 154 | 155 | if defined? puppet_facts['environment'] 156 | if puppet_facts['environment'] == nil 157 | puppet_facts['environment'] = env 158 | end 159 | 160 | if puppet_facts['environment'] == "" 161 | puppet_facts['environment'] = env 162 | end 163 | end 164 | 165 | # Apply the Hiera rules to the directory and get back a list of 166 | # files that would be matched by Hiera. The way we do this, is 167 | # by filling in each line in Hiera and essentially turning them 168 | # into a glob-able (is this even a word?) pattern which allows 169 | # us to determine what files we need to encrypt for this 170 | # particular node. 171 | 172 | # Iterate through the Hiera rules for values 173 | hiera_rules = [] 174 | hiera = YAML.load_file('hiera.yaml', safe: true, raise_on_unknown_tag: true) 175 | 176 | if defined? hiera[':hierarchy'] 177 | if hiera[':hierarchy'].is_a?(Array) 178 | for line in hiera[':hierarchy'] 179 | # Match syntax of %{::some_kinda_fact} 180 | line.scan(/%{::([[:word:]]*)}/) do |match| 181 | # Replace fact variable with actual value 182 | unless puppet_facts.key?(match[0]) 183 | missing_facts += 1 184 | $logger.debug "hiera.yaml references fact #{match[0]} but this fact doesn't exist in #{node}'s hieracrypt/node/#{node} JSON." 185 | $logger.debug "Possibly out of date data, re-run `pupistry hieracrypt --generate` on the node" 186 | else 187 | line = line.sub("%{::#{match[0]}}", puppet_facts[match[0]]) 188 | end 189 | end 190 | 191 | # Add processed line to the rules file 192 | hiera_rules.push(line) 193 | end 194 | else 195 | $logger.error "Use the array format of the hierachy entry in Hiera, string format not supported because why would you?" 196 | end 197 | end 198 | 199 | # We have the rules from Hiera for this machine, let's run 200 | # through them as globs and copy each match to a new location. 201 | begin 202 | FileUtils.rm_r "hieracrypt.#{node}" 203 | rescue Errno::ENOENT 204 | # Normal error if it doesn't exist yet. 205 | end 206 | 207 | FileUtils.mkdir "hieracrypt.#{node}" 208 | 209 | $logger.debug "Copying relevant hiera data files for #{node}..." 210 | 211 | hiera_rules.each do |rule| 212 | for file in Dir.glob("hieradata/#{rule}.*") 213 | if /\/\.\.?$/.match(file) 214 | # If we end up with /. or /.. in the glob, exclude. 215 | $logger.debug " - Excluding invalid file #{file}" 216 | else 217 | $logger.debug " - #{file}" 218 | 219 | file_rel = file.sub("hieradata/", "") 220 | FileUtils.mkdir_p "hieracrypt.#{node}/#{File.dirname(file_rel)}" 221 | FileUtils.cp file, "hieracrypt.#{node}/#{file_rel}" 222 | end 223 | end 224 | end 225 | 226 | 227 | # Generate the encrypted file 228 | tar = Pupistry::Config.which_tar 229 | $logger.debug "Using tar at #{tar}" 230 | 231 | unless system "#{tar} -c -z -f hieracrypt.#{node}.tar.gz hieracrypt.#{node}" 232 | $logger.error 'Unable to create tarball' 233 | fail 'An unexpected error occured when executing tar' 234 | end 235 | 236 | openssl = "openssl smime -encrypt -binary -aes256 -in hieracrypt.#{node}.tar.gz -out hieracrypt/encrypted/#{node}.tar.gz.enc hieracrypt/nodes/#{node}" 237 | $logger.debug "Executing: #{openssl}" 238 | 239 | unless system openssl 240 | $logger.error "Generation of encrypted file failed for node #{node}" 241 | fail 'An unexpected error occured when executing openssl' 242 | end 243 | 244 | # Cleanup Unencrypted 245 | FileUtils.rm_r "hieracrypt.#{node}.tar.gz" 246 | FileUtils.rm_r "hieracrypt.#{node}" 247 | end 248 | 249 | # Alert if we found missing facts 250 | if missing_facts > 0 251 | $logger.warn "Not all the values in hiera.yaml exist in the Hieracrypt data for #{missing_facts} node(s). Run with --verbose for more info" 252 | end 253 | else 254 | $logger.warn "No nodes could be found for branch #{env}, no encryption can take place there." 255 | break 256 | end 257 | 258 | # We don't do the purge of hieradata unencrypted directory here, 259 | # instead we tell the artifact creation process to exclude it from 260 | # the artifact generation if Hieracrypt is enabled. 261 | 262 | end 263 | end 264 | end 265 | 266 | end 267 | 268 | # Find & decrypt the data for this server, if any. This should be run 269 | # ALWAYS regardless of the Hieracrypt parameter, since we don't want people 270 | # to have to worry about rolling it out to clients, we can figure it out 271 | # based on what files do (or don't) exist. 272 | # 273 | # Runs after unpack, but before artifact install. We get the artifact class 274 | # to pass through the location to operate inside of. 275 | # 276 | def self.decrypt_hieradata puppetcode 277 | $logger.debug "Decrypting Hieracrypt..." 278 | 279 | hostname = get_hostname # Facter hostname value 280 | ssh_host_rsa_key = get_ssh_rsa_private_key # We generate the SSL cert using the SSH RSA Host key 281 | 282 | 283 | # Run through each environment. 284 | for env in Dir.glob(puppetcode +'/*') 285 | env = File.basename(env) 286 | 287 | if Dir.exists?(puppetcode + '/' + env) 288 | $logger.debug "Processing branch: #{env}" 289 | 290 | Dir.chdir(puppetcode + '/' + env) do 291 | unless Dir.exists?("hieracrypt/encrypted") 292 | $logger.debug "Environment #{env} is using unencrypted hieradata." 293 | else 294 | $logger.debug "Environment #{env} is using HieraCrypt, searching for host..." 295 | 296 | if File.exists?("hieracrypt/encrypted/#{hostname}.tar.gz.enc") 297 | $logger.info "Found encrypted Hieradata for #{hostname} in #{env} branch" 298 | 299 | # Perform decryption of this host. 300 | openssl = "openssl smime -decrypt -inkey #{ssh_host_rsa_key} < hieracrypt/encrypted/#{hostname}.tar.gz.enc | tar -xz -f -" 301 | 302 | unless system openssl 303 | $logger.error "A fault occured trying to decrypt the data for #{hostname}" 304 | end 305 | 306 | # Move unpacked host-specific Hieradata into final location 307 | FileUtils.mv "hieracrypt.#{hostname}", "hieradata" 308 | else 309 | $logger.error "Unable to find a HieraCrypt package for #{hostname} in branch #{env}, this machine will be missing all Hieradata" 310 | end 311 | end 312 | end 313 | end 314 | end 315 | 316 | end 317 | 318 | 319 | # Fetch the Puppet facts and the x509 cert from the server and export them 320 | # in a combined version for easy cut'n'paste to the puppetcode repo. 321 | def self.generate_nodedata 322 | $logger.info "Generating an export package of cert and facts..." 323 | 324 | # Setup the cache so we can park various files as we work. 325 | cache_dir = $config['general']['app_cache'] +'/hieracrypt' 326 | 327 | unless Dir.exists?(cache_dir) 328 | Dir.mkdir(cache_dir) 329 | end 330 | 331 | # Generate the SSH public cert. 332 | ssh_host_rsa_key = get_ssh_rsa_private_key # We generate the SSL cert using the SSH RSA Host key 333 | cert_days = '36500' # Valid for 100 years 334 | subject_string = '/C=XX/ST=Pupistry/L=Pupistry/O=Pupistry/OU=Pupistry/CN=Pupistry/emailAddress=pupistry@example.com' 335 | 336 | unless File.exists?(ssh_host_rsa_key) 337 | $logger.error "Unable to find ssh_host_rsa_key file at: #{ssh_host_rsa_key}, unable to proceed." 338 | end 339 | 340 | # TODO: Is there a native library we can use for invoking this and is anyone brave enough to face it? For now 341 | # system might be easier. 342 | openssl = 'openssl req -x509 -key '+ ssh_host_rsa_key +' -nodes -days '+ cert_days +' -newkey rsa:2048 -out '+ cache_dir +'/server.pem -subj '+ subject_string 343 | $logger.debug "Executing: #{openssl}" 344 | 345 | unless system openssl 346 | $logger.error "An error occured attempting to execute openssl" 347 | end 348 | 349 | # Grab all the facter values 350 | puppet_facts = facts_for_hiera($config['agent']['puppetcode']) 351 | 352 | # TODO: Hit facter natively via Rubylibs? 353 | unless system 'facter -p -j '+ puppet_facts.join(" ") +' >> '+ cache_dir +'/server.pem 2> /dev/null' 354 | $logger.error "An error occur attempting to execute facter" 355 | end 356 | 357 | # Output the whole file for the user 358 | hostname = get_hostname 359 | puts "The following output should be saved into `hieracrypt/nodes/#{hostname}`:" 360 | puts IO.read(cache_dir +'/server.pem') 361 | 362 | end 363 | 364 | 365 | # Iterate through the puppetcode environments for all hiera.yaml files 366 | # and suck out all the facts that Hiera cares about. We do this since 367 | # we want to selectively return only the facts we need, since it's 368 | # pretty common to have facts exposing stuff that's potentially a bit 369 | # private and unwanted in the puppetcode repo. 370 | # 371 | # Returns 372 | # Array of Facts 373 | 374 | def self.facts_for_hiera(path) 375 | $logger.debug "Searching for facts specified in Hiera rules..." 376 | 377 | puppet_facts = [] 378 | 379 | for env in Dir.entries(path) 380 | if Dir.exists?(path + '/' + env) 381 | # Directory env exists, check inside it for a hiera.yaml 382 | if File.exists?(path + '/' + env + '/hiera.yaml') 383 | $logger.debug 'Found hiera file '+ path + '/' + env + '/hiera.yaml, checking for facts' 384 | 385 | # Iterate through the Hiera rules for values 386 | hiera = YAML.load_file(path + '/' + env + '/hiera.yaml', safe: true, raise_on_unknown_tag: true) 387 | 388 | if defined? hiera[':hierarchy'] 389 | if hiera[':hierarchy'].is_a?(Array) 390 | for line in hiera[':hierarchy'] 391 | # Match syntax of %{::some_kinda_fact} 392 | line.scan(/%{::([[:word:]]*)}/) { |match| 393 | puppet_facts.push(match) unless puppet_facts.include?(match) 394 | } 395 | end 396 | else 397 | $logger.error "Use the array format of the hierachy entry in Hiera, string format not supported because why would you?" 398 | end 399 | end 400 | end 401 | end 402 | end 403 | 404 | if puppet_facts.count == 0 405 | $logger.warn "Couldn't find any facts mentioned in Hiera, possibly missing or very empty/basic hiera.yaml file in puppetcode repo" 406 | else 407 | $logger.debug "Facts specified in Hiera are: "+ puppet_facts.join(", ") 408 | end 409 | 410 | return puppet_facts 411 | end 412 | 413 | 414 | 415 | def self.get_ssh_rsa_private_key 416 | # Currently hard coded 417 | return '/etc/ssh/ssh_host_rsa_key' 418 | end 419 | 420 | def self.get_hostname 421 | # TODO: Ewwww 422 | hostname = `facter hostname` 423 | return hostname.chomp 424 | end 425 | 426 | end 427 | end 428 | 429 | # vim:shiftwidth=2:tabstop=2:softtabstop=2:expandtab:smartindent 430 | -------------------------------------------------------------------------------- /lib/pupistry/packer.rb: -------------------------------------------------------------------------------- 1 | # rubocop:disable Style/Documentation, Style/GlobalVars 2 | require 'rubygems' 3 | require 'erubis' 4 | 5 | module Pupistry 6 | # Pupistry::Packer 7 | 8 | class Packer 9 | attr_accessor :template_dir 10 | attr_accessor :contents 11 | 12 | def initialize 13 | # We need to find where the templates are located - either it should be 14 | # in the current working directory, or if we are an installed gem, we 15 | # can try the gem's installed path. 16 | 17 | if Dir.exist?('resources/packer/') 18 | # Use local PWD version first if possible 19 | @template_dir = Dir.pwd 20 | else 21 | # Check for GEM installed location 22 | begin 23 | @template_dir = Gem::Specification.find_by_name('pupistry').gem_dir 24 | rescue Gem::LoadError 25 | $logger.error "Unable to find packer templates directory, doesn't appear we are running from project dir nor as a Gem" 26 | return false 27 | end 28 | end 29 | 30 | @template_dir = @template_dir.chomp('/') + '/resources/packer/' 31 | 32 | if Dir.exist?(@template_dir) 33 | $logger.debug "Using directory #{@template_dir} for packer templates" 34 | else 35 | $logger.error "Unable to find packer templates dir at #{@template_dir}, unable to proceed." 36 | return false 37 | end 38 | end 39 | 40 | def list 41 | # Simply glob the templates directory and list their names. 42 | $logger.debug 'Finding all available templates' 43 | 44 | Dir.glob("#{@template_dir}/*.erb").each do |file| 45 | puts "- #{File.basename(file, '.json.erb')}" 46 | end 47 | end 48 | 49 | def build(template) 50 | # Build a template with the configured parameters already to go and save 51 | # into the object, so it can be outputted in the desired format. 52 | 53 | $logger.debug "Generating a packer template using #{template}" 54 | 55 | unless File.exist?("#{@template_dir}/#{template}.json.erb") 56 | $logger.error 'The requested template does not exist, unable to build' 57 | return 0 58 | end 59 | 60 | # Extract the OS bootstrap name from the template filename, we can then 61 | # generate the bootstrap commands to be inserted inline into the packer 62 | # configuration. 63 | 64 | matches = template.match(/^\S*_(\S*)$/) 65 | 66 | if matches[1] 67 | $logger.debug "Fetching bootstrap data for #{matches[1]}..." 68 | else 69 | $logger.error 'Unable to parse the packer filename properly' 70 | return 0 71 | end 72 | 73 | bootstrap = Pupistry::Bootstrap.new 74 | unless bootstrap.build matches[1] 75 | $logger.error 'An unexpected error occured when building the bootstrap data to go inside Packer' 76 | end 77 | 78 | # Pass the values we care about to the template 79 | template_values = { 80 | bootstrap_commands: bootstrap.output_array 81 | } 82 | 83 | # Generate template using ERB 84 | begin 85 | @contents = Erubis::Eruby.new(File.read("#{@template_dir}/#{template}.json.erb")).result(template_values) 86 | rescue StandardError => e 87 | $logger.error 'An unexpected error occured when trying to generate the packer template' 88 | raise e 89 | end 90 | end 91 | 92 | def output_plain 93 | # Do nothing clever, just output the template data. 94 | puts '-- Packer Start --' 95 | puts @contents 96 | puts '-- Packer End --' 97 | puts 'Tip: add --file output.json to write out the packer file directly and then run with `packer build output.json`' 98 | end 99 | 100 | def output_file(filename) 101 | # Write the template to the specified file 102 | begin 103 | File.open(filename, 'w') do |f| 104 | f.puts @contents 105 | end 106 | rescue StandardError => e 107 | $logger.error "An unexpected erorr occured when attempting to write the template to #{filename}" 108 | raise e 109 | else 110 | $logger.info "Wrote template into file #{filename} successfully." 111 | end 112 | end 113 | end 114 | end 115 | 116 | # vim:shiftwidth=2:tabstop=2:softtabstop=2:expandtab:smartindent 117 | -------------------------------------------------------------------------------- /lib/pupistry/storage_aws.rb: -------------------------------------------------------------------------------- 1 | # rubocop:disable Style/Documentation, Style/GlobalVars 2 | require 'rubygems' 3 | require 'yaml' 4 | require 'aws-sdk-v1' 5 | 6 | module Pupistry 7 | # Pupistry::StorageAWS 8 | 9 | class StorageAWS 10 | attr_accessor :s3 11 | attr_accessor :bucket 12 | 13 | def initialize(mode) 14 | # mode is either "build" or "agent", depending which we load a different 15 | # set of permissions. Awareness of both is intentional, since we want the 16 | # build machines to known the agent creds so we can generate bootstrap 17 | # template files. 18 | 19 | unless defined? $config['general']['s3_bucket'] 20 | $logger.fatal 'You must set the AWS s3_bucket' 21 | exit 0 22 | end 23 | 24 | # Define AWS configuration 25 | if defined? $config[mode]['access_key_id'] 26 | if $config[mode]['access_key_id'] == '' 27 | $logger.debug 'No AWS IAM credentials specified, defaulting to environmental discovery' 28 | $logger.debug 'If you get weird permissions errors, try setting the credentials explicity in config first.' 29 | else 30 | $logger.debug 'Loading AWS credentials from configuration file' 31 | 32 | AWS.config( 33 | access_key_id: $config[mode]['access_key_id'], 34 | secret_access_key: $config[mode]['secret_access_key'], 35 | region: $config[mode]['region'], 36 | proxy_uri: $config[mode]['proxy_uri'] 37 | ) 38 | end 39 | else 40 | $logger.debug 'No AWS IAM credentials specified, defaulting to environmental discovery' 41 | $logger.debug 'If you get weird permissions errors, try setting the credentials explicity in config first.' 42 | end 43 | 44 | # Setup S3 bucket 45 | if defined? $config['general']['s3_endpoint'] and $config['general']['s3_endpoint'] != nil 46 | $logger.debug 'Connecting to alternative endpoint ' + $config['general']['s3_endpoint'] 47 | @s3 = AWS::S3.new( 48 | s3_endpoint: $config['general']['s3_endpoint'], 49 | s3_force_path_style: true, 50 | ) 51 | else 52 | @s3 = AWS::S3.new 53 | end 54 | @bucket = @s3.buckets[$config[mode]['s3_bucket']] 55 | end 56 | 57 | def upload(src, dest) 58 | $logger.debug "Pushing file #{src} to s3://#{$config['general']['s3_bucket']}/#{$config['general']['s3_prefix']}#{dest}" 59 | 60 | begin 61 | # Generate the object name/key based on the relative file name and path. 62 | s3_obj_name = "#{$config['general']['s3_prefix']}#{dest}" 63 | s3_obj = @s3.buckets[$config['general']['s3_bucket']].objects[s3_obj_name] 64 | 65 | # Perform S3 upload 66 | s3_obj.write(file: src) 67 | 68 | rescue AWS::S3::Errors::NoSuchBucket 69 | $logger.fatal "S3 bucket #{$config['general']['s3_bucket']} does not exist" 70 | exit 0 71 | 72 | rescue AWS::S3::Errors::AccessDenied 73 | $logger.fatal "Access to S3 bucket #{$config['general']['s3_bucket']} denied" 74 | exit 0 75 | 76 | rescue AWS::S3::Errors::PermanentRedirect => e 77 | $logger.error "The wrong endpoint has been specified (or autodetected) for #{$config['general']['s3_bucket']}." 78 | raise e 79 | 80 | rescue AWS::S3::Errors::SignatureDoesNotMatch => e 81 | $logger.error "IAM signature error when accessing #{$config['general']['s3_bucket']}, probably invalid IAM credentials" 82 | raise e 83 | 84 | rescue AWS::S3::Errors::MissingCredentialsError 85 | $logger.error 'AWS credentials not supplied. You must either:' 86 | $logger.error 'a) Specify them in the config file for Pupistry' 87 | $logger.error 'b) Use IAM roles with an EC2 instance.' 88 | $logger.error 'c) Set them in ENV as AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY' 89 | return false 90 | 91 | rescue StandardError => e 92 | raise e 93 | end 94 | end 95 | 96 | def download(src, dest = 'stream') 97 | $logger.debug "Downloading file s3://#{$config['general']['s3_bucket']}/#{$config['general']['s3_prefix']}#{src} to #{dest}" 98 | 99 | begin 100 | # Generate the object name/key based on the relative file name and path. 101 | s3_obj_name = "#{$config['general']['s3_prefix']}#{src}" 102 | s3_obj = @s3.buckets[$config['general']['s3_bucket']].objects[s3_obj_name] 103 | 104 | # Download the file 105 | if dest == 'stream' 106 | # Return the contents rather than writing to disk. We assume stream mode 107 | # if the dest filename was unspecified 108 | return s3_obj.read 109 | else 110 | # Download to an ondisk file 111 | File.open(dest, 'wb') do |file| 112 | s3_obj.read do |chunk| 113 | file.write(chunk) 114 | end 115 | end 116 | end 117 | 118 | rescue AWS::S3::Errors::NoSuchKey 119 | $logger.debug 'No such file exists for download, this is normal at times.' 120 | return false 121 | 122 | rescue AWS::S3::Errors::NoSuchBucket 123 | $logger.fatal "S3 bucket #{$config['general']['s3_bucket']} does not exist" 124 | exit 0 125 | 126 | rescue AWS::S3::Errors::AccessDenied 127 | $logger.fatal "Access to S3 bucket #{$config['general']['s3_bucket']} denied" 128 | exit 0 129 | 130 | rescue AWS::S3::Errors::InvalidObjectState 131 | $logger.warn "Unable to download \"#{src}\", it has been archived off into Glacier and would need to be recovered first." 132 | 133 | # Do we need to restore it? 134 | begin 135 | if s3_obj.restore_in_progress? 136 | $logger.warn "A restore of this file is currently in progress, but can take up to 4 hours - please re-try later." 137 | else 138 | # Not being restored currently, let's file a request. This allows 139 | # us to cater for situations where a bunch of servers need to get 140 | # an old manifest/file, however the fastest solution is to simply 141 | # do a new `pupistry push` from a workstation to upload a new 142 | # manifest file. 143 | if s3_obj.restore(:days => 30) 144 | $logger.warn "Recover request has been issued, this could take up to 4 hours to complete." 145 | $logger.warn "Note that doing a `pupistry push` from the workstation would solve this faster." 146 | end 147 | end 148 | rescue StandardError => e 149 | $logger.error "Glacier restore request for #{src} failed. (#{e.class}), best option is to push the latest manifest from a workstation with `pupistry push`." 150 | end 151 | 152 | return false 153 | 154 | rescue AWS::S3::Errors::PermanentRedirect => e 155 | $logger.error "The wrong endpoint has been specified (or autodetected) for #{$config['general']['s3_bucket']}." 156 | raise e 157 | 158 | rescue AWS::S3::Errors::SignatureDoesNotMatch => e 159 | $logger.error "IAM signature error when accessing #{$config['general']['s3_bucket']}, probably invalid IAM credentials" 160 | raise e 161 | 162 | rescue AWS::S3::Errors::MissingCredentialsError 163 | $logger.error 'AWS credentials not supplied. You must either:' 164 | $logger.error 'a) Specify them in the config file for Pupistry' 165 | $logger.error 'b) Use IAM roles with an EC2 instance.' 166 | $logger.error 'c) Set them in ENV as AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY' 167 | return false 168 | 169 | rescue StandardError => e 170 | raise e 171 | end 172 | end 173 | end 174 | end 175 | # vim:shiftwidth=2:tabstop=2:softtabstop=2:expandtab:smartindent 176 | -------------------------------------------------------------------------------- /lib/pupistry/version.rb: -------------------------------------------------------------------------------- 1 | module Pupistry 2 | VERSION = '2.0.0' 3 | end 4 | -------------------------------------------------------------------------------- /pupistry.gemspec: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | lib = File.expand_path('../lib', __FILE__) 3 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 4 | require 'pupistry/version' 5 | 6 | Gem::Specification.new do |spec| 7 | spec.name = 'pupistry' 8 | spec.version = Pupistry::VERSION # See lib/pupistry/version.rb to change version 9 | spec.date = '2018-05-29' 10 | spec.summary = 'A workflow tool for Puppet Masterless Deployments' 11 | spec.description = 'Provides security, reliability and consistency to Puppet masterless environments' # rubocop:disable Metrics/LineLength 12 | spec.authors = ['Jethro Carr'] 13 | spec.email = 'jethro.carr@jethrocarr.com' 14 | spec.bindir = 'exe' 15 | spec.files = Dir[ 16 | 'exe/*', 17 | 'lib/**/*', 18 | 'resources/**/*', 19 | 'README.md', 20 | 'settings.example.yaml' 21 | ] 22 | spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) } 23 | spec.homepage = 'https://github.com/jethrocarr/pupistry' 24 | spec.license = 'Apache' 25 | 26 | spec.add_development_dependency 'bundler', '~> 1.9' 27 | spec.add_development_dependency 'rake', '~> 10.0' 28 | spec.add_development_dependency 'minitest', '~> 5.6' 29 | spec.add_development_dependency 'simplecov', '~> 0.10' 30 | spec.add_development_dependency 'rubocop' 31 | 32 | spec.add_runtime_dependency 'aws-sdk-v1' 33 | spec.add_runtime_dependency 'thor' 34 | spec.add_runtime_dependency 'whichr' 35 | spec.add_runtime_dependency 'erubis' 36 | spec.add_runtime_dependency 'safe_yaml' 37 | spec.add_runtime_dependency 'rufus-scheduler', '~> 3' 38 | 39 | # Now technically we don't call r10k from this gem, 40 | # instead we call it via system, but we can cheat 41 | # a bit and list it here to get it installed for uspec. 42 | spec.add_runtime_dependency 'r10k' 43 | 44 | # r10k requires Puppet to run, so the logial thing to do would be to 45 | # uncomment the below dependency. But the issue is that generally 46 | # Puppet is installed from system packages and we don't want to screw 47 | # up the environmnent by loading a different version on. 48 | # 49 | # We instead handle this dependency by checking for it at startup and 50 | # throwing an error if we cannot find puppet. 51 | # 52 | # spec.add_runtime_dependency 'puppet' 53 | end 54 | -------------------------------------------------------------------------------- /resources/aws/README_AWS.md: -------------------------------------------------------------------------------- 1 | # AWS Resources 2 | 3 | This directory contains resources for use with AWS and Pupistry 4 | 5 | 6 | ## cfn_pupistry_bucket_and_iam.template 7 | 8 | This is an template that can build an S3 bucket plus two IAM accounts, one for 9 | the Pupistry build host and another for the hosts running Pupistry itself and 10 | needing read access to the bucket. 11 | 12 | It's a perfectly functional stack which is parameterised so you can simply 13 | enter your specific details (like desired bucket name) and it will go and build 14 | a complete setup of the AWS resources needed for using Pupistry that is 15 | suitable for most end users. 16 | 17 | Alternatively, if you have complex requirements, feel free to incorporate the 18 | ideas and examples of this stack into your own design. 19 | 20 | Building the stack (simple): 21 | 22 | aws cloudformation create-stack \ 23 | --capabilities CAPABILITY_IAM \ 24 | --template-body file://cfn_pupistry_bucket_and_iam.template \ 25 | --stack-name pupistry-resources 26 | 27 | 28 | Building the stack and setting specific parameter values 29 | 30 | aws cloudformation create-stack \ 31 | --capabilities CAPABILITY_IAM \ 32 | --template-body file://cfn_pupistry_bucket_and_iam.template \ 33 | --stack-name pupistry-resources \ 34 | --parameters \ 35 | ParameterKey=S3BucketName,ParameterValue=pupistry-example-bucket \ 36 | ParameterKey=S3BucketArchive,ParameterValue=30 \ 37 | ParameterKey=S3BucketPurge,ParameterValue=365 38 | 39 | 40 | 41 | Make sure the stack has finished building/is built: 42 | 43 | aws cloudformation describe-stacks --query "Stacks[*].StackStatus" --stack-name pupistry-resources 44 | 45 | Status should be `COMPLETE`, if it is set to `ROLLBACK` then it has failed to 46 | build. If set to `CREATE_IN_PROGRESS` then you need to give it more time. 47 | 48 | 49 | Fetching details from the stack: 50 | 51 | aws cloudformation describe-stacks --query "Stacks[*].Outputs[*]" --stack-name pupistry-resources 52 | 53 | Deleting the stack: 54 | 55 | aws cloudformation delete-stack --stack-name PupistryResources 56 | 57 | Note that if the S3 bucket is not empty (ie you've used it for Pupistry 58 | artifacts) then it will fail to delete. Make sure you delete all items from 59 | the S3 bucket first, then delete the stack. This is generally considered a 60 | useful safety feature. ;-) 61 | 62 | You can delete all items with: 63 | 64 | aws s3 rm --recursive s3://pupistry-resources-changeme 65 | 66 | 67 | ## Developer Notes 68 | 69 | CloudFormation is an awesome and powerful tool, but it can be annoying to 70 | work with thanks to everything being written in the rather picky JSON format. 71 | 72 | When writing CFN files, you can validate the templates with: 73 | 74 | aws cloudformation validate-template --template-body file://filename.template 75 | 76 | 77 | It can often be easier to debug why stacks failed to build with the AWS web 78 | console due to better UI than reading JSON event output on the CLI. 79 | 80 | 81 | -------------------------------------------------------------------------------- /resources/aws/cfn_pupistry_bucket_and_iam.template: -------------------------------------------------------------------------------- 1 | { 2 | "AWSTemplateFormatVersion" : "2010-09-09", 3 | 4 | "Description" : "Pupistry S3 bucket and IAM users for both read (servers) and write (build workstation) roles. Note that deleting the stack will fail if the bucket is not empty.", 5 | 6 | "Parameters" : { 7 | "S3BucketName": { 8 | "Type": "String", 9 | "Description" : "Globally unique name of the S3 bucket to create", 10 | "Default" : "AWS::StackName" 11 | }, 12 | "S3BucketArchive": { 13 | "Type": "Number", 14 | "Description" : "Archive old artifacts in the S3 bucket to Glacier after specified number of days.", 15 | "Default" : "30" 16 | }, 17 | "S3BucketPurge": { 18 | "Type": "Number", 19 | "Description" : "Permanently delete old artifacts after specified number of days.", 20 | "Default" : "365" 21 | } 22 | 23 | }, 24 | 25 | "Conditions" : { 26 | "UseStackNameForBucket" : { 27 | "Fn::Equals": [ 28 | {"Ref": "S3BucketName"}, 29 | "AWS::StackName" 30 | ] 31 | } 32 | }, 33 | 34 | 35 | "Resources" : { 36 | 37 | "S3Bucket" : { 38 | "Type" : "AWS::S3::Bucket", 39 | "Properties" : { 40 | "BucketName" : { 41 | "Fn::If" : [ 42 | "UseStackNameForBucket", 43 | { "Ref" : "AWS::StackName" }, 44 | { "Ref" : "S3BucketName" } 45 | ] 46 | }, 47 | "AccessControl" : "Private", 48 | "LifecycleConfiguration" : { 49 | "Rules" : [{ 50 | "Status": "Enabled", 51 | "Prefix": "artifact.", 52 | "ExpirationInDays": { "Ref" : "S3BucketPurge" }, 53 | "Transition": { 54 | "StorageClass": "Glacier", 55 | "TransitionInDays": { "Ref" : "S3BucketArchive" } 56 | } 57 | }, 58 | { 59 | "Status": "Enabled", 60 | "Prefix": "manifest.", 61 | "ExpirationInDays": { "Ref" : "S3BucketPurge" } 62 | }] 63 | } 64 | }, 65 | "DeletionPolicy" : "Delete" 66 | }, 67 | 68 | "IAMReadOnly" : { 69 | "Type" : "AWS::IAM::User", 70 | "Properties" : { 71 | "Policies" : [{ 72 | "PolicyName" : "S3BucketReadOnly", 73 | "PolicyDocument" : { 74 | "Statement":[ 75 | { 76 | "Effect":"Allow", 77 | "Action":[ 78 | "s3:ListAllMyBuckets" 79 | ], 80 | "Resource": [{ "Fn::Join" : ["", [ "arn:aws:s3:::", { "Ref" : "S3Bucket" } ] ] }] 81 | }, 82 | { 83 | "Effect":"Allow", 84 | "Action":[ 85 | "s3:ListBucket", 86 | "s3:GetBucketLocation" 87 | ], 88 | "Resource": [{ "Fn::Join" : ["", [ "arn:aws:s3:::", { "Ref" : "S3Bucket" } ] ] }] 89 | }, 90 | { 91 | "Effect":"Allow", 92 | "Action":[ 93 | "s3:GetObject", 94 | "s3:RestoreObject" 95 | ], 96 | "Resource": [{ "Fn::Join" : ["", [ "arn:aws:s3:::", { "Ref" : "S3Bucket" } , "/*" ] ] }] 97 | } 98 | ] 99 | } 100 | }] 101 | } 102 | }, 103 | 104 | "IAMReadOnlyKeys" : { 105 | "Type" : "AWS::IAM::AccessKey", 106 | "Properties" : { 107 | "UserName" : { "Ref": "IAMReadOnly" } 108 | } 109 | }, 110 | 111 | "IAMReadWrite" : { 112 | "Type" : "AWS::IAM::User", 113 | "Properties" : { 114 | "Policies" : [{ 115 | "PolicyName" : "S3BucketReadAndAppend", 116 | "PolicyDocument" : { 117 | "Statement":[ 118 | { 119 | "Effect":"Allow", 120 | "Action":[ 121 | "s3:ListAllMyBuckets" 122 | ], 123 | "Resource": [{ "Fn::Join" : ["", [ "arn:aws:s3:::", { "Ref" : "S3Bucket" } ] ] }] 124 | }, 125 | { 126 | "Effect":"Allow", 127 | "Action":[ 128 | "s3:ListBucket", 129 | "s3:GetBucketLocation" 130 | ], 131 | "Resource": [{ "Fn::Join" : ["", [ "arn:aws:s3:::", { "Ref" : "S3Bucket" } ] ] }] 132 | }, 133 | { 134 | "Effect":"Allow", 135 | "Action":[ 136 | "s3:PutObject", 137 | "s3:GetObject", 138 | "s3:RestoreObject" 139 | ], 140 | "Resource": [{ "Fn::Join" : ["", [ "arn:aws:s3:::", { "Ref" : "S3Bucket" } , "/*" ] ] }] 141 | } 142 | ] 143 | } 144 | }] 145 | } 146 | }, 147 | 148 | "IAMReadWriteKeys" : { 149 | "Type" : "AWS::IAM::AccessKey", 150 | "Properties" : { 151 | "UserName" : { "Ref": "IAMReadWrite" } 152 | } 153 | } 154 | 155 | 156 | }, 157 | 158 | "Outputs" : { 159 | "S3Region" : { 160 | "Value" : { "Ref" : "AWS::Region" }, 161 | "Description" : "Region where the S3 bucket is located." 162 | }, 163 | "S3Bucket" : { 164 | "Value" : { "Ref" : "S3Bucket" }, 165 | "Description" : "Name of the S3 bucket for Pupistry artifacts" 166 | }, 167 | "AgentAccessKeyId" : { 168 | "Value" : { "Ref" : "IAMReadOnlyKeys" }, 169 | "Description" : "AWSAccessKeyId of the read-only IAM user account for use by agents." 170 | }, 171 | "AgentSecretKeyID" : { 172 | "Value" : { "Fn::GetAtt" : ["IAMReadOnlyKeys", "SecretAccessKey"] }, 173 | "Description" : "AWSSecretAccessKey of the read-only IAM user account for use by agents." 174 | }, 175 | "BuildAccessKeyId" : { 176 | "Value" : { "Ref" : "IAMReadWriteKeys" }, 177 | "Description" : "AWSAccessKeyId of the read-write (append-only) IAM user account for use by build workstations." 178 | }, 179 | "BuildSecretKeyID" : { 180 | "Value" : { "Fn::GetAtt" : ["IAMReadWriteKeys", "SecretAccessKey"] }, 181 | "Description" : "AWSSecretAccessKey of the read-write (append-only) IAM user account for use by build workstations." 182 | } 183 | 184 | } 185 | } 186 | -------------------------------------------------------------------------------- /resources/bootstrap/BOOTSTRAP_NOTES.md: -------------------------------------------------------------------------------- 1 | # Bootstrap Scripts 2 | 3 | Additional bootstrap scripts for major platforms are always welcome. Please 4 | submit a pull request for review and if acceptable, will be merged. 5 | 6 | 7 | # Development Guide Lines 8 | 9 | DO: 10 | 11 | * Install Puppet from the most OS-native source possible - either distribution repos, or Puppetlab's repos. 12 | * Install Pupistry from the most OS-native source - either distribution repos, or rubygems. 13 | * Install the latest OS updates for the platform - not all users will want this, but we should provide a good default security example. 14 | * Wrap the user data in a Bash subshell & log all output to syslog - most systems are headless and it's very useful for debug. Also remember to log the commands being run themselves (`#!/bin/bash -x` will do this for you). 15 | * Test the script both in cut & paste into your distro, but also via the user-data field of a major provider like AWS or Digital Ocean. Sometimes interesting bugs show up like user-data being run before networking is ready, or some distributions not defining key environmentals when running user data. 16 | 17 | DON'T: 18 | 19 | * Use third party respositories or download sites, it needs to be stock vendor OS and packages. 20 | * Execute code from third party sites (eg no `wget http://example.com/malware/myscript.sh`) 21 | * Tie user data to any particular cloud provider unless unavoidable for that platform. 22 | * Make the script any more complex than it needs to be. 23 | 24 | 25 | # Examples 26 | 27 | See the `centos-7` or `ubuntu-14.04` templates for examples on how the bootstrap 28 | templates should be written. The `fedora-any` template also shows an example of 29 | dealing with networking not being ready and also how to handle frequently 30 | changing distribution versions. 31 | 32 | 33 | # Life Span 34 | 35 | Any distribution that is EOL and no longer supported by either the distribution 36 | or by Puppetlabs will be subject to removal to keep the bootstrap selection 37 | modern and clean. Pull requests to clean up cruft are accepted. 38 | 39 | -------------------------------------------------------------------------------- /resources/bootstrap/amazon-any.erb: -------------------------------------------------------------------------------- 1 | #!/bin/bash -x 2 | # This bootstrap is specifcially for Amazon's Linux AMIs, if you are using 3 | # other distributions like Ubuntu or CentOS on AWS, use those bootstrap 4 | # templates. 5 | # 6 | # Amazon Linux is based on RHEL, but has a lot more variations that other 7 | # clones like CentOS, such as shipping with multiple versions of Puppet 8 | # and Ruby - which is useful, but can also make life.... interesting. 9 | ( 10 | exec 1> >(logger -s -t user-data) 2>&1 11 | 12 | export PATH=$PATH:/usr/local/bin 13 | 14 | yum update --assumeyes 15 | 16 | # Need to install a modern Ruby as the default to support current generation of 17 | # Pupistry and other common dependencies. 18 | yum install -y ruby24 ruby24-devel rubygems24 19 | alternatives --set ruby /usr/bin/ruby2.4 20 | alternatives --set gem /usr/bin/gem2.4 21 | 22 | yum install --assumeyes puppet3 gcc zlib-devel libxml2-devel patch gnupg2 23 | 24 | # Not sure why this doesn't get pulled down properly, maybe it's core and 25 | # Amazon didn't package it properly? Need it for Thor which is used by Pupistry 26 | gem install io-console --no-ri --no-rdoc 27 | 28 | gem install pupistry --no-ri --no-rdoc 29 | mkdir -p /etc/pupistry 30 | mkdir -p <%= puppetcode %> 31 | cat > /etc/pupistry/settings.yaml << "EOF" 32 | general: 33 | app_cache: ~/.pupistry/cache 34 | s3_bucket: <%= s3_bucket %> 35 | s3_prefix: <%= s3_prefix %> 36 | gpg_disable: <%= gpg_disable %> 37 | gpg_signing_key: <%= gpg_signing_key %> 38 | agent: 39 | puppetcode: <%= puppetcode %> 40 | access_key_id: <%= access_key_id %> 41 | secret_access_key: <%= secret_access_key %> 42 | region: <%= region %> 43 | proxy_uri: <%= proxy_uri %> 44 | daemon_frequency: <%= daemon_frequency %> 45 | daemon_minimal: <%= daemon_minimal %> 46 | environment: <%= environment %> 47 | EOF 48 | chmod 700 /etc/pupistry/settings.yaml 49 | chmod 700 <%= puppetcode %> 50 | pupistry apply --verbose 51 | 52 | ) 53 | -------------------------------------------------------------------------------- /resources/bootstrap/centos-7.erb: -------------------------------------------------------------------------------- 1 | #!/bin/bash -x 2 | # Bootstrap for CentOS 7 and maybe other EL-derived platforms. 3 | # 4 | # Note: Amusingly doesn't actually work on RHEL itself, since ruby-devel 5 | # does not seem to exist on it :-/ If you actually care about RHEL 6 | # itself, I'll happily accept a pull request that does whatever is 7 | # needed to fix ruby-devel on RHEL. 8 | # 9 | ( 10 | exec 1> >(logger -s -t user-data) 2>&1 11 | 12 | rpm -ivh https://yum.puppetlabs.com/puppetlabs-release-el-7.noarch.rpm 13 | 14 | yum update --assumeyes 15 | yum install --assumeyes puppet ruby-devel rubygems gcc zlib-devel libxml2-devel patch gnupg2 16 | 17 | # Pinned to old (and possibly insecure?) versions due to old version of Ruby being shipped 18 | gem install nokogiri --version 1.6.8.1 19 | gem install pupistry --no-ri --no-rdoc --version 1.5.0 20 | 21 | mkdir -p /etc/pupistry 22 | mkdir -p <%= puppetcode %> 23 | cat > /etc/pupistry/settings.yaml << "EOF" 24 | general: 25 | app_cache: ~/.pupistry/cache 26 | s3_bucket: <%= s3_bucket %> 27 | s3_prefix: <%= s3_prefix %> 28 | gpg_disable: <%= gpg_disable %> 29 | gpg_signing_key: <%= gpg_signing_key %> 30 | agent: 31 | puppetcode: <%= puppetcode %> 32 | access_key_id: <%= access_key_id %> 33 | secret_access_key: <%= secret_access_key %> 34 | region: <%= region %> 35 | proxy_uri: <%= proxy_uri %> 36 | daemon_frequency: <%= daemon_frequency %> 37 | daemon_minimal: <%= daemon_minimal %> 38 | environment: <%= environment %> 39 | EOF 40 | chmod 700 /etc/pupistry/settings.yaml 41 | chmod 700 <%= puppetcode %> 42 | pupistry apply --verbose 43 | 44 | ) 45 | -------------------------------------------------------------------------------- /resources/bootstrap/debian-8.erb: -------------------------------------------------------------------------------- 1 | #!/bin/bash -x 2 | # Bootstrap for Debian 8 stable (Jessie) 3 | # It will *probably* work with other Debian versions supported by Puppetlabs. 4 | # It *might* work with other Debian or Ubuntu derived systems. 5 | ( 6 | exec 1> >(logger -s -t user-data) 2>&1 7 | 8 | wget -O /tmp/puppetlabs-release.deb https://apt.puppetlabs.com/puppetlabs-release-`lsb_release -sc`.deb 9 | dpkg -i /tmp/puppetlabs-release.deb 10 | 11 | export DEBIAN_FRONTEND=noninteractive 12 | 13 | apt-get update 14 | apt-get -y upgrade 15 | 16 | apt-get install -y puppet ruby ruby-dev zlib1g-dev libxml2-dev gcc make patch gnupg2 17 | 18 | gem install pupistry --no-ri --no-rdoc 19 | mkdir -p /etc/pupistry 20 | mkdir -p <%= puppetcode %> 21 | cat > /etc/pupistry/settings.yaml << "EOF" 22 | general: 23 | app_cache: ~/.pupistry/cache 24 | s3_bucket: <%= s3_bucket %> 25 | s3_prefix: <%= s3_prefix %> 26 | gpg_disable: <%= gpg_disable %> 27 | gpg_signing_key: <%= gpg_signing_key %> 28 | agent: 29 | puppetcode: <%= puppetcode %> 30 | access_key_id: <%= access_key_id %> 31 | secret_access_key: <%= secret_access_key %> 32 | region: <%= region %> 33 | proxy_uri: <%= proxy_uri %> 34 | daemon_frequency: <%= daemon_frequency %> 35 | daemon_minimal: <%= daemon_minimal %> 36 | environment: <%= environment %> 37 | EOF 38 | chmod 700 /etc/pupistry/settings.yaml 39 | chmod 700 <%= puppetcode %> 40 | pupistry apply --verbose 41 | 42 | ) 43 | -------------------------------------------------------------------------------- /resources/bootstrap/debian-9.erb: -------------------------------------------------------------------------------- 1 | #!/bin/bash -x 2 | # Bootstrap for Debian 9 stable (stretch) 3 | # Uses distribution-supplied Puppet version (4.8.x) 4 | ( 5 | exec 1> >(logger -s -t user-data) 2>&1 6 | 7 | export DEBIAN_FRONTEND=noninteractive 8 | 9 | apt-get update 10 | apt-get -y upgrade 11 | 12 | apt-get install -y puppet ruby ruby-dev zlib1g-dev libxml2-dev gcc make patch gnupg2 13 | 14 | gem install pupistry --no-ri --no-rdoc 15 | mkdir -p /etc/pupistry 16 | mkdir -p <%= puppetcode %> 17 | cat > /etc/pupistry/settings.yaml << "EOF" 18 | general: 19 | app_cache: ~/.pupistry/cache 20 | s3_bucket: <%= s3_bucket %> 21 | s3_prefix: <%= s3_prefix %> 22 | gpg_disable: <%= gpg_disable %> 23 | gpg_signing_key: <%= gpg_signing_key %> 24 | agent: 25 | puppetcode: <%= puppetcode %> 26 | access_key_id: <%= access_key_id %> 27 | secret_access_key: <%= secret_access_key %> 28 | region: <%= region %> 29 | proxy_uri: <%= proxy_uri %> 30 | daemon_frequency: <%= daemon_frequency %> 31 | daemon_minimal: <%= daemon_minimal %> 32 | environment: <%= environment %> 33 | EOF 34 | chmod 700 /etc/pupistry/settings.yaml 35 | chmod 700 <%= puppetcode %> 36 | pupistry apply --verbose 37 | 38 | ) 39 | -------------------------------------------------------------------------------- /resources/bootstrap/fedora-any.erb: -------------------------------------------------------------------------------- 1 | #!/bin/bash -x 2 | # Bootstrap for Fedora, generally made to be compatible with any version to 3 | # keep up with the rapid rate of Fedora releases. We don't bother trying to 4 | # support any version of Fedora older than the current release due to the 5 | # 6 month EOL. 6 | ( 7 | # No need for logger with Fedora, cloud-init logs all the user-data output. 8 | 9 | # Sometimes Fedora runs user-data before networking is ready, so we should 10 | # make sure the network is ready before starting to try and downlod stuff! 11 | t=300; c=0; r=0; until ping -c 1 www.google.com >/dev/null 2>&1 || ((++c >= t)); do r=$?; echo "Waiting for network... ($r)"; done 12 | 13 | yum update --assumeyes 14 | yum install --assumeyes puppet ruby-devel rubygems gcc zlib-devel libxml2-devel libxslt-devel patch gnupg redhat-rpm-config make 15 | 16 | # Nokogiri system package seems... iffy 17 | gem install nokogiri -- --use-system-libraries 18 | 19 | gem install pupistry --no-ri --no-rdoc 20 | mkdir -p /etc/pupistry 21 | mkdir -p <%= puppetcode %> 22 | cat > /etc/pupistry/settings.yaml << "EOF" 23 | general: 24 | app_cache: ~/.pupistry/cache 25 | s3_bucket: <%= s3_bucket %> 26 | s3_prefix: <%= s3_prefix %> 27 | gpg_disable: <%= gpg_disable %> 28 | gpg_signing_key: <%= gpg_signing_key %> 29 | agent: 30 | puppetcode: <%= puppetcode %> 31 | access_key_id: <%= access_key_id %> 32 | secret_access_key: <%= secret_access_key %> 33 | region: <%= region %> 34 | proxy_uri: <%= proxy_uri %> 35 | daemon_frequency: <%= daemon_frequency %> 36 | daemon_minimal: <%= daemon_minimal %> 37 | environment: <%= environment %> 38 | EOF 39 | chmod 700 /etc/pupistry/settings.yaml 40 | chmod 700 <%= puppetcode %> 41 | pupistry apply --verbose 42 | 43 | ) 44 | -------------------------------------------------------------------------------- /resources/bootstrap/freebsd-10.erb: -------------------------------------------------------------------------------- 1 | #!/bin/tcsh -x 2 | # This bootstrap is for FreeBSD 10.x which has most of the same principals of 3 | # Linux, but we have had to make some variations to account for tcsh weirdness 4 | # vs the general behavior we expect from bash on Linux distributions 5 | 6 | # Known Issues: 7 | # * AWS and Digital Ocean issues: 8 | # http://www.jethrocarr.com/2015/04/19/freebsd-in-the-cloud/ 9 | # * We use Puppet 4 from Ports to avoid limitations in older Puppet 3 release: 10 | # https://www.jethrocarr.com/2015/04/22/puppet-3-and-4-on-freebsd/ 11 | # * tcsh makes capturing all the output to syslog difficult, so we don't do it. 12 | # * We can't rely on Bash, since it's not available in FreeBSD by default. 13 | # 14 | 15 | setenv PATH /sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin 16 | 17 | env ASSUME_ALWAYS_YES=YES pkg bootstrap 18 | env ASSUME_ALWAYS_YES=YES pkg upgrade --yes 19 | env ASSUME_ALWAYS_YES=YES pkg install --yes ruby devel/ruby-gems gnupg 20 | 21 | portsnap fetch 22 | portsnap extract 23 | 24 | cd /usr/ports/ports-mgmt/pkg 25 | make reinstall BATCH=yes 26 | 27 | cd /usr/ports/sysutils/puppet4 28 | make install BATCH=yes 29 | 30 | /usr/local/bin/gem install pupistry --no-ri --no-rdoc 31 | mkdir -p /usr/local/etc/pupistry 32 | mkdir -p /usr/local/etc/puppetlabs/code/environments 33 | cat > /usr/local/etc/pupistry/settings.yaml << EOF 34 | general: 35 | app_cache: ~/.pupistry/cache 36 | s3_bucket: <%= s3_bucket %> 37 | s3_prefix: <%= s3_prefix %> 38 | gpg_disable: <%= gpg_disable %> 39 | gpg_signing_key: <%= gpg_signing_key %> 40 | agent: 41 | puppetcode: /usr/local/etc/puppetlabs/code/environments 42 | access_key_id: <%= access_key_id %> 43 | secret_access_key: <%= secret_access_key %> 44 | region: <%= region %> 45 | proxy_uri: <%= proxy_uri %> 46 | daemon_frequency: <%= daemon_frequency %> 47 | daemon_minimal: <%= daemon_minimal %> 48 | environment: <%= environment %> 49 | EOF 50 | chmod 700 /usr/local/etc/pupistry 51 | chmod 700 /usr/local/etc/puppetlabs/code/environments 52 | /usr/local/bin/pupistry apply --verbose 53 | 54 | -------------------------------------------------------------------------------- /resources/bootstrap/openbsd-6.0.erb: -------------------------------------------------------------------------------- 1 | #!/bin/ksh 2 | 3 | echo \ 4 | 'installpath = http://YOURMIRRORHERE/pub/OpenBSD/%c/packages/%a/' \ 5 | > /etc/pkg.conf 6 | 7 | # need iconv for nokogiri gem build 8 | pkg_add ruby-2.3.1p2 libiconv 9 | ln -sf /usr/local/bin/ruby23 /usr/local/bin/ruby 10 | ln -sf /usr/local/bin/erb23 /usr/local/bin/erb 11 | ln -sf /usr/local/bin/irb23 /usr/local/bin/irb 12 | ln -sf /usr/local/bin/rdoc23 /usr/local/bin/rdoc 13 | ln -sf /usr/local/bin/ri23 /usr/local/bin/ri 14 | ln -sf /usr/local/bin/rake23 /usr/local/bin/rake 15 | ln -sf /usr/local/bin/gem23 /usr/local/bin/gem 16 | 17 | # modify RubyGems defaults so gem executables don't all get names 18 | # like thor23 and puppet23 and pupistry23 because frankly this 19 | # sucks and the alternative (moar symlinks) sucks even more 20 | osdefaults_path=/usr/local/lib/ruby/2.3/rubygems/defaults 21 | mkdir -p $osdefaults_path 22 | cat > $osdefaults_path/operating_system.rb << "OSDEFAULTSRB" 23 | module Gem 24 | def self.default_exec_format 25 | '%s' 26 | end 27 | end 28 | OSDEFAULTSRB 29 | 30 | gem install puppet pupistry --no-ri --no-rdoc 31 | 32 | mkdir -p /etc/pupistry 33 | mkdir -p <%= puppetcode %> 34 | cat > /etc/pupistry/settings.yaml << "EOF" 35 | general: 36 | app_cache: ~/.pupistry/cache 37 | s3_bucket: <%= s3_bucket %> 38 | s3_prefix: <%= s3_prefix %> 39 | gpg_disable: <%= gpg_disable %> 40 | gpg_signing_key: <%= gpg_signing_key %> 41 | agent: 42 | puppetcode: <%= puppetcode %> 43 | access_key_id: <%= access_key_id %> 44 | secret_access_key: <%= secret_access_key %> 45 | region: <%= region %> 46 | proxy_uri: <%= proxy_uri %> 47 | daemon_frequency: <%= daemon_frequency %> 48 | daemon_minimal: <%= daemon_minimal %> 49 | environment: <%= environment %> 50 | EOF 51 | chmod 700 /etc/pupistry/settings.yaml 52 | chmod 700 <%= puppetcode %> 53 | pupistry apply --verbose 54 | -------------------------------------------------------------------------------- /resources/bootstrap/ubuntu-14.04.erb: -------------------------------------------------------------------------------- 1 | #!/bin/bash -x 2 | # Bootstrap for Ubuntu 14.04 LTS (Trusty) 3 | # It will *probably* work with other Ubuntu versions supported by Puppetlabs. 4 | # It *might* work with other Ubuntu or Debian derived systems. 5 | ( 6 | exec 1> >(logger -s -t user-data) 2>&1 7 | 8 | wget -O /tmp/puppetlabs-release.deb https://apt.puppetlabs.com/puppetlabs-release-`lsb_release -sc`.deb 9 | dpkg -i /tmp/puppetlabs-release.deb 10 | 11 | export DEBIAN_FRONTEND=noninteractive 12 | 13 | apt-get update 14 | apt-get -y upgrade 15 | 16 | apt-get install -y puppet ruby ruby-dev zlib1g-dev libxml2-dev gcc make patch gnupg2 17 | 18 | # Pinned to old (and possibly insecure?) versions due to old version of Ruby being shipped 19 | gem install nokogiri --version 1.6.8.1 20 | gem install pupistry --no-ri --no-rdoc --version 1.5.0 21 | 22 | 23 | mkdir -p /etc/pupistry 24 | mkdir -p <%= puppetcode %> 25 | cat > /etc/pupistry/settings.yaml << "EOF" 26 | general: 27 | app_cache: ~/.pupistry/cache 28 | s3_bucket: <%= s3_bucket %> 29 | s3_prefix: <%= s3_prefix %> 30 | gpg_disable: <%= gpg_disable %> 31 | gpg_signing_key: <%= gpg_signing_key %> 32 | agent: 33 | puppetcode: <%= puppetcode %> 34 | access_key_id: <%= access_key_id %> 35 | secret_access_key: <%= secret_access_key %> 36 | region: <%= region %> 37 | proxy_uri: <%= proxy_uri %> 38 | daemon_frequency: <%= daemon_frequency %> 39 | daemon_minimal: <%= daemon_minimal %> 40 | environment: <%= environment %> 41 | EOF 42 | chmod 700 /etc/pupistry/settings.yaml 43 | chmod 700 <%= puppetcode %> 44 | pupistry apply --verbose 45 | 46 | ) 47 | -------------------------------------------------------------------------------- /resources/bootstrap/ubuntu-16.04-puppet4.erb: -------------------------------------------------------------------------------- 1 | #!/bin/bash -x 2 | # Bootstrap for Ubuntu 16.04 LTS (Xenial) 3 | # This version of the bootstrap file uses the upstream Puppet 4 series from 4 | # Puppetlabs, which differs from Puppet 3.8 series supplied with Ubuntu and 5 | # may break environments that are not prepared for Puppet 4. 6 | ( 7 | exec 1> >(logger -s -t user-data) 2>&1 8 | 9 | wget -O /tmp/puppetlabs-release.deb https://apt.puppetlabs.com/puppetlabs-release-pc1-`lsb_release -sc`.deb 10 | dpkg -i /tmp/puppetlabs-release.deb 11 | 12 | export DEBIAN_FRONTEND=noninteractive 13 | 14 | apt-get update 15 | apt-get -y upgrade 16 | 17 | apt-get install -y puppet-agent ruby ruby-dev zlib1g-dev libxml2-dev gcc make patch gnupg2 18 | 19 | update-alternatives --install /usr/bin/puppet puppet /opt/puppetlabs/bin/puppet 1 20 | update-alternatives --install /usr/bin/facter facter /opt/puppetlabs/bin/facter 1 21 | update-alternatives --install /usr/bin/hiera hiera /opt/puppetlabs/bin/hiera 1 22 | update-alternatives --install /usr/bin/mco mco /opt/puppetlabs/bin/mco 1 23 | 24 | gem install pupistry --no-ri --no-rdoc 25 | mkdir -p /etc/pupistry 26 | mkdir -p <%= puppetcode %> 27 | cat > /etc/pupistry/settings.yaml << "EOF" 28 | general: 29 | app_cache: ~/.pupistry/cache 30 | s3_bucket: <%= s3_bucket %> 31 | s3_prefix: <%= s3_prefix %> 32 | gpg_disable: <%= gpg_disable %> 33 | gpg_signing_key: <%= gpg_signing_key %> 34 | agent: 35 | puppetcode: <%= puppetcode %> 36 | access_key_id: <%= access_key_id %> 37 | secret_access_key: <%= secret_access_key %> 38 | region: <%= region %> 39 | proxy_uri: <%= proxy_uri %> 40 | daemon_frequency: <%= daemon_frequency %> 41 | daemon_minimal: <%= daemon_minimal %> 42 | environment: <%= environment %> 43 | EOF 44 | chmod 700 /etc/pupistry/settings.yaml 45 | chmod 700 <%= puppetcode %> 46 | pupistry apply --verbose 47 | 48 | ) 49 | -------------------------------------------------------------------------------- /resources/bootstrap/ubuntu-16.04.erb: -------------------------------------------------------------------------------- 1 | #!/bin/bash -x 2 | # Bootstrap for Ubuntu 16.04 LTS (Xenial) 3 | # It will *probably* work with other Ubuntu versions supported by Puppetlabs. 4 | # It *might* work with other Ubuntu or Debian derived systems. 5 | ( 6 | exec 1> >(logger -s -t user-data) 2>&1 7 | 8 | export DEBIAN_FRONTEND=noninteractive 9 | 10 | apt-get update 11 | apt-get -y upgrade 12 | 13 | apt-get install -y puppet ruby ruby-dev zlib1g-dev libxml2-dev gcc make patch gnupg2 14 | 15 | gem install pupistry --no-ri --no-rdoc 16 | mkdir -p /etc/pupistry 17 | mkdir -p <%= puppetcode %> 18 | cat > /etc/pupistry/settings.yaml << "EOF" 19 | general: 20 | app_cache: ~/.pupistry/cache 21 | s3_bucket: <%= s3_bucket %> 22 | s3_prefix: <%= s3_prefix %> 23 | gpg_disable: <%= gpg_disable %> 24 | gpg_signing_key: <%= gpg_signing_key %> 25 | agent: 26 | puppetcode: <%= puppetcode %> 27 | access_key_id: <%= access_key_id %> 28 | secret_access_key: <%= secret_access_key %> 29 | region: <%= region %> 30 | proxy_uri: <%= proxy_uri %> 31 | daemon_frequency: <%= daemon_frequency %> 32 | daemon_minimal: <%= daemon_minimal %> 33 | environment: <%= environment %> 34 | EOF 35 | chmod 700 /etc/pupistry/settings.yaml 36 | chmod 700 <%= puppetcode %> 37 | pupistry apply --verbose 38 | 39 | ) 40 | -------------------------------------------------------------------------------- /resources/packer/PACKER_NOTES.md: -------------------------------------------------------------------------------- 1 | # Packer Templates 2 | 3 | This directory contains templates for use with Packer (https://www.packer.io/). 4 | 5 | It can be very useful to use Packer with Pupistry, since it allows you to 6 | create your own image with Puppet, Pupistry and updates already loaded which 7 | is very useful when doing autoscaling and you need fast, consistent startup 8 | times. 9 | 10 | The packer templates provided will build an image which has Pupistry installed 11 | and will apply any manifests that match hostname of `packer`. This should give 12 | you a good general purpose image, but if you want to autoscale a particular app 13 | you may wish to build packer images using specific hostnames to match your 14 | Puppet manifests 15 | 16 | Additional packer templates for major platforms are always welcome. Please 17 | submit a pull request for review and if acceptable, will be merged. 18 | 19 | 20 | # Usage 21 | 22 | Refer to the main application `README.md` file for usage information. 23 | 24 | 25 | # Development Notes 26 | 27 | The filenames of the templates must be in the format of 28 | `PLATFORM_OPERATINGSYSTEM.json.erb`, this is intentional since `OPERATINGSYSTEM` 29 | then matches one of the OSes in the bootstrap directory and we can 30 | automatically populate the inline shell commands. 31 | 32 | When debugging broken packer template runs, add `-debug` to the build command 33 | to have control over stepping through the build process. This will give you 34 | the ability to log into the instance before it gets terminated to do any 35 | debugging on the system if needed. 36 | 37 | 38 | # Examples 39 | 40 | See the `aws_amazon-any.json.erb` template for an example on how the templates 41 | should be written for AWS. 42 | 43 | 44 | # Life Span 45 | 46 | Any distribution that is EOL and no longer supported by either the distribution 47 | or by Puppetlabs will be subject to removal to keep the bootstrap selection 48 | modern and clean. Pull requests to clean up cruft are accepted. 49 | 50 | -------------------------------------------------------------------------------- /resources/packer/aws_amazon-any.json.erb: -------------------------------------------------------------------------------- 1 | { 2 | "variables": { 3 | "aws_access_key": "", 4 | "aws_secret_key": "", 5 | "aws_ami": "ami-fd9cecc7", 6 | "aws_region": "ap-southeast-2", 7 | "aws_ami_name": "pupistry aws_amazon-any {{isotime \"2006-01-02\"}}", 8 | "aws_vpc_id": null, 9 | "aws_subnet_id": null, 10 | "hostname": "packer" 11 | }, 12 | "builders": [{ 13 | "type": "amazon-ebs", 14 | "access_key": "{{user `aws_access_key`}}", 15 | "secret_key": "{{user `aws_secret_key`}}", 16 | "region": "{{user `aws_region`}}", 17 | "source_ami": "{{user `aws_ami`}}", 18 | "instance_type": "t2.micro", 19 | "ssh_username": "ec2-user", 20 | "vpc_id": "{{user `aws_vpc_id`}}", 21 | "subnet_id": "{{user `aws_subnet_id`}}", 22 | "ami_name": "{{user `aws_ami_name`}}" 23 | }], 24 | "provisioners": [{ 25 | "type": "shell", 26 | "inline": ["sudo hostname {{user `hostname`}}"] 27 | }, 28 | { 29 | "type": "shell", 30 | "inline_shebang": "/bin/bash -x", 31 | "execute_command": "chmod +x {{ .Path }}; {{ .Vars }} sudo '{{ .Path }}'", 32 | "inline": <%= bootstrap_commands %> 33 | }, 34 | { 35 | "type": "shell", 36 | "inline": ["sudo rm -rf /tmp/*"] 37 | }] 38 | } 39 | -------------------------------------------------------------------------------- /resources/packer/aws_freebsd-10.json.erb: -------------------------------------------------------------------------------- 1 | { 2 | "variables": { 3 | "aws_access_key": "", 4 | "aws_secret_key": "", 5 | "aws_ami": "ami-193f5123", 6 | "aws_region": "ap-southeast-2", 7 | "aws_ami_name": "pupistry aws_freebsd-10 {{isotime \"2006-01-02\"}}", 8 | "aws_vpc_id": null, 9 | "aws_subnet_id": null, 10 | "hostname": "packer" 11 | }, 12 | "builders": [{ 13 | "type": "amazon-ebs", 14 | "access_key": "{{user `aws_access_key`}}", 15 | "secret_key": "{{user `aws_secret_key`}}", 16 | "region": "{{user `aws_region`}}", 17 | "source_ami": "{{user `aws_ami`}}", 18 | "instance_type": "t2.micro", 19 | "ssh_username": "ec2-user", 20 | "ssh_timeout": "15m", 21 | "vpc_id": "{{user `aws_vpc_id`}}", 22 | "subnet_id": "{{user `aws_subnet_id`}}", 23 | "ami_name": "{{user `aws_ami_name`}}" 24 | }], 25 | "provisioners": [{ 26 | "type": "shell", 27 | "inline": ["su -m root -c 'hostname {{user `hostname`}}'"] 28 | }, 29 | { 30 | "type": "shell", 31 | "inline_shebang": "/bin/tcsh", 32 | "execute_command": "chmod +x {{ .Path }}; {{ .Vars }} su -m root -c '/bin/tcsh {{ .Path }}'", 33 | "inline": <%= bootstrap_commands %> 34 | }, 35 | { 36 | "type": "shell", 37 | "inline": ["su -m root -c 'rm -rf /tmp/*'"] 38 | }] 39 | } 40 | -------------------------------------------------------------------------------- /resources/packer/aws_ubuntu-14.04.json.erb: -------------------------------------------------------------------------------- 1 | { 2 | "variables": { 3 | "aws_access_key": "", 4 | "aws_secret_key": "", 5 | "aws_ami": "ami-19c8b123", 6 | "aws_region": "ap-southeast-2", 7 | "aws_ami_name": "pupistry aws_ubuntu-14.04 {{isotime \"2006-01-02\"}}", 8 | "aws_vpc_id": null, 9 | "aws_subnet_id": null, 10 | "hostname": "packer" 11 | }, 12 | "builders": [{ 13 | "type": "amazon-ebs", 14 | "access_key": "{{user `aws_access_key`}}", 15 | "secret_key": "{{user `aws_secret_key`}}", 16 | "region": "{{user `aws_region`}}", 17 | "source_ami": "{{user `aws_ami`}}", 18 | "instance_type": "t2.micro", 19 | "ssh_username": "ubuntu", 20 | "vpc_id": "{{user `aws_vpc_id`}}", 21 | "subnet_id": "{{user `aws_subnet_id`}}", 22 | "ami_name": "{{user `aws_ami_name`}}" 23 | }], 24 | "provisioners": [{ 25 | "type": "shell", 26 | "inline": ["sudo hostname {{user `hostname`}}"] 27 | }, 28 | { 29 | "type": "shell", 30 | "inline_shebang": "/bin/bash -x", 31 | "execute_command": "chmod +x {{ .Path }}; {{ .Vars }} sudo '{{ .Path }}'", 32 | "inline": <%= bootstrap_commands %> 33 | }, 34 | { 35 | "type": "shell", 36 | "inline": ["sudo rm -rf /tmp/*"] 37 | }] 38 | } 39 | -------------------------------------------------------------------------------- /settings.example.yaml: -------------------------------------------------------------------------------- 1 | ## Configuration file for Pupistry. 2 | 3 | 4 | # The following settings apply for all use cases of Pupistry and must be set 5 | general: 6 | 7 | # We need somewhere to cache files like archives and git repos. This will 8 | # be as big as the total size of all your git repos when being used to 9 | # build artifacts. Agent-only systems will be far smaller as it only includes 10 | # the latest version of the artifacts. 11 | app_cache: ~/.pupistry/cache 12 | 13 | # Some users like to use Pupistry with a non-AWS S3 endpoint such as Minio 14 | # which requires setting an alternative endpoint below. DO NOT UNCOMMENT IF 15 | # USING STANDARD AWS S3. 16 | # s3_endpoint: s3.notaws.example.com 17 | 18 | # The S3 bucket must be set in order to have a place to push and 19 | # pull artifact and manifests from. This bucket should be PRIVATE, we 20 | # only want your servers accessing the files! 21 | # 22 | # REMEMBER - S3 buckets are a global namespace, other people might have 23 | # already picked the name you want. Make sure you update this default 24 | # with something you actually own :-) 25 | s3_bucket: example 26 | 27 | # S3 prefix is entirely optional, useful if you're reusing/sharing an S3 28 | # bucket with other applications. Leave blank if not needed. 29 | s3_prefix: 30 | 31 | # GPG key to use for signing & validating the artifacts. It is possible to 32 | # run pupistry in an unsigned mode, but you will lose the protection against 33 | # someone with access to the S3 bucket tampering with the files and pushing 34 | # malicious puppet manifests to your servers 35 | gpg_disable: true 36 | gpg_signing_key: XXXXXX 37 | 38 | 39 | # Settings for agents, these are required on the machines that will be 40 | # downloading and applying artifacts but also need to be set for build 41 | # machines so we can popular bootstrap templates for you and automatically 42 | # check stuff like IAM permissions before you roll your hosts. 43 | agent: 44 | # Puppet3 doesn't care what this is, but if using Puppet4, you need to set it 45 | # to /etc/puppetlabs/code/environments otherwise it blows up. 46 | puppetcode: /etc/puppetlabs/code/environments 47 | 48 | # The AWS credentials with READ permission to the S3 bucket for downloading 49 | # artifact files. If unset, we try to figure it out from any AWS creds 50 | access_key_id: 51 | secret_access_key: 52 | region: ap-southeast-2 53 | proxy_uri: 54 | 55 | # (If Daemonised) 56 | # Default is to check for a new artifact every 60 seconds, but only to 57 | # actually run Puppet if there has been a change to the artifact contents. 58 | # 59 | # At a polling rate of 60 seconds, the cost of S3 will be about $0.02 per 60 | # month per system running Pupistry. 61 | # 62 | # If you want to force regular Puppet runs regardless whether or not a new 63 | # artifact has been released, turn daemon_minimal off, but make sure the 64 | # frequency isn't too low - 300 seconds+ recommended otherwise Puppet will 65 | # be hammering your system resources. 66 | daemon_frequency: 60 67 | daemon_minimal: true 68 | 69 | 70 | # The following settings are only needed on the build machines (people building 71 | # new artifacts) and are not needed on the actual agent servers that will be 72 | # downloading and applying them. 73 | build: 74 | 75 | # Define the Git repo for the Puppet manifest & r10k data 76 | # (ie repo where your Puppetfile & site.pp is) 77 | puppetcode: git@github.com:jethrocarr/pupistry-samplepuppet.git 78 | 79 | 80 | # The AWS credentials with write permission to the S3 bucket for uploading 81 | # new artifact files. If unset, we try to figure it out from any AWS creds 82 | # set in the environmnt, but you're best to make it explicit here to avoid 83 | # surprises.... 84 | # 85 | access_key_id: 86 | secret_access_key: 87 | region: ap-southeast-2 88 | proxy_uri: 89 | 90 | 91 | # Enable the HieraCrypt feature 92 | # 93 | # Note - Once enabled, all your servers must have their definition added, 94 | # otherwise they will not recieve any Hiera information as it will no longer 95 | # be delivered in an unencrypted form. 96 | # 97 | # You will want to run `pupistry hieracrypt --generate` on each node to 98 | # generate a file which needs to be saved into `hieracrypt/nodes/hostname` 99 | # in your puppetcode repo (right alongside the `hieradata/` directory). 100 | # 101 | # If you later decide to disable hieracrypt, you should remove the entire 102 | # `hieracrypt` directory to avoid confusion. 103 | # 104 | hieracrypt: false 105 | 106 | -------------------------------------------------------------------------------- /test/data/empty.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | -------------------------------------------------------------------------------- /test/data/nonyaml.txt: -------------------------------------------------------------------------------- 1 | { foo 2 | this: is :' not yaml 3 | )) 4 | -------------------------------------------------------------------------------- /test/minitest_helper.rb: -------------------------------------------------------------------------------- 1 | $LOAD_PATH.unshift File.expand_path('../../lib', __FILE__) 2 | require 'simplecov' 3 | SimpleCov.start do 4 | add_filter '/test/' 5 | add_filter '/vendor/' 6 | end 7 | 8 | require 'pupistry' 9 | 10 | require 'minitest/spec' 11 | require 'minitest/autorun' 12 | -------------------------------------------------------------------------------- /test/test_config.rb: -------------------------------------------------------------------------------- 1 | require_relative './minitest_helper' 2 | 3 | describe Pupistry::Config do 4 | before do 5 | $logger = MiniTest::Mock.new 6 | end 7 | 8 | it 'exits with an error if told to use a non-existing config file' do 9 | $logger.expect :debug, nil, [String] 10 | $logger.expect :fatal, nil, [String] 11 | assert_raises(SystemExit) do 12 | Pupistry::Config.load('not_a_real_file') 13 | end 14 | assert $logger.verify 15 | end 16 | 17 | it 'exits with an error if a non-YAML file is specified for use' do 18 | $logger.expect :debug, nil, [String] 19 | $logger.expect :fatal, nil, [String] 20 | $logger.expect :debug, nil, [String] 21 | assert_raises(SystemExit) do 22 | Pupistry::Config.load('test/data/nonyaml.txt') 23 | end 24 | assert $logger.verify 25 | end 26 | 27 | it 'exits with an error if an empty YAML file is specified for use' do 28 | $logger.expect :debug, nil, [String] 29 | $logger.expect :fatal, nil, [String] 30 | $logger.expect :debug, nil, [String] 31 | assert_raises(SystemExit) do 32 | Pupistry::Config.load('test/data/empty.yaml') 33 | end 34 | assert $logger.verify 35 | end 36 | end 37 | -------------------------------------------------------------------------------- /test/test_pupistry.rb: -------------------------------------------------------------------------------- 1 | require_relative './minitest_helper' 2 | 3 | describe Pupistry do 4 | it 'has_a_version_number' do 5 | refute_nil ::Pupistry::VERSION 6 | end 7 | 8 | it 'does_something_useful' do 9 | skip 'we need to write tests!!' 10 | end 11 | end 12 | --------------------------------------------------------------------------------