├── .chef └── plugins │ └── knife │ └── deploy.rb ├── .gitignore ├── Cheffile ├── Cheffile.lock ├── Gemfile ├── Gemfile.lock ├── LICENSE ├── NOTICE ├── Vagrantfile ├── data_bags ├── README.md ├── package │ └── ruby.json └── pennyworth │ ├── install-dependency-packages.json │ ├── readme.org │ ├── ruby.json │ └── rubygem-bundler.json ├── readme.org └── roles └── pennyworth.rb /.chef/plugins/knife/deploy.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Author:: Darrin Eden 3 | # 4 | # Copyright 2011, Heavy Water Software Inc. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | # TODO: There are some EC2 assumptions here. We should be more generic. 20 | 21 | module HeavyWater 22 | class Deploy < Chef::Knife 23 | 24 | banner "knife deploy QUERY" 25 | 26 | option :attribute, 27 | :short => "-a ATTR", 28 | :long => "--attribute ATTR", 29 | :description => "The attribute to use for opening the connection - default is fqdn", 30 | :default => "fqdn" 31 | 32 | option :force, 33 | :short => "-f", 34 | :long => "--force", 35 | :description => "Force a deploy" 36 | 37 | option :identity_file, 38 | :short => "-i IDENTITY_FILE", 39 | :long => "--identity-file IDENTITY_FILE", 40 | :description => "The SSH identity file used for authentication" 41 | 42 | option :ssh_user, 43 | :short => "-x USERNAME", 44 | :long => "--ssh-user USERNAME", 45 | :description => "The ssh username" 46 | 47 | deps do 48 | require "chef/shef/ext" 49 | require "chef/knife/ssh" 50 | require "net/ssh" 51 | require "net/ssh/multi" 52 | require "net/ssh/gateway" 53 | end 54 | 55 | def upgrade( node) 56 | unload( node) 57 | set_version( node) 58 | converge( "name:#{node.name}") 59 | smoke( node) 60 | load( node) 61 | end 62 | 63 | def run 64 | Shef::Extensions.extend_context_object(self) 65 | 66 | unless name_args.size == 1 67 | puts "You need to supply a query" 68 | show_usage 69 | exit 1 70 | end 71 | 72 | @query = name_args.first 73 | 74 | if ( new_version == old_version) 75 | puts "Latest version deployed to all nodes." 76 | exit unless config[:force] 77 | end 78 | 79 | if search( :node, query).length < 2 80 | puts "Two or more nodes required to support rolling deploys." 81 | exit unless config[:force] 82 | end 83 | 84 | # Remove reference to terminated nodes 85 | cleanup() 86 | 87 | # Upgrade nodes not in the load balancer; assuming failure 88 | unloaded.each { |node| upgrade( node) } 89 | 90 | # Upgrade the rest of the nodes 91 | search( :node, query).each { |node| upgrade( node) } 92 | end 93 | 94 | def query 95 | @query 96 | end 97 | 98 | def app 99 | query[/\w*$/] 100 | end 101 | 102 | def new_version 103 | # Version of the newest project artifact generated by CI 104 | data_bag_item( :package, app)["version"] 105 | end 106 | 107 | def old_version 108 | # Oldest version deployed on any node 109 | versions = search( :node, query).map { |node| node[app][:version] } 110 | versions.sort.first 111 | end 112 | 113 | def set_version( node) 114 | puts "setting version: #{new_version}" 115 | node.set[app][:version] = new_version 116 | node.save 117 | end 118 | 119 | def loaded_ips 120 | # List of IP addresses being load balanced 121 | data_bag_item( :pools, "load_balancers")["loaded_ips"] 122 | end 123 | 124 | def cleanup 125 | # Remove reference to terminated nodes 126 | lbs = data_bag_item( :pools, "load_balancers") 127 | node_ips = search( :node, query).map { |node| node[:ipaddress] } 128 | lbs["loaded_ips"] = ( lbs["loaded_ips"] & node_ips) 129 | lbs.save 130 | end 131 | 132 | def unloaded 133 | # List of nodes not being load balanced 134 | ips = loaded_ips 135 | search( :node, query).reject { |node| ips.include?( node[:ipaddress]) } 136 | end 137 | 138 | def load_balance( ips) 139 | # Update load balancer pool data bag 140 | item = Chef::DataBagItem.new 141 | item.data_bag( "pools") 142 | item.raw_data = { "id" => "load_balancers", "loaded_ips" => ips } 143 | item.save 144 | end 145 | 146 | def load( node) 147 | # Add a node to load balancer 148 | puts "loading: #{node.name}" 149 | load_balance( loaded_ips + [ node[:ipaddress]]) 150 | converge( "role:lb") 151 | end 152 | 153 | def unload( node) 154 | # Remove a node from load balancer 155 | puts "unloading: #{node.name}" 156 | load_balance( loaded_ips - [ node[:ipaddress]]) 157 | converge( "role:lb") 158 | end 159 | 160 | def smoke( node) 161 | # Smoke test a node to test and warm it 162 | puts "smoking: #{node.name}" 163 | 164 | # We're wanting to load a URI directly from the node. 165 | # Since it's 166 | # on EC2's internal network maybe something like this so 167 | # our 168 | # workstation is able to connect? 169 | 170 | # gateway = Net::SSH::Gateway.new( 171 | # node[:ec2][:public_ipv4], "ubuntu" ) 172 | # gateway.open( node[:ec2][:ipaddress], 2999 ) do |port| 173 | # Net::HTTP.get_print("127.0.0.1", "/", port) 174 | # end 175 | # gateway.shutdown! 176 | 177 | true 178 | end 179 | 180 | def converge( search_term) 181 | puts "converging: #{search_term}" 182 | knife_ssh = Chef::Knife::Ssh.new 183 | knife_ssh.config = config 184 | knife_ssh.config[:manual] = true 185 | cmd = "sudo /opt/ruby/bin/chef-client --log_level error" 186 | nodes = search( :node, search_term) 187 | servers = nodes.map { |n| n[:ec2][:public_ipv4] }.join(" ") 188 | knife_ssh.name_args = [servers, cmd] 189 | knife_ssh.run 190 | true 191 | end 192 | end 193 | end 194 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | readme.html 2 | iso 3 | *.box 4 | .vagrant 5 | .bundle 6 | *.swp 7 | *.swo 8 | .\#* 9 | cookbooks 10 | tmp 11 | -------------------------------------------------------------------------------- /Cheffile: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | #^syntax detection 3 | 4 | site "http://community.opscode.com/api/v1" 5 | 6 | [ "runit", 7 | "leiningen", 8 | "build-essential", 9 | "apache2", 10 | "postfix", 11 | "java", 12 | "apt", 13 | "ant" ].each do |cb| 14 | cookbook cb 15 | end 16 | 17 | [ "pennyworth", 18 | "reprepro", 19 | "gpg" ].each do |git_cb| 20 | cookbook( git_cb, 21 | :git => "https://github.com/heavywater/chef-#{git_cb}", 22 | :ref => "develop" ) 23 | end 24 | -------------------------------------------------------------------------------- /Cheffile.lock: -------------------------------------------------------------------------------- 1 | SITE 2 | remote: http://community.opscode.com/api/v1 3 | specs: 4 | ant (0.10.1) 5 | java (>= 0) 6 | jpackage (>= 0) 7 | apache2 (1.1.4) 8 | apt (1.4.0) 9 | build-essential (1.0.0) 10 | java (1.4.2) 11 | jenkins (0.6.1) 12 | java (>= 0.0.0) 13 | runit (>= 0.0.0) 14 | jpackage (0.10.0) 15 | java (>= 0) 16 | leiningen (0.2.0) 17 | java (>= 0) 18 | postfix (0.8.4) 19 | runit (0.15.0) 20 | 21 | GIT 22 | remote: https://github.com/heavywater/chef-gpg 23 | ref: develop 24 | sha: b5a9afaf5dd7bd98dbaef6b5a5dc5e019e167e01 25 | specs: 26 | gpg (0.0.1) 27 | 28 | GIT 29 | remote: https://github.com/heavywater/chef-pennyworth 30 | ref: develop 31 | sha: 7ed86daafe46851849dbb03a6f8f1b7e23ebef0b 32 | specs: 33 | pennyworth (0.2.2) 34 | ant (>= 0.0.0) 35 | build-essential (>= 0.0.0) 36 | java (>= 0.0.0) 37 | jenkins (>= 0.0.0) 38 | leiningen (>= 0.0.0) 39 | postfix (>= 0.0.0) 40 | reprepro (>= 0.0.0) 41 | 42 | GIT 43 | remote: https://github.com/heavywater/chef-reprepro 44 | ref: develop 45 | sha: 65f4fc8ec93e0fa8eea5da0e6d9add725ce7d0a9 46 | specs: 47 | reprepro (0.2.2) 48 | apache2 (>= 0.0.0) 49 | apt (>= 0.0.0) 50 | build-essential (>= 0.0.0) 51 | gpg (>= 0.0.0) 52 | 53 | DEPENDENCIES 54 | ant (>= 0) 55 | apache2 (>= 0) 56 | apt (>= 0) 57 | build-essential (>= 0) 58 | gpg (>= 0) 59 | java (>= 0) 60 | leiningen (>= 0) 61 | pennyworth (>= 0) 62 | postfix (>= 0) 63 | reprepro (>= 0) 64 | runit (>= 0) 65 | 66 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source :rubygems 2 | gem "chef", ">= 0.10.10" 3 | gem "json", "= 1.5.2" 4 | gem "vagrant", "~> 1.0" 5 | gem "librarian", "~> 0.0.12" 6 | -------------------------------------------------------------------------------- /Gemfile.lock: -------------------------------------------------------------------------------- 1 | GEM 2 | remote: http://rubygems.org/ 3 | specs: 4 | archive-tar-minitar (0.5.2) 5 | bunny (0.7.9) 6 | chef (0.10.10) 7 | bunny (>= 0.6.0) 8 | erubis 9 | highline (>= 1.6.9) 10 | json (>= 1.4.4, <= 1.6.1) 11 | mixlib-authentication (>= 1.1.0) 12 | mixlib-cli (>= 1.1.0) 13 | mixlib-config (>= 1.1.2) 14 | mixlib-log (>= 1.3.0) 15 | mixlib-shellout (~> 1.0.0.rc) 16 | moneta 17 | net-ssh (~> 2.2.2) 18 | net-ssh-multi (~> 1.1.0) 19 | ohai (>= 0.6.0) 20 | rest-client (>= 1.0.4, < 1.7.0) 21 | treetop (~> 1.4.9) 22 | uuidtools 23 | yajl-ruby (>= 1.1.0) 24 | childprocess (0.3.2) 25 | ffi (~> 1.0.6) 26 | erubis (2.7.0) 27 | ffi (1.0.11) 28 | highline (1.6.12) 29 | i18n (0.6.0) 30 | ipaddress (0.8.0) 31 | json (1.5.2) 32 | librarian (0.0.23) 33 | archive-tar-minitar (>= 0.5.2) 34 | chef (>= 0.10) 35 | highline 36 | thor (~> 0.15) 37 | log4r (1.1.10) 38 | mime-types (1.18) 39 | mixlib-authentication (1.1.4) 40 | mixlib-log 41 | mixlib-cli (1.2.2) 42 | mixlib-config (1.1.2) 43 | mixlib-log (1.3.0) 44 | mixlib-shellout (1.0.0) 45 | moneta (0.6.0) 46 | net-scp (1.0.4) 47 | net-ssh (>= 1.99.1) 48 | net-ssh (2.2.2) 49 | net-ssh-gateway (1.1.0) 50 | net-ssh (>= 1.99.1) 51 | net-ssh-multi (1.1) 52 | net-ssh (>= 2.1.4) 53 | net-ssh-gateway (>= 0.99.0) 54 | ohai (0.6.12) 55 | ipaddress 56 | mixlib-cli 57 | mixlib-config 58 | mixlib-log 59 | systemu 60 | yajl-ruby 61 | polyglot (0.3.3) 62 | rest-client (1.6.7) 63 | mime-types (>= 1.16) 64 | systemu (2.5.0) 65 | thor (0.15.2) 66 | treetop (1.4.10) 67 | polyglot 68 | polyglot (>= 0.3.1) 69 | uuidtools (2.1.2) 70 | vagrant (1.0.3) 71 | archive-tar-minitar (= 0.5.2) 72 | childprocess (~> 0.3.1) 73 | erubis (~> 2.7.0) 74 | i18n (~> 0.6.0) 75 | json (~> 1.5.1) 76 | log4r (~> 1.1.9) 77 | net-scp (~> 1.0.4) 78 | net-ssh (~> 2.2.2) 79 | yajl-ruby (1.1.0) 80 | 81 | PLATFORMS 82 | ruby 83 | 84 | DEPENDENCIES 85 | chef (>= 0.10.10) 86 | json (= 1.5.2) 87 | librarian (~> 0.0.12) 88 | vagrant (~> 1.0) 89 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Pennyworth NOTICE 2 | =========== 3 | 4 | Developed by AJ Christensen (http://junglist.gen.nz) and Darrin Eden (http://www.hw-ops.com), for Heavy Water operations 5 | 6 | * Copyright 2011, AJ Christensen 7 | * Copyright 2011, Darrin Eden 8 | 9 | 10 | -------------------------------------------------------------------------------- /Vagrantfile: -------------------------------------------------------------------------------- 1 | Vagrant::Config.run do |config| 2 | config.vm.define :pennyworth do |pennyworth| 3 | pennyworth.vm.customize do |vm| 4 | vm.memory_size = ENV["PENNYWORTH_RAM"] || 4096 5 | vm.cpus = ENV["PENNYWORTH_CPUS"] || 2 6 | end 7 | pennyworth.vm.box = "oneiric64" 8 | pennyworth.vm.box_url = "https://s3.amazonaws.com/hw-vagrant/oneiric64.box" 9 | pennyworth.vm.forward_port 80, 8000 10 | pennyworth.vm.forward_port 8080, 8080 11 | pennyworth.vm.provision :chef_solo do |chef| 12 | chef.data_bags_path = "data_bags" 13 | chef.cookbooks_path = "cookbooks" 14 | chef.roles_path = "roles" 15 | chef.add_role "pennyworth" 16 | end 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /data_bags/README.md: -------------------------------------------------------------------------------- 1 | Data Bags 2 | --------- 3 | 4 | This directory contains directories of the various data bags you create for your infrastructure. Each subdirectory corresponds to a data bag on the Chef Server, and contains JSON files of the items that go in the bag. 5 | 6 | First, create a directory for the data bag. 7 | 8 | mkdir data_bags/BAG 9 | 10 | Then create the JSON files for items that will go into that bag. 11 | 12 | $EDITOR data_bags/BAG/ITEM.json 13 | 14 | The JSON for the ITEM must contain a key named "id" with a value equal to "ITEM". For example, 15 | 16 | { 17 | "id": "foo" 18 | } 19 | 20 | Next, create the data bag on the Chef Server. 21 | 22 | knife data bag create BAG 23 | 24 | Then upload the items in the data bag's directory to the Chef Server. 25 | 26 | knife data bag from file BAG ITEM.json 27 | 28 | 29 | Encrypted Data Bags 30 | ------------------- 31 | 32 | Added in Chef 0.10, encrypted data bags allow you to encrypt the contents of your data bags. The content of attributes will no longer be searchable. To use encrypted data bags, first you must have or create a secret key. 33 | 34 | openssl rand -base64 512 > secret_key 35 | 36 | You may use this secret_key to add items to a data bag during a create. 37 | 38 | knife data bag create --secret-file secret_key passwords mysql 39 | 40 | You may also use it when adding ITEMs from files, 41 | 42 | knife data bag create passwords 43 | knife data bag from file passwords data_bags/passwords/mysql.json --secret-file secret_key 44 | 45 | The JSON for the ITEM must contain a key named "id" with a value equal to "ITEM" and the contents will be encrypted when uploaded. For example, 46 | 47 | { 48 | "id": "mysql", 49 | "password": "abc123" 50 | } 51 | 52 | Without the secret_key, the contents are encrypted. 53 | 54 | knife data bag show passwords mysql 55 | id: mysql 56 | password: 2I0XUUve1TXEojEyeGsjhw== 57 | 58 | Use the secret_key to view the contents. 59 | 60 | knife data bag show passwords mysql --secret-file secret_key 61 | id: mysql 62 | password: abc123 63 | 64 | -------------------------------------------------------------------------------- /data_bags/package/ruby.json: -------------------------------------------------------------------------------- 1 | { "id" : "ruby", 2 | "version" : "0.0.0" 3 | } 4 | -------------------------------------------------------------------------------- /data_bags/pennyworth/install-dependency-packages.json: -------------------------------------------------------------------------------- 1 | { "id" : "install-dependency-packages", 2 | "project_type" : "package", 3 | "test_commands" : [ 4 | "true" 5 | ], 6 | "build_commands" : [ 7 | "sudo /usr/bin/apt-get update", 8 | "sudo /usr/bin/apt-get install ruby1.9.3-full -y" 9 | ], 10 | "package_commands" : [ 11 | "true" 12 | ], 13 | "child_projects" : [ 14 | "rubygem-bundler" 15 | ] 16 | } 17 | 18 | 19 | -------------------------------------------------------------------------------- /data_bags/pennyworth/readme.org: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heavywater/pennyworth/13dc28448380bc94f80e20ee19683d0bbb86e41e/data_bags/pennyworth/readme.org -------------------------------------------------------------------------------- /data_bags/pennyworth/ruby.json: -------------------------------------------------------------------------------- 1 | { "id" : "ruby", 2 | "version" : { 3 | "major" : "1.9", 4 | "minor" : "3" 5 | }, 6 | "project_type" : "package_from_git", 7 | "git_branch" : "ruby_1_9_3", 8 | "git_url" : "git://github.com/ruby/ruby.git", 9 | "test_commands" : [ 10 | "autoconf", 11 | "./configure --prefix=/usr/local --disable-install-doc --with-baseruby=/opt/opscode/embedded/bin/ruby", 12 | "make test" 13 | ], 14 | "build_commands" : [ 15 | "make install DESTDIR=\$DESTDIR" 16 | ], 17 | "package_commands": [ 18 | // Setup and build the deb package for Ruby, including a few paths 19 | // and adding the ldconfig dpkg-trigger. 20 | "mkdir -p fpm", 21 | "cd fpm; /opt/opscode/embedded/bin/fpm -s dir -t deb -n ruby1.9.3-full -v \$VERSION -C \$DESTDIR -p ruby1.9.3-full-VERSION_ARCH.deb --post-install /var/lib/jenkins/ldconfig usr/local/bin usr/local/include /usr/local/lib usr/local/share", 22 | // Include the package into reprepro 23 | "sudo /usr/bin/reprepro -Vb /srv/apt includedeb \$LSB_CODENAME \$DESTDIR/ruby1.9.3-full-\$VERSION_*.deb" 24 | ], 25 | "child_projects" : [ 26 | "install-dependency-packages" 27 | ], 28 | "remote_poll": false, 29 | "clean" : true, 30 | "wipeoutworkspace" : true 31 | } 32 | -------------------------------------------------------------------------------- /data_bags/pennyworth/rubygem-bundler.json: -------------------------------------------------------------------------------- 1 | { "id" : "rubygem-bundler", 2 | "project_type" : "package", 3 | "build_commands" : [ 4 | "fpm -s gem -t deb bundler" 5 | ], 6 | "package_commands" : [ 7 | //"sudo /usr/bin/reprepro -Vb /srv/apt remove \$LSB_CODENAME rubygem-bundler", 8 | "sudo /usr/bin/reprepro -Vb /srv/apt includedeb \$LSB_CODENAME $\WORKSPACE/rubygem-bundler*.deb" 9 | ] 10 | } 11 | -------------------------------------------------------------------------------- /readme.org: -------------------------------------------------------------------------------- 1 | * NOTE: This project has been abandoned. 2 | 3 | * pennyworth 0.2.2 4 | 5 | #+BEGIN_QUOTE 6 | Bruce Wayne: Rebuild it. Just the way it was, brick for brick. 7 | 8 | Alfred Pennyworth: *Just* the way it was, sir? 9 | 10 | Bruce Wayne: Yeah. Why? 11 | 12 | Alfred Pennyworth: I thought this might be a good opportunity for improving the foundations. 13 | #+END_QUOTE 14 | 15 | Based upon the principles of continuous integration, pennyworth will 16 | help you build assets for any software you use and provides a 17 | reasonable means to distribute it throughout your nodes. By using a 18 | well tested package management system under the hood for deployment, you can easily roll back versions and manage node 19 | level conflicts. 20 | 21 | Jenkins can be configured to search for feature branches, for 22 | advanced workflows. We've included some examples for basic branch tracking. 23 | 24 | Many example data bags are included that show how to test, build and 25 | package a variety of projects, including Ruby and Clojure projects. 26 | ** Librarian support 27 | - To test, run: 28 | #+BEGIN_EXAMPLE 29 | bundle install 30 | bundle exec librarian-chef install --verbose 31 | #+END_EXAMPLE 32 | 33 | ** vagrant support 34 | - you can use 'vagrant up' to launch a virtualbox instance based on 35 | Ubuntu Natty 11.04 amd64 to run pennyworth 36 | #+BEGIN_EXAMPLE 37 | bundle exec vagrant up 38 | #+END_EXAMPLE 39 | ** chef cookbooks 40 | *** jenkins 41 | - Based on Fletcher Nichols' [[https://github.com/fnichol/chef-jenkins][chef-jenkins cookbook]] 42 | *** pennyworth 43 | **** recipe: ssh_keys 44 | - drops keypairs off for Jenkins, so it can talk to your Git infrastructure later. 45 | **** recipe: remount_ephemeral 46 | - on ec2, remount the ephemeral device (with an appropriate 47 | filesystem and such) at a convenient location for jenkins 48 | **** recipe: riak 49 | - in one of our deployments a pennyworth project required riak, 50 | so we shipped it 51 | **** recipe: default 52 | - consumes the pennyworth databag items individually, adding them 53 | to jenkins and so on. 54 | **** recipe: update_version 55 | - ships the update_version script which updates the "package" 56 | data bag versions for each job 57 | *** [[http://mirrorer.alioth.debian.org/][reprepro]] 58 | - reprepro (formerly known as mirrorer) is a nasty old debian tool 59 | that you can use to manage an APT repository 60 | - could easily slot something like 61 | [[https://github.com/rcrowley/freight][Richard Crowley's freight]] project in here 62 | *** apt 63 | - upstream opscode cookbook, provides apt repository LWRP which is used to 64 | install the repository hosted on the jenkins node 65 | *** java 66 | - upstream opscode cookbook, installs java 67 | *** leiningen 68 | - installs the leiningen build tool for clojure 69 | ** data bags 70 | *** package 71 | - this allows for an API client to query the chef server for 72 | available versions 73 | - can be updated at end-of-build by Jenkins (examples included) 74 | *** pennyworth 75 | - multiple types of jenkins jobs 76 | **** testing 77 | - Git based, runs test commands 78 | **** packaging 79 | - Git based, runs test, build and package commands. 80 | - Examples included for building packages with fpm 81 | - Packages built with fpm can be included into a Reprepro package 82 | repository on the system 83 | ** other tools 84 | *** [[https://github.com/jordansissel/fpm][fpm]] 85 | - fpm allows us to build debian packages out of basically anything, 86 | including cool stuff like ruby gems and python eggs 87 | - we use it inside of pennyworth jenkins projects to build 88 | architecture-specific assets of varying types of projects 89 | ** knife deploy plugin 90 | - to be documented 91 | -------------------------------------------------------------------------------- /roles/pennyworth.rb: -------------------------------------------------------------------------------- 1 | name "pennyworth" 2 | description "Components required for the pennyworth continuous deployment pipeline" 3 | run_list( "recipe[reprepro]", 4 | "recipe[java]", 5 | "recipe[postfix]", 6 | "recipe[pennyworth::ssh_key]", 7 | "recipe[jenkins]", 8 | "recipe[pennyworth::update_version]", 9 | "recipe[pennyworth::dependencies]", 10 | "recipe[pennyworth]" ) 11 | 12 | override_attributes( "java" => { "install_flavor" => "openjdk" }, 13 | "jenkins" => { 14 | "http_proxy" => { 15 | "variant" => "apache2" 16 | }, 17 | "server" => { 18 | "url" => "http://127.0.0.1:8080", 19 | "host" => "127.0.0.1", 20 | "port" => "8080", 21 | "plugins" => [ 22 | 'setenv', 23 | 'git', 24 | 'python', 25 | 'ruby', 26 | 'rake', 27 | 'greenballs', 28 | 'xunit', 29 | 'campfire' 30 | ] 31 | } }, 32 | "postfix" => { 33 | "mail_type" => "master" 34 | }, 35 | "pennyworth" => { 36 | "package_dependencies" => [ 37 | "git", 38 | "libtool", 39 | "uuid-dev" 40 | ], 41 | "ruby_gem_dependencies" => [ 42 | "fpm", 43 | "bundler", 44 | "rake", 45 | "rdoc", 46 | "rspec", 47 | "cucumber" 48 | ], 49 | } ) 50 | --------------------------------------------------------------------------------