├── README.markdown ├── deploy.rb ├── edit_env.rb ├── grep.rb ├── key_check.rb ├── lib └── opscode_deploy.rb ├── push_env.rb ├── role_tree.rb ├── set_rev.rb └── show_rev.rb /README.markdown: -------------------------------------------------------------------------------- 1 | # KNIFE PLUGINS 2 | 3 | This is my knife plugins directory. 4 | 5 | ## Grep 6 | `knife grep` allows you to find hosts by specifying a partial hostname, 7 | role, or IP address. 8 | 9 | ## Push Env, Edit Env, Set Rev 10 | At Opscode, we always deploy from tags, which we set in data bag items 11 | in the _environments_ data bag. Editing these is more work than it 12 | should be, so imma make the computer do it. These things may or may not 13 | be useful in your environment. 14 | 15 | I have several Chef configurations layed out like this: 16 | 17 | TOPLEVEL 18 | | - ENVIRONMENT 19 | | - .chef/knife.rb 20 | ` - chef-repo 21 | | - cookbooks/ 22 | | - data_bags 23 | | ` - environments 24 | | | - prod.json 25 | | ` - preprod.json 26 | ` - roles/ 27 | 28 | The chef-repo contents are shared between all environments with 29 | symlinks. 30 | 31 | So, chances are your setup is different and these plugins will 32 | not work out of the box for you. But maybe you will find them 33 | interesting. 34 | 35 | These plugins are also compatible with the following layout: 36 | 37 | | - ENVIRONMENT 38 | | - .chef/knife.rb 39 | | - cookbooks/ 40 | | - data_bags 41 | | ` - environments 42 | | | - prod.json 43 | | ` - preprod.json 44 | ` - roles/ 45 | 46 | ## Deploy 47 | 48 | The `knife deploy` plugin is essentially a wrapper for `knife ssh`, 49 | but with a number of safety checks in place to help avoid mistakes due 50 | to out of date cookbooks when deploying new code. Knife deploy helps 51 | you by: 52 | 53 | - Making sure your local git repo is in sync with the remote repo 54 | (specified in knife.rb config, see below). 55 | 56 | - Collecting the cookbooks used by the nodes you will be deploying to 57 | and comparing the cookbooks checksums on the server with those in 58 | your local cookbooks directory. This helps avoid running a deploy 59 | when you have forgotten to upload modified cookbooks. 60 | 61 | - After these checks are complete, you will get tmux, screen, or 62 | macterm sessions on the hosts matching your deploy query. 63 | 64 | - Saves you typing. The query you enter will be matched glob style 65 | against roles unless the query contains a ':', in which case it will 66 | be interpreted directly as a search query. 67 | 68 | ### configuration 69 | 70 | In your knife.rb, add a stanza like this: 71 | 72 | deploy({ 73 | "prod" => { 74 | :remote => "origin", 75 | :branch => "prod", 76 | :default_command => "screen" 77 | } 78 | }) 79 | -------------------------------------------------------------------------------- /deploy.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path('../lib/opscode_deploy', __FILE__) 2 | 3 | module OpscodeDeploy 4 | class Deploy < Chef::Knife 5 | include EnvironmentNames 6 | 7 | category "OPSCODE DEPLOYMENT" 8 | 9 | banner "knife deploy [ROLE-ISH|QUERY] [COMMAND]" 10 | 11 | option :concurrency, 12 | :short => "-C NUM", 13 | :long => "--concurrency NUM", 14 | :description => "The number of concurrent connections", 15 | :default => nil, 16 | :proc => lambda { |o| o.to_i } 17 | 18 | option :attribute, 19 | :short => "-a ATTR", 20 | :long => "--attribute ATTR", 21 | :description => "The attribute to use for opening the connection - default is fqdn", 22 | :default => "fqdn" 23 | 24 | option :ssh_user, 25 | :short => "-x USERNAME", 26 | :long => "--ssh-user USERNAME", 27 | :description => "The ssh username" 28 | 29 | option :ssh_password, 30 | :short => "-P PASSWORD", 31 | :long => "--ssh-password PASSWORD", 32 | :description => "The ssh password" 33 | 34 | option :ssh_port, 35 | :short => "-p PORT", 36 | :long => "--ssh-port PORT", 37 | :description => "The ssh port", 38 | :default => "22", 39 | :proc => Proc.new { |key| Chef::Config[:knife][:ssh_port] = key } 40 | 41 | option :identity_file, 42 | :short => "-i IDENTITY_FILE", 43 | :long => "--identity-file IDENTITY_FILE", 44 | :description => "The SSH identity file used for authentication" 45 | 46 | option :no_host_key_verify, 47 | :long => "--no-host-key-verify", 48 | :description => "Disable host key verification", 49 | :boolean => true, 50 | :default => false 51 | 52 | deps do 53 | require 'yajl' 54 | require 'chef/search/query' 55 | require 'chef/cookbook_version' 56 | require 'chef/checksum_cache' 57 | require 'chef/knife/ssh' 58 | require 'net/ssh' 59 | require 'net/ssh/multi' 60 | require 'set' 61 | require 'pp' 62 | end 63 | 64 | def run 65 | get_env_from_args! 66 | # in order to deploy, your local git repo must match the 67 | # configured remote 68 | assert_git_rev_matches_remote 69 | 70 | nodes = find_nodes(name_args[0]) 71 | 72 | # The union of run lists of the nodes is used to determine a set 73 | # of cookbooks. We verify that the checksums for these 74 | # cookbooks on the server match what is on disk locally 75 | assert_server_vs_local_cookbooks_match(nodes) 76 | 77 | # use knife ssh to launch sessions 78 | knife_ssh = Chef::Knife::Ssh.new 79 | knife_ssh.config = config 80 | knife_ssh.config[:manual] = true 81 | cmd = name_args[1] || deploy_config[:default_command] || "tmux" 82 | servers = nodes.map { |n| n[config[:attribute]] }.join(" ") 83 | knife_ssh.name_args = [servers, cmd] 84 | knife_ssh.run 85 | exit 0 86 | end 87 | 88 | def git_branch 89 | @git_branch ||= deploy_config[:branch] 90 | required_config(":branch", @git_branch) 91 | end 92 | 93 | def git_remote 94 | @git_remote ||= deploy_config[:remote] 95 | required_config(":remote", @git_remote) 96 | end 97 | 98 | def deploy_config 99 | @deploy_config ||= (Chef::Config[:deploy][environment] rescue nil) 100 | if @deploy_config.nil? || !@deploy_config.is_a?(Hash) 101 | ui.error "missing deploy({#{environment} => {...}}) section in knife.rb" 102 | exit 1 103 | end 104 | @deploy_config 105 | end 106 | 107 | def required_config(label, value) 108 | if value.nil? 109 | ui.error "missing key deploy({#{environment} => {#{label} => ???}}) in knife.rb" 110 | exit 1 111 | end 112 | value 113 | end 114 | 115 | def assert_git_rev_matches_remote 116 | Dir.chdir(repo_file('')) do 117 | local_sha = `git rev-parse HEAD`.chomp 118 | remote_sha = `git ls-remote #{git_remote} #{git_branch}`[/^[0-9a-f]+/] 119 | if local_sha != remote_sha 120 | ui.error "your git repo is out of sync #{git_remote}/#{git_branch}" 121 | ui.msg "#{local_sha} (local)" 122 | ui.msg "#{remote_sha} (remote)" 123 | exit 1 124 | end 125 | end 126 | end 127 | 128 | def find_nodes(project_spec) 129 | query = query_for_project_spec 130 | searcher = Chef::Search::Query.new 131 | rows, _start, _total = searcher.search(:node, query) 132 | if rows.empty? 133 | ui.error "No nodes matched the query: #{query}" 134 | exit 1 135 | end 136 | rows 137 | end 138 | 139 | def query_for_project_spec 140 | query = case spec = name_args[0] 141 | when /:/ 142 | spec 143 | else 144 | "role:#{role_from_rolish(spec)}" 145 | end 146 | "app_environment:#{environment} AND (#{query})" 147 | end 148 | 149 | def role_from_rolish(spec) 150 | role_matches = Dir.glob("#{repo_file("roles")}/*#{spec}*.json").map do |f| 151 | File.basename(f, ".json") 152 | end 153 | case role_matches.size 154 | when 1 155 | role_matches.first 156 | when 0 157 | ui.error "No roles matched '#{spec}' in roles dir #{repo_file("roles")}" 158 | exit 1 159 | else 160 | # choice? 161 | ui.msg "Multiple role matches for '#{spec}', pick one:" 162 | choice = ui.highline.choose(*(role_matches.push("oops, nevermind"))) 163 | if choice == "oops, nevermind" 164 | exit 0 165 | end 166 | choice 167 | end 168 | end 169 | 170 | def assert_server_vs_local_cookbooks_match(nodes) 171 | remote_cookbooks = cookbooks_for_nodes(nodes) 172 | local_cookbooks = cookbooks_from_repo(remote_cookbooks[:names]) 173 | compare_cookbooks(local_cookbooks, remote_cookbooks) 174 | end 175 | 176 | def cookbooks_for_nodes(nodes) 177 | run_list = nodes.inject(nodes.first.run_list) do |list, node| 178 | node.run_list.to_a.each { |ri| list << ri } 179 | list 180 | end 181 | chef_rest = Chef::REST.new(Chef::Config[:chef_server_url]) 182 | # FIXME: customize for real environments 183 | path = "environments/_default/cookbook_versions" 184 | cookbook_versions = chef_rest.post_rest(path, 185 | {"run_list" => run_list}) 186 | file_checksums = {} 187 | checksums_for_cookbooks(cookbook_versions.values) 188 | end 189 | 190 | def checksums_for_cookbooks(cookbook_versions) 191 | file_checksums = {} 192 | cookbook_versions.each do |cookbook_version| 193 | Chef::CookbookVersion::COOKBOOK_SEGMENTS.each do |segment| 194 | cookbook_version.manifest[segment].each do |file| 195 | file_checksums["#{cookbook_version.name}/#{file["path"]}"] = file["checksum"] 196 | end 197 | end 198 | end 199 | { 200 | :names => cookbook_versions.map { |cv| cv.name.to_s }.sort, 201 | :checksums => file_checksums 202 | } 203 | end 204 | 205 | def cookbooks_from_repo(names) 206 | cookbook_versions = [] 207 | names.each do |name| 208 | cookbook_path = repo_file("cookbooks/#{name}") 209 | cvl = Chef::Cookbook::CookbookVersionLoader.new(cookbook_path) 210 | cvl.load_cookbooks 211 | cookbook_version = cvl.cookbook_version 212 | # will get nil if no such cookbook in local repo 213 | if cookbook_version 214 | cookbook_versions << cookbook_version 215 | end 216 | end 217 | checksums_for_cookbooks(cookbook_versions) 218 | end 219 | 220 | def compare_cookbook_names(local, remote) 221 | if local[:names] != remote[:names] 222 | only_local, only_remote = difference_report(local[:names], remote[:names]) 223 | ui.error "Local cookbook repo does not match server" 224 | if !only_local.empty? 225 | ui.msg "The following cookbooks are not on the server:" 226 | only_local.each { |c| ui.msg "\t#{c}" } 227 | end 228 | if !only_remote.empty? 229 | ui.msg "The following cookbooks are not in your cookbooks dir:" 230 | only_remote.each { |c| ui.msg "\t#{c}" } 231 | end 232 | false 233 | end 234 | true 235 | end 236 | 237 | def compare_cookbook_files(local, remote) 238 | local_files = local[:checksums].keys.sort 239 | remote_files = remote[:checksums].keys.sort 240 | they_match = true 241 | only_local, only_remote = difference_report(local_files, remote_files) 242 | if !only_local.empty? 243 | they_match = false 244 | ui.msg "The following cookbook files are not on the server:" 245 | only_local.each { |c| ui.msg "\t#{c}" } 246 | end 247 | if !only_remote.empty? 248 | they_match = false 249 | ui.msg "The following cookbook files are not in your cookbooks dir:" 250 | only_remote.each { |c| ui.msg "\t#{c}" } 251 | end 252 | mismatches = [] 253 | local_files.each do |file| 254 | if local[:checksums][file] != remote[:checksums][file] 255 | mismatches << file 256 | end 257 | end 258 | if !mismatches.empty? 259 | they_match = false 260 | ui.error "mismatches!" 261 | mismatches.each { |m| ui.msg m } 262 | end 263 | they_match 264 | end 265 | 266 | def compare_cookbooks(local, remote) 267 | names_match = compare_cookbook_names(local, remote) 268 | files_match = compare_cookbook_files(local, remote) 269 | exit 1 unless (names_match && files_match) 270 | end 271 | 272 | def difference_report(a, b) 273 | a_set = Set.new(a) 274 | b_set = Set.new(b) 275 | only_a = a_set.difference(b_set) 276 | only_b = b_set.difference(a_set) 277 | [only_a, only_b] 278 | end 279 | end 280 | end 281 | -------------------------------------------------------------------------------- /edit_env.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path('../lib/opscode_deploy', __FILE__) 2 | 3 | module OpscodeDeploy 4 | class EditEnv < Chef::Knife 5 | include EnvironmentNames 6 | 7 | category "OPSCODE DEPLOYMENT" 8 | 9 | banner "knife edit env [OPSCODE_ENV]" 10 | 11 | deps do 12 | end 13 | 14 | def run 15 | get_env_from_args! 16 | data_bag_file = repo_file("data_bags/environments/#{environment}.json") 17 | ui.msg(data_bag_file) 18 | exec "#{ENV['EDITOR']} #{data_bag_file}" 19 | end 20 | end 21 | 22 | end 23 | -------------------------------------------------------------------------------- /grep.rb: -------------------------------------------------------------------------------- 1 | require 'chef/knife' 2 | 3 | module Kallistec 4 | class Grep < Chef::Knife 5 | 6 | deps do 7 | require 'chef/search/query' 8 | require 'chef/knife/search' 9 | end 10 | 11 | banner "knife grep QUERY" 12 | 13 | def run 14 | unless @query = name_args.first 15 | ui.error "You need to specify a query term" 16 | exit 1 17 | end 18 | 19 | 20 | fuzzy_query = "tags:*#{@query}* OR roles:*#{@query}* OR fqdn:*#{@query}* OR addresses:*#{@query}*" 21 | knife_search = Chef::Knife::Search.new 22 | knife_search.name_args = ['node', fuzzy_query] 23 | knife_search.run 24 | 25 | end 26 | end 27 | end 28 | -------------------------------------------------------------------------------- /key_check.rb: -------------------------------------------------------------------------------- 1 | require 'chef/knife' 2 | 3 | module Kallistec 4 | class KeyCheck < Chef::Knife 5 | 6 | deps do 7 | require 'openssl' 8 | end 9 | 10 | banner 'knife key check CLIENT PATH_TO_private_key_file' 11 | 12 | def run 13 | unless @client_name = name_args[0] and @private_key_file = name_args[1] 14 | show_usage 15 | exit 1 16 | end 17 | @private_key_file = File.expand_path(@private_key_file) 18 | unless File.exist?(@private_key_file) 19 | ui.error "No such file for private key: #{@private_key_file}" 20 | exit 1 21 | end 22 | 23 | @auth_creds = Chef::REST::AuthCredentials.new(@client_name, @private_key_file) 24 | @private_key = @auth_creds.key 25 | #extract the public key from the private key: 26 | @public_key_from_local = @private_key.public_key 27 | 28 | @public_key_from_server = fetch_public_from_server 29 | if @public_key_from_local.to_s == @public_key_from_server.to_s 30 | ui.msg "Match." 31 | ui.msg "#{@private_key_file} is a valid key for client #{@client_name}" 32 | else 33 | ui.error "Mismatch:" 34 | ui.error "#{@private_key_file} is not a valid key for client #{@client_name}" 35 | ui.msg "Public key extracted from private key:\n#{@public_key_from_local}" 36 | ui.msg "Public key from server:\n#{@public_key_from_server}" 37 | exit 1 38 | end 39 | 40 | end 41 | 42 | def fetch_public_from_server 43 | api_client = Chef::REST.new(Chef::Config[:chef_server_url]) 44 | client_info = api_client.get_rest("clients/#{@client_name}") 45 | if client_info.has_key?("certificate") 46 | OpenSSL::X509::Certificate.new(client_info["certificate"]).public_key 47 | elsif client_info.has_key?("public_key") 48 | OpenSSL::PKey::RSA.new(client_info["public_key"]) 49 | else 50 | ui.error "The server did not return a cert or public key for this client, cannot verify key." 51 | exit 1 52 | end 53 | end 54 | 55 | end 56 | 57 | end 58 | -------------------------------------------------------------------------------- /lib/opscode_deploy.rb: -------------------------------------------------------------------------------- 1 | module OpscodeDeploy 2 | module EnvironmentNames 3 | OC_ENVS = {'prod' => 'rs-prod', 'preprod' => 'rs-preprod'} 4 | 5 | attr_reader :environment 6 | 7 | def get_env_from_args! 8 | unless @environment = guess_env_name 9 | ui.error "Environment to edit could not be determined by magic and you did not provide one" 10 | exit 1 11 | end 12 | end 13 | 14 | def guess_env_name 15 | pwd = File.basename(Dir.pwd) 16 | if OC_ENVS.key?(pwd) 17 | OC_ENVS[pwd] 18 | elsif OC_ENVS.key?(@name_args[0]) 19 | @name_args[0] 20 | else 21 | nil 22 | end 23 | end 24 | 25 | # Takes a relative path and expands it relative to your chef-repo 26 | # Looks for data_bags in cwd, if found then assume we are in the 27 | # chef-repo, otherwise look for 'chef-repo' 28 | def repo_file(relative_path) 29 | if File.directory?("data_bags") 30 | File.expand_path(relative_path, Dir.pwd) 31 | else 32 | repo_path = File.expand_path('chef-repo', Dir.pwd) 33 | File.expand_path(relative_path, repo_path) 34 | end 35 | end 36 | end 37 | end 38 | -------------------------------------------------------------------------------- /push_env.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path('../lib/opscode_deploy', __FILE__) 2 | 3 | module OpscodeDeploy 4 | class PushEnv < Chef::Knife 5 | include EnvironmentNames 6 | 7 | category "OPSCODE DEPLOYMENT" 8 | 9 | K = Chef::Knife 10 | 11 | banner "knife env push OPSCODE_ENV" 12 | 13 | deps do 14 | require 'chef/knife/data_bag_from_file' 15 | K::DataBagFromFile.load_deps 16 | end 17 | 18 | def run 19 | get_env_from_args! 20 | 21 | dbff = K::DataBagFromFile.new 22 | json_file = repo_file("data_bags/environments/#{environment}.json") 23 | dbff.name_args = %W{environments #{json_file}} 24 | dbff.run 25 | 26 | end 27 | end 28 | 29 | end 30 | -------------------------------------------------------------------------------- /role_tree.rb: -------------------------------------------------------------------------------- 1 | require 'chef/knife' 2 | 3 | module Kallistec 4 | class RoleTree < Chef::Knife 5 | 6 | deps do 7 | require 'chef/node' 8 | require 'chef/run_list' 9 | require 'chef/run_list/run_list_expansion' 10 | end 11 | 12 | banner "knife role tree NODE" 13 | 14 | attr_reader :node_name 15 | attr_reader :node 16 | 17 | def run 18 | unless Chef::RunList::RunListExpansion.instance_methods.map(&:to_s).include?("run_list_trace") 19 | ui.error "knife role tree requires Chef 10.14.0 beta or newer" 20 | exit 1 21 | end 22 | unless @node_name = name_args.first 23 | ui.error "You must specify a the name of the node you want to print the run list tree for" 24 | exit 1 25 | end 26 | 27 | @node = Chef::Node.load(node_name) 28 | 29 | tree_print("top level", expansion.run_list_trace) 30 | 31 | end 32 | 33 | def run_list 34 | @node.run_list 35 | end 36 | 37 | def expansion 38 | @expansion ||= run_list.expand("server") 39 | end 40 | 41 | def puts_indented(item, indentation) 42 | prefix = "" 43 | unless indentation == 0 44 | prefix = "| " * (indentation - 1) 45 | prefix << "|-" 46 | end 47 | puts "#{prefix}#{item}" 48 | end 49 | 50 | def tree_print(item, trace, depth=0) 51 | puts_indented(item, depth) 52 | trace[item.to_s].each { |sub_item| tree_print(sub_item, trace, depth + 1)} 53 | end 54 | 55 | end 56 | end 57 | 58 | 59 | 60 | 61 | 62 | -------------------------------------------------------------------------------- /set_rev.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path('../lib/opscode_deploy', __FILE__) 2 | 3 | module OpscodeDeploy 4 | class SetRev < Chef::Knife 5 | include EnvironmentNames 6 | 7 | category "OPSCODE DEPLOYMENT" 8 | 9 | banner "knife set rev PROJECT REVISION" 10 | 11 | deps do 12 | require 'yajl' 13 | end 14 | 15 | def env_dbag_file 16 | repo_file("data_bags/environments/#{environment}.json") 17 | end 18 | 19 | def assert_order_preserving_hashes 20 | if RUBY_VERSION !~ /^1\.9/ 21 | ui.error "Ruby 1.9 required so you don't reorder hashes (found: Ruby #{RUBY_VERSION})" 22 | exit 1 23 | end 24 | end 25 | 26 | def run 27 | assert_order_preserving_hashes 28 | get_env_from_args! 29 | @project, @rev = name_args[0], name_args[1] 30 | if @project.nil? || @rev.nil? 31 | ui.message "provide a project and a revision yo" 32 | exit 1 33 | end 34 | env_dbag_data = Yajl::Parser.parse(IO.read(env_dbag_file)) 35 | project_keys = env_dbag_data.keys.grep(/.*#{@project}.*\-revision/) 36 | @project_key = case project_keys.size 37 | when 1 38 | project_keys.first 39 | when 0 40 | ui.error "No project matches the name #@project" 41 | else 42 | ui.msg "Multiple projects match #@project, pick one:" 43 | ui.highline.choose(*project_keys) 44 | end 45 | ui.msg "#@project_key #{env_dbag_data[@project_key]} => #@rev" 46 | 47 | old_rev = env_dbag_data[@project_key] 48 | env_dbag_data[@project_key] = @rev 49 | File.open(env_dbag_file, "w"){|f| f.puts(Yajl::Encoder.encode(env_dbag_data, :pretty => true))} 50 | 51 | git_commit_pid = fork do 52 | Dir.chdir(repo_file("")) 53 | exec "git commit -v -e -m 'Bump #{environment} #@project_key from #{old_rev} to #@rev' data_bags/environments/#{environment}.json" 54 | end 55 | pid, status = Process.waitpid2(git_commit_pid) 56 | 57 | if status.success? 58 | git_push_pid = fork do 59 | Dir.chdir(repo_file("")) 60 | ui.msg "Hey, don't forget to git push" 61 | end 62 | else 63 | ui.error "Commit failed, exiting" 64 | exit 1 65 | end 66 | end 67 | 68 | end 69 | end 70 | -------------------------------------------------------------------------------- /show_rev.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path('../lib/opscode_deploy', __FILE__) 2 | 3 | module OpscodeDeploy 4 | class ShowRev < Chef::Knife 5 | include EnvironmentNames 6 | 7 | category "OPSCODE DEPLOYMENT" 8 | 9 | banner "knife show rev PROJECT REVISION" 10 | 11 | deps do 12 | require 'yajl' 13 | end 14 | 15 | def env_dbag_file 16 | repo_file("data_bags/environments/#{environment}.json") 17 | end 18 | 19 | def run 20 | get_env_from_args! 21 | @project = name_args[0] 22 | if @project.nil? 23 | ui.error "provide a project and a revision yo" 24 | exit 1 25 | end 26 | env_dbag_data = Yajl::Parser.parse(IO.read(env_dbag_file)) 27 | project_keys = env_dbag_data.keys.grep(/.*#{@project}.*\-revision/) 28 | selected_data = project_keys.inject({}) {|data, key| data[key] = env_dbag_data[key]; data} 29 | output(format_for_display(selected_data)) 30 | end 31 | 32 | end 33 | end 34 | --------------------------------------------------------------------------------