├── .gitignore ├── CHANGELOG.md ├── Gemfile ├── LICENSE ├── README.md ├── Rakefile ├── kitchen-sync.gemspec └── lib ├── kitchen-sync.rb ├── kitchen-sync ├── checksums.rb ├── core_ext.rb └── version.rb └── kitchen └── transport ├── rsync.rb └── sftp.rb /.gitignore: -------------------------------------------------------------------------------- 1 | Gemfile.lock 2 | pkg/ 3 | 4 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Kitchen-Sync Changelog 2 | 3 | ## v2.2.1 4 | 5 | Fix compatibility with current `kitchen-inspec` verifier. 6 | 7 | ## v2.2.0 8 | 9 | Add configurable `ruby_path` for `sftp` transport. 10 | 11 | ## v2.1.2 12 | 13 | Catch errors from closed sockets during instance shutdown. Fixes compat with 14 | net-ssh 4.x. 15 | 16 | ## v2.1.1 17 | 18 | Allow mixing kitchen-sync transports with other transports. 19 | 20 | ## v2.1.0 21 | 22 | Compatibility with new Test Kitchen features. 23 | 24 | ## v2.0.0 25 | 26 | Fully revamped at last for Test Kitchen's new modular transports. 27 | 28 | ## v1.1.1 29 | 30 | Bugfix for the new SFTP transport. 31 | 32 | ## v1.1.0 33 | 34 | First stab at a Test Kitchen 1.4 transport plugin. 35 | 36 | ## v1.0.1 37 | 38 | Bugfix for the SFTP transport provider. 39 | 40 | ## v1.0.0 41 | 42 | Initial release! 43 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | # 2 | # Author:: Noah Kantrowitz 3 | # 4 | # Copyright 2014, Noah Kantrowitz 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | source 'http://rubygems.org' 20 | 21 | gemspec 22 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | kitchen-sync 2 | ============ 3 | 4 | Do you wish your test-kitchen runs were faster? Do I ever have the gem for you! 5 | 6 | kitchen-sync provides alternate file transfer implementations for test-kitchen, 7 | most of which are faster than the default, thus speeding up your test runs. 8 | 9 | Quick Start 10 | ----------- 11 | 12 | Run `chef gem install kitchen-sync` and then set your transport to `sftp`: 13 | 14 | ``` 15 | transport: 16 | name: sftp 17 | ``` 18 | 19 | Available Transfer Methods 20 | -------------------------- 21 | 22 | ### `sftp` 23 | 24 | The default mode uses SFTP for file transfers, as well as a helper script to 25 | avoid recopying files that are already present on the test host. If SFTP is 26 | disabled, this will automatically fall back to the SCP mode. 27 | 28 | By default this will use the Chef omnibus Ruby, you can customize the path to 29 | Ruby via `ruby_path`: 30 | 31 | ``` 32 | transport: 33 | name: sftp 34 | ruby_path: /usr/bin/ruby 35 | ``` 36 | 37 | ### `rsync` 38 | 39 | The Rsync mode is based on the work done by [Mikhail Bautin](https://github.com/test-kitchen/test-kitchen/pull/359). 40 | This is the fastest mode, but it does have a few downsides. The biggest is that 41 | you must be using `ssh-agent` and have an identity loaded for it to use. It also 42 | requires that rsync be available on the remote side. Consider this implementation 43 | more experimental than `sftp` at this time. 44 | 45 | Windows Guests 46 | -------------- 47 | 48 | Windows is not specifically supported at this time, though if you have an SSH 49 | server it will probably work. There is no support for WinRM. 50 | 51 | Upgrading from 1.x 52 | ------------------ 53 | 54 | As of version 2.0, kitchen-sync uses Test Kitchen's modular transport system 55 | rather than monkey patch overrides. To upgrade, remove the `<% require 'kitchen-sync' %>` 56 | from your `.kitchen.yml` and add the transport configuration mentioned above. 57 | The `$KITCHEN_SYNC_MODE` environment variable is no longer needed as configuration 58 | can happen in the normal Yaml file. 59 | 60 | License 61 | ------- 62 | 63 | Copyright 2014-2016, Noah Kantrowitz 64 | 65 | Licensed under the Apache License, Version 2.0 (the "License"); 66 | you may not use this file except in compliance with the License. 67 | You may obtain a copy of the License at 68 | 69 | http://www.apache.org/licenses/LICENSE-2.0 70 | 71 | Unless required by applicable law or agreed to in writing, software 72 | distributed under the License is distributed on an "AS IS" BASIS, 73 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 74 | See the License for the specific language governing permissions and 75 | limitations under the License. 76 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env rake 2 | # 3 | # Author:: Noah Kantrowitz 4 | # 5 | # Copyright 2014, Noah Kantrowitz 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | require 'bundler/gem_tasks' 21 | -------------------------------------------------------------------------------- /kitchen-sync.gemspec: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | lib = File.expand_path('../lib', __FILE__) 3 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 4 | require 'kitchen-sync/version' 5 | 6 | Gem::Specification.new do |spec| 7 | spec.name = 'kitchen-sync' 8 | spec.version = KitchenSync::VERSION 9 | spec.authors = ['Noah Kantrowitz'] 10 | spec.email = ['noah@coderanger.net'] 11 | spec.description = %q{Improved file transfers for for test-kitchen} 12 | spec.summary = spec.description 13 | spec.homepage = 'https://github.com/coderanger/kitchen-sync' 14 | spec.license = 'Apache 2.0' 15 | 16 | spec.files = `git ls-files`.split($/) 17 | spec.executables = [] 18 | spec.test_files = spec.files.grep(%r{^(test|spec|features)/}) 19 | spec.require_paths = ['lib'] 20 | 21 | spec.add_dependency 'test-kitchen', '>= 1.0.0' 22 | spec.add_dependency 'net-sftp' 23 | 24 | spec.add_development_dependency 'bundler' 25 | spec.add_development_dependency 'rake' 26 | end 27 | -------------------------------------------------------------------------------- /lib/kitchen-sync.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2014-2016, Noah Kantrowitz 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | 18 | class KitchenSync 19 | autoload :VERSION, 'kitchen-sync/version' 20 | end 21 | -------------------------------------------------------------------------------- /lib/kitchen-sync/checksums.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Author:: Noah Kantrowitz 3 | # 4 | # Copyright 2014, Noah Kantrowitz 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | require 'json' 20 | require 'digest/sha1' 21 | 22 | glob_path = base = ARGV.first 23 | glob_path = File.join(glob_path, '**', '*') if File.directory?(glob_path) 24 | d = Digest::SHA1.new 25 | STDOUT.write( 26 | Dir.glob(glob_path, File::FNM_PATHNAME | File::FNM_DOTMATCH).inject({}) do |memo, path| 27 | rel_path = path[base.length..-1] 28 | if File.file?(path) && File.readable?(path) 29 | d.reset 30 | memo[rel_path] = d.file(path).hexdigest 31 | elsif File.directory?(path) 32 | memo[rel_path] = true 33 | end 34 | memo 35 | end.to_json 36 | ) 37 | -------------------------------------------------------------------------------- /lib/kitchen-sync/core_ext.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2014-2016, Noah Kantrowitz 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | # ┻━┻ ︵ヽ(`Д´)ﾉ︵ ┻━┻ 18 | module Kitchen 19 | # Monkey patch to prevent the deletion of everything 20 | module Provisioner 21 | class ChefBase < Base 22 | 23 | old_init_command = instance_method(:init_command) 24 | 25 | define_method(:init_command) do 26 | if (defined?(Kitchen::Transport::Sftp) && instance.transport.is_a?(Kitchen::Transport::Sftp)) || \ 27 | (defined?(Kitchen::Transport::Rsync) && instance.transport.is_a?(Kitchen::Transport::Rsync)) 28 | "mkdir -p #{config[:root_path]}" 29 | else 30 | old_init_command.bind(self).() 31 | end 32 | end 33 | 34 | end 35 | end 36 | end 37 | -------------------------------------------------------------------------------- /lib/kitchen-sync/version.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2014-2016, Noah Kantrowitz 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | 18 | class KitchenSync 19 | VERSION = '2.2.2.pre' 20 | end 21 | -------------------------------------------------------------------------------- /lib/kitchen/transport/rsync.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2014-2016, Noah Kantrowitz 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | require 'base64' 18 | 19 | require 'kitchen/transport/ssh' 20 | require 'net/ssh' 21 | 22 | require 'kitchen-sync/core_ext' 23 | 24 | module Kitchen 25 | module Transport 26 | class Rsync < Ssh 27 | def finalize_config!(instance) 28 | super.tap do 29 | if defined?(Kitchen::Verifier::Inspec) && instance.verifier.is_a?(Kitchen::Verifier::Inspec) 30 | instance.verifier.send(:define_singleton_method, :runner_options_for_rsync) do |config_data| 31 | runner_options_for_ssh(config_data) 32 | end 33 | end 34 | end 35 | end 36 | 37 | # Copy-pasta from Ssh#create_new_connection because I need the Rsync 38 | # connection class. 39 | # Tracked in https://github.com/test-kitchen/test-kitchen/pull/726 40 | def create_new_connection(options, &block) 41 | if @connection 42 | logger.debug("[SSH] shutting previous connection #{@connection}") 43 | @connection.close 44 | end 45 | 46 | @connection_options = options 47 | @connection = self.class::Connection.new(options, &block) 48 | end 49 | 50 | class Connection < Ssh::Connection 51 | def upload(locals, remote) 52 | if @rsync_failed || !File.exists?('/usr/bin/rsync') 53 | logger.debug('[rsync] Rsync already failed or not installed, not trying it') 54 | return super 55 | end 56 | 57 | locals = Array(locals) 58 | # We only try to sync folders for now and ignore the cache folder 59 | # because we don't want to --delete that. 60 | rsync_candidates = locals.select {|path| File.directory?(path) && File.basename(path) != 'cache' } 61 | ssh_command = "ssh #{ssh_args.join(' ')}" 62 | copy_identity 63 | rsync_cmd = "/usr/bin/rsync -e '#{ssh_command}' -az#{logger.level == :debug ? 'vv' : ''} --delete #{rsync_candidates.join(' ')} #{@session.options[:user]}@#{@session.host}:#{remote}" 64 | logger.debug("[rsync] Running rsync command: #{rsync_cmd}") 65 | ret = [] 66 | time = Benchmark.realtime do 67 | ret << system(rsync_cmd) 68 | end 69 | logger.info("[rsync] Time taken to upload #{rsync_candidates.join(';')} to #{self}:#{remote}: %.2f sec" % time) 70 | unless ret.first 71 | logger.warn("[rsync] rsync exited with status #{$?.exitstatus}, using SCP instead") 72 | @rsync_failed = true 73 | end 74 | 75 | # Fall back to SCP 76 | remaining = if @rsync_failed 77 | locals 78 | else 79 | locals - rsync_candidates 80 | end 81 | logger.debug("[rsync] Using fallback to upload #{remaining.join(';')}") 82 | super(remaining, remote) unless remaining.empty? 83 | end 84 | 85 | # Copy your SSH identity, creating a new one if needed 86 | def copy_identity 87 | return if @copied_identity 88 | identities = Net::SSH::Authentication::Agent.connect.identities 89 | raise 'No SSH identities found. Please run ssh-add.' if identities.empty? 90 | key = identities.first 91 | enc_key = Base64.encode64(key.to_blob).gsub("\n", '') 92 | identitiy = "ssh-rsa #{enc_key} #{key.comment}" 93 | @session.exec! <<-EOT 94 | test -e ~/.ssh || mkdir ~/.ssh 95 | test -e ~/.ssh/authorized_keys || touch ~/.ssh/authorized_keys 96 | if ! grep -q "#{identitiy}" ~/.ssh/authorized_keys ; then 97 | chmod go-w ~ ~/.ssh ~/.ssh/authorized_keys ; \ 98 | echo "#{identitiy}" >> ~/.ssh/authorized_keys 99 | fi 100 | EOT 101 | @copied_identity = true 102 | end 103 | 104 | def ssh_args 105 | args = %W{ -o UserKnownHostsFile=/dev/null } 106 | args += %W{ -o StrictHostKeyChecking=no } 107 | args += %W{ -o IdentitiesOnly=yes } if @options[:keys] 108 | args += %W{ -o LogLevel=#{@logger.debug? ? "VERBOSE" : "ERROR"} } 109 | args += %W{ -o ForwardAgent=#{options[:forward_agent] ? "yes" : "no"} } if @options.key? :forward_agent 110 | Array(@options[:keys]).each { |ssh_key| args += %W{ -i #{ssh_key}} } 111 | args += %W{ -p #{@session.options[:port]}} 112 | end 113 | end 114 | 115 | end 116 | end 117 | end 118 | -------------------------------------------------------------------------------- /lib/kitchen/transport/sftp.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2014-2016, Noah Kantrowitz 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | require 'benchmark' 18 | require 'digest/sha1' 19 | require 'json' 20 | 21 | require 'kitchen/transport/ssh' 22 | require 'net/sftp' 23 | 24 | require 'kitchen-sync/core_ext' 25 | 26 | 27 | module Kitchen 28 | module Transport 29 | class Sftp < Ssh 30 | CHECKSUMS_PATH = File.expand_path('../../../kitchen-sync/checksums.rb', __FILE__) 31 | CHECKSUMS_HASH = Digest::SHA1.file(CHECKSUMS_PATH) 32 | CHECKSUMS_REMOTE_PATH = "/tmp/checksums-#{CHECKSUMS_HASH}.rb" # This won't work on Windows targets 33 | MAX_TRANSFERS = 64 34 | 35 | default_config :ruby_path, '/opt/chef/embedded/bin/ruby' 36 | 37 | def finalize_config!(instance) 38 | super.tap do 39 | if defined?(Kitchen::Verifier::Inspec) && instance.verifier.is_a?(Kitchen::Verifier::Inspec) 40 | instance.verifier.send(:define_singleton_method, :runner_options_for_sftp) do |config_data| 41 | runner_options_for_ssh(config_data) 42 | end 43 | end 44 | end 45 | end 46 | 47 | # Copy-pasta from Ssh#create_new_connection because I need the SFTP 48 | # connection class. 49 | # Tracked in https://github.com/test-kitchen/test-kitchen/pull/726 50 | def create_new_connection(options, &block) 51 | if @connection 52 | logger.debug("[SSH] shutting previous connection #{@connection}") 53 | @connection.close 54 | end 55 | 56 | @connection_options = options 57 | @connection = self.class::Connection.new(config, options, &block) 58 | end 59 | 60 | class Connection < Ssh::Connection 61 | def initialize(config, options, &block) 62 | @config = config 63 | super(options, &block) 64 | end 65 | 66 | # Wrap Ssh::Connection#close to also shut down the SFTP connection. 67 | def close 68 | if @sftp_session 69 | logger.debug("[SFTP] closing connection to #{self}") 70 | begin 71 | sftp_session.close_channel 72 | rescue Net::SSH::Disconnect 73 | # Welp, we tried. 74 | rescue IOError 75 | # Can happen with net-ssh 4.x, no idea why. 76 | # See https://github.com/net-ssh/net-ssh/pull/493 77 | end 78 | end 79 | ensure 80 | @sftp_session = nil 81 | # Make sure we can turn down the session even if closing the channels 82 | # fails in the middle because of a remote disconnect. 83 | saved_session = @session 84 | begin 85 | super 86 | rescue Net::SSH::Disconnect 87 | # Boooo zlib warnings. 88 | saved_session.transport.close if saved_session 89 | end 90 | end 91 | 92 | def upload(locals, remote) 93 | Array(locals).each do |local| 94 | full_remote = File.join(remote, File.basename(local)) 95 | options = { 96 | recursive: File.directory?(local), 97 | purge: File.basename(local) != 'cache', 98 | } 99 | recursive = File.directory?(local) 100 | time = Benchmark.realtime do 101 | sftp_upload!(local, full_remote, options) 102 | end 103 | logger.info("[SFTP] Time taken to upload #{local} to #{self}:#{full_remote}: %.2f sec" % time) 104 | end 105 | end 106 | 107 | private 108 | 109 | def sftp_upload!(local, remote, recursive: true, purge: true) 110 | # Fast path check, if the remote path doesn't exist at all we just run a direct transfer 111 | unless safe_stat(remote) 112 | logger.debug("[SFTP] Fast path upload from #{local} to #{remote}") 113 | sftp_session.mkdir!(remote) if recursive 114 | sftp_session.upload!(local, remote, requests: MAX_TRANSFERS) 115 | return 116 | end 117 | # Get checksums for existing files on the remote side. 118 | logger.debug("[SFTP] Slow path upload from #{local} to #{remote}") 119 | copy_checksums_script! 120 | checksum_cmd = "#{@config[:ruby_path]} #{CHECKSUMS_REMOTE_PATH} #{remote}" 121 | logger.debug("[SFTP] Running #{checksum_cmd}") 122 | checksums = JSON.parse(session.exec!(checksum_cmd)) 123 | # Sync files that have changed. 124 | files_to_upload(checksums, local, recursive).each do |rel_path| 125 | upload_file(checksums, local, remote, rel_path) 126 | end 127 | purge_files(checksums, remote) if purge 128 | # Wait until all xfers are complete. 129 | sftp_loop(0) 130 | end 131 | 132 | # Bug fix for session.loop never terminating if there is an SFTP conn active 133 | # since as far as it is concerned there is still active stuff. 134 | # This function is Copyright Fletcher Nichol 135 | # Tracked in https://github.com/test-kitchen/test-kitchen/pull/724 136 | def execute_with_exit_code(command) 137 | exit_code = nil 138 | closed = false 139 | session.open_channel do |channel| 140 | 141 | channel.request_pty 142 | 143 | channel.exec(command) do |_ch, _success| 144 | 145 | channel.on_data do |_ch, data| 146 | logger << data 147 | end 148 | 149 | channel.on_extended_data do |_ch, _type, data| 150 | logger << data 151 | end 152 | 153 | channel.on_request("exit-status") do |_ch, data| 154 | exit_code = data.read_long 155 | end 156 | 157 | channel.on_close do |ch| # This block is new. 158 | closed = true 159 | end 160 | end 161 | end 162 | session.loop { exit_code.nil? && !closed } # THERE IS A CHANGE ON THIS LINE, PAY ATTENTION!!!!!! 163 | exit_code 164 | end 165 | 166 | # Create the SFTP session and block until it is ready. 167 | # 168 | # @return [Net::SFTP::Session] 169 | def sftp_session 170 | @sftp_session ||= session.sftp 171 | end 172 | 173 | # Return if the path exists (because net::sftp uses exceptions for that 174 | # and it makes code gross) and also raise an exception if the path is a 175 | # symlink. 176 | # 177 | # @param path [String] Remote path to check. 178 | # @return [Boolean] 179 | def safe_stat(path) 180 | stat = sftp_session.lstat!(path) 181 | raise "#{path} is a symlink, possible security threat, bailing out" if stat.symlink? 182 | true 183 | rescue Net::SFTP::StatusException 184 | false 185 | end 186 | 187 | # Upload the checksum script if needed. 188 | # 189 | # @return [void] 190 | def copy_checksums_script! 191 | # Fast path because upload itself is called multiple times. 192 | return if @checksums_copied 193 | # Only try to transfer the script if it isn't present. a stat takes about 194 | # 1/3rd the time of the transfer, so worst case here is still okay. 195 | sftp_session.upload!(CHECKSUMS_PATH, CHECKSUMS_REMOTE_PATH) unless safe_stat(CHECKSUMS_REMOTE_PATH) 196 | @checksums_copied = true 197 | end 198 | 199 | def files_to_upload(checksums, local, recursive) 200 | glob_path = if recursive 201 | File.join(local, '**', '*') 202 | else 203 | local 204 | end 205 | pending = [] 206 | Dir.glob(glob_path, File::FNM_PATHNAME | File::FNM_DOTMATCH).each do |path| 207 | next unless File.file?(path) 208 | rel_path = path[local.length..-1] 209 | remote_hash = checksums.delete(rel_path) 210 | pending << rel_path unless remote_hash && remote_hash == Digest::SHA1.file(path).hexdigest 211 | end 212 | pending 213 | end 214 | 215 | def upload_file(checksums, local, remote, rel_path) 216 | parts = rel_path.split('/') 217 | parts.pop # Drop the filename since we are only checking dirs 218 | parts_to_check = [] 219 | until parts.empty? 220 | parts_to_check << parts.shift 221 | path_to_check = parts_to_check.join('/') 222 | unless checksums[path_to_check] 223 | logger.debug("[SFTP] Creating directory #{remote}#{path_to_check}") 224 | add_xfer(sftp_session.mkdir("#{remote}#{path_to_check}")) 225 | checksums[path_to_check] = true 226 | end 227 | end 228 | logger.debug("[SFTP] Uploading #{local}#{rel_path} to #{remote}#{rel_path}") 229 | add_xfer(sftp_session.upload("#{local}#{rel_path}", "#{remote}#{rel_path}")) 230 | end 231 | 232 | def purge_files(checksums, remote) 233 | checksums.each do |key, value| 234 | # Check if the file was uploaded in #upload_file. 235 | if value != true 236 | logger.debug("[SFTP] Removing #{remote}#{key}") 237 | add_xfer(sftp_session.remove("#{remote}#{key}")) 238 | end 239 | end 240 | end 241 | 242 | def sftp_xfers 243 | @sftp_xfers ||= [] 244 | end 245 | 246 | def add_xfer(xfer) 247 | sftp_xfers << xfer 248 | sftp_loop 249 | end 250 | 251 | def sftp_loop(n_xfers=MAX_TRANSFERS) 252 | sftp_session.loop do 253 | sftp_xfers.delete_if {|x| !(x.is_a?(Net::SFTP::Request) ? x.pending? : x.active?) } # Purge any completed operations, which has two different APIs for some reason 254 | sftp_xfers.length > n_xfers # Run until we have fewer than max 255 | end 256 | end 257 | 258 | 259 | end 260 | 261 | end 262 | end 263 | end 264 | --------------------------------------------------------------------------------