├── CHANGELOG ├── test ├── helpers.rb ├── test_virb.rb └── test_vmth.rb ├── CONFIG.rdoc ├── Manifest ├── DESCRIPTION ├── Rakefile ├── lib ├── defaults.yaml ├── virb.rb └── vmth.rb ├── bin ├── virb └── vmth ├── README.rdoc ├── vmth.gemspec ├── QUICKSTART.rdoc ├── sample_config.yaml └── LICENSE /CHANGELOG: -------------------------------------------------------------------------------- 1 | v0.1. Initial Release 2 | -------------------------------------------------------------------------------- /test/helpers.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/ruby 2 | # 3 | $LOAD_PATH << File.expand_path( File.dirname(__FILE__) + '/../lib' ) 4 | require 'test/unit' 5 | 6 | -------------------------------------------------------------------------------- /CONFIG.rdoc: -------------------------------------------------------------------------------- 1 | == Config file format 2 | 3 | Here's the format of the test config file. it is a yaml file and specified 4 | by the '-c' parameter to vmth. It is included with the gem named 'sample_config.yaml' 5 | 6 | :include: sample_config.yaml 7 | 8 | -------------------------------------------------------------------------------- /test/test_virb.rb: -------------------------------------------------------------------------------- 1 | #!/bin/ruby 2 | require File.dirname(__FILE__)+"/helpers.rb" 3 | require 'virb' 4 | 5 | class VirbTest < Test::Unit::TestCase 6 | def setup 7 | true 8 | end 9 | def test_new 10 | assert defined?(p_test) 11 | assert defined?(f_apply) 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /Manifest: -------------------------------------------------------------------------------- 1 | CHANGELOG 2 | CONFIG.rdoc 3 | DESCRIPTION 4 | LICENSE 5 | Manifest 6 | QUICKSTART.rdoc 7 | README.rdoc 8 | Rakefile 9 | bin/virb 10 | bin/vmth 11 | lib/defaults.yaml 12 | lib/virb.rb 13 | lib/vmth.rb 14 | sample_config.yaml 15 | test/helpers.rb 16 | test/test_virb.rb 17 | test/test_vmth.rb 18 | vmth.gemspec 19 | -------------------------------------------------------------------------------- /DESCRIPTION: -------------------------------------------------------------------------------- 1 | The VMTH (Virtual Machine Test Harness) provides a mechanism to unit-test your infrastructure automation - puppet policies, chef recipes, release deployment scripts, etc. It uses features of modern VM monitors (like qemu) to snapshot system state, and then reset that state after each test, so that a series of tests can be performed on a VM instance in rapid succession. 2 | -------------------------------------------------------------------------------- /test/test_vmth.rb: -------------------------------------------------------------------------------- 1 | #!/bin/ruby 2 | require File.dirname(__FILE__)+"/helpers.rb" 3 | require 'vmth' 4 | 5 | class VmthTest < Test::Unit::TestCase 6 | def setup 7 | true 8 | end 9 | def test_true 10 | assert true 11 | end 12 | 13 | 14 | def test_new 15 | assert_nothing_raised do 16 | @vmth = Vmth.new() 17 | end 18 | end 19 | def test_vmcl 20 | @vmth = Vmth.new() 21 | vmcl = @vmth.vmcl() 22 | assert vmcl.class == String, "vmcl did not return a string" 23 | assert ! vmcl.empty? 24 | end 25 | end 26 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require 'rubygems' 2 | require 'rake' 3 | require 'echoe' 4 | 5 | Echoe.new('vmth', '0.0.2') do |p| 6 | p.description = File.open(File.dirname(__FILE__+"/DESCRIPTION")).read 7 | p.summary = "A VM test harness for testing operational configurations" 8 | p.url = "http://github.com/gregretkowski/vmth" 9 | p.author = "Greg Retkowski" 10 | p.email = "greg@rage.net" 11 | p.ignore_pattern = ["tmp/*", "script/*", "ol/*"] 12 | p.rdoc_template = nil 13 | p.rdoc_pattern = /^(lib|bin|tasks|ext)|^README|^CHANGELOG|^TODO|^LICENSE|^QUICKSTART|^CONFIG|^COPYING$/ 14 | # p.rdoc_template = "" 15 | p.development_dependencies = [] 16 | p.runtime_dependencies = [ 17 | 'formatr', 18 | 'net-ssh', 19 | 'net-scp', 20 | ] 21 | 22 | end 23 | -------------------------------------------------------------------------------- /lib/defaults.yaml: -------------------------------------------------------------------------------- 1 | # This is the defaults file for the vmth. You can use this to set system-wide 2 | # defaults for testing, for stuff that changes frequently you should specify 3 | # a config.yaml on the command line. 4 | vmm: 5 | cmdline: qemu-kvm -usb -usbdevice tablet -m 1024 -smp 1 -hda <%=@image_file%> -vnc :<%=@vm_vnc_port%> -net nic,macaddr=<%=@vm_mac_addr%> -net user -redir tcp:<%=@vm_ssh_port%>::22 -no-reboot -monitor stdio 6 | loadinit: loadvm init-test 7 | saveteststate: savevm test-freeze 8 | loadteststate: loadvm test-freeze 9 | quitvmm: quit 10 | ssh_port_start: 2224 11 | ssh_port_end: 2233 12 | vnc_port_start: 5903 13 | vnc_port_end: 5912 14 | mca_start: "00:50:56:36:b3:" 15 | timeout: 30 16 | prompt: "(qemu)" 17 | ssh: 18 | host: localhost 19 | user: root 20 | auth_methods: password 21 | password: "" 22 | paranoid: false 23 | init_scenario: 0nulltest 24 | prep: 25 | applying: 26 | testing: 27 | teardown: 28 | 29 | -------------------------------------------------------------------------------- /bin/virb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/ruby 2 | 3 | 4 | =begin rdoc 5 | 6 | This tool helps you to debug and browse through results from the 7 | VM test harness. Use to examine the output yaml produced by vmth. 8 | 9 | It is an extension to irb, so all irb commands work. The loaded 10 | yaml is stored in global variable $y. To get started use: 11 | 12 | load_obj 'filename' 13 | 14 | .. that will load the yaml output into virb.. Then use these 15 | commands to examine the outout.. 16 | 17 | p_apply 'scenario' # Shows output of apply step 18 | 19 | p_test 'scenario' # Shows output of test step 20 | 21 | f_apply 'scenario','filename' # Write out a scenario's output to a file 22 | 23 | =end 24 | 25 | #-- 26 | # Copyright 2011 Greg Retkowski 27 | # 28 | # Licensed under the Apache License, Version 2.0 (the "License"); 29 | # you may not use this file except in compliance with the License. 30 | # You may obtain a copy of the License at 31 | # 32 | # http://www.apache.org/licenses/LICENSE-2.0 33 | # 34 | # Unless required by applicable law or agreed to in writing, software 35 | # distributed under the License is distributed on an "AS IS" BASIS, 36 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 37 | # See the License for the specific language governing permissions and 38 | # limitations under the License. 39 | #++ 40 | 41 | exec "irb -r rubygems -r virb --simple-prompt" 42 | -------------------------------------------------------------------------------- /lib/virb.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/ruby 2 | 3 | #-- 4 | # Copyright 2011 Greg Retkowski 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | #++ 18 | 19 | require 'yaml' 20 | 21 | 22 | def load_obj(filename) 23 | $y = YAML.load_file(filename) 24 | true 25 | end 26 | def p_apply(service) 27 | puts $y['tests'][service]['apply'] 28 | true 29 | end 30 | def p_test(service) 31 | puts $y['tests'][service]['test'] 32 | true 33 | 34 | end 35 | # Write out the output of the 'apply' stage to a file. 36 | def f_apply(service,file) 37 | File.open(file,'w') do |f| 38 | f.puts $y['tests'][service]['apply'] 39 | end 40 | end 41 | def helpme() 42 | use = [] 43 | use << "Usage:" 44 | use << "" 45 | use << "load_obj 'filename' # Load the output of your vmth run" 46 | use << "p_apply 'scenario' # Shows output of apply step" 47 | use << "p_test 'scenario' # Shows output of test step" 48 | use << "f_apply 'scenario','filename' # Write out a scenario's output to a file" 49 | use << "helpme() # This help message" 50 | return use.join("\n") 51 | end 52 | 53 | puts helpme() 54 | -------------------------------------------------------------------------------- /README.rdoc: -------------------------------------------------------------------------------- 1 | === Introduction 2 | 3 | The VMTH (Virtual Machine Test Harness) provides a mechanism to unit-test your infrastructure automation - puppet policies, chef recipes, release deployment scripts, etc. It uses features of modern VM monitors (like qemu) to snapshot system state, and then reset that state after each test, so that a series of tests can be performed on a VM instance in rapid succession. 4 | 5 | It can be integrated with your continuous integration environment and triggered each time a commit is made to your automation code. 6 | 7 | 8 | === What it does 9 | 10 | It will go through a series of scenarios (a scenario is usualy a service name). It will run the 'prep' steps, freeze the vm, then run one scenario at a time, and unfreeze after 11 | each scenario. 12 | 13 | == Getting started 14 | 15 | See QUICKSTART[link:files/QUICKSTART_rdoc.html] to get started. 16 | 17 | == Command-line tools 18 | 19 | * vmth[link:files/bin/vmth.html]: vm test harness command-line tool 20 | * virb[link:files/bin/virb.html]: Interactive vm test debugger... 21 | 22 | == Gochas 23 | 24 | Took a while to track down a bug where our VM instance was running out of memory and then crashing. It just appeared that qemu just plain exited without error. traced it to the guest having vm.panic_on_oom to 1 - which causes the kernel to panic and exit if we run out of memory. 25 | 26 | Here is the Vmth[link:classes/Vmth.html] class documentation. 27 | 28 | == Config file format 29 | 30 | See sample_config.yaml[link:files/CONFIG_rdoc.html] for the vmth config file format. 31 | 32 | == Support 33 | 34 | For support or to report bugs open an issue at the gitub issue tracker: 35 | 36 | https://github.com/gregretkowski/vmth/issues 37 | 38 | -------------------------------------------------------------------------------- /vmth.gemspec: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | 3 | Gem::Specification.new do |s| 4 | s.name = %q{vmth} 5 | s.version = "0.0.1" 6 | 7 | s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version= 8 | s.authors = ["Greg Retkowski"] 9 | s.date = %q{2011-04-26} 10 | s.description = %q{require 'rubygems' require 'rake' require 'echoe' Echoe.new('vmth', '0.0.1') do |p| p.description = File.open(File.dirname(__FILE__+"/DESCRIPTION")).read p.summary = "A VM test harness for testing operational configurations" p.url = "http://github.com/gregretkowski/vmth" p.author = "Greg Retkowski" p.email = "greg@rage.net" p.ignore_pattern = ["tmp/*", "script/*", "ol/*"] p.rdoc_template = nil p.rdoc_pattern = /^(lib|bin|tasks|ext)|^README|^CHANGELOG|^TODO|^LICENSE|^QUICKSTART|^COPYING$/ # p.rdoc_template = "" p.development_dependencies = [] p.runtime_dependencies = [ 'formatr', 'net-ssh', 'net-scp', ] end} 11 | s.email = %q{greg@rage.net} 12 | s.executables = ["virb", "vmth"] 13 | s.extra_rdoc_files = ["CHANGELOG", "LICENSE", "QUICKSTART.rdoc", "README.rdoc", "bin/virb", "bin/vmth", "lib/defaults.yaml", "lib/virb.rb", "lib/vmth.rb"] 14 | s.files = ["CHANGELOG", "DESCRIPTION", "LICENSE", "Manifest", "QUICKSTART.rdoc", "README.rdoc", "Rakefile", "bin/virb", "bin/vmth", "lib/defaults.yaml", "lib/virb.rb", "lib/vmth.rb", "sample_config.yaml", "test/helpers.rb", "test/test_virb.rb", "test/test_vmth.rb", "vmth.gemspec"] 15 | s.has_rdoc = true 16 | s.homepage = %q{http://github.com/gregretkowski/vmth} 17 | s.rdoc_options = ["--line-numbers", "--inline-source", "--title", "Vmth", "--main", "README.rdoc"] 18 | s.require_paths = ["lib"] 19 | s.rubyforge_project = %q{vmth} 20 | s.rubygems_version = %q{1.3.1} 21 | s.summary = %q{A VM test harness for testing operational configurations} 22 | s.test_files = ["test/test_vmth.rb", "test/test_virb.rb"] 23 | 24 | if s.respond_to? :specification_version then 25 | current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION 26 | s.specification_version = 2 27 | 28 | if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then 29 | s.add_runtime_dependency(%q, [">= 0"]) 30 | s.add_runtime_dependency(%q, [">= 0"]) 31 | s.add_runtime_dependency(%q, [">= 0"]) 32 | else 33 | s.add_dependency(%q, [">= 0"]) 34 | s.add_dependency(%q, [">= 0"]) 35 | s.add_dependency(%q, [">= 0"]) 36 | end 37 | else 38 | s.add_dependency(%q, [">= 0"]) 39 | s.add_dependency(%q, [">= 0"]) 40 | s.add_dependency(%q, [">= 0"]) 41 | end 42 | end 43 | -------------------------------------------------------------------------------- /QUICKSTART.rdoc: -------------------------------------------------------------------------------- 1 | This is a not-so-quickstart introduction to getting vmth up and running in your environment. 2 | 3 | == Setting up system images 4 | 5 | Get a copy of one of your system images 6 | 7 | One way to do this would be to boot from a rescue CD, then 'dd if=/dev/sda of=/some/other/filesystem/my-system.img'. presumably you'd want your system to have a fairly small harddrive to do this! 8 | 9 | If your system image is shrunk down (as we do) you can add space to the end of it via: 'dd if=/dev/zero bs=10240 count=5000000 >>your.img' 10 | 11 | Next convert the system image to a qemu compatable cow (copy-on-write) file. 12 | 13 | qemu-img convert -f raw linux-img-6.img -O qcow2 linux-img-6-large.cow 14 | 15 | == Setting up your 'host' system.. 16 | Make sure you have qemu & kvm installed - and the kernel module loaded. Performance was unacceptably slow when running with the CPU emulated instead of directly accessed via kvm. 17 | 18 | == Editing your disk image. 19 | 20 | use 'vmth -q -i /your/image/file.cow 21 | 22 | This will spit out the qemu command line to launch your image. 23 | Run that - log into your system and configure it to be ready for vmth to interact with it. Some changes you may want to make are: removing the root password, ensuring sshd will allow root to login w/o a password.. change /etc/hosts if you have to hard-code some of the hosts your system will talk to, etc... 24 | 25 | Once you are done enter 'savevm init-test' and quit qemu. 26 | 27 | This saves the current state of the VM - so that when the vmth runs the test it doesn't have to boot the vm - it starts from that precise moment. 28 | 29 | == Creating the yaml specification for your tests 30 | 31 | Next you'll want to edit the test config yaml for your test. You can use the 'sample_config.yaml' as a starting point. The 'prep' steps are completed once before testing starts - the 'applying' applies changes to your system in preparation for a test. The 'testing' runs your unit test script. and the teardown is done after applying and testing has completed. Most commands are passed through ERB, so you can put expansions here. 32 | 33 | Put a list of scenarios you want to test into a file, like testable_scenarios.txt. Each scenario should be a single word on its own line. 34 | 35 | == Running the test 36 | sudo bin/vmth -e testable_scenarios.txt -i /data/vms/system-test.cow -p /path/to/my/automation/code -c my_config.yaml -d -o my_config.output.yaml 37 | 38 | After your run completes, the vmth will print a report of what passed or failed via the 'applying' and the 'testing' steps. You can use 'virb' to look at your output yaml and understand where tests may have failed. 39 | 40 | == Debugging 41 | 42 | use 'vmth --console' for debugging; it'll launch the vm, apply all the prep steps, and then leave you at a qemu prompt.. From here you can ssh into the machine and interact with it at will.. and you can always reset via 'loadvm init-test' or 'loadvm test-freeze' to get you back to the start. 43 | 44 | 45 | -------------------------------------------------------------------------------- /sample_config.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # Strings are passed through ERB, so you can use this to templatize your config 3 | # Variables you might use via ERB are... 4 | # @source_path - specified on the command-line via '-p' 5 | # @scenario - name of the scenario currently under test 6 | # 7 | # Each section below provides an array of directives. 8 | # Directives are processed in order. Valid directives are: 9 | # upload: [ host-path, vm-path ] - copy a file to the vm 10 | # upload_recurse: [ host-path, vm-path ] - recursively copy files to vm 11 | # download: [ vm-path, host-path ] - copy a file down from the vm 12 | # download_recurse: [ vm-path, host-path ] - same, but recursive 13 | # cmd: some-shell-command - execute a command on the vm 14 | # cmdtest: [ "some-shell-command", !ruby/regexp /output-when-cmd-fails/ ] 15 | # This command will be run and its output will be evaluated for a string 16 | # which shows the command failed. If this string is found or the 17 | # command exits non-zero the section is considered 'FAILED' and reported 18 | # as such in the text output 19 | # foreach: [ args [array-of-args], directive1, directive2, .. ] 20 | # This is used to loop over a series of commands. 21 | 22 | # You can also override settings from defaults.yaml in this file. 23 | 24 | prep: 25 | - cmd: sysctl vm.panic_on_oom=0 26 | - cmd: mkdir -p /home/ops-envs/development 27 | - foreach: 28 | - args: [ dnsdata, rubylib, scripts, service ] 29 | - cmd: svn export --force https://svn/trunk/<%=@arg%> /home/ops-envs/development/<%=@arg%> 30 | - upload_recurse: [ "<%=@source_path%>/.", /home/ops-envs/development/puppet ] 31 | - foreach: 32 | - args: [ ldap.conf, nsswitch.conf ] 33 | - upload: [ "<%=@source_path%>/dist/ldap-client/<%=@arg%>", "/etc/<%=@arg%>" ] 34 | - upload: [ "<%=@source_path%>/dist/puppetmaster/fileserver.rb", /usr/lib/ruby/site_ruby/1.8/puppet/network/handler/fileserver.rb ] 35 | - cmd: cp /home/ops-envs/development/puppet/dist/puppet/puppet.conf /etc/puppet/puppet.conf 36 | - cmd: cp /home/ops-envs/development/puppet/dist/puppetmaster/fileserver.conf /etc/puppet 37 | - cmd: rm /etc/puppet/puppetd.conf 38 | - cmd: rm /etc/puppet/puppetmasterd.conf 39 | - cmd: mkdir -p /var/cache/rsync 40 | - cmd: /sbin/service puppetmaster restart 41 | - upload: [ "<%=@source_path%>/../get_device_services.rb", /etc/puppet/get_device_services.rb ] 42 | - upload: [ "<%=@source_path%>/../machdb_hostdata.yaml", /etc/machdb_hostdata.yaml ] 43 | - cmd: chattr +i /etc/hosts 44 | - cmd: chattr +i /etc/ssh/sshd_config 45 | - cmd: sleep 5 46 | applying: 47 | - cmd: cp /etc/shadow /etc/shadow.bak 48 | - cmd: /sbin/service puppetmaster restart 49 | - cmd: "echo <%=@scenario%> >/etc/test_this_service" 50 | - cmdtest: [ "pp", !ruby/regexp /Not using cache on failed catalog/ ] 51 | testing: 52 | - cmdtest: [ "ruby /home/ops-envs/development/puppet/tests/test_<%=@scenario%>.rb 2>&1", !ruby/regexp /Failure:/ ] 53 | teardown: 54 | - cmd: cp /etc/shadow.bak /etc/shadow 55 | -------------------------------------------------------------------------------- /bin/vmth: -------------------------------------------------------------------------------- 1 | #!/usr/bin/ruby 2 | 3 | =begin rdoc 4 | This is the command line tool to invoke the puppet test harness. 5 | 6 | Usage: vmth [options] 7 | -s, --service service_name Test this service 8 | -q, --vmm-cmd-line Just output the vm monitor command line and exit 9 | -t, --console Prep a vm and allocate it to a user 10 | -c, --config-yaml file.yaml The file used to configure the vmth 11 | -d, --debug Run in debugging mode 12 | -n, --no-vmm Run w/o VM management (for dev/debug) 13 | -i /path/to/image.cow Specify the vm disk image file 14 | --image-file 15 | -y, --yaml Output Yaml instead of formatted text 16 | -e /path/to/scenarios Specify the scenarios list file 17 | --scenarios-file 18 | -o, --output-file output.yaml Output results to a yaml file 19 | -p /path/to/your/source/dir Path to your source directory 20 | --path 21 | -h, -? Show this message 22 | 23 | One example command-line may look like: 24 | 25 | sudo vmth -e testable_services.txt -i /data/greg-test.cow -p /home/greg/puppet -c testpup26.yaml -d -o testpup26.output.yaml 26 | 27 | See sample_config.yaml for what the config file specified by '-c' should contain. 28 | 29 | The output of the test harness looks something like... 30 | 31 | VM Test Harness Results.... 32 | Elapsed time to complete testing: 06:08:14 33 | ------------------+------------+------------+----------------+ 34 | Scenario | Apply | Test | Execution Time | 35 | ------------------+------------+------------+----------------+ 36 | 0nulltest | Passed | Passed | 00:22:37 | 37 | dbserver | Passed | Passed | 00:09:29 | 38 | loadbalancer | Passed | Passed | 00:04:04 | 39 | webserver | Passed | Passed | 00:04:03 | 40 | ... 41 | 42 | =end 43 | 44 | #-- 45 | # Copyright 2011 Greg Retkowski 46 | # 47 | # Licensed under the Apache License, Version 2.0 (the "License"); 48 | # you may not use this file except in compliance with the License. 49 | # You may obtain a copy of the License at 50 | # 51 | # http://www.apache.org/licenses/LICENSE-2.0 52 | # 53 | # Unless required by applicable law or agreed to in writing, software 54 | # distributed under the License is distributed on an "AS IS" BASIS, 55 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 56 | # See the License for the specific language governing permissions and 57 | # limitations under the License. 58 | #++ 59 | 60 | require 'rubygems' 61 | require 'vmth' 62 | require 'optparse' 63 | require 'formatr' 64 | progname = File.basename($0) 65 | 66 | # PARSE COMMAND LINE OPTIONS 67 | 68 | options = { 69 | :action => "all", 70 | :qemu => true, 71 | :path => ".", 72 | :output => 'text', 73 | :debug => false 74 | } 75 | op = OptionParser.new do |opts| 76 | opts.banner = "Usage: #{progname} [options]" 77 | opts.on("-s", "--service service_name", "Test this service") do |s| 78 | options[:action] = 'service' 79 | options[:services] ||= Array.new() 80 | options[:services] << s 81 | end 82 | opts.on("-q", "--vmm-cmd-line", "Just output the vm monitor command line and exit") do 83 | options[:action] = "vmm-command-line" 84 | end 85 | opts.on("-t", "--console", "Prep a vm and allocate it to a user") do 86 | options[:action] = "console" 87 | end 88 | 89 | opts.on("-c", "--config-yaml file.yaml", "The file used to configure the vmth") do |f| 90 | options[:config_file] = f 91 | end 92 | opts.on("-d", "--debug", "Run in debugging mode") do 93 | options[:debug] = true 94 | end 95 | opts.on("-n", "--no-vmm","Run w/o VM management (for dev/debug)") do 96 | options[:vmm_enabled] = false 97 | end 98 | opts.on("-i", "--image-file /path/to/image.cow","Specify the vm disk image file") do |i| 99 | options[:image_file] = i 100 | end 101 | opts.on("-y", "--yaml","Output Yaml instead of formatted text") do 102 | options[:out_format] = 'yaml' 103 | end 104 | opts.on("-e", "--scenarios-file /path/to/scenarios","Specify the scenarios list file") do |e| 105 | options[:scenarios_file] = e 106 | end 107 | opts.on("-o", "--output-file output.yaml","Output results to a yaml file") do |o| 108 | options[:out_file] = o 109 | end 110 | opts.on("-p", "--path /path/to/your/source/dir","Path to your source directory") do |p| 111 | options[:source_path] = p 112 | end 113 | opts.on_tail("-h", "-?", "Show this message") do 114 | puts "Runs the VM Test Harness" 115 | puts 116 | puts opts 117 | exit 118 | end 119 | end.order! 120 | 121 | # FORMATTING FOR PRETTY OUTPUT 122 | 123 | include FormatR 124 | @top_ex = <>>>>>>>>> |@>>>>>>>>>> |@>>>>>>>>>>>>>> | 134 | s[:name], s[:apply], s[:test], s[:elapsed] 135 | TO 136 | @footer_ex = < k, 157 | :apply => (v['apply']['passed'] ? "Passed" : "FAILED" ), 158 | :test => (v['test']['passed'] ? "Passed" : "FAILED" ), 159 | :elapsed => (pretty_secs(v['elapsed_time'])), 160 | :total_elapsed => (pretty_secs(results['elapsed_time'])) 161 | } 162 | @body_fmt.printFormat(binding) 163 | end 164 | end 165 | 166 | # EXECUTE TESTS AND PRINT RESULTS 167 | 168 | def err_catcher() #:nodoc: 169 | begin 170 | yield 171 | rescue PTY::ChildExited, Errno::EIO => e 172 | @qemu_r.flush rescue nil 173 | $stderr.puts "Bad things happened to vmm!" 174 | $stderr.puts "#{e.class}: #{e.message}" 175 | $stderr.puts "Last output:" 176 | $stderr.puts @qemu_r 177 | raise e.class, e.message 178 | end 179 | end 180 | 181 | @test = Vmth.new(options) 182 | 183 | 184 | unless options[:image_file] 185 | $stderr.puts "Image file is mandatory" 186 | exit 1 187 | end 188 | unless options[:config_file] 189 | $stderr.puts "Config file is mandatory" 190 | exit 1 191 | end 192 | 193 | if options[:action] == "vmm-command-line" 194 | puts @test.vmcl 195 | exit 196 | elsif options[:action] == 'console' 197 | @test.console 198 | exit 199 | elsif options[:action] == 'service' 200 | err_catcher { results = @test.test_services(options[:services]) } 201 | elsif options[:action] == 'all' 202 | err_catcher { results = @test.test } 203 | end 204 | results = @test.results 205 | 206 | if options[:out_file] 207 | File.open(options[:out_file],"w") do |f| 208 | f.puts YAML.dump(results) 209 | end 210 | end 211 | if options[:output] == 'yaml' 212 | puts YAML.dump(results) 213 | elsif options[:output] == 'text' 214 | print_pretty(results) 215 | end 216 | 217 | # Handle the exit code.. Exit with the number of 'fails' 218 | # we have.. That'll indicate to a continuous integration 219 | # system that it 'failed' if exits != 0. 220 | def get_exit_code(results) #:nodoc: 221 | pass_array = results['tests'].collect do |k,v| 222 | v['test']['passed'] and v['apply']['passed'] 223 | end 224 | if pass_array.size == 0 225 | return 255 226 | end 227 | return pass_array.select{|g| g == false}.size 228 | end 229 | # remove state file. 230 | @test.cleanup 231 | exit get_exit_code(results) 232 | 233 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright 2011 Greg Retkowski 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /lib/vmth.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/ruby 2 | 3 | #-- 4 | # Copyright 2011 Greg Retkowski 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | #++ 18 | 19 | require 'rubygems' 20 | require 'net/ssh' 21 | require 'net/scp' 22 | require 'pty' 23 | require 'expect' 24 | require 'fileutils' 25 | require 'tempfile' 26 | require 'erb' 27 | 28 | 29 | class VmthError < RuntimeError;end 30 | 31 | =begin rdoc 32 | This class provides a VM test harness to allow testing of operational 33 | code (puppet policies, chef configs, etc..) against a environment 34 | similar to your production environment. 35 | 36 | The VM test harness uses features of the VM monitor (qemu) to freeze 37 | and re-use system memory/disk state so that a series of test scenarios 38 | can be rapidly tested. 39 | 40 | This class provides all the logic to implement the VM test harness. It 41 | manages the VM, loads and runs tests for each scenario, and produces 42 | a 'results' hash with the results of the test. 43 | =end 44 | 45 | class Vmth 46 | # Set the directory where your puppet code directory is. 47 | attr_accessor :source_dir 48 | # A boolean, should we use QEMU or not? Should almost always be true. 49 | attr_accessor :vmm_enabled 50 | # The machdb services.yaml location. Describes which services should be tested. 51 | attr_accessor :scenarios_file 52 | # The system/disk image file booted by QEMU 53 | attr_accessor :image_file 54 | # Contains the hash of all test output and results. Read this after 55 | # you've completed your test run. 56 | attr_reader :results 57 | # So we can flush this if there's an error. 58 | attr_accessor :vmm_r 59 | attr_reader :options 60 | # 61 | # new takes no arguments. 62 | # 63 | DEFAULT_OPTIONS={ 64 | :source_path => ".", 65 | :vmm_enabled => true, 66 | :config_file => nil, 67 | :scenarios_file => nil, 68 | :image_file => nil, 69 | :debug => false, 70 | :action => 'all', 71 | :outfile => self.class.to_s.downcase+"_out.yaml", 72 | :out_format => 'text', 73 | :services => [] 74 | } 75 | 76 | def initialize(options={}) 77 | @options=DEFAULT_OPTIONS.merge(options) 78 | @config = YAML.load_file(File.dirname(__FILE__)+'/defaults.yaml') 79 | if @options[:config_file] 80 | @config.merge!(YAML.load_file(@options[:config_file])) 81 | end 82 | @log = Logger.new(STDERR) 83 | if @options[:debug] 84 | @log.level = Logger::DEBUG 85 | else 86 | @log.level = Logger::WARN 87 | end 88 | @tmp_state = Tempfile.new("pth").path 89 | @results = { 90 | 'tests' => {} 91 | } 92 | ssh_port_range=@config['vmm']['ssh_port_start']..@config['vmm']['ssh_port_end'] 93 | @vm_ssh_port = Vmth.allocate_tcp_port(ssh_port_range) 94 | vnc_port_range=@config['vmm']['vnc_port_start']..@config['vmm']['vnc_port_end'] 95 | @vm_vnc_port = Vmth.allocate_tcp_port(vnc_port_range) - 5900 96 | @vm_mac_addr = @config['vmm']['mca_start'] + "%02x" % (rand()*256).round 97 | @vmm_prompt = eb(@config['vmm']['prompt']) 98 | @vmm_timeout = @config['vmm']['timeout'] 99 | @image_file=@options[:image_file] 100 | @source_path=@options[:source_path] 101 | # Try to cleanly shutdown the vmm. 102 | trap("INT") do 103 | if @vm_running 104 | stop_vm() 105 | end 106 | raise 107 | end 108 | end 109 | # Expand a string with ERB. 110 | def eb(string) 111 | renderer = ERB.new(string) 112 | return renderer.result(binding) 113 | end 114 | # Change the loglevel of the logger. Argument should 115 | # be a loglevel constant, i.e. Logger::INFO 116 | def loglevel=(level) 117 | @log.level=level 118 | end 119 | # Test all testable services - this is indicated by if a service in machdb 120 | # has the 'testable' field set to true. It takes no arguments and returns 121 | # an array of booleans, indicating the success or failure of tests. You 122 | # should query results() for your results. 123 | def test_all 124 | @results['test_start'] = Time.now() 125 | passed = [] 126 | boot_vm() if @options[:vmm_enabled] 127 | prep 128 | freeze_vm() if @options[:vmm_enabled] 129 | @log.info "RUNNING NO-SERVICE TEST" 130 | passed << one_test(@config['init_scenario']) 131 | # Stop testing if our initial test fails. 132 | unless passed.first == true 133 | @log.error "Initial setup failed.. sleeping 60 seconds for debugging." 134 | sleep 60 135 | stop_vm() if @options[:vmm_enabled] 136 | return passed 137 | end 138 | freeze_vm() if @options[:vmm_enabled] 139 | @log.info "RUNNING TESTS" 140 | scenarios = get_scenarios 141 | test_counter = 0 142 | scenarios.each do |scenario| 143 | test_counter += 1 144 | @log.info "Running test for #{scenario} - #{test_counter} of #{scenarios.size}" 145 | passed << one_test(scenario) 146 | end 147 | stop_vm() if @config[:vmm_enabled] 148 | all_passed = passed.select{|p| p == false}.size == 0 149 | @log.info "Number of tests run : #{passed.size}" 150 | @log.info "Result of ALL tests: Passed? #{all_passed}" 151 | @results['test_stop'] = Time.now() 152 | @results['elapsed_time'] = @results['test_stop'] - @results['test_start'] 153 | return all_passed 154 | end 155 | alias :test :test_all 156 | 157 | # Set up a vm, and drop it off for a developer to use. 158 | def console 159 | create_private_disk 160 | @results['test_start'] = Time.now() 161 | passed = [] 162 | boot_vm() if @options[:vmm_enabled] 163 | prep 164 | freeze_vm() if @options[:vmm_enabled] 165 | # Print out ssh & vnc port, and freeze name. 166 | @log.info "Handing off VM to you.. Type #{@config['vmm']['quitvmm']} to end session." 167 | @log.info "Ports - SSH: #{@vm_ssh_port} VNC: #{@vm_vnc_port}" 168 | 169 | # hand off console. 170 | print @config['vmm']['prompt'] 171 | begin 172 | system('stty raw -echo') 173 | Thread.new{ loop { @vmm_w.print $stdin.getc.chr } } 174 | loop { $stdout.print @vmm_r.readpartial(512); STDOUT.flush } 175 | rescue 176 | nil # User probably caused the VMM to exit. 177 | ensure 178 | system "stty -raw echo" 179 | end 180 | # Done via the user? 181 | # stop_vm() 182 | cleanup_private_disk 183 | return 184 | end 185 | def create_private_disk 186 | @orig_image_file = @image_file 187 | @image_file = "#{@orig_image_file}.#{$$}" 188 | @log.debug "Copying #{@orig_image_file} to #{@image_file}" 189 | FileUtils.cp(@orig_image_file,@image_file) 190 | end 191 | def cleanup_private_disk 192 | @log.debug "Removing tmp imagefile #{@image_file}" 193 | if defined?(@orig_image_file) and @orig_image_file != @image_file 194 | File.delete(@image_file) 195 | end 196 | end 197 | 198 | # Cleanup state file, but only if everything is done! 199 | def cleanup 200 | File.delete(@tmp_state) rescue nil 201 | end 202 | # Really only for development/testing of this class. 203 | # Will run tests against an already running VM (presumably the 204 | # developer is running it in another window) 205 | def test_without_vm 206 | prep 207 | test_services 208 | end 209 | # Test a bunch of services. Pass in an array containing the names 210 | # of services to test. Returns an array of booleans, indicating 211 | # the success or failure of the tests. You should read detailed 212 | # results from results() 213 | def test_services(services) 214 | @results['test_start'] = Time.now() 215 | boot_vm() if @options[:vmm_enabled] 216 | prep 217 | freeze_vm() if @options[:vmm_enabled] 218 | passed = [] 219 | @log.info "RUNNING NO-SERVICE TEST" 220 | passed << one_test(eb(@config["init_scenario"])) 221 | # Stop testing if our initial test fails. 222 | unless passed.first == true 223 | stop_vm() if @options[:vmm_enabled] 224 | return passed 225 | end 226 | freeze_vm() if @options[:vmm_enabled] 227 | @log.info "RUNNING TESTS" 228 | test_counter = 0 229 | services.each do |service| 230 | test_counter += 1 231 | @log.info "Running test for #{service} - #{test_counter} of #{services.size}" 232 | passed << one_test(service) 233 | end 234 | stop_vm() if @options[:vmm_enabled] 235 | @results['test_stop'] = Time.now() 236 | @results['elapsed_time']= @results['test_stop'] - @results['test_start'] 237 | return passed 238 | end 239 | # Return the command-line that would have been used to start QEMU. 240 | # This can be used for developing this library, or to get a new 241 | # disk image prepped to be used with the test harness. 242 | def vmcl 243 | return vmm_command_line 244 | end 245 | # 246 | # START PRIVATE METHODS 247 | # 248 | private 249 | # This starts the QEMU instance for the test VM. Spawns the VM 250 | # and then sets @qemu_r (read socket for qemu), @qemu_w (write 251 | # socket for qemu) and @qemu_pid (qemu process ID). 252 | # These class variables are used to interact with the QEMU 253 | # supervisor. 254 | def start_vm 255 | unless File.exists?(@image_file) and File.owned?(@image_file) 256 | @log.error "Image file #{@image_file} doesn't exist or is not owned by you!" 257 | exit 255 258 | end 259 | @log.info "VM Will use SSH Port #{@vm_ssh_port} and VNC Port #{@vm_vnc_port}" 260 | @log.info "Starting vmm now..." 261 | @log.debug "vmm command line is: " + vmm_command_line() 262 | @vmm_r, @vmm_w, @vmm_pid = PTY.spawn vmm_command_line() 263 | @vmm_r.expect(@vmm_prompt,@vmm_timeout) do |line| 264 | true 265 | end 266 | @vm_running = true 267 | @log.debug "vmm instance pid is #{@vmm_pid}" 268 | end 269 | # Read in the scenarios file and return it as an array. 270 | def get_scenarios 271 | scenarios = [] 272 | File.open(@options[:scenarios_file]) do |f| 273 | f.each_line do |line| 274 | scenarios << line.chomp 275 | end 276 | end 277 | return scenarios.sort 278 | end 279 | # Returns a command-line for invoking QEMU. Used by 280 | # start_qemu 281 | def vmm_command_line 282 | return eb(@config['vmm']['cmdline']) 283 | end 284 | # Stops the vmm process. First tries to issue the 'quit' 285 | # command on the qemu console. 286 | def stop_vm 287 | exit_status = nil 288 | @vm_running = false 289 | begin 290 | exit_status = vmm_command(eb(@config['vmm']['quitvmm'])) 291 | sleep 1 292 | # Check to see if it is still running. 293 | is_alive = (Process.kill(0, @vmm_pid) rescue 0) 294 | if is_alive != 0 295 | @log.warn "Warning, vmm didn't die.. killing manually" 296 | Process.kill("TERM",@vmm_pid) 297 | sleep 2 298 | end 299 | rescue PTY::ChildExited 300 | true # expected 301 | end 302 | return exit_status 303 | end 304 | # Issue a command to the QEMU supervisor. Used 305 | # for saving or restoring VM state between tests. 306 | def vmm_command(command) 307 | return nil unless @options[:vmm_enabled] 308 | result = nil 309 | @log.debug "Issuing '#{command}' to vmm" 310 | return nil unless @vmm_w 311 | @vmm_w.puts("#{command}\n") 312 | begin 313 | @vmm_r.expect(@vmm_prompt,@vmm_timeout) do |line| 314 | @log.debug "Expect line was: #{line}" 315 | result = line 316 | end 317 | # Handle quick exit on 'quit' commands. 318 | rescue PTY::ChildExited, Errno::EIO => e 319 | if command == eb(@config['vmm']['quitvmm']) 320 | @log.debug "Command 'quit' exited before completion." 321 | else 322 | raise e 323 | end 324 | end 325 | @log.debug "Command completed with result '#{result}'" 326 | return result 327 | end 328 | def one_test(service) 329 | reset_vm 330 | passed = true 331 | @log.info "Running test for #{service}" 332 | passed = run_vmth_test(service) 333 | @log.info "Did it pass?: #{passed}" 334 | return passed 335 | end 336 | # Run a command on a ssh channel. Return false if we get a 337 | # match on bad_match - otherwise return true. bad_match 338 | # is used to pattern match text that indicates a bad exit 339 | # state - used when running something that'll trip a test 340 | def run_on_channel(session,command,bad_match) 341 | if bad_match.class == Regexp 342 | bad_match_regexp = bad_match 343 | else 344 | bad_match_regexp = /#{bad_match}/ 345 | end 346 | output = [] 347 | test_passed = true 348 | @log.debug "Running #{command}" 349 | session.open_channel do |ch| 350 | ch.exec command do |ch, success| 351 | unless success 352 | @log.info "could not execute #{command}" 353 | test_passed = false 354 | end 355 | ch.on_data do |ch, data| 356 | @log.debug data 357 | output << data 358 | if data =~ bad_match_regexp 359 | test_passed = false 360 | end 361 | end 362 | # Test failed if program/script exited nonzero 363 | ch.on_request("exit-status") do |ch,data| 364 | exit_code = data.read_long 365 | @log.debug "Command exited with #{exit_code.to_s}" 366 | if exit_code != 0 367 | test_passed = false 368 | end 369 | end 370 | end 371 | end 372 | # Causes this to block until the command completes. 373 | session.loop 374 | # So far if there's no output, the command failed.. 375 | if output.empty? 376 | test_passed = false 377 | end 378 | return ({"passed" => test_passed, "output" => output.join("\n") }) 379 | end 380 | 381 | # Run a test for a specific scenario on the guest VM. Will set 'service' 382 | # class on the VM and then execute puppet - which will invoke all 383 | # rules related to that class. It will then execute any unit 384 | # tests associated with that service. 385 | # Fills in the @results instance variable with information 386 | # about the test then returns true|false indicating pass|fail 387 | def run_vmth_test(scenario) 388 | @scenario=scenario 389 | service=scenario # legacy/lazy 390 | start_timer = Time.now() 391 | @results['tests'][service] = {} 392 | test_passed = true 393 | begin 394 | ssh_session do |session| 395 | @results['tests'][service]['apply'] = 396 | _recursor(@config['applying'],session) 397 | @results['tests'][service]['test'] = 398 | _recursor(@config['testing'],session) 399 | @results['tests'][service]['passed'] = @results['tests'][service]['apply']['passed'] \ 400 | and @results['tests'][service]['test']['passed'] 401 | _recursor(@config['teardown'],session) 402 | @results['tests'][service]['teardown'] = 403 | _recursor(@config['teardown'],session) 404 | end 405 | rescue => e 406 | # If anything was raised here it is big problems yo. 407 | @results['tests'][service]['apply'] ||= {} 408 | @results['tests'][service]['apply']['passed'] = false 409 | @results['tests'][service]['test'] ||= {} 410 | @results['tests'][service]['test']['passed'] = false 411 | @results['tests'][service]['passed'] = false 412 | @results['tests'][service]['error'] = { 413 | 'class' => e.class.to_s, 'message' => e.message, 'backtrace' => e.backtrace 414 | } 415 | end 416 | @results['tests'][service]['elapsed_time'] = (Time.now() - start_timer) 417 | write_out_state() 418 | return @results['tests'][service]['passed'] 419 | end 420 | 421 | # Write out a state file, handy for debugging later. 422 | def write_out_state 423 | if @options[:out_file] 424 | filename = @options[:out_file] 425 | else 426 | filename = @tmp_state 427 | end 428 | @log.debug "Writing out state into #{filename}" 429 | File.open(filename,'w') do |f| 430 | f.puts YAML.dump(@results) 431 | end 432 | end 433 | # Starts the QEMU instance and then immediately loads the saved 434 | # VM via 'loadvm foo' 435 | def boot_vm 436 | start_vm() 437 | @log.debug "Loading initial vm..." 438 | vmm_command(eb(@config['vmm']['loadinit'])) 439 | end 440 | # Freeze the current state of the VM - so we can use it later 441 | # to reset the VM so that it is immediately ready for the next test. 442 | def freeze_vm() 443 | @log.debug "Freezing vm for test series" 444 | vmm_command(eb(@config['vmm']['saveteststate'])) 445 | end 446 | # Reset the VM for the next test - using the instance saved by 'freeze' 447 | def reset_vm() 448 | @log.debug "Reseting vm for next test" 449 | vmm_command(eb(@config['vmm']['loadteststate'])) 450 | # Give it a half a tic to reset... 451 | sleep 0.5 452 | end 453 | # Set up an ssh session. 454 | def ssh_session 455 | retry_flag=true 456 | @log.debug "ssh is #{@config['ssh'].inspect}" 457 | ssh_config = @config['ssh'].clone 458 | host = ssh_config['host'] 459 | ssh_config.delete('host') 460 | user = ssh_config['user'] 461 | ssh_config.delete('user') 462 | # Convert strings to symbols.. 463 | ssh_config = ssh_config.inject({}){|memo,(k,v)| memo[k.to_sym] = v; memo} 464 | ssh_config[:port] = @vm_ssh_port 465 | begin 466 | Net::SSH.start(host,user,ssh_config) do |session| 467 | yield session 468 | end 469 | rescue EOFError => e 470 | raise(e) unless retry_flag 471 | retry_flag = false 472 | @log.info "SSH session creation failed, retrying!" 473 | retry 474 | end 475 | end 476 | # This function executes all the commands on the just-started VM to 477 | # sync over all files and state needed before testing can start. 478 | def prep 479 | ssh_session do |session| 480 | @results['prep'] = _recursor(@config['prep'],session) 481 | end 482 | @log.info "FINISHED PREP STEPS...." 483 | end 484 | 485 | private 486 | # Allocate a TCP port. 487 | def self.allocate_tcp_port(valid_ports=[]) 488 | last_error = ArgumentError.new("Port range not given.") 489 | # Try to bind to each port until we don't error out 490 | # because of permission or it already being used. 491 | valid_ports.each do |port| 492 | begin 493 | s = TCPServer.open('0.0.0.0',port) 494 | s.close 495 | return port 496 | rescue => e 497 | last_error = e 498 | next 499 | end 500 | end 501 | # If we can't allocate a port raise an error. 502 | raise last_error.class, last_error.message 503 | end 504 | # Execute commands on vm, recurse if required. 505 | def _recursor(cmds,session) 506 | results = [] 507 | passed = true 508 | @log.debug "Processing #{cmds.inspect}" 509 | cmds.each do |myhash| 510 | if myhash.size != 1 511 | @log.error "Config format problem with #{myhash.inspect}" 512 | raise VmthError 513 | end 514 | cmd = myhash.keys.first 515 | values = myhash[cmd] 516 | @log.debug "Values is #{values.inspect}" 517 | if cmd=='foreach' 518 | args = values.shift['args'] 519 | args.each do |arg| 520 | @log.debug "Arg is #{arg.inspect}" 521 | @arg = arg 522 | res_hash = _recursor(values,session) 523 | results << res_hash['output'] 524 | passed = res_hash['passed'] and passed 525 | end 526 | elsif cmd=='cmd' 527 | command_string = eb(values)+" 2>&1" 528 | @log.debug "Running on vm.. '#{command_string}" 529 | result = session.exec!(command_string) 530 | @log.debug "output is: #{result}" 531 | results << result 532 | elsif %{upload download upload_recurse download_recurse}.include?(cmd) 533 | first=eb(values[0]) 534 | second=eb(values[1]) 535 | @log.debug "File transfer with #{first} => #{second}" 536 | if cmd=='upload' 537 | results << session.scp.upload!(first,second) 538 | elsif cmd=='upload_recurse' 539 | results << session.scp.upload!(first,second,{:recursive=>true}) 540 | elsif cmd=='download' 541 | results << session.scp.download!(first,second ) 542 | elsif cmd=='download_recurse' 543 | results << session.scp.download!(first,second,{:recursive=>true}) 544 | end 545 | elsif cmd=='cmdtest' 546 | res_hash = run_on_channel(session,eb(values[0]),values[1]) 547 | results << res_hash['output'] 548 | passed = res_hash['passed'] and passed 549 | else 550 | @log.error "unknown command #{cmd.inspect}" 551 | raise VmthError 552 | end 553 | end 554 | return {'output'=>results,'passed'=>passed} 555 | end 556 | end # Class 557 | 558 | 559 | --------------------------------------------------------------------------------