├── .github └── workflows │ └── main.yml ├── .gitignore ├── .k8sharness.example ├── .ruby-version ├── .todos ├── done.txt ├── report.txt ├── todo.txt └── todo.txt.bak ├── Gemfile ├── Gemfile.lock ├── Makefile ├── README.md ├── VERSION ├── bin └── k8s-harness ├── conf └── required_software.yaml ├── content └── how_it_works.png ├── include ├── Vagrantfile ├── inventory └── site.yml ├── k8s_harness.gemspec ├── lib ├── k8s_harness.rb └── k8s_harness │ ├── cli.rb │ ├── clusters.rb │ ├── clusters │ ├── ansible.rb │ ├── cluster_info.rb │ ├── constants.rb │ ├── metadata.rb │ ├── required_software.rb │ └── vagrant.rb │ ├── harness_file.rb │ ├── logging.rb │ ├── paths.rb │ ├── shell_command.rb │ └── subcommand.rb ├── scripts └── verify_runner_doesnt_suck.sh └── tests ├── fixtures ├── .k8sharness.key_already_has_sh_command ├── .k8sharness.key_references_script ├── .k8sharness.missing_setup_key ├── .k8sharness.missing_teardown_key ├── .k8sharness.missing_tests_key └── .k8sharness.valid_with_all_keys ├── harness_file_spec.rb ├── integration ├── .k8sharness └── run_spec.rb ├── k8s_harness_cli_spec.rb ├── k8s_harness_cluster_info_spec.rb ├── k8s_harness_clusters_spec.rb ├── shell_command_spec.rb ├── spec_helper.rb └── subcommand_spec_depcrecated.rb /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: k8s-harness CI/CD 3 | on: 4 | pull_request: 5 | branches: 6 | - master 7 | push: 8 | paths-ignore: 9 | - "README.md" 10 | - 'content/**' 11 | 12 | jobs: 13 | unit: 14 | runs-on: ubuntu-latest 15 | if: startsWith(github.ref, 'refs/tags/') != true 16 | steps: 17 | - uses: actions/checkout@v1 18 | - uses: actions/setup-ruby@v1 19 | with: 20 | ruby-version: '2.7' 21 | - name: Run unit tests 22 | run: make test 23 | # Our integration tests require Vagrant and VirtualBox. 24 | # Only the MacOS builders supported VT-x, which is required by VirtualBox. 25 | integration: 26 | needs: unit 27 | if: startsWith(github.ref, 'refs/tags/') != true 28 | runs-on: macos-latest 29 | steps: 30 | - uses: actions/checkout@v1 31 | - uses: actions/setup-ruby@v1 32 | - name: Install Ansible 33 | run: brew install ansible 34 | # Vagrant is super flaky on these runners. 35 | - name: Run integration tests 36 | run: make integration 37 | deploy: 38 | runs-on: ubuntu-latest 39 | needs: integration 40 | if: github.ref == 'refs/heads/master' 41 | steps: 42 | - uses: actions/checkout@v1 43 | - uses: actions/setup-ruby@v1 44 | with: 45 | ruby-version: '2.7' 46 | - name: Ensure that the VERSION file has been updated. 47 | run: make ensure_version 48 | - name: Deploy to RubyGems 49 | run: make deploy 50 | env: 51 | GEM_HOST_API_KEY: "${{ secrets.GEM_HOST_API_KEY }}" 52 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | output/ 2 | .k8sharness_data/ 3 | -------------------------------------------------------------------------------- /.k8sharness.example: -------------------------------------------------------------------------------- 1 | --- 2 | # vi: set ft=yaml: 3 | 4 | # This is a sample .k8sharness file. Use this to define the test bench to execute in your 5 | # disposable Kubernetes cluster. 6 | 7 | # Define the command that runs your test here! It can be in any language you want. 8 | tests: echo 'change me' 9 | 10 | # Use `setup:` to run a command, script, or other application that provisions your cluster 11 | # before running your test. You can use this to do things like: 12 | # - Deploy or modify your application's Helm chart into your test cluster, or 13 | # - Store temporary metadata required by your application. 14 | setup: echo 'optionally change me' 15 | 16 | # Use `teardown:` to run cleanup tasks before your cluster gets destroyed. 17 | teardown: echo 'optionally change me' 18 | -------------------------------------------------------------------------------- /.ruby-version: -------------------------------------------------------------------------------- 1 | 2.7.0@k8s-harness 2 | -------------------------------------------------------------------------------- /.todos/done.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carlosonunez/k8s-harness/935f4fe94060715d917a0dcf84810477f698c431/.todos/done.txt -------------------------------------------------------------------------------- /.todos/report.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carlosonunez/k8s-harness/935f4fe94060715d917a0dcf84810477f698c431/.todos/report.txt -------------------------------------------------------------------------------- /.todos/todo.txt: -------------------------------------------------------------------------------- 1 | +testing @portable_tests Have tests run in Docker Compose (ironic) instead of on user's local machine 2 | +features @allow_any_lang Allow users to run tests in any language. 3 | +features @no_harcoded_refs Remove hardcoded refs to k3s clusters in clusters.rb 4 | +feature @command_tracing Add a trace ID for every command call 5 | +feature @prepackaged_k8s_clusters OH MY GOD. We could package Kubernetes clusters for k8s-harness like Vagrantfiles! 6 | +bugfix @destroy Cleaner warning if no clusters found in ccwd 7 | -------------------------------------------------------------------------------- /.todos/todo.txt.bak: -------------------------------------------------------------------------------- 1 | +testing @portable_tests Have tests run in Docker Compose (ironic) instead of on user's local machine 2 | +features @allow_any_lang Allow users to run tests in any language. 3 | +features @no_harcoded_refs Remove hardcoded refs to k3s clusters in clusters.rb 4 | +feature @command_tracing Add a trace ID for every command call 5 | +feature @prepackaged_k8s_clusters OH MY GOD. We could package Kubernetes clusters for k8s-harness like Vagrantfiles! 6 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | source 'https://rubygems.org' 4 | group :test do 5 | gem 'fakefs' 6 | gem 'pry' 7 | gem 'rspec' 8 | gem 'rspec-expectations' 9 | gem 'rubocop' 10 | end 11 | -------------------------------------------------------------------------------- /Gemfile.lock: -------------------------------------------------------------------------------- 1 | GEM 2 | remote: https://rubygems.org/ 3 | specs: 4 | ast (2.4.1) 5 | coderay (1.1.3) 6 | diff-lcs (1.4.4) 7 | fakefs (1.2.2) 8 | method_source (1.0.0) 9 | parallel (1.19.2) 10 | parser (2.7.2.0) 11 | ast (~> 2.4.1) 12 | pry (0.13.1) 13 | coderay (~> 1.1) 14 | method_source (~> 1.0) 15 | rainbow (3.0.0) 16 | regexp_parser (1.8.2) 17 | rexml (3.2.4) 18 | rspec (3.9.0) 19 | rspec-core (~> 3.9.0) 20 | rspec-expectations (~> 3.9.0) 21 | rspec-mocks (~> 3.9.0) 22 | rspec-core (3.9.3) 23 | rspec-support (~> 3.9.3) 24 | rspec-expectations (3.9.3) 25 | diff-lcs (>= 1.2.0, < 2.0) 26 | rspec-support (~> 3.9.0) 27 | rspec-mocks (3.9.1) 28 | diff-lcs (>= 1.2.0, < 2.0) 29 | rspec-support (~> 3.9.0) 30 | rspec-support (3.9.4) 31 | rubocop (1.0.0) 32 | parallel (~> 1.10) 33 | parser (>= 2.7.1.5) 34 | rainbow (>= 2.2.2, < 4.0) 35 | regexp_parser (>= 1.8) 36 | rexml 37 | rubocop-ast (>= 0.6.0) 38 | ruby-progressbar (~> 1.7) 39 | unicode-display_width (>= 1.4.0, < 2.0) 40 | rubocop-ast (1.0.1) 41 | parser (>= 2.7.1.5) 42 | ruby-progressbar (1.10.1) 43 | unicode-display_width (1.7.0) 44 | 45 | PLATFORMS 46 | ruby 47 | 48 | DEPENDENCIES 49 | fakefs 50 | pry 51 | rspec 52 | rspec-expectations 53 | rubocop 54 | 55 | BUNDLED WITH 56 | 2.1.2 57 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env make 2 | MAKEFLAGS += --silent 3 | SHELL := /usr/bin/env bash 4 | ROOTDIR := $(shell git rev-parse --show-toplevel) 5 | ifeq ($(CI),true) 6 | VAGRANT_HOME := $(PWD)/.vagrant_home 7 | VAGRANT_DOTFILE_PATH := $(PWD)/.vagrant_dotfile_path 8 | else 9 | VAGRANT_HOME := $(HOME)/.vagrant.d 10 | VAGRANT_DOTFILE_PATH := $(HOME)/.vagrant 11 | endif 12 | 13 | 14 | .PHONY: is_rvm_installed \ 15 | create_env \ 16 | recreate_env \ 17 | delete_env \ 18 | install_rvm \ 19 | test 20 | 21 | 22 | is_rvm_installed: 23 | if test "$(CI)" == "true"; \ 24 | then \ 25 | >&2 echo "INFO: Running in CI; no RVM required."; \ 26 | exit 0; \ 27 | fi; \ 28 | if ! rvm --version &>/dev/null; \ 29 | then \ 30 | >&2 echo "ERROR: rvm is not installed. Install it by running this: \ 31 | make install_rvm"; \ 32 | exit 1; \ 33 | fi 34 | 35 | create_env: 36 | if ! ( echo "$(GEM_HOME)" | grep -q 'k8s-harness' ) || \ 37 | ! ( find "$(GEM_HOME)/gems" -type d -maxdepth 1 | grep -qi rspec ); \ 38 | then \ 39 | if test "$(CI)" != "true"; \ 40 | then \ 41 | source ~/.rvm/scripts/rvm && \ 42 | rvm --quiet use; \ 43 | gem install bundler --quiet; \ 44 | bundle install --quiet; \ 45 | else \ 46 | gem install bundler; \ 47 | bundle install; \ 48 | fi; \ 49 | gem which rspec; \ 50 | fi 51 | 52 | recreate_env: 53 | { test "$(CI)" == "true" || source ~/.rvm/scripts/rvm && rvm --quiet use; }; \ 54 | { test "$(CI)" == "true" && bundle install || bundle install --quiet }; 55 | 56 | delete_env: 57 | test "$(CI)" == "true" && { >&2 echo "INFO: In CI. Nothing to delete."; exit 0; }; \ 58 | rvm --force gemset delete k8s-harness 59 | 60 | deploy: build 61 | deploy: 62 | if test -z "$(GEM_HOST_API_KEY)"; \ 63 | then \ 64 | >&2 echo "ERROR: Please define GEM_HOST_API_KEY before deploying."; \ 65 | exit 1; \ 66 | fi; \ 67 | gem push output/k8s_harness.gem 68 | 69 | install_rvm: 70 | test "$(CI)" == "true" && { >&2 echo "INFO: In CI. rvm not required."; exit 0; }; \ 71 | source ~/.rvm/scripts/rvm && \ 72 | rvm --version &>/dev/null || curl -sSL https://get.rvm.io | bash -s stable --ruby 73 | 74 | ensure_version: ensure_version_diff ensure_version_tag 75 | 76 | ensure_version_diff: 77 | if test -z "$$(git diff HEAD..HEAD~1 VERSION)"; \ 78 | then \ 79 | >&2 echo 'ERROR: The VERSION file seems to not have been updated?'; \ 80 | exit 1; \ 81 | fi 82 | 83 | ensure_version_tag: 84 | this_commit=$$(git rev-parse HEAD); \ 85 | version_tag_matching_commit=$$(git show-ref --tags | grep $$this_commit); \ 86 | if test -z "$$version_tag_matching_commit"; \ 87 | then \ 88 | >&2 echo 'ERROR: The HEAD commit does not have a tag. Fix this.'; \ 89 | exit 1; \ 90 | fi; \ 91 | this_version=$$(cat VERSION); \ 92 | version_from_version_tag=$$(echo "$$version_tag_matching_commit" | \ 93 | awk '{print $$2}' | \ 94 | rev | \ 95 | cut -f1 -d "/" | \ 96 | rev); \ 97 | if test "$$this_version" != "$$version_from_version_tag"; \ 98 | then \ 99 | >&2 echo "ERROR: Expected [$$this_version] in this version tag, but found [$$version_from_version_tag]."; \ 100 | exit 1; \ 101 | fi 102 | 103 | test: is_rvm_installed create_env 104 | test: 105 | bundle exec rspec -I $(ROOTDIR)/tests -I $(ROOTDIR)/lib \ 106 | --tag ~@wip \ 107 | --tag ~@integration \ 108 | --fail-fast \ 109 | --format \ 110 | documentation tests/ 111 | 112 | build: is_rvm_installed 113 | build: 114 | mkdir -p output; \ 115 | if test "$(CI)" == "true"; \ 116 | then \ 117 | gem build -o output/k8s_harness.gem k8s_harness.gemspec && \ 118 | gem install output/k8s_harness.gem; \ 119 | else \ 120 | gem build --quiet --silent -o output/k8s_harness.gem k8s_harness.gemspec && \ 121 | gem install --quiet --silent output/k8s_harness.gem; \ 122 | fi; 123 | 124 | test_verbose: is_rvm_installed create_env 125 | test_verbose: 126 | LOG_LEVEL=DEBUG bundle exec rspec -I $(ROOTDIR)/tests -I $(ROOTDIR)/lib \ 127 | --tag ~@wip \ 128 | --tag ~@integration\ 129 | --fail-fast \ 130 | --format documentation tests/ 131 | 132 | test_debug: test_verbose 133 | 134 | integration: is_rvm_installed create_env build verify_runner_doesnt_suck 135 | integration: 136 | cp tests/integration/.k8sharness .; \ 137 | VAGRANT_HOME="$(VAGRANT_HOME)" \ 138 | VAGRANT_DOTFILE_PATH="$(VAGRANT_DOTFILE_PATH)" \ 139 | bundle exec rspec -I $(ROOTDIR)/tests -I $(ROOTDIR)/lib --tag @integration --fail-fast \ 140 | --format documentation \ 141 | tests/integration 142 | 143 | integration_verbose: is_rvm_installed create_env build verify_runner_doesnt_suck 144 | integration_verbose: 145 | cp tests/integration/.k8sharness .; \ 146 | VAGRANT_HOME="$(VAGRANT_HOME)" \ 147 | VAGRANT_DOTFILE_PATH="$(VAGRANT_DOTFILE_PATH)" \ 148 | LOG_LEVEL=DEBUG \ 149 | bundle exec rspec -I $(ROOTDIR)/tests -I $(ROOTDIR)/lib --tag @integration --fail-fast \ 150 | --format documentation \ 151 | tests/integration 152 | 153 | integration_debug: integration_verbose 154 | 155 | verify_runner_doesnt_suck: 156 | VAGRANT_HOME="$(VAGRANT_HOME)" \ 157 | VAGRANT_DOTFILE_PATH="$(VAGRANT_DOTFILE_PATH)" \ 158 | ./scripts/verify_runner_doesnt_suck.sh 159 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # k8s-harness 2 | 3 | 🚀 Test your apps in disposable, prod-like Kubernetes clusters 🚀 4 | 5 | [[ _insert gif here when ready_ ]] 6 | 7 | ## But why? 8 | 9 | `k8s-harness` is for you if: 10 | 11 | - You have apps that run on Kubernetes in production (GKE, EKS, AKS, Rancher, etc.), but 12 | - you don't want to create (and pay for) Kubernetes clusters yourself, and 13 | - you want a prod-like Kubernetes experience on your laptop without the hassles, and 14 | - you just want to run your tests in a clean cluster every time. 15 | 16 | `k8s-harness` is probably not for you if: 17 | 18 | - You want to learn how Kubernetes works under the hood (check out Hightower's 19 | [kubernetes-the-hard-way](https://github.com/kelseyhightower/kubernetes-the-hard-way) 20 | for that), or 21 | - you want to run long-lived clusters on your laptop that you manage. 22 | 23 | ## How it works 24 | 25 | ![](./content/how_it_works.png) 26 | 27 | `k8s-harness` is simple: 28 | 29 | 1. [Create](#.k8sharness) a new `.k8sharness` file. 30 | 2. Run your tests! `k8s-harness run` 31 | 32 | `run` can take a long time depending on your network connection. If you want to see more 33 | details behind what's going on, enable debug output like this: 34 | 35 | ```sh 36 | $: k8s-harness --debug run 37 | ``` 38 | 39 | or this: 40 | 41 | ```sh 42 | $: LOG_LEVEL=debug k8s-harness run 43 | ``` 44 | 45 | ### The nitty-gritty 46 | 47 | Here's how it really works: 48 | 49 | 1. `k8s-harness` will look for a `.k8sharness` file in the root of your repository, 50 | 2. Once found, it will create a two-node [k3s](https://github.com/rancher/k3s) cluster 51 | on your machine with [Vagrant](https://vagrantup.com) and [Ansible](https://ansible.io), 52 | 3. `k8s-harness` will also provision a local insecure Docker registry into which you can push 53 | your app's Docker images, 54 | 4. `k8s-harness` will run your tests as defined by `.k8sharness` in a Bash subshell, 55 | 5. `k8s-harness` destroys the cluster (unless you keep it up with `--disable-teardown`). 56 | 57 | (If you're interested in the nitty-gritty of how `k8s-harness` works, check out 58 | [its tests](https://github.com/carlosonunez/k8s-harness/blob/master/tests) for the details.) 59 | 60 | ## .k8sharness 61 | 62 | `k8s-harness` uses `.k8sharness` files to determine what it should do once its cluster is 63 | provisioned. An example file is provided at [`.k8sharness.example`](./.k8sharness.example), 64 | but the crux of how it works is this: 65 | 66 | Define your test like this: 67 | 68 | ```yaml 69 | test: make test 70 | ``` 71 | 72 | You can optionally add setup or teardown instructions too: 73 | 74 | 75 | ```yaml 76 | test: make test 77 | setup: helm install -f testenv.yaml 78 | teardown: make report 79 | ``` 80 | 81 | Every command in `test`, `setup`, and `teardown` runs in a `sh` subshell, but you can 82 | provide a script as well: 83 | 84 | ```yaml 85 | test: make test 86 | setup: helm install -f testenv.yaml 87 | teardown: ./scripts/run_teardown.sh 88 | ``` 89 | 90 | If you need to have more control over the subshell, just start your test command with 91 | `sh -c`: 92 | 93 | ```yaml 94 | test: make test 95 | setup: helm install -f testenv.yaml 96 | teardown: sh -xc "echo 'I\'m gonna wreck it!'" 97 | ``` 98 | 99 | ## Installing 100 | 101 | - `gem install k8s-harness` if you're installing this standalone, or 102 | - Include `k8s-harness` into your app's `Gemfile` if you're building a Ruby or Rails app. 103 | 104 | ## Options 105 | 106 | * See [`.k8sharness.example`](https://github.com/carlosonunez/k8s-harness/blob/master/.k8sharness.example) 107 | for documentation on how to configure your `.k8sharness` file. 108 | * Run `k8s-harness --help` to learn how to configure `k8s-harness` to your liking. 109 | 110 | ## Questions 111 | 112 | ### Does this replace Docker Compose? 113 | 114 | Nope! Docker Compose is excellent for locally running your apps and testing that your app 115 | works in Docker. However, I've found Compose to be lacking for testing whether my app can run 116 | with Kubernete's extra features, like `Secret`s and `Ingress` objects. As well, for writing 117 | Ansible playbooks that provision "hard" Kuberntes infrastructure like installing CRDs, I've found 118 | having clean Kubernetes clusters that resemble what I'm provisioning in production to be fast 119 | (or at least faster than waiting for CI/CD to apply my manifests) and cost-effective (since 120 | I don't need to provision my own Kubernetes clusters externally). 121 | 122 | ### My production Kubernetes clusters has `$x`. How do I install `$x` in k8s-harness? 123 | 124 | This is not supported yet, but is on the roadmap! 125 | 126 | ## Contributing! 127 | 128 | Thanks for helping make `k8s-harness` better! 129 | 130 | Contributing is simple. 131 | 132 | ### What You'll Need To Install 133 | 134 | - Vagrant 135 | - Virtualbox 136 | - Ansible 137 | 138 | ### How to contribute 139 | 140 | 1. Fork this repository. 141 | 2. Add a test in `tests/`. 142 | 143 | **NOTE**: If you're adding a new feature to `k8s-harness`, you'll also need to add 144 | an integration test in `tests/integration` to describe what the feature does, how users 145 | should use it, and what they can expect when it runs. 146 | See [the integration test for `run`](./tests/integration/run_spec.rb) for an example. 147 | 148 | 3. Add your code in `lib`. 149 | 4. Run your unit tests: `make unit` 150 | 5. Run the integration test to ensure that everything works: `make integration` 151 | 6. Push your commits up to your fork, then submit a new pull request into this repo! 152 | 153 | ### A note about pushing new gems 154 | 155 | You're free to fork this and create your own gems from a forked instance of this codebase. Note 156 | that the CD that's included in the [`.github`](./.github) directory requires that you 157 | define `GEM_HOST_API_KEY` in your build. 158 | -------------------------------------------------------------------------------- /VERSION: -------------------------------------------------------------------------------- 1 | 1.0.0 -------------------------------------------------------------------------------- /bin/k8s-harness: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # frozen_string_literal: true 3 | 4 | $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib')) 5 | require 'k8s_harness' 6 | 7 | KubernetesHarness::CLI.parse(ARGV) 8 | -------------------------------------------------------------------------------- /conf/required_software.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: vagrant 3 | version_check: vagrant --version 4 | - name: ansible-playbook 5 | version_check: ansible-playbook --version 6 | -------------------------------------------------------------------------------- /content/how_it_works.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carlosonunez/k8s-harness/935f4fe94060715d917a0dcf84810477f698c431/content/how_it_works.png -------------------------------------------------------------------------------- /include/Vagrantfile: -------------------------------------------------------------------------------- 1 | # vim: set ft=ruby: 2 | num_nodes = ENV["K3S_NUMBER_OF_NODES"] || 2 3 | memory_per_node = ENV["K3S_NODE_MEMORY_GB"] || 1024 4 | ssh_pub_key = File.read("#{ENV['VAGRANT_CWD']}/ssh_key.pub").gsub("\n","") 5 | install_ansible_command = <<-COMMAND 6 | apk update 7 | if ! apk add ansible 8 | then 9 | echo "ERROR: Failed to install Ansible on this machine." 10 | exit 1 11 | fi 12 | COMMAND 13 | 14 | Vagrant.configure("2") do |config| 15 | config.vm.box = "maier/alpine-3.6-x86_64" 16 | config.vm.provider "virtualbox" do |vb| 17 | vb.customize [ "modifyvm", :id, "--memory", memory_per_node ] 18 | end 19 | 20 | config.vm.define "k3s-registry" do |node| 21 | node.vm.hostname = "k3s-registry" 22 | node.vm.network "private_network", ip: "192.168.50.200" 23 | node.vm.network "forwarded_port", guest: 5000, host: 5000 24 | node.vm.provision "shell", 25 | inline: "echo '#{ssh_pub_key}' >> /home/vagrant/.ssh/authorized_keys" 26 | node.vm.provision "shell", inline: install_ansible_command 27 | end 28 | 29 | num_nodes.times do |node_id| 30 | config.vm.define "k3s-node-#{node_id}" do |node| 31 | node.vm.hostname = "k3s-node-#{node_id}" 32 | node.vm.network "private_network", ip: "192.168.50.#{node_id+2}" 33 | node.vm.network "forwarded_port", guest: 6443, host: 6443 if node_id == 0 34 | node.vm.provision "shell", 35 | inline: "echo '#{ssh_pub_key}' >> /home/vagrant/.ssh/authorized_keys" 36 | node.vm.provision "shell", inline: install_ansible_command 37 | end 38 | end 39 | end 40 | -------------------------------------------------------------------------------- /include/inventory: -------------------------------------------------------------------------------- 1 | # These IP addresses are statically allocated. 2 | # See scripts/deploy_k3s for more info. 3 | [master] 4 | 192.168.50.2 5 | 6 | [worker] 7 | 192.168.50.3 8 | 9 | [registry] 10 | 192.168.50.200 11 | -------------------------------------------------------------------------------- /include/site.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: master 3 | become: true 4 | gather_facts: no 5 | tasks: 6 | - name: Get this host's IP address 7 | shell: "echo $(ip -4 -o addr show eth1 | awk '{print $4}' | cut -f1 -d '/')" 8 | register: result 9 | 10 | - set_fact: 11 | ip_address: "{{ result.stdout }}" 12 | 13 | - name: Create directories 14 | file: 15 | path: "{{ item }}" 16 | state: directory 17 | with_items: 18 | - /etc/rancher/k3s 19 | - /etc/docker 20 | 21 | - name: Create registry files 22 | file: 23 | path: "{{ item }}" 24 | state: touch 25 | with_items: 26 | - /etc/rancher/k3s/registries.yaml 27 | - /etc/docker/daemon.json 28 | 29 | - name: Configure insecure registries for k3s 30 | blockinfile: 31 | path: /etc/rancher/k3s/registries.yaml 32 | block: | 33 | mirrors: 34 | "10.0.2.2:5000": 35 | endpoint: 36 | - "http://10.0.2.2:5000" 37 | 38 | - name: Configure insecure regsitries for containerd 39 | block: 40 | - name: Create the daemon file 41 | blockinfile: 42 | path: /etc/docker/daemon.json 43 | marker: "" 44 | block: | 45 | { "insecure-registries": [ "10.0.2.2:5000" ] } 46 | 47 | - name: Remove blank lines 48 | lineinfile: 49 | path: /etc/docker/daemon.json 50 | state: absent 51 | regexp: '^$' 52 | 53 | - name: Install Rancher k3s 54 | shell: curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="--node-ip={{ ip_address }} --flannel-iface=eth1" K3S_TOKEN={{ k3s_token }} sh - 55 | 56 | - name: Check if extlinux updated 57 | shell: "grep -q cgroup_enable=cpuset /etc/update-extlinux.conf" 58 | register: extlinux_enabled_result 59 | ignore_errors: true 60 | 61 | - name: Update extlinux per documentation 62 | shell: "echo 'default_kernel_opts=\"... cgroup_enable=cpuset cgroup_memory=1 cgroup_enable=memory\" >> /etc/update-extlinux.conf'" 63 | when: extlinux_enabled_result.rc != 0 64 | 65 | - name: Apply extlinux updates 66 | shell: update-extlinux 67 | when: extlinux_enabled_result.rc != 0 68 | 69 | - name: Reboot 70 | shell: /sbin/reboot 71 | when: extlinux_enabled_result.rc != 0 72 | 73 | - name: "Wait for machine" 74 | become: false 75 | register: wait_result 76 | local_action: wait_for host={{ ip_address }} port=22 timeout=300 connect_timeout=300 77 | 78 | - hosts: worker 79 | become: true 80 | tasks: 81 | - name: Get this host's IP address 82 | shell: "echo $(ip -4 -o addr show eth1 | awk '{print $4}' | cut -f1 -d '/')" 83 | register: result 84 | 85 | - set_fact: 86 | ip_address: "{{ result.stdout }}" 87 | 88 | - name: Wait for master to become available 89 | register: wait_result 90 | wait_for: 91 | timeout: 300 92 | connect_timeout: 300 93 | host: 192.168.50.2 94 | port: 6443 95 | 96 | 97 | - name: Create directories 98 | file: 99 | path: "{{ item }}" 100 | state: directory 101 | with_items: 102 | - /etc/rancher/k3s 103 | - /etc/docker 104 | 105 | - name: Create registry files 106 | file: 107 | path: "{{ item }}" 108 | state: touch 109 | with_items: 110 | - /etc/rancher/k3s/registries.yaml 111 | - /etc/docker/daemon.json 112 | 113 | - name: Configure insecure registries for k3s 114 | blockinfile: 115 | path: /etc/rancher/k3s/registries.yaml 116 | block: | 117 | mirrors: 118 | "10.0.2.2:5000": 119 | endpoint: 120 | - "http://10.0.2.2:5000" 121 | 122 | - name: Configure insecure regsitries for containerd 123 | block: 124 | - name: Create the daemon file 125 | blockinfile: 126 | path: /etc/docker/daemon.json 127 | marker: "" 128 | block: | 129 | { "insecure-registries": [ "10.0.2.2:5000" ] } 130 | 131 | - name: Remove blank lines 132 | lineinfile: 133 | path: /etc/docker/daemon.json 134 | state: absent 135 | regexp: '^$' 136 | 137 | - name: Install k3s as worker 138 | shell: curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="--node-ip={{ ip_address }} --flannel-iface=eth1" K3S_URL=https://192.168.50.2:6443 K3S_TOKEN={{ k3s_token }} sh - 139 | 140 | - name: Check if extlinux updated 141 | shell: "grep -q cgroup_enable=cpuset /etc/update-extlinux.conf" 142 | register: extlinux_enabled_result 143 | ignore_errors: true 144 | 145 | - name: Update extlinux per documentation 146 | shell: "echo 'default_kernel_opts=\"... cgroup_enable=cpuset cgroup_memory=1 cgroup_enable=memory\" >> /etc/update-extlinux.conf'" 147 | when: extlinux_enabled_result.rc != 0 148 | 149 | - name: Apply extlinux updates 150 | shell: update-extlinux 151 | when: extlinux_enabled_result.rc != 0 152 | 153 | - name: Reboot 154 | shell: /sbin/reboot 155 | when: extlinux_enabled_result.rc != 0 156 | 157 | - name: "Wait for machine" 158 | become: false 159 | register: wait_result 160 | local_action: wait_for host={{ ip_address }} port=22 timeout=300 connect_timeout=300 161 | 162 | 163 | - hosts: registry 164 | become: true 165 | tasks: 166 | - name: Get this host's IP address 167 | shell: "echo $(ip -4 -o addr show eth1 | awk '{print $4}' | cut -f1 -d '/')" 168 | register: result 169 | 170 | - set_fact: 171 | ip_address: "{{ result.stdout }}" 172 | 173 | 174 | - name: Install Docker 175 | apk: 176 | name: 177 | - docker 178 | 179 | - name: Add docker as service 180 | shell: "rc-update add docker boot" 181 | 182 | - name: Reboot 183 | shell: /sbin/reboot 184 | 185 | - name: "Wait for machine" 186 | become: false 187 | register: wait_result 188 | local_action: wait_for host={{ ip_address }} port=22 timeout=300 connect_timeout=300 189 | 190 | - name: Start docker daemon 191 | shell: "service docker start" 192 | retries: 5 193 | delay: 2 194 | 195 | - name: Confirm Docker available 196 | shell: "docker run --rm hello-world" 197 | retries: 5 198 | delay: 2 199 | 200 | - name: Check for instances of registry 201 | shell: "sudo docker ps | grep -q registry" 202 | register: result 203 | ignore_errors: true 204 | 205 | - name: Start Docker Registry 206 | shell: "sudo docker run -d --restart=always -p 5000:5000 --name registry registry:2" 207 | when: result.rc != 0 208 | -------------------------------------------------------------------------------- /k8s_harness.gemspec: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | Gem::Specification.new do |s| 4 | s.name = 'k8s-harness' 5 | s.required_ruby_version = '~> 2.7.0' 6 | s.executables << 'k8s-harness' 7 | s.version = File.read('VERSION') 8 | s.date = '2020-10-28' 9 | s.summary = 'Test your apps in disposable, prod-like Kubernetes clusters' 10 | s.description = 'Please visit the README in the Github repo linked to this gem for more info.' 11 | s.authors = ['Carlos Nunez'] 12 | s.email = 'dev@carlosnunez.me' 13 | s.files = Dir['./lib/**/*.rb', './include/**', './conf/**'] 14 | s.homepage = 'https://github.com/carlosonunez/k8s-harness' 15 | s.license = 'MIT' 16 | end 17 | -------------------------------------------------------------------------------- /lib/k8s_harness.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'k8s_harness/cli' 4 | require 'k8s_harness/clusters' 5 | require 'k8s_harness/logging' 6 | -------------------------------------------------------------------------------- /lib/k8s_harness/cli.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'optparse' 4 | require 'k8s_harness/subcommand' 5 | 6 | # KubernetesHarness 7 | module KubernetesHarness 8 | # This module contains everything CLI-related. 9 | # We're using it as the entry-point for k8s-harness. 10 | module CLI 11 | @options = { 12 | base: {} 13 | } 14 | @subcommands = { 15 | run: { 16 | description: 'Runs tests', 17 | option_parser: OptionParser.new do |opts| 18 | opts.banner = 'Usage: k8s-harness run [options]' 19 | opts.separator 'Runs tests' 20 | opts.separator '' 21 | opts.separator 'Commands:' 22 | opts.on('-h', '--help', 'Displays this help message') do 23 | add_option(options: { show_usage: true }, subcommand: :run) 24 | puts opts 25 | end 26 | opts.on('--disable-teardown', 'Keeps the cluster up for local testing') do 27 | add_option(options: { disable_teardown: true }, subcommand: :run) 28 | end 29 | end 30 | }, 31 | validate: { 32 | description: 'Validates .k8sharness files', 33 | option_parser: OptionParser.new do |opts| 34 | opts.banner = 'Usage: k8s-harness validate [options]' 35 | opts.separator 'Validates that a .k8sharness file is correct' 36 | opts.separator '' 37 | opts.separator 'Commands:' 38 | opts.on('-h', '--help', 'Displays this help message') do 39 | add_option(options: { show_usage: true }, subcommand: :validate) 40 | puts opts 41 | end 42 | end 43 | }, 44 | destroy: { 45 | description: 'Deletes a live cluster provisioned by k8s-harness WITHOUT WARNING.', 46 | option_parser: OptionParser.new do |opts| 47 | opts.banner = 'Usage: k8s-harness destroy [options]' 48 | opts.separator 'Deletes live clusters provisioned by k8s-harness WITHOUT WARNING' 49 | opts.separator '' 50 | opts.separator 'Commands:' 51 | opts.on('-h', '--help', 'Displays this help message') do 52 | add_option(options: { show_usage: true }, subcommand: :destroy) 53 | puts opts 54 | end 55 | end 56 | } 57 | } 58 | 59 | @base_command = OptionParser.new do |opts| 60 | opts.banner = 'Usage: k8s-harness [subcommand] [options]' 61 | opts.separator 'Test your apps in disposable Kubernetes clusters' 62 | opts.separator '' 63 | opts.separator 'Sub-commands:' 64 | opts.separator '' 65 | @subcommands.each_key do |subcommand| 66 | opts.separator " #{subcommand.to_s.ljust(20)} #{@subcommands[subcommand][:description]}" 67 | end 68 | opts.separator '' 69 | opts.separator 'See k8s-harness [subcommand] --help for more specific options.' 70 | opts.separator '' 71 | opts.separator 'Global options:' 72 | opts.on('-d', '--debug', 'Show debug output') do 73 | add_option(options: { enable_debug_logging: true }) 74 | end 75 | opts.on('-h', '--help', 'Displays this help message') do 76 | add_option(options: { help: opts.help }) 77 | end 78 | end 79 | 80 | def self.parse(args) 81 | args.push('-h') if args.empty? || subcommands_missing?(args) 82 | @base_command.order!(args) 83 | subcommand = args.shift 84 | if subcommand.nil? 85 | puts @options[:base][:help] 86 | else 87 | enable_debug_logging_if_present 88 | @subcommands[subcommand.to_sym][:option_parser].order!(args) 89 | call_entrypoint(subcommand) 90 | end 91 | end 92 | 93 | def self.enable_debug_logging_if_present 94 | KubernetesHarness::Logging.enable_debug_logging if @options[:base][:enable_debug_logging] 95 | end 96 | 97 | def self.subcommands_missing?(args) 98 | args.select { |arg| arg.match?(/^[a-z]/) }.empty? 99 | end 100 | 101 | def self.add_option(options:, subcommand: nil) 102 | if subcommand.nil? 103 | @options[:base].merge!(options) 104 | else 105 | @options[subcommand] = {} unless @options.key subcommand 106 | @options[subcommand].merge!(options) 107 | end 108 | end 109 | 110 | def self.call_entrypoint(subcommand) 111 | KubernetesHarness::Subcommand.method(subcommand.to_sym).call(@options[subcommand.to_sym]) 112 | end 113 | 114 | private_class_method :call_entrypoint, :add_option, :subcommands_missing? 115 | end 116 | end 117 | -------------------------------------------------------------------------------- /lib/k8s_harness/clusters.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'k8s_harness/clusters/ansible' 4 | require 'k8s_harness/clusters/constants' 5 | require 'k8s_harness/clusters/cluster_info' 6 | require 'k8s_harness/clusters/metadata' 7 | require 'k8s_harness/clusters/required_software' 8 | require 'k8s_harness/clusters/vagrant' 9 | require 'k8s_harness/shell_command' 10 | 11 | module KubernetesHarness 12 | # Handles bring up and deletion of disposable clusters. 13 | module Clusters 14 | def self.create! 15 | RequiredSoftware.ensure_installed_or_exit! 16 | Metadata.initialize! 17 | create_ssh_key! 18 | vagrant_up_disposable_cluster_or_exit! 19 | cluster = ClusterInfo.new(master_ip_address_command: master_ip_address_command, 20 | worker_ip_addresses_command: worker_ip_addresses_command, 21 | docker_registry_command: docker_registry_command, 22 | kubeconfig_path: 'not_yet', 23 | ssh_key_path: cluster_ssh_key) 24 | Metadata.write!('cluster.yaml', cluster.to_yaml) 25 | cluster 26 | end 27 | 28 | def self.create_ssh_key! 29 | ssh_key_fp = File.join(Metadata.default_dir, 'ssh_key') 30 | return if File.exist? ssh_key_fp 31 | 32 | KubernetesHarness.nice_logger.info 'Creating a new SSH key for the cluster.' 33 | ssh_key_command = ShellCommand.new( 34 | "ssh-keygen -t rsa -f '#{ssh_key_fp}' -q -N ''" 35 | ) 36 | raise 'Unable to create a SSH key for the cluster' unless ssh_key_command.execute! 37 | end 38 | 39 | def self.provision!(cluster_info) 40 | all_results = provision_nodes_in_parallel!(cluster_info) 41 | failures = all_results.filter { |thread| !thread.success? } 42 | raise failed_cluster_error(failures) unless failures.empty? 43 | 44 | cluster_info.kubeconfig_path = cluster_kubeconfig 45 | true 46 | end 47 | 48 | # TODO: tests missing 49 | def self.teardown! 50 | destroy_nodes_in_parallel! 51 | 52 | true 53 | end 54 | 55 | # TODO: tests missing 56 | def self.destroy_existing! 57 | destroy_nodes_in_parallel! 58 | end 59 | 60 | def self.destroy_nodes_in_parallel! 61 | if cluster_running? 62 | KubernetesHarness.logger.debug('🚨 Deleting all nodes! 🚨') 63 | vagrant_threads = [] 64 | Constants::ALL_NODES.each do |node| 65 | KubernetesHarness.logger.debug("Starting thread for node #{node}") 66 | vagrant_threads << Thread.new do 67 | vagrant_command = Vagrant.new_command('destroy', ['-f', node]) 68 | vagrant_command.execute! 69 | vagrant_command 70 | end 71 | end 72 | results = vagrant_threads.each(&:join).map(&:value) 73 | failures = results.filter { |result| !result.success? } 74 | raise failed_cluster_destroy_error(failures) unless failures.empty? 75 | 76 | delete_cluster_yaml_and_ssh_key! 77 | else 78 | KubernetesHarness.nice_logger.info('No clusters found to destroy. Stopping.') 79 | end 80 | end 81 | 82 | def self.provision_nodes_in_parallel!(cluster_info) 83 | ansible_threads = [] 84 | ssh_key_path = cluster_info.ssh_key_path 85 | [cluster_info.master_ip_address, 86 | cluster_info.worker_ip_addresses, 87 | cluster_info.docker_registry_address].flatten.each do |addr| 88 | ansible_threads << Thread.new do 89 | command = Ansible::Playbook.create_run_against_single_host( 90 | playbook_fp: playbook_path, 91 | ssh_key_path: ssh_key_path, 92 | inventory_fp: inventory_path, 93 | ip_address: addr, 94 | extra_vars: ["k3s_token=#{cluster_info.kubernetes_cluster_token}"] 95 | ) 96 | command.execute! 97 | command 98 | end 99 | end 100 | ansible_threads.each(&:join).map(&:value) 101 | end 102 | 103 | def self.worker_ip_addresses_command 104 | Constants::WORKER_NODE_NAMES.map do |node| 105 | Vagrant.create_and_execute_new_ssh_command(node, Constants::IP_ETH1_COMMAND) 106 | end 107 | end 108 | 109 | def self.master_ip_address_command 110 | Vagrant.create_and_execute_new_ssh_command(Constants::MASTER_NODE_NAME, 111 | Constants::IP_ETH1_COMMAND) 112 | end 113 | 114 | def self.docker_registry_command 115 | Vagrant.create_and_execute_new_ssh_command(Constants::DOCKER_REGISTRY_NAME, 116 | Constants::IP_ETH1_COMMAND) 117 | end 118 | 119 | def self.cluster_kubeconfig 120 | args = [ 121 | '-c', 122 | '"sudo cat /etc/rancher/k3s/k3s.yaml"', 123 | Constants::MASTER_NODE_NAME.to_s 124 | ] 125 | command = Vagrant.new_command('ssh', args) 126 | command.execute! 127 | if command.stdout.empty? 128 | KubernetesHarness.logger.warn('No kubeconfig created!') 129 | return 130 | end 131 | Metadata.write!('kubeconfig', command.stdout) 132 | File.join Metadata.default_dir, 'kubeconfig' 133 | end 134 | 135 | def self.cluster_ssh_key 136 | File.join KubernetesHarness::Clusters::Metadata.default_dir, '/ssh_key' 137 | end 138 | 139 | def self.playbook_path 140 | File.join Metadata.default_dir, 'site.yml' 141 | end 142 | 143 | def self.inventory_path 144 | File.join Metadata.default_dir, 'inventory' 145 | end 146 | 147 | def self.vagrant_up_disposable_cluster_or_exit! 148 | KubernetesHarness.logger.debug('🚀 Creating node new disposable cluster 🚀') 149 | vagrant_threads = [] 150 | Constants::ALL_NODES.each do |node| 151 | KubernetesHarness.logger.debug("Starting thread for node #{node}") 152 | vagrant_threads << Thread.new do 153 | vagrant_command = Vagrant.new_command('up', [node]) 154 | vagrant_command.execute! 155 | vagrant_command 156 | end 157 | end 158 | results = vagrant_threads.each(&:join).map(&:value) 159 | failures = results.filter { |result| !result.success? } 160 | raise failed_cluster_error(failures) unless failures.empty? 161 | end 162 | 163 | def self.generate_err_msg(cmd) 164 | header = "From command '#{cmd.command}':" 165 | separator = '-' * (header.length + 4) 166 | 167 | <<~MESSAGE 168 | #{header} 169 | #{separator} 170 | 171 | Output: 172 | #{cmd.stdout} 173 | 174 | Errors: 175 | #{cmd.stderr} 176 | MESSAGE 177 | end 178 | 179 | def self.failed_cluster_error(command) 180 | stderr = if command.is_a? Array 181 | command.map do |cmd| 182 | generate_err_msg(cmd) 183 | end.flatten.join("\n\n") 184 | else 185 | generate_err_msg(command) 186 | end 187 | raise "Failed to start Kubernetes cluster. Here's why:\n\n#{stderr}" 188 | end 189 | 190 | def self.failed_cluster_destroy_error(command) 191 | stderr = if command.is_a? Array 192 | command.map(&:stderr).uniq!.join("\n") 193 | else 194 | command.stderr 195 | end 196 | raise "Failed to delete Kubernetes cluster. Here's why:\n\n#{stderr}" 197 | end 198 | 199 | def self.delete_cluster_yaml_and_ssh_key! 200 | ['ssh_key', 'ssh_key.pub', 'cluster.yaml'].each do |file| 201 | Metadata.delete!(file) 202 | end 203 | end 204 | 205 | def self.cluster_running? 206 | vagrant_status_command = Vagrant.new_command('global-status') 207 | vagrant_status_command.execute! 208 | vagrant_status_command.stdout.match?(/k3s-/) 209 | end 210 | end 211 | end 212 | -------------------------------------------------------------------------------- /lib/k8s_harness/clusters/ansible.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'yaml' 4 | require 'k8s_harness/paths' 5 | 6 | module KubernetesHarness 7 | module Clusters 8 | # Simple module for interacting with Vagrant. 9 | module Ansible 10 | # for ansible-playbook 11 | module Playbook 12 | def self.create_run_against_single_host(playbook_fp:, 13 | inventory_fp:, 14 | ssh_key_path:, 15 | ip_address:, 16 | extra_vars:) 17 | log_new_run(playbook_fp, inventory_fp, ssh_key_path, ip_address, extra_vars) 18 | command_env = { 19 | ANSIBLE_HOST_KEY_CHECKING: 'no', 20 | ANSIBLE_SSH_ARGS: '-o IdentitiesOnly=true', 21 | ANSIBLE_COMMAND_WARNINGS: 'False', 22 | ANSIBLE_PYTHON_INTERPRETER: '/usr/bin/python' 23 | } 24 | KubernetesHarness::ShellCommand.new( 25 | command(playbook_fp, inventory_fp, ssh_key_path, ip_address, extra_vars), 26 | environment: command_env 27 | ) 28 | end 29 | 30 | def self.command(playbook_fp, inventory_fp, ssh_key_path, ip_address, extra_vars) 31 | [ 32 | 'ansible-playbook', 33 | "-i #{inventory_fp}", 34 | "-e \"ansible_ssh_user=\\\"#{ENV['ANSIBLE_SSH_USER'] || 'vagrant'}\\\"\"", 35 | extra_vars.map { |var| "-e \"#{var}\"" }, 36 | "-l #{ip_address}", 37 | "--private-key #{ssh_key_path}", 38 | playbook_fp 39 | ].flatten.join(' ') 40 | end 41 | 42 | def self.log_new_run(playbook_fp, inventory_fp, ssh_key_path, ip_address = '', extra_vars) 43 | KubernetesHarness.logger.info( 44 | <<~MESSAGE.strip 45 | Creating a new single-host Ansible Playbook run! \ 46 | playbook: #{playbook_fp}, \ 47 | inventory: #{inventory_fp}, \ 48 | ssh_key: #{ssh_key_path}, \ 49 | ip_address: #{ip_address}, \ 50 | extra_vars: #{extra_vars} 51 | MESSAGE 52 | ) 53 | end 54 | end 55 | end 56 | end 57 | end 58 | -------------------------------------------------------------------------------- /lib/k8s_harness/clusters/cluster_info.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'yaml' 4 | require 'digest/md5' 5 | 6 | module KubernetesHarness 7 | module Clusters 8 | # This class provides a handy set of information that might be useful for k8s-harness 9 | # users after creating their clusters. 10 | class ClusterInfo 11 | attr_reader :master_ip_address, 12 | :worker_ip_addresses, 13 | :docker_registry_address, 14 | :ssh_key_path, 15 | :kubernetes_cluster_token 16 | attr_accessor :kubeconfig_path 17 | 18 | IP_ADDRESS_REGEX = / 19 | \b(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\. 20 | (25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\. 21 | (25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\. 22 | (25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b 23 | /x.freeze 24 | 25 | def initialize(master_ip_address_command:, 26 | worker_ip_addresses_command:, 27 | docker_registry_command:, 28 | kubeconfig_path:, 29 | ssh_key_path:) 30 | @kubeconfig_path = kubeconfig_path 31 | @ssh_key_path = ssh_key_path 32 | @master_ip_address = get_ip_addresses_from_command(master_ip_address_command).first 33 | @docker_registry_address = get_ip_addresses_from_command(docker_registry_command).first 34 | @worker_ip_addresses = 35 | worker_ip_addresses_command.map do |command| 36 | get_ip_addresses_from_command(command) 37 | end.flatten 38 | @kubernetes_cluster_token = generate_k8s_token( 39 | @master_ip_address, 40 | @worker_ip_addresses, 41 | @docker_registry_address 42 | ) 43 | end 44 | 45 | def to_yaml 46 | YAML.dump({ 47 | master_ip_address: @master_ip_address, 48 | worker_ip_addresses: @worker_ip_addresses, 49 | docker_registry_address: @docker_registry_address, 50 | kubeconfig_path: @kubeconfig_path, 51 | ssh_key_path: @ssh_key_path, 52 | kubernetes_cluster_token: @kubernetes_cluster_token 53 | }) 54 | end 55 | 56 | private 57 | 58 | def generate_k8s_token(master_ip, worker_ip, docker_registry) 59 | combined_addresses = [master_ip, worker_ip, docker_registry].flatten.join('') 60 | Digest::MD5.hexdigest(combined_addresses) 61 | end 62 | 63 | def get_ip_addresses_from_command(command) 64 | command.execute! 65 | command.stdout.split("\n").select { |line| line.match? IP_ADDRESS_REGEX } 66 | end 67 | end 68 | end 69 | end 70 | -------------------------------------------------------------------------------- /lib/k8s_harness/clusters/constants.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module KubernetesHarness 4 | module Clusters 5 | # Just constants. 6 | module Constants 7 | MASTER_NODE_NAME = 'k3s-node-0' 8 | WORKER_NODE_NAMES = ['k3s-node-1'].freeze 9 | DOCKER_REGISTRY_NAME = 'k3s-registry' 10 | IP_ETH1_COMMAND = 11 | "\"ip addr show dev eth1 | grep \'\\\' | awk \'{print \\$2}\' | cut -f1 -d \'/\'\"" 12 | ALL_NODES = [MASTER_NODE_NAME, WORKER_NODE_NAMES, DOCKER_REGISTRY_NAME].flatten 13 | end 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /lib/k8s_harness/clusters/metadata.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'fileutils' 4 | require 'k8s_harness/paths' 5 | 6 | module KubernetesHarness 7 | # This module is for everything around CRUD'ing disposable clusters. 8 | module Clusters 9 | # k8s-harness relies on storing things like Ansible playbooks for our 10 | # disposable cluster and extra files that users might use. 11 | # This module handles all of that. 12 | module Metadata 13 | def self.default_dir 14 | "#{ENV['PWD']}/.k8sharness_data" 15 | end 16 | 17 | def self.create_dir! 18 | ::FileUtils.mkdir_p default_dir unless Dir.exist? default_dir 19 | end 20 | 21 | def self.initialize! 22 | create_dir! 23 | FileUtils.cp_r("#{KubernetesHarness::Paths.include_dir}/.", default_dir) 24 | end 25 | 26 | def self.write!(file_name, content) 27 | KubernetesHarness.logger.debug "Creating new metadata: #{file_name}" 28 | fp = File.join default_dir, file_name 29 | File.write(fp, content) 30 | end 31 | 32 | def self.delete!(file_name) 33 | KubernetesHarness.logger.debug "Deleting from metadata: #{file_name}" 34 | fp = File.join default_dir, file_name 35 | FileUtils.rm(fp) 36 | end 37 | end 38 | end 39 | end 40 | -------------------------------------------------------------------------------- /lib/k8s_harness/clusters/required_software.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'yaml' 4 | require 'k8s_harness/paths' 5 | require 'k8s_harness/shell_command' 6 | 7 | module KubernetesHarness 8 | module Clusters 9 | # This module ensures that we have the software we need to run k8s-harness 10 | # on the user's machine. 11 | module RequiredSoftware 12 | def self.software 13 | YAML.safe_load( 14 | File.read(File.join(KubernetesHarness::Paths.conf_dir, 'required_software.yaml')), 15 | symbolize_names: true 16 | ) 17 | end 18 | 19 | def self.ensure_installed_or_exit! 20 | missing = [] 21 | software.each do |app_data| 22 | name = app_data[:name] 23 | version_check = app_data[:version_check] 24 | KubernetesHarness.logger.debug("Checking that this is installed: #{name}") 25 | command_string = "sh -c '#{version_check}; exit $?'" 26 | command = KubernetesHarness::ShellCommand.new(command_string) 27 | command.execute! 28 | missing.push name unless command.success? 29 | end 30 | 31 | raise show_missing_software_message(missing) unless missing.empty? 32 | end 33 | 34 | def self.show_missing_software_message(apps) 35 | <<~MESSAGE.strip 36 | You are missing the following software: 37 | 38 | #{apps.map { |app| "- #{app}" }.join("\n")} 39 | 40 | Please consult the README to learn what you'll need to install before using k8s-harness. 41 | MESSAGE 42 | end 43 | 44 | private_class_method :show_missing_software_message 45 | end 46 | end 47 | end 48 | -------------------------------------------------------------------------------- /lib/k8s_harness/clusters/vagrant.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module KubernetesHarness 4 | module Clusters 5 | # Simple module for interacting with Vagrant. 6 | module Vagrant 7 | def self.new_command(command, args = nil) 8 | command_env = { 9 | VAGRANT_CWD: Metadata.default_dir 10 | } 11 | command = "vagrant #{command}" 12 | command = "#{command} #{[args].flatten.join(' ')}" unless args.nil? 13 | KubernetesHarness::ShellCommand.new(command, environment: command_env) 14 | end 15 | 16 | def self.create_and_execute_new_ssh_command(node_name, command) 17 | args = ['-c', command, node_name] 18 | command = Vagrant.new_command('ssh', args) 19 | command.execute! 20 | command 21 | end 22 | end 23 | end 24 | end 25 | -------------------------------------------------------------------------------- /lib/k8s_harness/harness_file.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'shellwords' 4 | require 'yaml' 5 | require 'k8s_harness/shell_command' 6 | 7 | module KubernetesHarness 8 | # This module handles reading and validating .k8sharness files. 9 | module HarnessFile 10 | def self.execute_setup!(options) 11 | exec_command!(options, :setup, 'Setting up your tests.') 12 | end 13 | 14 | # TODO: Tests missing (but execute_setup! has a test and implementation is the same.) 15 | def self.execute_tests!(options) 16 | exec_command!(options, :test, 'Running your tests.') 17 | end 18 | 19 | # TODO: Tests missing (but execute_setup! has a test and implementation is the same.) 20 | def self.execute_teardown!(options) 21 | exec_command!(options, :teardown, 'Tearing down your test bench.') 22 | end 23 | 24 | def self.exec_command!(options, key, message) 25 | rendered = render(options) 26 | raise 'No tests found' if (key == :test) && !rendered.key?(:test) 27 | 28 | KubernetesHarness.logger.debug "Checking for: #{key}" 29 | return nil unless rendered.key? key 30 | 31 | KubernetesHarness.nice_logger.info message 32 | command = KubernetesHarness::ShellCommand.new(rendered[key]) 33 | command.execute! 34 | KubernetesHarness.logger.error command.stderr unless command.stderr.empty? 35 | puts command.stdout 36 | end 37 | 38 | def self.test_present?(options) 39 | harness_file(options).key? :test 40 | end 41 | 42 | def self.convert_to_commands(options) 43 | # TODO: Currently, we are assuming that the steps provided in the .k8sharness 44 | # will always be invoked in a shell. 45 | # First, we shouldn't assume that the user will want to use `sh` for these commands. 46 | # Second, we should allow users to invoke code in the language of their preference to 47 | # maximize codebase homogeneity. 48 | rendered = harness_file(options) 49 | rendered.each_key do |key| 50 | if rendered[key].match?(/.(sh|bash|zsh)$/) 51 | rendered[key] = "sh #{rendered[key]}" 52 | else 53 | rendered[key] = "sh -c '#{Shellwords.escape(rendered[key])}'" \ 54 | unless rendered[key].match?(/^(sh|bash|zsh) -c/) 55 | end 56 | end 57 | end 58 | 59 | def self.render(options = {}) 60 | fp = harness_file_path(options) 61 | raise "k8s-harness file not found at: #{fp}" unless File.exist? fp 62 | return convert_to_commands(options) if test_present?(options) 63 | 64 | raise KeyError, <<~MESSAGE.strip 65 | It appears that your test isn't defined in #{fp}. Ensure that \ 66 | a key called 'test' is in #{fp}. See .k8sharness.example for \ 67 | an example of what a valid .k8sharness looks like. 68 | MESSAGE 69 | end 70 | 71 | def self.validate(options) 72 | puts YAML.dump(render(options.to_h)) 73 | end 74 | 75 | def self.default_harness_file_path 76 | "#{Dir.pwd}/.k8sharness" 77 | end 78 | 79 | def self.harness_file_path(options) 80 | if !options.nil? && options.key?(:alternate_harnessfile) 81 | options[:alternate_harnessfile] 82 | else 83 | default_harness_file_path 84 | end 85 | end 86 | 87 | def self.harness_file(options) 88 | YAML.safe_load(File.read(harness_file_path(options)), symbolize_names: true) 89 | end 90 | end 91 | end 92 | -------------------------------------------------------------------------------- /lib/k8s_harness/logging.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'logger' 4 | 5 | # KubernetesHarness. 6 | module KubernetesHarness 7 | @logger = Logger.new($stdout) 8 | @logger.level = ENV['LOG_LEVEL'] || Logger::WARN 9 | @nice_logger = Logger.new($stdout) 10 | @nice_logger.formatter = proc do |_sev, datetime, _app, message| 11 | if @logger.level == Logger::DEBUG 12 | "--> [#{datetime.strftime('%F %T %z')}] #{message}\n" 13 | else 14 | "--> #{message}\n" 15 | end 16 | end 17 | def self.logger 18 | @logger 19 | end 20 | 21 | def self.nice_logger 22 | @nice_logger 23 | end 24 | 25 | # Functions to manipulate log control. 26 | module Logging 27 | def self.enable_debug_logging 28 | KubernetesHarness.logger.level = Logger::DEBUG 29 | end 30 | end 31 | end 32 | -------------------------------------------------------------------------------- /lib/k8s_harness/paths.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module KubernetesHarness 4 | # The canonical source of all toplevel paths 5 | module Paths 6 | def self.root_dir 7 | File.expand_path '../..', __dir__ 8 | end 9 | 10 | def self.include_dir 11 | File.join root_dir, 'include' 12 | end 13 | 14 | def self.conf_dir 15 | File.join root_dir, 'conf' 16 | end 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /lib/k8s_harness/shell_command.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'English' 4 | require 'open3' 5 | 6 | module KubernetesHarness 7 | # Handles all interactions with shells 8 | class ShellCommand 9 | attr_accessor :command, :stdout, :stderr 10 | 11 | # Ruby 2.7 deprecated keyword arguments in favor of passing in Hashes. 12 | # TODO: Refactor to account for this. 13 | def initialize(command, environment: {}) 14 | KubernetesHarness.logger.debug("Creating new command #{command} with env #{environment}") 15 | @command = command 16 | @environment = environment 17 | @exitcode = nil 18 | @stdout = nil 19 | @stderr = nil 20 | end 21 | 22 | def execute! 23 | @stdout, @stderr, @exitcode = read_output_in_chunks(@environment) 24 | 25 | show_debug_command_output 26 | end 27 | 28 | def success?(exit_code: 0) 29 | @exitcode == exit_code 30 | end 31 | 32 | private 33 | 34 | def read_output_in_chunks(environment = {}) 35 | # Courtesy of: https://gist.github.com/chrisn/7450808 36 | def all_eof(files) 37 | files.find { |f| !f.eof }.nil? 38 | end 39 | block_size = 1024 40 | final_stdout = '' 41 | final_stderr = '' 42 | final_process = nil 43 | KubernetesHarness.logger.debug("Running #{@command} with env #{environment}") 44 | Open3.popen3(environment.transform_keys(&:to_s), @command) do |stdin, stdout, stderr, thread| 45 | stdin.close_write 46 | 47 | begin 48 | files = [stdout, stderr] 49 | until all_eof(files) 50 | ready = IO.select(files) 51 | next unless ready 52 | 53 | readable = ready[0] 54 | readable.each do |f| 55 | data = f.read_nonblock(block_size) 56 | stdout_chunk = f == stdout ? data : '' 57 | stderr_chunk = f == stderr ? data : '' 58 | if f == stdout 59 | final_stdout += stdout_chunk 60 | else 61 | final_stderr += stderr_chunk 62 | end 63 | KubernetesHarness.logger.debug("command: #{@command}, stdout_chunk: #{stdout_chunk}") 64 | KubernetesHarness.logger.debug("command: #{@command}, stderr_chunk: #{stderr_chunk}") 65 | rescue EOFError 66 | KubernetesHarness.logger.debug("command: #{@command}, stream has EOF'ed") 67 | end 68 | end 69 | rescue IOError => e 70 | puts "IOError: #{e}" 71 | end 72 | final_process = thread.value.exitstatus 73 | end 74 | 75 | [final_stdout, final_stderr, final_process] 76 | end 77 | 78 | def show_debug_command_output 79 | message = <<~MESSAGE.strip 80 | Running #{@command} done, \ 81 | rc = #{@exitcode}, \ 82 | stdout = '#{@stdout}', \ 83 | stderr = '#{@stderr}' 84 | MESSAGE 85 | KubernetesHarness.logger.debug message 86 | end 87 | end 88 | end 89 | -------------------------------------------------------------------------------- /lib/k8s_harness/subcommand.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'k8s_harness/clusters' 4 | require 'k8s_harness/clusters/cluster_info' 5 | require 'k8s_harness/clusters/metadata' 6 | require 'k8s_harness/harness_file' 7 | 8 | module KubernetesHarness 9 | # All entrypoints for our subcommands live here. 10 | module Subcommand 11 | def self.run(options = {}) 12 | fail_if_validate_fails!(options) 13 | disable_teardown = !options.nil? && options[:disable_teardown] 14 | return true if !options.nil? && options[:show_usage] 15 | 16 | print_warning_if_teardown_disabled(disable_teardown) 17 | cluster_info = create! 18 | provision!(cluster_info) 19 | print_post_create_message(cluster_info) 20 | setup!(options) 21 | run_tests!(options) 22 | teardown!(options) 23 | destroy_cluster!(disable_teardown) 24 | end 25 | 26 | def self.validate(options = {}) 27 | return true if options.to_h[:show_usage] 28 | 29 | KubernetesHarness::HarnessFile.validate(options) 30 | end 31 | 32 | def self.destroy(options = {}) 33 | return true if options.to_h[:show_usage] 34 | 35 | KubernetesHarness.nice_logger.info('Destroying your cluster (if any found).') 36 | KubernetesHarness::Clusters.destroy_existing! 37 | end 38 | 39 | def self.print_warning_if_teardown_disabled(teardown_flag) 40 | return unless teardown_flag 41 | 42 | KubernetesHarness.nice_logger.warn( 43 | <<~MESSAGE.strip 44 | Teardown is disabled. Your cluster will stay up until you run \ 45 | 'k8s-harness destroy'. 46 | MESSAGE 47 | ) 48 | end 49 | 50 | def self.create! 51 | KubernetesHarness.nice_logger.info( 52 | 'Creating your cluster now. Provisioning will occur in a few minutes.' 53 | ) 54 | KubernetesHarness::Clusters.create! 55 | end 56 | 57 | def self.provision!(cluster_info) 58 | KubernetesHarness.nice_logger.info('Provisioning the cluster. This will take a few minutes.') 59 | KubernetesHarness::Clusters.provision!(cluster_info) 60 | end 61 | 62 | def self.setup!(options) 63 | KubernetesHarness::HarnessFile.execute_setup!(options) 64 | end 65 | 66 | def self.run_tests!(options) 67 | KubernetesHarness.nice_logger.info('Running your tests.') 68 | KubernetesHarness::HarnessFile.execute_tests!(options) 69 | end 70 | 71 | def self.teardown!(options) 72 | KubernetesHarness::HarnessFile.execute_teardown!(options) 73 | end 74 | 75 | def self.destroy_cluster!(disable_teardown) 76 | KubernetesHarness.nice_logger.info('Done. Tearing down the cluster.') 77 | KubernetesHarness::Clusters.teardown! unless disable_teardown 78 | end 79 | 80 | def self.fail_if_validate_fails!(options) 81 | _ = KubernetesHarness::HarnessFile.render(options) 82 | end 83 | 84 | def self.print_post_create_message(cluster_info) 85 | # TODO: Make this not hardcoded. 86 | cluster_info_yaml_path = File.join Clusters::Metadata.default_dir, 'cluster.yaml' 87 | KubernetesHarness.nice_logger.info( 88 | <<~MESSAGE.strip 89 | Cluster has been created. Details are below and in YAML at #{cluster_info_yaml_path}: 90 | 91 | * Master address: '#{cluster_info.master_ip_address}' 92 | * Worker addresses: #{cluster_info.worker_ip_addresses} 93 | * Docker registry address: '#{cluster_info.docker_registry_address}' 94 | * Kubeconfig path: #{cluster_info.kubeconfig_path} 95 | * SSH key path: #{cluster_info.ssh_key_path} 96 | MESSAGE 97 | ) 98 | end 99 | end 100 | end 101 | -------------------------------------------------------------------------------- /scripts/verify_runner_doesnt_suck.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | usage() { 4 | cat <<-USAGE 5 | $(basename $0) 6 | Ensures that the GitHub Actions runner we are using is configured properly. 7 | 8 | OPTIONS 9 | 10 | -f, --force Run this script regardless of whether we're in CI or not. 11 | USAGE 12 | } 13 | 14 | show_info() { 15 | >&2 printf "\033[1;32m--> [INFO]\033[m $1\n" 16 | } 17 | 18 | show_error() { 19 | >&2 printf "\033[1;31m--> [ERROR]\033[m $1\n" 20 | } 21 | 22 | see_if_we_can_download_boxes() { 23 | # Vagrant kept bombing in integration due to boxes not being pulled correctly. 24 | # Let's try to download the boxes we need to run our integration tests ahead of them 25 | # executing. 26 | show_info "Checking if we can download and unzip Vagrant boxes." 27 | vagrant_boxes_used_by_k8s_harness=$(cat "$(dirname $0)/../include/Vagrantfile" | 28 | grep config.vm.box | 29 | sed 's/.*config.vm.box = "\(.*\)"$/\1/' 30 | ) 31 | for box in $vagrant_boxes_used_by_k8s_harness 32 | do 33 | vagrant box add "$box" --force 34 | done 35 | } 36 | 37 | ensure_ci_or_force_run_enabled() { 38 | test "$CI" == "true" || \ 39 | (echo "$*" | grep -wq -- '-f') || \ 40 | (echo "$*" | grep -wq -- '--force') 41 | } 42 | 43 | show_diagnostics() { 44 | consumed_disk_space() { 45 | du -sh "$PWD" 46 | } 47 | 48 | free_disk_space() { 49 | df -h "$PWD" 50 | } 51 | 52 | echo "Here are some helpful diagnostics:" 53 | cat <<-DIAGNOSTICS 54 | Diagnostic info 55 | ==================== 56 | 57 | Consumed disk space 58 | -------------------- 59 | $(consumed_disk_space) 60 | 61 | Free disk space 62 | ---------------- 63 | $(free_disk_space) 64 | DIAGNOSTICS 65 | } 66 | 67 | if [ "$1" == '-h' ] || [ "$1" == '--help' ] 68 | then 69 | usage 70 | exit 0 71 | fi 72 | 73 | if [ "$1" == '-v' ] || [ "$1" == '--version' ] 74 | then 75 | printf "$(basename $0) version $(cat $(dirname $0)/../VERSION)" 76 | exit 0 77 | fi 78 | 79 | if ! ensure_ci_or_force_run_enabled $* 80 | then 81 | show_error "This script only runs in CI environments. \ 82 | Add \033[1;36m[-f | --force]\033[m to override this." 83 | exit 1 84 | fi 85 | 86 | if ! { see_if_we_can_download_boxes; } 87 | then 88 | show_error "Integration environment is broken. Please re-run this on another runner." 89 | show_diagnostics 90 | exit 1 91 | fi 92 | -------------------------------------------------------------------------------- /tests/fixtures/.k8sharness.key_already_has_sh_command: -------------------------------------------------------------------------------- 1 | --- 2 | setup: sh -c "Foo" 3 | test: bar 4 | -------------------------------------------------------------------------------- /tests/fixtures/.k8sharness.key_references_script: -------------------------------------------------------------------------------- 1 | --- 2 | test: bar 3 | setup: foo.sh 4 | -------------------------------------------------------------------------------- /tests/fixtures/.k8sharness.missing_setup_key: -------------------------------------------------------------------------------- 1 | --- 2 | test: foo 3 | teardown: bar 4 | -------------------------------------------------------------------------------- /tests/fixtures/.k8sharness.missing_teardown_key: -------------------------------------------------------------------------------- 1 | --- 2 | test: foo 3 | setup: bar 4 | -------------------------------------------------------------------------------- /tests/fixtures/.k8sharness.missing_tests_key: -------------------------------------------------------------------------------- 1 | --- 2 | setup: "nope" 3 | teardown: "nope" 4 | -------------------------------------------------------------------------------- /tests/fixtures/.k8sharness.valid_with_all_keys: -------------------------------------------------------------------------------- 1 | # vi: set ft=yaml: 2 | setup: setup.sh 3 | test: sh -c "echo Look! A test!" 4 | teardown: teardown.sh 5 | -------------------------------------------------------------------------------- /tests/harness_file_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'spec_helper' 4 | 5 | # rubocop:disable Metrics/BlockLength 6 | describe 'Given a function that renders .k8sharness files' do 7 | context 'When it tries to find where .k8sharness is' do 8 | context 'And I provide it with no options' do 9 | example 'Then it tries to render the .k8sharness file at the root of your pwd' do 10 | FakeFS do 11 | FileUtils.mkdir_p '/foo' 12 | File.write('/foo/.k8sharness', '{}') 13 | allow(Dir).to receive(:pwd).and_return '/foo' 14 | allow(KubernetesHarness::HarnessFile).to receive(:test_present?).and_return true 15 | expect(KubernetesHarness::HarnessFile.render({})).to eq({}) 16 | end 17 | end 18 | end 19 | 20 | context 'And I provide it with an alternate Harness file' do 21 | example 'Then it tries to render the .k8sharness file at the path provided' do 22 | FakeFS do 23 | test_options = { alternate_harnessfile: '/bar/.k8sharness' } 24 | FileUtils.mkdir_p '/bar' 25 | File.write('/bar/.k8sharness', '{}') 26 | allow(KubernetesHarness::HarnessFile).to receive(:test_present?).and_return true 27 | expect(KubernetesHarness::HarnessFile.render(test_options)).to eq({}) 28 | end 29 | end 30 | end 31 | end 32 | 33 | context 'When it renders them' do 34 | test_cases = { 35 | invalid_missing_test_key: { 36 | name: 'missing "test" key', 37 | file: '.k8sharness.missing_tests_key', 38 | failure: true, 39 | error: <<~MESSAGE.strip 40 | It appears that your test isn't defined in [HARNESS_FILE]. Ensure that \ 41 | a key called 'test' is in [HARNESS_FILE]. See .k8sharness.example for \ 42 | an example of what a valid .k8sharness looks like. 43 | MESSAGE 44 | }, 45 | valid_missing_setup_key: { 46 | name: 'missing "setup" key', 47 | file: '.k8sharness.missing_setup_key', 48 | failure: false, 49 | expected: { test: "sh -c 'foo'", teardown: "sh -c 'bar'" } 50 | }, 51 | valid_missing_teardown_key: { 52 | name: 'missing "teardown" key', 53 | file: '.k8sharness.missing_teardown_key', 54 | failure: false, 55 | expected: { test: "sh -c 'foo'", setup: "sh -c 'bar'" } 56 | }, 57 | key_already_has_sh_command: { 58 | name: 'file is valid but one of the keys already has a "sh" command in it', 59 | file: '.k8sharness.key_already_has_sh_command', 60 | failure: false, 61 | expected: { test: "sh -c 'bar'", setup: 'sh -c "Foo"' } 62 | }, 63 | key_references_a_script: { 64 | name: 'file is valid but one of the keys references a script', 65 | file: '.k8sharness.key_references_script', 66 | failure: false, 67 | expected: { test: "sh -c 'bar'", setup: 'sh foo.sh' } 68 | } 69 | } 70 | test_cases.each_key do |test_case| 71 | expected_result = test_cases[test_case][:failure] ? 'fail' : 'succeed' 72 | condition = test_cases[test_case][:name] 73 | expected_object = test_cases[test_case][:expected] || {} 74 | expected_error = if test_cases[test_case].key? :error 75 | test_cases[test_case][:error].gsub(/\[HARNESS_FILE\]/, '/foo/.k8sharness') 76 | else 77 | String.new 78 | end 79 | test_file = "#{ENV['PWD']}/tests/fixtures/#{test_cases[test_case][:file]}" 80 | example "Then render should #{expected_result} because '#{condition}'" do 81 | FakeFS do 82 | FakeFS::FileSystem.clone(test_file) 83 | FileUtils.mkdir_p('/foo') 84 | FileUtils.copy(test_file, '/foo/.k8sharness') 85 | allow(Dir).to receive(:pwd).and_return('/foo') 86 | if test_cases[test_case][:failure] 87 | expect { KubernetesHarness::HarnessFile.render } 88 | .to raise_error(KeyError, expected_error) 89 | else 90 | expect(KubernetesHarness::HarnessFile.render) 91 | .to eq(expected_object) 92 | end 93 | end 94 | end 95 | end 96 | end 97 | end 98 | 99 | describe 'Given a function that validates .k8sharness files' do 100 | context 'When I run it' do 101 | example 'Then I am given a YAML representation of my rendered file', :wip do 102 | test_file = "#{ENV['PWD']}/tests/fixtures/.k8sharness.valid_with_all_keys" 103 | expected_output = { 104 | setup: 'sh setup.sh', 105 | test: "sh -c 'echo Look! A test!'", 106 | teardown: 'sh teardown.sh' 107 | } 108 | FakeFS do 109 | FakeFS::FileSystem.clone(test_file) 110 | FileUtils.mkdir_p '/foo' 111 | FileUtils.cp(test_file, '/foo/.k8sharness') 112 | allow(Dir).to receive(:pwd).and_return('/foo') 113 | expect(KubernetesHarness::HarnessFile.validate) 114 | .to eq expected_output 115 | end 116 | end 117 | end 118 | end 119 | 120 | describe 'Given a function that sets up a test suite' do 121 | context 'When the Harness file does not have a setup instruction' do 122 | example 'Then it does not run' do 123 | harness_file_double = { 124 | test: 'foo' 125 | } 126 | allow(KubernetesHarness::HarnessFile) 127 | .to receive(:render) 128 | .and_return(harness_file_double) 129 | expect(KubernetesHarness::HarnessFile.execute_setup!({})) 130 | .to be nil 131 | end 132 | end 133 | context 'When the Harness file does have a setup instruction' do 134 | example 'Then it runs' do 135 | command_double = double(KubernetesHarness::ShellCommand, 136 | command: 'sh -c "bar"', 137 | execute!: true, 138 | stderr: '', 139 | stdout: 'Bar.') 140 | harness_file_mock = { 141 | setup: 'sh -c "bar"', 142 | test: 'sh -c "foo"' 143 | } 144 | allow(KubernetesHarness::HarnessFile) 145 | .to receive(:render) 146 | .and_return(harness_file_mock) 147 | allow(KubernetesHarness::ShellCommand) 148 | .to receive(:new) 149 | .with('sh -c "bar"') 150 | .and_return(command_double) 151 | expect { KubernetesHarness::HarnessFile.execute_setup!({}) } 152 | .to output("Bar.\n") 153 | .to_stdout 154 | end 155 | end 156 | end 157 | # rubocop:enable Metrics/BlockLength 158 | -------------------------------------------------------------------------------- /tests/integration/.k8sharness: -------------------------------------------------------------------------------- 1 | --- 2 | test: sh -c 'echo "Your test ran successfully!"' 3 | -------------------------------------------------------------------------------- /tests/integration/run_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'open3' 4 | require 'spec_helper' 5 | 6 | describe 'Given the k8s-harness app' do 7 | context 'When I execute "run"' do 8 | before(:each) do 9 | FakeFS.deactivate! 10 | ENV['PWD'] = "#{ENV['PWD']}/tests/integration" 11 | end 12 | 13 | after(:each) do |example| 14 | KubernetesHarness::CLI.parse(['destroy']) if example.exception 15 | FakeFS.activate! 16 | end 17 | 18 | # We need $STDOUT and $STDERR for seeing whether our test ran. 19 | example 'Then a test should have executed', :integration do 20 | expect { KubernetesHarness::CLI.parse(['run']) } 21 | .to output(/Your test ran successfully!/) 22 | .to_stdout 23 | end 24 | end 25 | end 26 | -------------------------------------------------------------------------------- /tests/k8s_harness_cli_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'spec_helper' 4 | 5 | # rubocop:disable Metrics/BlockLength 6 | describe 'Given a CLI that runs k8s-harness' do 7 | before(:all) do 8 | @helpdoc = <<~HELPDOC 9 | Usage: k8s-harness [subcommand] [options] 10 | Test your apps in disposable Kubernetes clusters 11 | 12 | Sub-commands: 13 | 14 | #{'run'.ljust(20)} Runs tests 15 | #{'validate'.ljust(20)} Validates .k8sharness files 16 | #{'destroy'.ljust(20)} Deletes a live cluster provisioned by k8s-harness WITHOUT WARNING. 17 | 18 | See k8s-harness [subcommand] --help for more specific options. 19 | 20 | Global options: 21 | #{'-d, --debug'.ljust(32)} Show debug output 22 | #{'-h, --help'.ljust(32)} Displays this help message 23 | HELPDOC 24 | end 25 | 26 | context 'When I run it with no options' do 27 | example 'Then it prints a usage doc' do 28 | example_args = [] 29 | expect { KubernetesHarness::CLI.parse(example_args) } 30 | .to output(@helpdoc).to_stdout 31 | end 32 | end 33 | 34 | ['-h', '--help'].each do |help_option| 35 | context "When I run #{help_option}" do 36 | example 'Then it prints a usage doc' do 37 | example_args = [help_option] 38 | expect { KubernetesHarness::CLI.parse(example_args) }.to output(@helpdoc).to_stdout 39 | end 40 | end 41 | end 42 | 43 | context "When I run 'validate'" do 44 | example 'Then it runs the validate function' do 45 | allow(KubernetesHarness::CLI) 46 | .to receive(:call_entrypoint) 47 | .with('validate') 48 | .and_return("Hi, I'm TestFile!") 49 | example_args = ['validate'] 50 | expect(KubernetesHarness::CLI.parse(example_args)).to eq "Hi, I'm TestFile!" 51 | end 52 | end 53 | end 54 | # rubocop:enable Metrics/BlockLength 55 | -------------------------------------------------------------------------------- /tests/k8s_harness_cluster_info_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # rubocop:disable Metrics/BlockLength 4 | describe 'Given a class that provides information about a cluster from Vagrant info' do 5 | context 'When I create a new instance of it' do 6 | before(:each) do 7 | allow(Digest::MD5).to receive(:hexdigest).and_return '12345' 8 | @master_ip_address_double = double(KubernetesHarness::ShellCommand, 9 | command: 'anything', 10 | exitcode: 87, 11 | execute!: true, 12 | stdout: '1.2.3.4') 13 | @docker_registry_command_double = double(KubernetesHarness::ShellCommand, 14 | command: 'anything', 15 | exitcode: 87, 16 | execute!: true, 17 | stdout: '8.9.0.1') 18 | @worker_ip_addresses_double = double(KubernetesHarness::ShellCommand, 19 | command: 'anything', 20 | exitcode: 14, 21 | execute!: true, 22 | stdout: "1.2.3.5\n1.2.3.6\n1.2.3.7") 23 | @cluster_info = KubernetesHarness::Clusters::ClusterInfo.new( 24 | master_ip_address_command: @master_ip_address_double, 25 | worker_ip_addresses_command: [@worker_ip_addresses_double], 26 | docker_registry_command: @docker_registry_command_double, 27 | kubeconfig_path: '/path/to/kubeconfig', 28 | ssh_key_path: '/path/to/ssh_key' 29 | ) 30 | end 31 | example 'Then it should have the correct master IP address' do 32 | expect(@cluster_info.master_ip_address).to eq '1.2.3.4' 33 | end 34 | 35 | example 'Then it should have the correct worker IP address' do 36 | expect(@cluster_info.worker_ip_addresses).to eq ['1.2.3.5', 37 | '1.2.3.6', 38 | '1.2.3.7'] 39 | end 40 | 41 | example 'Then it should have the correct Docker registry address' do 42 | expect(@cluster_info.docker_registry_address).to eq '8.9.0.1' 43 | end 44 | 45 | example 'Then it should have the correct cluser token' do 46 | expect(@cluster_info.kubernetes_cluster_token).to eq '12345' 47 | end 48 | 49 | example 'Then it should have the correct kubeconfig and SSH key paths' do 50 | expect(@cluster_info.kubeconfig_path).to eq '/path/to/kubeconfig' 51 | expect(@cluster_info.ssh_key_path).to eq '/path/to/ssh_key' 52 | end 53 | end 54 | end 55 | # rubocop:enable Metrics/BlockLength 56 | -------------------------------------------------------------------------------- /tests/k8s_harness_clusters_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # rubocop:disable Metrics/BlockLength 4 | describe 'Given a function that creates new disposable clusters' do 5 | context 'When I run it' do 6 | before(:all) do 7 | @harness_spec = { 8 | setup: "sh -c 'echo \"Running setup\"'", 9 | test: "sh -c 'echo \"Running tests\"'", 10 | teardown: "sh -c 'echo \"Running teardown\"'" 11 | } 12 | end 13 | context "And I'm creating metadata" do 14 | example 'Then it creates a hidden directory for metadata' do 15 | ENV['PWD'] = '/foo' 16 | FakeFS do 17 | FakeFS::FileSystem.clone(KubernetesHarness::Paths.include_dir) 18 | KubernetesHarness::Clusters::Metadata.create_dir! 19 | expect(File).to exist '/foo/.k8sharness_data' 20 | end 21 | end 22 | 23 | example 'Then it copies the right stuff into that directory' do 24 | expected_files = %w[Vagrantfile inventory site.yml] 25 | allow(KubernetesHarness::Clusters::Metadata) 26 | .to receive(:create_dir!) 27 | .and_return true 28 | ENV['PWD'] = '/foo' 29 | FakeFS do 30 | FileUtils.mkdir_p '/foo/.k8sharness_data' 31 | KubernetesHarness::Clusters::Metadata.initialize! 32 | expected_files.each do |file| 33 | expected_file = "/foo/.k8sharness_data/#{file}" 34 | expect(File).to exist(expected_file), 35 | "File expected to exist but does not: #{expected_file}" 36 | end 37 | end 38 | end 39 | end 40 | 41 | context "And I'm verifying that my machine is configured correctly" do 42 | example "Then it fails if I don't have necessary software" do 43 | mocked_software = [ 44 | { 45 | name: 'foo', 46 | version_check: 'foo --version' 47 | }, 48 | { 49 | name: 'bar', 50 | version_check: 'bar --version' 51 | }, 52 | { 53 | name: 'baz', 54 | version_check: 'baz --version' 55 | } 56 | ] 57 | allow(KubernetesHarness::Clusters::RequiredSoftware) 58 | .to receive(:software) 59 | .and_return(mocked_software) 60 | mocked_software.each do |app| 61 | mocked_result = app[:name] == 'baz' 62 | command_string = "sh -c '#{app[:version_check]}; exit $?'" 63 | shellcommand_double = double(KubernetesHarness::ShellCommand, 64 | success?: mocked_result, 65 | execute!: nil) 66 | allow(KubernetesHarness::ShellCommand) 67 | .to receive(:new) 68 | .with(command_string) 69 | .and_return shellcommand_double 70 | end 71 | error_message = <<~MESSAGE.strip 72 | You are missing the following software: 73 | 74 | - foo 75 | - bar 76 | 77 | Please consult the README to learn what you'll need to install \ 78 | before using k8s-harness. 79 | MESSAGE 80 | expect { KubernetesHarness::Clusters::RequiredSoftware.ensure_installed_or_exit! } 81 | .to raise_error(error_message) 82 | end 83 | example 'Then it passes if I have all of the necessary software' do 84 | mocked_software = [ 85 | { 86 | name: 'foo', 87 | version_check: 'foo --version' 88 | }, 89 | { 90 | name: 'bar', 91 | version_check: 'bar --version' 92 | } 93 | ] 94 | allow(KubernetesHarness::Clusters::RequiredSoftware) 95 | .to receive(:software) 96 | .and_return(mocked_software) 97 | mocked_software.each do |app| 98 | shellcommand_double = double(KubernetesHarness::ShellCommand, 99 | success?: true, 100 | execute!: nil) 101 | command_string = "sh -c '#{app[:version_check]}; exit $?'" 102 | allow(KubernetesHarness::ShellCommand) 103 | .to receive(:new) 104 | .with(command_string) 105 | .and_return shellcommand_double 106 | end 107 | expect { KubernetesHarness::Clusters::RequiredSoftware.ensure_installed_or_exit! } 108 | .not_to raise_error 109 | end 110 | end 111 | end 112 | 113 | context 'When I create SSH keys' do 114 | example 'Then it creates the keypair' do 115 | ssh_keygen_command = "ssh-keygen -t rsa -f '/foo/.k8sharness_data/ssh_key' -q -N ''" 116 | command_double = double(KubernetesHarness::ShellCommand, 117 | command: ssh_keygen_command, 118 | execute!: true, 119 | exitcode: 0) 120 | expect(KubernetesHarness::ShellCommand) 121 | .to receive(:new) 122 | .with(ssh_keygen_command) 123 | .and_return(command_double) 124 | KubernetesHarness::Clusters.create_ssh_key! 125 | end 126 | end 127 | context 'When I create the disposable cluster' do 128 | before(:each) do 129 | ENV['PWD'] = '/foo' 130 | @mocked_env = { 131 | VAGRANT_CWD: KubernetesHarness::Clusters::Metadata.default_dir, 132 | ANSIBLE_HOST_KEY_CHECKING: 'no', 133 | ANSIBLE_COMMAND_WARNINGS: 'False', 134 | ANSIBLE_SSH_ARGS: '-o IdentitiesOnly=true', 135 | ANSIBLE_PYTHON_INTERPRETER: '/usr/bin/python' 136 | } 137 | end 138 | 139 | example 'Then a cluster is created if all commands process succesfully' do 140 | base_ip_address_command = [ 141 | 'vagrant ssh', 142 | '-c', 143 | "\"ip addr show dev eth1 | grep \'\\\' | awk \'{print \\$2}\' | cut -f1 -d \'/\'\"", 144 | '%%node%%' 145 | ].join(' ') 146 | master_ip_address_command = { master: base_ip_address_command.gsub('%%node%%', 'k3s-node-0') } 147 | worker_ip_address_command = { worker: base_ip_address_command.gsub('%%node%%', 'k3s-node-1') } 148 | docker_registry_command = { registry: base_ip_address_command.gsub('%%node%%', 'k3s-registry') } 149 | all_command_mocks = {} 150 | [master_ip_address_command, 151 | worker_ip_address_command, 152 | docker_registry_command].each do |kvp| 153 | double_name = kvp.keys.first 154 | command = kvp[double_name] 155 | vagrant_env = @mocked_env.select { |k| k.to_s.match?(/VAGRANT/) } 156 | shell_command_mock = double(KubernetesHarness::ShellCommand, 157 | command: command, 158 | execute!: true, 159 | stdout: '', 160 | exitcode: 0, 161 | environment: vagrant_env) 162 | allow(KubernetesHarness::ShellCommand) 163 | .to receive(:new) 164 | .with(command, environment: vagrant_env) 165 | .and_return(shell_command_mock) 166 | all_command_mocks[double_name] = shell_command_mock 167 | end 168 | allow(KubernetesHarness::Clusters) 169 | .to receive(:create_ssh_key!) 170 | .and_return true 171 | allow(KubernetesHarness::Clusters::RequiredSoftware) 172 | .to receive(:ensure_installed_or_exit!) 173 | .and_return(true) 174 | allow(KubernetesHarness::Clusters) 175 | .to receive(:vagrant_up_disposable_cluster_or_exit!) 176 | .and_return true 177 | allow(KubernetesHarness::Clusters) 178 | .to receive(:cluster_ssh_key) 179 | .and_return '/path/to/ssh/key' 180 | expect(KubernetesHarness::Clusters::ClusterInfo) 181 | .to receive(:new) 182 | .with(master_ip_address_command: all_command_mocks[:master], 183 | worker_ip_addresses_command: [all_command_mocks[:worker]], 184 | docker_registry_command: all_command_mocks[:registry], 185 | kubeconfig_path: 'not_yet', 186 | ssh_key_path: '/path/to/ssh/key') 187 | expect(KubernetesHarness::Clusters::Metadata) 188 | .to receive(:write!) 189 | .with('cluster.yaml', /^--- {0,}\n/) 190 | FileUtils.mkdir_p('/foo') 191 | KubernetesHarness::Clusters.create! 192 | end 193 | end 194 | 195 | context 'When I provision the cluster' do 196 | before(:each) do 197 | ENV['PWD'] = '/foo' 198 | @mocked_env = { 199 | VAGRANT_CWD: KubernetesHarness::Clusters::Metadata.default_dir, 200 | ANSIBLE_HOST_KEY_CHECKING: 'no', 201 | ANSIBLE_COMMAND_WARNINGS: 'False', 202 | ANSIBLE_SSH_ARGS: '-o IdentitiesOnly=true', 203 | ANSIBLE_PYTHON_INTERPRETER: '/usr/bin/python' 204 | } 205 | end 206 | example 'Then a cluster is provisioned once it is created' do 207 | # No, rubocop, the quotes are needed here. 208 | # rubocop: disable Lint/PercentStringArray 209 | ansible_playbook_base_command = %w[ 210 | ansible-playbook 211 | -i /metadata_dir/inventory 212 | -e "ansible_ssh_user=\"vagrant\"" 213 | -e "k3s_token=12345" 214 | -l 215 | --private-key /metadata_dir/ssh_key 216 | /metadata_dir/site.yml 217 | ].join(' ') 218 | # rubocop: enable Lint/PercentStringArray 219 | cluster_info_double = double( 220 | KubernetesHarness::Clusters::ClusterInfo, 221 | master_ip_address: '1.2.3.4', 222 | worker_ip_addresses: ['4.5.6.7'], 223 | docker_registry_address: '8.9.0.1', 224 | kubernetes_cluster_token: '12345', 225 | ssh_key_path: '/metadata_dir/ssh_key' 226 | ) 227 | ansible_playbook_master_command = ansible_playbook_base_command.gsub('', '1.2.3.4') 228 | ansible_playbook_worker_command = ansible_playbook_base_command.gsub('', '4.5.6.7') 229 | ansible_playbook_registry_command = ansible_playbook_base_command.gsub('', '8.9.0.1') 230 | [ 231 | ansible_playbook_worker_command, 232 | ansible_playbook_registry_command, 233 | ansible_playbook_master_command 234 | ].each do |command| 235 | shell_command_mock = double(KubernetesHarness::ShellCommand, 236 | command: command, 237 | execute!: true, 238 | stdout: '', 239 | exitcode: 0, 240 | success?: true) 241 | ansible_env = @mocked_env.select { |key| key.match? 'ANSIBLE' } 242 | allow(KubernetesHarness::ShellCommand) 243 | .to receive(:new) 244 | .with(command, environment: ansible_env) 245 | .and_return(shell_command_mock) 246 | end 247 | allow(KubernetesHarness::Clusters) 248 | .to receive(:cluster_kubeconfig) 249 | .and_return '/path/to/kubeconfig' 250 | allow(KubernetesHarness::Clusters) 251 | .to receive(:create_ssh_key!) 252 | .and_return true 253 | allow(KubernetesHarness::Clusters::Metadata) 254 | .to receive(:default_dir) 255 | .and_return('/metadata_dir') 256 | expect(cluster_info_double) 257 | .to receive(:kubeconfig_path=) 258 | .with '/path/to/kubeconfig' 259 | expect(KubernetesHarness::Clusters.provision!(cluster_info_double)) 260 | .to be true 261 | end 262 | end 263 | end 264 | # rubocop:enable Metrics/BlockLength 265 | -------------------------------------------------------------------------------- /tests/shell_command_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'spec_helper' 4 | 5 | describe 'Given an object that runs shell commands' do 6 | context 'When it is given a shell command' do 7 | before(:each) do 8 | @test_command = 'my awesome command' 9 | @test_stdout = 'foo' 10 | @test_rc = 42 11 | end 12 | context 'And no environment variables are provided' do 13 | example 'Then it executes it' do 14 | allow_any_instance_of(KubernetesHarness::ShellCommand) 15 | .to receive(:read_output_in_chunks) 16 | .and_return([@test_stdout, '', @test_rc]) 17 | test_command = KubernetesHarness::ShellCommand.new(@test_command) 18 | test_command.execute! 19 | expect(test_command.stdout).to eq @test_stdout 20 | expect(test_command.success?(exit_code: @test_rc)).to be true 21 | end 22 | end 23 | 24 | context 'And environment variables are provided' do 25 | example 'Then it executes it' do 26 | mocked_env = { 'FOO' => 'bar' } 27 | allow_any_instance_of(KubernetesHarness::ShellCommand) 28 | .to receive(:read_output_in_chunks) 29 | .with(mocked_env) 30 | .and_return([@test_stdout, '', @test_rc]) 31 | test_command = KubernetesHarness::ShellCommand.new(@test_command, environment: mocked_env) 32 | test_command.execute! 33 | expect(test_command.stdout).to eq @test_stdout 34 | expect(test_command.success?(exit_code: @test_rc)).to be true 35 | end 36 | end 37 | end 38 | end 39 | -------------------------------------------------------------------------------- /tests/spec_helper.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'pp' 4 | require 'fakefs' 5 | require 'rspec' 6 | require 'k8s_harness' 7 | -------------------------------------------------------------------------------- /tests/subcommand_spec_depcrecated.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'spec_helper' 4 | require 'k8s_harness/subcommand' 5 | 6 | # rubocop: disable Metrics/BlockLength 7 | describe 'Given a module that contains subcommands' do 8 | context 'When I execute run' do 9 | context 'And I ask for usage' do 10 | example 'It should exit because the CLI handles displaying usage' do 11 | expect(KubernetesHarness::Subcommand.run({ show_usage: true })).to be true 12 | end 13 | end 14 | 15 | context 'And I run it by itself' do 16 | before(:each) do 17 | ENV['PWD'] = '/foo' 18 | cluster_info = { 19 | master_ip_address: 'foo', 20 | worker_ip_addresses: ['bar'], 21 | docker_registry_address: 'baz', 22 | kubeconfig_path: '/foo', 23 | ssh_key_path: '/bar' 24 | } 25 | @cluster_info_double = double( 26 | KubernetesHarness::Clusters::ClusterInfo, 27 | cluster_info 28 | ) 29 | @logger_double = double(Logger, 30 | info: nil, 31 | debug: nil) 32 | @expected_yaml_file = YAML.dump(cluster_info) 33 | # This is an anti-pattern since it is chaining tests together. 34 | # This means that any method that involves creating clusters will have to 35 | # update this hash in order for tests to pass. 36 | # TODO: Fix this. 37 | @all_stdout_expected = { 38 | create: ['Creating your cluster now. It will be ready in a few minutes.', 39 | 'Cluster has been created. Details are below and in YAML at /foo/.k8sharness_data/cluster.yaml:', 40 | " Master address: 'foo'", 41 | ' Worker addresses: ["bar"]', 42 | ' Kubeconfig path: /foo', 43 | ' SSH key path: /bar'], 44 | provision: [ 45 | 'Provisioning your cluster. Hang tight; almost there!' 46 | ] 47 | } 48 | end 49 | example 'It should create a cluster' do 50 | FakeFS do 51 | allow(Logger).to receive(:new).and_return(@logger_double) 52 | allow(KubernetesHarness::Clusters) 53 | .to receive(:provision!) 54 | .and_return(true) 55 | expect(KubernetesHarness::Clusters) 56 | .to receive(:create!) 57 | .at_least(1).times 58 | .and_return(@cluster_info_double) 59 | @all_stdout_expected[:create].each do |message| 60 | expect(@logger_double) 61 | .to receive(:info) 62 | .with(message) 63 | end 64 | KubernetesHarness::Subcommand.run 65 | end 66 | end 67 | example 'It should provision the cluster' do 68 | FakeFS do 69 | allow(Logger).to receive(:new).and_return(@logger_double) 70 | allow(KubernetesHarness::Clusters) 71 | .to receive(:provision_nodes_in_parallel!) 72 | .and_return([command_double]) 73 | allow(KubernetesHarness::Clusters) 74 | .to receive(:create!) 75 | .and_return(@cluster_info_double) 76 | expect(KubernetesHarness::Clusters) 77 | .to receive(:provision!) 78 | .with(@cluster_info_double) 79 | .and_return(true) 80 | (@all_stdout_expected[:create] + @all_stdout_expected[:provision]).each do |message| 81 | expect(@logger_double) 82 | .to receive(:info) 83 | .with(message) 84 | end 85 | KubernetesHarness::Subcommand.run 86 | end 87 | end 88 | end 89 | end 90 | context 'When I execute validate' do 91 | context 'And I ask for usage' do 92 | example 'It should exit because the CLI handles displaying usage' do 93 | expect(KubernetesHarness::Subcommand.validate({ show_usage: true })).to be true 94 | end 95 | end 96 | 97 | context 'And I run it by itself' do 98 | example 'It should validate a Harness file' do 99 | expect(KubernetesHarness::HarnessFile).to receive(:validate) 100 | KubernetesHarness::Subcommand.validate 101 | end 102 | end 103 | end 104 | end 105 | # rubocop: enable Metrics/BlockLength 106 | --------------------------------------------------------------------------------