├── OWNERS ├── .gitignore ├── requirements.yml ├── test ├── remote │ ├── test_install_home.sh │ ├── test_install_home_nogo_nogopath.sh │ ├── test_install_home_nogo.sh │ ├── test_install_local_bare.sh │ ├── common.sh │ └── e2e-simple.exp ├── gce_prepare.sh ├── test_vagrant.sh ├── Vagrantfile ├── provision.sh └── gce_run_tests.sh ├── ansible-via-docker ├── ansible-galaxy └── ansible-playbook ├── provisioning ├── devbox_vm.yml ├── resizefs.yml ├── install_vagrant.yml ├── install_virtualbox.yml ├── files │ ├── motd │ └── k8s-devenv.sh ├── install_dev_packages.yml ├── install_vagrant_libvirt.yml ├── user_devenv.yml └── playbook.yml ├── Vagrantfile ├── scripts └── gce.sh ├── README.md └── install.sh /OWNERS: -------------------------------------------------------------------------------- 1 | Ivan Shvedunov 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.retry 2 | .vagrant 3 | roles 4 | *.log 5 | vagrant*deb 6 | -------------------------------------------------------------------------------- /requirements.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - src: joshualund.golang 3 | - src: angstwad.docker_ubuntu 4 | -------------------------------------------------------------------------------- /test/remote/test_install_home.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | . test/remote/common.sh 3 | 4 | ./install.sh home https://github.com/kubernetes/kubernetes.git 5 | devbox-test-e2e-simple 6 | -------------------------------------------------------------------------------- /test/gce_prepare.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -u -e -x -o pipefail 3 | cd "$(dirname "${BASH_SOURCE[0]}")" 4 | . ../scripts/gce.sh 5 | 6 | devbox::gce::cleanup-test-stuff 7 | devbox::gce::make-provisioned-image "${test_image_name}" provision.sh 8 | 9 | # TBD: /bin/bash ++ rm 'mapfile' 10 | -------------------------------------------------------------------------------- /test/remote/test_install_home_nogo_nogopath.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | . test/remote/common.sh 3 | 4 | export DEBIAN_FRONTEND=noninteractive 5 | sudo -E apt-get update 6 | sudo -E apt-get install -y golang-go 7 | sudo bash -c "CGO_ENABLED=0 go install -a -installsuffix cgo std" 8 | mkdir -p $HOME/go 9 | 10 | ./install.sh home -nogo https://github.com/kubernetes/kubernetes.git 11 | devbox-test-e2e-simple 12 | -------------------------------------------------------------------------------- /ansible-via-docker/ansible-galaxy: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set >/tmp/ans-galaxy.cmd 3 | echo "$*" >/tmp/ans-galaxy.cmd 4 | devbox_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" 5 | docker run --rm \ 6 | -e ANSIBLE_ROLES_PATH=$ANSIBLE_ROLES_PATH \ 7 | -e SSH_AUTH_SOCK=/ssh-agent \ 8 | -v $SSH_AUTH_SOCK:/ssh-agent \ 9 | -v "$devbox_dir:$devbox_dir" \ 10 | williamyeh/ansible:alpine3 \ 11 | ansible-galaxy "$@" 12 | -------------------------------------------------------------------------------- /test/remote/test_install_home_nogo.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | . test/remote/common.sh 3 | 4 | export DEBIAN_FRONTEND=noninteractive 5 | sudo -E apt-get update 6 | sudo -E apt-get install -y golang-go 7 | sudo bash -c "CGO_ENABLED=0 go install -a -installsuffix cgo std" 8 | mkdir -p $HOME/go 9 | export GOPATH=$HOME/go 10 | export PATH=$HOME/go/bin:$PATH 11 | 12 | ./install.sh home -nogo https://github.com/kubernetes/kubernetes.git 13 | devbox-test-e2e-simple 'export GOPATH=$HOME/go; export PATH=$HOME/go/bin:$PATH' 14 | -------------------------------------------------------------------------------- /provisioning/devbox_vm.yml: -------------------------------------------------------------------------------- 1 | - name: DevBox 2 | hosts: all 3 | roles: 4 | - { role: joshualund.golang, become: true } 5 | tasks: 6 | - include: install.yml 7 | become: true 8 | 9 | - name: mark VM as vagrant devbox 10 | file: path=/vagrant_devbox state=touch 11 | become: true 12 | 13 | - name: install /etc/motd 14 | # we don't clobber /etc/motd on 'real' systems, 15 | # but for vagrant VM it's ok 16 | copy: src=files/motd dest=/etc 17 | become: true 18 | 19 | - include: install_vagrant.yml 20 | - include: user_devenv.yml 21 | -------------------------------------------------------------------------------- /test/remote/test_install_local_bare.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | . test/remote/common.sh 3 | 4 | echo "deb http://ppa.launchpad.net/ansible/ansible/ubuntu xenial main" | sudo tee /etc/apt/sources.list.d/ansible.list 5 | echo "deb-src http://ppa.launchpad.net/ansible/ansible/ubuntu xenial main" | sudo tee -a /etc/apt/sources.list.d/ansible.list 6 | sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 7BB9C367 7 | export DEBIAN_FRONTEND=noninteractive 8 | sudo -E apt-get update 9 | sudo -E apt-get install -y ansible expect 10 | 11 | ./install.sh local https://github.com/kubernetes/kubernetes.git 12 | devbox-test-e2e-simple 13 | -------------------------------------------------------------------------------- /provisioning/resizefs.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: configure volume group 3 | lvg: vg=primary pvs=/dev/vda5,/dev/vdb 4 | 5 | - name: resize the logical volume 6 | lvol: vg=primary lv=root size=+100%FREE 7 | 8 | # http://stackoverflow.com/questions/28088486/ansible-is-it-possible-to-resize-ext4-filesystem 9 | - name: Assert root partition is expanded 10 | assert: { that: item.mount != '/' or item.size_total > 10737418240 } # 10 GB 11 | with_items: '{{ ansible_mounts }}' 12 | ignore_errors: yes 13 | register: expanded 14 | 15 | # make it idempotent 16 | - name: resize the file system 17 | command: resize2fs /dev/mapper/primary-root 18 | when: expanded|failed 19 | -------------------------------------------------------------------------------- /ansible-via-docker/ansible-playbook: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set >/tmp/ans-play.cmd 3 | echo "$*" >/tmp/ans-play.cmd 4 | devbox_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" 5 | docker run --rm \ 6 | -e PYTHONUNBUFFERED="$PYTHONUNBUFFERED"\ 7 | -e ANSIBLE_ROLES_PATH="$ANSIBLE_ROLES_PATH" \ 8 | -e ANSIBLE_FORCE_COLOR="$ANSIBLE_FORCE_COLOR" \ 9 | -e ANSIBLE_HOST_KEY_CHECKING="$ANSIBLE_HOST_KEY_CHECKING" \ 10 | -e ANSIBLE_SSH_ARGS="$ANSIBLE_SSH_ARGS" \ 11 | -e SSH_AUTH_SOCK=/ssh-agent \ 12 | -v $SSH_AUTH_SOCK:/ssh-agent \ 13 | -v "$devbox_dir:$devbox_dir" \ 14 | williamyeh/ansible:alpine3 \ 15 | ansible-playbook "$@" 16 | -------------------------------------------------------------------------------- /test/test_vagrant.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -u -e -x 3 | tempdir="$(mktemp -d)" 4 | cp -av "$(dirname "${BASH_SOURCE[0]}")/.." "$tempdir"/devbox 5 | cd "$tempdir"/devbox 6 | trap 'vagrant destroy || true; rm -rf "$tempdir"' EXIT 7 | 8 | ./install.sh vagrant https://github.com/kubernetes/kubernetes.git 9 | vagrant ssh -- ' 10 | export PATH="/usr/local/go/bin:$PATH" 11 | . /etc/profile.d/k8s-devenv.sh 12 | dind-up && 13 | list_e2e DNS | grep "should provide DNS for the cluster" && 14 | e2e "existing RC" && 15 | dind-down && 16 | dind-up quick 6 && 17 | conformance 18 | dind-down && 19 | testit pkg/api/validation TestValidateEvent 20 | ' 21 | 22 | # TODO: test local-up 23 | # TODO: test vagrant-up 24 | -------------------------------------------------------------------------------- /provisioning/install_vagrant.yml: -------------------------------------------------------------------------------- 1 | - name: check whether vagrant 1.8.4+ is installed 2 | # http://stackoverflow.com/questions/4023830/how-compare-two-strings-in-dot-separated-version-format-in-bash 3 | shell: "dpkg --compare-versions $(dpkg-query -W --showformat='${Version}' vagrant 2>/dev/null || echo 0) ge 1.8.4" 4 | register: vagrant_version_check_result 5 | ignore_errors: True 6 | changed_when: False 7 | 8 | - name: install fresh vagrant package 9 | apt: deb=https://releases.hashicorp.com/vagrant/1.8.4/vagrant_1.8.4_x86_64.deb 10 | when: vagrant_version_check_result|failed 11 | become: true 12 | 13 | - include: install_vagrant_libvirt.yml 14 | when: vm_type == 'libvirt' 15 | 16 | - include: install_virtualbox.yml 17 | when: vm_type == 'virtualbox' 18 | -------------------------------------------------------------------------------- /test/remote/common.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -u -e 3 | 4 | devbox_dir="$(pwd)" 5 | 6 | # FIXME: rm/comment K8S_DIND_REPO_URL/BRANCH when optimize-docker-images is merged 7 | dind_override='export K8S_DIND_REPO_URL="https://github.com/ivan4th/kubernetes-dind-cluster"; export K8S_DIND_BRANCH=optimize-docker-images; export DIND_PREPULL_BASE=ivan4th/kubernetes-dind-base:v1' 8 | eval "$dind_override" 9 | echo "$dind_override" >>~/.profile 10 | 11 | function devbox-test-e2e-simple () { 12 | local extra_cmds="${1:-true}" 13 | echo "${extra_cmds}" >/tmp/extra_cmds 14 | # "sudo su" is needed for "relogin" because in some tests 15 | # the user was just added to 'docker' group 16 | time sudo su - "${USER}" -c "expect -f ${devbox_dir}/test/remote/e2e-simple.exp" 17 | } 18 | -------------------------------------------------------------------------------- /provisioning/install_virtualbox.yml: -------------------------------------------------------------------------------- 1 | - name: install virtualbox 2 | apt: name=virtualbox state=latest 3 | when: vm_type == 'virtualbox' 4 | become: true 5 | 6 | # FIXME: use 'service' when it's fixed (and rm changed_when: False) 7 | # https://github.com/ansible/ansible-modules-core/issues/3868 8 | - name: disable libvirt service 9 | shell: (systemctl disable libvirt-bin && systemctl stop libvirt-bin) || (systemctl disable libvirtd && systemctl stop libvirtd) 10 | changed_when: False 11 | become: true 12 | ignore_errors: true 13 | 14 | # FIXME: use 'service' when it's fixed (and rm changed_when: False) 15 | # https://github.com/ansible/ansible-modules-core/issues/3868 16 | - name: enable virtualbox service 17 | shell: systemctl enable virtualbox && systemctl start virtualbox 18 | changed_when: False 19 | become: true 20 | -------------------------------------------------------------------------------- /test/Vagrantfile: -------------------------------------------------------------------------------- 1 | # -*- mode: ruby -*- 2 | # vi: set ft=ruby : 3 | 4 | Vagrant.configure("2") do |config| 5 | if ENV['USE_VIRTUALBOX'].to_s != '' then 6 | config.vm.box = "bento/ubuntu-16.04" 7 | else 8 | config.vm.box = "nrclark/xenial64-minimal-libvirt" 9 | end 10 | 11 | config.vm.hostname = "devbox" 12 | #config.vm.network :private_network, ip: "192.168.129.100" 13 | 14 | config.vm.provider :virtualbox do |v| 15 | v.memory = 6000 16 | v.cpus = 2 17 | end 18 | 19 | config.vm.provider :libvirt do |v| 20 | # avoid domain name conflicts 21 | v.random_hostname = true 22 | 23 | v.memory = 16384 24 | v.cpus = 6 25 | v.storage :file, :size => '100G' 26 | 27 | v.nested = true 28 | v.volume_cache = 'none' 29 | #v.management_network_name = 'vagrant-libvirt-devbox-test' 30 | #v.management_network_address = '192.168.129.0/24' 31 | #v.graphics_port = 5998 32 | end 33 | config.vm.provision "shell", path: "provision.sh" 34 | end 35 | -------------------------------------------------------------------------------- /test/provision.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -u -e 3 | # TBD: use playbooks for this 4 | echo "127.0.1.1 $(hostname)" >>/etc/hosts 5 | target_user="${SUDO_USER:-vagrant}" 6 | modprobe overlay || true 7 | echo overlay >/etc/modules-load.d/overlay.conf 8 | export DEBIAN_FRONTEND=noninteractive 9 | apt-get update 10 | apt-get install -y apt-transport-https ca-certificates 11 | apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D 12 | echo 'deb https://apt.dockerproject.org/repo ubuntu-xenial main' >/etc/apt/sources.list.d/docker.list 13 | apt-get update 14 | apt-get install -y docker-engine git build-essential curl expect 15 | usermod -aG docker "$target_user" 16 | echo "DOCKER_OPTS='--storage-driver=overlay2'" > /etc/default/docker 17 | service docker restart 18 | docker pull ivan4th/kubernetes-dind-base:v1 19 | docker pull cfssl/cfssl:latest 20 | docker pull gcr.io/google_containers/etcd-amd64:3.0.4 21 | docker pull jpetazzo/nsenter:latest 22 | -------------------------------------------------------------------------------- /test/remote/e2e-simple.exp: -------------------------------------------------------------------------------- 1 | #!/usr/bin/expect -f 2 | set hostname [exec hostname -s] 3 | set prompt "$hostname.*:.*~.*\\\$ *" 4 | 5 | proc waitprompt {timeout} { 6 | global prompt 7 | expect { 8 | -timeout $timeout 9 | timeout {puts "\nTimed out waiting for prompt\n"; exit 1} 10 | -re "$prompt" 11 | } 12 | } 13 | 14 | proc cmd {timeout cmd} { 15 | send "$cmd || exit 1\n" 16 | waitprompt $timeout 17 | } 18 | 19 | spawn bash -lis 20 | 21 | waitprompt 10 22 | 23 | send "\[ -f /tmp/extra_cmds \] && . /tmp/extra_cmds\n" 24 | waitprompt 5 25 | cmd 5 "cdk" 26 | 27 | set prompt "$hostname.*:.*kubernetes.*\\\$ *" 28 | 29 | cmd 500 "dind-up 8" 30 | cmd 120 "(list_e2e DNS | grep \"should provide DNS for the cluster\")" 31 | cmd 200 "e2e \"existing RC\" || exit 1" 32 | cmd 30 "dind-down" 33 | 34 | send "exit 0\n" 35 | 36 | expect eof 37 | 38 | lassign [wait] pid spawnid os_error_flag value 39 | 40 | if {$os_error_flag == 0} { 41 | exit $value 42 | } else { 43 | exit $value 44 | } 45 | -------------------------------------------------------------------------------- /provisioning/files/motd: -------------------------------------------------------------------------------- 1 | 2 | * k8s local development environment 3 | 4 | cdk = chdir to k8s source directory 5 | dind-up [N] = bring up N-node DIND (Docker-in-Docker) cluster 6 | dind-up quick [N] = bring up N-node DIND cluster without rebuilding 7 | the containers 8 | dind-down = bring down DIND cluster 9 | vagrant-up = bring up Vagrant cluster 10 | vagrant-down = bring down Vagrant cluster 11 | local-up = bring up the local cluster 12 | use-dind = switch to using DIND cluster 13 | use-local = switch to 'local' provider (use with local-up) 14 | use-vagrant = switch to 'vagrant' provider 15 | list_e2e = list available e2e tests using Ginkgo dry run 16 | e2e [focus] = run e2e test(s) optionally filtering them 17 | by name using 'focus' regexp 18 | update-kubelet = update kubelet on the vagrant-based nodes 19 | testit [pkg] [regex] = run some of the unit tests 20 | devhelp = display this help 21 | 22 | * native k8s commands 23 | 24 | make = build k8s binaries 25 | make quick-release = build k8s release for use with kube-up 26 | make test = run unit tests 27 | 28 | -------------------------------------------------------------------------------- /provisioning/install_dev_packages.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: install dev packages 3 | apt: "name={{ item }} state=latest" 4 | with_items: 5 | - git 6 | - htop 7 | - mercurial 8 | - build-essential 9 | - nfs-kernel-server 10 | - screen 11 | - curl 12 | - nano 13 | become: true 14 | 15 | - name: install etcd 16 | unarchive: 17 | src: https://github.com/coreos/etcd/releases/download/v3.0.4/etcd-v3.0.4-linux-amd64.tar.gz 18 | dest: /opt 19 | remote_src: yes 20 | # work around https://github.com/ansible/ansible-modules-core/issues/4202 21 | copy: no 22 | become: true 23 | 24 | - name: create etcd symlink 25 | file: 26 | src: /opt/etcd-v3.0.4-linux-amd64/etcd 27 | dest: /usr/local/bin/etcd 28 | state: link 29 | become: true 30 | 31 | - name: create etcdctl symlink 32 | file: 33 | src: /opt/etcd-v3.0.4-linux-amd64/etcdctl 34 | dest: /usr/local/bin/etcdctl 35 | state: link 36 | become: true 37 | 38 | - name: install cgo & fix stdlib 39 | shell: "CGO_ENABLED=0 /usr/local/go/bin/go install -a -installsuffix cgo std" 40 | args: 41 | creates: /usr/local/go/pkg/linux_amd64_cgo 42 | become: true 43 | 44 | - name: add the user to 'docker' group 45 | user: "name={{ ansible_env.USER }} groups=docker append=yes" 46 | become: true 47 | -------------------------------------------------------------------------------- /provisioning/install_vagrant_libvirt.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: install packages required by vagrant-libvirt 3 | apt: "name={{ item }} state=latest" 4 | with_items: 5 | - build-essential 6 | - libvirt-bin 7 | - qemu-system 8 | - qemu-system-x86 9 | - qemu-kvm 10 | - libxslt-dev 11 | - libxml2-dev 12 | - libvirt-dev 13 | - zlib1g-dev 14 | - ruby-dev 15 | become: true 16 | 17 | # FIXME: use 'service' when it's fixed (and rm changed_when: False) 18 | # https://github.com/ansible/ansible-modules-core/issues/3868 19 | - name: disable virtualbox service 20 | shell: systemctl disable virtualbox && systemctl stop virtualbox 21 | changed_when: False 22 | become: true 23 | ignore_errors: true 24 | 25 | # FIXME: use 'service' when it's fixed (and rm changed_when: False) 26 | # https://github.com/ansible/ansible-modules-core/issues/3868 27 | - name: enable libvirt service 28 | shell: (systemctl enable libvirt-bin && systemctl start libvirt-bin) || (systemctl enable libvirtd && systemctl start libvirtd) 29 | changed_when: false 30 | become: true 31 | 32 | - name: check for vagrant libvirt plugin 33 | shell: "vagrant plugin list|grep -q vagrant-libvirt" 34 | register: libvirt_plugin_check_result 35 | ignore_errors: true 36 | changed_when: false 37 | 38 | - name: install libvirt plugin 39 | shell: vagrant plugin install vagrant-libvirt 40 | when: libvirt_plugin_check_result|failed 41 | -------------------------------------------------------------------------------- /provisioning/user_devenv.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: make directory structure for source tree 3 | file: 4 | path: "{{ ansible_env.HOME }}/work/kubernetes/src/k8s.io" 5 | state: directory 6 | 7 | - name: make go-tools directory 8 | file: 9 | path: "{{ ansible_env.HOME }}/go-tools" 10 | state: directory 11 | mode: 0755 12 | 13 | - name: install go tools 14 | shell: "go get -u {{ item }}" 15 | args: 16 | creates: "{{ ansible_env.HOME }}/go-tools/src/{{ item }}" 17 | environment: 18 | GOPATH: "{{ ansible_env.HOME }}/go-tools" 19 | PATH: "{{ ansible_env.HOME }}/go-tools/bin:/usr/local/go/bin:{{ ansible_env.PATH }}" 20 | with_items: 21 | - github.com/tools/godep 22 | - github.com/jteeuwen/go-bindata/go-bindata 23 | - github.com/nsf/gocode 24 | - github.com/rogpeppe/godef 25 | - github.com/golang/lint/golint 26 | 27 | - name: install k8s-devenv.sh into /etc/profile.d 28 | copy: src=files/k8s-devenv.sh dest=/etc/profile.d 29 | become: true 30 | 31 | - name: source k8s-devenv.sh from bashrc 32 | lineinfile: "dest={{ ansible_env.HOME }}/.bashrc regexp='^.*/k8s-devenv.sh' line='. /etc/profile.d/k8s-devenv.sh #added-by-k8s-devbox'" 33 | 34 | - name: check out k8s source tree 35 | git: 36 | repo: "{{ k8s_repo_url }}" 37 | dest: "{{ ansible_env.HOME }}/work/kubernetes/src/k8s.io/kubernetes" 38 | update: no 39 | accept_hostkey: true 40 | when: k8s_repo_url is defined 41 | -------------------------------------------------------------------------------- /provisioning/playbook.yml: -------------------------------------------------------------------------------- 1 | # the following deployment types are supported: 2 | # devbox_type == 'vm_host': prepare a libvirt VM host for devbox VM install 3 | # devbox_type == 'host': provision the host to become devbox 4 | # vm_type == 'libvirt': use libvirt for kube-up 5 | # vm_type == 'virtualbox': use VirtualBox for kube-up 6 | # devbox_type == 'vm': provision a devbox VM 7 | # vm_type == 'libvirt': use libvirt (kube-up will spawn nested VMs) 8 | # vm_type == 'virtualbox': use VirtualBox (kube-up will not work inside this VM) 9 | # 10 | - name: DevBox 11 | hosts: all 12 | roles: 13 | - role: joshualund.golang 14 | # FIXME: this should be provided by install.sh script 15 | go_tarball: go1.7.1.linux-amd64.tar.gz 16 | go_tarball_checksum: "sha256:43ad621c9b014cde8db17393dc108378d37bc853aa351a6c74bf6432c1bbd182" 17 | go_version_target: go version go1.7.1 linux/amd64 18 | when: devbox_type != 'vm_host' 19 | become: true 20 | - role: angstwad.docker_ubuntu 21 | docker_opts: "--storage-driver=overlay2" 22 | become: true 23 | tasks: 24 | - name: Update apt cache 25 | apt: update_cache=yes cache_valid_time=3600 26 | become: true 27 | 28 | - include: install_vagrant.yml 29 | # virtualbox doesn't support nested virtualization 30 | when: devbox_type != 'vm' or vm_type != 'virtualbox' 31 | 32 | - block: 33 | - include: resizefs.yml 34 | # libvirt VM image we sue has smallish rootfs 35 | when: vm_type == 'libvirt' 36 | become: true 37 | 38 | - name: mark VM as vagrant devbox 39 | file: path=/vagrant_devbox state=touch 40 | become: true 41 | 42 | - name: install /etc/motd 43 | # we don't clobber /etc/motd on 'real' systems, 44 | # but for vagrant VM it's ok 45 | copy: src=files/motd dest=/etc 46 | become: true 47 | when: devbox_type == 'vm' 48 | 49 | - block: 50 | - include: install_dev_packages.yml 51 | become: true 52 | 53 | - include: user_devenv.yml 54 | 55 | when: devbox_type != 'vm_host' 56 | -------------------------------------------------------------------------------- /test/gce_run_tests.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -u -e -o pipefail 3 | cd "$(dirname "${BASH_SOURCE[0]}")" 4 | . ../scripts/gce.sh 5 | 6 | debug= 7 | while getopts "d" opt; do 8 | case $opt in 9 | d) 10 | debug=y 11 | ;; 12 | \?) 13 | echo "Usage: $0 [-d] [test_script_names.sh_separated_by_spaces...]" >&2 14 | exit 1 15 | ;; 16 | esac 17 | done 18 | shift $((OPTIND-1)) 19 | 20 | devbox::gce::cleanup-test-instances 21 | if [[ ! "$debug" ]]; then 22 | trap devbox::gce::cleanup-test-instances EXIT 23 | fi 24 | 25 | # TBD: run local tests (./install.sh remote) -- local/ dir 26 | i=0 27 | if [ "$#" -eq 0 ]; then 28 | test_cases=($(cd remote && ls test_*.sh)) 29 | else 30 | test_cases=("$@") 31 | fi 32 | status=0 33 | test_pids=() 34 | for name in "${test_cases[@]}"; do 35 | if [ ! -f "remote/${name}" ]; then 36 | status=1 37 | echo "Invalid test name: ${name}" >&2 38 | continue 39 | fi 40 | ( 41 | echo "************* ${name} START" 42 | instance_name="devbox-tester-${i}" 43 | while true; do 44 | if [[ "${name}" =~ _bare\.sh$ ]]; then 45 | devbox::gce::make-bare-instance "${instance_name}" 46 | else 47 | devbox::gce::make-test-instance "${instance_name}" 48 | fi 49 | devbox::gce::copy-devbox "${instance_name}" 50 | if devbox::gce::ssh "${instance_name}" "cd k8s-devbox && test/remote/$name" >&"remote-${name}".log; then 51 | echo "************* ${name} OK" 52 | devbox::gce::delete-instance "${instance_name}" 53 | break 54 | elif [[ $? -ne 255 ]]; then 55 | status=1 56 | echo "************* ${name} FAIL" 57 | if [[ ! "$debug" ]]; then 58 | devbox::gce::delete-instance "${instance_name}" 59 | else 60 | echo "************* Kept instance for ${name}: ${instance_name}" 61 | fi 62 | break 63 | fi 64 | echo "*** $name RETRY (possible instance preemption)" 65 | devbox::gce::delete-instance "${instance_name}" 66 | sleep 20 67 | done 68 | ) & 69 | test_pids[$((i++))]=$! 70 | done 71 | 72 | if [ ${#test_pids[@]} -gt 0 ]; then 73 | for pid in ${test_pids[*]}; do 74 | wait ${pid} 75 | done 76 | fi 77 | 78 | exit ${status} 79 | -------------------------------------------------------------------------------- /Vagrantfile: -------------------------------------------------------------------------------- 1 | # -*- mode: ruby -*- 2 | # vi: set ft=ruby : 3 | 4 | Vagrant.configure("2") do |config| 5 | if ENV['USE_VIRTUALBOX'].to_s != '' then 6 | config.vm.box = "bento/ubuntu-16.04" 7 | else 8 | config.vm.box = "nrclark/xenial64-minimal-libvirt" 9 | end 10 | 11 | config.vm.hostname = "devbox" 12 | #config.vm.network :private_network, ip: "192.168.128.100" 13 | 14 | config.vm.provider :virtualbox do |v| 15 | v.memory = 6000 16 | v.cpus = 2 17 | end 18 | 19 | config.vm.provider :libvirt do |v| 20 | # avoid domain name conflicts 21 | v.random_hostname = true 22 | 23 | # TBD: make these customizable 24 | v.memory = 16384 25 | v.cpus = 6 26 | v.storage :file, :size => '100G' 27 | 28 | # Another possibility for bigger VM disk 29 | # (harder to deal with when using Ansible LVM modules) 30 | # v.machine_virtual_size = 40 31 | 32 | v.nested = true 33 | v.volume_cache = 'none' 34 | #v.management_network_name = 'vagrant-libvirt-devbox' 35 | #v.management_network_address = '192.168.128.0/24' 36 | #v.graphics_port = 5999 37 | end 38 | 39 | if ENV['USE_VIRTUALBOX'].to_s != '' then 40 | config.vm.provision "shell", 41 | inline: "DEBIAN_FRONTEND=noninteractive apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y python2.7 python2.7-dev && ln -fs /usr/bin/python2.7 /usr/bin/python" 42 | end 43 | 44 | # ansible_local provisioning doesn't work because libvirt provider 45 | # fails to provide /vagrant shared holder 46 | config.vm.provision "ansible" do |ansible| 47 | ansible.playbook = File.dirname(__FILE__) + "/provisioning/playbook.yml" 48 | ansible.extra_vars = { 49 | devbox_type: 'vm' 50 | } 51 | 52 | # use dirname here because dockerized ansible will have 53 | # different current directory 54 | if ENV['USE_VIRTUALBOX'].to_s != '' then 55 | ansible.extra_vars[:vm_type] = 'virtualbox' 56 | else 57 | ansible.extra_vars[:vm_type] = 'libvirt' 58 | end 59 | ansible.galaxy_role_file = File.dirname(__FILE__) + "/requirements.yml" 60 | 61 | if ENV['K8S_REPO_URL'].to_s != '' then 62 | ansible.extra_vars[:k8s_repo_url] = ENV['K8S_REPO_URL'] 63 | end 64 | 65 | # Disable sudo to avoid problems with ssh agent. 66 | # Sudo is used via 'become' on per-include/per-task basis in playbooks. 67 | ansible.sudo = false 68 | 69 | # uncomment for Ansible debugging 70 | # ansible.verbose = "vvv" 71 | end 72 | 73 | config.ssh.forward_agent = true 74 | end 75 | -------------------------------------------------------------------------------- /scripts/gce.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | preempt="--preemptible" 3 | devbox_dir="$(dirname "${BASH_SOURCE[0]}")/.." 4 | zone="${DEVBOX_GCE_ZONE:-us-central1-b}" 5 | project="${DEVBOX_GCE_PROJECT:-$(gcloud config list --format 'value(core.project)' 2>/dev/null)}" 6 | image_family="${DEVBOX_GCE_IMAGE:-ubuntu-1604-lts}" 7 | image_project="${DEVBOX_GCE_IMAGE_PROJECT:-ubuntu-os-cloud}" 8 | setup_machine_type="${DEVBOX_GCE_MACHINE_TYPE:-n1-standard-1}" 9 | machine_type="${DEVBOX_GCE_MACHINE_TYPE:-n1-standard-4}" 10 | projzone=(--project "${project}" --zone "${zone}") 11 | projzones=(--project "${project}" --zones "${zone}") 12 | tmp_instance="${DEVBOX_GCE_TMP_INSTANCE_NAME:-devbox-test-tmp}" 13 | test_image_name="${DEVBOX_GCE_TEST_IMAGE_NAME:-devbox-test-image}" 14 | # disk_type="${DEVBOX_GCE_DISK_TYPE:-pd-ssd}" 15 | # disk_size="${DEVBOX_GCE_DISK_SIZE:-20GB}" 16 | 17 | function devbox::gce::cleanup-test-instances () { 18 | local -a instances 19 | mapfile -t instances < <(gcloud compute instances list \ 20 | --regexp='^devbox-test.*' \ 21 | --format='value(name)' \ 22 | "${projzones[@]}") 23 | if [ ${#instances[@]} -ne 0 ]; then 24 | gcloud compute instances delete "${instances[@]}" -q "${projzone[@]}" 25 | fi 26 | } 27 | 28 | function devbox::gce::cleanup-test-stuff () { 29 | devbox::gce::cleanup-test-instances 30 | local image_name="$(gcloud compute images list \ 31 | --regexp='^devbox-test.*' \ 32 | --format='value(name)' \ 33 | --project "${project}" \ 34 | --no-standard-images)" 35 | if [[ "$image_name" ]]; then 36 | gcloud compute images delete "$image_name" -q --project "${project}" 37 | fi 38 | 39 | local disk_name="$(gcloud compute disks list \ 40 | --regexp='^devbox-test.*' \ 41 | --format='value(name)' \ 42 | "${projzones[@]}")" 43 | if [[ "$disk_name" ]]; then 44 | gcloud compute disks delete "$disk_name" -q "${projzone[@]}" 45 | fi 46 | } 47 | 48 | function devbox::gce::ssh () { 49 | local host="$1" 50 | local cmd="$2" 51 | gcloud compute ssh \ 52 | --ssh-flag="-o LogLevel=quiet" --ssh-flag="-o ConnectTimeout=30" \ 53 | "${projzone[@]}" "${host}" --command \ 54 | "$cmd" 55 | } 56 | 57 | function devbox::gce::wait-for-ssh () { 58 | local host="$1" 59 | for n in {1..5}; do 60 | if devbox::gce::ssh "${host}" true 2>/dev/null; then 61 | break 62 | fi 63 | sleep 5 64 | done 65 | } 66 | 67 | function devbox::gce::make-bare-instance () { 68 | local name="$1" 69 | local mtype="${2:-${machine_type}}" 70 | gcloud compute instances create "${name}" \ 71 | --image-project "${image_project}" \ 72 | --image-family "${image_family}" \ 73 | --machine-type "${mtype}" \ 74 | "${projzone[@]}" 75 | devbox::gce::wait-for-ssh "${name}" 76 | } 77 | 78 | function devbox::gce::make-provisioned-image () { 79 | local image_name="$1" 80 | local provision_script="$2" 81 | devbox::gce::make-bare-instance "${tmp_instance}" "${setup_machine_type}" 82 | devbox::gce::ssh "${tmp_instance}" "sudo bash -s" <"$provision_script" 83 | gcloud compute instances delete -q "${tmp_instance}" --keep-disks boot "${projzone[@]}" 84 | gcloud compute images create "${image_name}" \ 85 | --source-disk "${tmp_instance}" \ 86 | --source-disk-zone "${zone}" \ 87 | --project "${project}" 88 | gcloud compute disks delete -q "${tmp_instance}" "${projzone[@]}" 89 | } 90 | 91 | function devbox::gce::make-test-instance () { 92 | local name="$1" 93 | gcloud compute instances create "${name}" \ 94 | $preempt \ 95 | --image "${test_image_name}" \ 96 | --machine-type "${machine_type}" \ 97 | "${projzone[@]}" 98 | # --boot-disk-type "${disk_type}" \ 99 | # --boot-disk-size "${disk_size}" \ 100 | devbox::gce::wait-for-ssh "${name}" 101 | } 102 | 103 | function devbox::gce::delete-instance () { 104 | local name="$1" 105 | gcloud compute instances delete "${name}" -q "${projzone[@]}" 106 | } 107 | 108 | function devbox::gce::copy-devbox () { 109 | local host="$1" 110 | tar -C "${devbox_dir}" -c --exclude .vagrant --exclude .git . | 111 | devbox::gce::ssh ${host} 'rm -rf k8s-devbox && mkdir k8s-devbox && tar -C k8s-devbox -x' 112 | } 113 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Kubernetes Development Environment 2 | 3 | k8s-devbox provides a reproducible development environment 4 | for working on Kubernetes project. 5 | 6 | It's currently tested on Ubuntu Xenial (x86_64) and Mac OS X 10.11. 7 | 8 | For easy creation of local k8s clusters k8s-devbox uses 9 | [kubernetes-dind-cluster](https://github.com/sttts/kubernetes-dind-cluster) tool 10 | written by Dr. Stefan Schimanski. 11 | 12 | Demo - Mac OS X, installation using `home` method, DIND cluster: 13 | [![asciicast](https://asciinema.org/a/85690.png)](https://asciinema.org/a/85690) 14 | 15 | Demo - Linux, installation using `vagrant` method, local and DIND clusters, 16 | conformance tests: 17 | [![asciicast](https://asciinema.org/a/cjemrekkurdkhe19j539wii8l.png)](https://asciinema.org/a/cjemrekkurdkhe19j539wii8l) 18 | 19 | ## Installation 20 | 21 | First, clone k8s-devbox repository: 22 | ``` 23 | git clone https://github.com/mirantis/k8s-devbox.git 24 | cd k8s-devbox 25 | ``` 26 | 27 | To install the devbox in user's home directory without creating 28 | any VMs or changing configuration of the host machine, use 29 | ``` 30 | ./install.sh home git@github.com:YOUR_GITHUB_USERNAME/kubernetes 31 | ``` 32 | You'll need to have Docker installed on your machine for this to 33 | work. `home` mode is supported on Linux and Mac OS X, but 34 | unfortunately as of now it only supports bash (zsh support is 35 | planned). 36 | 37 | For other installation modes you need to have 38 | [Ansible](http://docs.ansible.com/ansible/intro_installation.html#installation) 39 | >= 2.1.0 installed on your machine. For Mac OS X, you'll also need to 40 | install [Vagrant](https://www.vagrantup.com/) and 41 | [VirtualBox](https://en.wikipedia.org/wiki/VirtualBox). 42 | 43 | In case of VM-based installation on Ubuntu, use the following command 44 | to prepare the host: 45 | ``` 46 | ./install.sh host 47 | ``` 48 | or 49 | ``` 50 | USE_VIRTUALBOX=1 ./install.sh host 51 | ``` 52 | to force the use of VirtualBox event if vagrant-libvirt plugin is installed. 53 | You may need to relogin after that. 54 | 55 | To install using Vagrant: 56 | ``` 57 | ./install.sh vagrant git@github.com:YOUR_GITHUB_USERNAME/kubernetes 58 | ``` 59 | (specify your kubernetes fork) 60 | 61 | If you want to force use of VirtualBox for the wrapper VM on Linux, use 62 | ``` 63 | USE_VIRTUALBOX=1 ./install.sh vagrant git@github.com:YOUR_GITHUB_USERNAME/kubernetes 64 | ``` 65 | But note that VirtualBox doesn't support nested virtualization and you 66 | will not be able to use `kube-up` inside your VM. 67 | 68 | After installation, you may log into the box via 69 | ``` 70 | vagrant ssh 71 | ``` 72 | 73 | You can also provision a remote machine to become a k8s dev environment, 74 | but this parts needs some testing: 75 | ``` 76 | ./install.sh remote HOSTNAME git@github.com:YOUR_GITHUB_USERNAME/kubernetes 77 | ``` 78 | 79 | The same goes for the local machine: 80 | ``` 81 | ./install.sh local git@github.com:YOUR_GITHUB_USERNAME/kubernetes 82 | ``` 83 | 84 | You can prepend `USE_VIRTUALBOX=1` to `./install.sh remote ...` or 85 | `./install.sh local ...` to use VirtualBox instead of libvirt for 86 | `vagrant-up`. 87 | 88 | **Do not** invoke any of these commands as root, because they need to 89 | use your user account. 90 | 91 | ## Usage 92 | 93 | The following shortcuts are provided in the shell: 94 | 95 | ``` 96 | cdk 97 | ``` 98 | Chdir to Kubernetes source directory. 99 | 100 | ``` 101 | dind-up [quick] [N] 102 | ``` 103 | Bring up `N`-node DIND (Docker-in-Docker) cluster. `quick` mode can be 104 | used to start DIND cluster without rebuilding Docker images it uses. 105 | `N` (number of nodes) defaults to 2. 106 | 107 | ``` 108 | dind-down 109 | ``` 110 | Stop DIND cluster. 111 | 112 | ``` 113 | vagrant-up 114 | ``` 115 | Bring up a 2-node vagrant based cluster and switches to `vagrant` provider. 116 | 117 | ``` 118 | vagrant-down 119 | ``` 120 | Bring down the vagrant cluster. 121 | 122 | ``` 123 | list_e2e 124 | ``` 125 | List available e2e tests 126 | 127 | ``` 128 | e2e [focus] 129 | ``` 130 | Run e2e test(s). You can specify a filter as regular expression. If 131 | no filter (focus) is specified, the same set of e2e tests as in 132 | upstream CI is used. Note that you need to do `make quick-release` and 133 | start the cluster via either `local-up` or `kube-up` before you can 134 | run e2e tests. Be advised that running e2e tests against `local-up` 135 | cluster may be unreliable. 136 | 137 | ``` 138 | local-up 139 | ``` 140 | Bring up a local cluster using `hack/local-up-cluster.sh` 141 | with DNS support. You may want to use this command inside 142 | `screen`. 143 | 144 | ``` 145 | use-dind 146 | ``` 147 | Switch to using DIND cluster that's currently active. 148 | 149 | ``` 150 | use-local 151 | ``` 152 | Switch to 'local' provider (use with local-up). You may need to 153 | do this in every terminal session you're using to work with 154 | the local cluster. 155 | 156 | ``` 157 | use-vagrant 158 | ``` 159 | Switch to 'vagrant' provider. 160 | 161 | ``` 162 | update-kubelet 163 | ``` 164 | Update kubelet on vagrant-based nodes. 165 | 166 | ``` 167 | testit [pkg] [regex] 168 | ``` 169 | Run unit test(s). `pkg` (package) and `regex` can be used to specify 170 | which tests to run, e.g. 171 | ``` 172 | testit pkg/api/validation TestValidateEvent 173 | ``` 174 | 175 | ``` 176 | devhelp 177 | ``` 178 | Display help on devbox commands. 179 | 180 | ## "Native" k8s commands 181 | 182 | The following commands may be useful for Kubernetes development: 183 | 184 | ``` 185 | make 186 | ``` 187 | Build Kubernetes binaries. 188 | 189 | ``` 190 | make quick-release 191 | ``` 192 | Build k8s release for use with kube-up. 193 | 194 | ``` 195 | make test 196 | ``` 197 | Run unit tests. 198 | 199 | ## Additional notes 200 | 201 | There must be no symlinks in the path to Kubernetes source directory 202 | as this will cause e2e test scripts to fail. 203 | 204 | If you started `kube-up` without doing `fix-influxdb` first and 205 | e2e tests refuse to run, you can fix your vagrant cluster using following 206 | commands: 207 | ``` 208 | cdk 209 | vagrant ssh master -- sudo rm -rf /etc/kubernetes/addons/cluster-monitoring 210 | kubectl delete --now --namespace=kube-system pod monitoring-influxdb-grafana-v3-0 211 | ``` 212 | -------------------------------------------------------------------------------- /install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -u -e 3 | 4 | # workaround for https://github.com/ansible/ansible-modules-core/issues/3706 5 | # (fixed in Ansible 2.1.1). 6 | # joshualund.golang role fails without this 7 | export LANG=C 8 | export LC_ALL=C 9 | unset LANGUAGE 10 | 11 | script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 12 | nogo= 13 | k8s_repo_url= 14 | ansible_via_docker= 15 | target_hostname= 16 | provider_args= 17 | vm_type=libvirt 18 | USE_VIRTUALBOX="${USE_VIRTUALBOX:-}" 19 | ansible_ask_become_pass= 20 | if ! sudo -n true 2>/dev/null; then 21 | ansible_ask_become_pass="--ask-become-pass" 22 | fi 23 | export ANSIBLE_ROLES_PATH=$script_dir/provisioning/roles 24 | 25 | function usage { 26 | echo "Usage:" 1>&2 27 | echo " ./install.sh local [K8S_REPO_URL] - provision the local machine" 28 | echo " ./install.sh remote HOST [K8S_REPO_URL] - provision the remote host" 29 | echo " ./install.sh home [-nogo] [K8S_REPO_URL] - install in current user's home directory" 30 | echo " (WiP; currently bash-only)" 31 | echo " -nogo prevents the script from installing Go" 32 | echo " ./install.sh vagrant [K8S_REPO_URL] - install inside a Vagrant VM" 33 | echo " ./install.sh host - prepare host machine for devbox VM" 34 | exit 1 35 | } 36 | 37 | function install_roles { 38 | mkdir -p $ANSIBLE_ROLES_PATH 39 | ansible-galaxy install -r requirements.yml 40 | } 41 | 42 | function update_profile { 43 | if [ ! -f "$1" -a ! -L "$1" ]; then 44 | return 1 45 | fi 46 | if ! grep -q '#added-by-k8s-devbox' "$1"; then 47 | echo "Adding k8s-devenv.sh to $1.." 48 | echo >>"$1" 49 | echo ". '$devbox_dir'/k8s-devenv.sh #added-by-k8s-devbox" >> "$1" 50 | fi 51 | } 52 | 53 | function install_go { 54 | rm -rf "$devbox_dir/go" 55 | cd "$devbox_dir" 56 | if [ "$(uname)" == "Darwin" ]; then 57 | go_tarball=go1.7.1.darwin-amd64.tar.gz 58 | go_tarball_sha256="9fd80f19cc0097f35eaa3a52ee28795c5371bb6fac69d2acf70c22c02791f912" 59 | else 60 | go_tarball=go1.7.1.linux-amd64.tar.gz 61 | go_tarball_sha256="43ad621c9b014cde8db17393dc108378d37bc853aa351a6c74bf6432c1bbd182" 62 | fi 63 | rm -f "$go_tarball" 64 | wget https://storage.googleapis.com/golang/"$go_tarball" 65 | if ! echo "$go_tarball_sha256 $go_tarball" | sha256sum -c -; then 66 | echo "Go tarball checksum verification failed" 1>&2 67 | exit 1 68 | fi 69 | tar -xzf "$go_tarball" 70 | rm -f "$go_tarball" 71 | export GOROOT="$devbox_dir/go" 72 | export PATH="$devbox_dir/go/bin:$PATH" 73 | CGO_ENABLED=0 go install -a -installsuffix cgo std 74 | } 75 | 76 | function install_go_tools { 77 | if [ -z "$nogo" -o -z "${GOPATH:-}" ]; then 78 | mkdir -p "$devbox_dir/go-tools" 79 | export GOPATH="$devbox_dir/go-tools" 80 | fi 81 | go get -u github.com/tools/godep 82 | go get -u github.com/jteeuwen/go-bindata/go-bindata 83 | } 84 | 85 | function install_docker_compose { 86 | curl -L https://github.com/docker/compose/releases/download/1.8.0/docker-compose-`uname -s`-`uname -m` > "$devbox_dir/bin/docker-compose" 87 | chmod +x "$devbox_dir/bin/docker-compose" 88 | } 89 | 90 | function install_to_home_dir { 91 | devbox_dir="$HOME"/.k8s-devbox 92 | mkdir -p "$devbox_dir" "$devbox_dir/bin" 93 | cp "$script_dir"/provisioning/files/k8s-devenv.sh "$devbox_dir" 94 | cp "$script_dir"/provisioning/files/motd "$devbox_dir/help.txt" 95 | # TBD: verify prereqs 96 | if [ -z "$nogo" ]; then 97 | install_go 98 | fi 99 | install_go_tools 100 | if [ "$(uname)" != "Darwin" ]; then 101 | # docker-compose is bundled with Docker on Mac OS X 102 | install_docker_compose 103 | fi 104 | # based on from https://github.com/moovweb/gvm/blob/604e702e2a155b33c2f217f1f4931188344d4926/binscripts/gvm-installer#L96 105 | if [ -n "${ZSH_NAME:-}" ]; then 106 | update_profile "$HOME/.zshrc" 107 | elif [ "$(uname)" = "Linux" ]; then 108 | update_profile "$HOME/.bashrc" || update_profile "$HOME/.bash_profile" 109 | elif [ "$(uname)" = "Darwin" ]; then 110 | update_profile "$HOME/.profile" || update_profile "$HOME/.bash_profile" 111 | fi 112 | if [ -n "$k8s_repo_url" -a ! -d $HOME/work/kubernetes/src/k8s.io/kubernetes ]; then 113 | mkdir -p "$HOME/work/kubernetes/src/k8s.io" 114 | git clone "$k8s_repo_url" "$HOME/work/kubernetes/src/k8s.io/kubernetes" 115 | fi 116 | echo 1>&2 117 | echo "Please restart your shell to start using k8s-devbox or use . ~/.k8s-devbox/k8s-devenv.sh" 1>&2 118 | } 119 | 120 | function install_using_vagrant { 121 | if ! hash vagrant 2>/dev/null; then 122 | echo "You need to install Vagrant" 1>&2 123 | exit 1 124 | fi 125 | 126 | if [ -z "$USE_VIRTUALBOX" ] && vagrant plugin list | grep -q '^vagrant-libvirt'; then 127 | export VAGRANT_DEFAULT_PROVIDER=libvirt 128 | provider_args="--provider=libvirt" 129 | else 130 | export USE_VIRTUALBOX=1 131 | fi 132 | 133 | if [ -n "$USE_VIRTUALBOX" ]; then 134 | vm_type=virtualbox 135 | fi 136 | 137 | # https://kushaldas.in/posts/storage-volume-error-in-libvirt-with-vagrant.html 138 | # FIXME: happens too often for me for some reason 139 | virsh pool-refresh tmp >& /dev/null || true 140 | K8S_REPO_URL=$k8s_repo_url vagrant up $provider_args 141 | } 142 | 143 | function provision_vm_host { 144 | ansible-playbook -i localhost, -c local ${ansible_ask_become_pass} \ 145 | -e "devbox_type=vm_host vm_type=$vm_type" provisioning/playbook.yml 146 | } 147 | 148 | function install_via_ansible { 149 | conn_opts="$*" 150 | install_roles 151 | extra_vars="devbox_type=host vm_type=$vm_type" 152 | if [ -n "$k8s_repo_url" ]; then 153 | extra_vars="$extra_vars k8s_repo_url=$k8s_repo_url" 154 | fi 155 | ansible-playbook $conn_opts ${ansible_ask_become_pass} \ 156 | -e "$extra_vars" provisioning/playbook.yml 157 | } 158 | 159 | function install_locally { 160 | install_via_ansible -i localhost, -c local 161 | } 162 | 163 | function install_remotely { 164 | install_via_ansible -i "$target_hostname", --ssh-extra-args="-oForwardAgent=yes" 165 | } 166 | 167 | if [ $# -gt 0 ]; then 168 | if [ "$1" = "-d" ]; then 169 | ansible_via_docker=y 170 | shift 171 | fi 172 | fi 173 | 174 | if [ $# -eq 0 ]; then 175 | usage 176 | fi 177 | 178 | cmd="$1" 179 | shift 180 | 181 | if [ "$cmd" = "remote" ]; then 182 | if [ $# -eq 0 ]; then 183 | echo "must specify target hostname" 1>& 2 184 | fi 185 | target_hostname="$1" 186 | shift 187 | fi 188 | 189 | if [ "${1:-}" = "-nogo" ]; then 190 | nogo=1 191 | shift 192 | fi 193 | 194 | k8s_repo_url= 195 | if [ $# -gt 0 ]; then 196 | k8s_repo_url="$1" 197 | fi 198 | 199 | if [ "$cmd" != "home" ]; then 200 | if ! hash ansible-playbook 2>/dev/null; then 201 | ansible_via_docker=y 202 | fi 203 | 204 | if [ -n "$ansible_via_docker" ]; then 205 | echo "WiP: ansible invocation via docker doesn't work currently due to ssh & file perm problems" 1>& 2 206 | exit 1 207 | # export PATH="$script_dir/ansible-via-docker:$PATH" 208 | fi 209 | fi 210 | 211 | case "$cmd" in 212 | vagrant) 213 | install_using_vagrant 214 | ;; 215 | host) 216 | provision_vm_host 217 | ;; 218 | local) 219 | install_locally 220 | ;; 221 | remote) 222 | install_remotely 223 | ;; 224 | home) 225 | install_to_home_dir 226 | ;; 227 | *) 228 | usage 229 | ;; 230 | esac 231 | -------------------------------------------------------------------------------- /provisioning/files/k8s-devenv.sh: -------------------------------------------------------------------------------- 1 | K8S_DEVBOX_FULL_ENV="${K8S_DEVBOX_FULL_ENV:-}" 2 | if [ -f /vagrant_devbox ]; then 3 | K8S_DEVBOX_FULL_ENV=true 4 | fi 5 | 6 | if [ -n "$K8S_DEVBOX_FULL_ENV" ]; then 7 | # In case of vagrant VM, provide a useful default prompt and go to k8s directory 8 | # The prompts gets overridden in default ~/.bashrc, 9 | # so reset it even if this script was already sourced 10 | export GIT_PS1_SHOWDIRTYSTATE=1 11 | export GIT_PS1_SHOWUNTRACKEDFILES=1 12 | export PS1='\[\033[1;95m\]\u@\h\[\e[0m\]:\[\e[1;32m\]\w\[\033[0;33m\]$(__git_ps1 " (%s) ")\[\e[0m\]\$ ' 13 | fi 14 | 15 | K8S_DIND_BRANCH="master" 16 | K8S_DIND_REPO_URL="${K8S_DIND_REPO_URL:-https://github.com/sttts/kubernetes-dind-cluster.git}" 17 | export KPATH=$HOME/work/kubernetes 18 | if [ -n "${GOPATH:-}" ]; then 19 | # If there was some GOPATH already, make sure bin directory from the first 20 | # entry is added to path. That's because `./install.sh home` may have 21 | # installed go tools there 22 | export PATH="${GOPATH//://bin:}/bin:$PATH" 23 | fi 24 | export GOPATH=$KPATH 25 | export KUBERNETES_SRC_DIR=$KPATH/src/k8s.io/kubernetes 26 | export PATH="$KPATH/bin:$KUBERNETES_SRC_DIR/_output/bin:$KUBERNETES_SRC_DIR/cluster:$PATH" 27 | if [ -d "$HOME/.k8s-devbox/go" ]; then 28 | # installed via ./install.sh home 29 | export GOROOT="$HOME/.k8s-devbox/go" 30 | export PATH="$HOME/.k8s-devbox/bin:$HOME/.k8s-devbox/go-tools/bin:$GOROOT/bin:$PATH" 31 | elif [ -d "$HOME/.k8s-devbox/go-tools" ]; then 32 | # installed via ./install.sh home -nogo 33 | export PATH="$HOME/.k8s-devbox/go-tools/bin:$GOROOT/bin:$PATH" 34 | else 35 | export PATH="$HOME/go-tools/bin:$PATH" 36 | fi 37 | 38 | if [ -d "$HOME/.k8s-devbox/bin" ]; then 39 | export PATH="$HOME/.k8s-devbox/bin:$PATH" 40 | fi 41 | 42 | if hash systemctl 2>/dev/null; then 43 | if systemctl -q is-active virtualbox; then 44 | export VAGRANT_DEFAULT_PROVIDER=virtualbox 45 | elif systemctl -q is-active libvirt-bin; then 46 | export VAGRANT_DEFAULT_PROVIDER=libvirt 47 | # The following was causing ansible's scp to fail 48 | # with 'Received message too long' 49 | #else 50 | # echo "WARNING: no active virtualbox or libvirt-bin service detected" 51 | fi 52 | fi 53 | 54 | trace_highlight_start="$(echo -ne "\x1b[100m\x1b[97m"; echo -n "+")" 55 | trace_highlight_end="$(echo -ne "\x1b[49m\x1b[39m")" 56 | 57 | function quote_arg { 58 | if [[ "$arg" =~ " " ]]; then 59 | echo -n " '$arg'" 1>&2 60 | else 61 | echo -n " $arg" 1>&2 62 | fi 63 | } 64 | 65 | function quote_var { 66 | if [[ "$1" =~ ([^=]+)=(.*\ .*) ]]; then 67 | echo -n " ${BASH_REMATCH[1]}='${BASH_REMATCH[2]}'" 68 | else 69 | echo -n " $1" 70 | fi 71 | } 72 | 73 | function fake_trace { 74 | echo -n "$trace_highlight_start" 75 | for arg in "$@"; do 76 | quote_arg "$arg" 77 | done 78 | echo "$trace_highlight_end" 79 | } 80 | 81 | function trace { 82 | # have to use this tricky alternative to set -x becuause 83 | # we want to highlight the trace lines 84 | echo -n "$trace_highlight_start" 85 | 86 | if [ "$1" = "export" ]; then 87 | # don't execute 'export' in a subshell 88 | echo -n " export" 89 | shift 90 | for arg in "$@"; do 91 | quote_var "$arg" 92 | done 93 | echo "$trace_highlight_end" 94 | export "$@" 95 | return 0 96 | fi 97 | 98 | ( 99 | # Use subshell because var assignments are translated 100 | # to 'export' because "$@" below can't parse them, 101 | # but we don't want to 102 | while [[ "$1" =~ "=" ]]; do 103 | quote_var "$1" 104 | export "$1" 105 | shift 106 | done 107 | for arg in "$@"; do 108 | quote_arg "$arg" 109 | done 110 | echo "$trace_highlight_end" 111 | "$@" 112 | ) 113 | } 114 | 115 | function escape_test_name() { 116 | sed 's/[]\$*.^|()[]/\\&/g; s/\s\+/\\s+/g' <<< "$1" | tr -d '\n' 117 | } 118 | 119 | function cdk { 120 | cd "$KUBERNETES_SRC_DIR" 121 | } 122 | 123 | if [ -n "$K8S_DEVBOX_FULL_ENV" -a -d "$KUBERNETES_SRC_DIR" ]; then 124 | cdk 125 | fi 126 | 127 | alias kubectl=$KUBERNETES_SRC_DIR/cluster/kubectl.sh 128 | 129 | function use-vagrant { 130 | trace export KUBERNETES_PROVIDER=vagrant 131 | trace kubectl config use-context vagrant 132 | } 133 | 134 | function dind-up { 135 | cdk 136 | if [ ! -d dind ]; then 137 | trace git clone -b "${K8S_DIND_BRANCH}" "${K8S_DIND_REPO_URL}" dind 138 | fi 139 | if [ "$(uname)" = "Darwin" ]; then 140 | if [ ! -f _output/dockerized/bin/linux/amd64/hyperkube ]; then 141 | trace build/run.sh make WHAT=cmd/hyperkube 142 | fi 143 | elif [ ! -f _output/bin/hyperkube ]; then 144 | trace make WHAT=cmd/hyperkube 145 | fi 146 | if [ ! -f _output/bin/kubectl ]; then 147 | make WHAT=cmd/kubectl 148 | fi 149 | local -a args 150 | while true; do 151 | if [ "${1:-}" = "quick" ]; then 152 | args=("${args[@]}" DOCKER_IN_DOCKER_SKIP_BUILD=true) 153 | shift 154 | elif [[ "${1:-}" =~ [0-9]+ ]]; then 155 | args=("${args[@]}" NUM_NODES="$1") 156 | shift 157 | else 158 | break 159 | fi 160 | done 161 | trace "${args[@]}" dind/dind-up-cluster.sh 162 | trace export KUBERNETES_PROVIDER=dind 163 | } 164 | 165 | function dind-down { 166 | cdk 167 | if [ ! -d dind ]; then 168 | echo "DIND cluster isn't installed, try dind-up first" 1>& 2 169 | return 1 170 | fi 171 | trace dind/dind-down-cluster.sh 172 | } 173 | 174 | function use-dind { 175 | if [ ! -d dind ]; then 176 | echo "DIND cluster isn't installed, try dind-up first" 1>& 2 177 | return 1 178 | fi 179 | trace export KUBERNETES_PROVIDER=dind 180 | trace kubectl config use-context dind 181 | } 182 | 183 | function vagrant-up { 184 | trace make quick-release 185 | trace KUBERNETES_VAGRANT_USE_NFS=true \ 186 | KUBERNETES_NODE_MEMORY=1024 \ 187 | NUM_NODES=2 \ 188 | KUBERNETES_PROVIDER=vagrant \ 189 | cluster/kube-up.sh 190 | use-vagrant 191 | } 192 | 193 | function vagrant-down { 194 | trace NUM_NODES=2 \ 195 | KUBERNETES_PROVIDER=vagrant \ 196 | cluster/kube-down.sh 197 | } 198 | 199 | function get-ext-ip { 200 | ip route get 1 | awk '{print $NF;exit}' 201 | } 202 | 203 | function list_e2e { 204 | ( 205 | cdk 206 | if [ ! -f _output/bin/e2e.test ]; then 207 | trace make WHAT=test/e2e/e2e.test 208 | fi 209 | if [ ! -f _output/bin/ginkgo ]; then 210 | make WHAT=vendor/github.com/onsi/ginkgo/ginkgo 211 | fi 212 | extra_opts= 213 | if [ $# -gt 0 ]; then 214 | focus="$(escape_test_name "$1")" 215 | extra_opts="--ginkgo.focus=${focus}" 216 | fi 217 | fake_trace ginkgo _output/bin/e2e.test -- --prefix=e2e --network=e2e --ginkgo.dryRun --ginkgo.noColor --ginkgo.noisyPendings=false $extra_opts '2>&1' '|' 'some_ugly_filter...' 218 | ginkgo _output/bin/e2e.test -- --prefix=e2e --network=e2e --ginkgo.dryRun --ginkgo.noColor --ginkgo.noisyPendings=false $extra_opts 2>&1 | 219 | egrep -v '^[•SP ]*$'|awk '/^Will run [0-9]* of [0-9]*/{flag=1;next}/^Ran [0-9]* of [0-9]*/{flag=0}flag' 220 | ) 221 | } 222 | 223 | function e2e_setup { 224 | cdk 225 | if [ ! -f _output/bin/e2e.test ]; then 226 | trace make WHAT=test/e2e/e2e.test 227 | fi 228 | if [ ! -f _output/bin/ginkgo ]; then 229 | make WHAT=vendor/github.com/onsi/ginkgo/ginkgo 230 | fi 231 | if [ ! -f _output/bin/kubectl ]; then 232 | make WHAT=cmd/kubectl 233 | fi 234 | 235 | extra_opts="" 236 | extra_test_args="" 237 | # work around test_args problems with spaces 238 | if [ "$KUBERNETES_PROVIDER" = "local" ]; then 239 | ext_ip="$(get-ext-ip)" 240 | # thanks to @asalkeld 241 | trace export KUBE_MASTER_IP="$ext_ip" 242 | trace export KUBE_MASTER="$ext_ip" 243 | extra_opts="--check_node_count=false" 244 | extra_test_args=" --host=http://$KUBE_MASTER_IP:8080" 245 | elif [ "$KUBERNETES_PROVIDER" = "dind" ]; then 246 | trace export KUBE_MASTER_IP="localhost" 247 | trace export KUBE_MASTER="localhost" 248 | extra_test_args=" --host=https://$KUBE_MASTER_IP:6443" 249 | fi 250 | } 251 | 252 | function e2e { 253 | ( 254 | e2e_setup 255 | status=0 256 | if [ $# -gt 0 ]; then 257 | focus="$(escape_test_name "$1")" 258 | trace go run hack/e2e.go -v -check_version_skew=false --test --test_args="--ginkgo.focus=${focus}${extra_test_args}" $extra_opts 259 | else 260 | # run 'upstream' set of tests 261 | trace go run ./hack/e2e.go -v --test -check_version_skew=false \ 262 | --test_args="--ginkgo.skip=\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]$extra_test_args" $extra_opts 263 | fi 264 | ) 265 | } 266 | 267 | function conformance { 268 | ( 269 | num_nodes="$(kubectl get nodes -o name|wc -l)" 270 | e2e_setup 271 | trace KUBERNETES_CONFORMANCE_TEST=y \ 272 | GINKGO_PARALLEL_NODES=$num_nodes \ 273 | GINKGO_PARALLEL=y \ 274 | go run hack/e2e.go --v --test -check_version_skew=false \ 275 | --test_args="--ginkgo.focus=\[Conformance\] --ginkgo.skip=\[Serial\]${extra_test_args}" $extra_opts 276 | # [Serial] tests fail on DIND cluster as of now 277 | trace KUBERNETES_CONFORMANCE_TEST=y \ 278 | go run hack/e2e.go --v --test -check_version_skew=false \ 279 | --test_args="--ginkgo.focus=\[Serial\].*\[Conformance\]${extra_test_args}" $extra_opts 280 | ) 281 | } 282 | 283 | function local-up { 284 | ext_ip="$(get-ext-ip)" 285 | trace KUBE_ENABLE_CLUSTER_DNS=true \ 286 | KUBELET_HOST="$ext_ip" \ 287 | HOSTNAME_OVERRIDE="$ext_ip" \ 288 | API_HOST="$ext_ip" \ 289 | ALLOW_SECURITY_CONTEXT=true \ 290 | hack/local-up-cluster.sh 291 | } 292 | 293 | function use-local { 294 | ext_ip="$(get-ext-ip)" 295 | trace export KUBERNETES_PROVIDER=local 296 | trace kubectl config set-cluster local --server="http://$ext_ip:8080" --insecure-skip-tls-verify=true 297 | trace kubectl config set-context local --cluster=local 298 | trace kubectl config use-context local 299 | } 300 | 301 | function update-kubelet { 302 | cdk 303 | trace make 304 | for node in node-1 node-2; do 305 | trace NUM_NODES=2 vagrant ssh $node -- sudo systemctl stop kubelet.service 306 | trace NUM_NODES=2 vagrant ssh $node -- 'sudo tee /usr/local/bin/kubelet>/dev/null' <_output/bin/kubelet 307 | trace NUM_NODES=2 vagrant ssh $node -- sudo systemctl start kubelet.service 308 | done 309 | } 310 | 311 | function testit { 312 | cdk 313 | if [ $# -eq 0 ]; then 314 | trace make test 315 | elif [ $# -eq 1 ]; then 316 | trace make test WHAT="$1" KUBE_GOFLAGS="-v" 317 | else 318 | trace make test WHAT="$1" KUBE_GOFLAGS="-v" KUBE_TEST_ARGS="-run $2" 319 | fi 320 | } 321 | 322 | function devhelp { 323 | if [ -f /vagrant_devbox ]; then 324 | cat /etc/motd 325 | elif [ -f ~/.k8s-devbox/help.txt ]; then 326 | cat ~/.k8s-devbox/help.txt 327 | else 328 | echo "Help file not found" 1>&2 329 | fi 330 | } 331 | 332 | if [ -f ~/.kube/config ] && context_str="$(grep -o 'current-context:.*' ~/.kube/config)" && [[ "$context_str" =~ :\ *([^ ]+) ]]; then 333 | context="${BASH_REMATCH[1]}" 334 | case "$context" in 335 | dind|local|vagrant) 336 | if [ "${KUBERNETES_PROVIDER:-}" != "$context" ]; then 337 | if [ -n "$K8S_DEVBOX_FULL_ENV" ]; then 338 | trace export KUBERNETES_PROVIDER="$context" 339 | else 340 | export KUBERNETES_PROVIDER="$context" 341 | fi 342 | fi 343 | esac 344 | fi 345 | --------------------------------------------------------------------------------