├── roles ├── node │ ├── files │ │ └── join.sh │ └── tasks │ │ └── main.yml ├── base │ ├── files │ │ ├── copy_hosts │ │ └── docker_profile.sh │ ├── defaults │ │ └── main.yml │ ├── templates │ │ ├── wpa_supplicant.conf │ │ └── hosts │ └── tasks │ │ ├── wifi.yml │ │ ├── main.yml │ │ ├── user.yml │ │ ├── apt.yml │ │ ├── swap.yml │ │ └── system.yml ├── dashboard │ └── tasks │ │ └── main.yml ├── update │ └── tasks │ │ └── main.yml └── master │ └── tasks │ └── main.yml ├── .gitignore ├── config.example.yml ├── update.yml ├── nodes.yml ├── master.yml ├── setup.yml ├── hosts.example ├── scripts └── nodes │ ├── download.sh │ └── flash.sh ├── LICENSE └── README.md /roles/node/files/join.sh: -------------------------------------------------------------------------------- 1 | kubeadm join --token $1 $2 -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.retry 2 | kubeconfig 3 | config.yml 4 | hosts 5 | **/downloads/** -------------------------------------------------------------------------------- /roles/base/files/copy_hosts: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | cp /etc/dhcp/hosts /etc/hosts 3 | -------------------------------------------------------------------------------- /roles/base/defaults/main.yml: -------------------------------------------------------------------------------- 1 | ssh: 2 | pub_key_path: "~/.ssh/id_rsa.pub" 3 | timezone: "Etc/UTC" 4 | -------------------------------------------------------------------------------- /config.example.yml: -------------------------------------------------------------------------------- 1 | # You timezone 2 | timezone: "Europe/Madrid" 3 | arch: arm 4 | token: 506f2b.1a51ab3d42ed9d10 5 | master: master.cluster.local 6 | cidr: 10.244.0.0/16 7 | reset: false -------------------------------------------------------------------------------- /update.yml: -------------------------------------------------------------------------------- 1 | - name: Update HypriotOS and Kubernetes components 2 | hosts: pis 3 | gather_facts: yes 4 | remote_user: pi 5 | become: true 6 | become_method: sudo 7 | 8 | roles: 9 | - update -------------------------------------------------------------------------------- /nodes.yml: -------------------------------------------------------------------------------- 1 | - name: Kubernetes nodes configuration 2 | hosts: nodes 3 | gather_facts: yes 4 | remote_user: pi 5 | become: true 6 | vars_files: 7 | - config.yml 8 | 9 | roles: 10 | - node -------------------------------------------------------------------------------- /roles/base/templates/wpa_supplicant.conf: -------------------------------------------------------------------------------- 1 | country=GB 2 | ctrl_interface=DIR=/var/run/wpa_supplicant GROUP=netdev 3 | update_config=1 4 | network={ 5 | ssid="{{ wifi.ssid }}" 6 | psk="{{ wifi.password }}" 7 | } 8 | -------------------------------------------------------------------------------- /roles/dashboard/tasks/main.yml: -------------------------------------------------------------------------------- 1 | - name: Install Kubernetes Dashboard 2 | shell: curl -sSL 'https://rawgit.com/kubernetes/dashboard/master/src/deploy/kubernetes-dashboard.yaml' | sed 's/amd64/{{arch}}/g' | kubectl create -f - 3 | -------------------------------------------------------------------------------- /master.yml: -------------------------------------------------------------------------------- 1 | - name: Kubernetes master configuration 2 | hosts: master 3 | gather_facts: yes 4 | remote_user: pi 5 | become: true 6 | vars_files: 7 | - config.yml 8 | 9 | roles: 10 | - master 11 | - dashboard -------------------------------------------------------------------------------- /setup.yml: -------------------------------------------------------------------------------- 1 | - name: Setup Raspberry Pi Base system 2 | hosts: pis 3 | gather_facts: yes 4 | remote_user: pirate 5 | become: true 6 | become_method: sudo 7 | vars_files: 8 | - config.yml 9 | 10 | roles: 11 | - base -------------------------------------------------------------------------------- /roles/node/tasks/main.yml: -------------------------------------------------------------------------------- 1 | - name: Reset Kubernetes installation 2 | shell: "kubeadm reset" 3 | when: "reset==true" 4 | 5 | - name: Adding node to cluster 6 | script: files/join.sh {{ token }} {{ master }} 7 | 8 | register: out 9 | 10 | - debug: var=out.stdout_lines 11 | -------------------------------------------------------------------------------- /roles/update/tasks/main.yml: -------------------------------------------------------------------------------- 1 | - name: Update APT package 2 | command: apt-get update 3 | 4 | - name: Upgrade APT package 5 | command: apt-get upgrade -y 6 | 7 | - name: Reboot 8 | shell: sleep 2 && reboot 9 | async: 1 10 | poll: 0 11 | ignore_errors: true 12 | 13 | - name: Waiting for servers 14 | local_action: wait_for host={{ inventory_hostname }} state=started delay=10 timeout=60 15 | sudo: no -------------------------------------------------------------------------------- /hosts.example: -------------------------------------------------------------------------------- 1 | [pis] 2 | master.cluster.local name=master.cluster.local 3 | node1.cluster.local name=node1.cluster.local 4 | node2.cluster.local name=node2.cluster.local 5 | node3.cluster.local name=node3.cluster.local 6 | node4.cluster.local name=node3.cluster.local 7 | 8 | [master] 9 | master.cluster.local 10 | 11 | [nodes] 12 | node1.cluster.local 13 | node2.cluster.local 14 | node3.cluster.local 15 | node4.cluster.local -------------------------------------------------------------------------------- /roles/base/tasks/wifi.yml: -------------------------------------------------------------------------------- 1 | # This is only needed if not already setup. E.g. when using Raspian instead of the Hypriot image, this 2 | # might be useful 3 | - name: Configure WiFi 4 | template: src=wpa_supplicant.conf dest=/etc/wpa_supplicant/wpa_supplicant.conf mode=0600 5 | 6 | - name: Switch off power management for WiFi 7 | lineinfile: dest=/etc/network/interfaces.d/wlan0 state=present line="wireless-power off" insertafter="^iface wlan0" 8 | -------------------------------------------------------------------------------- /roles/base/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Add and update packages 3 | include: apt.yml 4 | 5 | - name: Add Swapfile 6 | include: swap.yml 7 | vars: 8 | swap_file_path: "/swapfile" 9 | swap_file_size: "1G" 10 | 11 | - name: Setup system parameters (boot, hosts, timzone) 12 | include: system.yml 13 | 14 | - name: Configure Wifi 15 | include: wifi.yml 16 | when: wifi is defined 17 | 18 | - name: Setup user 19 | include: user.yml 20 | -------------------------------------------------------------------------------- /scripts/nodes/download.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | export VERSION=v1.12.0 3 | 4 | if [[ $1 ]]; then 5 | VERSION=$1 6 | fi 7 | 8 | export URL=https://github.com/hypriot/image-builder-rpi/releases/download/$VERSION/hypriotos-rpi-$VERSION.img.zip 9 | export FILENAME=hypriotos-rpi-$VERSION.img.zip 10 | 11 | echo Downloading hypriot/image-builder-rpi $VERSION 12 | mkdir downloads 13 | cd downloads 14 | curl -Lo $FILENAME $URL 15 | unzip $FILENAME 16 | rm $FILENAME 17 | -------------------------------------------------------------------------------- /scripts/nodes/flash.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | export VERSION=v1.12.0 3 | export IMG=downloads/hypriotos-rpi-$VERSION.img 4 | #export DEVICE=/dev/disk2 # Use at your own risk! Put your SD device here and uncomment to perform an unattended flash 5 | 6 | if [[ $1 ]]; then 7 | VERSION=$1 8 | fi 9 | 10 | if [ ! -f "$IMG" ]; then 11 | ./download.sh $VERSION 12 | fi 13 | 14 | if [[ $DEVICE ]]; then 15 | flash -d $DEVICE -f $IMG 16 | else 17 | flash $IMG 18 | fi -------------------------------------------------------------------------------- /roles/base/templates/hosts: -------------------------------------------------------------------------------- 1 | # {{ ansible_managed }} 2 | 3 | 127.0.0.1 localhost 4 | ::1 localhost ip6-localhost ip6-loopback 5 | ff02::1 ip6-allnodes 6 | ff02::2 ip6-allrouters 7 | 8 | {% for item in groups['pis'] %} 9 | {% if hostvars[item].ansible_default_ipv4 is defined %} 10 | {{ hostvars[item].ansible_default_ipv4.address }} {{ hostvars[item].name }}{% if hostvars[item].host_extra is defined %} {{ hostvars[item].host_extra }}{% endif %} 11 | 12 | {% endif %} 13 | {% endfor %} 14 | -------------------------------------------------------------------------------- /roles/base/files/docker_profile.sh: -------------------------------------------------------------------------------- 1 | function docker_nuke() { 2 | docker stop $(docker ps -q) >/dev/null 2>&1 3 | docker rm $(docker ps -q -a) >/dev/null 2>&1 4 | docker rmi $(docker images -qaf 'dangling=true') >/dev/null 2>&1 5 | docker volume rm $(docker volume ls -qf dangling=true) >/dev/null 2>&1 6 | } 7 | 8 | function docker_rmi_none() { 9 | docker rmi $(docker images -qaf 'dangling=true') 10 | } 11 | 12 | function docker_rmi_all() { 13 | docker images | grep $1 | perl -n -e '@a = split; print "$a[0]:$a[1]\n"' | xargs docker rmi 14 | } 15 | -------------------------------------------------------------------------------- /roles/master/tasks/main.yml: -------------------------------------------------------------------------------- 1 | - name: Reset Kubernetes installation 2 | shell: "kubeadm reset" 3 | when: "reset==true" 4 | 5 | - name: Initialize Kubernetes cluster for ARM with flannel support 6 | shell: "kubeadm init --token {{ token }} --pod-network-cidr={{cidr}} --api-external-dns-names kubernetes {{ master }}" 7 | 8 | - name: Install Networking Pods 9 | shell: "curl -sSL 'https://github.com/coreos/flannel/blob/master/Documentation/kube-flannel.yml?raw=true' | 'sed s/amd64/{{arch}}/g' | kubectl create -f -" 10 | register: out 11 | 12 | - debug: var=out.stdout_lines 13 | 14 | - name: Download cluster configuration 15 | fetch: src=/etc/kubernetes/admin.conf dest=./kubeconfig/ -------------------------------------------------------------------------------- /roles/base/tasks/user.yml: -------------------------------------------------------------------------------- 1 | - name: Add a group pi 2 | group: name=pi 3 | 4 | - name: Add a group docker 5 | group: name=docker 6 | 7 | - name: Add user pi to group docker 8 | user: name=pi groups=docker,pi append=yes shell=/bin/bash 9 | 10 | - name: Add pi to to sudoers 11 | lineinfile: 12 | dest: /etc/sudoers 13 | state: present 14 | line: "pi ALL=(ALL) NOPASSWD: ALL" 15 | insertafter: "EOF" 16 | 17 | - name: Verify ~/.ssh 18 | file: path="/home/pi/.ssh" state=directory recurse=no owner=pi group=pi 19 | 20 | - name: Copy SSH Key 21 | copy: src="{{ ssh.pub_key_path }}" dest=/home/pi/.ssh/authorized_keys mode=0600 owner=pi group=pi 22 | 23 | - name: Add user pi to group docker 24 | user: name=pi groups=docker append=yes 25 | 26 | - name: Set user password 27 | user: name=pi password="{{ user.password }}" 28 | when: user is defined and user.password is defined 29 | -------------------------------------------------------------------------------- /roles/base/tasks/apt.yml: -------------------------------------------------------------------------------- 1 | - name: Add apt-transport-https 2 | apt: 3 | name: 'apt-transport-https' 4 | state: present 5 | force: yes 6 | 7 | - name: Add Hypriot Repo Key 8 | apt_key: url=https://packagecloud.io/gpg.key 9 | 10 | - name: Add Hypriot Repo 11 | apt_repository: repo='deb https://packagecloud.io/Hypriot/Schatzkiste/debian/ wheezy main' state=present 12 | 13 | - name: Add Kubernetes Repo Key 14 | apt_key: url=https://packages.cloud.google.com/apt/doc/apt-key.gpg 15 | 16 | - name: Add Kubernetes Repo 17 | apt_repository: repo='deb http://apt.kubernetes.io/ kubernetes-xenial main' state=present 18 | 19 | - name: Update APT package cache 20 | apt: 21 | update_cache: yes 22 | upgrade: safe 23 | 24 | - name: Install Packages 25 | apt: 26 | name: "{{ item }}" 27 | force: yes 28 | state: present 29 | with_items: 30 | - kubelet 31 | - kubeadm 32 | - kubectl 33 | - kubernetes-cni 34 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2016-2020 Sergio Sisternes 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /roles/base/tasks/swap.yml: -------------------------------------------------------------------------------- 1 | # Taken directly from http://stackoverflow.com/questions/24765930/add-swap-memory-with-ansible 2 | - name: Set swap_file variable 3 | set_fact: 4 | swap_file: "{{ swap_file_path }}" 5 | tags: 6 | - swap.set.file.path 7 | 8 | - name: Check if swap file exists 9 | stat: 10 | path: "{{swap_file}}" 11 | register: swap_file_check 12 | tags: 13 | - swap.file.check 14 | 15 | - name: Create swap file 16 | command: fallocate -l {{ swap_file_size }} {{swap_file}} 17 | when: not swap_file_check.stat.exists 18 | tags: 19 | - swap.file.create 20 | 21 | - name: Change swap file permissions 22 | file: path="{{swap_file}}" 23 | owner=root 24 | group=root 25 | mode=0600 26 | tags: 27 | - swap.file.permissions 28 | 29 | - name: Format swap file 30 | command: "mkswap {{swap_file}}" 31 | when: not swap_file_check.stat.exists 32 | tags: 33 | - swap.file.mkswap 34 | 35 | - name: Write swap entry in fstab 36 | mount: name=none 37 | src={{swap_file}} 38 | fstype=swap 39 | opts=sw 40 | passno=0 41 | dump=0 42 | state=present 43 | tags: 44 | - swap.fstab 45 | 46 | - name: Turn on swap 47 | command: swapon -a 48 | when: not swap_file_check.stat.exists 49 | tags: 50 | - swap.turn.on 51 | 52 | - name: Set swappiness 53 | sysctl: 54 | name: vm.swappiness 55 | value: "1" 56 | tags: 57 | - swap.set.swappiness 58 | -------------------------------------------------------------------------------- /roles/base/tasks/system.yml: -------------------------------------------------------------------------------- 1 | - name: Add cgroup for Memory limits to bootparams 2 | lineinfile: 3 | dest: /boot/cmdline.txt 4 | regexp: '^(.*?)(\s*cgroup_enable=cpuset\s*)?$' 5 | line: '\1 cgroup_enable=cpuset' 6 | backrefs: true 7 | state: present 8 | 9 | - name: Add overlay filesystem module 10 | lineinfile: dest=/etc/modules state=present line="overlay" insertafter="EOF" 11 | 12 | - name: Load overlay module 13 | modprobe: name=overlay state=present 14 | 15 | - name: Setup profile 16 | copy: src=docker_profile.sh dest=/etc/profile.d/docker.sh mode=0644 17 | 18 | - name: Set timezone variables 19 | copy: 20 | content: "{{ timezone }}" 21 | dest: /etc/timezone 22 | owner: root 23 | group: root 24 | mode: 0644 25 | backup: yes 26 | 27 | - name: update timezone 28 | command: dpkg-reconfigure --frontend noninteractive tzdata 29 | 30 | - name: Add hosts 31 | template: src=hosts dest=/etc/hosts 32 | 33 | - stat: path=/etc/dhcp/dhclient-enter-hooks.d 34 | register: dhclient_hooks 35 | - name: Copy hosts to DHCP setup 36 | template: src=hosts dest=/etc/dhcp 37 | when: dhclient_hooks.stat.exists 38 | - name: Add DHCP hook to copy etc hosts 39 | copy: src=copy_hosts dest=/etc/dhcp/dhclient-enter-hooks.d/ mode=0755 40 | 41 | - name: Set hostname 42 | hostname: name="{{ name }}" 43 | 44 | - name: Restart hostname 45 | shell: "hostnamectl set-hostname {{ name }}" 46 | 47 | - name: Check for ld-linux-armhf.so.3 48 | stat: path=/lib/ld-linux-armhf.so.3 49 | register: ld 50 | 51 | - name: Link ld-linux-armhf.so.3 --> ld-linux.so.3 52 | file: 53 | src: /lib/ld-linux-armhf.so.3 54 | dest: /lib/ld-linux.so.3 55 | state: link 56 | when: ld.stat.exists 57 | 58 | # Needed only for hypriot 59 | - stat: path=/boot/device-init.yaml 60 | register: device_init_file 61 | - name: Set hostname in boot configuration 62 | lineinfile: 63 | dest: /boot/device-init.yaml 64 | regexp: '^(.*hostname:)\s*(.*?)\s*$' 65 | line: '\1 {{ name }}' 66 | backrefs: true 67 | state: present 68 | when: device_init_file.stat.exists 69 | 70 | - name: Add useDns=no to /etc/ssh/sshd_config 71 | lineinfile: dest=/etc/ssh/sshd_config state=present line="UseDNS no" insertafter="EOF" 72 | 73 | - name: Restart hostname 74 | shell: "hostnamectl set-hostname {{ name }}" 75 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Ansible 2.0 Playbook for Kubernetes on Raspberry Pi 3 2 | Create an Kubernetes cluster with Raspberry Pi 3 (Also work with 1/2) and set it up in 20 minutes using Ansible and kubeadm. 3 | 4 | Based on [Roland Huß](https://github.com/Project31/ansible-kubernetes-openshift-pi3)'s job and originally inspired by the post [Creating a Raspberry Pi cluster running Kubernetes](http://blog.kubernetes.io/2015/11/creating-a-Raspberry-Pi-cluster-running-Kubernetes-the-shopping-list-Part-1.html) by Arjen Wassink and Ray Tsang. 5 | 6 | This project automates all the configuration steps described in the [Kubernetes documentation](http://kubernetes.io/docs/getting-started-guides/kubeadm/) + some basic setup for all nodes. 7 | 8 | ## Project goals 9 | * Create a demo cluster to show Kubernetes capabilities 10 | * Learn more about Kubernetes installation and configuration 11 | * Learn the basics of Ansible 12 | 13 | ## HW List 14 | 15 | Here is the hardware you will need to complete the project: 16 | 17 | | Amount | Part | Price | 18 | | ------ | ---- | ----- | 19 | | 5 | [Raspberry Pi 3](http://amzn.eu/0Gxy4ku) | 5 * 39 EUR | 20 | | 5 | [Micro SD Card 32 GB](http://amzn.eu/5IMqzRx) | 5 * 11 EUR | 21 | | 1 | [WLAN Router](http://amzn.eu/3Zzxmpt) | 24 EUR | 22 | | 1 | [100 MB Switch](http://amzn.eu/ixBjdx3) | 12 EUR | 23 | | 5 | Micro-USB wires | 5 * 1 EUR | 24 | | 1 | [Power Supply](https://www.modmypi.com/raspberry-pi/accessories/usb-hubs/anidees-6-port-smart-ic-usb-charger-50-watt/) | 43 EUR | 25 | | 3 | [Multi stackable case](https://www.modmypi.com/raspberry-pi/cases/multi-pi-stacker/multi-pi-stackable-raspberry-pi-case/) | 3 * 16 EUR | 26 | | 1 | [Bolt pack](https://www.modmypi.com/raspberry-pi/cases/multi-pi-stacker/multi-pi-stackable-raspberry-pi-case-bolt-pack/) | 6 EUR | 27 | 28 | ## SW List 29 | 30 | Here is the software you will need to complete the project: 31 | 32 | | Name | Origin | URL | 33 | | ------ | ---- | ----- | 34 | | HypriotOS | GitHub | [hypriot/rpi-image-builder](https://github.com/hypriot/image-builder-rpi/releases/) | 35 | | flash | GitHub | [hypriot/flash](https://github.com/hypriot/flash) | 36 | | Ansible | Website | [Ansible installation guide](http://docs.ansible.com/ansible/intro_installation.html) 37 | 38 | ## Quick start 39 | 40 | Download the latest HypriotOS image from the hypriot/rpi-image-builder [release page](https://github.com/hypriot/image-builder-rpi/releases/) 41 | 42 | For each SD card, flash the image (v.1.1.3 in this case).: 43 | 44 | ``` $ flash hypriotos-rpi-v1.1.3.img ``` 45 | 46 | Put the SD card in all the Pis. 47 | 48 | RECOMMENDED: For each Pi, turn it on, find out its MAC address (e.g. using your router), set a static IP from them and reboot them to get the new IP. 49 | 50 | Copy "hosts.example" to "hosts" and edit the file. 51 | * Describe in "Pis" all your Raspberry Pi devices' IP (or hostname) (both master and nodes). Don't forget to set the "name" in order to rename each node during setup 52 | * Describe in "Master" ONE of your Raspberry Pi devices that will act as cluster master 53 | * Describe in "Nodes" the rest of Raspberry Pi devices that will act as cluster nodes (Please, do not include here the master!) 54 | 55 | Apply the base configuration for all Pi: 56 | ``` $ ansible-playbook -k -i hosts setup.yml ``` 57 | 58 | IMPORTANT: Amongs others, the setup copies your public SSH key to all Pis and associates it to the user "Pi". Is important to check that the key exists at "~/.ssh/id_rsa.pub". 59 | You can create a new key using the command: 60 | ```ssh-keygen -t rsa -b 4096 ``` 61 | 62 | You can set another path in /roles/base/defaults/main.yml. 63 | 64 | Copy config.example.yml to config.yml 65 | * Put a random Kubernetes token (<6 character string>.<16 character string>) into the "token" parameter. This token will be used for both master and nodes creation 66 | * Put your master hostname or IP address in the "master" variable 67 | 68 | 69 | Create the Kubernetes cluster: 70 | ``` $ ansible-playbook -i hosts master.yml ``` 71 | 72 | Join the nodes to the cluster: 73 | ```$ ansible-playbook -i hosts nodes.yml ``` 74 | 75 | OPTIONAL: The "master.yml" file has copied for you the admin.config file from the master. This file is required to use kubectl from your computer. 76 | * Move the file to ${HOME}/.kube/config or use the flag --kubeconfig when calling to kubectl from your computer [More info](http://kubernetes.io/docs/user-guide/kubectl/kubectl_config/) 77 | * Alternativelly, you can connect to the cluster master and execute kubectl from there. --------------------------------------------------------------------------------