├── .gitignore ├── LICENSE ├── README.md ├── ansible.cfg ├── cluster ├── files ├── bootstrap-17.03.sh ├── bootstrap-up.sh ├── bootstrap.sh └── examples │ ├── alpine.yml │ ├── nginx-deploy.yml │ ├── nginx-service.yml │ └── nginx.yml ├── images └── EdgeMax-DHCP-configure.png ├── install_flash.sh ├── kubernetes-up.yml ├── kubernetes.yml ├── pictures ├── Kubernetes-systemdesign-raspberrypi-edge.png ├── Kubernetes-systemdesign-raspberrypi.graffle └── Kubernetes-systemdesign-raspberrypi.png ├── reset.yml └── user-data-example /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Kubernetes-On-Embedded 2 | 3 | Dies ist das Repo für unseren Blogpost https://bee42.com/de/blog/tutorials/kubernetes-cluster-on-embedded/, schaut mal rein! 4 | 5 | ## Kubernetes 1.9 mit Docker 17.03 6 | 7 | _Achtung: Mit der Hypriot-Version 1.4.0 gab es einen schwerwiegenden Fehler: Die Machine-ID wurde nicht neu generiert und war deshalb bei allen RPI's identisch. Das führte zu [Problemen](https://stackoverflow.com/questions/49113253/kubernetes-pods-stuck-at-containercreating) mit dem Netzwerk-Layer!_ 8 | 9 | ![RPI Systemdesign:](pictures/Kubernetes-systemdesign-raspberrypi.png) 10 | 11 | Alternative kann man den Cluster auch folgendermassen aufbauen: 12 | 13 | Wir bauen einen kompleten Kubernetes-Cluster auf Embedded-Hardware. Dazu benutzen wir einen EdgeRouterX, 2 WLAN-Router, 2 Switches, ein Intel UP-Board und für den eigentlichen K8S-Cluster Raspberry Pi's. 14 | 15 | ![RPI Systemdesign edge:](pictures/Kubernetes-systemdesign-raspberrypi-edge.png) 16 | 17 | ## Einkaufswagen füllen, bestellen, bezahlen, installieren und Spaß haben! 18 | 19 | Alle Komponenten die Ihr braucht um einen Docker PI-Cluster aufzubauen können in der Regel preiswert und zuverlässig bestellt werden. Wir haben uns an der Liste von [Roland Huss](https://ro14nd.de/kubernetes-on-raspberry-pi3) orientiert: 20 | 21 | Danke Roland :-) 22 | 23 | Wir haben hier den aktuellsten RPI Modell 3B+ genommen, ein RPI 2 funktioniert aber auch, natürlich mit Abstrichen in der Leistung. 24 | 25 | Stand 2018-10 ca. 290 Euro 26 | 27 | | Anzahl | Systemkomponente | Preis | 28 | |:-------|:--------------------------------------------------------------|:-----------| 29 | | 3 | [Raspberry Pi 3 B+](https://www.amazon.de/dp/B07BDR5PDW) | 3 * 41 EUR | 30 | | 3 | [Micro SD Card 32 GB](https://www.amazon.de//dp/B073JWXGNT) | 3 * 8 EUR | 31 | | 1 | [WLAN Router](https://www.amazon.de/dp/B01MY5JIJ0) | 1 * 34 EUR | 32 | | 4 | [USB Kabel](https://www.amazon.de/dp/B016BEVNK4) | 1 * 7 EUR | 33 | | 1 | [USB Stromgerät](https://www.amazon.de/dp/B00PTLSH9G) | 1 * 29 EUR | 34 | | 1 | [Gehäuse](https://www.amazon.de/dp/B00NB1WPEE) | 1 * 10 EUR | 35 | | 2 | [Zwischenplatten](https://www.amazon.de/dp/B00NB1WQZW) | 2 * 7 EUR | 36 | | 5 | [Ethernet Kabel](https://www.amazon.de/dp/B005VFZ00E) | 1 * 8 EUR | 37 | | 1 | [Ethernet 8 Port Switch](https://www.amazon.de/dp/B000092RRM) | 1 * 32 EUR | 38 | 39 | ### Optionen 40 | 41 | * Kühler für die PI kaufen und installieren 42 | * OnOff Shim von Pimoroni installieren oder selber bauen 43 | * https://shop.pimoroni.com/products/onoff-shim 44 | * http://www.netzmafia.de/skripten/hardware/RasPi/Projekt-OnOff/index.html 45 | * Nicht vergessen es wird ein Ethernet Switch und Ethernetkabel benötigt. Die WLAN Verbindungen sind in der Regel zu störanfällig. 46 | * https://www.amazon.de/dp/B01MU3WUX1 47 | * https://www.amazon.de/dp/B01N362YPG 48 | * POE Switch und POE 3+ Hat installieren. 49 | * https://shop.pimoroni.com/products/raspberry-pi-poe-hat 50 | * TP-Link-TL-WR902AC-AC750 51 | * https://www.amazon.de/dp/B01MY5JIJ0 52 | * Konfiguration via ROOT WLAN Access Point spart den Edge Router und die zweiten TP Link Access Point. 53 | 54 | ## Kubernetes 1.12.x mit Docker 18.06.x 55 | 56 | Stand: 2018-10-06 57 | 58 | Wer die aktuelle Docker 18.06 mit dem aktuellen Kubernetes 1.12.x zum laufen bringen möchte, 59 | muss etwas in die Trickkisten greifen und das OS Image für die RPIs selber herstellen. 60 | 61 | ### Selbsterstellen des Hypriot OS mit aktuallisierten Packages 62 | 63 | * https://github.com/hypriot/image-builder-rpi 64 | 65 | Installieren von virtualbox und vagrant mit Homebrew 66 | 67 | * https://brew.sh/ 68 | * http://caskroom.io/ 69 | 70 | ``` 71 | $ brew cask install virtualbox 72 | $ brew cask install virtualbox-extension-pack 73 | $ brew cask install vagrant 74 | ``` 75 | 76 | ``` 77 | $ git clone https://github.com/hypriot/image-builder-rpi 78 | $ cd image-builder-rpi 79 | $ vagrant up 80 | $ export DOCKER_HOST=tcp://127.0.0.1:2375 81 | # check OS version and docker version 82 | $ docker info | grep 'Operating System' 83 | Operating System: Ubuntu 16.04.5 LTS 84 | $ mv versions.config versions-orginal.config 85 | $ cat >versions.config <docker-compose.yml <versions.config < 252 | ``` 253 | __Frage__: Wie bekommt eigentlich heraus welche IP dem PI vom DHCP Server zugeordnet wurde? 254 | 255 | ``` 256 | # install nmap 257 | $ brew install nmap 258 | $ nmap -sn 192.168.42.0/24 # Durch Euer Netz ersetzen 259 | ``` 260 | 261 | Das Passwort für den Nutzer __pirate__ lautet: **hypriot**. 262 | 263 | Im Blog der Hypriot Piraten findet Ihr jede Mengen Erklärungen zum Thema Docker on ARM: 264 | 265 | * https://blog.hypriot.com/getting-started-with-docker-on-your-arm-device/ 266 | * https://hub.docker.com/u/hypriot/ 267 | 268 | ## Kubernetes-Cluster installieren 269 | 270 | ### Installation des Kubernetes-Masters 271 | 272 | Zur Ausführung unseres Installationsscripts auf den einzelen RPIs benutzen wir [Ansible](https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html). 273 | 274 | ``` 275 | $ brew install ansible 276 | ``` 277 | 278 | Dafür benötigen wir zunächst ein Inhaltsverzeichnis. Wir haben mehrere Cluster, deshalb ist die Konfiguration in der Datei `cluster` etwas größer angelegt: 279 | 280 | ``` 281 | [cluster-1-master] 282 | 192.168.42.11 283 | 284 | [cluster-1-nodes] 285 | 192.168.42.12 286 | 192.168.42.13 287 | 288 | [cluster-1:children] 289 | cluster-1-master 290 | cluster-1-nodes 291 | 292 | [cluster-1:vars] 293 | fqdn_master="bee42-crew-01-001.local" 294 | network_address_master="192.168.42.11" 295 | 296 | [master:children] 297 | cluster-1-master 298 | 299 | [nodes:children] 300 | cluster-1-nodes 301 | 302 | ``` 303 | 304 | Für eine K8s Cluster sollten die Maschinen eine fixe IP Adresse besitzen. 305 | Wir fixieren also die DHCP Records im Edge Max. 306 | 307 | ![](images/EdgeMax-DHCP-configure.png) 308 | 309 | Die Maschinen müssen dann alle rebooted werden. 310 | 311 | Auf einem MAC kann man den DNS Cache mit folgendem Kommando erneuern: 312 | 313 | ``` 314 | sudo killall -HUP mDNSResponder 315 | ``` 316 | 317 | Ansible verbindet sich per SSH auf die zu verwaltenden Rechner, dort muss also öffentlicher SSH-Key hinterlegt sein. Falls Ihr noch keinen habt, hier ein kleines Beispiel: 318 | 319 | ```bash 320 | # Schlüssel erzeugen 321 | $ ssh-keygen -t ed25519 -C "name@example.org" 322 | 323 | # Öffentlichen Schlüssel auf alle RPIs kopieren 324 | $ ssh-copy-id pirate@192.168.42.11 325 | $ ssh-copy-id pirate@192.168.42.12 326 | ... 327 | 328 | # Testen der Maschinenverfügbarkeit : 329 | $ export SSH_KEY=~/.ssh/id_ed25519 330 | $ export K8sCLUSTER=cluster-1 331 | $ ansible -u pirate --key=$SSH_KEY -i cluster -l $K8sCLUSTER -m ping all 332 | 192.168.42.11 | success >> { 333 | "changed": false, 334 | "ping": "pong" 335 | } 336 | 337 | 192.168.2.12 | success >> { 338 | "changed": false, 339 | "ping": "pong" 340 | } 341 | ``` 342 | 343 | ### K8s-Cluster erzeugen 344 | 345 | So, wenn alle Vorbereitungen abgeschlossen sind, kann der Kubernetes-Cluster erzeugt werden. Ein RPI wird zum Master und die restlichen werden als Nodes angeschlossen. 346 | 347 | ```bash 348 | $ export K8sCLUSTER=cluster-1 349 | $ export KUBERNETES_VERSION=1.13.2 350 | $ export DOCKER_VERSION=18.09.1 351 | $ ansible-playbook -u pirate --key=$SSH_KEY -i cluster -l $K8sCLUSTER kubernetes.yml 352 | $ ssh 192.168.42.11 353 | $ mkdir -p .kube 354 | $ sudo cp /etc/kubernetes/admin.conf .kube/config 355 | $ sudo chown -R pirate:pirate .kube/ 356 | $ kubectl get nodes 357 | NAME STATUS ROLES AGE VERSION 358 | bee42-crew-04-001 Ready master 8m53s v1.12.0 359 | bee42-crew-04-002 Ready 6m32s v1.12.0 360 | bee42-crew-04-003 Ready 6m22s v1.12.0 361 | $ kubectl run my-shell --rm -i --tty --image alpine -- /bin/sh 362 | > ls 363 | ``` 364 | 365 | ``` 366 | ssh 367 | sudo useradd -m -p hypriot -s /bin/bash pirate 368 | sudo usermod -aG sudo pirate 369 | exit 370 | ssh-copy-id -i $SSH_KEY pirate@192.168.42.35 371 | ``` 372 | 373 | I2C: 374 | 375 | * http://www.netzmafia.de/skripten/hardware/RasPi/RasPi_I2C.html 376 | 377 | __Todo__: 378 | 379 | * Kopieren der Images eines Release in eine Registry 380 | * Installation der PI's auf der Basis dieser Registry 381 | * Einrichten eines Mirrors 382 | * Einrichten eines DNS Server für alle Cluster 383 | * Test mit einem Unify Router via Ethernet 384 | * WLAN Router nur für Crew 385 | * Test mit weiteren AccessPoints 386 | * DHCP? 387 | * Loadbalancer 388 | * Metall LB 389 | * https://medium.com/@JockDaRock/kubernetes-metal-lb-for-on-prem-baremetal-cluster-in-10-minutes-c2eaeb3fe813 390 | * External DNS 391 | * Noip Dynamic DNS 392 | 393 | ### Cluster vernichten 394 | 395 | Manchmal möchte man den Cluster abreissen und vielleicht in einer anderen Version neu bauen, deshalb gibt es auch ein Reset-Playbook: 396 | 397 | ```bash 398 | $ ansible-playbook -u pirate --key=$SSH_KEY -i cluster -l $K8sCLUSTER reset.yml 399 | ``` 400 | 401 | __TIPP__: Phönix aus der Asche entstehen lassen 402 | 403 | Wer einen wirklich unbelastete Maschine benötigt, sollte allerdings lieber die SD Karte flashen und den RPI neu starten. 404 | 405 | ## Trouble on arm64 406 | 407 | * https://github.com/kubernetes/kubernetes/issues/64649 408 | * https://github.com/kubernetes/kubernetes/pull/66458/files 409 | 410 | ``` 411 | --audit-log-maxage=30 \ 412 | --audit-log-maxbackup=3 \ 413 | --audit-log-maxsize=100 \ 414 | --authorization-mode=Node,RBAC \ 415 | --enable-swagger-ui=true \ 416 | --event-ttl=1h \ 417 | --insecure-bind-address=127.0.0.1 \ 418 | --runtime-config=rbac.authorization.k8s.io/v1alpha1 \ 419 | --v=2 420 | ``` 421 | 422 | Liebe Grüße
423 | Peter 424 | 425 | * https://gist.github.com/alexellis/fdbc90de7691a1b9edb545c17da2d975 426 | * https://www.trion.de/news/2018/07/06/kubernetes-raspberry-pi.html -------------------------------------------------------------------------------- /ansible.cfg: -------------------------------------------------------------------------------- 1 | [defaults] 2 | inventory = ./inventory 3 | roles_path = ./roles 4 | retry_files_enabled = false 5 | host_key_checking=False 6 | 7 | -------------------------------------------------------------------------------- /cluster: -------------------------------------------------------------------------------- 1 | [cluster-1-master] 2 | 192.168.42.11 3 | 4 | [cluster-1-nodes] 5 | 192.168.42.12 6 | 192.168.42.13 7 | 8 | [cluster-1:children] 9 | cluster-1-master 10 | cluster-1-nodes 11 | 12 | [cluster-1:vars] 13 | fqdn_master="bee42-crew-01-001.local" 14 | network_address_master="192.168.42.11" 15 | 16 | [cluster-2-master] 17 | 192.168.42.21 18 | 19 | [cluster-2-nodes] 20 | 192.168.42.22 21 | 192.168.42.23 22 | 23 | [cluster-2:children] 24 | cluster-2-master 25 | cluster-2-nodes 26 | 27 | [cluster-2:vars] 28 | fqdn_master="bee42-crew-02-001.local" 29 | network_address_master="192.168.42.21" 30 | 31 | [cluster-3-master] 32 | 192.168.42.31 33 | 34 | [cluster-3-nodes] 35 | 192.168.42.32 36 | 192.168.42.33 37 | 38 | [cluster-3-nodes-up] 39 | 192.168.42.35 40 | 41 | [cluster-3:children] 42 | cluster-3-master 43 | cluster-3-nodes 44 | cluster-3-nodes-up 45 | 46 | [cluster-3:vars] 47 | fqdn_master="bee42-crew-03-001.local" 48 | network_address_master="192.168.42.31" 49 | 50 | [cluster-4-master] 51 | 192.168.42.41 52 | 53 | [cluster-4-nodes] 54 | 192.168.42.42 55 | 192.168.42.43 56 | 57 | [cluster-4:children] 58 | cluster-4-master 59 | cluster-4-nodes 60 | 61 | [cluster-4:vars] 62 | fqdn_master="bee42-crew-04-001.local" 63 | network_address_master="192.168.42.41" 64 | 65 | [cluster-5-master] 66 | 192.168.42.51 67 | 68 | [cluster-5-nodes] 69 | 192.168.42.52 70 | 192.168.42.53 71 | 72 | [cluster-5:children] 73 | cluster-5-master 74 | cluster-5-nodes 75 | 76 | [cluster-5:vars] 77 | fqdn_master="bee42-crew-05-001.local" 78 | network_address_master="192.168.42.51" 79 | 80 | [master:children] 81 | cluster-1-master 82 | cluster-2-master 83 | cluster-3-master 84 | cluster-4-master 85 | cluster-5-master 86 | 87 | [nodes:children] 88 | cluster-1-nodes 89 | cluster-2-nodes 90 | cluster-3-nodes 91 | cluster-4-nodes 92 | cluster-5-nodes 93 | 94 | [nodes-up:children] 95 | cluster-3-nodes-up 96 | -------------------------------------------------------------------------------- /files/bootstrap-17.03.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Parameter: 3 | # 1 - Master/node 4 | # 2 - IP Master 5 | # 3 - Kubernetes-version 6 | 7 | set -eu 8 | 9 | K8S_VERSION=$3 10 | K8S_VERSION=${K8S_VERSION:-1.9.6} 11 | DOCKER_VERSION=${DOCKER_VERSION:-17.03} 12 | 13 | ##### 14 | # Disable swap 15 | ##### 16 | swapoff -a 17 | sed -e 's-^\(.*swap.*\)-#\1-' -i /etc/fstab||true 18 | swapon -s 19 | 20 | ##### 21 | # Setup docker 22 | ##### 23 | apt-get update 24 | apt-get install -y \ 25 | apt-transport-https \ 26 | ca-certificates \ 27 | curl \ 28 | software-properties-common 29 | 30 | cat > /etc/docker/daemon.json < /tmp/kubeadm_join 91 | 92 | echo copy kube-config... 93 | mkdir -p ~/.kube/ 94 | cp /etc/kubernetes/admin.conf ~/.kube/config 95 | 96 | echo Deploying Network Layer 97 | 98 | MSG="Wating for Kubernets-API to get ready" 99 | COUNT=10 100 | DELAY=6 101 | COMMAND='kubectl get no' 102 | while eval $COMMAND 2> /dev/null ; [ $? -ne 0 -a $COUNT -gt 0 ];do 103 | sleep $DELAY 104 | COUNT=$(( $COUNT-1 )) 105 | echo $MSG - Counter: $COUNT 106 | done 107 | 108 | # Network-Layer 109 | kubectl apply -f \ 110 | "https://cloud.weave.works/k8s/net?k8s-version=$(kubectl version | base64 | tr -d '\n')" 111 | #curl https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml| sed "s/amd64/arm/g" | kubectl create -f - 112 | ;; 113 | 114 | node) 115 | bash /tmp/kubeadm_join 116 | ;; 117 | esac 118 | -------------------------------------------------------------------------------- /files/bootstrap-up.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Parameter: 3 | # 1 - Master/node 4 | # 2 - IP Master 5 | # 3 - Kubernetes-version 6 | # 4 - Docker-version 7 | 8 | set -eu 9 | DOCKER_VERSION=$4 10 | export DOCKER_VERSION=${DOCKER_VERSION:-18.09.1} 11 | echo " 12 | Package: docker-ce 13 | Pin: version ${DOCKER_VERSION}.* 14 | Pin-Priority: 1000 15 | " > /etc/apt/preferences.d/docker-ce 16 | 17 | apt-get update 18 | apt-get install -y \ 19 | apt-transport-https \ 20 | ca-certificates \ 21 | curl \ 22 | software-properties-common 23 | curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - 24 | add-apt-repository \ 25 | "deb [arch=amd64] https://download.docker.com/linux/ubuntu \ 26 | $(lsb_release -cs) \ 27 | stable" 28 | apt-get update && apt-get install -y docker-ce 29 | 30 | cat > /etc/docker/daemon.json < /tmp/kubeadm_join 91 | 92 | echo copy kube-config... 93 | mkdir -p ~/.kube/ 94 | cp /etc/kubernetes/admin.conf ~/.kube/config 95 | 96 | echo Deploying Network Layer 97 | 98 | MSG="Wating for Kubernets-API to get ready" 99 | COUNT=10 100 | DELAY=6 101 | COMMAND='kubectl get no' 102 | while eval $COMMAND 2> /dev/null ; [ $? -ne 0 -a $COUNT -gt 0 ];do 103 | sleep $DELAY 104 | COUNT=$(( $COUNT-1 )) 105 | echo $MSG - Counter: $COUNT 106 | done 107 | 108 | # Network-Layer 109 | kubectl apply -f \ 110 | "https://cloud.weave.works/k8s/net?k8s-version=$(kubectl version | base64 | tr -d '\n')" 111 | #curl https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml| sed "s/amd64/arm/g" | kubectl create -f - 112 | ;; 113 | 114 | node) 115 | bash /tmp/kubeadm_join 116 | ;; 117 | esac 118 | -------------------------------------------------------------------------------- /files/bootstrap.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Parameter: 3 | # 1 - Master/node 4 | # 2 - IP Master 5 | # 3 - Kubernetes-version 6 | 7 | set -eu 8 | 9 | K8S_VERSION=$3 10 | K8S_VERSION=${K8S_VERSION:-1.13.2} 11 | K8S_CNI_VERSION=${K8S_CNI_VERSION:-0.6.0-00} 12 | 13 | ##### 14 | # Disable swap 15 | ##### 16 | swapoff -a 17 | sed -e 's-^\(.*swap.*\)-#\1-' -i /etc/fstab||true 18 | swapon -s 19 | 20 | ##### 21 | #Setup Kubernetes 22 | ##### 23 | tee /etc/apt/preferences.d/k8s < /tmp/kubeadm_join 60 | 61 | echo copy kube-config... 62 | mkdir -p ~/.kube/ 63 | cp /etc/kubernetes/admin.conf ~/.kube/config 64 | 65 | echo Deploying Network Layer 66 | 67 | MSG="Wating for Kubernets-API to get ready" 68 | COUNT=10 69 | DELAY=6 70 | COMMAND='kubectl get no' 71 | while eval $COMMAND 2> /dev/null ; [ $? -ne 0 -a $COUNT -gt 0 ];do 72 | sleep $DELAY 73 | COUNT=$(( $COUNT-1 )) 74 | echo $MSG - Counter: $COUNT 75 | done 76 | 77 | # Network-Layer 78 | kubectl apply -f \ 79 | "https://cloud.weave.works/k8s/net?k8s-version=$(kubectl version | base64 | tr -d '\n')" 80 | #curl https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml| sed "s/amd64/arm/g" | kubectl create -f - 81 | ;; 82 | 83 | node) 84 | bash /tmp/kubeadm_join 85 | ;; 86 | esac 87 | -------------------------------------------------------------------------------- /files/examples/alpine.yml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: alpine 5 | namespace: default 6 | spec: 7 | containers: 8 | - image: alpine:3.8 9 | command: 10 | - /bin/sh 11 | - "-c" 12 | - "sleep 60m" 13 | imagePullPolicy: IfNotPresent 14 | name: alpine 15 | restartPolicy: Always -------------------------------------------------------------------------------- /files/examples/nginx-deploy.yml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: nginx-deployment 5 | spec: 6 | selector: 7 | matchLabels: 8 | app: nginx 9 | replicas: 2 10 | template: 11 | metadata: 12 | labels: 13 | app: nginx 14 | spec: 15 | containers: 16 | - name: nginx 17 | image: nginx:1.15.5 18 | ports: 19 | - containerPort: 80 20 | -------------------------------------------------------------------------------- /files/examples/nginx-service.yml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: my-nginx 5 | labels: 6 | run: my-nginx 7 | spec: 8 | type: NodePort 9 | ports: 10 | - port: 8080 11 | targetPort: 80 12 | protocol: TCP 13 | name: http 14 | selector: 15 | app: nginx 16 | -------------------------------------------------------------------------------- /files/examples/nginx.yml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: nginx 5 | labels: 6 | env: test 7 | spec: 8 | containers: 9 | - name: nginx 10 | image: nginx 11 | imagePullPolicy: IfNotPresent 12 | nodeSelector: 13 | beta.kubernetes.io/arch: amd64 -------------------------------------------------------------------------------- /images/EdgeMax-DHCP-configure.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bee42/kubernetes-on-embedded/2b01f532955c303352ea9a9bf26cff7cc236d242/images/EdgeMax-DHCP-configure.png -------------------------------------------------------------------------------- /install_flash.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | set -o pipefail 4 | 5 | 6 | curl -O https://raw.githubusercontent.com/hypriot/flash/master/flash && \ 7 | chmod +x flash && \ 8 | sudo mv flash /usr/local/bin/flash 9 | -------------------------------------------------------------------------------- /kubernetes-up.yml: -------------------------------------------------------------------------------- 1 | - hosts: master 2 | become: true 3 | gather_facts: False 4 | vars: 5 | kubernetes_version: "{{ lookup('env', 'KUBERNETES_VERSION')| default('1.13.2',true) }}" 6 | tasks: 7 | - name: install k8s 8 | script: "bootstrap.sh master {{network_address_master}} {{ kubernetes_version }}" 9 | - name: Copy k8s-config 10 | fetch: 11 | src: /etc/kubernetes/admin.conf 12 | dest: "{{playbook_dir}}/secrets" 13 | - name: Copy join-command 14 | fetch: 15 | src: /tmp/kubeadm_join 16 | dest: "{{playbook_dir}}/secrets" 17 | 18 | - hosts: nodes 19 | become: true 20 | gather_facts: False 21 | vars: 22 | kubernetes_version: "{{ lookup('env', 'KUBERNETES_VERSION')| default('1.13.2',true) }}" 23 | tasks: 24 | - name: Copy join-command 25 | copy: 26 | src: "{{playbook_dir}}/secrets/{{network_address_master}}/tmp/kubeadm_join" 27 | dest: /tmp/kubeadm_join 28 | - name: install and join k8s 29 | script: "bootstrap.sh node {{network_address_master}} {{ kubernetes_version }}" 30 | 31 | - hosts: nodes-up 32 | become: true 33 | gather_facts: False 34 | vars: 35 | kubernetes_version: "{{ lookup('env', 'KUBERNETES_VERSION')| default('1.13.2',true) }}" 36 | docker_version: "{{ lookup('env', 'DOCKER_VERSION')| default('18.09.1',true) }}" 37 | tasks: 38 | - name: Copy join-command 39 | copy: 40 | src: "{{playbook_dir}}/secrets/{{network_address_master}}/tmp/kubeadm_join" 41 | dest: /tmp/kubeadm_join 42 | - name: install and join k8s 43 | script: "bootstrap-up.sh node {{network_address_master}} {{ kubernetes_version }} {{ docker_version }}" 44 | -------------------------------------------------------------------------------- /kubernetes.yml: -------------------------------------------------------------------------------- 1 | - hosts: master 2 | become: true 3 | gather_facts: False 4 | vars: 5 | kubernetes_version: "{{ lookup('env', 'KUBERNETES_VERSION')| default('1.13.2',true) }}" 6 | tasks: 7 | - name: install k8s 8 | script: "bootstrap.sh master {{network_address_master}} {{ kubernetes_version }}" 9 | - name: Copy k8s-config 10 | fetch: 11 | src: /etc/kubernetes/admin.conf 12 | dest: "{{playbook_dir}}/secrets" 13 | - name: Copy join-command 14 | fetch: 15 | src: /tmp/kubeadm_join 16 | dest: "{{playbook_dir}}/secrets" 17 | 18 | - hosts: nodes 19 | become: true 20 | gather_facts: False 21 | vars: 22 | kubernetes_version: "{{ lookup('env', 'KUBERNETES_VERSION')| default('1.13.2',true) }}" 23 | tasks: 24 | - name: Copy join-command 25 | copy: 26 | src: "{{playbook_dir}}/secrets/{{network_address_master}}/tmp/kubeadm_join" 27 | dest: /tmp/kubeadm_join 28 | - name: install and join k8s 29 | script: "bootstrap.sh node {{network_address_master}} {{ kubernetes_version }}" 30 | -------------------------------------------------------------------------------- /pictures/Kubernetes-systemdesign-raspberrypi-edge.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bee42/kubernetes-on-embedded/2b01f532955c303352ea9a9bf26cff7cc236d242/pictures/Kubernetes-systemdesign-raspberrypi-edge.png -------------------------------------------------------------------------------- /pictures/Kubernetes-systemdesign-raspberrypi.graffle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bee42/kubernetes-on-embedded/2b01f532955c303352ea9a9bf26cff7cc236d242/pictures/Kubernetes-systemdesign-raspberrypi.graffle -------------------------------------------------------------------------------- /pictures/Kubernetes-systemdesign-raspberrypi.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bee42/kubernetes-on-embedded/2b01f532955c303352ea9a9bf26cff7cc236d242/pictures/Kubernetes-systemdesign-raspberrypi.png -------------------------------------------------------------------------------- /reset.yml: -------------------------------------------------------------------------------- 1 | - hosts: all 2 | become: true 3 | gather_facts: False 4 | 5 | tasks: 6 | - name: delete k8s-cluster 7 | command: kubeadm reset --force 8 | - name: remove kubernetes-config-folder 9 | file: 10 | path: /etc/kubernetes 11 | state: absent 12 | - name: remove k8s-pinning 13 | file: 14 | path: /etc/apt/preferences.d/k8s 15 | state: absent 16 | - name: uninstall k8s-binaries 17 | apt: 18 | name: kubeadm,kubelet,kubectl,kubernetes-cni 19 | state: absent 20 | purge: yes 21 | 22 | -------------------------------------------------------------------------------- /user-data-example: -------------------------------------------------------------------------------- 1 | #cloud-config 2 | # vim: syntax=yaml 3 | # 4 | 5 | # The current version of cloud-init in the Hypriot rpi-64 is 0.7.6 6 | # When dealing with cloud-init, it is SUPER important to know the version 7 | # I have wasted many hours creating servers to find out the module I was trying to use wasn't in the cloud-init version I had 8 | # Documentation: http://cloudinit.readthedocs.io/en/0.7.9/index.html 9 | 10 | # Set your hostname here, the manage_etc_hosts will update the hosts file entries as well 11 | hostname: REPLACE_ME 12 | manage_etc_hosts: true 13 | 14 | # You could modify this for your own user information 15 | users: 16 | - name: pirate 17 | gecos: "Hypriot Pirate" 18 | sudo: ALL=(ALL) NOPASSWD:ALL 19 | shell: /bin/bash 20 | groups: users,docker,video,input 21 | plain_text_passwd: hypriot 22 | lock_passwd: false 23 | ssh_pwauth: true 24 | chpasswd: { expire: false } 25 | ssh_authorized_keys: 26 | - 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFaMxEwf5lwpyjn1aazYqCsmegzmjBKGxJjK2+f5UarS nmietz@digitalocean' 27 | - 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFl546jxIfE2XAxHOBTsKL/KRC11eoNHP7Q8y4InJrfp peter.rossbach@bee42.com' 28 | 29 | # # Set the locale of the system 30 | # locale: "en_US.UTF-8" 31 | 32 | # # Set the timezone 33 | # # Value of 'timezone' must exist in /usr/share/zoneinfo 34 | # timezone: "America/Los_Angeles" 35 | 36 | # # Update apt packages on first boot 37 | # package_update: true 38 | # package_upgrade: true 39 | # package_reboot_if_required: true 40 | package_upgrade: false 41 | 42 | # # Install any additional apt packages you need here 43 | # packages: 44 | # - ntp 45 | 46 | # # WiFi connect to HotSpot 47 | # # - use `wpa_passphrase SSID PASSWORD` to encrypt the psk 48 | # write_files: 49 | # - content: | 50 | # allow-hotplug wlan0 51 | # iface wlan0 inet dhcp 52 | # wpa-conf /etc/wpa_supplicant/wpa_supplicant.conf 53 | # iface default inet dhcp 54 | # path: /etc/network/interfaces.d/wlan0 55 | # - content: | 56 | # country=de 57 | # ctrl_interface=DIR=/var/run/wpa_supplicant GROUP=netdev 58 | # update_config=1 59 | # network={ 60 | # ssid="YOUR_WIFI_SSID" 61 | # psk="YOUR_WIFI_PASSWORD" 62 | # proto=RSN 63 | # key_mgmt=WPA-PSK 64 | # pairwise=CCMP 65 | # auth_alg=OPEN 66 | # } 67 | # path: /etc/wpa_supplicant/wpa_supplicant.conf 68 | 69 | 70 | # These commands will be ran once on first boot only 71 | runcmd: 72 | # Pickup the hostname changes 73 | - 'systemctl restart avahi-daemon' 74 | 75 | --------------------------------------------------------------------------------