├── .envrc ├── .gitattributes ├── .gitignore ├── .pre-commit-config.yaml ├── .woodpecker ├── helm-diff.yaml └── static-checks.yaml ├── .yamllint.yaml ├── LICENSE.md ├── Makefile ├── README.md ├── apps ├── actualbudget │ ├── Chart.yaml │ └── values.yaml ├── blog │ ├── Chart.yaml │ └── values.yaml ├── excalidraw │ ├── Chart.yaml │ └── values.yaml ├── homepage │ ├── Chart.yaml │ └── values.yaml ├── jellyfin │ ├── Chart.yaml │ └── values.yaml ├── matrix │ ├── Chart.yaml │ └── values.yaml ├── ollama │ ├── Chart.yaml │ └── values.yaml ├── pairdrop │ ├── Chart.yaml │ └── values.yaml ├── paperless │ ├── Chart.yaml │ ├── templates │ │ └── secret.yaml │ └── values.yaml ├── speedtest │ ├── Chart.yaml │ └── values.yaml ├── tailscale │ ├── Chart.yaml │ ├── templates │ │ ├── role.yaml │ │ ├── rolebinding.yaml │ │ ├── secret.yaml │ │ └── serviceaccount.yaml │ └── values.yaml └── wireguard │ ├── Chart.yaml │ └── values.yaml ├── docs ├── .gitignore ├── concepts │ ├── certificate-management.md │ ├── development-shell.md │ ├── pxe-boot.md │ ├── secrets-management.md │ └── testing.md ├── getting-started │ ├── install-pre-commit-hooks.md │ ├── user-onboarding.md │ └── vpn-setup.md ├── how-to-guides │ ├── add-or-remove-nodes.md │ ├── alternate-dns-setup.md │ ├── backup-and-restore.md │ ├── disable-dhcp-proxy-in-dnsmasq.md │ ├── expose-services-to-the-internet.md │ ├── media-management.md │ ├── run-commands-on-multiple-nodes.md │ ├── single-node-cluster-adjustments.md │ ├── troubleshooting │ │ └── pxe-boot.md │ ├── updating-documentation.md │ └── use-both-github-and-gitea.md ├── index.md ├── installation │ ├── post-installation.md │ ├── production │ │ ├── configuration.md │ │ ├── deployment.md │ │ ├── external-resources.md │ │ └── prerequisites.md │ └── sandbox.md └── reference │ ├── architecture │ ├── decision-records.md │ ├── networking.md │ └── overview.md │ ├── changelog.md │ ├── contributing.md │ ├── faq.md │ ├── license.md │ └── roadmap.md ├── external ├── .gitignore ├── Makefile ├── main.tf ├── modules │ ├── cloudflare │ │ ├── main.tf │ │ ├── variables.tf │ │ └── versions.tf │ ├── extra-secrets │ │ ├── main.tf │ │ ├── variables.tf │ │ └── versions.tf │ └── ntfy │ │ ├── main.tf │ │ ├── variables.tf │ │ └── versions.tf ├── namespaces.yml ├── terraform.tfvars.example ├── variables.tf └── versions.tf ├── flake.lock ├── flake.nix ├── metal ├── Makefile ├── ansible.cfg ├── boot.yml ├── cluster.yml ├── group_vars │ └── all.yml ├── inventories │ ├── prod.yml │ └── stag.yml ├── k3d-dev.yaml └── roles │ ├── automatic_upgrade │ ├── files │ │ └── automatic.conf │ └── tasks │ │ └── main.yml │ ├── cilium │ ├── defaults │ │ └── main.yml │ ├── tasks │ │ └── main.yml │ └── templates │ │ ├── ciliuml2announcementpolicy.yaml │ │ └── ciliumloadbalancerippool.yaml │ ├── k3s │ ├── defaults │ │ └── main.yml │ ├── files │ │ └── bin │ │ │ └── .gitignore │ ├── tasks │ │ └── main.yml │ └── templates │ │ ├── config.yaml.j2 │ │ ├── k3s.service.j2 │ │ └── kube-vip.yaml.j2 │ ├── prerequisites │ └── tasks │ │ └── main.yml │ ├── pxe_server │ ├── defaults │ │ └── main.yml │ ├── files │ │ ├── data │ │ │ ├── init-config │ │ │ │ └── .gitignore │ │ │ ├── iso │ │ │ │ └── .gitignore │ │ │ ├── os │ │ │ │ └── .gitignore │ │ │ └── pxe-config │ │ │ │ └── .gitignore │ │ ├── dnsmasq │ │ │ └── Dockerfile │ │ ├── docker-compose.yml │ │ └── http │ │ │ └── Dockerfile │ ├── tasks │ │ └── main.yml │ └── templates │ │ ├── dnsmasq.conf.j2 │ │ ├── grub.cfg.j2 │ │ └── kickstart.ks.j2 │ └── wake │ └── tasks │ └── main.yml ├── mkdocs.yml ├── platform ├── dex │ ├── Chart.yaml │ ├── templates │ │ └── secret.yaml │ └── values.yaml ├── external-secrets │ └── Chart.yaml ├── gitea │ ├── Chart.yaml │ ├── files │ │ └── config │ │ │ ├── config.yaml │ │ │ ├── go.mod │ │ │ ├── go.sum │ │ │ └── main.go │ ├── templates │ │ ├── admin-secret.yaml │ │ ├── config-job.yaml │ │ └── config-source.yaml │ └── values.yaml ├── global-secrets │ ├── Chart.yaml │ ├── files │ │ └── secret-generator │ │ │ ├── config.yaml │ │ │ ├── go.mod │ │ │ ├── go.sum │ │ │ └── main.go │ └── templates │ │ ├── clustersecretstore │ │ ├── clustersecretstore.yaml │ │ ├── role.yaml │ │ ├── rolebinding.yaml │ │ └── serviceaccount.yaml │ │ └── secret-generator │ │ ├── configmap.yaml │ │ ├── job.yaml │ │ ├── role.yaml │ │ ├── rolebinding.yaml │ │ └── serviceaccount.yaml ├── grafana │ ├── Chart.yaml │ ├── templates │ │ └── secret.yaml │ └── values.yaml ├── kanidm │ ├── Chart.yaml │ ├── templates │ │ ├── certificate.yaml │ │ └── issuer.yaml │ └── values.yaml ├── renovate │ ├── Chart.yaml │ ├── templates │ │ └── secret.yaml │ └── values.yaml ├── woodpecker │ ├── Chart.yaml │ ├── templates │ │ └── secret.yaml │ └── values.yaml └── zot │ ├── Chart.yaml │ ├── templates │ └── admin-secret.yaml │ └── values.yaml ├── renovate.json5 ├── scripts ├── argocd-admin-password ├── backup ├── configure ├── get-dns-config ├── get-status ├── get-wireguard-config ├── hacks ├── helm-diff ├── kanidm-reset-password ├── new-service ├── onboard-user ├── pxe-logs └── take-screenshots ├── system ├── Makefile ├── argocd │ ├── Chart.yaml │ ├── values-seed.yaml │ └── values.yaml ├── bootstrap.yml ├── cert-manager │ ├── Chart.yaml │ ├── templates │ │ └── clusterissuer.yaml │ └── values.yaml ├── cloudflared │ ├── Chart.yaml │ └── values.yaml ├── external-dns │ ├── Chart.yaml │ └── values.yaml ├── ingress-nginx │ ├── Chart.yaml │ └── values.yaml ├── kured │ ├── Chart.yaml │ └── values.yaml ├── loki │ ├── Chart.yaml │ └── values.yaml ├── monitoring-system │ ├── Chart.yaml │ ├── files │ │ └── webhook-transformer │ │ │ └── alertmanager-to-ntfy.jsonnet │ ├── templates │ │ └── configmap.yaml │ └── values.yaml ├── rook-ceph │ ├── Chart.yaml │ └── values.yaml └── volsync-system │ └── Chart.yaml └── test ├── Makefile ├── benchmark ├── security │ └── kube-bench.yaml └── storage │ ├── dbench-rwo.yaml │ └── dbench-rwx.yaml ├── external_test.go ├── go.mod ├── go.sum ├── integration_test.go ├── smoke_test.go └── tools_test.go /.envrc: -------------------------------------------------------------------------------- 1 | use flake 2 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | external/** -linguist-vendored 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .venv/ 2 | book/ 3 | 4 | *.iso 5 | *.log 6 | *.png 7 | *.tgz 8 | *kubeconfig.yaml 9 | Chart.lock 10 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/pre-commit/pre-commit-hooks 3 | rev: v4.3.0 4 | hooks: 5 | - id: check-added-large-files 6 | - id: check-executables-have-shebangs 7 | - id: check-merge-conflict 8 | - id: check-shebang-scripts-are-executable 9 | - id: detect-private-key 10 | - id: end-of-file-fixer 11 | - id: mixed-line-ending 12 | - id: trailing-whitespace 13 | - repo: https://github.com/adrienverge/yamllint 14 | rev: v1.27.1 15 | hooks: 16 | - id: yamllint 17 | - repo: https://github.com/gruntwork-io/pre-commit 18 | rev: v0.1.24 19 | hooks: 20 | - id: helmlint 21 | - id: shellcheck 22 | - repo: https://github.com/tofuutils/pre-commit-opentofu 23 | rev: v2.1.0 24 | hooks: 25 | - id: tofu_fmt 26 | -------------------------------------------------------------------------------- /.woodpecker/helm-diff.yaml: -------------------------------------------------------------------------------- 1 | when: 2 | branch: ${CI_REPO_DEFAULT_BRANCH} 3 | 4 | matrix: 5 | STACK: 6 | - system 7 | - platform 8 | - apps 9 | 10 | steps: 11 | # TODO DRY with nix develop and custom entrypoint https://github.com/woodpecker-ci/woodpecker/pull/2985, 12 | # but first we need a Nix cache. See the nix-cache branch for the WIP. 13 | diff: 14 | image: nixery.dev/shell/git/python3/kubernetes-helm/diffutils/dyff # TODO replace with nix develop 15 | commands: 16 | - ./scripts/helm-diff --repository "${CI_REPO_CLONE_URL}" --source "${CI_COMMIT_SOURCE_BRANCH}" --target "${CI_COMMIT_TARGET_BRANCH}" --subpath "${STACK}" 17 | when: 18 | - event: pull_request 19 | path: '${STACK}/**' 20 | depends_on: [] 21 | -------------------------------------------------------------------------------- /.woodpecker/static-checks.yaml: -------------------------------------------------------------------------------- 1 | when: 2 | branch: ${CI_REPO_DEFAULT_BRANCH} 3 | 4 | steps: 5 | # TODO DRY with nix develop and custom entrypoint https://github.com/woodpecker-ci/woodpecker/pull/2985, 6 | # but first we need a Nix cache. See the nix-cache branch for the WIP. 7 | tools-versions: 8 | image: nixos/nix 9 | commands: 10 | - echo 'experimental-features = flakes nix-command' >> /etc/nix/nix.conf 11 | # - echo 'trusted-substituters = http://nix-cache.nix-cache' >> /etc/nix/nix.conf 12 | # - echo 'substituters = http://nix-cache.nix-cache' >> /etc/nix/nix.conf 13 | - nix develop --command make -C test filter=ToolsVersions 14 | when: 15 | - event: pull_request 16 | path: 17 | include: 18 | - 'flake.*' 19 | depends_on: [] 20 | pre-commit: 21 | image: nixery.dev/shell/git/pre-commit # TODO replace with nix develop 22 | commands: 23 | - pre-commit run --color=always 24 | when: 25 | - event: pull_request 26 | depends_on: [] 27 | -------------------------------------------------------------------------------- /.yamllint.yaml: -------------------------------------------------------------------------------- 1 | ignore: | 2 | templates/ 3 | mkdocs.yml 4 | 5 | extends: default 6 | 7 | rules: 8 | document-start: disable 9 | line-length: disable 10 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .POSIX: 2 | .PHONY: * 3 | .EXPORT_ALL_VARIABLES: 4 | 5 | KUBECONFIG = $(shell pwd)/metal/kubeconfig.yaml 6 | KUBE_CONFIG_PATH = $(KUBECONFIG) 7 | 8 | default: metal system external smoke-test post-install clean 9 | 10 | configure: 11 | ./scripts/configure 12 | git status 13 | 14 | metal: 15 | make -C metal 16 | 17 | system: 18 | make -C system 19 | 20 | external: 21 | make -C external 22 | 23 | smoke-test: 24 | make -C test filter=Smoke 25 | 26 | post-install: 27 | @./scripts/hacks 28 | 29 | # TODO maybe there's a better way to manage backup with GitOps? 30 | backup: 31 | ./scripts/backup --action setup --namespace=actualbudget --pvc=actualbudget-data 32 | ./scripts/backup --action setup --namespace=jellyfin --pvc=jellyfin-data 33 | 34 | restore: 35 | ./scripts/backup --action restore --namespace=actualbudget --pvc=actualbudget-data 36 | ./scripts/backup --action restore --namespace=jellyfin --pvc=jellyfin-data 37 | 38 | test: 39 | make -C test 40 | 41 | clean: 42 | docker compose --project-directory ./metal/roles/pxe_server/files down 43 | 44 | docs: 45 | mkdocs serve 46 | 47 | git-hooks: 48 | pre-commit install 49 | -------------------------------------------------------------------------------- /apps/actualbudget/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: actualbudget 3 | version: 0.0.0 4 | dependencies: 5 | - name: app-template 6 | version: 2.6.0 7 | repository: https://bjw-s.github.io/helm-charts 8 | -------------------------------------------------------------------------------- /apps/actualbudget/values.yaml: -------------------------------------------------------------------------------- 1 | app-template: 2 | controllers: 3 | main: 4 | containers: 5 | main: 6 | image: 7 | repository: docker.io/actualbudget/actual-server 8 | tag: 24.12.0-alpine 9 | service: 10 | main: 11 | ports: 12 | http: 13 | port: 5006 14 | protocol: HTTP 15 | ingress: 16 | main: 17 | enabled: true 18 | className: nginx 19 | annotations: 20 | cert-manager.io/cluster-issuer: letsencrypt-prod 21 | hosts: 22 | - host: &host budget.khuedoan.com 23 | paths: 24 | - path: / 25 | pathType: Prefix 26 | service: 27 | name: main 28 | port: http 29 | tls: 30 | - hosts: 31 | - *host 32 | secretName: actualbudget-tls-certificate 33 | persistence: 34 | data: 35 | accessMode: ReadWriteOnce 36 | size: 1Gi 37 | globalMounts: 38 | - path: /data 39 | -------------------------------------------------------------------------------- /apps/blog/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: blog 3 | version: 0.0.0 4 | dependencies: 5 | - name: app-template 6 | version: 2.2.0 7 | repository: https://bjw-s.github.io/helm-charts 8 | -------------------------------------------------------------------------------- /apps/blog/values.yaml: -------------------------------------------------------------------------------- 1 | app-template: 2 | controllers: 3 | main: 4 | containers: 5 | main: 6 | image: 7 | repository: registry.khuedoan.com/blog 8 | tag: latest 9 | service: 10 | main: 11 | ports: 12 | http: 13 | port: 3000 14 | protocol: HTTP 15 | ingress: 16 | main: 17 | enabled: true 18 | className: nginx 19 | annotations: 20 | cert-manager.io/cluster-issuer: letsencrypt-prod 21 | external-dns.alpha.kubernetes.io/target: homelab-tunnel.khuedoan.com 22 | external-dns.alpha.kubernetes.io/cloudflare-proxied: 'true' 23 | hosts: 24 | - host: &host www.khuedoan.com 25 | paths: 26 | - path: / 27 | pathType: Prefix 28 | service: 29 | name: main 30 | port: http 31 | tls: 32 | - hosts: 33 | - *host 34 | secretName: blog-tls-certificate 35 | -------------------------------------------------------------------------------- /apps/excalidraw/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: excalidraw 3 | version: 0.0.0 4 | dependencies: 5 | - name: app-template 6 | version: 2.2.0 7 | repository: https://bjw-s.github.io/helm-charts 8 | -------------------------------------------------------------------------------- /apps/excalidraw/values.yaml: -------------------------------------------------------------------------------- 1 | app-template: 2 | controllers: 3 | main: 4 | containers: 5 | main: 6 | image: 7 | repository: docker.io/excalidraw/excalidraw 8 | tag: latest 9 | service: 10 | main: 11 | ports: 12 | http: 13 | port: 80 14 | protocol: HTTP 15 | ingress: 16 | main: 17 | enabled: true 18 | className: nginx 19 | annotations: 20 | cert-manager.io/cluster-issuer: letsencrypt-prod 21 | external-dns.alpha.kubernetes.io/target: homelab-tunnel.khuedoan.com 22 | external-dns.alpha.kubernetes.io/cloudflare-proxied: 'true' 23 | hosts: 24 | - host: &host draw.khuedoan.com 25 | paths: 26 | - path: / 27 | pathType: Prefix 28 | service: 29 | name: main 30 | port: http 31 | tls: 32 | - hosts: 33 | - *host 34 | secretName: excalidraw-tls-certificate 35 | -------------------------------------------------------------------------------- /apps/homepage/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: homepage 3 | version: 0.0.0 4 | dependencies: 5 | - name: app-template 6 | version: 2.6.0 7 | repository: https://bjw-s.github.io/helm-charts 8 | -------------------------------------------------------------------------------- /apps/jellyfin/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: jellyfin 3 | version: 0.0.0 4 | dependencies: 5 | - name: app-template 6 | version: 2.6.0 7 | repository: https://bjw-s.github.io/helm-charts 8 | -------------------------------------------------------------------------------- /apps/jellyfin/values.yaml: -------------------------------------------------------------------------------- 1 | app-template: 2 | defaultPodOptions: 3 | securityContext: 4 | fsGroup: 1000 5 | controllers: 6 | main: 7 | containers: 8 | main: 9 | image: 10 | repository: docker.io/jellyfin/jellyfin 11 | tag: 10.8.13 12 | transmission: 13 | image: 14 | repository: lscr.io/linuxserver/transmission 15 | tag: 4.0.5 16 | prowlarr: 17 | image: 18 | repository: lscr.io/linuxserver/prowlarr 19 | tag: 1.13.3 20 | radarr: 21 | image: 22 | repository: lscr.io/linuxserver/radarr 23 | tag: 5.3.6 24 | sonarr: 25 | image: 26 | repository: lscr.io/linuxserver/sonarr 27 | tag: 4.0.2 28 | jellyseerr: 29 | image: 30 | repository: docker.io/fallenbagel/jellyseerr 31 | tag: 1.7.0 32 | service: 33 | main: 34 | ports: 35 | http: 36 | port: 8096 37 | protocol: HTTP 38 | transmission: 39 | port: 9091 40 | protocol: HTTP 41 | prowlarr: 42 | port: 9696 43 | protocol: HTTP 44 | radarr: 45 | port: 7878 46 | protocol: HTTP 47 | sonarr: 48 | port: 8989 49 | protocol: HTTP 50 | jellyseerr: 51 | port: 5055 52 | protocol: HTTP 53 | ingress: 54 | main: 55 | enabled: true 56 | className: nginx 57 | annotations: 58 | cert-manager.io/cluster-issuer: letsencrypt-prod 59 | hosts: 60 | - host: &jellyfinHost jellyfin.khuedoan.com 61 | paths: 62 | - path: / 63 | pathType: Prefix 64 | service: 65 | name: main 66 | port: http 67 | - host: &transmissionHost transmission.khuedoan.com 68 | paths: 69 | - path: / 70 | pathType: Prefix 71 | service: 72 | name: main 73 | port: transmission 74 | - host: &prowlarrHost prowlarr.khuedoan.com 75 | paths: 76 | - path: / 77 | pathType: Prefix 78 | service: 79 | name: main 80 | port: prowlarr 81 | - host: &radarrHost radarr.khuedoan.com 82 | paths: 83 | - path: / 84 | pathType: Prefix 85 | service: 86 | name: main 87 | port: radarr 88 | - host: &sonarrHost sonarr.khuedoan.com 89 | paths: 90 | - path: / 91 | pathType: Prefix 92 | service: 93 | name: main 94 | port: sonarr 95 | - host: &jellyseerrHost jellyseerr.khuedoan.com 96 | paths: 97 | - path: / 98 | pathType: Prefix 99 | service: 100 | name: main 101 | port: jellyseerr 102 | tls: 103 | - secretName: jellyfin-tls-certificate 104 | hosts: 105 | - *jellyfinHost 106 | - *transmissionHost 107 | - *prowlarrHost 108 | - *radarrHost 109 | - *sonarrHost 110 | - *jellyseerrHost 111 | persistence: 112 | data: 113 | accessMode: ReadWriteOnce 114 | size: 50Gi 115 | advancedMounts: 116 | main: 117 | main: 118 | - path: /config 119 | subPath: jellyfin/config 120 | - path: /media/movies 121 | subPath: movies 122 | - path: /media/shows 123 | subPath: shows 124 | transmission: 125 | - path: /config 126 | subPath: transmission/config 127 | - path: /downloads 128 | subPath: transmission/downloads 129 | prowlarr: 130 | - path: /config 131 | subPath: prowlarr/config 132 | radarr: 133 | - path: /config 134 | subPath: radarr/config 135 | - path: /downloads/complete 136 | subPath: transmission/downloads/complete 137 | - path: /movies 138 | subPath: movies 139 | sonarr: 140 | - path: /config 141 | subPath: sonarr/config 142 | - path: /downloads/complete 143 | subPath: transmission/downloads/complete 144 | - path: /shows 145 | subPath: shows 146 | jellyseerr: 147 | - path: /app/config 148 | subPath: jellyseerr/config 149 | -------------------------------------------------------------------------------- /apps/matrix/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: elementweb 3 | version: 0.0.0 4 | dependencies: 5 | - name: elementweb 6 | version: 0.0.6 7 | repository: https://locmai.github.io/charts # TODO switch to official chart 8 | - name: dendrite 9 | version: 0.13.5 10 | repository: https://matrix-org.github.io/dendrite 11 | -------------------------------------------------------------------------------- /apps/matrix/values.yaml: -------------------------------------------------------------------------------- 1 | elementweb: 2 | ingress: 3 | enabled: true 4 | className: nginx 5 | annotations: 6 | cert-manager.io/cluster-issuer: letsencrypt-prod 7 | external-dns.alpha.kubernetes.io/target: "homelab-tunnel.khuedoan.com" 8 | external-dns.alpha.kubernetes.io/cloudflare-proxied: "true" 9 | hosts: 10 | - host: &frontend_host chat.khuedoan.com 11 | paths: 12 | - path: / 13 | pathType: Prefix 14 | tls: 15 | - secretName: element-tls-certificate 16 | hosts: 17 | - *frontend_host 18 | config: 19 | default: 20 | base_url: https://matrix.khuedoan.com 21 | server_name: khuedoan.com 22 | 23 | dendrite: 24 | dendrite_config: 25 | global: 26 | server_name: matrix.khuedoan.com 27 | ingress: 28 | enabled: true 29 | className: nginx 30 | annotations: 31 | cert-manager.io/cluster-issuer: letsencrypt-prod 32 | hostName: matrix.khuedoan.com 33 | tls: 34 | - hosts: 35 | - matrix.khuedoan.com 36 | secretName: matrix-tls-certificate 37 | postgresql: 38 | enabled: true 39 | -------------------------------------------------------------------------------- /apps/ollama/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: ollama 3 | version: 0.0.0 4 | dependencies: 5 | - name: app-template 6 | version: 2.6.0 7 | repository: https://bjw-s.github.io/helm-charts 8 | -------------------------------------------------------------------------------- /apps/ollama/values.yaml: -------------------------------------------------------------------------------- 1 | app-template: 2 | controllers: 3 | main: 4 | containers: 5 | main: 6 | image: 7 | repository: docker.io/ollama/ollama 8 | tag: 0.1.29 9 | ui: 10 | containers: 11 | main: 12 | image: 13 | repository: ghcr.io/open-webui/open-webui 14 | tag: latest 15 | env: 16 | OLLAMA_BASE_URL: http://ollama:11434 17 | service: 18 | main: 19 | ports: 20 | http: 21 | port: 11434 22 | protocol: HTTP 23 | ui: 24 | controller: ui 25 | ports: 26 | http: 27 | port: 8080 28 | protocol: HTTP 29 | ingress: 30 | main: 31 | enabled: true 32 | className: nginx 33 | annotations: 34 | cert-manager.io/cluster-issuer: letsencrypt-prod 35 | hosts: 36 | - host: &ollamaHost ollama.khuedoan.com 37 | paths: 38 | - path: / 39 | pathType: Prefix 40 | service: 41 | name: main 42 | port: http 43 | - host: &uiHost ai.khuedoan.com 44 | paths: 45 | - path: / 46 | pathType: Prefix 47 | service: 48 | name: ui 49 | port: http 50 | tls: 51 | - hosts: 52 | - *ollamaHost 53 | - *uiHost 54 | secretName: ollama-tls-certificate 55 | persistence: 56 | data: 57 | accessMode: ReadWriteOnce 58 | size: 10Gi 59 | advancedMounts: 60 | main: 61 | main: 62 | - path: /root/.ollama 63 | -------------------------------------------------------------------------------- /apps/pairdrop/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: pairdrop 3 | version: 0.0.0 4 | dependencies: 5 | - name: app-template 6 | version: 2.6.0 7 | repository: https://bjw-s.github.io/helm-charts 8 | -------------------------------------------------------------------------------- /apps/pairdrop/values.yaml: -------------------------------------------------------------------------------- 1 | app-template: 2 | controllers: 3 | main: 4 | containers: 5 | main: 6 | image: 7 | repository: lscr.io/linuxserver/pairdrop 8 | tag: 1.10.7 9 | service: 10 | main: 11 | ports: 12 | http: 13 | port: 3000 14 | protocol: HTTP 15 | ingress: 16 | main: 17 | enabled: true 18 | className: nginx 19 | annotations: 20 | cert-manager.io/cluster-issuer: letsencrypt-prod 21 | hosts: 22 | - host: &host drop.khuedoan.com 23 | paths: 24 | - path: / 25 | pathType: Prefix 26 | service: 27 | name: main 28 | port: http 29 | tls: 30 | - hosts: 31 | - *host 32 | secretName: pairdrop-tls-certificate 33 | -------------------------------------------------------------------------------- /apps/paperless/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: paperless 3 | version: 0.0.0 4 | dependencies: 5 | - name: app-template 6 | version: 2.6.0 7 | repository: https://bjw-s.github.io/helm-charts 8 | -------------------------------------------------------------------------------- /apps/paperless/templates/secret.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: external-secrets.io/v1beta1 2 | kind: ExternalSecret 3 | metadata: 4 | name: {{ .Release.Name }}-secret 5 | namespace: {{ .Release.Namespace }} 6 | spec: 7 | secretStoreRef: 8 | kind: ClusterSecretStore 9 | name: global-secrets 10 | dataFrom: 11 | - extract: 12 | key: paperless.admin 13 | -------------------------------------------------------------------------------- /apps/paperless/values.yaml: -------------------------------------------------------------------------------- 1 | app-template: 2 | controllers: 3 | main: 4 | containers: 5 | main: 6 | image: 7 | repository: ghcr.io/paperless-ngx/paperless-ngx 8 | tag: 2.5.4 9 | env: 10 | PAPERLESS_PORT: 8000 11 | PAPERLESS_ADMIN_USER: admin 12 | PAPERLESS_URL: https://paperless.khuedoan.com 13 | envFrom: 14 | - secret: "{{ .Release.Name }}-secret" 15 | redis: 16 | image: 17 | repository: docker.io/library/redis 18 | tag: 7.2.4 19 | service: 20 | main: 21 | ports: 22 | http: 23 | port: 8000 24 | protocol: HTTP 25 | ingress: 26 | main: 27 | enabled: true 28 | className: nginx 29 | annotations: 30 | cert-manager.io/cluster-issuer: letsencrypt-prod 31 | hosts: 32 | - host: &host paperless.khuedoan.com 33 | paths: 34 | - path: / 35 | pathType: Prefix 36 | service: 37 | name: main 38 | port: http 39 | tls: 40 | - hosts: 41 | - *host 42 | secretName: paperless-tls-certificate 43 | persistence: 44 | data: 45 | accessMode: ReadWriteOnce 46 | size: 10Gi 47 | advancedMounts: 48 | main: 49 | main: 50 | - path: /usr/src/paperless/data 51 | subPath: data 52 | - path: /usr/src/paperless/media 53 | subPath: media 54 | -------------------------------------------------------------------------------- /apps/speedtest/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: speedtest 3 | version: 0.0.0 4 | dependencies: 5 | - name: app-template 6 | version: 2.6.0 7 | repository: https://bjw-s.github.io/helm-charts 8 | -------------------------------------------------------------------------------- /apps/speedtest/values.yaml: -------------------------------------------------------------------------------- 1 | app-template: 2 | controllers: 3 | main: 4 | containers: 5 | main: 6 | image: 7 | repository: docker.io/openspeedtest/latest 8 | tag: latest 9 | service: 10 | main: 11 | ports: 12 | http: 13 | port: 3000 14 | protocol: HTTP 15 | ingress: 16 | main: 17 | enabled: true 18 | className: nginx 19 | annotations: 20 | nginx.ingress.kubernetes.io/proxy-body-size: 50m 21 | cert-manager.io/cluster-issuer: letsencrypt-prod 22 | hosts: 23 | - host: &host speedtest.khuedoan.com 24 | paths: 25 | - path: / 26 | pathType: Prefix 27 | service: 28 | name: main 29 | port: http 30 | tls: 31 | - hosts: 32 | - *host 33 | secretName: speedtest-tls-certificate 34 | -------------------------------------------------------------------------------- /apps/tailscale/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: tailscale 3 | version: 0.0.0 4 | dependencies: 5 | - name: app-template 6 | version: 3.1.0 7 | repository: https://bjw-s.github.io/helm-charts 8 | -------------------------------------------------------------------------------- /apps/tailscale/templates/role.yaml: -------------------------------------------------------------------------------- 1 | # https://github.com/tailscale/tailscale/blob/main/docs/k8s/role.yaml 2 | # Copyright (c) Tailscale Inc & AUTHORS 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | apiVersion: rbac.authorization.k8s.io/v1 5 | kind: Role 6 | metadata: 7 | name: tailscale 8 | namespace: {{ .Release.Namespace }} 9 | rules: 10 | - apiGroups: [""] 11 | resources: ["secrets"] 12 | verbs: ["create"] 13 | - apiGroups: [""] 14 | resourceNames: ["tailscale"] 15 | resources: ["secrets"] 16 | verbs: ["get", "update", "patch"] 17 | -------------------------------------------------------------------------------- /apps/tailscale/templates/rolebinding.yaml: -------------------------------------------------------------------------------- 1 | # https://github.com/tailscale/tailscale/blob/main/docs/k8s/rolebinding.yaml 2 | # Copyright (c) Tailscale Inc & AUTHORS 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | apiVersion: rbac.authorization.k8s.io/v1 5 | kind: RoleBinding 6 | metadata: 7 | name: tailscale 8 | namespace: {{ .Release.Namespace }} 9 | subjects: 10 | - kind: ServiceAccount 11 | name: tailscale 12 | roleRef: 13 | kind: Role 14 | name: tailscale 15 | apiGroup: rbac.authorization.k8s.io 16 | -------------------------------------------------------------------------------- /apps/tailscale/templates/secret.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: external-secrets.io/v1beta1 2 | kind: ExternalSecret 3 | metadata: 4 | name: tailscale-auth 5 | namespace: {{ .Release.Namespace }} 6 | spec: 7 | secretStoreRef: 8 | kind: ClusterSecretStore 9 | name: global-secrets 10 | data: 11 | - secretKey: TS_AUTHKEY 12 | remoteRef: 13 | key: external 14 | property: tailscale-auth-key 15 | -------------------------------------------------------------------------------- /apps/tailscale/templates/serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | # https://github.com/tailscale/tailscale/blob/main/docs/k8s/sa.yaml 2 | # Copyright (c) Tailscale Inc & AUTHORS 3 | # SPDX-License-Identifier: BSD-3-Clause 4 | apiVersion: v1 5 | kind: ServiceAccount 6 | metadata: 7 | name: tailscale 8 | namespace: {{ .Release.Namespace }} 9 | -------------------------------------------------------------------------------- /apps/tailscale/values.yaml: -------------------------------------------------------------------------------- 1 | app-template: 2 | serviceAccount: 3 | name: tailscale 4 | controllers: 5 | tailscale: 6 | containers: 7 | app: 8 | image: 9 | repository: ghcr.io/tailscale/tailscale 10 | tag: latest 11 | env: 12 | TS_HOSTNAME: homelab-router 13 | TS_USERSPACE: false 14 | TS_KUBE_SECRET: tailscale 15 | TS_ROUTES: 192.168.1.224/27 16 | TS_AUTHKEY: 17 | valueFrom: 18 | secretKeyRef: 19 | name: tailscale-auth 20 | key: TS_AUTHKEY 21 | securityContext: 22 | capabilities: 23 | add: 24 | - NET_ADMIN 25 | -------------------------------------------------------------------------------- /apps/wireguard/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: wireguard 3 | version: 0.0.0 4 | dependencies: 5 | - name: app-template 6 | version: 3.5.0 7 | repository: https://bjw-s.github.io/helm-charts 8 | -------------------------------------------------------------------------------- /apps/wireguard/values.yaml: -------------------------------------------------------------------------------- 1 | app-template: 2 | controllers: 3 | wireguard: 4 | containers: 5 | app: 6 | image: 7 | repository: lscr.io/linuxserver/wireguard 8 | tag: latest 9 | env: 10 | LOG_CONFS: false 11 | USE_COREDNS: true 12 | securityContext: 13 | capabilities: 14 | add: 15 | - NET_ADMIN 16 | service: 17 | wireguard: 18 | controller: wireguard 19 | type: LoadBalancer 20 | ports: 21 | http: 22 | port: 51820 23 | protocol: UDP 24 | persistence: 25 | config: 26 | type: secret 27 | name: "{{ .Release.Name }}-secret" 28 | globalMounts: 29 | - path: /config/wg_confs 30 | rawResources: 31 | secret: 32 | apiVersion: external-secrets.io/v1beta1 33 | kind: ExternalSecret 34 | spec: 35 | spec: 36 | secretStoreRef: 37 | kind: ClusterSecretStore 38 | name: global-secrets 39 | data: 40 | - secretKey: WIREGUARD_PRIVATE_KEY 41 | remoteRef: 42 | key: external 43 | property: wireguard-private-key 44 | target: 45 | template: 46 | data: 47 | wg0.conf: | 48 | [Interface] 49 | Address = 172.16.0.1/32 50 | ListenPort = 51820 51 | PrivateKey = {{ `{{ .WIREGUARD_PRIVATE_KEY }}` }} 52 | PostUp = iptables -A FORWARD -i %i -j ACCEPT; iptables -A FORWARD -o %i -j ACCEPT; iptables -t nat -A POSTROUTING -o eth+ -j MASQUERADE 53 | PostDown = iptables -D FORWARD -i %i -j ACCEPT; iptables -D FORWARD -o %i -j ACCEPT; iptables -t nat -D POSTROUTING -o eth+ -j MASQUERADE 54 | 55 | # Note that WireGuard will ignore a peer whose public key matches 56 | # the interface's private key. So you can distribute a single 57 | # list of peers everywhere. 58 | # https://lists.zx2c4.com/pipermail/wireguard/2018-December/003703.html 59 | 60 | # Servers 61 | 62 | [Peer] 63 | # homelab 64 | PublicKey = sSAZS1Z3vB7Wx8e2yVqXfeHjgWTa80wnSYoma3mZkiU= 65 | AllowedIPs = 172.16.0.1/32, 192.168.1.100/32, 192.168.1.224/27 66 | 67 | [Peer] 68 | # horus 69 | PublicKey = zVwYqwvGn/IL7o6CD84y4/Y/OnRAUl/jw6T7DtNqWGM= 70 | Endpoint = horus.khuedoan.com:51820 71 | PersistentKeepalive = 25 72 | AllowedIPs = 172.16.0.2/32 73 | 74 | # Clients 75 | 76 | [Peer] 77 | # khuedoan-ryzentower 78 | PublicKey = qnDY23ZWUOlCRUmB8anpXH1uzX0A17F1YtQJmVi7GWM= 79 | AllowedIPs = 172.16.0.10/32 80 | 81 | [Peer] 82 | # khuedoan-thinkpadz13 83 | PublicKey = hSVWn2lBasJeueHApQYmYR0s2zNpXrqZX6F8gyEqpy0= 84 | AllowedIPs = 172.16.0.11/32 85 | 86 | [Peer] 87 | # khuedoan-phone 88 | PublicKey = nITHFdgTkNZOTWeSWqnGXjgwlCJMKRCnnUsjMx2yp2U= 89 | AllowedIPs = 172.16.0.12/32 90 | -------------------------------------------------------------------------------- /docs/.gitignore: -------------------------------------------------------------------------------- 1 | mermaid*.js 2 | -------------------------------------------------------------------------------- /docs/concepts/certificate-management.md: -------------------------------------------------------------------------------- 1 | # Certificate management 2 | 3 | Certificates are generated and managed by [cert-manager](https://cert-manager.io) with [Let's Encrypt](https://letsencrypt.org). 4 | By default certificates are valid for 90 days and will be renewed after 60 days. 5 | 6 | cert-manager watches `Ingress` resources across the cluster. When you create an `Ingress` with a [supported annotation](https://cert-manager.io/docs/usage/ingress/#supported-annotations): 7 | 8 | ```yaml hl_lines="5 13 14" 9 | apiVersion: networking.k8s.io/v1 10 | kind: Ingress 11 | metadata: 12 | annotations: 13 | cert-manager.io/cluster-issuer: letsencrypt-prod 14 | name: foo 15 | spec: 16 | rules: 17 | - host: foo.example.com 18 | # ... 19 | tls: 20 | - hosts: 21 | - foo.example.com 22 | secretName: foo-tls-certificate 23 | ``` 24 | 25 | ```mermaid 26 | flowchart LR 27 | User -- 6 --> Ingress 28 | 29 | subgraph cluster[Homelab cluster] 30 | Ingress --- Secret 31 | Ingress -. 1 .-> Certificate 32 | Certificate -. 5 .-> Secret 33 | Certificate -- 2 --> CertificateRequest -- 3 --> Order -- 4 --> Challenge 34 | end 35 | 36 | Order -.- ACMEServer[ACME server] 37 | 38 | subgraph dnsprovider[DNS provider] 39 | TXT 40 | end 41 | 42 | Challenge -- 4.a --> TXT 43 | ACMEServer -.- Challenge 44 | ACMEServer -. 4.b .-> TXT 45 | ``` 46 | 47 | 1. cert-manager creates a corresponding `Certificate` resources 48 | 2. Based on the `Certificate` resource, cert-manager creates a `CertificateRequest` resource to request a signed certificate from the configured `ClusterIssuer` 49 | 3. The `CertificateRequest` will create an order with an ACME server (we use Let's Encrypt), which is represented by the `Order` resource 50 | 4. Then cert-manager will perform a [DNS-01](https://cert-manager.io/docs/configuration/acme/dns01) `Challenge`: 51 | 1. Create a DNS TXT record (contains a computed key) 52 | 2. The ACME server retrieve this key via a DNS lookup and validate that we own the domain for the requested certificate 53 | 7. cert-manager stores the certificate (typically `tls.crt` and `tls.key`) in the `Secret` specified in the `Ingress` configuration 54 | 8. Now you can access the HTTPS website with a valid certificate 55 | 56 | A much more detailed diagram can be found in the official documentation under [certificate lifecycle](https://cert-manager.io/docs/concepts/certificate/#certificate-lifecycle). 57 | -------------------------------------------------------------------------------- /docs/concepts/development-shell.md: -------------------------------------------------------------------------------- 1 | # Development shell 2 | 3 | The development shell makes it easy to get all of the dependencies needed to interact with the homelab. 4 | 5 | ## Prerequisites 6 | 7 | !!! info 8 | 9 | NixOS users can skip this step. 10 | 11 | Install Nix using one of the following methods: 12 | 13 | - [Official Nix installer](https://nixos.org/download) 14 | - [Determinate Nix Installer](https://docs.determinate.systems/getting-started/#installer) 15 | 16 | If you're using the official installer, add the following to your 17 | `~/.config/nix/nix.conf` to enable [Flakes](https://nixos.wiki/wiki/Flakes): 18 | 19 | ```conf 20 | experimental-features = nix-command flakes 21 | ``` 22 | 23 | ## How to open it 24 | 25 | Run the following command: 26 | 27 | ```sh 28 | nix develop 29 | ``` 30 | 31 | It will open a shell with all the dependencies defined in `./flake.nix`: 32 | 33 | ``` 34 | [khuedoan@ryzentower:~/Documents/homelab]$ which kubectl 35 | /nix/store/0558zzzqynzw7rx9dp2i7jymvznd1cqx-kubectl-1.30.1/bin/kubectl 36 | ``` 37 | 38 | !!! tip 39 | 40 | If you have [`direnv`](https://direnv.net) installed, you can run `direnv 41 | allow` once and it will automatically enter the Nix shell every time you 42 | `cd` into the project. 43 | -------------------------------------------------------------------------------- /docs/concepts/pxe-boot.md: -------------------------------------------------------------------------------- 1 | # PXE boot 2 | 3 | ```mermaid 4 | flowchart TD 5 | subgraph controller[Initial controller] 6 | Ansible 7 | dhcp[DHCP server] 8 | tftp[TFTP server] 9 | http[HTTP server] 10 | end 11 | 12 | machine[Bare metal machine] 13 | 14 | Ansible -. 1 .-> machine 15 | machine <-. 2, 3 .-> dhcp 16 | machine <-. 4, 5 .-> tftp 17 | machine <-. 6, 7 .-> http 18 | ``` 19 | 20 | 1. Ansible: Hey MAC address `xx:xx:xx:xx:xx:xx`, wake up! 21 | 2. Machine: Hello everyone, I just woke up in network mode, could someone please show me how to boot? 22 | 3. DHCP server: I hear you, here's your IP address, proceed to the next server to obtain your bootloader. 23 | 4. Machine: Hello, could you please send me my bootloader? 24 | 5. TFTP server: Here you go. Grab your boot configuration, kernel, and initial ramdisk as well. 25 | 6. Machine: Hi, I just booted into my bootloader, and my boot parameters instructed me to get the installation instructions, packages, etc. from this site. 26 | 7. HTTP server: It's all yours. 27 | 8. Machine: Great, now I can install the OS and reboot! 28 | 29 | Here's how it looks like in action: 30 | 31 | 32 | -------------------------------------------------------------------------------- /docs/concepts/secrets-management.md: -------------------------------------------------------------------------------- 1 | # Secrets management 2 | 3 | ## Overview 4 | 5 | - Global secrets are stored in the `global-secrets` namespace. 6 | - Integrate with GitOps using [External Secrets Operator](https://external-secrets.io). 7 | - Secrets that can be generated are automatically generated and stored in the `global-secrets` namespace. 8 | 9 | !!! info 10 | 11 | Despite the name _External_ Secrets Operator, global secrets are created in the same cluster and synced 12 | to other namespaces using the [Kubernetes provider](https://external-secrets.io/latest/provider/kubernetes). 13 | 14 | While not supported by default in this project, you can also use other external providers such as HashiCorp Vault, 15 | AWS Secret Manager, Google Cloud Secret Manager, Azure Key Vault, 1Password, etc. 16 | 17 | ```mermaid 18 | flowchart TD 19 | subgraph global-secrets-namespace[global-secrets namespace] 20 | secret-generator[Secret Generator] -- generate if not exist --> source-secrets[Source Secrets] 21 | end 22 | 23 | subgraph app-namespace[application namespace] 24 | ExternalSecret -- create --> Secret 25 | App -- read --> Secret 26 | end 27 | 28 | ClusterSecretStore -- read --> source-secrets 29 | ExternalSecret --- ClusterSecretStore 30 | ``` 31 | 32 | ## Randomly generated secrets 33 | 34 | This is useful when you want to generate random secrets like admin password and store in global secrets. 35 | 36 | ```yaml title="./platform/global-secrets/files/secret-generator/config.yaml" hl_lines="2-6" 37 | --8<-- 38 | ./platform/global-secrets/files/secret-generator/config.yaml 39 | --8<-- 40 | ``` 41 | 42 | ## Extra third-party secrets 43 | 44 | For third-party secrets that you don't control, add them to `external/terraform.tfvars` under the `extra_secrets` key, 45 | then run `make external`. 46 | 47 | They will be available as a Secret named `external` in the `global-secrets` namespace. 48 | You can use it with `ExternalSecret` just like any other global secret. 49 | 50 | ## How secrets are pulled from global secrets to other namespaces 51 | 52 | When you apply an `ExternalSecret` object, for example: 53 | 54 | ```yaml hl_lines="4 21-23" 55 | apiVersion: external-secrets.io/v1beta1 56 | kind: ExternalSecret 57 | metadata: 58 | name: gitea-admin-secret 59 | namespace: gitea 60 | spec: 61 | data: 62 | - remoteRef: 63 | conversionStrategy: Default 64 | key: gitea.admin 65 | property: password 66 | secretKey: password 67 | refreshInterval: 1h 68 | secretStoreRef: 69 | kind: ClusterSecretStore 70 | name: global-secrets 71 | target: 72 | creationPolicy: Owner 73 | deletionPolicy: Retain 74 | template: 75 | data: 76 | password: '{{ .password }}' 77 | username: gitea_admin 78 | engineVersion: v2 79 | ``` 80 | 81 | This will create a corresponding Kubernetes secret: 82 | 83 | `kubectl describe secrets -n gitea gitea-admin-secret` 84 | 85 | ```yaml hl_lines="1 8-11" 86 | Name: gitea-admin-secret 87 | Namespace: gitea 88 | Labels: 89 | Annotations: reconcile.external-secrets.io/data-hash: 90 | 91 | Type: Opaque 92 | 93 | Data 94 | ==== 95 | password: 32 bytes 96 | username: 11 bytes 97 | ``` 98 | 99 | Please see the official documentation for more information: 100 | 101 | - [External Secrets Operator](https://external-secrets.io) 102 | - [API specification](https://external-secrets.io/latest/spec) 103 | -------------------------------------------------------------------------------- /docs/concepts/testing.md: -------------------------------------------------------------------------------- 1 | # Testing infrastructure code 2 | 3 | We use [Terratest](https://terratest.gruntwork.io) for automated tests. 4 | The tests are written in Go and can be found at `./test`. 5 | 6 | TODO: more docs here (PR welcomed) 7 | -------------------------------------------------------------------------------- /docs/getting-started/install-pre-commit-hooks.md: -------------------------------------------------------------------------------- 1 | # Install pre-commit hooks 2 | 3 | Git hook scripts are useful for identifying simple issues before commiting changes. 4 | 5 | Install [pre-commit](https://pre-commit.com/#install) first, one-liner for Arch users: 6 | 7 | ```sh 8 | sudo pacman -S python-pre-commit 9 | ``` 10 | 11 | Then install git hook scripts: 12 | 13 | ```sh 14 | make git-hooks 15 | ``` 16 | 17 | If you want to enable pre-commit on all repositories without enabling it manually, see [automatically enabling pre-commit on repositories](https://pre-commit.com/#automatically-enabling-pre-commit-on-repositories). 18 | -------------------------------------------------------------------------------- /docs/getting-started/user-onboarding.md: -------------------------------------------------------------------------------- 1 | # User onboarding 2 | 3 | === "For user" 4 | 5 | ## Create user 6 | 7 | Ask an admin to create your account, provide the following information: 8 | 9 | - [ ] Full name (John Doe) 10 | - [ ] Select a username (`johndoe`) 11 | - [ ] Email address (`johndoe@example.com`) 12 | 13 | ## Install companion apps 14 | 15 | For all users: 16 | 17 | - [ ] A password manager (I personally recommend [Bitwarden](https://bitwarden.com/download)) 18 | - [ ] A [Matrix chat client](https://matrix.org/clients) (optional, you can use the web version) 19 | 20 | For technical users: 21 | 22 | - [ ] [Docker](https://docs.docker.com/engine/install) 23 | - [ ] [Nix](https://nixos.org/download) and [direnv](https://direnv.net) (optional, but highly recommended) 24 | - [ ] [Lens](https://k8slens.dev) (optional, you can use the included `kubectl` or `k9s` command in the tools container) 25 | 26 | === "For admin" 27 | 28 | Run the following script: 29 | 30 | ```sh 31 | ./scripts/onboard-user johndoe "John Doe" "johndoe@example.com" 32 | ``` 33 | 34 | Let the user scan the QR code or follow the link to set up passkeys or password + TOTP. 35 | -------------------------------------------------------------------------------- /docs/getting-started/vpn-setup.md: -------------------------------------------------------------------------------- 1 | # VPN setup 2 | 3 | You can choose between [Tailscale](https://tailscale.com), 4 | [Wireguard](https://www.wireguard.com), or use both like me. I primarily use 5 | WireGuard but keep Tailscale as a backup for when the WireGuard server is down. 6 | 7 | ## Tailscale (requires third-party account) 8 | 9 | Get an [auth key](https://tailscale.com/kb/1085/auth-keys) from [Tailscale admin console](https://login.tailscale.com/admin/authkeys): 10 | 11 | - Description: homelab 12 | - Reusable: optionally set this to true 13 | 14 | Add it to `external/terraform.tfvars` as an extra secret: 15 | 16 | ```hcl 17 | extra_secrets = { 18 | tailscale-auth-key = "tskey-auth-myauthkeyhere" 19 | } 20 | ``` 21 | 22 | You may want to back up the `external/terraform.tfvars` file to a secure location. 23 | 24 | Apply the secret: 25 | 26 | ```sh 27 | make external 28 | ``` 29 | 30 | Finally, [enable subnet routes](https://tailscale.com/kb/1019/subnets#step-3-enable-subnet-routes-from-the-admin-console) for `homelab-router` 31 | from the [admin console](https://login.tailscale.com/admin/machines). 32 | 33 | You can now connect to your homelab via Tailscale and [invite user to your Tailscale network](https://tailscale.com/kb/1371/invite-users). 34 | 35 | ## Wireguard (requires port-forwarding) 36 | 37 | ### Prerequisites 38 | 39 | Find your public IP address using: 40 | 41 | ```sh 42 | curl -4 ifconfig.me 43 | ``` 44 | 45 | If you don’t have a static IP address, use dynamic DNS and replace the IP with 46 | your domain name. 47 | 48 | Next, configure port forwarding in your router for the WireGuard service. 49 | 50 | !!! example 51 | 52 | Each router is different, here's mine for reference: 53 | 54 | - Protocol: `UDP` 55 | - Start Port: `51820` 56 | - End Port: `51820` 57 | - Local IP Address: `192.168.1.226` (find it with `kubectl get service -n wireguard wireguard`) 58 | - Start Port Local: `51820` 59 | - End Port Local: `51820` 60 | 61 | Generate a key pair for the server: 62 | 63 | ```sh 64 | wg genkey | tee /dev/tty | wg pubkey 65 | ``` 66 | 67 | This will generate a private key and a public key, in that order. Add the 68 | private key to `external/terraform.tfvars` as an extra secret: 69 | 70 | ```hcl 71 | extra_secrets = { 72 | wireguard-private-key = "privatekeyhere" 73 | } 74 | ``` 75 | 76 | You may want to back up the `external/terraform.tfvars` file to a secure location. 77 | 78 | Apply the secret: 79 | 80 | ```sh 81 | make external 82 | ``` 83 | 84 | I use `172.16.0.0/12` as the private IP range for WireGuard, but you can choose 85 | any private IP address range you prefer in `./apps/wireguard/values.yaml`. I 86 | also recommend removing my peers and adding your own. 87 | 88 | ### Add a new device to the server 89 | 90 | !!! info 91 | 92 | Each device requires its own configuration. 93 | 94 | Generate a new key pair for the device. You can generate it for the user, or 95 | they can generate it themselves if they prefer to keep the private key 96 | confidential: 97 | 98 | ```sh 99 | wg genkey | tee /dev/tty | wg pubkey 100 | ``` 101 | 102 | This will generate a private key and a public key, in that order. The private 103 | key must be saved in a secure password manager, and save the public key for the 104 | next step. 105 | 106 | Update the list of peers in `./apps/wireguard/values.yaml`, make sure you 107 | replace all of my peers with yours. 108 | 109 | !!! example 110 | 111 | Example configuration for my phone: 112 | 113 | ```ini 114 | [Peer] 115 | PublicKey = nITHFdgTkNZOTWeSWqnGXjgwlCJMKRCnnUsjMx2yp2U= 116 | AllowedIPs = 172.16.0.12/32 117 | ``` 118 | 119 | - The public key is the one generated in the previous step. 120 | - `172.16.0.12/32` is the device's private IP address, manually selected from 121 | the `172.16.0.0/12` range mentioned above. 122 | 123 | ### Add the Wireguard config to the device 124 | 125 | Create a new configuration file for the device: 126 | 127 | ```ini 128 | [Interface] 129 | Address = /32 130 | PrivateKey = 131 | 132 | [Peer] 133 | PublicKey = 134 | Endpoint = :51820 135 | AllowedIPs = /32, 136 | ``` 137 | 138 | Replace placeholders with actual values and save as `wg0.conf`. 139 | 140 | !!! example 141 | 142 | Example configuration for my phone: 143 | 144 | ```ini 145 | [Interface] 146 | Address = 172.16.0.12/32 147 | PrivateKey = 148 | 149 | [Peer] 150 | PublicKey = sSAZS1Z3vB7Wx8e2yVqXfeHjgWTa80wnSYoma3mZkiU 151 | Endpoint = :51820 152 | AllowedIPs = 172.16.0.1/32, 192.168.1.224/27 153 | ``` 154 | 155 | The client can now import this configuration and connect to your WireGuard 156 | mesh. Make sure you clean up the `wg0.conf` file after importing it to the 157 | client. 158 | 159 | === "Mobile" 160 | 161 | Generate a QR code from the configuration file: 162 | 163 | ```sh 164 | qrencode -t ansiutf8 -r wg0.conf 165 | ``` 166 | 167 | Then scan the QR code using the official WireGuard app. 168 | 169 | === "Linux" 170 | 171 | Import the WireGuard configuration using NetworkManager: 172 | 173 | ```sh 174 | nmcli connection import type wireguard file wg0.conf 175 | ``` 176 | 177 | Activate the connection: 178 | 179 | ```sh 180 | nmcli connection up wg0 181 | ``` 182 | -------------------------------------------------------------------------------- /docs/how-to-guides/add-or-remove-nodes.md: -------------------------------------------------------------------------------- 1 | # Add or remove nodes 2 | 3 | Or how to scale vertically. To replace the same node with a clean OS, remove it and add it again. 4 | 5 | ## Add new nodes 6 | 7 | !!! tip 8 | 9 | You can add multiple nodes at the same time 10 | 11 | Add its details to the inventory **at the end of the group** (masters or workers): 12 | 13 | ```diff title="metal/inventories/prod.yml" 14 | diff --git a/metal/inventories/prod.yml b/metal/inventories/prod.yml 15 | index 7f6474a..1bb2cbc 100644 16 | --- a/metal/inventories/prod.yml 17 | +++ b/metal/inventories/prod.yml 18 | @@ -8,3 +8,4 @@ metal: 19 | workers: 20 | hosts: 21 | metal3: {ansible_host: 192.168.1.113, mac: '00:23:24:d1:f5:69', disk: sda, network_interface: eno1} 22 | + metal4: {ansible_host: 192.168.1.114, mac: '00:11:22:33:44:55', disk: sda, network_interface: eno1} 23 | ``` 24 | 25 | Install the OS and join the cluster: 26 | 27 | ``` 28 | make metal 29 | ``` 30 | 31 | That's it! 32 | 33 | ## Remove a node 34 | 35 | !!! danger 36 | 37 | It is recommended to remove nodes one at a time 38 | 39 | Remove it from the inventory: 40 | 41 | ```diff title="metal/inventories/prod.yml" 42 | diff --git a/metal/inventories/prod.yml b/metal/inventories/prod.yml 43 | index 7f6474a..d12b50a 100644 44 | --- a/metal/inventories/prod.yml 45 | +++ b/metal/inventories/prod.yml 46 | @@ -4,7 +4,6 @@ metal: 47 | hosts: 48 | metal0: {ansible_host: 192.168.1.110, mac: '00:23:24:d1:f3:f0', disk: sda, network_interface: eno1} 49 | metal1: {ansible_host: 192.168.1.111, mac: '00:23:24:d1:f4:d6', disk: sda, network_interface: eno1} 50 | - metal2: {ansible_host: 192.168.1.112, mac: '00:23:24:e7:04:60', disk: sda, network_interface: eno1} 51 | workers: 52 | hosts: 53 | metal3: {ansible_host: 192.168.1.113, mac: '00:23:24:d1:f5:69', disk: sda, network_interface: eno1} 54 | ``` 55 | 56 | Drain the node: 57 | 58 | ```sh 59 | kubectl drain ${NODE_NAME} --delete-emptydir-data --ignore-daemonsets --force 60 | ``` 61 | 62 | Remove the node from the cluster 63 | 64 | ```sh 65 | kubectl delete node ${NODE_NAME} 66 | ``` 67 | 68 | Shutdown the node: 69 | 70 | ``` 71 | ssh root@${NODE_IP} poweroff 72 | ``` 73 | -------------------------------------------------------------------------------- /docs/how-to-guides/alternate-dns-setup.md: -------------------------------------------------------------------------------- 1 | # Alternate DNS setup 2 | 3 | !!! info 4 | 5 | Skip this step if you already use the included Cloudflare setup 6 | 7 | Before you can access the home page at , you'll need to update your DNS config. 8 | 9 | Some options for DNS config (choose one): 10 | 11 | - Change the DNS config at your domain registrar (already included and automated) 12 | - Change the DNS config in your router (also works if you don't own a domain) 13 | - Use [nip.io](https://nip.io) (suitable for a test environment) 14 | 15 | ## At your domain registrar (recommended) 16 | 17 | The default configuration is for Cloudflare DNS, but you can change the code to use other providers. 18 | 19 | ## In your router 20 | 21 | !!! tip 22 | 23 | If you don't have a domain, you can use the `home.arpa` domain (according to [RFC-8375](https://datatracker.ietf.org/doc/html/rfc8375)). 24 | 25 | You can add each subdomain one by one, or use a wildcard `*.example.com` and point it to the IP address of the load balancer. 26 | To acquire a list of subdomains and their addresses, use this command: 27 | 28 | ```sh 29 | ./scripts/get-dns-config 30 | ``` 31 | 32 | ## Use [nip.io](https://nip.io) 33 | 34 | Preconfigured in the `dev` branch. 35 | -------------------------------------------------------------------------------- /docs/how-to-guides/backup-and-restore.md: -------------------------------------------------------------------------------- 1 | # Backup and restore 2 | 3 | ## Prerequisites 4 | 5 | Create an S3 bucket to store backups. You can use AWS S3, Minio, or 6 | any other S3-compatible provider. 7 | 8 | - For AWS S3, your bucket URL might look something like this: 9 | `https://s3.amazonaws.com/my-homelab-backup`. 10 | - For Minio, your bucket URL might look something like this: 11 | `https://my-s3-host.example.com/homelab-backup`. 12 | 13 | Follow your provider's documentation to create a service account with the 14 | following policy (replace `my-homelab-backup` with your actual bucket name): 15 | 16 | ```json 17 | { 18 | "Version": "2012-10-17", 19 | "Statement": [ 20 | { 21 | "Effect": "Allow", 22 | "Action": [ 23 | "s3:GetObject", 24 | "s3:PutObject", 25 | "s3:DeleteObject", 26 | "s3:ListBucket" 27 | ], 28 | "Resource": [ 29 | "arn:aws:s3:::my-homelab-backup", 30 | "arn:aws:s3:::my-homelab-backup/*" 31 | ] 32 | } 33 | ] 34 | } 35 | ``` 36 | 37 | Save the access key and secret key to a secure location, such as a password 38 | manager. While you're at it, generate a new password for Restic encryption and 39 | save it there as well. 40 | 41 | !!! example 42 | 43 | I use Minio for my homelab backups. Here's how I set it up: 44 | 45 | - Create a bucket named `homelab-backup`. 46 | - Create a service account under Identity -> Service Accounts -> Create 47 | Service Account: 48 | - Enable Restrict beyond user policy. 49 | - Paste the policy above. 50 | - Click Create and copy the access key and secret key 51 | - I also set up Minio replication to store backups in two locations: one in 52 | my house and one remotely. 53 | 54 | ## Add backup credentials to global secrets 55 | 56 | Add the following to `external/terraform.tfvars`: 57 | 58 | ```hcl 59 | extra_secrets = { 60 | restic-password = "xxxxxxxxxxxxxxxxxxxxxxxx" 61 | restic-s3-bucket = "https://s3.amazonaws.com/my-homelab-backup-xxxxxxxxxx" 62 | restic-s3-access-key = "xxxxxxxxxxxxxxxx" 63 | restic-s3-secret-key = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" 64 | } 65 | ``` 66 | 67 | Then apply the changes: 68 | 69 | ```sh 70 | make external 71 | ``` 72 | 73 | You may want to back up the `external/terraform.tfvars` file to a secure location as well. 74 | 75 | ## Add backup configuration for volumes 76 | 77 | !!! warning 78 | 79 | Do not run the backup command when building a new cluster where you intend 80 | to restore backups, as it may overwrite existing backup data. To restore 81 | data on a new cluster, refer to the [restore from 82 | backup](#restore-from-backup) section. 83 | 84 | For now, you need to run a command to opt-in volumes until we have a better 85 | GitOps solution: 86 | 87 | ```sh 88 | make backup 89 | ``` 90 | 91 | This command will set up Restic repositories and back up the volumes configured 92 | in `./Makefile`. You can adjust the list there to add or remove volumes from the 93 | backup. You only need to run this command once, the backup configuration will 94 | be stored in the cluster and run on a schedule. 95 | 96 | ## Restore from backup 97 | 98 | The restore process is ad-hoc, you need to run a command to restore application volumes: 99 | 100 | ```sh 101 | make restore 102 | ``` 103 | 104 | The command above will restore the latest backup of recommended volumes. Like 105 | with backups, you can modify `./Makefile` to adjust the list of volumes you 106 | want to restore. 107 | -------------------------------------------------------------------------------- /docs/how-to-guides/disable-dhcp-proxy-in-dnsmasq.md: -------------------------------------------------------------------------------- 1 | # Disable DHCP proxy in dnsmasq 2 | 3 | ## Overview 4 | 5 | Dnsmasq is used as either a DHCP server or DHCP proxy server for PXE metal provisioning. 6 | 7 | Proxy mode is enabled by default allowing the use of existing DHCP servers on the network. 8 | A good description on how DHCP Proxy works can be found on the related [FOG project wiki page](https://wiki.fogproject.org/wiki/index.php?title=ProxyDHCP_with_dnsmasq). 9 | 10 | ## Disabling Proxy mode 11 | 12 | Certain scenarios will require this project to use a DHCP server, such as an air-gap deployment or dedicated VLAN. 13 | 14 | To disable proxy mode thereby using dnsmasq as a DHCP server, modify `metal/roles/pxe_server/defaults/main.yml` and set `dhcp_proxy` to `false`. 15 | -------------------------------------------------------------------------------- /docs/how-to-guides/expose-services-to-the-internet.md: -------------------------------------------------------------------------------- 1 | # Expose services to the internet 2 | 3 | !!! info 4 | 5 | This tutorial is for Cloudflare Tunnel users, please skip if you use port forwarding. 6 | 7 | Apply the `./external` layer to create a tunnel if you haven't already, 8 | then add the following annotations to your `Ingress` object (replace `example.com` with your domain): 9 | 10 | ```yaml 11 | apiVersion: networking.k8s.io/v1 12 | kind: Ingress 13 | metadata: 14 | annotations: 15 | external-dns.alpha.kubernetes.io/target: "homelab-tunnel.example.com" 16 | external-dns.alpha.kubernetes.io/cloudflare-proxied: "true" 17 | # ... 18 | ``` 19 | -------------------------------------------------------------------------------- /docs/how-to-guides/media-management.md: -------------------------------------------------------------------------------- 1 | # Media management 2 | 3 | !!! warning 4 | 5 | This is for educational purposes only :wink: Use it at your own risk! 6 | 7 | ## Initial setup 8 | 9 | - Jellyfin `https://jellyfin.example.com`: 10 | - Create an `admin` user and save the credentials to your password manager 11 | - Add media libraries: 12 | - Movies at `/media/movies` 13 | - Shows at `/media/shows` 14 | - Radarr `https://radarr.example.com`: 15 | - Authentication method: Forms 16 | - Create an `admin` user and save the credentials to your password manager 17 | - Navigate to Settings -> Download Clients -> Add -> Transmission (you can keep the default address and port) 18 | - Navigate to Settings -> Media Management -> Add Root Folder `/movies` 19 | - Navigate to Settings -> General -> API Key -> copy it for the next steps (or save it to your password manager) 20 | - Sonarr `https://sonarr.example.com`: same as Radarr but use `/shows` for the root folder 21 | - Prowlarr `https://prowlarr.example.com`: 22 | - Authentication method: Forms 23 | - Create an `admin` user and save the credentials to your password manager 24 | - Navigate to Settings -> Apps -> Add: 25 | - Radarr: paste the API key (you can keep the default address and port) 26 | - Sonarr: same as Radarr 27 | - Go back to Indexers -> Add New Indexers 28 | - Jellyseerr `https://jellyseerr.example.com`: 29 | - Sign In 30 | - Use your Jellyfin account 31 | - URL: `https://jellyfin.example.com` 32 | - Email: you can enter anything 33 | - Username: `admin` 34 | - Password: same as Jellyfin 35 | - Configure Media Server 36 | - Enable Movies and Shows 37 | - Configure Services: 38 | - Add Radarr Server: 39 | - Default Server: true 40 | - Name: Radarr 41 | - Hostname: localhost 42 | - Port: use default 43 | - API Key: from previous step 44 | - Click Test 45 | - Quality Profile: choose whatever suits you 46 | - Root folder: `/movies` 47 | - External URL: `https://radarr.example.com` 48 | - Enable Scan: true 49 | - Add Sonarr Server: similar to Radarr 50 | 51 | Optionally, for convenience, you can add a `guest` account without a password in Jellyfin, 52 | allow access to all libraries, and allow that account to manage requests in Jellyseerr. 53 | 54 | ## Usage 55 | 56 | Here's a suggested flow: 57 | 58 | - Users using the `guest` account can request media in Jellyseerr 59 | - Admins approve the request (or you can enable auto-approve) 60 | - Wait for the media to be downloaded 61 | - Watch on Jellyfin 62 | 63 | You may need to increase the volume size depending on your usage. 64 | -------------------------------------------------------------------------------- /docs/how-to-guides/run-commands-on-multiple-nodes.md: -------------------------------------------------------------------------------- 1 | # Run commands on multiple nodes 2 | 3 | Use [ansible-console](https://docs.ansible.com/ansible/latest/cli/ansible-console.html): 4 | 5 | ```sh 6 | cd metal 7 | make console 8 | ``` 9 | 10 | Then enter the command(s) you want to run. 11 | 12 | !!! example 13 | 14 | `root@all (4)[f:5]$ uptime` 15 | 16 | ```console 17 | metal0 | CHANGED | rc=0 >> 18 | 10:52:02 up 2 min, 1 user, load average: 0.17, 0.15, 0.06 19 | metal1 | CHANGED | rc=0 >> 20 | 10:52:02 up 2 min, 1 user, load average: 0.14, 0.11, 0.04 21 | metal3 | CHANGED | rc=0 >> 22 | 10:52:02 up 2 min, 1 user, load average: 0.03, 0.02, 0.00 23 | metal2 | CHANGED | rc=0 >> 24 | 10:52:02 up 2 min, 1 user, load average: 0.06, 0.06, 0.02 25 | ``` 26 | -------------------------------------------------------------------------------- /docs/how-to-guides/single-node-cluster-adjustments.md: -------------------------------------------------------------------------------- 1 | # Single node cluster adjustments 2 | 3 | !!! danger 4 | 5 | This is not officially supported and I don't regularly test it, 6 | I highly recommend using multiple nodes. 7 | 8 | Using a single node could lead to data loss unless your backup strategy is rock solid, 9 | make sure you are **ABSOLUTELY CERTAIN** this is what you want. 10 | 11 | Update the following changes, then commit and push. 12 | 13 | ## Remove storage redundancy 14 | 15 | Set pod counts and number of data copies to `1`: 16 | 17 | ```yaml title="system/rook-ceph/values.yaml" hl_lines="4 6 11 12 18 22 25" 18 | rook-ceph-cluster: 19 | cephClusterSpec: 20 | mon: 21 | count: 1 22 | mgr: 23 | count: 1 24 | cephBlockPools: 25 | - name: standard-rwo 26 | spec: 27 | replicated: 28 | size: 1 29 | requireSafeReplicaSize: false 30 | cephFileSystems: 31 | - name: standard-rwx 32 | spec: 33 | metadataPool: 34 | replicated: 35 | size: 1 36 | dataPools: 37 | - name: data0 38 | replicated: 39 | size: 1 40 | metadataServer: 41 | activeCount: 1 42 | activeStandby: false 43 | ``` 44 | 45 | ## Disable automatic upgrade 46 | 47 | Because they will try to drain the only node, the pods will have no place to go. 48 | Remove them entirely: 49 | 50 | ```sh 51 | rm -rf system/kured 52 | ``` 53 | 54 | Commit and push the change. 55 | You can revert it later when you add more nodes. 56 | -------------------------------------------------------------------------------- /docs/how-to-guides/troubleshooting/pxe-boot.md: -------------------------------------------------------------------------------- 1 | # PXE boot 2 | 3 | ## PXE server logs 4 | 5 | To view PXE server (includes DHCP, TFTP and HTTP server) logs: 6 | 7 | ```sh 8 | ./scripts/pxe-logs 9 | ``` 10 | 11 | !!! tip 12 | 13 | You can view the logs of one or more containers selectively, for example: 14 | 15 | ```sh 16 | ./scripts/pxe-logs dnsmasq 17 | ./scripts/pxe-logs http 18 | ``` 19 | 20 | ## Nodes not booting from the network 21 | 22 | - Plug a monitor and a keyboard to one of the bare metal node if possible to make the debugging process easier 23 | - Check if the controller (PXE server) is on the same subnet with bare metal nodes (sometimes Wifi will not work or conflict with wired Ethernet, try to turn it off) 24 | - Check if bare metal nodes are configured to boot from the network 25 | - Check if Wake-on-LAN is enabled 26 | - Check if the operating system ISO file is mounted 27 | - Check the controller's firewall config to make sure that the following ports are open: 28 | - DHCP (67/68) 29 | - TFTP (69) 30 | - HTTP (80) 31 | - Check PXE server Docker logs 32 | - Check if the servers are booting to the correct OS (Fedora Server installer instead of the previously installed OS), if not try to select it manually or remove the previous OS boot entry 33 | - Examine the network boot process with [Wireshark](https://www.wireshark.org) or [Termshark](https://termshark.io) 34 | -------------------------------------------------------------------------------- /docs/how-to-guides/updating-documentation.md: -------------------------------------------------------------------------------- 1 | # Updating documentation (this website) 2 | 3 | This project uses the [Diátaxis](https://diataxis.fr) technical documentation framework. 4 | The website is generated using [Material for MkDocs](https://squidfunk.github.io/mkdocs-material) and can be viewed at [homelab.khuedoan.com](https://homelab.khuedoan.com). 5 | 6 | There are 4 main parts: 7 | 8 | - [Getting started (tutorials)](https://diataxis.fr/tutorials): learning-oriented 9 | - [Concepts (explanation)](https://diataxis.fr/explanation): understanding-oriented 10 | - [How-to guides](https://diataxis.fr/how-to-guides): goal-oriented 11 | - [Reference](https://diataxis.fr/reference): information-oriented 12 | 13 | ## Local development 14 | 15 | To edit and view locally, run: 16 | 17 | ```sh 18 | make docs 19 | ``` 20 | 21 | Then visit [localhost:8000](http://localhost:8000) 22 | 23 | ## Deployment 24 | 25 | It's running on my other cluster in the [khuedoan/horus](https://github.com/khuedoan/horus) project 26 | (so if the homelab goes down everyone can still read the documentation). 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | -------------------------------------------------------------------------------- /docs/how-to-guides/use-both-github-and-gitea.md: -------------------------------------------------------------------------------- 1 | # Use both GitHub and Gitea 2 | 3 | Even though we self-host Gitea, you may still want to use GitHub as a backup and for discovery. 4 | 5 | Add both push URLs (replace my repositories with yours): 6 | 7 | ```sh 8 | git remote set-url --add --push origin git@git.khuedoan.com:ops/homelab 9 | git remote set-url --add --push origin git@github.com:khuedoan/homelab 10 | ``` 11 | 12 | Now you can just run `git push` like usual and it will push to both GitHub and Gitea. 13 | -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | --8<-- 2 | README.md 3 | --8<-- 4 | -------------------------------------------------------------------------------- /docs/installation/post-installation.md: -------------------------------------------------------------------------------- 1 | # Post-installation 2 | 3 | ## Backup secrets 4 | 5 | Save the following files to a safe location like a password manager (if you're using the sandbox, you can skip this step): 6 | 7 | - `~/.ssh/id_ed25519` 8 | - `~/.ssh/id_ed25519.pub` 9 | - `./metal/kubeconfig.yaml` 10 | - `~/.terraform.d/credentials.tfrc.json` 11 | - `./external/terraform.tfvars` 12 | 13 | ## Admin credentials 14 | 15 | - ArgoCD: 16 | - Username: `admin` 17 | - Password: run `./scripts/argocd-admin-password` 18 | - Gitea: 19 | - Username: `gitea_admin` 20 | - Password: get from `global-secrets` namespace 21 | - Kanidm: 22 | - Usernames: `admin` and `idm_admin` 23 | - Password: run `./scripts/kanidm-reset-password admin` and `./scripts/kanidm-reset-password idm_admin` 24 | - Jellyfin and other applications in the \*arr stack: see the [dedicated guide for media management](../how-to-guides/media-management.md) 25 | - Other apps: 26 | - Username: `admin` 27 | - Password: get from `global-secrets` namespace 28 | 29 | ## Backup 30 | 31 | Now is a good time to set up backups for your homelab. 32 | Follow the [backup and restore guide](../how-to-guides/backup-and-restore.md) to get started. 33 | 34 | ## Run the full test suite 35 | 36 | After the homelab has been stabilized, you can run the full test suite to ensure that everything is working properly: 37 | 38 | ```sh 39 | make test 40 | ``` 41 | 42 | !!! info 43 | 44 | The "full" test suit is still in its early stages, so any contribution is greatly appreciated. 45 | -------------------------------------------------------------------------------- /docs/installation/production/configuration.md: -------------------------------------------------------------------------------- 1 | # Configuration 2 | 3 | Open the [development shell](../../concepts/development-shell.md), which includes all the tools needed: 4 | 5 | ```sh 6 | nix develop 7 | ``` 8 | 9 | Run the following script to configure the homelab: 10 | 11 | ```sh 12 | make configure 13 | ``` 14 | 15 | !!! example 16 | 17 | 18 | 19 | ``` 20 | Text editor (nvim): 21 | Enter seed repo (github.com/khuedoan/homelab): github.com/example/homelab 22 | Enter your domain (khuedoan.com): example.com 23 | ``` 24 | 25 | It will prompt you to edit the inventory: 26 | 27 | - IP address: the desired one, not the current one, since your servers have no operating system installed yet 28 | - Disk: based on `/dev/$DISK`, in my case it's `sda`, but yours can be `sdb`, `nvme0n1`... 29 | - Network interface: usually it's `eth0`, mine is `eno1` 30 | - MAC address: the **lowercase, colon separated** MAC address of the above network interface 31 | 32 | !!! example 33 | 34 | ```yaml title="metal/inventories/prod.yml" 35 | --8<-- 36 | metal/inventories/prod.yml 37 | --8<-- 38 | ``` 39 | 40 | At the end it will show what has changed. After examining the diff, commit and push the changes. 41 | -------------------------------------------------------------------------------- /docs/installation/production/deployment.md: -------------------------------------------------------------------------------- 1 | # Deployment 2 | 3 | Open the development shell if you haven't already: 4 | 5 | ```sh 6 | nix develop 7 | ``` 8 | 9 | Build the lab: 10 | 11 | ```sh 12 | make 13 | ``` 14 | 15 | Yes it's that simple! 16 | 17 | !!! example 18 | 19 | 20 | 21 | It will take a while to download everything, 22 | you can read the [architecture document](../../reference/architecture/overview.md) while waiting for the deployment to complete. 23 | -------------------------------------------------------------------------------- /docs/installation/production/external-resources.md: -------------------------------------------------------------------------------- 1 | # External resources 2 | 3 | !!! info 4 | 5 | These resources are optional, the homelab still works without them but will lack some features like trusted certificates and offsite backup 6 | 7 | Although I try to keep the amount of external resources to the minimum, there's still need for a few of them. 8 | Below is a list of external resources and why we need them (also see some [alternatives](#alternatives) below). 9 | 10 | | Provider | Resource | Purpose | 11 | | -------- | -------- | ------- | 12 | | Terraform Cloud | Workspace | Terraform state backend | 13 | | Cloudflare | DNS | DNS and [DNS-01 challenge](https://letsencrypt.org/docs/challenge-types/#dns-01-challenge) for certificates | 14 | | Cloudflare | Tunnel | Public services to the internet without port forwarding | 15 | | ntfy | Topic | External notification service to receive alerts | 16 | 17 | ## Create credentials 18 | 19 | You'll be asked to provide these credentials on first build. 20 | 21 | ### Create Terraform workspace 22 | 23 | Terraform is stateful, which means it needs somewhere to store its state. Terraform Cloud is one option for a state backend with a generous free tier, perfect for a homelab. 24 | 25 | 1. Sign up for a [Terraform Cloud](https://cloud.hashicorp.com/products/terraform) account 26 | 2. Create a workspace named `homelab-external`, this is the workspace where your homelab state will be stored. 27 | 3. Change the "Execution Mode" from "Remote" to "Local". This will ensure your local machine, which can access your lab, is the one executing the Terraform plan rather than the cloud runners. 28 | 29 | If you decide to use a [different Terraform backend](https://www.terraform.io/language/settings/backends#available-backends), you'll need to edit the `external/versions.tf` file as required. 30 | 31 | ### Cloudflare 32 | 33 | - Buy a domain and [transfer it to Cloudflare](https://developers.cloudflare.com/registrar/get-started/transfer-domain-to-cloudflare) if you haven't already 34 | - Get Cloudflare email and account ID 35 | - Global API key: 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | ### ntfy 60 | 61 | - Choose a topic name like (treat it like you password) 62 | 63 | ## Alternatives 64 | 65 | To avoid vendor lock-in, each external provider must have an equivalent alternative that is easy to replace: 66 | 67 | - Terraform Cloud: 68 | - Any other [Terraform backends](https://www.terraform.io/language/settings/backends) 69 | - Cloudflare DNS: 70 | - Update cert-manager and external-dns to use a different provider 71 | - [Alternate DNS setup](../../how-to-guides/alternate-dns-setup.md) 72 | - Cloudflare Tunnel: 73 | - Use port forwarding if it's available 74 | - Create a small VPS in the cloud and utilize Wireguard to route traffic via it 75 | - Access everything via VPN 76 | - See also [awesome tunneling](https://github.com/anderspitman/awesome-tunneling) 77 | - ntfy: 78 | - [Self-host your own ntfy server](https://docs.ntfy.sh/install) 79 | - Any other [integration supported by Grafana Alerting](https://grafana.com/docs/grafana/latest/alerting/alerting-rules/manage-contact-points/integrations/#list-of-supported-integrations) 80 | -------------------------------------------------------------------------------- /docs/installation/production/prerequisites.md: -------------------------------------------------------------------------------- 1 | # Prerequisites 2 | 3 | ## Fork this repository 4 | 5 | Because [this project](https://github.com/khuedoan/homelab) applies GitOps practices, 6 | it's the source of truth for _my_ homelab, so you'll need to fork it to make it yours: 7 | 8 | [:fontawesome-solid-code-fork: Fork khuedoan/homelab](https://github.com/khuedoan/homelab/fork){ .md-button } 9 | 10 | By using this project you agree to [the license](../../reference/license.md). 11 | 12 | 13 | !!! summary "License TL;DR" 14 | 15 | - This project is free to use for any purpose, but it comes with no warranty 16 | - You must use the same [GPLv3 license](https://www.gnu.org/licenses/gpl-3.0.en.html) in `LICENSE.md` 17 | - You must keep the copy right notice and/or include an acknowledgement 18 | - Your project must remain open-source 19 | 20 | ## Hardware requirements 21 | 22 | ### Initial controller 23 | 24 | !!! info 25 | 26 | The initial controller is the machine used to bootstrap the cluster, we only need it once, you can use your laptop or desktop 27 | 28 | - A **Linux** machine that can run Docker (because the `host` networking driver used for PXE boot [only supports Linux](https://docs.docker.com/network/host/), you can use a Linux virtual machine with bridged networking if you're on macOS or Windows). 29 | 30 | ### Servers 31 | 32 | Any modern `x86_64` computer(s) should work, you can use old PCs, laptops or servers. 33 | 34 | !!! info 35 | 36 | This is the requirements for _each_ node 37 | 38 | | Component | Minimum | Recommended | 39 | | :-- | :-- | :-- | 40 | | CPU | 2 cores | 4 cores | 41 | | RAM | 8 GB | 16 GB | 42 | | Hard drive | 128 GB | 512 GB (depending on your storage usage, the base installation will not use more than 128GB) | 43 | | Node count | 1 (checkout the [single node cluster adjustments](../../how-to-guides/single-node-cluster-adjustments.md) tutorial) | 3 or more for high availability | 44 | 45 | Additional capabilities: 46 | 47 | - Ability to boot from the network (PXE boot) 48 | - Wake-on-LAN capability, used to wake the machines up automatically without physically touching the power button 49 | 50 | ### Network setup 51 | 52 | - All servers must be connected to the same **wired** network with the initial controller 53 | - You have the access to change DNS config (on your router or at your domain registrar) 54 | 55 | ## Domain 56 | 57 | Buying a domain is highly recommended, but if you don't have one, see [alternate DNS setup](../../how-to-guides/alternate-dns-setup.md). 58 | 59 | ## BIOS setup 60 | 61 | !!! info 62 | 63 | You need to do it once per machine if the default config is not sufficent, 64 | usually for consumer hardware this can not be automated 65 | (it requires something like [IPMI](https://en.wikipedia.org/wiki/Intelligent_Platform_Management_Interface) to automate). 66 | 67 | Common settings: 68 | 69 | - Enable Wake-on-LAN (WoL) and network boot 70 | - Use UEFI mode and disable CSM (legacy) mode 71 | - Disable secure boot 72 | 73 | Boot order options (select one, each has their pros and cons): 74 | 75 | 1. Only boot from the network if no operating system found: works on most hardware but you need to manually wipe your hard drive or delete the existing boot record for the current OS 76 | 2. Prefer booting from the network if turned on via WoL: more convenience but your BIOS must support it, and you must test it throughly to ensure you don't accidentally wipe your servers 77 | 78 | !!! example 79 | 80 | Below is my BIOS setup for reference. Your motherboard may have a different name for the options, so you'll need to adapt it to your hardware. 81 | 82 | ```yaml 83 | Devices: 84 | NetworkSetup: 85 | PXEIPv4: true 86 | PXEIPv6: false 87 | Advanced: 88 | CPUSetup: 89 | VT-d: true 90 | Power: 91 | AutomaticPowerOn: 92 | WoL: Automatic # Use network boot if Wake-on-LAN 93 | Security: 94 | SecureBoot: false 95 | Startup: 96 | CSM: false 97 | ``` 98 | 99 | ## Gather information 100 | 101 | - [ ] MAC address for each machine 102 | - [ ] OS disk name (for example `/dev/sda`) 103 | - [ ] Network interface name (for example `eth0`) 104 | - [ ] Choose a static IP address for each machine (just the desired address, we don't set anything up yet) 105 | -------------------------------------------------------------------------------- /docs/installation/sandbox.md: -------------------------------------------------------------------------------- 1 | # Development sandbox 2 | 3 | The sandbox is intended for trying out the homelab without any hardware or testing changes before applying them to the production environment. 4 | 5 | ## Prerequisites 6 | 7 | Host machine: 8 | 9 | - Recommended hardware specifications: 10 | - CPU: 4 cores 11 | - RAM: 16 GiB 12 | - OS: Linux (Windows and macOS are untested, please let me know if it works) 13 | - Available ports: `80` and `443` 14 | 15 | Install the following packages: 16 | 17 | - `docker` 18 | - `nix` (see [development shell](../concepts/development-shell.md) for the installation guide) 19 | 20 | Clone the repository and checkout the development branch: 21 | 22 | ```sh 23 | git clone https://github.com/khuedoan/homelab 24 | git checkout dev 25 | ``` 26 | 27 | ## Build 28 | 29 | Open the development shell, which includes all the tools needed: 30 | 31 | ```sh 32 | nix develop 33 | ``` 34 | 35 | Build a development cluster and bootstrap it: 36 | 37 | ``` 38 | make 39 | ``` 40 | 41 | !!! note 42 | 43 | It will take about 15 to 30 minutes to build depending on your internet connection 44 | 45 | ## Explore 46 | 47 | The homepage should be available at (ignore the security warning because we don't have valid certificates). 48 | 49 | See [admin credentials](../post-installation/#admin-credentials) for default passwords. 50 | 51 | If you want to make some changes, simply commit to the local `dev` branch and push it to Gitea in the sandbox: 52 | 53 | ```sh 54 | git remote add sandbox https://git.127-0-0-1.nip.io/ops/homelab 55 | git config http.https://git.127-0-0-1.nip.io.sslVerify false 56 | 57 | git add foobar.txt 58 | git commit -m "feat: harness the power of the sun" 59 | git push sandbox # you can use the gitea_admin account 60 | ``` 61 | 62 | ## Clean up 63 | 64 | Delete the cluster: 65 | 66 | ```sh 67 | k3d cluster delete homelab-dev 68 | ``` 69 | 70 | ## Caveats compare to production environment 71 | 72 | The development cluster doesn't have the following features: 73 | 74 | - There is no valid domain name, hence no SSL certificates (some services require valid SSL certificates) 75 | - Only accessible on the host machine 76 | - No backup 77 | 78 | Please keep in mind that the development cluster may be unstable and things may break (it's for development after all). 79 | -------------------------------------------------------------------------------- /docs/reference/architecture/networking.md: -------------------------------------------------------------------------------- 1 | # Networking 2 | 3 | ```mermaid 4 | flowchart TD 5 | subgraph LAN 6 | laptop/desktop/phone <--> LoadBalancer 7 | subgraph k8s[Kubernetes cluster] 8 | Pod --> Service 9 | Service --> Ingress 10 | 11 | LoadBalancer 12 | 13 | cloudflared 14 | cloudflared <--> Ingress 15 | end 16 | LoadBalancer <--> Ingress 17 | end 18 | 19 | cloudflared -- outbound --> Cloudflare 20 | Internet -- inbound --> Cloudflare 21 | ``` 22 | 23 | TODO (PR welcomed) 24 | -------------------------------------------------------------------------------- /docs/reference/architecture/overview.md: -------------------------------------------------------------------------------- 1 | # Overview 2 | 3 | ## Components 4 | 5 | ``` 6 | +--------------+ 7 | | ./apps | 8 | |--------------| 9 | | ./platform | 10 | |--------------| +------------+ 11 | | ./system |- - - -| ./external | 12 | |--------------| +------------+ 13 | | ./metal | 14 | |--------------| 15 | | HARDWARE | 16 | +--------------+ 17 | ``` 18 | 19 | Main components: 20 | 21 | - `./metal`: bare metal management, install Linux and Kubernetes 22 | - `./system`: critical system components for the cluster (load balancer, storage, ingress, operation tools...) 23 | - `./platform`: essential components for service hosting platform (git, build runners, dashboards...) 24 | - `./apps`: user facing applications 25 | - `./external` (optional): externally managed services 26 | 27 | Support components: 28 | 29 | - `./docs`: all documentation go here, this will generate a searchable web UI 30 | - `./scripts`: scripts to automate common tasks 31 | 32 | ## Provisioning flow 33 | 34 | Everything is automated, after you edit the configuration files, you just need to run a single `make` command and it will: 35 | 36 | - (1) Build the `./metal` layer: 37 | - Create an ephemeral, stateless PXE server 38 | - Install Linux on all servers in parallel 39 | - Build a Kubernetes cluster (based on k3s) 40 | - (2) Bootstrap the `./system` layer: 41 | - Install ArgoCD and the root app to manage itself and other layers, from now on ArgoCD will do the rest 42 | - Install the remaining components (storage, monitoring, etc) 43 | - (3) Build the `./platform` layer (Gitea, Grafana, SSO, etc) 44 | - (4) Deploy applications in the `./apps` layer 45 | 46 | ```mermaid 47 | flowchart TD 48 | subgraph metal[./metal] 49 | pxe[PXE Server] -.-> linux[Fedora Server] --> k3s 50 | end 51 | 52 | subgraph system[./system] 53 | argocd[ArgoCD and root app] 54 | nginx[NGINX] 55 | rook-ceph[Rook Ceph] 56 | cert-manager 57 | external-dns[External DNS] 58 | cloudflared 59 | end 60 | 61 | subgraph external[./external] 62 | letsencrypt[Let's Encrypt] 63 | cloudflare[Cloudflare] 64 | end 65 | 66 | letsencrypt -.-> cert-manager 67 | cloudflare -.-> cert-manager 68 | cloudflare -.-> external-dns 69 | cloudflare -.-> cloudflared 70 | 71 | subgraph platform[./platform] 72 | Gitea 73 | Woodpecker 74 | Grafana 75 | end 76 | 77 | subgraph apps[./apps] 78 | homepage[Homepage] 79 | jellyfin[Jellyfin] 80 | matrix[Matrix] 81 | paperless[Paperless] 82 | end 83 | 84 | make[Run make] -- 1 --> metal -- 2 --> system -. 3 .-> platform -. 4 .-> apps 85 | ``` 86 | -------------------------------------------------------------------------------- /docs/reference/changelog.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## v0.0.8 4 | 5 | Notable changes: 6 | 7 | - **build:** run post install scripts by default 8 | - **build:** set `KUBECONFIG` from global Makefile 9 | - **feat(external-dns)!:** add cluster name as owner ID 10 | - **feat(tools):** install `yamllint`, `ansible-lint` and `k9s` 11 | - **feat(tools):** set `KUBECONFIG` by default 12 | - **feat:** add pre-commit hooks 13 | - **feat:** add script to setup Gitea tokens and OAuth apps 14 | - **perf(argocd):** turning on selective sync 15 | - **refactor(docs):** migrate to [mkdocs](https://squidfunk.github.io/mkdocs-material) 16 | - **refactor(metal):** migrate to Fedora 36 for newer packages 17 | - **refactor(pxe)!:** combine dhcpd and tftpd to dnsmasq 18 | - Many bug fixes 19 | 20 | Please see git log for full change log. 21 | 22 | ## 0.0.7-alpha 23 | 24 | - Replace standard Vault with Vault Operator 25 | - Automatically initialize and unseal Vault 26 | - Declarative secret generation and management 27 | - Declarative Gitea configuration with YAML 28 | - Automatic OS rolling upgrade 29 | - Automatic Kubernetes rolling upgrade 30 | - Automatic application updates using Renovate (still require manual token generation) 31 | - Add script to wait for essential services after deployment 32 | - Add icons and bookmarks to the home page 33 | - Deploy Matrix chat 34 | - Replace Authentik with Dex for SSO (still require manual token generation) 35 | - Switch to Mermaid for diagrams in documentation 36 | - Replace Vagrant with k3d for development environment 37 | - Use nip.io domain for development environment 38 | - Remove Backblaze (S3 Glacier and/or Minio will be added in future version) 39 | - Enable monitor for the majority of applications 40 | - Many code refactorings and bug fixes 41 | 42 | ## 0.0.6-alpha 43 | 44 | - Upgrade to Kubernetes 1.23 45 | - Support external resources: 46 | - Cloudflare DNS and Tunnel 47 | - Backblaze for backup 48 | - Auto inject secrets to required namespaces 49 | - Replace self-signed certificates with Let's Encrypt production (with API token injected from the `external` layer) 50 | - Add DNS records automatically using external-dns 51 | - Easy Cloudflare Tunnel configuration with annotations 52 | - Offsite backup to Backblaze B2 bucket using k8up-operator 53 | - Add private container registry 54 | - Remove Knative to save resources (temporarily) 55 | - Enable encryption at rest for Kubernetes Secrets 56 | - Add more Tekton tasks and pipelines 57 | - Initialize GitOps repository on Gitea automatically after install 58 | - Generate MetalLB address pool automatically (default to the last `/27` subnet) 59 | - Some bug fixes 60 | 61 | ## 0.0.5-alpha 62 | 63 | - Add convenience scripts 64 | - Add Loki for logging 65 | - Add custom health check for Application and ApplicationSet 66 | - Use Vault with dev mode on (temporarily until we hit beta) 67 | - Replace Authelia with Authentik 68 | - Upgrade to Kubernetes 1.22 69 | - Upgrade most services to the latest version 70 | - Set ingress class and storage class explicitly 71 | - Initial Linkerd and Knative setup (not working yet) 72 | - Set up Hajimari for home page with automatic ingress discovery 73 | - Add dev VM for local development or evaluation 74 | - Optimize bare metal provisioning performance 75 | - Replace Syncthing with Seafile (may use both in the feature) 76 | - Enable Gitea SSH cloning via Ingress 77 | - Various code clean up 78 | - Add more documents 79 | 80 | ## 0.0.4-alpha 81 | 82 | - Switch to Rocky Linux 83 | - Some optimization for bare metal provisioning 84 | - Switch to k3s and combine Kubernetes cluster config in `./infra` layer to `./metal` layer (because k3s is also configured using Ansible) 85 | - Split boostrap Helm charts in `./infra` layer to `./bootstrap` layer (with new ArgoCD pattern) and `./system` layer 86 | - Add `./platform` layer for some applications like Gitea, Tekton... 87 | - User only need to provision `./metal` and `bootstrap` layer, the `./bootstrap` layer will deploy the remaining layers 88 | - Provisioning time from empty disk to running services is significantly reduced (thanks to k3s and new bootstrap pattern) 89 | - Use [mdBook](https://rust-lang.github.io/mdBook/) for documents 90 | - Replace Drone CI with Tekton 91 | - Enable TLS on all Ingresses (using [cert-manager](https://cert-manager.io)) 92 | - Add some new applications 93 | 94 | ## 0.0.3-alpha 95 | 96 | - Generate Terraform backend config automatically 97 | - Switch to CoreOS 98 | - Better PXE boot setup 99 | - Diagrams as code 100 | 101 | ## 0.0.2-alpha 102 | 103 | - Ensure idempotency for bare metal provisioning 104 | - Extract instead of mounting the OS ISO file 105 | - Easy initial controller setup (with only Docker) 106 | - Switch to Fedora 107 | - Remove LXD 108 | - Move etcd (Terraform state backend) back to Docker 109 | 110 | ## 0.0.1-alpha 111 | 112 | - Bare metal provisioning with PXE 113 | - LXD cluster 114 | - Terraform state backend (etcd) 115 | - RKE cluster 116 | - Core services (Vault, Gitea, ArgoCD,...) 117 | - Public services to the internet (via port forwarding or Cloudflare Tunnel) 118 | -------------------------------------------------------------------------------- /docs/reference/contributing.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | ## How to contribute 4 | 5 | ### Bug report 6 | 7 | You can [create a new GitHub issue](https://github.com/khuedoan/homelab/issues/new/choose) with the bug report template. 8 | 9 | ### Submitting patches 10 | 11 | Because you may have a lot of changes in your fork, you can't create a pull request directly from your `master` branch. 12 | Instead, create a branch from the upstream repository and commit your changes there: 13 | 14 | ```sh 15 | git remote add upstream https://github.com/khuedoan/homelab 16 | git fetch upstream 17 | git checkout upstream/master 18 | git checkout -b contrib-fix-something 19 | 20 | # Make your changes here 21 | # 22 | # nvim README.md 23 | # git cherry-pick a1b2c3 24 | # 25 | # commit, push, etc. as usual 26 | ``` 27 | 28 | Then you can send the patch using [GitHub pull request](https://github.com/khuedoan/homelab/pulls) or `git send-email` to . 29 | -------------------------------------------------------------------------------- /docs/reference/faq.md: -------------------------------------------------------------------------------- 1 | # FAQ 2 | 3 | ## Is it necessary to install Linux on my servers before setting up the homelab? 4 | 5 | No, and that's the beauty of this setup. 6 | You start with empty hard drives, type a single command on your laptop/PC, 7 | and it will install the operating system for you automatically and in parallel over the network. 8 | 9 | ## Is it necessary to keep the PXE server running? 10 | 11 | No, the ephemeral PXE server is stateless, once Linux is installed on your servers, 12 | you can shut it down (or not, ideally, you don't even need to be aware of its existence). 13 | The Ansible setup in `./metal` is idempotent and will start the PXE server if needed. 14 | -------------------------------------------------------------------------------- /docs/reference/license.md: -------------------------------------------------------------------------------- 1 | Copyright © 2020 - 2024 Khue Doan 2 | 3 | --8<-- 4 | LICENSE.md 5 | --8<-- 6 | -------------------------------------------------------------------------------- /docs/reference/roadmap.md: -------------------------------------------------------------------------------- 1 | # Roadmap 2 | 3 | !!! info 4 | 5 | Current status: **ALPHA** 6 | 7 | ## Alpha requirements 8 | 9 | Literally anything that works. 10 | 11 | ## Beta requirements 12 | 13 | Good enough for tinkering and personal usage, and reasonably secure. 14 | 15 | - [x] Automated bare metal provisioning 16 | - [x] Controller set up (Docker) 17 | - [x] OS installation (PXE boot) 18 | - [x] Automated cluster creation (k3s) 19 | - [x] Automated application deployment (ArgoCD) 20 | - [x] Automated DNS management 21 | - [x] Initialize GitOps repository on Gitea automatically 22 | - [x] Observability 23 | - [x] Monitoring 24 | - [x] Logging 25 | - [x] Alerting 26 | - [x] SSO 27 | - [ ] Reasonably secure 28 | - [x] Automated certificate management 29 | - [x] Declarative secret management 30 | - [ ] Replace all default passwords with randomly generated ones 31 | - [x] Expose services to the internet securely with Cloudflare Tunnel 32 | - [x] Only use open-source technologies (except external managed services in `./external`) 33 | - [x] Everything is defined as code 34 | - [ ] Backup solution (3 copies, 2 seperate devices, 1 offsite) 35 | - [ ] Define [SLOs](https://en.wikipedia.org/wiki/Service-level_objective): 36 | - [ ] 70% availability (might break in the weekend due to new experimentation) 37 | - [x] Core applications 38 | - [x] Gitea 39 | - [x] Woodpecker 40 | - [x] Private container registry 41 | - [x] Homepage 42 | 43 | ## Stable requirements 44 | 45 | Can be used in "production" (for family or even small scale businesses). 46 | 47 | - [x] A single command to deploy everything 48 | - [x] Fast deployment time (from empty hard drive to running services in under 1 hour) 49 | - [ ] Fully _automatic_, not just _automated_ 50 | - [x] Bare-metal OS rolling upgrade 51 | - [x] Kubernetes version rolling upgrade 52 | - [x] Application version upgrade 53 | - [ ] Encrypted backups 54 | - [ ] Secrets rotation 55 | - [x] Self healing 56 | - [ ] Secure by default 57 | - [ ] SELinux 58 | - [ ] Network policies 59 | - [ ] Static code analysis 60 | - [ ] Chaos testing 61 | - [x] Minimal dependency on external services 62 | - [ ] Complete documentation 63 | - [x] Diagram as code 64 | - [x] Book (this book) 65 | - [ ] Walkthrough tutorial and feature demo (video) 66 | - [x] Configuration script for new users 67 | - [ ] More dashboards and alert rules 68 | - [ ] SLOs: 69 | - [ ] 99,9% availability (less than 9 hours of downtime per year) 70 | - [ ] 99,99% data durability 71 | - [ ] Clear upgrade path 72 | - [ ] Additional applications 73 | - [ ] Matrix with bridges 74 | - [ ] VPN server 75 | - [ ] PeerTube 76 | - [x] Blog 77 | - [ ] [Development dashboard](https://github.com/khuedoan/homelab-backstage) 78 | 79 | ## Unplanned 80 | 81 | Nice to have 82 | 83 | - [ ] Addition applications 84 | - [ ] Mail server 85 | - [ ] Air-gap install 86 | - [ ] Automated testing 87 | - [ ] Security audit 88 | - [ ] Serverless ([Knative](https://knative.dev)) 89 | - [ ] Cluster API ([last attempt](https://github.com/khuedoan/homelab/pull/2)) 90 | - [ ] Split DNS (requires a better router) 91 | -------------------------------------------------------------------------------- /external/.gitignore: -------------------------------------------------------------------------------- 1 | .terraform* 2 | terraform.tfstate 3 | terraform.tfvars 4 | -------------------------------------------------------------------------------- /external/Makefile: -------------------------------------------------------------------------------- 1 | .POSIX: 2 | 3 | default: apply 4 | 5 | ~/.terraform.d/credentials.tfrc.json: 6 | tofu login 7 | 8 | terraform.tfvars: 9 | cp terraform.tfvars.example ${@} 10 | nvim ${@} 11 | 12 | .terraform.lock.hcl: ~/.terraform.d/credentials.tfrc.json versions.tf terraform.tfvars 13 | tofu init 14 | touch ${@} 15 | 16 | namespaces: 17 | ansible-playbook namespaces.yml 18 | 19 | plan: .terraform.lock.hcl 20 | tofu plan 21 | 22 | apply: .terraform.lock.hcl namespaces 23 | tofu apply -auto-approve 24 | -------------------------------------------------------------------------------- /external/main.tf: -------------------------------------------------------------------------------- 1 | module "cloudflare" { 2 | source = "./modules/cloudflare" 3 | cloudflare_account_id = var.cloudflare_account_id 4 | cloudflare_email = var.cloudflare_email 5 | cloudflare_api_key = var.cloudflare_api_key 6 | } 7 | 8 | module "ntfy" { 9 | source = "./modules/ntfy" 10 | auth = var.ntfy 11 | } 12 | 13 | module "extra_secrets" { 14 | source = "./modules/extra-secrets" 15 | data = var.extra_secrets 16 | } 17 | -------------------------------------------------------------------------------- /external/modules/cloudflare/main.tf: -------------------------------------------------------------------------------- 1 | data "cloudflare_zone" "zone" { 2 | name = "khuedoan.com" 3 | } 4 | 5 | data "cloudflare_api_token_permission_groups" "all" {} 6 | 7 | resource "random_password" "tunnel_secret" { 8 | length = 64 9 | special = false 10 | } 11 | 12 | resource "cloudflare_tunnel" "homelab" { 13 | account_id = var.cloudflare_account_id 14 | name = "homelab" 15 | secret = base64encode(random_password.tunnel_secret.result) 16 | } 17 | 18 | # Not proxied, not accessible. Just a record for auto-created CNAMEs by external-dns. 19 | resource "cloudflare_record" "tunnel" { 20 | zone_id = data.cloudflare_zone.zone.id 21 | type = "CNAME" 22 | name = "homelab-tunnel" 23 | value = "${cloudflare_tunnel.homelab.id}.cfargotunnel.com" 24 | proxied = false 25 | ttl = 1 # Auto 26 | } 27 | 28 | resource "kubernetes_secret" "cloudflared_credentials" { 29 | metadata { 30 | name = "cloudflared-credentials" 31 | namespace = "cloudflared" 32 | 33 | annotations = { 34 | "app.kubernetes.io/managed-by" = "Terraform" 35 | } 36 | } 37 | 38 | data = { 39 | "credentials.json" = jsonencode({ 40 | AccountTag = var.cloudflare_account_id 41 | TunnelName = cloudflare_tunnel.homelab.name 42 | TunnelID = cloudflare_tunnel.homelab.id 43 | TunnelSecret = base64encode(random_password.tunnel_secret.result) 44 | }) 45 | } 46 | } 47 | 48 | resource "cloudflare_api_token" "external_dns" { 49 | name = "homelab_external_dns" 50 | 51 | policy { 52 | permission_groups = [ 53 | data.cloudflare_api_token_permission_groups.all.zone["Zone Read"], 54 | data.cloudflare_api_token_permission_groups.all.zone["DNS Write"] 55 | ] 56 | resources = { 57 | "com.cloudflare.api.account.zone.*" = "*" 58 | } 59 | } 60 | } 61 | 62 | resource "kubernetes_secret" "external_dns_token" { 63 | metadata { 64 | name = "cloudflare-api-token" 65 | namespace = "external-dns" 66 | 67 | annotations = { 68 | "app.kubernetes.io/managed-by" = "Terraform" 69 | } 70 | } 71 | 72 | data = { 73 | "value" = cloudflare_api_token.external_dns.value 74 | } 75 | } 76 | 77 | resource "cloudflare_api_token" "cert_manager" { 78 | name = "homelab_cert_manager" 79 | 80 | policy { 81 | permission_groups = [ 82 | data.cloudflare_api_token_permission_groups.all.zone["Zone Read"], 83 | data.cloudflare_api_token_permission_groups.all.zone["DNS Write"] 84 | ] 85 | resources = { 86 | "com.cloudflare.api.account.zone.*" = "*" 87 | } 88 | } 89 | } 90 | 91 | resource "kubernetes_secret" "cert_manager_token" { 92 | metadata { 93 | name = "cloudflare-api-token" 94 | namespace = "cert-manager" 95 | 96 | annotations = { 97 | "app.kubernetes.io/managed-by" = "Terraform" 98 | } 99 | } 100 | 101 | data = { 102 | "api-token" = cloudflare_api_token.cert_manager.value 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /external/modules/cloudflare/variables.tf: -------------------------------------------------------------------------------- 1 | variable "cloudflare_email" { 2 | type = string 3 | } 4 | 5 | variable "cloudflare_api_key" { 6 | type = string 7 | sensitive = true 8 | } 9 | 10 | variable "cloudflare_account_id" { 11 | type = string 12 | } 13 | -------------------------------------------------------------------------------- /external/modules/cloudflare/versions.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | cloudflare = { 4 | source = "cloudflare/cloudflare" 5 | version = "~> 4.30.0" 6 | } 7 | 8 | kubernetes = { 9 | source = "hashicorp/kubernetes" 10 | version = "~> 2.26.0" 11 | } 12 | 13 | http = { 14 | source = "hashicorp/http" 15 | version = "~> 3.4.0" 16 | } 17 | } 18 | } 19 | 20 | provider "cloudflare" { 21 | email = var.cloudflare_email 22 | api_key = var.cloudflare_api_key 23 | } 24 | -------------------------------------------------------------------------------- /external/modules/extra-secrets/main.tf: -------------------------------------------------------------------------------- 1 | resource "kubernetes_secret" "external" { 2 | metadata { 3 | name = var.name 4 | namespace = var.namespace 5 | 6 | annotations = { 7 | "app.kubernetes.io/managed-by" = "Terraform" 8 | } 9 | } 10 | 11 | data = var.data 12 | } 13 | -------------------------------------------------------------------------------- /external/modules/extra-secrets/variables.tf: -------------------------------------------------------------------------------- 1 | variable "name" { 2 | type = string 3 | default = "external" 4 | } 5 | 6 | variable "namespace" { 7 | type = string 8 | default = "global-secrets" 9 | } 10 | 11 | variable "data" { 12 | type = map(string) 13 | } 14 | -------------------------------------------------------------------------------- /external/modules/extra-secrets/versions.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | kubernetes = { 4 | source = "hashicorp/kubernetes" 5 | version = "~> 2.26.0" 6 | } 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /external/modules/ntfy/main.tf: -------------------------------------------------------------------------------- 1 | resource "kubernetes_secret" "ntfy_auth" { 2 | metadata { 3 | name = "webhook-transformer" 4 | namespace = "monitoring-system" 5 | 6 | annotations = { 7 | "app.kubernetes.io/managed-by" = "Terraform" 8 | } 9 | } 10 | 11 | data = { 12 | NTFY_URL = var.auth.url 13 | NTFY_TOPIC = var.auth.topic 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /external/modules/ntfy/variables.tf: -------------------------------------------------------------------------------- 1 | variable "auth" { 2 | type = object({ 3 | url = string 4 | topic = string 5 | }) 6 | } 7 | -------------------------------------------------------------------------------- /external/modules/ntfy/versions.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | kubernetes = { 4 | source = "hashicorp/kubernetes" 5 | version = "~> 2.26.0" 6 | } 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /external/namespaces.yml: -------------------------------------------------------------------------------- 1 | - hosts: localhost 2 | tasks: 3 | - name: Ensure required namespaces exist 4 | kubernetes.core.k8s: 5 | api_version: v1 6 | kind: Namespace 7 | name: "{{ item }}" 8 | state: present 9 | loop: 10 | - cert-manager 11 | - cloudflared 12 | - external-dns 13 | - global-secrets 14 | - k8up-operator 15 | - monitoring-system 16 | -------------------------------------------------------------------------------- /external/terraform.tfvars.example: -------------------------------------------------------------------------------- 1 | # https://dash.cloudflare.com/profile 2 | cloudflare_email = "myaccount@example.com" 3 | # https://developers.cloudflare.com/fundamentals/setup/find-account-and-zone-ids 4 | cloudflare_account_id = "foobarid" 5 | # https://dash.cloudflare.com/profile/api-tokens 6 | cloudflare_api_key = "foobarkey" 7 | 8 | ntfy = { 9 | # https://ntfy.sh or your own instance 10 | url = "https://ntfy.sh" 11 | # Your topic name 12 | topic = "random_topic_name_here_a8sd7fkjxlkcjasdw33813" 13 | } 14 | 15 | extra_secrets = { 16 | # Try to keep this to a minimum with third-party secrets 17 | # Consider using the secret generator if possible 18 | # ../platform/global-secrets/files/secret-generator/config.yaml 19 | # Here's some examples of what you might want to add: 20 | # 21 | # wireguard-private-key = "wg genkey output here" 22 | # tailscale-auth-key = "tskey-auth-xxxxxxxxxxxxxxxxx-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" 23 | # restic-password = "xxxxxxxxxxxxxxxxxxxxxxxx" 24 | # restic-s3-bucket = "https://s3.amazonaws.com/my-homelab-backup-xxxxxxxxxx" 25 | # restic-s3-access-key = "xxxxxxxxxxxxxxxx" 26 | # restic-s3-secret-key = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" 27 | } 28 | -------------------------------------------------------------------------------- /external/variables.tf: -------------------------------------------------------------------------------- 1 | variable "cloudflare_email" { 2 | type = string 3 | } 4 | 5 | variable "cloudflare_api_key" { 6 | type = string 7 | sensitive = true 8 | } 9 | 10 | variable "cloudflare_account_id" { 11 | type = string 12 | } 13 | 14 | variable "ntfy" { 15 | type = object({ 16 | url = string 17 | topic = string 18 | }) 19 | 20 | sensitive = true 21 | } 22 | 23 | variable "extra_secrets" { 24 | type = map(string) 25 | description = "Key-value pairs of extra secrets that cannot be randomly generated (e.g. third party API tokens)" 26 | sensitive = true 27 | default = {} 28 | } 29 | -------------------------------------------------------------------------------- /external/versions.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = "~> 1.7" 3 | 4 | backend "remote" { 5 | hostname = "app.terraform.io" 6 | organization = "khuedoan" 7 | 8 | workspaces { 9 | name = "homelab-external" 10 | } 11 | } 12 | 13 | required_providers { 14 | cloudflare = { 15 | source = "cloudflare/cloudflare" 16 | version = "~> 4.30.0" 17 | } 18 | 19 | kubernetes = { 20 | source = "hashicorp/kubernetes" 21 | version = "~> 2.26.0" 22 | } 23 | 24 | http = { 25 | source = "hashicorp/http" 26 | version = "~> 3.4.0" 27 | } 28 | } 29 | } 30 | 31 | provider "cloudflare" { 32 | email = var.cloudflare_email 33 | api_key = var.cloudflare_api_key 34 | } 35 | 36 | provider "kubernetes" { 37 | # Use KUBE_CONFIG_PATH environment variables 38 | # Or in cluster service account 39 | } 40 | -------------------------------------------------------------------------------- /flake.lock: -------------------------------------------------------------------------------- 1 | { 2 | "nodes": { 3 | "flake-utils": { 4 | "inputs": { 5 | "systems": "systems" 6 | }, 7 | "locked": { 8 | "lastModified": 1731533236, 9 | "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", 10 | "owner": "numtide", 11 | "repo": "flake-utils", 12 | "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", 13 | "type": "github" 14 | }, 15 | "original": { 16 | "owner": "numtide", 17 | "repo": "flake-utils", 18 | "type": "github" 19 | } 20 | }, 21 | "nixpkgs": { 22 | "locked": { 23 | "lastModified": 1731797254, 24 | "narHash": "sha256-df3dJApLPhd11AlueuoN0Q4fHo/hagP75LlM5K1sz9g=", 25 | "owner": "NixOS", 26 | "repo": "nixpkgs", 27 | "rev": "e8c38b73aeb218e27163376a2d617e61a2ad9b59", 28 | "type": "github" 29 | }, 30 | "original": { 31 | "owner": "NixOS", 32 | "ref": "nixos-24.05", 33 | "repo": "nixpkgs", 34 | "type": "github" 35 | } 36 | }, 37 | "root": { 38 | "inputs": { 39 | "flake-utils": "flake-utils", 40 | "nixpkgs": "nixpkgs" 41 | } 42 | }, 43 | "systems": { 44 | "locked": { 45 | "lastModified": 1681028828, 46 | "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", 47 | "owner": "nix-systems", 48 | "repo": "default", 49 | "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", 50 | "type": "github" 51 | }, 52 | "original": { 53 | "owner": "nix-systems", 54 | "repo": "default", 55 | "type": "github" 56 | } 57 | } 58 | }, 59 | "root": "root", 60 | "version": 7 61 | } 62 | -------------------------------------------------------------------------------- /flake.nix: -------------------------------------------------------------------------------- 1 | { 2 | description = "Homelab"; 3 | 4 | inputs = { 5 | nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.05"; 6 | flake-utils.url = "github:numtide/flake-utils"; 7 | }; 8 | 9 | outputs = { self, nixpkgs, flake-utils }: 10 | flake-utils.lib.eachDefaultSystem (system: 11 | let 12 | pkgs = import nixpkgs { inherit system; }; 13 | in 14 | with pkgs; 15 | { 16 | devShells.default = mkShell { 17 | packages = [ 18 | ansible 19 | ansible-lint 20 | bmake 21 | diffutils 22 | docker 23 | docker-compose 24 | dyff 25 | git 26 | go 27 | gotestsum 28 | iproute2 29 | jq 30 | k9s 31 | kanidm 32 | kube3d 33 | kubectl 34 | kubernetes-helm 35 | kustomize 36 | libisoburn 37 | neovim 38 | openssh 39 | opentofu # Drop-in replacement for Terraform 40 | p7zip 41 | pre-commit 42 | qrencode 43 | shellcheck 44 | wireguard-tools 45 | yamllint 46 | 47 | (python3.withPackages (p: with p; [ 48 | jinja2 49 | kubernetes 50 | mkdocs-material 51 | netaddr 52 | pexpect 53 | rich 54 | ])) 55 | ]; 56 | }; 57 | } 58 | ); 59 | } 60 | -------------------------------------------------------------------------------- /metal/Makefile: -------------------------------------------------------------------------------- 1 | .POSIX: 2 | 3 | env ?= prod 4 | export KUBECONFIG = $(shell pwd)/kubeconfig.yaml 5 | 6 | default: boot cluster 7 | 8 | ~/.ssh/id_ed25519: 9 | ssh-keygen -t ed25519 -P '' -f "$@" 10 | 11 | boot: ~/.ssh/id_ed25519 12 | ansible-playbook \ 13 | --inventory inventories/${env}.yml \ 14 | boot.yml 15 | 16 | cluster: 17 | ansible-playbook \ 18 | --inventory inventories/${env}.yml \ 19 | cluster.yml 20 | 21 | console: 22 | ansible-console \ 23 | --inventory inventories/${env}.yml 24 | -------------------------------------------------------------------------------- /metal/ansible.cfg: -------------------------------------------------------------------------------- 1 | [defaults] 2 | host_key_checking=false 3 | stdout_callback=debug 4 | stderr_callback=debug 5 | force_color=true 6 | -------------------------------------------------------------------------------- /metal/boot.yml: -------------------------------------------------------------------------------- 1 | - name: Start PXE server 2 | hosts: localhost 3 | roles: 4 | - pxe_server 5 | 6 | - name: Provision bare metal machines 7 | hosts: metal 8 | gather_facts: false 9 | roles: 10 | - wake 11 | -------------------------------------------------------------------------------- /metal/cluster.yml: -------------------------------------------------------------------------------- 1 | - name: Create Kubernetes cluster 2 | hosts: metal 3 | roles: 4 | - prerequisites 5 | - k3s 6 | - automatic_upgrade 7 | 8 | - name: Install Kubernetes addons 9 | hosts: localhost 10 | roles: 11 | - cilium 12 | -------------------------------------------------------------------------------- /metal/group_vars/all.yml: -------------------------------------------------------------------------------- 1 | ansible_user: root 2 | ansible_ssh_private_key_file: ~/.ssh/id_ed25519 3 | ssh_public_key: "{{ lookup('file', '~/.ssh/id_ed25519.pub') }}" 4 | dns_server: "8.8.8.8" 5 | -------------------------------------------------------------------------------- /metal/inventories/prod.yml: -------------------------------------------------------------------------------- 1 | all: 2 | vars: 3 | control_plane_endpoint: 192.168.1.100 4 | load_balancer_ip_pool: 5 | - 192.168.1.224/27 6 | metal: 7 | children: 8 | masters: 9 | hosts: 10 | metal0: {ansible_host: 192.168.1.110, mac: '00:23:24:d1:f5:69', disk: sda, network_interface: eno1} 11 | metal1: {ansible_host: 192.168.1.111, mac: '00:23:24:d1:f3:f0', disk: sda, network_interface: eno1} 12 | metal2: {ansible_host: 192.168.1.112, mac: '00:23:24:e7:04:60', disk: sda, network_interface: eno1} 13 | workers: 14 | hosts: 15 | metal3: {ansible_host: 192.168.1.113, mac: '00:23:24:d1:f4:d6', disk: sda, network_interface: eno1} 16 | -------------------------------------------------------------------------------- /metal/inventories/stag.yml: -------------------------------------------------------------------------------- 1 | metal: 2 | children: 3 | masters: 4 | hosts: 5 | proxmox0: {ansible_host: 192.168.1.169, mac: 'c2:f5:cf:1f:3e:c0', disk: sda, network_interface: ens18} 6 | workers: 7 | hosts: {} 8 | -------------------------------------------------------------------------------- /metal/k3d-dev.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: k3d.io/v1alpha4 2 | kind: Simple 3 | metadata: 4 | name: homelab-dev 5 | image: docker.io/rancher/k3s:v1.27.1-k3s1 6 | servers: 1 7 | agents: 0 8 | options: 9 | k3s: 10 | extraArgs: 11 | - arg: --disable=traefik 12 | nodeFilters: 13 | - server:* 14 | - arg: --disable-cloud-controller 15 | nodeFilters: 16 | - server:* 17 | ports: 18 | - port: 80:80 19 | nodeFilters: 20 | - loadbalancer 21 | - port: 443:443 22 | nodeFilters: 23 | - loadbalancer 24 | -------------------------------------------------------------------------------- /metal/roles/automatic_upgrade/files/automatic.conf: -------------------------------------------------------------------------------- 1 | [commands] 2 | upgrade_type = default 3 | apply_updates = yes 4 | -------------------------------------------------------------------------------- /metal/roles/automatic_upgrade/tasks/main.yml: -------------------------------------------------------------------------------- 1 | - name: Install packages for automatic upgrade 2 | ansible.builtin.dnf: 3 | name: 4 | - dnf-automatic 5 | - dnf-utils 6 | 7 | - name: Copy automatic upgrade config file 8 | ansible.builtin.copy: 9 | src: automatic.conf 10 | dest: /etc/dnf/automatic.conf 11 | mode: 0644 12 | 13 | - name: Enable automatic upgrade service 14 | ansible.builtin.systemd: 15 | name: dnf-automatic.timer 16 | state: started 17 | enabled: true 18 | -------------------------------------------------------------------------------- /metal/roles/cilium/defaults/main.yml: -------------------------------------------------------------------------------- 1 | cilium_repo_url: https://helm.cilium.io 2 | cilium_version: 1.16.1 3 | cilium_namespace: kube-system 4 | cilium_values: 5 | operator: 6 | replicas: 1 7 | kubeProxyReplacement: true 8 | l2announcements: 9 | enabled: true 10 | # TODO the host and port are k3s-specific, generic solution is in progress 11 | # https://github.com/cilium/cilium/issues/19038 12 | # https://github.com/cilium/cilium/pull/28741 13 | k8sServiceHost: 127.0.0.1 14 | k8sServicePort: 6444 15 | hubble: 16 | relay: 17 | enabled: true 18 | ui: 19 | enabled: true 20 | -------------------------------------------------------------------------------- /metal/roles/cilium/tasks/main.yml: -------------------------------------------------------------------------------- 1 | - name: Install Cilium 2 | kubernetes.core.helm: 3 | name: cilium 4 | chart_ref: cilium 5 | chart_repo_url: "{{ cilium_repo_url }}" 6 | chart_version: "{{ cilium_version }}" 7 | release_namespace: "{{ cilium_namespace }}" 8 | values: "{{ cilium_values }}" 9 | 10 | - name: Wait for Cilium CRDs 11 | kubernetes.core.k8s_info: 12 | kind: CustomResourceDefinition 13 | name: "{{ item }}" 14 | loop: 15 | - ciliuml2announcementpolicies.cilium.io 16 | - ciliumloadbalancerippools.cilium.io 17 | register: crd 18 | until: crd.resources | length > 0 19 | retries: 5 20 | delay: 10 21 | 22 | - name: Apply Cilium resources 23 | kubernetes.core.k8s: 24 | template: "{{ item }}" 25 | loop: 26 | - ciliuml2announcementpolicy.yaml 27 | - ciliumloadbalancerippool.yaml 28 | -------------------------------------------------------------------------------- /metal/roles/cilium/templates/ciliuml2announcementpolicy.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: cilium.io/v2alpha1 2 | kind: CiliumL2AnnouncementPolicy 3 | metadata: 4 | name: default 5 | spec: 6 | externalIPs: true 7 | loadBalancerIPs: true 8 | -------------------------------------------------------------------------------- /metal/roles/cilium/templates/ciliumloadbalancerippool.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: cilium.io/v2alpha1 2 | kind: CiliumLoadBalancerIPPool 3 | metadata: 4 | name: default 5 | spec: 6 | blocks: 7 | {% for cidr in load_balancer_ip_pool %} 8 | - cidr: {{ cidr }} 9 | {% endfor %} 10 | -------------------------------------------------------------------------------- /metal/roles/k3s/defaults/main.yml: -------------------------------------------------------------------------------- 1 | k3s_version: v1.30.4+k3s1 2 | k3s_config_file: /etc/rancher/k3s/config.yaml 3 | k3s_token_file: /etc/rancher/node/password 4 | k3s_service_file: /etc/systemd/system/k3s.service 5 | k3s_data_dir: /var/lib/rancher/k3s 6 | k3s_kubeconfig_file: /etc/rancher/k3s/k3s.yaml 7 | k3s_server_config: 8 | tls-san: 9 | - "{{ control_plane_endpoint }}" 10 | disable: 11 | - local-storage 12 | - servicelb 13 | - traefik 14 | disable-helm-controller: true 15 | disable-kube-proxy: true 16 | disable-network-policy: true 17 | flannel-backend: none 18 | secrets-encryption: true 19 | -------------------------------------------------------------------------------- /metal/roles/k3s/files/bin/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | !.gitignore 3 | -------------------------------------------------------------------------------- /metal/roles/k3s/tasks/main.yml: -------------------------------------------------------------------------------- 1 | - name: Download k3s binary 2 | ansible.builtin.get_url: 3 | url: https://github.com/k3s-io/k3s/releases/download/{{ k3s_version }}/k3s 4 | checksum: sha256:https://github.com/k3s-io/k3s/releases/download/{{ k3s_version }}/sha256sum-amd64.txt 5 | dest: "{{ role_path }}/files/bin/k3s" 6 | mode: 0755 7 | delegate_to: localhost 8 | run_once: true 9 | register: k3s_binary 10 | 11 | - name: Copy k3s binary to nodes 12 | ansible.builtin.copy: 13 | src: bin/k3s 14 | dest: /usr/local/bin/k3s 15 | owner: root 16 | group: root 17 | mode: 0755 18 | 19 | - name: Ensure config directories exist 20 | ansible.builtin.file: 21 | path: "{{ item }}" 22 | state: directory 23 | mode: 0755 24 | loop: 25 | - /etc/rancher/k3s 26 | - /etc/rancher/node 27 | - "{{ k3s_data_dir }}/agent/pod-manifests" 28 | 29 | - name: Check if k3s token file exists on the first node 30 | run_once: true 31 | ansible.builtin.stat: 32 | path: "{{ k3s_token_file }}" 33 | register: k3s_token_file_stat 34 | 35 | - name: Generate k3s token file on the first node if not exist yet 36 | run_once: true 37 | when: not k3s_token_file_stat.stat.exists 38 | ansible.builtin.copy: 39 | content: "{{ lookup('community.general.random_string', length=32) }}" 40 | dest: "{{ k3s_token_file }}" 41 | mode: 0600 42 | 43 | - name: Get k3s token from the first node 44 | run_once: true 45 | ansible.builtin.slurp: 46 | src: "{{ k3s_token_file }}" 47 | register: k3s_token_base64 48 | 49 | - name: Ensure all nodes has the same token 50 | ansible.builtin.copy: 51 | content: "{{ k3s_token_base64.content | b64decode }}" 52 | dest: "{{ k3s_token_file }}" 53 | mode: 0600 54 | 55 | - name: Copy k3s config files 56 | ansible.builtin.template: 57 | src: "{{ item.src }}" 58 | dest: "{{ item.dest }}" 59 | mode: 0644 60 | loop: 61 | - src: config.yaml.j2 62 | dest: "{{ k3s_config_file }}" 63 | - src: k3s.service.j2 64 | dest: "{{ k3s_service_file }}" 65 | 66 | - name: Copy kube-vip manifests 67 | when: "'masters' in group_names" 68 | ansible.builtin.template: 69 | src: "{{ item.src }}" 70 | dest: "{{ item.dest }}" 71 | mode: 0644 72 | loop: 73 | - src: kube-vip.yaml.j2 74 | dest: "{{ k3s_data_dir }}/agent/pod-manifests/kube-vip.yaml" 75 | 76 | - name: Enable k3s service 77 | ansible.builtin.systemd: 78 | name: k3s 79 | enabled: true 80 | state: started 81 | register: k3s_service 82 | until: k3s_service is succeeded 83 | retries: 5 84 | 85 | - name: Get Kubernetes config file 86 | run_once: true 87 | ansible.builtin.slurp: 88 | src: "{{ k3s_kubeconfig_file }}" 89 | register: kubeconfig_base64 90 | 91 | - name: Write Kubernetes config file with the correct cluster address 92 | ansible.builtin.copy: 93 | content: "{{ kubeconfig_base64.content | b64decode | replace('127.0.0.1', control_plane_endpoint) }}" 94 | dest: "{{ playbook_dir }}/kubeconfig.yaml" 95 | mode: 0600 96 | delegate_to: localhost 97 | run_once: true 98 | -------------------------------------------------------------------------------- /metal/roles/k3s/templates/config.yaml.j2: -------------------------------------------------------------------------------- 1 | {% if inventory_hostname == groups['masters'][0] %} 2 | cluster-init: true 3 | {% else %} 4 | server: https://{{ control_plane_endpoint }}:6443 5 | {% endif %} 6 | token-file: {{ k3s_token_file }} 7 | {% if 'masters' in group_names %} 8 | {{ k3s_server_config | to_nice_yaml }} 9 | {% endif %} 10 | snapshotter: stargz 11 | -------------------------------------------------------------------------------- /metal/roles/k3s/templates/k3s.service.j2: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Lightweight Kubernetes 3 | Documentation=https://k3s.io 4 | After=network-online.target 5 | 6 | [Service] 7 | Type=notify 8 | ExecStartPre=-/sbin/modprobe br_netfilter 9 | ExecStartPre=-/sbin/modprobe overlay 10 | ExecStart=/usr/local/bin/k3s {{ 'server' if 'masters' in group_names else 'agent' }} 11 | KillMode=process 12 | Delegate=yes 13 | # Having non-zero Limit*s causes performance problems due to accounting overhead 14 | # in the kernel. We recommend using cgroups to do container-local accounting. 15 | LimitNOFILE=1048576 16 | LimitNPROC=infinity 17 | LimitCORE=infinity 18 | TasksMax=infinity 19 | TimeoutStartSec=0 20 | Restart=always 21 | RestartSec=5s 22 | 23 | [Install] 24 | WantedBy=multi-user.target 25 | -------------------------------------------------------------------------------- /metal/roles/k3s/templates/kube-vip.yaml.j2: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: kube-vip 5 | namespace: kube-system 6 | spec: 7 | containers: 8 | - name: kube-vip 9 | image: ghcr.io/kube-vip/kube-vip:v0.6.4 10 | args: 11 | - manager 12 | env: 13 | - name: address 14 | value: {{ control_plane_endpoint }} 15 | - name: vip_arp 16 | value: "true" 17 | - name: cp_enable 18 | value: "true" 19 | - name: vip_leaderelection 20 | value: "true" 21 | - name: lb_enable 22 | value: "true" 23 | securityContext: 24 | capabilities: 25 | add: 26 | - NET_ADMIN 27 | - NET_RAW 28 | volumeMounts: 29 | - mountPath: /etc/kubernetes/admin.conf 30 | name: kubeconfig 31 | hostAliases: 32 | - hostnames: 33 | - kubernetes 34 | ip: 127.0.0.1 35 | hostNetwork: true 36 | volumes: 37 | - hostPath: 38 | path: {{ k3s_kubeconfig_file }} 39 | name: kubeconfig 40 | -------------------------------------------------------------------------------- /metal/roles/prerequisites/tasks/main.yml: -------------------------------------------------------------------------------- 1 | - name: Adjust kernel parameters 2 | ansible.posix.sysctl: 3 | name: "{{ item.name }}" 4 | value: "{{ item.value }}" 5 | loop: 6 | - {name: "fs.inotify.max_queued_events", value: 16384} 7 | - {name: "fs.inotify.max_user_instances", value: 8192} 8 | - {name: "fs.inotify.max_user_watches", value: 524288} 9 | -------------------------------------------------------------------------------- /metal/roles/pxe_server/defaults/main.yml: -------------------------------------------------------------------------------- 1 | iso_url: "https://download.fedoraproject.org/pub/fedora/linux/releases/39/Server/x86_64/iso/Fedora-Server-dvd-x86_64-39-1.5.iso" 2 | iso_checksum: "sha256:2755cdff6ac6365c75be60334bf1935ade838fc18de53d4c640a13d3e904f6e9" 3 | timezone: Asia/Ho_Chi_Minh 4 | dhcp_proxy: true 5 | -------------------------------------------------------------------------------- /metal/roles/pxe_server/files/data/init-config/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | !.gitignore 3 | -------------------------------------------------------------------------------- /metal/roles/pxe_server/files/data/iso/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | !.gitignore 3 | -------------------------------------------------------------------------------- /metal/roles/pxe_server/files/data/os/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | !.gitignore 3 | -------------------------------------------------------------------------------- /metal/roles/pxe_server/files/data/pxe-config/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | !.gitignore 3 | -------------------------------------------------------------------------------- /metal/roles/pxe_server/files/dnsmasq/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:3.19 2 | 3 | RUN apk --no-cache add dnsmasq 4 | 5 | ENTRYPOINT ["dnsmasq", "-k"] 6 | -------------------------------------------------------------------------------- /metal/roles/pxe_server/files/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | dnsmasq: 3 | build: ./dnsmasq 4 | volumes: 5 | - ./data/pxe-config/dnsmasq.conf:/etc/dnsmasq.conf 6 | - ./data/pxe-config/grub.cfg:/tftp/grub.cfg 7 | - ./data/os/EFI/BOOT/grubx64.efi:/tftp/grubx64.efi 8 | - ./data/os/images/pxeboot/initrd.img:/tftp/initrd.img 9 | - ./data/os/images/pxeboot/vmlinuz:/tftp/vmlinuz 10 | network_mode: host 11 | cap_add: 12 | - NET_ADMIN 13 | http: 14 | build: ./http 15 | network_mode: host 16 | volumes: 17 | - ./data/os:/usr/share/nginx/html/os 18 | - ./data/init-config/:/usr/share/nginx/html/init-config 19 | environment: 20 | NGINX_PORT: 80 21 | -------------------------------------------------------------------------------- /metal/roles/pxe_server/files/http/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM nginx:1.25-alpine 2 | -------------------------------------------------------------------------------- /metal/roles/pxe_server/tasks/main.yml: -------------------------------------------------------------------------------- 1 | - name: Get Docker info 2 | docker_host_info: {} 3 | register: docker_info_result 4 | 5 | - name: Ensure Docker is running on a supported operating system 6 | fail: 7 | msg: Docker host networking driver only works on Linux hosts, and is not supported on Docker Desktop for Mac or Windows (you can use a Linux VM with bridged networking instead) 8 | when: 9 | - docker_info_result.host_info.OperatingSystem == "Docker Desktop" 10 | 11 | - name: Download boot image 12 | ansible.builtin.get_url: 13 | url: "{{ iso_url }}" 14 | dest: "{{ role_path }}/files/data/iso/{{ iso_url | basename }}" 15 | checksum: "{{ iso_checksum }}" 16 | mode: 0644 17 | register: iso 18 | 19 | - name: Extract boot image 20 | ansible.builtin.command: 21 | cmd: "xorriso -osirrox on -indev {{ iso.dest }} -extract / {{ role_path }}/files/data/os" 22 | creates: "{{ role_path }}/files/data/os/.treeinfo" 23 | 24 | - name: Generate dnsmasq config 25 | ansible.builtin.template: 26 | src: dnsmasq.conf.j2 27 | dest: "{{ role_path }}/files/data/pxe-config/dnsmasq.conf" 28 | mode: 0644 29 | 30 | - name: Generate GRUB config 31 | ansible.builtin.template: 32 | src: grub.cfg.j2 33 | dest: "{{ role_path }}/files/data/pxe-config/grub.cfg" 34 | mode: 0644 35 | 36 | - name: Generate init config for each machine 37 | ansible.builtin.template: 38 | src: kickstart.ks.j2 39 | dest: "{{ role_path }}/files/data/init-config/{{ hostvars[item]['mac'] }}.ks" 40 | mode: 0644 41 | loop: "{{ groups['metal'] }}" 42 | 43 | - name: Start the ephemeral PXE server 44 | community.docker.docker_compose_v2: 45 | project_src: "{{ role_path }}/files" 46 | state: present 47 | build: always 48 | -------------------------------------------------------------------------------- /metal/roles/pxe_server/templates/dnsmasq.conf.j2: -------------------------------------------------------------------------------- 1 | # Disable DNS Server. 2 | port=0 3 | {% if dhcp_proxy == true %} 4 | # We're DHCP proxying on the network of the homelab host 5 | dhcp-range={{ ansible_default_ipv4.address }},proxy 6 | pxe-service=X86-64_EFI, "Boot From Network, (UEFI)", grubx64.efi 7 | {% else %} 8 | # We're DHCP configuring on this range 9 | dhcp-range={{ ansible_default_ipv4.network | ansible.netcommon.ipmath(1) }},{{ ansible_default_ipv4.broadcast | ansible.netcommon.ipmath(-1) }},{{ ansible_default_ipv4.netmask }},12h 10 | dhcp-option=3,{{ ansible_default_ipv4.gateway }} 11 | 12 | # Match Arch Types efi x86 and x64 13 | dhcp-match=set:efi-x86_64,option:client-arch,7 14 | dhcp-match=set:efi-x86_64,option:client-arch,9 15 | 16 | # Set the Boot file based on the tag from above 17 | dhcp-boot=tag:efi-x86_64,grubx64.efi 18 | {% endif %} 19 | # Log DHCP queries to stdout 20 | log-queries 21 | log-dhcp 22 | log-facility=- 23 | 24 | # Enable TFTP server 25 | enable-tftp 26 | tftp-root=/tftp 27 | -------------------------------------------------------------------------------- /metal/roles/pxe_server/templates/grub.cfg.j2: -------------------------------------------------------------------------------- 1 | set timeout=1 2 | 3 | menuentry '{{ iso_url | basename | splitext | first }} (PXE)' { 4 | linux vmlinuz \ 5 | ip=dhcp \ 6 | inst.ks=http://{{ ansible_default_ipv4.address }}/init-config/${net_default_mac}.ks 7 | initrd initrd.img 8 | } 9 | -------------------------------------------------------------------------------- /metal/roles/pxe_server/templates/kickstart.ks.j2: -------------------------------------------------------------------------------- 1 | #version=RHEL8 2 | 3 | # Do not use graphical install 4 | text 5 | 6 | # Keyboard layouts 7 | keyboard --xlayouts='us' 8 | # System language 9 | lang en_US.UTF-8 10 | 11 | # Partition clearing information 12 | clearpart --all --drives={{ hostvars[item]['disk'] }} 13 | # Partitioning 14 | ignoredisk --only-use={{ hostvars[item]['disk'] }} 15 | partition /boot/efi --fstype=vfat --size=512 16 | partition / --fstype=ext4 --size=65536 17 | 18 | # Network information 19 | network --bootproto=static --device={{ hostvars[item]['network_interface'] }} --ip={{ hostvars[item]['ansible_host'] }} --gateway={{ ansible_default_ipv4.gateway }} --nameserver={{ dns_server }} --netmask={{ ansible_default_ipv4.netmask }} --ipv6=auto --hostname={{ hostvars[item]['inventory_hostname'] }} --activate 20 | 21 | # Use network installation 22 | repo --name="Repository" --baseurl=http://{{ ansible_default_ipv4.address }}/os 23 | url --url="http://{{ ansible_default_ipv4.address }}/os" 24 | # Disable Setup Agent on first boot 25 | firstboot --disable 26 | # Do not configure the X Window System 27 | skipx 28 | # Enable NTP 29 | services --enabled="chronyd" 30 | # System timezone 31 | timezone {{ timezone }} --utc 32 | 33 | # Create user (locked by default) 34 | user --groups=wheel --name=admin 35 | # Add SSH key 36 | sshkey --username=root "{{ ssh_public_key }}" 37 | # Disable root password login 38 | rootpw --lock 39 | 40 | # Disable SELinux 41 | selinux --disabled 42 | 43 | # Disable firewall 44 | firewall --disabled 45 | 46 | %packages 47 | @^custom-environment 48 | openssh-server 49 | %end 50 | 51 | # Create a raw partition for Ceph using the remaining space 52 | # Using a post script because there is no built-in feature in Kickstart 53 | # The three empty lines are equivalent to pressing Enter to use the default values for: 54 | # - Partition number 55 | # - First sector 56 | # - Last sector 57 | %post 58 | fdisk /dev/{{ hostvars[item]['disk'] }} << EOF 59 | new 60 | 61 | 62 | 63 | write 64 | EOF 65 | %end 66 | 67 | reboot 68 | -------------------------------------------------------------------------------- /metal/roles/wake/tasks/main.yml: -------------------------------------------------------------------------------- 1 | - name: Send Wake-on-LAN magic packets 2 | community.general.wakeonlan: 3 | mac: "{{ mac }}" 4 | delegate_to: localhost 5 | 6 | - name: Wait for the machines to come online 7 | ansible.builtin.wait_for_connection: 8 | timeout: 600 9 | -------------------------------------------------------------------------------- /mkdocs.yml: -------------------------------------------------------------------------------- 1 | # yaml-language-server: $schema=https://squidfunk.github.io/mkdocs-material/schema.json 2 | 3 | site_name: Khue's Homelab 4 | copyright: Copyright © 2020 - 2024 Khue Doan 5 | 6 | repo_url: https://github.com/khuedoan/homelab 7 | 8 | theme: 9 | favicon: https://github.com/khuedoan/homelab/assets/27996771/d33be1af-3687-4712-a671-4370da13cc92 10 | name: material 11 | palette: 12 | primary: black 13 | features: 14 | - navigation.expand 15 | - navigation.instant 16 | - navigation.sections 17 | - search.highlight 18 | - search.share 19 | 20 | markdown_extensions: 21 | - pymdownx.emoji: 22 | emoji_index: !!python/name:material.extensions.emoji.twemoji 23 | emoji_generator: !!python/name:material.extensions.emoji.to_svg 24 | - attr_list 25 | - admonition 26 | - pymdownx.details 27 | - pymdownx.snippets: 28 | check_paths: true 29 | - def_list 30 | - pymdownx.tasklist: 31 | - pymdownx.superfences: 32 | custom_fences: 33 | - name: mermaid 34 | class: mermaid 35 | format: !!python/name:pymdownx.superfences.fence_code_format 36 | - pymdownx.tabbed: 37 | alternate_style: true 38 | 39 | nav: 40 | - Home: index.md 41 | - Installation: 42 | - installation/sandbox.md 43 | - Production: 44 | - installation/production/prerequisites.md 45 | - installation/production/external-resources.md 46 | - installation/production/configuration.md 47 | - installation/production/deployment.md 48 | - installation/post-installation.md 49 | - Getting started: 50 | - getting-started/vpn-setup.md 51 | - getting-started/user-onboarding.md 52 | - getting-started/install-pre-commit-hooks.md 53 | - Concepts: 54 | - concepts/pxe-boot.md 55 | - concepts/secrets-management.md 56 | - concepts/certificate-management.md 57 | - concepts/development-shell.md 58 | - concepts/testing.md 59 | - How-to guides: 60 | - how-to-guides/alternate-dns-setup.md 61 | - how-to-guides/expose-services-to-the-internet.md 62 | - how-to-guides/backup-and-restore.md 63 | - how-to-guides/use-both-github-and-gitea.md 64 | - how-to-guides/add-or-remove-nodes.md 65 | - how-to-guides/run-commands-on-multiple-nodes.md 66 | - how-to-guides/single-node-cluster-adjustments.md 67 | - how-to-guides/disable-dhcp-proxy-in-dnsmasq.md 68 | - how-to-guides/media-management.md 69 | - how-to-guides/updating-documentation.md 70 | - Troubleshooting: 71 | - how-to-guides/troubleshooting/pxe-boot.md 72 | - Reference: 73 | - Architecture: 74 | - reference/architecture/overview.md 75 | - reference/architecture/networking.md 76 | - reference/architecture/decision-records.md 77 | - reference/license.md 78 | - reference/changelog.md 79 | - reference/roadmap.md 80 | - reference/contributing.md 81 | - reference/faq.md 82 | -------------------------------------------------------------------------------- /platform/dex/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: dex 3 | version: 0.0.0 4 | dependencies: 5 | - name: dex 6 | version: 0.16.0 7 | repository: https://charts.dexidp.io 8 | -------------------------------------------------------------------------------- /platform/dex/templates/secret.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: external-secrets.io/v1beta1 2 | kind: ExternalSecret 3 | metadata: 4 | name: dex-secrets 5 | namespace: {{ .Release.Namespace }} 6 | spec: 7 | secretStoreRef: 8 | kind: ClusterSecretStore 9 | name: global-secrets 10 | target: 11 | name: dex-secrets 12 | data: 13 | # Connectors 14 | - secretKey: KANIDM_CLIENT_ID 15 | remoteRef: 16 | key: kanidm.dex 17 | property: client_id 18 | - secretKey: KANIDM_CLIENT_SECRET 19 | remoteRef: 20 | key: kanidm.dex 21 | property: client_secret 22 | # Clients 23 | - secretKey: GRAFANA_SSO_CLIENT_SECRET 24 | remoteRef: 25 | key: dex.grafana 26 | property: client_secret 27 | - secretKey: GITEA_CLIENT_SECRET 28 | remoteRef: 29 | key: dex.gitea 30 | property: client_secret 31 | -------------------------------------------------------------------------------- /platform/dex/values.yaml: -------------------------------------------------------------------------------- 1 | dex: 2 | config: 3 | issuer: https://dex.khuedoan.com 4 | storage: 5 | type: kubernetes 6 | config: 7 | inCluster: true 8 | oauth2: 9 | skipApprovalScreen: true 10 | connectors: 11 | - type: oidc 12 | id: kanidm 13 | name: Kanidm 14 | config: 15 | clientID: $KANIDM_CLIENT_ID 16 | clientSecret: $KANIDM_CLIENT_SECRET 17 | redirectURI: https://dex.khuedoan.com/callback 18 | issuer: https://auth.khuedoan.com/oauth2/openid/dex 19 | # TODO https://github.com/dexidp/dex/pull/3188 20 | # enablePKCE: true 21 | scopes: 22 | - openid 23 | - profile 24 | - email 25 | - groups 26 | staticClients: 27 | - id: grafana-sso 28 | name: Grafana 29 | redirectURIs: 30 | - 'https://grafana.khuedoan.com/login/generic_oauth' 31 | secretEnv: GRAFANA_SSO_CLIENT_SECRET 32 | - id: gitea 33 | name: Gitea 34 | redirectURIs: 35 | - 'https://git.khuedoan.com/user/oauth2/Dex/callback' 36 | secretEnv: GITEA_CLIENT_SECRET 37 | envFrom: 38 | - secretRef: 39 | name: dex-secrets 40 | ingress: 41 | enabled: true 42 | className: nginx 43 | annotations: 44 | cert-manager.io/cluster-issuer: letsencrypt-prod 45 | hosts: 46 | - host: &host dex.khuedoan.com 47 | paths: 48 | - path: / 49 | pathType: ImplementationSpecific 50 | tls: 51 | - secretName: dex-tls-certificate 52 | hosts: 53 | - *host 54 | -------------------------------------------------------------------------------- /platform/external-secrets/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: external-secrets 3 | version: 0.0.0 4 | dependencies: 5 | - name: external-secrets 6 | version: 0.10.2 7 | repository: https://charts.external-secrets.io 8 | -------------------------------------------------------------------------------- /platform/gitea/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: gitea 3 | version: 0.0.0 4 | dependencies: 5 | - name: gitea 6 | version: 10.1.3 7 | repository: https://dl.gitea.io/charts/ 8 | -------------------------------------------------------------------------------- /platform/gitea/files/config/config.yaml: -------------------------------------------------------------------------------- 1 | # TODO create user and access token 2 | # users: 3 | # - name: renovate 4 | # fullName: Renovate 5 | # email: bot@renovateapp.com 6 | # tokenSecretRef: renovate-secret # ??? 7 | organizations: 8 | - name: ops 9 | description: Operations 10 | teams: 11 | - name: Owners 12 | members: 13 | - renovate 14 | repositories: 15 | - name: homelab 16 | owner: ops 17 | private: false 18 | migrate: 19 | source: https://github.com/khuedoan/homelab 20 | mirror: false 21 | - name: blog 22 | owner: khuedoan 23 | migrate: 24 | source: https://github.com/khuedoan/blog 25 | mirror: true 26 | - name: backstage 27 | owner: khuedoan 28 | migrate: 29 | source: https://github.com/khuedoan/backstage 30 | mirror: true 31 | -------------------------------------------------------------------------------- /platform/gitea/files/config/go.mod: -------------------------------------------------------------------------------- 1 | module git.khuedoan.com/khuedoan/homelab/gitea/config 2 | 3 | go 1.19 4 | 5 | require ( 6 | code.gitea.io/sdk/gitea v0.15.1 7 | gopkg.in/yaml.v2 v2.4.0 8 | ) 9 | 10 | require github.com/hashicorp/go-version v1.2.1 // indirect 11 | -------------------------------------------------------------------------------- /platform/gitea/files/config/go.sum: -------------------------------------------------------------------------------- 1 | code.gitea.io/gitea-vet v0.2.1/go.mod h1:zcNbT/aJEmivCAhfmkHOlT645KNOf9W2KnkLgFjGGfE= 2 | code.gitea.io/sdk/gitea v0.15.1 h1:WJreC7YYuxbn0UDaPuWIe/mtiNKTvLN8MLkaw71yx/M= 3 | code.gitea.io/sdk/gitea v0.15.1/go.mod h1:klY2LVI3s3NChzIk/MzMn7G1FHrfU7qd63iSMVoHRBA= 4 | github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= 5 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 6 | github.com/hashicorp/go-version v1.2.1 h1:zEfKbn2+PDgroKdiOzqiE8rsmLqU2uwi5PB5pBJ3TkI= 7 | github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= 8 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 9 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 10 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 11 | github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= 12 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 13 | github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 14 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 15 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 16 | golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 17 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 18 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 19 | golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 20 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 21 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 22 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 23 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 24 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 25 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 26 | golang.org/x/tools v0.0.0-20200325010219-a49f79bcc224/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= 27 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 28 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 29 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 30 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 31 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 32 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 33 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 34 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 35 | -------------------------------------------------------------------------------- /platform/gitea/files/config/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | // TODO WIP clean this up 4 | 5 | import ( 6 | "log" 7 | "os" 8 | 9 | "code.gitea.io/sdk/gitea" 10 | "gopkg.in/yaml.v2" 11 | ) 12 | 13 | type Organization struct { 14 | Name string 15 | Description string 16 | } 17 | 18 | type Repository struct { 19 | Name string 20 | Owner string 21 | Private bool 22 | Migrate struct { 23 | Source string 24 | Mirror bool 25 | } 26 | } 27 | 28 | type Config struct { 29 | Organizations []Organization 30 | Repositories []Repository 31 | } 32 | 33 | func main() { 34 | data, err := os.ReadFile("./config.yaml") 35 | 36 | if err != nil { 37 | log.Fatalf("Unable to read config file: %v", err) 38 | } 39 | 40 | config := Config{} 41 | 42 | err = yaml.Unmarshal([]byte(data), &config) 43 | 44 | if err != nil { 45 | log.Fatalf("error: %v", err) 46 | } 47 | 48 | gitea_host := os.Getenv("GITEA_HOST") 49 | gitea_user := os.Getenv("GITEA_USER") 50 | gitea_password := os.Getenv("GITEA_PASSWORD") 51 | 52 | options := (gitea.SetBasicAuth(gitea_user, gitea_password)) 53 | client, err := gitea.NewClient(gitea_host, options) 54 | 55 | if err != nil { 56 | log.Fatal(err) 57 | } 58 | 59 | for _, org := range config.Organizations { 60 | _, _, err = client.CreateOrg(gitea.CreateOrgOption{ 61 | Name: org.Name, 62 | Description: org.Description, 63 | }) 64 | 65 | if err != nil { 66 | log.Printf("Create organization %s: %v", org.Name, err) 67 | } 68 | } 69 | 70 | for _, repo := range config.Repositories { 71 | if repo.Migrate.Source != "" { 72 | _, _, err = client.MigrateRepo(gitea.MigrateRepoOption{ 73 | RepoName: repo.Name, 74 | RepoOwner: repo.Owner, 75 | CloneAddr: repo.Migrate.Source, 76 | Service: gitea.GitServicePlain, 77 | Mirror: repo.Migrate.Mirror, 78 | Private: repo.Private, 79 | MirrorInterval: "10m", 80 | }) 81 | 82 | if err != nil { 83 | log.Printf("Migrate %s/%s: %v", repo.Owner, repo.Name, err) 84 | } 85 | } else { 86 | _, _, err = client.AdminCreateRepo(repo.Owner, gitea.CreateRepoOption{ 87 | Name: repo.Name, 88 | // Description: "TODO", 89 | Private: repo.Private, 90 | }) 91 | } 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /platform/gitea/templates/admin-secret.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: external-secrets.io/v1beta1 2 | kind: ExternalSecret 3 | metadata: 4 | name: {{ .Values.gitea.gitea.admin.existingSecret }} 5 | namespace: {{ .Release.Namespace }} 6 | spec: 7 | secretStoreRef: 8 | kind: ClusterSecretStore 9 | name: global-secrets 10 | target: 11 | template: 12 | engineVersion: v2 13 | data: 14 | username: gitea_admin 15 | password: {{` "{{ .password }}" `}} 16 | data: 17 | - secretKey: password 18 | remoteRef: 19 | key: gitea.admin 20 | property: password 21 | -------------------------------------------------------------------------------- /platform/gitea/templates/config-job.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: batch/v1 2 | kind: Job 3 | metadata: 4 | name: gitea-config-{{ include (print $.Template.BasePath "/config-source.yaml") . | sha256sum | trunc 7 }} 5 | namespace: {{ .Release.Namespace }} 6 | annotations: 7 | argocd.argoproj.io/sync-wave: "1" 8 | spec: 9 | backoffLimit: 10 10 | template: 11 | spec: 12 | restartPolicy: Never 13 | containers: 14 | - name: apply 15 | image: golang:1.19-alpine 16 | env: 17 | - name: GITEA_HOST 18 | value: http://gitea-http:3000 19 | - name: GITEA_USER 20 | valueFrom: 21 | secretKeyRef: 22 | name: gitea-admin-secret 23 | key: username 24 | - name: GITEA_PASSWORD 25 | valueFrom: 26 | secretKeyRef: 27 | name: gitea-admin-secret 28 | key: password 29 | workingDir: /go/src/gitea-config 30 | command: 31 | - sh 32 | - -c 33 | args: 34 | - | 35 | go get . 36 | go run . 37 | volumeMounts: 38 | - name: source 39 | mountPath: /go/src/gitea-config 40 | volumes: 41 | - name: source 42 | configMap: 43 | name: gitea-config-source 44 | -------------------------------------------------------------------------------- /platform/gitea/templates/config-source.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: gitea-config-source 5 | namespace: {{ .Release.Namespace }} 6 | data: 7 | {{ (.Files.Glob "files/config/*").AsConfig | indent 2 }} 8 | -------------------------------------------------------------------------------- /platform/gitea/values.yaml: -------------------------------------------------------------------------------- 1 | gitea: 2 | ingress: 3 | enabled: true 4 | className: nginx 5 | annotations: 6 | cert-manager.io/cluster-issuer: letsencrypt-prod 7 | hosts: 8 | - host: &host git.khuedoan.com 9 | paths: 10 | - path: / 11 | pathType: Prefix 12 | tls: 13 | - secretName: gitea-tls-certificate 14 | hosts: 15 | - *host 16 | gitea: 17 | admin: 18 | existingSecret: gitea-admin-secret 19 | config: 20 | server: 21 | LANDING_PAGE: explore 22 | ROOT_URL: https://git.khuedoan.com 23 | OFFLINE_MODE: true 24 | repository: 25 | DISABLED_REPO_UNITS: repo.wiki,repo.projects,repo.packages 26 | DISABLE_STARS: true 27 | DEFAULT_BRANCH: master 28 | # TODO it's not reading the username from Dex correctly for now, related issues: 29 | # https://github.com/go-gitea/gitea/issues/25725 30 | # https://github.com/go-gitea/gitea/issues/24957 31 | # oauth2_client: 32 | # ENABLE_AUTO_REGISTRATION: true 33 | # USERNAME: userid 34 | service.explore: 35 | DISABLE_USERS_PAGE: true 36 | actions: 37 | ENABLED: false 38 | webhook: 39 | ALLOWED_HOST_LIST: private 40 | -------------------------------------------------------------------------------- /platform/global-secrets/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: global-secrets 3 | version: 0.0.0 4 | -------------------------------------------------------------------------------- /platform/global-secrets/files/secret-generator/config.yaml: -------------------------------------------------------------------------------- 1 | # Gitea 2 | - name: gitea.admin 3 | data: 4 | - key: password 5 | length: 32 6 | special: true 7 | 8 | # Dex 9 | - name: dex.grafana 10 | data: 11 | - key: client_secret 12 | length: 32 13 | special: false 14 | - name: dex.gitea 15 | data: 16 | - key: client_secret 17 | length: 32 18 | special: false 19 | 20 | # Registry 21 | - name: registry.admin 22 | data: 23 | - key: password 24 | length: 32 25 | special: true 26 | 27 | # Woodpecker 28 | - name: woodpecker.agent 29 | data: 30 | - key: secret 31 | length: 32 32 | special: false 33 | 34 | # Paperless 35 | - name: paperless.admin 36 | data: 37 | - key: PAPERLESS_ADMIN_PASSWORD 38 | length: 32 39 | special: true 40 | -------------------------------------------------------------------------------- /platform/global-secrets/files/secret-generator/go.mod: -------------------------------------------------------------------------------- 1 | module git.khuedoan.com/khuedoan/homelab/platform/secret-generator 2 | 3 | go 1.19 4 | 5 | require ( 6 | github.com/sethvargo/go-password v0.2.0 7 | gopkg.in/yaml.v2 v2.4.0 8 | k8s.io/api v0.28.4 9 | k8s.io/apimachinery v0.28.4 10 | k8s.io/client-go v0.28.4 11 | ) 12 | 13 | require ( 14 | github.com/davecgh/go-spew v1.1.1 // indirect 15 | github.com/emicklei/go-restful/v3 v3.9.0 // indirect 16 | github.com/go-logr/logr v1.2.4 // indirect 17 | github.com/go-openapi/jsonpointer v0.19.6 // indirect 18 | github.com/go-openapi/jsonreference v0.20.2 // indirect 19 | github.com/go-openapi/swag v0.22.3 // indirect 20 | github.com/gogo/protobuf v1.3.2 // indirect 21 | github.com/golang/protobuf v1.5.3 // indirect 22 | github.com/google/gnostic-models v0.6.8 // indirect 23 | github.com/google/go-cmp v0.5.9 // indirect 24 | github.com/google/gofuzz v1.2.0 // indirect 25 | github.com/google/uuid v1.3.0 // indirect 26 | github.com/imdario/mergo v0.3.6 // indirect 27 | github.com/josharian/intern v1.0.0 // indirect 28 | github.com/json-iterator/go v1.1.12 // indirect 29 | github.com/mailru/easyjson v0.7.7 // indirect 30 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 31 | github.com/modern-go/reflect2 v1.0.2 // indirect 32 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 33 | github.com/spf13/pflag v1.0.5 // indirect 34 | golang.org/x/net v0.17.0 // indirect 35 | golang.org/x/oauth2 v0.8.0 // indirect 36 | golang.org/x/sys v0.13.0 // indirect 37 | golang.org/x/term v0.13.0 // indirect 38 | golang.org/x/text v0.13.0 // indirect 39 | golang.org/x/time v0.3.0 // indirect 40 | google.golang.org/appengine v1.6.7 // indirect 41 | google.golang.org/protobuf v1.31.0 // indirect 42 | gopkg.in/inf.v0 v0.9.1 // indirect 43 | gopkg.in/yaml.v3 v3.0.1 // indirect 44 | k8s.io/klog/v2 v2.100.1 // indirect 45 | k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect 46 | k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 // indirect 47 | sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect 48 | sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect 49 | sigs.k8s.io/yaml v1.3.0 // indirect 50 | ) 51 | -------------------------------------------------------------------------------- /platform/global-secrets/files/secret-generator/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "log" 7 | "math" 8 | "os" 9 | 10 | "github.com/sethvargo/go-password/password" 11 | "gopkg.in/yaml.v2" 12 | v1 "k8s.io/api/core/v1" 13 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 14 | "k8s.io/client-go/kubernetes" 15 | "k8s.io/client-go/tools/clientcmd" 16 | ) 17 | 18 | const namespace = "global-secrets" 19 | 20 | type RandomSecret struct { 21 | Name string 22 | Data []struct { 23 | Key string 24 | Length int 25 | Special bool 26 | } 27 | } 28 | 29 | func getClient() (*kubernetes.Clientset, error) { 30 | rules := clientcmd.NewDefaultClientConfigLoadingRules() 31 | overrides := &clientcmd.ConfigOverrides{} 32 | 33 | config, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(rules, overrides).ClientConfig() 34 | if err != nil { 35 | return nil, fmt.Errorf("Error building client config: %v", err) 36 | } 37 | 38 | return kubernetes.NewForConfig(config) 39 | } 40 | 41 | func generateRandomPassword(length int, special bool) (string, error) { 42 | numDigits := int(math.Ceil(float64(length) * 0.2)) 43 | numSymbols := 0 44 | 45 | if special { 46 | numSymbols = int(math.Ceil(float64(length) * 0.2)) 47 | } 48 | 49 | return password.Generate(length, numDigits, numSymbols, false, true) 50 | } 51 | 52 | func readConfigFile(filename string) ([]RandomSecret, error) { 53 | data, err := os.ReadFile(filename) 54 | if err != nil { 55 | return nil, fmt.Errorf("Unable to read config file: %v", err) 56 | } 57 | 58 | var randomSecrets []RandomSecret 59 | err = yaml.Unmarshal(data, &randomSecrets) 60 | if err != nil { 61 | return nil, fmt.Errorf("Error parsing config file: %v", err) 62 | } 63 | 64 | return randomSecrets, nil 65 | } 66 | 67 | func createOrUpdateSecret(client *kubernetes.Clientset, name string, randomSecret RandomSecret) error { 68 | secret, err := client.CoreV1().Secrets(namespace).Get(context.Background(), name, metav1.GetOptions{}) 69 | 70 | if err != nil { 71 | // Secret not found, create a new one 72 | secretData := map[string][]byte{} 73 | 74 | for _, randomPassword := range randomSecret.Data { 75 | password, err := generateRandomPassword(randomPassword.Length, randomPassword.Special) 76 | if err != nil { 77 | log.Printf("Error generating password for key '%s': %v", randomPassword.Key, err) 78 | continue 79 | } 80 | 81 | secretData[randomPassword.Key] = []byte(password) 82 | } 83 | 84 | newSecret := &v1.Secret{ 85 | ObjectMeta: metav1.ObjectMeta{ 86 | Name: name, 87 | Namespace: namespace, 88 | }, 89 | Data: secretData, 90 | } 91 | 92 | _, err := client.CoreV1().Secrets(namespace).Create(context.Background(), newSecret, metav1.CreateOptions{}) 93 | if err != nil { 94 | return fmt.Errorf("Unable to create secret: %v", err) 95 | } 96 | log.Printf("Secret '%s' created successfully.", name) 97 | } else { 98 | // Secret exists, check for new keys 99 | for _, randomKey := range randomSecret.Data { 100 | if _, exists := secret.Data[randomKey.Key]; !exists { 101 | // New key found, generate new password 102 | log.Printf("New key '%s' found in config for secret '%s', generating new password", randomKey.Key, name) 103 | password, err := generateRandomPassword(randomKey.Length, randomKey.Special) 104 | if err != nil { 105 | log.Printf("Error generating password for key '%s': %v", randomKey.Key, err) 106 | continue 107 | } 108 | secret.Data[randomKey.Key] = []byte(password) 109 | } 110 | } 111 | 112 | // Update the secret 113 | _, err := client.CoreV1().Secrets(namespace).Update(context.Background(), secret, metav1.UpdateOptions{}) 114 | if err != nil { 115 | return fmt.Errorf("Unable to update secret: %v", err) 116 | } 117 | } 118 | 119 | return nil 120 | } 121 | 122 | func main() { 123 | configFilename := "./config.yaml" 124 | randomSecrets, err := readConfigFile(configFilename) 125 | if err != nil { 126 | log.Fatalf("Error reading config file: %v", err) 127 | } 128 | 129 | client, err := getClient() 130 | if err != nil { 131 | log.Fatalf("Unable to create Kubernetes client: %v", err) 132 | } 133 | 134 | for _, randomSecret := range randomSecrets { 135 | err := createOrUpdateSecret(client, randomSecret.Name, randomSecret) 136 | if err != nil { 137 | log.Printf("Error processing secret %s: %v", randomSecret.Name, err) 138 | } 139 | } 140 | } 141 | -------------------------------------------------------------------------------- /platform/global-secrets/templates/clustersecretstore/clustersecretstore.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: external-secrets.io/v1beta1 2 | kind: ClusterSecretStore 3 | metadata: 4 | name: global-secrets 5 | spec: 6 | provider: 7 | kubernetes: 8 | remoteNamespace: {{ .Release.Namespace }} 9 | server: 10 | caProvider: 11 | type: ConfigMap 12 | name: kube-root-ca.crt 13 | namespace: {{ .Release.Namespace }} 14 | key: ca.crt 15 | auth: 16 | serviceAccount: 17 | name: external-secrets-kubernetes-global-secrets 18 | namespace: {{ .Release.Namespace }} 19 | -------------------------------------------------------------------------------- /platform/global-secrets/templates/clustersecretstore/role.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: Role 3 | metadata: 4 | name: external-secrets-kubernetes-global-secrets 5 | namespace: {{ .Release.Namespace }} 6 | rules: 7 | - apiGroups: 8 | - "" 9 | resources: 10 | - secrets 11 | verbs: 12 | - get 13 | - list 14 | - watch 15 | - apiGroups: 16 | - authorization.k8s.io 17 | resources: 18 | - selfsubjectrulesreviews 19 | verbs: 20 | - create 21 | -------------------------------------------------------------------------------- /platform/global-secrets/templates/clustersecretstore/rolebinding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: RoleBinding 3 | metadata: 4 | name: external-secrets-kubernetes-global-secrets 5 | namespace: {{ .Release.Namespace }} 6 | roleRef: 7 | apiGroup: rbac.authorization.k8s.io 8 | kind: Role 9 | name: external-secrets-kubernetes-global-secrets 10 | subjects: 11 | - kind: ServiceAccount 12 | name: external-secrets-kubernetes-global-secrets 13 | namespace: {{ .Release.Namespace }} 14 | -------------------------------------------------------------------------------- /platform/global-secrets/templates/clustersecretstore/serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: external-secrets-kubernetes-global-secrets 5 | namespace: {{ .Release.Namespace }} 6 | -------------------------------------------------------------------------------- /platform/global-secrets/templates/secret-generator/configmap.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: secret-generator 5 | namespace: {{ .Release.Namespace }} 6 | data: 7 | {{ (.Files.Glob "files/secret-generator/*").AsConfig | indent 2 }} 8 | -------------------------------------------------------------------------------- /platform/global-secrets/templates/secret-generator/job.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: batch/v1 2 | kind: Job 3 | metadata: 4 | name: secret-generator-{{ include (print $.Template.BasePath "/secret-generator/configmap.yaml") . | sha256sum | trunc 7 }} 5 | namespace: {{ .Release.Namespace }} 6 | spec: 7 | backoffLimit: 3 8 | template: 9 | spec: 10 | restartPolicy: Never 11 | containers: 12 | - name: secret-generator 13 | image: golang:1.19-alpine 14 | workingDir: /go/src/secret-generator 15 | command: 16 | - sh 17 | - -c 18 | args: 19 | - | 20 | go get . 21 | go run . 22 | volumeMounts: 23 | - name: source 24 | mountPath: /go/src/secret-generator 25 | serviceAccount: secret-generator 26 | volumes: 27 | - name: source 28 | configMap: 29 | name: secret-generator 30 | -------------------------------------------------------------------------------- /platform/global-secrets/templates/secret-generator/role.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: Role 3 | metadata: 4 | name: secret-generator 5 | namespace: {{ .Release.Namespace }} 6 | rules: 7 | - apiGroups: 8 | - "" 9 | resources: 10 | - secrets 11 | verbs: 12 | - get 13 | - list 14 | - create 15 | - update 16 | - patch 17 | - apiGroups: 18 | - authorization.k8s.io 19 | resources: 20 | - selfsubjectrulesreviews 21 | verbs: 22 | - create 23 | -------------------------------------------------------------------------------- /platform/global-secrets/templates/secret-generator/rolebinding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: RoleBinding 3 | metadata: 4 | name: secret-generator 5 | namespace: {{ .Release.Namespace }} 6 | roleRef: 7 | apiGroup: rbac.authorization.k8s.io 8 | kind: Role 9 | name: secret-generator 10 | subjects: 11 | - kind: ServiceAccount 12 | name: secret-generator 13 | namespace: {{ .Release.Namespace }} 14 | -------------------------------------------------------------------------------- /platform/global-secrets/templates/secret-generator/serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: secret-generator 5 | namespace: {{ .Release.Namespace }} 6 | -------------------------------------------------------------------------------- /platform/grafana/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: grafana 3 | version: 0.0.0 4 | dependencies: 5 | - name: grafana 6 | repository: https://grafana.github.io/helm-charts 7 | version: 7.3.3 8 | -------------------------------------------------------------------------------- /platform/grafana/templates/secret.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: external-secrets.io/v1beta1 2 | kind: ExternalSecret 3 | metadata: 4 | name: grafana-secrets 5 | namespace: {{ .Release.Namespace }} 6 | spec: 7 | secretStoreRef: 8 | kind: ClusterSecretStore 9 | name: global-secrets 10 | target: 11 | name: grafana-secrets 12 | data: 13 | - secretKey: GRAFANA_SSO_CLIENT_SECRET 14 | remoteRef: 15 | key: dex.grafana 16 | property: client_secret 17 | -------------------------------------------------------------------------------- /platform/grafana/values.yaml: -------------------------------------------------------------------------------- 1 | grafana: 2 | ingress: 3 | enabled: true 4 | ingressClassName: nginx 5 | annotations: 6 | cert-manager.io/cluster-issuer: letsencrypt-prod 7 | hosts: 8 | - &host grafana.khuedoan.com 9 | tls: 10 | - secretName: grafana-general-tls 11 | hosts: 12 | - *host 13 | sidecar: 14 | dashboards: 15 | enabled: true 16 | searchNamespace: monitoring-system 17 | datasources: 18 | enabled: true 19 | searchNamespace: monitoring-system 20 | envFromSecret: grafana-secrets 21 | grafana.ini: 22 | server: 23 | root_url: https://grafana.khuedoan.com 24 | auth.generic_oauth: 25 | enabled: true 26 | allow_sign_up: true 27 | name: Dex 28 | client_id: grafana-sso 29 | client_secret: $__env{GRAFANA_SSO_CLIENT_SECRET} 30 | scopes: openid profile email groups 31 | auth_url: https://dex.khuedoan.com/auth 32 | token_url: https://dex.khuedoan.com/token 33 | api_url: https://dex.khuedoan.com/userinfo 34 | -------------------------------------------------------------------------------- /platform/kanidm/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: kanidm 3 | version: 0.0.0 4 | dependencies: 5 | - name: app-template 6 | version: 2.2.0 7 | repository: https://bjw-s.github.io/helm-charts 8 | -------------------------------------------------------------------------------- /platform/kanidm/templates/certificate.yaml: -------------------------------------------------------------------------------- 1 | # TODO https://github.com/kanidm/kanidm/issues/1227 2 | apiVersion: cert-manager.io/v1 3 | kind: Certificate 4 | metadata: 5 | name: kanidm-selfsigned 6 | namespace: {{ .Release.Namespace }} 7 | spec: 8 | secretName: kanidm-selfsigned-certificate 9 | issuerRef: 10 | kind: Issuer 11 | name: kanidm-selfsigned 12 | dnsNames: 13 | - home.arpa 14 | -------------------------------------------------------------------------------- /platform/kanidm/templates/issuer.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: cert-manager.io/v1 2 | kind: Issuer 3 | metadata: 4 | name: kanidm-selfsigned 5 | namespace: {{ .Release.Namespace }} 6 | spec: 7 | selfSigned: {} 8 | -------------------------------------------------------------------------------- /platform/kanidm/values.yaml: -------------------------------------------------------------------------------- 1 | app-template: 2 | controllers: 3 | main: 4 | type: statefulset 5 | containers: 6 | main: 7 | image: 8 | repository: docker.io/kanidm/server 9 | tag: 1.3.3 10 | statefulset: 11 | volumeClaimTemplates: 12 | - name: data 13 | size: 1Gi 14 | globalMounts: 15 | - path: /data 16 | accessMode: "ReadWriteOnce" 17 | configMaps: 18 | config: 19 | enabled: true 20 | data: 21 | server.toml: | 22 | bindaddress = "[::]:443" 23 | ldapbindaddress = "[::]:636" 24 | trust_x_forward_for = true 25 | db_path = "/data/kanidm.db" 26 | tls_chain = "/data/ca.crt" 27 | tls_key = "/data/tls.key" 28 | domain = "auth.khuedoan.com" 29 | origin = "https://auth.khuedoan.com" 30 | service: 31 | main: 32 | ports: 33 | http: 34 | enabled: false 35 | https: 36 | port: 443 37 | protocol: HTTPS 38 | ldap: 39 | port: 636 40 | protocol: TCP 41 | ingress: 42 | main: 43 | enabled: true 44 | className: nginx 45 | annotations: 46 | cert-manager.io/cluster-issuer: letsencrypt-prod 47 | nginx.ingress.kubernetes.io/backend-protocol: HTTPS 48 | hosts: 49 | - host: &host auth.khuedoan.com 50 | paths: 51 | - path: / 52 | pathType: Prefix 53 | service: 54 | name: main 55 | port: https 56 | tls: 57 | - hosts: 58 | - *host 59 | secretName: kanidm-tls-certificate 60 | persistence: 61 | config: 62 | enabled: true 63 | type: configMap 64 | name: kanidm-config 65 | globalMounts: 66 | - path: /data/server.toml 67 | subPath: server.toml 68 | tls: 69 | enabled: true 70 | type: secret 71 | name: kanidm-selfsigned-certificate 72 | globalMounts: 73 | - path: /data/ca.crt 74 | subPath: ca.crt 75 | - path: /data/tls.key 76 | subPath: tls.key 77 | -------------------------------------------------------------------------------- /platform/renovate/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: renovate 3 | version: 0.0.0 4 | dependencies: 5 | - name: renovate 6 | version: 31.97.3 7 | repository: https://docs.renovatebot.com/helm-charts 8 | -------------------------------------------------------------------------------- /platform/renovate/templates/secret.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: external-secrets.io/v1beta1 2 | kind: ExternalSecret 3 | metadata: 4 | name: {{ .Values.renovate.existingSecret }} 5 | namespace: {{ .Release.Namespace }} 6 | spec: 7 | secretStoreRef: 8 | kind: ClusterSecretStore 9 | name: global-secrets 10 | target: 11 | template: 12 | engineVersion: v2 13 | data: 14 | RENOVATE_TOKEN: {{` "{{ .token }}" `}} 15 | data: 16 | - secretKey: token 17 | remoteRef: 18 | key: gitea.renovate 19 | property: token 20 | -------------------------------------------------------------------------------- /platform/renovate/values.yaml: -------------------------------------------------------------------------------- 1 | renovate: 2 | cronjob: 3 | schedule: '0 9 * * *' # Everyday at 09:00 4 | renovate: 5 | config: | 6 | { 7 | "platform": "gitea", 8 | "endpoint": "https://git.khuedoan.com/api/v1", 9 | "gitAuthor": "Renovate Bot ", 10 | "autodiscover": true 11 | } 12 | existingSecret: renovate-secret 13 | -------------------------------------------------------------------------------- /platform/woodpecker/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: woodpecker 3 | version: 0.0.0 4 | dependencies: 5 | - name: woodpecker 6 | version: 1.5.1 7 | repository: https://woodpecker-ci.org 8 | -------------------------------------------------------------------------------- /platform/woodpecker/templates/secret.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: external-secrets.io/v1beta1 2 | kind: ExternalSecret 3 | metadata: 4 | name: woodpecker-secret 5 | namespace: {{ .Release.Namespace }} 6 | spec: 7 | secretStoreRef: 8 | kind: ClusterSecretStore 9 | name: global-secrets 10 | data: 11 | - secretKey: WOODPECKER_GITEA_CLIENT 12 | remoteRef: 13 | key: gitea.woodpecker 14 | property: client_id 15 | - secretKey: WOODPECKER_GITEA_SECRET 16 | remoteRef: 17 | key: gitea.woodpecker 18 | property: client_secret 19 | - secretKey: WOODPECKER_AGENT_SECRET 20 | remoteRef: 21 | key: woodpecker.agent 22 | property: secret 23 | -------------------------------------------------------------------------------- /platform/woodpecker/values.yaml: -------------------------------------------------------------------------------- 1 | woodpecker: 2 | agent: 3 | replicaCount: 2 4 | env: 5 | WOODPECKER_BACKEND_K8S_STORAGE_RWX: false 6 | # Agents will spawn pods to run workflow steps using the 7 | # Kubernetes backend instead of running them directly on 8 | # the agent pod, so we can run many workflows per agent. 9 | WOODPECKER_MAX_WORKFLOWS: 10 10 | server: 11 | env: 12 | WOODPECKER_HOST: https://ci.khuedoan.com 13 | WOODPECKER_WEBHOOK_HOST: http://woodpecker-server.woodpecker 14 | WOODPECKER_GITEA: true 15 | WOODPECKER_GITEA_URL: https://git.khuedoan.com 16 | WOODPECKER_OPEN: true 17 | WOODPECKER_ADMIN: gitea_admin 18 | ingress: 19 | enabled: true 20 | annotations: 21 | cert-manager.io/cluster-issuer: letsencrypt-prod 22 | ingressClassName: nginx 23 | hosts: 24 | - host: &host ci.khuedoan.com 25 | paths: 26 | - path: / 27 | tls: 28 | - secretName: woodpecker-tls-certificate 29 | hosts: 30 | - *host 31 | -------------------------------------------------------------------------------- /platform/zot/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: zot 3 | version: 0.0.0 4 | dependencies: 5 | - name: zot 6 | version: 0.1.52 7 | repository: http://zotregistry.dev/helm-charts 8 | -------------------------------------------------------------------------------- /platform/zot/templates/admin-secret.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: external-secrets.io/v1beta1 2 | kind: ExternalSecret 3 | metadata: 4 | name: registry-admin-secret 5 | namespace: {{ .Release.Namespace }} 6 | spec: 7 | secretStoreRef: 8 | kind: ClusterSecretStore 9 | name: global-secrets 10 | target: 11 | template: 12 | engineVersion: v2 13 | data: 14 | username: admin 15 | password: {{` "{{ .password }}" `}} 16 | data: 17 | - secretKey: password 18 | remoteRef: 19 | key: registry.admin 20 | property: password 21 | -------------------------------------------------------------------------------- /platform/zot/values.yaml: -------------------------------------------------------------------------------- 1 | zot: 2 | ingress: 3 | enabled: true 4 | annotations: 5 | cert-manager.io/cluster-issuer: letsencrypt-prod 6 | nginx.ingress.kubernetes.io/proxy-body-size: "0" 7 | className: nginx 8 | hosts: 9 | - host: &host registry.khuedoan.com 10 | paths: 11 | - path: / 12 | tls: 13 | - secretName: zot-tls-certificate 14 | hosts: 15 | - *host 16 | # TODO enable auth 17 | persistence: true 18 | pvc: 19 | create: true 20 | storage: 10Gi 21 | -------------------------------------------------------------------------------- /renovate.json5: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://docs.renovatebot.com/renovate-schema.json", 3 | "extends": [ 4 | "config:base" 5 | ], 6 | "packageRules": [ 7 | { 8 | "matchPackagePatterns": [ 9 | "*" 10 | ], 11 | "matchUpdateTypes": [ 12 | "minor", 13 | "patch" 14 | ], 15 | "groupName": "all non-major dependencies", 16 | "groupSlug": "all-minor-patch" 17 | } 18 | ] 19 | } 20 | -------------------------------------------------------------------------------- /scripts/argocd-admin-password: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | echo "WARNING: ArgoCD admin can do anything in the cluster, only use it for just enough initial setup or in emergencies." >&2 4 | export KUBECONFIG=./metal/kubeconfig.yaml 5 | kubectl -n argocd get secret argocd-initial-admin-secret -o jsonpath="{.data.password}" | base64 -d 6 | -------------------------------------------------------------------------------- /scripts/configure: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # WIP 4 | # TODO clean this up 5 | 6 | """ 7 | Basic configure script for new users 8 | """ 9 | 10 | import fileinput 11 | import subprocess 12 | import sys 13 | 14 | from rich.prompt import Confirm, Prompt 15 | 16 | upstream_config = { 17 | "seed_repo": "https://github.com/khuedoan/homelab", 18 | "domain": "khuedoan.com", 19 | "timezone": "Asia/Ho_Chi_Minh", 20 | "terraform_workspace": "khuedoan", 21 | "loadbalancer_ip_range": "192.168.1.224/27", 22 | } 23 | 24 | 25 | def check_python_version(required_version: str) -> None: 26 | if sys.version_info < tuple(map(int, required_version.split('.'))): 27 | raise Exception(f"Must be using Python >= {required_version}") 28 | 29 | 30 | def find_and_replace(pattern: str, replacement: str, paths: list[str]) -> None: 31 | files_with_matches = subprocess.run( 32 | ["git", "grep", "--files-with-matches", pattern, "--"] + paths, 33 | capture_output=True, 34 | text=True 35 | ).stdout.splitlines() 36 | 37 | for file_with_maches in files_with_matches: 38 | with fileinput.FileInput(file_with_maches, inplace=True) as file: 39 | for line in file: 40 | print(line.replace(pattern, replacement), end='') 41 | 42 | 43 | def main() -> None: 44 | check_python_version( 45 | required_version='3.10.0' 46 | ) 47 | 48 | editor = Prompt.ask("Select text editor", default='nvim') 49 | domain = Prompt.ask("Enter your domain", default=upstream_config['domain']) 50 | seed_repo = Prompt.ask("Enter seed repo", default=upstream_config['seed_repo']) 51 | timezone = Prompt.ask("Enter time zone", default=upstream_config['timezone']) 52 | loadbalancer_ip_range = Prompt.ask("Enter IP range for load balancer", default=upstream_config['loadbalancer_ip_range']) 53 | 54 | find_and_replace( 55 | pattern=upstream_config['domain'], 56 | replacement=domain, 57 | paths=[ 58 | ".ci", 59 | "apps", 60 | "platform", 61 | "system", 62 | "external" 63 | ] 64 | ) 65 | 66 | find_and_replace( 67 | pattern=upstream_config['seed_repo'], 68 | replacement=seed_repo, 69 | paths=[ 70 | "system", 71 | "platform" 72 | ] 73 | ) 74 | 75 | find_and_replace( 76 | pattern=upstream_config['timezone'], 77 | replacement=timezone, 78 | paths=[ 79 | "apps", 80 | "system", 81 | "metal" 82 | ] 83 | ) 84 | 85 | find_and_replace( 86 | pattern=upstream_config['loadbalancer_ip_range'], 87 | replacement=loadbalancer_ip_range, 88 | paths=[ 89 | "metal/inventories/prod.yml", 90 | "apps/tailscale/values.yaml", 91 | ] 92 | ) 93 | 94 | if Confirm.ask("Update server list?", default=True): 95 | subprocess.run( 96 | [editor, 'metal/inventories/prod.yml'] 97 | ) 98 | 99 | 100 | if Confirm.ask("Do you want to use managed services?"): 101 | terraform_workspace = Prompt.ask("Enter Terraform Workspace", default=upstream_config['terraform_workspace']) 102 | 103 | find_and_replace( 104 | pattern=upstream_config['terraform_workspace'], 105 | replacement=terraform_workspace, 106 | paths=[ 107 | "external/versions.tf" 108 | ] 109 | ) 110 | 111 | 112 | if __name__ == '__main__': 113 | main() 114 | -------------------------------------------------------------------------------- /scripts/get-dns-config: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | export KUBECONFIG=./metal/kubeconfig.yaml 4 | kubectl get ingress --all-namespaces --no-headers --output custom-columns="ADDRESS:.status.loadBalancer.ingress[0].ip,HOST:.spec.rules[0].host" 5 | -------------------------------------------------------------------------------- /scripts/get-status: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | export KUBECONFIG=./metal/kubeconfig.yaml 4 | 5 | kubectl get applicationsets --namespace argocd 6 | kubectl get applications --namespace argocd 7 | kubectl get ingress --all-namespaces 8 | -------------------------------------------------------------------------------- /scripts/get-wireguard-config: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -eu 4 | 5 | PEER="${1}" 6 | 7 | export KUBECONFIG=./metal/kubeconfig.yaml 8 | 9 | kubectl -n wireguard exec -it deployment/wireguard -- /app/show-peer "${PEER}" 10 | kubectl -n wireguard exec -it deployment/wireguard -- cat "/config/peer_${PEER}/peer_${PEER}.conf" 11 | -------------------------------------------------------------------------------- /scripts/helm-diff: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from argparse import ArgumentParser 4 | from glob import glob 5 | from os import path 6 | from subprocess import run 7 | from tempfile import mkdtemp, NamedTemporaryFile 8 | 9 | 10 | def clone_repository(repo, branch, target_path): 11 | run( 12 | ['git', 'clone', repo, '--depth', '1', '--branch', branch, target_path], 13 | check=True 14 | ) 15 | 16 | 17 | def render_helm_chart(chart_path, namespace, release_name, rendered_path): 18 | # Even if there is no Helm chart at the specified chart path, do not raise an error. 19 | # This accommodates cases where the entire chart is removed, or a new chart is added. 20 | # In such cases, the rendered file will simply be empty. 21 | if path.isdir(chart_path): 22 | run( 23 | ['helm', 'dependency', 'update', chart_path], 24 | check=True 25 | ) 26 | 27 | run( 28 | ['helm', 'template', '--namespace', namespace, release_name, chart_path], 29 | stdout=open(rendered_path, 'w'), 30 | check=True 31 | ) 32 | 33 | 34 | def changed_charts(source_path, target_path, subpath): 35 | changed_charts = [] 36 | 37 | # Convert to set for deduplication 38 | all_charts = set( 39 | glob(f"*", root_dir=f"{source_path}/{subpath}") 40 | + glob(f"*", root_dir=f"{target_path}/{subpath}") 41 | ) 42 | 43 | for chart in all_charts: 44 | source_chart_path = path.join(source_path, subpath, chart) 45 | target_chart_path = path.join(target_path, subpath, chart) 46 | 47 | if run(['diff', source_chart_path, target_chart_path], capture_output=True).returncode != 0: 48 | changed_charts.append(chart) 49 | 50 | return changed_charts 51 | 52 | 53 | def main(): 54 | parser = ArgumentParser(description='Compare Helm charts in a directory between two Git revisions.') 55 | parser.add_argument('--repository', required=True, help='Repository to clone') 56 | parser.add_argument('--source', required=True, help='Source branch (e.g. pull request branch)') 57 | parser.add_argument('--target', required=True, help='Target branch (e.g. master branch)') 58 | parser.add_argument('--subpath', required=True, help='Subpath containing the charts (e.g. system)') 59 | 60 | args = parser.parse_args() 61 | 62 | source_path = mkdtemp() 63 | target_path = mkdtemp() 64 | 65 | clone_repository(args.repository, args.source, source_path) 66 | clone_repository(args.repository, args.target, target_path) 67 | 68 | for chart in changed_charts(source_path, target_path, args.subpath): 69 | with NamedTemporaryFile(suffix='.yaml', mode='w+', delete=False) as f_source, NamedTemporaryFile(suffix='.yaml', mode='w+', delete=False) as f_target: 70 | render_helm_chart(f"{source_path}/{args.subpath}/{chart}", chart, chart, f_source.name) 71 | render_helm_chart(f"{target_path}/{args.subpath}/{chart}", chart, chart, f_target.name) 72 | 73 | diff_result = run( 74 | ['dyff', 'between', '--omit-header', '--use-go-patch-style', '--color=on', '--truecolor=off', f_target.name, f_source.name], 75 | capture_output=True, 76 | text=True, 77 | check=True 78 | ) 79 | 80 | print(diff_result.stdout) 81 | 82 | 83 | if __name__ == "__main__": 84 | main() 85 | -------------------------------------------------------------------------------- /scripts/kanidm-reset-password: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -eu 4 | 5 | account="${1}" 6 | 7 | echo "WARNING: Kanidm admin can do anything in the cluster, only use it for just enough initial setup or in emergencies." >&2 8 | export KUBECONFIG=./metal/kubeconfig.yaml 9 | kubectl exec -it -n kanidm statefulset/kanidm -- kanidmd recover-account "${account}" 10 | -------------------------------------------------------------------------------- /scripts/new-service: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | mkdir -p "apps/${1}" 4 | 5 | cat << EOT > "apps/${1}/Chart.yaml" 6 | apiVersion: v2 7 | name: CHANGEME 8 | version: 0.0.0 9 | dependencies: 10 | - name: CHANGEME 11 | version: CHANGEME 12 | repository: CHANGEME 13 | EOT 14 | 15 | touch "apps/${1}/values.yaml" 16 | -------------------------------------------------------------------------------- /scripts/onboard-user: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | username="${1}" 4 | fullname="${2}" 5 | mail="${3}" 6 | 7 | export KUBECONFIG=./metal/kubeconfig.yaml 8 | host="$(kubectl get ingress --namespace kanidm kanidm --output jsonpath='{.spec.rules[0].host}')" 9 | 10 | kanidm person create "${username}" "${fullname}" --url "https://${host}" --name idm_admin 11 | kanidm person update "${username}" --url "https://${host}" --name idm_admin --mail "${mail}" 12 | # TODO better group management 13 | kanidm group add-members "editor" "${username}" --url "https://${host}" --name idm_admin 14 | kanidm person credential create-reset-token "${username}" --url "https://${host}" --name idm_admin 15 | -------------------------------------------------------------------------------- /scripts/pxe-logs: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | docker compose \ 4 | --project-directory ./metal/roles/pxe_server/files/ \ 5 | logs \ 6 | --follow \ 7 | "${@}" 8 | -------------------------------------------------------------------------------- /scripts/take-screenshots: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # WIP 4 | # - [x] take screenshot 5 | # - [ ] self contained 6 | # - [ ] login automatically credentials from Kubernetes Secrets (is this really needed?) 7 | 8 | # TODO put this in ../flake.nix or use Docker 9 | # pip install selenium 10 | # sudo pacman -S geckodriver 11 | 12 | import time 13 | from selenium import webdriver 14 | 15 | apps = [ 16 | { 17 | 'name': 'home', 18 | 'url': 'https://home.khuedoan.com' 19 | }, 20 | { 21 | 'name': 'gitea', 22 | 'url': 'https://git.khuedoan.com/ops/homelab' 23 | }, 24 | { 25 | 'name': 'argocd', 26 | 'url': 'https://argocd.khuedoan.com/applications/root' 27 | }, 28 | { 29 | 'name': 'matrix', 30 | 'url': 'https://chat.khuedoan.com/#/room/#random:matrix.khuedoan.com' 31 | }, 32 | { 33 | 'name': 'grafana', 34 | 'url': 'https://grafana.khuedoan.com/d/efa86fd1d0c121a26444b636a3f509a8/kubernetes-compute-resources-cluster' # wtf is this ID 35 | }, 36 | ] 37 | 38 | options = webdriver.firefox.options.Options() 39 | options.headless = True 40 | 41 | driver = webdriver.Firefox( 42 | options=options, 43 | firefox_profile=webdriver.FirefoxProfile( # TODO deprecated 44 | profile_directory="/home/khuedoan/.mozilla/firefox/h05irklw.default-release" # TODO do not hard code 45 | ) 46 | ) 47 | driver.set_window_size(1920, 1080) 48 | 49 | for app in apps: 50 | print(f"Opening {app['url']}") 51 | driver.get(app['url']) 52 | time.sleep(3) # TODO wait for full page load instead of sleep 53 | driver.save_screenshot(f"{app['name']}.png") 54 | print(f"Screenshot saved to {app['name']}.png") 55 | 56 | driver.close() 57 | -------------------------------------------------------------------------------- /system/Makefile: -------------------------------------------------------------------------------- 1 | .POSIX: 2 | 3 | export KUBECONFIG = $(shell pwd)/../metal/kubeconfig.yaml 4 | 5 | .PHONY: bootstrap 6 | bootstrap: 7 | ansible-playbook \ 8 | bootstrap.yml 9 | -------------------------------------------------------------------------------- /system/argocd/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: argocd 3 | version: 0.0.0 4 | dependencies: 5 | - name: argo-cd 6 | version: 7.5.2 7 | repository: https://argoproj.github.io/argo-helm 8 | - name: argocd-apps 9 | version: 2.0.0 10 | repository: https://argoproj.github.io/argo-helm 11 | -------------------------------------------------------------------------------- /system/argocd/values-seed.yaml: -------------------------------------------------------------------------------- 1 | argo-cd: 2 | server: 3 | metrics: &metrics 4 | enabled: false 5 | serviceMonitor: 6 | enabled: false 7 | controller: 8 | metrics: *metrics 9 | repoServer: 10 | metrics: *metrics 11 | redis: 12 | metrics: *metrics 13 | argocd-apps: 14 | applicationsets: 15 | root: 16 | generators: 17 | - git: 18 | repoURL: &repoURL https://github.com/khuedoan/homelab 19 | revision: &revision master 20 | directories: 21 | - path: system/* 22 | - path: platform/* 23 | - path: apps/* 24 | template: 25 | spec: 26 | source: 27 | repoURL: *repoURL 28 | targetRevision: *revision 29 | -------------------------------------------------------------------------------- /system/argocd/values.yaml: -------------------------------------------------------------------------------- 1 | argo-cd: 2 | global: 3 | domain: argocd.khuedoan.com 4 | configs: 5 | params: 6 | server.insecure: true 7 | controller.diff.server.side: true 8 | cm: 9 | resource.ignoreResourceUpdatesEnabled: true 10 | resource.customizations.ignoreResourceUpdates.all: | 11 | jsonPointers: 12 | - /status 13 | server: 14 | ingress: 15 | enabled: true 16 | ingressClassName: nginx 17 | annotations: 18 | cert-manager.io/cluster-issuer: letsencrypt-prod 19 | tls: true 20 | metrics: &metrics 21 | enabled: true 22 | serviceMonitor: 23 | enabled: true 24 | dex: 25 | enabled: false 26 | controller: 27 | metrics: *metrics 28 | repoServer: 29 | metrics: *metrics 30 | redis: 31 | metrics: *metrics 32 | argocd-apps: 33 | applicationsets: 34 | root: 35 | namespace: argocd 36 | generators: 37 | - git: 38 | repoURL: &repoURL http://gitea-http.gitea:3000/ops/homelab 39 | revision: &revision master 40 | directories: 41 | - path: system/* 42 | - path: platform/* 43 | - path: apps/* 44 | template: 45 | metadata: 46 | name: '{{path.basename}}' 47 | spec: 48 | destination: 49 | name: in-cluster 50 | namespace: '{{path.basename}}' 51 | project: default # TODO 52 | source: 53 | repoURL: *repoURL 54 | path: '{{path}}' 55 | targetRevision: *revision 56 | syncPolicy: 57 | automated: 58 | prune: true 59 | selfHeal: true 60 | retry: 61 | limit: 10 62 | backoff: 63 | duration: 1m 64 | factor: 2 65 | maxDuration: 16m 66 | syncOptions: 67 | - CreateNamespace=true 68 | - ApplyOutOfSyncOnly=true 69 | - ServerSideApply=true 70 | managedNamespaceMetadata: 71 | annotations: 72 | # Enable privileged VolSync movers by default for all namespaces 73 | # TODO this may be refactored in the future for finer granularity 74 | # See also https://volsync.readthedocs.io/en/stable/usage/permissionmodel.html 75 | volsync.backube/privileged-movers: "true" 76 | -------------------------------------------------------------------------------- /system/bootstrap.yml: -------------------------------------------------------------------------------- 1 | - name: Bootstrapping the cluster 2 | hosts: localhost 3 | tasks: 4 | - name: Create ArgoCD namespace 5 | kubernetes.core.k8s: 6 | api_version: v1 7 | kind: Namespace 8 | name: argocd 9 | state: present 10 | 11 | - name: Check if this is the first installation 12 | kubernetes.core.k8s_info: 13 | kind: Pod 14 | label_selectors: 15 | - app.kubernetes.io/instance=gitea 16 | field_selectors: 17 | - status.phase=Running 18 | register: first_install 19 | 20 | - name: Render ArgoCD manifests from Helm chart 21 | kubernetes.core.helm_template: 22 | chart_ref: ./argocd 23 | include_crds: true 24 | release_name: argocd 25 | release_namespace: argocd 26 | dependency_update: true 27 | values_files: 28 | - "argocd/{{ (first_install.resources | length == 0) | ternary('values-seed.yaml', 'values.yaml') }}" 29 | register: argocd_manifests 30 | 31 | - name: Apply ArgoCD manifests 32 | kubernetes.core.k8s: 33 | resource_definition: "{{ argocd_manifests.stdout }}" 34 | apply: true 35 | server_side_apply: 36 | field_manager: argocd-controller 37 | -------------------------------------------------------------------------------- /system/cert-manager/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: cert-manager 3 | version: 0.0.0 4 | dependencies: 5 | - name: cert-manager 6 | version: v1.15.3 7 | repository: https://charts.jetstack.io 8 | -------------------------------------------------------------------------------- /system/cert-manager/templates/clusterissuer.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: cert-manager.io/v1 2 | kind: ClusterIssuer 3 | metadata: 4 | name: letsencrypt-prod 5 | spec: 6 | acme: 7 | server: https://acme-v02.api.letsencrypt.org/directory 8 | privateKeySecretRef: 9 | name: letsencrypt-prod 10 | solvers: 11 | - dns01: 12 | cloudflare: 13 | apiTokenSecretRef: 14 | name: cloudflare-api-token 15 | key: api-token 16 | -------------------------------------------------------------------------------- /system/cert-manager/values.yaml: -------------------------------------------------------------------------------- 1 | cert-manager: 2 | installCRDs: true 3 | prometheus: 4 | enabled: true 5 | servicemonitor: 6 | enabled: true 7 | -------------------------------------------------------------------------------- /system/cloudflared/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: cloudflared 3 | version: 0.0.0 4 | dependencies: 5 | - name: app-template 6 | version: 3.1.0 7 | repository: https://bjw-s.github.io/helm-charts 8 | -------------------------------------------------------------------------------- /system/cloudflared/values.yaml: -------------------------------------------------------------------------------- 1 | app-template: 2 | controllers: 3 | cloudflared: 4 | containers: 5 | app: 6 | image: 7 | repository: docker.io/cloudflare/cloudflared 8 | tag: 2024.4.0 9 | args: 10 | - tunnel 11 | - --config 12 | - /etc/cloudflared/config.yaml 13 | - run 14 | configMaps: 15 | config: 16 | enabled: true 17 | data: 18 | config.yaml: | 19 | tunnel: homelab 20 | credentials-file: /etc/cloudflared/credentials.json 21 | metrics: 0.0.0.0:2000 22 | no-autoupdate: true 23 | ingress: 24 | - hostname: '*.khuedoan.com' 25 | service: https://ingress-nginx-controller.ingress-nginx 26 | originRequest: 27 | noTLSVerify: true 28 | - service: http_status:404 29 | persistence: 30 | config: 31 | enabled: true 32 | type: configMap 33 | name: cloudflared-config 34 | globalMounts: 35 | - path: /etc/cloudflared/config.yaml 36 | subPath: config.yaml 37 | credentials: 38 | enabled: true 39 | type: secret 40 | # Created by ../../external/cloudflared 41 | name: cloudflared-credentials 42 | globalMounts: 43 | - path: /etc/cloudflared/credentials.json 44 | subPath: credentials.json 45 | -------------------------------------------------------------------------------- /system/external-dns/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: external-dns 3 | version: 0.0.0 4 | dependencies: 5 | - name: external-dns 6 | version: 1.14.3 7 | repository: https://kubernetes-sigs.github.io/external-dns 8 | -------------------------------------------------------------------------------- /system/external-dns/values.yaml: -------------------------------------------------------------------------------- 1 | external-dns: 2 | provider: cloudflare 3 | txtOwnerId: homelab 4 | env: 5 | - name: CF_API_TOKEN 6 | valueFrom: 7 | secretKeyRef: 8 | name: cloudflare-api-token 9 | key: value 10 | extraArgs: 11 | - --annotation-filter=external-dns.alpha.kubernetes.io/exclude notin (true) 12 | interval: 5m 13 | triggerLoopOnEvent: true 14 | metrics: 15 | enabled: true 16 | serviceMonitor: 17 | enabled: true 18 | -------------------------------------------------------------------------------- /system/ingress-nginx/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: ingress-nginx 3 | version: 0.0.0 4 | dependencies: 5 | - name: ingress-nginx 6 | version: 4.11.2 7 | repository: https://kubernetes.github.io/ingress-nginx 8 | -------------------------------------------------------------------------------- /system/ingress-nginx/values.yaml: -------------------------------------------------------------------------------- 1 | ingress-nginx: 2 | controller: 3 | admissionWebhooks: 4 | timeoutSeconds: 30 5 | metrics: 6 | enabled: true 7 | serviceMonitor: 8 | enabled: true 9 | tcp: 10 | 22: gitea/gitea-ssh:22 11 | -------------------------------------------------------------------------------- /system/kured/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: kured 3 | version: 0.0.0 4 | dependencies: 5 | - name: kured 6 | version: 4.7.0 7 | repository: https://kubereboot.github.io/charts 8 | -------------------------------------------------------------------------------- /system/kured/values.yaml: -------------------------------------------------------------------------------- 1 | kured: 2 | configuration: 3 | annotateNodes: true 4 | rebootSentinelCommand: sh -c "! needs-restarting --reboothint" 5 | timeZone: Asia/Ho_Chi_Minh 6 | -------------------------------------------------------------------------------- /system/loki/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: loki 3 | version: 0.0.0 4 | dependencies: 5 | - name: loki-stack 6 | version: 2.10.1 7 | repository: https://grafana.github.io/helm-charts 8 | -------------------------------------------------------------------------------- /system/loki/values.yaml: -------------------------------------------------------------------------------- 1 | loki-stack: 2 | loki: 3 | serviceMonitor: 4 | enabled: true 5 | -------------------------------------------------------------------------------- /system/monitoring-system/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: kube-prometheus-stack 3 | version: 0.0.0 4 | dependencies: 5 | - name: kube-prometheus-stack 6 | version: 56.19.0 7 | repository: https://prometheus-community.github.io/helm-charts 8 | -------------------------------------------------------------------------------- /system/monitoring-system/files/webhook-transformer/alertmanager-to-ntfy.jsonnet: -------------------------------------------------------------------------------- 1 | local get_tags(status, severity) = 2 | // https://docs.ntfy.sh/emojis 3 | if status == "resolved" then 4 | ["tada"] 5 | else 6 | std.get({ 7 | critical: ["rotating_light"], 8 | warning: ["warning"], 9 | info: ["newspaper"], 10 | }, severity, ["question"]); 11 | 12 | local get_priority(status, severity) = 13 | // https://docs.ntfy.sh/publish/#message-priority 14 | if status == "resolved" then 15 | 2 16 | else 17 | std.get({ 18 | critical: 5, 19 | warning: 3, 20 | info: 1, 21 | }, severity, 3); 22 | 23 | local get_actions(status, annotations) = 24 | // https://docs.ntfy.sh/publish/#action-buttons 25 | if status == "resolved" || !("runbook_url" in annotations) then 26 | [] 27 | else 28 | [ 29 | { 30 | action: "view", 31 | label: "Open runbook", 32 | url: annotations.runbook_url, 33 | }, 34 | ]; 35 | 36 | // TODO support multiple alerts 37 | { 38 | topic: env.NTFY_TOPIC, 39 | title: "[" + std.asciiUpper(body.status) + "] " + body.alerts[0].labels.alertname, 40 | message: body.alerts[0].annotations.description, 41 | tags: get_tags(body.status, body.alerts[0].labels.severity), 42 | priority: get_priority(body.status, body.alerts[0].labels.severity), 43 | actions: get_actions(body.status, body.alerts[0].annotations), 44 | } 45 | -------------------------------------------------------------------------------- /system/monitoring-system/templates/configmap.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: webhook-transformer 5 | namespace: {{ .Release.Namespace }} 6 | data: 7 | {{ (.Files.Glob "files/webhook-transformer/*").AsConfig | indent 2 }} 8 | -------------------------------------------------------------------------------- /system/monitoring-system/values.yaml: -------------------------------------------------------------------------------- 1 | kube-prometheus-stack: 2 | grafana: 3 | enabled: false 4 | forceDeployDatasources: true 5 | forceDeployDashboards: true 6 | additionalDataSources: 7 | - name: Loki 8 | type: loki 9 | url: http://loki.loki:3100 10 | prometheus: 11 | prometheusSpec: 12 | ruleSelectorNilUsesHelmValues: false 13 | serviceMonitorSelectorNilUsesHelmValues: false 14 | podMonitorSelectorNilUsesHelmValues: false 15 | probeSelectorNilUsesHelmValues: false 16 | alertmanager: 17 | alertmanagerSpec: 18 | containers: 19 | - name: ntfy-relay 20 | image: ghcr.io/khuedoan/webhook-transformer:v0.0.3 21 | args: 22 | - --port=8081 23 | - --config=/config/alertmanager-to-ntfy.jsonnet 24 | - --upstream-host=https://ntfy.sh 25 | envFrom: 26 | - secretRef: 27 | name: webhook-transformer 28 | volumeMounts: 29 | - name: config 30 | mountPath: /config 31 | volumes: 32 | - name: config 33 | configMap: 34 | name: webhook-transformer 35 | config: 36 | route: 37 | receiver: ntfy 38 | group_by: 39 | - namespace 40 | group_wait: 30s 41 | group_interval: 5m 42 | repeat_interval: 12h 43 | routes: 44 | - receiver: ntfy 45 | matchers: 46 | - alertname = "Watchdog" 47 | receivers: 48 | - name: ntfy 49 | webhook_configs: 50 | - url: http://localhost:8081 51 | send_resolved: true 52 | -------------------------------------------------------------------------------- /system/rook-ceph/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: rook-ceph 3 | version: 0.0.0 4 | dependencies: 5 | - name: rook-ceph 6 | version: 1.13.5 7 | repository: https://charts.rook.io/release 8 | - name: rook-ceph-cluster 9 | version: 1.13.5 10 | repository: https://charts.rook.io/release 11 | # TODO switch to official chart when there is one 12 | # https://github.com/kubernetes-csi/external-snapshotter/issues/812 13 | - name: snapshot-controller 14 | version: 2.2.1 15 | repository: https://piraeus.io/helm-charts 16 | -------------------------------------------------------------------------------- /system/rook-ceph/values.yaml: -------------------------------------------------------------------------------- 1 | rook-ceph: 2 | monitoring: 3 | enabled: true 4 | rook-ceph-cluster: 5 | monitoring: 6 | enabled: true 7 | createPrometheusRules: true 8 | cephClusterSpec: 9 | mon: 10 | count: 3 11 | mgr: 12 | count: 2 13 | dashboard: 14 | ssl: false 15 | logCollector: 16 | enabled: false 17 | removeOSDsIfOutAndSafeToRemove: true 18 | resources: 19 | mgr: 20 | limits: 21 | memory: "1Gi" 22 | requests: 23 | cpu: "100m" 24 | memory: "512Mi" 25 | mon: 26 | limits: 27 | memory: "2Gi" 28 | requests: 29 | cpu: "100m" 30 | memory: "100Mi" 31 | osd: 32 | limits: 33 | memory: "4Gi" 34 | requests: 35 | cpu: "100m" 36 | memory: "512Mi" 37 | cephBlockPools: 38 | - name: standard-rwo 39 | spec: 40 | replicated: 41 | size: 2 42 | storageClass: 43 | enabled: true 44 | name: standard-rwo 45 | isDefault: true 46 | allowVolumeExpansion: true 47 | parameters: 48 | imageFeatures: layering,fast-diff,object-map,deep-flatten,exclusive-lock 49 | csi.storage.k8s.io/provisioner-secret-name: rook-csi-rbd-provisioner 50 | csi.storage.k8s.io/provisioner-secret-namespace: "{{ .Release.Namespace }}" 51 | csi.storage.k8s.io/controller-expand-secret-name: rook-csi-rbd-provisioner 52 | csi.storage.k8s.io/controller-expand-secret-namespace: "{{ .Release.Namespace }}" 53 | csi.storage.k8s.io/node-stage-secret-name: rook-csi-rbd-node 54 | csi.storage.k8s.io/node-stage-secret-namespace: "{{ .Release.Namespace }}" 55 | cephBlockPoolsVolumeSnapshotClass: 56 | enabled: true 57 | isDefault: true 58 | cephFileSystems: 59 | - name: standard-rwx 60 | spec: 61 | metadataPool: 62 | replicated: 63 | size: 2 64 | dataPools: 65 | - name: data0 66 | replicated: 67 | size: 2 68 | metadataServer: 69 | activeCount: 1 70 | activeStandby: true 71 | resources: 72 | limits: 73 | memory: "4Gi" 74 | requests: 75 | cpu: "100m" 76 | memory: "100Mi" 77 | priorityClassName: system-cluster-critical 78 | storageClass: 79 | enabled: true 80 | name: standard-rwx 81 | isDefault: false 82 | allowVolumeExpansion: true 83 | pool: data0 84 | parameters: 85 | csi.storage.k8s.io/provisioner-secret-name: rook-csi-cephfs-provisioner 86 | csi.storage.k8s.io/provisioner-secret-namespace: "{{ .Release.Namespace }}" 87 | csi.storage.k8s.io/controller-expand-secret-name: rook-csi-cephfs-provisioner 88 | csi.storage.k8s.io/controller-expand-secret-namespace: "{{ .Release.Namespace }}" 89 | csi.storage.k8s.io/node-stage-secret-name: rook-csi-cephfs-node 90 | csi.storage.k8s.io/node-stage-secret-namespace: "{{ .Release.Namespace }}" 91 | cephFileSystemVolumeSnapshotClass: 92 | enabled: true 93 | isDefault: false 94 | cephObjectStores: [] 95 | -------------------------------------------------------------------------------- /system/volsync-system/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: volsync 3 | version: 0.0.0 4 | dependencies: 5 | - name: volsync 6 | version: 0.9.1 7 | repository: https://backube.github.io/helm-charts 8 | -------------------------------------------------------------------------------- /test/Makefile: -------------------------------------------------------------------------------- 1 | .POSIX: 2 | 3 | filter=. 4 | 5 | default: test 6 | 7 | test: 8 | gotestsum --format testname -- -timeout 30m -run "${filter}" 9 | -------------------------------------------------------------------------------- /test/benchmark/security/kube-bench.yaml: -------------------------------------------------------------------------------- 1 | # kubectl apply -f kube-bench.yaml 2 | # https://github.com/aquasecurity/kube-bench/blob/main/job.yaml 3 | apiVersion: batch/v1 4 | kind: Job 5 | metadata: 6 | name: kube-bench 7 | spec: 8 | template: 9 | metadata: 10 | labels: 11 | app: kube-bench 12 | spec: 13 | containers: 14 | - command: ["kube-bench"] 15 | image: docker.io/aquasec/kube-bench:v0.7.2 16 | name: kube-bench 17 | volumeMounts: 18 | - name: var-lib-cni 19 | mountPath: /var/lib/cni 20 | readOnly: true 21 | - mountPath: /var/lib/etcd 22 | name: var-lib-etcd 23 | readOnly: true 24 | - mountPath: /var/lib/kubelet 25 | name: var-lib-kubelet 26 | readOnly: true 27 | - mountPath: /var/lib/kube-scheduler 28 | name: var-lib-kube-scheduler 29 | readOnly: true 30 | - mountPath: /var/lib/kube-controller-manager 31 | name: var-lib-kube-controller-manager 32 | readOnly: true 33 | - mountPath: /etc/systemd 34 | name: etc-systemd 35 | readOnly: true 36 | - mountPath: /lib/systemd/ 37 | name: lib-systemd 38 | readOnly: true 39 | - mountPath: /srv/kubernetes/ 40 | name: srv-kubernetes 41 | readOnly: true 42 | - mountPath: /etc/kubernetes 43 | name: etc-kubernetes 44 | readOnly: true 45 | - mountPath: /usr/local/mount-from-host/bin 46 | name: usr-bin 47 | readOnly: true 48 | - mountPath: /etc/cni/net.d/ 49 | name: etc-cni-netd 50 | readOnly: true 51 | - mountPath: /opt/cni/bin/ 52 | name: opt-cni-bin 53 | readOnly: true 54 | hostPID: true 55 | restartPolicy: Never 56 | volumes: 57 | - name: var-lib-cni 58 | hostPath: 59 | path: /var/lib/cni 60 | - hostPath: 61 | path: /var/lib/etcd 62 | name: var-lib-etcd 63 | - hostPath: 64 | path: /var/lib/kubelet 65 | name: var-lib-kubelet 66 | - hostPath: 67 | path: /var/lib/kube-scheduler 68 | name: var-lib-kube-scheduler 69 | - hostPath: 70 | path: /var/lib/kube-controller-manager 71 | name: var-lib-kube-controller-manager 72 | - hostPath: 73 | path: /etc/systemd 74 | name: etc-systemd 75 | - hostPath: 76 | path: /lib/systemd 77 | name: lib-systemd 78 | - hostPath: 79 | path: /srv/kubernetes 80 | name: srv-kubernetes 81 | - hostPath: 82 | path: /etc/kubernetes 83 | name: etc-kubernetes 84 | - hostPath: 85 | path: /usr/bin 86 | name: usr-bin 87 | - hostPath: 88 | path: /etc/cni/net.d/ 89 | name: etc-cni-netd 90 | - hostPath: 91 | path: /opt/cni/bin/ 92 | name: opt-cni-bin 93 | -------------------------------------------------------------------------------- /test/benchmark/storage/dbench-rwo.yaml: -------------------------------------------------------------------------------- 1 | # kubectl apply -f dbench-rwo.yaml 2 | --- 3 | kind: PersistentVolumeClaim 4 | apiVersion: v1 5 | metadata: 6 | name: dbench-rwo 7 | spec: 8 | storageClassName: standard-rwo 9 | accessModes: 10 | - ReadWriteOnce 11 | resources: 12 | requests: 13 | storage: 10Gi 14 | --- 15 | apiVersion: batch/v1 16 | kind: Job 17 | metadata: 18 | name: dbench-rwo 19 | spec: 20 | template: 21 | spec: 22 | containers: 23 | - name: dbench 24 | image: zayashv/dbench:latest 25 | imagePullPolicy: Always 26 | env: 27 | - name: DBENCH_MOUNTPOINT 28 | value: /data 29 | - name: DBENCH_QUICK 30 | value: "no" 31 | volumeMounts: 32 | - name: data 33 | mountPath: /data 34 | restartPolicy: Never 35 | volumes: 36 | - name: data 37 | persistentVolumeClaim: 38 | claimName: dbench-rwo 39 | -------------------------------------------------------------------------------- /test/benchmark/storage/dbench-rwx.yaml: -------------------------------------------------------------------------------- 1 | # kubectl apply -f dbench-rwx.yaml 2 | --- 3 | kind: PersistentVolumeClaim 4 | apiVersion: v1 5 | metadata: 6 | name: dbench-rwx 7 | spec: 8 | storageClassName: standard-rwx 9 | accessModes: 10 | - ReadWriteOnce 11 | resources: 12 | requests: 13 | storage: 10Gi 14 | --- 15 | apiVersion: batch/v1 16 | kind: Job 17 | metadata: 18 | name: dbench-rwx 19 | spec: 20 | template: 21 | spec: 22 | containers: 23 | - name: dbench 24 | image: zayashv/dbench:latest 25 | imagePullPolicy: Always 26 | env: 27 | - name: DBENCH_MOUNTPOINT 28 | value: /data 29 | - name: DBENCH_QUICK 30 | value: "no" 31 | volumeMounts: 32 | - name: data 33 | mountPath: /data 34 | restartPolicy: Never 35 | volumes: 36 | - name: data 37 | persistentVolumeClaim: 38 | claimName: dbench-rwx 39 | -------------------------------------------------------------------------------- /test/external_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/gruntwork-io/terratest/modules/terraform" 7 | test_structure "github.com/gruntwork-io/terratest/modules/test-structure" 8 | ) 9 | 10 | func TestTerraformExternal(t *testing.T) { 11 | t.Parallel() 12 | 13 | // Make a copy of the terraform module to a temporary directory. This allows running multiple tests in parallel 14 | // against the same terraform module. 15 | exampleFolder := test_structure.CopyTerraformFolderToTemp(t, "../external", ".") 16 | 17 | terraformOptions := terraform.WithDefaultRetryableErrors(t, &terraform.Options{ 18 | TerraformDir: exampleFolder, 19 | }) 20 | 21 | terraform.Init(t, terraformOptions) 22 | terraform.Validate(t, terraformOptions) 23 | } 24 | -------------------------------------------------------------------------------- /test/go.mod: -------------------------------------------------------------------------------- 1 | module git.khuedoan.com/ops/homelab 2 | 3 | go 1.21 4 | 5 | toolchain go1.21.4 6 | 7 | require github.com/gruntwork-io/terratest v0.46.1 8 | 9 | require ( 10 | cloud.google.com/go v0.105.0 // indirect 11 | cloud.google.com/go/compute v1.12.1 // indirect 12 | cloud.google.com/go/compute/metadata v0.2.1 // indirect 13 | cloud.google.com/go/iam v0.7.0 // indirect 14 | cloud.google.com/go/storage v1.27.0 // indirect 15 | github.com/agext/levenshtein v1.2.3 // indirect 16 | github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect 17 | github.com/aws/aws-sdk-go v1.44.122 // indirect 18 | github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect 19 | github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect 20 | github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect 21 | github.com/davecgh/go-spew v1.1.1 // indirect 22 | github.com/emicklei/go-restful/v3 v3.9.0 // indirect 23 | github.com/go-errors/errors v1.0.2-0.20180813162953-d98b870cc4e0 // indirect 24 | github.com/go-logr/logr v1.2.3 // indirect 25 | github.com/go-openapi/jsonpointer v0.19.6 // indirect 26 | github.com/go-openapi/jsonreference v0.20.1 // indirect 27 | github.com/go-openapi/swag v0.22.3 // indirect 28 | github.com/go-sql-driver/mysql v1.4.1 // indirect 29 | github.com/gogo/protobuf v1.3.2 // indirect 30 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect 31 | github.com/golang/protobuf v1.5.3 // indirect 32 | github.com/google/gnostic v0.5.7-v3refs // indirect 33 | github.com/google/go-cmp v0.5.9 // indirect 34 | github.com/google/gofuzz v1.1.0 // indirect 35 | github.com/google/uuid v1.3.0 // indirect 36 | github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect 37 | github.com/googleapis/gax-go/v2 v2.7.0 // indirect 38 | github.com/gruntwork-io/go-commons v0.8.0 // indirect 39 | github.com/hashicorp/errwrap v1.0.0 // indirect 40 | github.com/hashicorp/go-cleanhttp v0.5.2 // indirect 41 | github.com/hashicorp/go-getter v1.7.1 // indirect 42 | github.com/hashicorp/go-multierror v1.1.0 // indirect 43 | github.com/hashicorp/go-safetemp v1.0.0 // indirect 44 | github.com/hashicorp/go-version v1.6.0 // indirect 45 | github.com/hashicorp/hcl/v2 v2.9.1 // indirect 46 | github.com/hashicorp/terraform-json v0.13.0 // indirect 47 | github.com/imdario/mergo v0.3.11 // indirect 48 | github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a // indirect 49 | github.com/jmespath/go-jmespath v0.4.0 // indirect 50 | github.com/josharian/intern v1.0.0 // indirect 51 | github.com/json-iterator/go v1.1.12 // indirect 52 | github.com/klauspost/compress v1.15.11 // indirect 53 | github.com/mailru/easyjson v0.7.7 // indirect 54 | github.com/mattn/go-zglob v0.0.2-0.20190814121620-e3c945676326 // indirect 55 | github.com/mitchellh/go-homedir v1.1.0 // indirect 56 | github.com/mitchellh/go-testing-interface v1.14.1 // indirect 57 | github.com/mitchellh/go-wordwrap v1.0.1 // indirect 58 | github.com/moby/spdystream v0.2.0 // indirect 59 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 60 | github.com/modern-go/reflect2 v1.0.2 // indirect 61 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 62 | github.com/pmezard/go-difflib v1.0.0 // indirect 63 | github.com/pquerna/otp v1.2.0 // indirect 64 | github.com/russross/blackfriday/v2 v2.1.0 // indirect 65 | github.com/spf13/pflag v1.0.5 // indirect 66 | github.com/stretchr/testify v1.8.1 // indirect 67 | github.com/tmccombs/hcl2json v0.3.3 // indirect 68 | github.com/ulikunitz/xz v0.5.10 // indirect 69 | github.com/urfave/cli v1.22.2 // indirect 70 | github.com/zclconf/go-cty v1.9.1 // indirect 71 | go.opencensus.io v0.24.0 // indirect 72 | golang.org/x/crypto v0.14.0 // indirect 73 | golang.org/x/net v0.17.0 // indirect 74 | golang.org/x/oauth2 v0.1.0 // indirect 75 | golang.org/x/sys v0.13.0 // indirect 76 | golang.org/x/term v0.13.0 // indirect 77 | golang.org/x/text v0.13.0 // indirect 78 | golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect 79 | golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect 80 | google.golang.org/api v0.103.0 // indirect 81 | google.golang.org/appengine v1.6.7 // indirect 82 | google.golang.org/genproto v0.0.0-20221201164419-0e50fba7f41c // indirect 83 | google.golang.org/grpc v1.51.0 // indirect 84 | google.golang.org/protobuf v1.31.0 // indirect 85 | gopkg.in/inf.v0 v0.9.1 // indirect 86 | gopkg.in/yaml.v2 v2.4.0 // indirect 87 | gopkg.in/yaml.v3 v3.0.1 // indirect 88 | k8s.io/api v0.27.2 // indirect 89 | k8s.io/apimachinery v0.27.2 // indirect 90 | k8s.io/client-go v0.27.2 // indirect 91 | k8s.io/klog/v2 v2.90.1 // indirect 92 | k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect 93 | k8s.io/utils v0.0.0-20230209194617-a36077c30491 // indirect 94 | sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect 95 | sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect 96 | sigs.k8s.io/yaml v1.3.0 // indirect 97 | ) 98 | 99 | // TODO https://github.com/gruntwork-io/terratest/pull/1182 100 | replace github.com/gruntwork-io/terratest v0.46.1 => github.com/khuedoan/terratest v0.0.0-20231027122225-118d656063b1 101 | -------------------------------------------------------------------------------- /test/integration_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "crypto/tls" 5 | "fmt" 6 | "testing" 7 | "time" 8 | 9 | http_helper "github.com/gruntwork-io/terratest/modules/http-helper" 10 | "github.com/gruntwork-io/terratest/modules/k8s" 11 | ) 12 | 13 | func TestArgoCDCheck(t *testing.T) { 14 | t.Parallel() 15 | 16 | // Setup the kubectl config and context. Here we choose to use the defaults, which is: 17 | // - $KUBECONFIG for the kubectl config file 18 | // - Current context of the kubectl config file 19 | options := k8s.NewKubectlOptions("", "", "argocd") 20 | 21 | // This will wait up to 10 seconds for the service to become available, to ensure that we can access it 22 | k8s.WaitUntilIngressAvailable(t, options, "argocd-server", 10, 1*time.Second) 23 | 24 | // Now we verify that the service will successfully boot and start serving requests 25 | ingress := k8s.GetIngress(t, options, "argocd-server") 26 | 27 | // Setup a TLS configuration to submit with the helper, a blank struct is acceptable 28 | tlsConfig := tls.Config{} 29 | 30 | // Test the endpoint for up to 5 minutes. This will only fail if we timeout waiting for the service to return a 200 response 31 | http_helper.HttpGetWithRetryWithCustomValidation( 32 | t, 33 | fmt.Sprintf("https://%s", ingress.Spec.Rules[0].Host), 34 | &tlsConfig, 35 | 30, 36 | 30*time.Second, 37 | func(statusCode int, body string) bool { 38 | return statusCode == 200 39 | }, 40 | ) 41 | } 42 | -------------------------------------------------------------------------------- /test/smoke_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "crypto/tls" 5 | "fmt" 6 | "os" 7 | "testing" 8 | "time" 9 | 10 | http_helper "github.com/gruntwork-io/terratest/modules/http-helper" 11 | "github.com/gruntwork-io/terratest/modules/k8s" 12 | ) 13 | 14 | func TestSmoke(t *testing.T) { 15 | t.Parallel() 16 | 17 | var mainApps = []struct { 18 | name string 19 | namespace string 20 | }{ 21 | {"argocd-server", "argocd"}, 22 | {"gitea", "gitea"}, 23 | {"grafana", "grafana"}, 24 | {"homepage", "homepage"}, 25 | {"kanidm", "kanidm"}, 26 | {"zot", "zot"}, 27 | } 28 | 29 | for _, app := range mainApps { 30 | app := app // https://github.com/golang/go/wiki/CommonMistakes#using-goroutines-on-loop-iterator-variables 31 | t.Run(app.name, func(t *testing.T) { 32 | t.Parallel() 33 | 34 | options := k8s.NewKubectlOptions("", "", app.namespace) 35 | 36 | // Wait the service to become available to ensure that we can access it 37 | k8s.WaitUntilIngressAvailable(t, options, app.name, 30, 60*time.Second) 38 | 39 | // Now we verify that the service will successfully boot and start serving requests 40 | ingress := k8s.GetIngress(t, options, app.name) 41 | 42 | // Setup a TLS configuration, ignore the certificate because we may not use cert-manager (like the sandbox environment) 43 | tlsConfig := tls.Config{ 44 | InsecureSkipVerify: os.Getenv("INSECURE_SKIP_VERIFY") != "", 45 | } 46 | 47 | // Test the endpoint, this will only fail if we timeout waiting for the service to return a 200 response 48 | http_helper.HttpGetWithRetryWithCustomValidation( 49 | t, 50 | fmt.Sprintf("https://%s", ingress.Spec.Rules[0].Host), 51 | &tlsConfig, 52 | 30, 53 | 60*time.Second, 54 | func(statusCode int, body string) bool { 55 | return statusCode == 200 56 | }, 57 | ) 58 | }) 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /test/tools_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "path/filepath" 5 | "testing" 6 | 7 | "github.com/gruntwork-io/terratest/modules/shell" 8 | "github.com/gruntwork-io/terratest/modules/version-checker" 9 | ) 10 | 11 | func TestToolsVersions(t *testing.T) { 12 | t.Parallel() 13 | 14 | var tools = []struct { 15 | binaryPath string 16 | versionArg string 17 | versionConstraint string 18 | }{ 19 | {"ansible", "--version", ">= 2.12.6, < 3.0.0"}, 20 | {"docker", "--version", ">= 25.0.0, < 26.0.0"}, 21 | {"git", "--version", ">= 2.37.1, < 3.0.0"}, 22 | {"go", "version", ">= 1.22.0, < 1.23.0"}, 23 | {"helm", "version", ">= 3.9.4, < 4.0.0"}, 24 | {"kubectl", "version", ">= 1.30.0, < 1.32.0"}, // https://kubernetes.io/releases/version-skew-policy/#kubectl 25 | {"kustomize", "version", ">= 5.0.3, < 6.0.0"}, 26 | {"pre-commit", "--version", ">= 3.3.2, < 4.0.0"}, 27 | {"tofu", "--version", ">= 1.7.0, < 1.9.0"}, 28 | } 29 | 30 | for _, tool := range tools { 31 | tool := tool // https://github.com/golang/go/wiki/CommonMistakes#using-goroutines-on-loop-iterator-variables 32 | t.Run(tool.binaryPath, func(t *testing.T) { 33 | t.Parallel() 34 | params := version_checker.CheckVersionParams{ 35 | BinaryPath: tool.binaryPath, 36 | VersionConstraint: tool.versionConstraint, 37 | VersionArg: tool.versionArg, 38 | WorkingDir: ".", 39 | } 40 | 41 | version_checker.CheckVersion(t, params) 42 | }) 43 | } 44 | } 45 | 46 | func TestToolsNixShell(t *testing.T) { 47 | t.Parallel() 48 | 49 | projectRoot, err := filepath.Abs("../") 50 | if err != nil { 51 | t.FailNow() 52 | } 53 | 54 | command := shell.Command{ 55 | Command: "nix", 56 | Args: []string{ 57 | "develop", 58 | "--experimental-features", "nix-command flakes", 59 | "--command", "true", 60 | }, 61 | WorkingDir: projectRoot, 62 | } 63 | 64 | shell.RunCommand(t, command) 65 | } 66 | --------------------------------------------------------------------------------