├── .gitignore ├── 01-clusters └── main.tf ├── 02-setup └── main.tf ├── 03-demo-cron ├── 01-podinfo.yaml └── 02-scaled-object.yaml ├── 04-demo-proactive ├── 01-podinfo.yaml └── 02-placeholder.yaml ├── 05-demo-hpa ├── 01-rate-limiter.yaml ├── 02-placeholder.yaml └── 03-scaled-object.yaml ├── README.md ├── app ├── Dockerfile └── index.js ├── assets ├── preview.gif └── quick-scaling.mp4 ├── dashboard ├── app.js └── index.html └── modules ├── autoscaling ├── locust.yaml └── main.tf ├── cluster └── main.tf └── label └── main.tf /.gitignore: -------------------------------------------------------------------------------- 1 | istio 2 | karmada 3 | terraform.tfstate 4 | terraform.tfstate.backup 5 | karmada-config 6 | kube-config-* 7 | kubeconfig-* 8 | node_ip.txt 9 | .terraform.lock.hcl 10 | terraform.tfstate.* 11 | .terraform 12 | .terraform.tfstate.lock.info 13 | kubeconfig -------------------------------------------------------------------------------- /01-clusters/main.tf: -------------------------------------------------------------------------------- 1 | module "standard" { 2 | source = "../modules/cluster" 3 | 4 | name = "standard" 5 | region = "eu-west" 6 | } 7 | 8 | resource "local_file" "kubeconfig_standard" { 9 | filename = "../kubeconfig" 10 | content = module.standard.kubeconfig 11 | } 12 | 13 | output "standard_scale_to_3" { 14 | value = "linode-cli lke pool-update ${module.standard.cluster_id} ${module.standard.pool_id} --count 2" 15 | } 16 | -------------------------------------------------------------------------------- /02-setup/main.tf: -------------------------------------------------------------------------------- 1 | module "label_standard" { 2 | source = "../modules/label" 3 | 4 | kubeconfig_path = abspath("../kubeconfig") 5 | } 6 | 7 | module "autoscaling_hpa" { 8 | source = "../modules/autoscaling" 9 | 10 | kubeconfig_path = abspath("../kubeconfig") 11 | } -------------------------------------------------------------------------------- /03-demo-cron/01-podinfo.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: podinfo 5 | spec: 6 | replicas: 1 7 | selector: 8 | matchLabels: 9 | app: podinfo 10 | template: 11 | metadata: 12 | labels: 13 | app: podinfo 14 | annotations: 15 | prometheus.io/scrape: "true" 16 | spec: 17 | affinity: 18 | nodeAffinity: 19 | requiredDuringSchedulingIgnoredDuringExecution: 20 | nodeSelectorTerms: 21 | - matchExpressions: 22 | - key: node 23 | operator: NotIn 24 | values: 25 | - primary 26 | containers: 27 | - name: podinfo 28 | image: stefanprodan/podinfo 29 | ports: 30 | - containerPort: 9898 31 | resources: 32 | requests: 33 | memory: 0.9G 34 | --- 35 | apiVersion: v1 36 | kind: Service 37 | metadata: 38 | name: podinfo 39 | spec: 40 | ports: 41 | - port: 80 42 | targetPort: 9898 43 | selector: 44 | app: podinfo -------------------------------------------------------------------------------- /03-demo-cron/02-scaled-object.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: keda.sh/v1alpha1 2 | kind: ScaledObject 3 | metadata: 4 | name: cron-scaledobject 5 | namespace: default 6 | spec: 7 | maxReplicaCount: 10 8 | minReplicaCount: 1 9 | scaleTargetRef: 10 | name: podinfo 11 | triggers: 12 | - type: cron 13 | metadata: 14 | timezone: Europe/London 15 | start: 3 * * * * 16 | end: 4 * * * * 17 | desiredReplicas: "5" 18 | -------------------------------------------------------------------------------- /04-demo-proactive/01-podinfo.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: podinfo 5 | spec: 6 | replicas: 1 7 | selector: 8 | matchLabels: 9 | app: podinfo 10 | template: 11 | metadata: 12 | labels: 13 | app: podinfo 14 | annotations: 15 | prometheus.io/scrape: "true" 16 | spec: 17 | affinity: 18 | nodeAffinity: 19 | requiredDuringSchedulingIgnoredDuringExecution: 20 | nodeSelectorTerms: 21 | - matchExpressions: 22 | - key: node 23 | operator: NotIn 24 | values: 25 | - primary 26 | containers: 27 | - name: podinfo 28 | image: stefanprodan/podinfo 29 | ports: 30 | - containerPort: 9898 31 | resources: 32 | requests: 33 | memory: 0.9G 34 | --- 35 | apiVersion: v1 36 | kind: Service 37 | metadata: 38 | name: podinfo 39 | spec: 40 | ports: 41 | - port: 80 42 | targetPort: 9898 43 | selector: 44 | app: podinfo -------------------------------------------------------------------------------- /04-demo-proactive/02-placeholder.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: scheduling.k8s.io/v1 2 | kind: PriorityClass 3 | metadata: 4 | name: overprovisioning 5 | value: -1 6 | globalDefault: false 7 | description: "Priority class used by overprovisioning." 8 | --- 9 | apiVersion: apps/v1 10 | kind: Deployment 11 | metadata: 12 | name: overprovisioning 13 | spec: 14 | replicas: 1 15 | selector: 16 | matchLabels: 17 | run: overprovisioning 18 | template: 19 | metadata: 20 | labels: 21 | run: overprovisioning 22 | spec: 23 | priorityClassName: overprovisioning 24 | containers: 25 | - name: pause 26 | image: k8s.gcr.io/pause 27 | resources: 28 | requests: 29 | cpu: 900m 30 | memory: 3.9G -------------------------------------------------------------------------------- /05-demo-hpa/01-rate-limiter.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: podinfo 5 | spec: 6 | replicas: 4 7 | selector: 8 | matchLabels: 9 | app: podinfo 10 | template: 11 | metadata: 12 | labels: 13 | app: podinfo 14 | annotations: 15 | prometheus.io/scrape: "true" 16 | spec: 17 | affinity: 18 | nodeAffinity: 19 | requiredDuringSchedulingIgnoredDuringExecution: 20 | nodeSelectorTerms: 21 | - matchExpressions: 22 | - key: node 23 | operator: NotIn 24 | values: 25 | - primary 26 | containers: 27 | - name: podinfo 28 | image: learnk8s/rate-limiter:1.0.0 29 | imagePullPolicy: Always 30 | args: ["/app/index.js", "10"] 31 | ports: 32 | - containerPort: 8080 33 | resources: 34 | requests: 35 | memory: 0.9G 36 | --- 37 | apiVersion: v1 38 | kind: Service 39 | metadata: 40 | name: podinfo 41 | spec: 42 | ports: 43 | - port: 80 44 | targetPort: 8080 45 | selector: 46 | app: podinfo -------------------------------------------------------------------------------- /05-demo-hpa/02-placeholder.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: scheduling.k8s.io/v1 2 | kind: PriorityClass 3 | metadata: 4 | name: overprovisioning 5 | value: -1 6 | globalDefault: false 7 | description: "Priority class used by overprovisioning." 8 | --- 9 | apiVersion: apps/v1 10 | kind: Deployment 11 | metadata: 12 | name: overprovisioning 13 | spec: 14 | replicas: 1 15 | selector: 16 | matchLabels: 17 | run: overprovisioning 18 | template: 19 | metadata: 20 | labels: 21 | run: overprovisioning 22 | spec: 23 | priorityClassName: overprovisioning 24 | containers: 25 | - name: pause 26 | image: k8s.gcr.io/pause 27 | resources: 28 | requests: 29 | cpu: 900m 30 | memory: 3.9G -------------------------------------------------------------------------------- /05-demo-hpa/03-scaled-object.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: keda.sh/v1alpha1 2 | kind: ScaledObject 3 | metadata: 4 | name: podinfo 5 | spec: 6 | scaleTargetRef: 7 | kind: Deployment 8 | name: podinfo 9 | minReplicaCount: 4 10 | maxReplicaCount: 30 11 | cooldownPeriod: 30 12 | pollingInterval: 1 13 | triggers: 14 | - type: prometheus 15 | metadata: 16 | serverAddress: http://prometheus-server 17 | metricName: connections_active_keda 18 | query: | 19 | sum(increase(http_requests_total{app="podinfo"}[60s])) 20 | threshold: "420" # 8rps * 60s -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Kubernetes scaling strategies 2 | 3 | This project helps you create a cluster with idle nodes ready to deploy new workloads. 4 | 5 | ![Proactively autoscaling in Kubernetes](assets/preview.gif) 6 | 7 | ## Getting started 8 | 9 | You need to create a Linode token to access the API: 10 | 11 | ```bash 12 | linode-cli profile token-create 13 | export LINODE_TOKEN= 14 | ``` 15 | 16 | ```bash 17 | # Create the cluster 18 | terraform -chdir=01-clusters init 19 | terraform -chdir=01-clusters apply -auto-approve 20 | 21 | # Label the node, install KEDA + Prometheus 22 | terraform -chdir=02-setup init 23 | terraform -chdir=02-setup apply -auto-approve 24 | 25 | # Scale to 2 nodes 26 | linode-cli lke pool-update --count 2 # cluster 1 27 | 28 | # # Disable scaling down (you can't label nodes in LKE) 29 | # kubectl annotate node cluster-autoscaler.kubernetes.io/scale-down-disabled=true 30 | # kubectl annotate node cluster-autoscaler.kubernetes.io/scale-down-disabled=true 31 | 32 | # Cleanup 33 | terraform -chdir=02-setup destroy -auto-approve 34 | terraform -chdir=01-clusters destroy -auto-approve 35 | ``` 36 | 37 | _Why scale to 3 nodes after the first step?_ 38 | 39 | This is to ensure that all LKE's controllers end up in the same node, and you can uniquely tag that node. 40 | 41 | ## Demo the cron scaling 42 | 43 | Make sure that your kubectl is configured with the current kubeconfig file: 44 | 45 | ```bash 46 | export KUBECONFIG="${PWD}/kubeconfig" 47 | ``` 48 | 49 | The execute: 50 | 51 | ```bash 52 | kubectl apply -f 03-demo-cron/01-podinfo.yaml 53 | ``` 54 | 55 | Amend the time to be 1 minute in the future. 56 | 57 | Submit the ScaledObject: 58 | 59 | ```bash 60 | kubectl apply -f 03-demo-cron/02-scaled-object.yaml 61 | ``` 62 | 63 | ## Dashboard 64 | 65 | ```bash 66 | kubectl proxy --www=./dashboard 67 | ``` 68 | 69 | ## Placeholder demo 70 | 71 | Make sure that your kubectl is configured with the current kubeconfig file: 72 | 73 | ```bash 74 | export KUBECONFIG="${PWD}/kubeconfig" 75 | ``` 76 | 77 | Then execute: 78 | 79 | ```bash 80 | kubectl apply -f 04-demo-proactive/01-podinfo.yaml 81 | ``` 82 | 83 | When ready, observe the node scaling up with: 84 | 85 | ```bash 86 | kubectl scale deployment/podinfo --replicas=5 87 | ``` 88 | 89 | [**Or use the dashboard.**](#dashboard) 90 | 91 | The total scaling time for the 5th pod should take ~2m. 92 | 93 | Scale back to 1 and submit the placeholder pod with: 94 | 95 | ```bash 96 | kubectl scale deployment/podinfo --replicas=1 97 | kubectl apply -f 04-demo-proactive/02-placeholder.yaml 98 | ``` 99 | 100 | Click on the scale button to scale up to 5. Then show the new node is being provisioned. 101 | 102 | ```bash 103 | kubectl proxy --www=./dashboard --www-prefix=/static & 104 | open http://localhost:8001/static 105 | ``` 106 | Repeat the experiment. The total scaling time should go down to ~10s. 107 | 108 | ## HPA demo 109 | 110 | Make sure that your kubectl is configured with the current kubeconfig file: 111 | 112 | ```bash 113 | export KUBECONFIG="${PWD}/kubeconfig-hpa" 114 | ``` 115 | 116 | Then execute: 117 | 118 | ```bash 119 | kubectl apply -f 05-demo-hpa/01-rate-limiter.yaml 120 | kubectl apply -f 05-demo-hpa/03-scaled-object.yaml 121 | ``` 122 | 123 | At this point, you should have at least two nodes. One has four podinfo pods. 124 | 125 | Scale your workers to three instances so that you have an empty node. 126 | 127 | Retrieve the IP address of the load balancer for Locust with: 128 | 129 | ```bash 130 | kubectl get service locust -o jsonpath='{.status.loadBalancer.ingress[0].ip}' 131 | ``` 132 | 133 | Drive traffic to the instance: 134 | 135 | - Number of users: 300 136 | - Concurrent users: 0.4 137 | - Url: `http://podinfo` 138 | 139 | Repeat the experiment. 140 | 141 | ```bash 142 | kubectl delete -f 05-demo-hpa/03-scaled-object.yaml 143 | kubectl scale deployment/podinfo --replicas=1 144 | ``` 145 | 146 | Wait for the Cluster autoscaler to drain the nodes. By the end, you should have only 2. 147 | 148 | Submit the placeholder pod: 149 | 150 | ```bash 151 | kubectl apply -f 05-demo-hpa/placeholder.yaml 152 | ``` 153 | 154 | The pod stays Pending until the cluster autoscaler creates the third node. 155 | 156 | Open Locust and drive the traffic to the deployment with: 157 | 158 | - Number of users: 300 159 | - Concurrent users: 0.4 160 | - Url: `http://podinfo` 161 | 162 | ## Building the rate limiter 163 | 164 | ```bash 165 | cd app 166 | docker build -t learnk8s/rate-limiter:1.0.0 . 167 | docker push learnk8s/rate-limiter:1.0.0 168 | ``` -------------------------------------------------------------------------------- /app/Dockerfile: -------------------------------------------------------------------------------- 1 | 2 | FROM gcr.io/distroless/nodejs:18 3 | 4 | WORKDIR /app 5 | 6 | COPY index.js . 7 | 8 | CMD [ "/app/index.js" ] -------------------------------------------------------------------------------- /app/index.js: -------------------------------------------------------------------------------- 1 | const http = require("http"); 2 | const port = Number.isFinite(parseInt(process.env.PORT, 10)) 3 | ? parseInt(process.env.PORT, 10) 4 | : 8080; 5 | 6 | let counter = 0; 7 | let requestsForCurrentWindow = 0; 8 | 9 | const arg = parseFloat(process.argv.slice(2)[0]); 10 | const rps = Number.isFinite(arg) ? arg : 1; 11 | 12 | const requestListener = function (req, res) { 13 | res.setHeader("Content-Type", "text/html"); 14 | res.writeHead(200); 15 | 16 | switch (req.url) { 17 | case "/metrics": { 18 | res.end( 19 | [ 20 | `# HELP http_requests_total The total number of HTTP requests.`, 21 | `# TYPE http_requests_total counter`, 22 | `http_requests_total{status="200"} ${counter}`, 23 | ].join("\n") 24 | ); 25 | break; 26 | } 27 | default: 28 | counter++; 29 | requestsForCurrentWindow++; 30 | reply(res); 31 | break; 32 | } 33 | }; 34 | 35 | const server = http.createServer(requestListener); 36 | server.listen(port); 37 | 38 | console.log(`Starting server http://localhost:${port} with ${rps} rps`); 39 | 40 | setInterval(() => { 41 | requestsForCurrentWindow = 0; 42 | }, 1000); 43 | 44 | function reply(res, retryAttempt = 0) { 45 | if (requestsForCurrentWindow > rps) { 46 | const ms = Math.pow(2, retryAttempt) * 1000; 47 | console.log(`RATE LIMITED, waiting ${ms / 1000} seconds`); 48 | setTimeout(() => reply(res, retryAttempt + 1), ms); 49 | } else { 50 | res.end("OK"); 51 | } 52 | } 53 | 54 | process.on("SIGINT", async () => { 55 | console.log("[SIGINT]"); 56 | process.exit(0); 57 | }); 58 | process.on("SIGTERM", async () => { 59 | console.log("[SIGTERM]"); 60 | process.exit(0); 61 | }); 62 | process.on("SIGHUP", async () => { 63 | console.log("[SIGHUP]"); 64 | process.exit(0); 65 | }); 66 | process.on("uncaughtException", async (error) => { 67 | console.log("[uncaughtException] ", error); 68 | process.exit(1); 69 | }); 70 | -------------------------------------------------------------------------------- /assets/preview.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/learnk8s/combining-autoscalers-kubernetes/63a9945289ef9f7904b0918a5fbe6d60357374b6/assets/preview.gif -------------------------------------------------------------------------------- /assets/quick-scaling.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/learnk8s/combining-autoscalers-kubernetes/63a9945289ef9f7904b0918a5fbe6d60357374b6/assets/quick-scaling.mp4 -------------------------------------------------------------------------------- /dashboard/app.js: -------------------------------------------------------------------------------- 1 | const app = App(); 2 | let lastPodResourceVersion; 3 | let lastNodeResourceVersion; 4 | let startTime = undefined; 5 | const timer = document.querySelector("#timer"); 6 | 7 | fetch("/api/v1/pods") 8 | .then((response) => response.json()) 9 | .then((response) => { 10 | const pods = response.items; 11 | lastPodResourceVersion = response.metadata.resourceVersion; 12 | pods.forEach((pod) => { 13 | const podId = `${pod.metadata.namespace}-${pod.metadata.name}`; 14 | app.upsertPod(podId, pod); 15 | }); 16 | }) 17 | .then(() => streamUpdatesPods()); 18 | 19 | fetch("/api/v1/nodes") 20 | .then((response) => response.json()) 21 | .then((response) => { 22 | const nodes = response.items; 23 | lastNodeResourceVersion = response.metadata.resourceVersion; 24 | nodes.forEach((node) => { 25 | const nodeId = node.metadata.name; 26 | app.upsertPod(nodeId, node); 27 | }); 28 | }) 29 | .then(() => streamUpdatesNodes()); 30 | 31 | function streamUpdatesPods() { 32 | fetch(`/api/v1/pods?watch=1&resourceVersion=${lastPodResourceVersion}`) 33 | .then((response) => { 34 | const stream = response.body.getReader(); 35 | const utf8Decoder = new TextDecoder("utf-8"); 36 | let buffer = ""; 37 | 38 | return stream.read().then(function processText({ done, value }) { 39 | if (done) { 40 | console.log("Request terminated"); 41 | return; 42 | } 43 | buffer += utf8Decoder.decode(value); 44 | buffer = onNewLine(buffer, (chunk) => { 45 | if (chunk.trim().length === 0) { 46 | return; 47 | } 48 | try { 49 | const event = JSON.parse(chunk); 50 | console.log("PROCESSING EVENT: ", event); 51 | const pod = { ...event.object, phase: event.object.status.phase }; 52 | const podId = `${pod.metadata.namespace}-${pod.metadata.name}`; 53 | switch (event.type) { 54 | case "ADDED": { 55 | app.upsertPod(podId, pod); 56 | break; 57 | } 58 | case "DELETED": { 59 | app.removePod(podId); 60 | break; 61 | } 62 | case "MODIFIED": { 63 | app.upsertPod(podId, pod); 64 | break; 65 | } 66 | default: 67 | break; 68 | } 69 | lastPodResourceVersion = event.object.metadata.resourceVersion; 70 | } catch (error) { 71 | console.log("Error while parsing", chunk, "\n", error); 72 | } 73 | }); 74 | return stream.read().then(processText); 75 | }); 76 | }) 77 | .catch(() => { 78 | console.log("Error! Retrying in 5 seconds..."); 79 | setTimeout(() => streamUpdatesPods(), 5000); 80 | }); 81 | } 82 | 83 | function streamUpdatesNodes() { 84 | fetch(`/api/v1/nodes?watch=1&resourceVersion=${lastNodeResourceVersion}`) 85 | .then((response) => { 86 | const stream = response.body.getReader(); 87 | const utf8Decoder = new TextDecoder("utf-8"); 88 | let buffer = ""; 89 | 90 | return stream.read().then(function processText({ done, value }) { 91 | if (done) { 92 | console.log("Request terminated"); 93 | return; 94 | } 95 | buffer += utf8Decoder.decode(value); 96 | buffer = onNewLine(buffer, (chunk) => { 97 | if (chunk.trim().length === 0) { 98 | return; 99 | } 100 | try { 101 | const event = JSON.parse(chunk); 102 | console.log("PROCESSING EVENT: ", event); 103 | const node = event.object; 104 | switch (event.type) { 105 | case "ADDED": { 106 | app.upsertNode(node.metadata.name, node); 107 | break; 108 | } 109 | case "DELETED": { 110 | app.removeNode(node.metadata.name); 111 | break; 112 | } 113 | case "MODIFIED": { 114 | app.upsertNode(node.metadata.name, node); 115 | break; 116 | } 117 | default: 118 | break; 119 | } 120 | lastNodeResourceVersion = event.object.metadata.resourceVersion; 121 | } catch (error) { 122 | console.log("Error while parsing", chunk, "\n", error); 123 | } 124 | }); 125 | return stream.read().then(processText); 126 | }); 127 | }) 128 | .catch(() => { 129 | console.log("Error! Retrying in 5 seconds..."); 130 | setTimeout(() => streamUpdatesNodes(), 5000); 131 | }); 132 | } 133 | 134 | function onNewLine(buffer, fn) { 135 | const newLineIndex = buffer.indexOf("\n"); 136 | if (newLineIndex === -1) { 137 | return buffer; 138 | } 139 | const chunk = buffer.slice(0, buffer.indexOf("\n")); 140 | const newBuffer = buffer.slice(buffer.indexOf("\n") + 1); 141 | fn(chunk); 142 | return onNewLine(newBuffer, fn); 143 | } 144 | 145 | function App() { 146 | const allPods = new Map(); 147 | const allNodes = new Map(); 148 | const content = document.querySelector("#content"); 149 | 150 | function render() { 151 | const pods = Array.from(allPods.values()); 152 | 153 | if (pods.length === 0) { 154 | return; 155 | } 156 | 157 | const podsByNode = Array.from(allNodes.values()).reduce( 158 | (acc, it) => { 159 | if (it.name in acc) { 160 | return acc; 161 | } else { 162 | acc[it.name] = []; 163 | } 164 | return acc; 165 | }, 166 | groupBy(pods, (it) => it.nodeName) 167 | ); 168 | const nodeTemplates = Object.keys(podsByNode) 169 | .filter((nodeName) => { 170 | const pods = podsByNode[nodeName]; 171 | return !pods.some((it) => /csi-linode-controller/i.test(it.name)); 172 | }) 173 | .sort((a, b) => { 174 | if ( 175 | podsByNode[a].find((pod) => pod.name.startsWith("overprovisioning")) 176 | ) { 177 | return 1; 178 | } 179 | if ( 180 | podsByNode[b].find((pod) => pod.name.startsWith("overprovisioning")) 181 | ) { 182 | return -1; 183 | } 184 | return a.localeCompare(b); 185 | }) 186 | .map((nodeName) => { 187 | const pods = podsByNode[nodeName] 188 | .sort((a, b) => a.name.localeCompare(b.name)) 189 | .filter( 190 | (it) => 191 | ![ 192 | "calico", 193 | "csi", 194 | "kube-proxy", 195 | "coredns", 196 | "locust", 197 | "prom", 198 | ].some((prefix) => it.name.startsWith(prefix)) 199 | ); 200 | return [ 201 | '
  • ', 202 | `

    ${nodeName}

    `, 203 | "
    ", 204 | `
    ${renderNode( 205 | pods 206 | )}
    `, 207 | "
    ", 208 | "
    • ", 209 | ].join(""); 210 | }); 211 | 212 | content.innerHTML = `
      ${nodeTemplates.join( 213 | "" 214 | )}
    `; 215 | 216 | function renderNode(pods) { 217 | return [ 218 | '
      ', 219 | pods 220 | .map((pod) => { 221 | if (pod.name.includes("overprovisioning")) { 222 | return [ 223 | '
    • ', 224 | `
      Placeholder
      `, 225 | "
      • ", 226 | ].join(""); 227 | } 228 | return [ 229 | '
    • ', 230 | `
      `, 231 | "
      • ", 232 | ].join(""); 233 | }) 234 | .join(""), 235 | "
    ", 236 | ].join(""); 237 | } 238 | } 239 | 240 | return { 241 | upsertPod(podId, pod) { 242 | if (!pod.spec.nodeName) { 243 | return; 244 | } 245 | allPods.set(podId, { 246 | name: pod.metadata.name, 247 | namespace: pod.metadata.namespace, 248 | nodeName: pod.spec.nodeName, 249 | phase: pod.status.phase, 250 | }); 251 | render(); 252 | }, 253 | removePod(podId) { 254 | allPods.delete(podId); 255 | render(); 256 | }, 257 | upsertNode(nodeId, node) { 258 | allNodes.set(nodeId, { name: node.metadata.name }); 259 | render(); 260 | }, 261 | removeNode(nodeId) { 262 | allNodes.delete(nodeId); 263 | render(); 264 | }, 265 | pods() { 266 | return Array.from(allPods.values()); 267 | }, 268 | }; 269 | } 270 | 271 | function groupBy(arr, groupByKeyFn) { 272 | return arr.reduce((acc, c) => { 273 | const key = groupByKeyFn(c); 274 | if (!(key in acc)) { 275 | acc[key] = []; 276 | } 277 | acc[key].push(c); 278 | return acc; 279 | }, {}); 280 | } 281 | 282 | document 283 | .querySelector("#go-1") 284 | ?.addEventListener("click", createEventListener(1)); 285 | document 286 | .querySelector("#go-5") 287 | ?.addEventListener("click", createEventListener(5)); 288 | document 289 | .querySelector("#go-9") 290 | ?.addEventListener("click", createEventListener(9)); 291 | document 292 | .querySelector("#placeholder-1") 293 | ?.addEventListener("click", createPlaceholderEventListener(1)); 294 | document 295 | .querySelector("#placeholder-0") 296 | ?.addEventListener("click", createPlaceholderEventListener(0)); 297 | 298 | function createEventListener(replicas) { 299 | return (e) => { 300 | startTime = Date.now(); 301 | timer.innerHTML = `00:00:00`; 302 | fetch("/apis/apps/v1/namespaces/default/deployments/podinfo", { 303 | method: "PATCH", 304 | body: JSON.stringify({ spec: { replicas } }), 305 | headers: { 306 | "Content-Type": "application/strategic-merge-patch+json", 307 | }, 308 | }).then((response) => response.json()); 309 | const timerId = setInterval(() => { 310 | timer.innerHTML = `${toHHMMSS(Date.now() - startTime)}`; 311 | const podinfo = app 312 | .pods() 313 | .filter((it) => /podinfo/.test(it.name) && it.phase === "Running"); 314 | if (podinfo.length === replicas) { 315 | clearInterval(timerId); 316 | startTime = undefined; 317 | } 318 | }, 16); 319 | }; 320 | } 321 | 322 | function createPlaceholderEventListener(replicas) { 323 | return (e) => { 324 | startTime = Date.now(); 325 | timer.innerHTML = `00:00:00`; 326 | fetch("/apis/apps/v1/namespaces/default/deployments/overprovisioning", { 327 | method: "PATCH", 328 | body: JSON.stringify({ spec: { replicas } }), 329 | headers: { 330 | "Content-Type": "application/strategic-merge-patch+json", 331 | }, 332 | }).then((response) => response.json()); 333 | }; 334 | } 335 | 336 | function toHHMMSS(value) { 337 | const sec_num = value / 1000; 338 | var hours = Math.floor(sec_num / 3600); 339 | var minutes = Math.floor((sec_num - hours * 3600) / 60); 340 | var seconds = Math.ceil(sec_num - hours * 3600 - minutes * 60); 341 | 342 | if (hours < 10) { 343 | hours = "0" + hours; 344 | } 345 | if (minutes < 10) { 346 | minutes = "0" + minutes; 347 | } 348 | if (seconds < 10) { 349 | seconds = "0" + seconds; 350 | } 351 | return hours + ":" + minutes + ":" + seconds; 352 | } 353 | -------------------------------------------------------------------------------- /dashboard/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | K8bit 8 | 9 | 38 | 39 | 40 | 41 |
    42 |
    43 |
    44 | 45 |
    46 | 47 |
    48 | 49 |
    50 | 51 | 52 |
    53 |
    54 |

    00:00:00

    55 |

    time elapsed

    56 |
    57 |
    58 |
    59 |
    60 | 61 | 62 | 63 | -------------------------------------------------------------------------------- /modules/autoscaling/locust.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: locust-script 5 | data: 6 | locustfile.py: |- 7 | from locust import HttpUser, task, between 8 | 9 | class QuickstartUser(HttpUser): 10 | @task 11 | def hello_world(self): 12 | self.client.get("/", headers={"Host": "example.com"}) 13 | --- 14 | apiVersion: apps/v1 15 | kind: Deployment 16 | metadata: 17 | name: locust 18 | spec: 19 | selector: 20 | matchLabels: 21 | app: locust-primary 22 | template: 23 | metadata: 24 | labels: 25 | app: locust-primary 26 | spec: 27 | containers: 28 | - name: locust 29 | image: locustio/locust 30 | args: ["--master"] 31 | ports: 32 | - containerPort: 5557 33 | name: comm 34 | - containerPort: 5558 35 | name: comm-plus-1 36 | - containerPort: 8089 37 | name: web-ui 38 | volumeMounts: 39 | - mountPath: /home/locust 40 | name: locust-script 41 | volumes: 42 | - name: locust-script 43 | configMap: 44 | name: locust-script 45 | --- 46 | apiVersion: v1 47 | kind: Service 48 | metadata: 49 | name: locust 50 | spec: 51 | ports: 52 | - port: 5557 53 | name: communication 54 | - port: 5558 55 | name: communication-plus-1 56 | - port: 80 57 | targetPort: 8089 58 | name: web-ui 59 | selector: 60 | app: locust-primary 61 | type: LoadBalancer 62 | --- 63 | apiVersion: apps/v1 64 | kind: DaemonSet 65 | metadata: 66 | name: locust 67 | spec: 68 | selector: 69 | matchLabels: 70 | app: locust-worker 71 | template: 72 | metadata: 73 | labels: 74 | app: locust-worker 75 | spec: 76 | containers: 77 | - name: locust 78 | image: locustio/locust 79 | args: ["--worker", "--master-host=locust"] 80 | volumeMounts: 81 | - mountPath: /home/locust 82 | name: locust-script 83 | volumes: 84 | - name: locust-script 85 | configMap: 86 | name: locust-script 87 | -------------------------------------------------------------------------------- /modules/autoscaling/main.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | kubernetes = { 4 | source = "hashicorp/kubernetes" 5 | version = "2.23.0" 6 | } 7 | helm = { 8 | source = "hashicorp/helm" 9 | version = "2.11.0" 10 | } 11 | } 12 | } 13 | 14 | provider "kubernetes" { 15 | config_path = var.kubeconfig_path 16 | } 17 | 18 | provider "helm" { 19 | kubernetes { 20 | config_path = var.kubeconfig_path 21 | } 22 | } 23 | 24 | variable "kubeconfig_path" { 25 | type = string 26 | } 27 | 28 | locals { 29 | keda_namespace = "default" 30 | prometheus_namespace = "default" 31 | } 32 | 33 | resource "helm_release" "keda" { 34 | name = "keda" 35 | chart = "https://kedacore.github.io/charts/keda-2.11.2.tgz" 36 | namespace = local.keda_namespace 37 | depends_on = [ 38 | null_resource.node_label 39 | ] 40 | } 41 | 42 | resource "null_resource" "node_label" { 43 | provisioner "local-exec" { 44 | command = "kubectl label nodes $(kubectl get nodes -o jsonpath='{.items[0].metadata.name}' --kubeconfig=${var.kubeconfig_path}) node=primary --kubeconfig=${var.kubeconfig_path} --overwrite" 45 | } 46 | } 47 | 48 | resource "helm_release" "prometheus" { 49 | name = "prometheus" 50 | chart = "https://github.com/prometheus-community/helm-charts/releases/download/prometheus-25.0.0/prometheus-25.0.0.tgz" 51 | namespace = local.prometheus_namespace 52 | 53 | set { 54 | name = "server.global.scrape_interval" 55 | value = "10s" 56 | } 57 | 58 | set { 59 | name = "server.global.evaluation_interval" 60 | value = "10s" 61 | } 62 | depends_on = [ 63 | null_resource.node_label 64 | ] 65 | } 66 | 67 | # https://medium.com/@danieljimgarcia/dont-use-the-terraform-kubernetes-manifest-resource-6c7ff4fe629a 68 | resource "null_resource" "expose" { 69 | triggers = { 70 | invokes_me_everytime = uuid() 71 | kubeconfig_path = var.kubeconfig_path 72 | current_path = path.module 73 | } 74 | 75 | provisioner "local-exec" { 76 | command = "kubectl apply --kubeconfig=${var.kubeconfig_path} -f ${path.module}/locust.yaml" 77 | } 78 | 79 | # https://github.com/hashicorp/terraform/issues/23679#issuecomment-885063851 80 | provisioner "local-exec" { 81 | command = "kubectl delete --kubeconfig=${self.triggers.kubeconfig_path} --ignore-not-found=true -f ${self.triggers.current_path}/locust.yaml" 82 | when = destroy 83 | } 84 | } -------------------------------------------------------------------------------- /modules/cluster/main.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | linode = { 4 | source = "linode/linode" 5 | version = "2.7.2" 6 | } 7 | } 8 | } 9 | 10 | variable "name" { 11 | type = string 12 | } 13 | 14 | variable "region" { 15 | type = string 16 | } 17 | 18 | resource "linode_lke_cluster" "this" { 19 | label = var.name 20 | k8s_version = "1.26" 21 | region = var.region 22 | 23 | pool { 24 | type = "g6-standard-2" 25 | count = 1 26 | 27 | autoscaler { 28 | min = 1 29 | max = 10 30 | } 31 | } 32 | 33 | # Prevent the count field from overriding autoscaler-created nodes 34 | lifecycle { 35 | ignore_changes = [ 36 | pool.0.count 37 | ] 38 | } 39 | } 40 | 41 | output "kubeconfig" { 42 | value = base64decode(linode_lke_cluster.this.kubeconfig) 43 | sensitive = true 44 | } 45 | 46 | output "cluster_id" { 47 | value = linode_lke_cluster.this.id 48 | } 49 | 50 | output "pool_id" { 51 | value = linode_lke_cluster.this.pool.0.id 52 | } 53 | -------------------------------------------------------------------------------- /modules/label/main.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | kubernetes = { 4 | source = "hashicorp/kubernetes" 5 | version = "2.23.0" 6 | } 7 | } 8 | } 9 | 10 | provider "kubernetes" { 11 | config_path = var.kubeconfig_path 12 | } 13 | 14 | variable "kubeconfig_path" { 15 | type = string 16 | } 17 | 18 | resource "null_resource" "node_label" { 19 | provisioner "local-exec" { 20 | command = "kubectl label nodes $(kubectl get nodes -o jsonpath='{.items[0].metadata.name}' --kubeconfig=${var.kubeconfig_path}) node=primary --kubeconfig=${var.kubeconfig_path} --overwrite" 21 | } 22 | } 23 | 24 | --------------------------------------------------------------------------------