├── .gitignore
├── README.md
├── canIdo.sh
├── countPodsAndContainerPerNodeCSV.sh
├── findEmptyNamespaces.sh
├── getNodesLoadCSV.sh
├── getPodsLoad.sh
├── getPodsTopCSV.sh
├── getResourcesCSV.sh
├── getRestartingPods.sh
├── podReady.sh
├── runCommandOnPods.sh
└── yaml
    ├── kind-config.yaml
    ├── memory.yaml
    ├── metrics-server-patch.yaml
    ├── podWithHostFS.yaml
    ├── podWithTools.yaml
    ├── podmanPod.yaml
    └── superPod.yaml


/.gitignore:
--------------------------------------------------------------------------------
1 | .idea
2 | resources.csv*
3 | top-*
4 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # Kubernetes Scripts
  2 | A collection of scripts and commands for various tasks in [Kubernetes](https://kubernetes.io/).<br>
  3 | These were all written during my work with Kubernetes on various project. Enjoy and share. Contributions are more than welcome!
  4 | 
  5 | ## Usage
  6 | Each script has a `usage` function. See usage with
  7 | ```shell script
  8 | <script> --help
  9 | ```
 10 | 
 11 | ## Scripts
 12 | * [countPodsAndContainerPerNodeCSV.sh](countPodsAndContainerPerNodeCSV.sh): Count number of pods and containers per node. Print in CSV format.
 13 | * [findEmptyNamespaces.sh](findEmptyNamespaces.sh): Loop over all namespaces in a cluster and find empty ones.
 14 | * [getPodsLoad.sh](getPodsLoad.sh): Get formatted results of pods in a namespace underlying node's load average (using cat /proc/loadavg).
 15 | * [getPodsTopCSV.sh](getPodsTopCSV.sh): Get a pod's cpu and memory usage (optionally per container) written as CSV formatted file.
 16 | * [getResourcesCSV.sh](getResourcesCSV.sh): Get all pods resources requests, limits and actual usage per container in a CSV format with values normalized.
 17 | * [getRestartingPods.sh](getRestartingPods.sh): Get all pods (all or single namespace) that have restarts detected in one or more containers. Formatted in CSV.
 18 | * [podReady.sh](podReady.sh): Simple script to check if pod is really ready. Check status is 'Running' and that all containers are ready. Returns 0 if ready. Returns 1 if not ready.
 19 | * [getNodesLoadCSV.sh](getNodesLoadCSV.sh): Traverse over the `kube-proxy` pods to get the nodes load average and number of CPUs in a CSV format. Will also mark high load node with big `YES` in the output.
 20 | * [runCommandOnPods.sh](runCommandOnPods.sh): Run a command on a list of pods.
 21 | * [canIdo.sh](canIdo.sh): Check all or some permissions current user has in a namespace on all or some resources using the `kubectl auth can-i` command.
 22 | 
 23 | ## YAML
 24 | * [memory.yaml](yaml/memory.yaml): A pod using a given block of memory for a given time.<br>
 25 | * [podWithTools.yaml](yaml/podWithTools.yaml): A pod with some basic tools (`vi` and `curl`) for easy debugging.<br>
 26 | * [podWithHostFS.yaml](yaml/podWithHostFS.yaml): A pod with the host root file system mounted into it.<br>
 27 | * [podmanPod.yaml](yaml/podmanPod.yaml): A pod with [podman](https://podman.io/) in it.<br>
 28 | * [superPod](yaml/superPod.yaml): A pod with the host file system and process IDs mounted for allowing to manipulate the host system and restarting kubelet.<br>
 29 |   **WARNING:** There is danger of corrupting your Kubernetes host. Use with extra care!
 30 | * [kind-config.yaml](yaml/kind-config.yaml): An example [kind](https://kind.sigs.k8s.io/) configuration for a multi node K8s cluster
 31 |   **WARNING:** There is danger of corrupting your Kubernetes host. Use with extra care!
 32 | 
 33 | ## Commands
 34 | ### Kubectl
 35 | #### See all cluster nodes load (top)
 36 | ```shell script
 37 | kubectl top nodes
 38 | ```
 39 | 
 40 | #### Get cluster events
 41 | ```shell script
 42 | # All cluster
 43 | kubectl get events
 44 | 
 45 | # Specific namespace events
 46 | kubectl get events --namespace=kube-system
 47 | ```
 48 | 
 49 | #### Get all cluster nodes IPs and names
 50 | ```shell script
 51 | # Single call to K8s API
 52 | kubectl get nodes -o json | grep -A 12 addresses
 53 | 
 54 | # A loop for more flexibility
 55 | for n in $(kubectl get nodes -o name); do \
 56 |   echo -e "\nNode ${n}"; \
 57 |   kubectl get ${n} -o jsonpath='{.status.addresses}'; \
 58 | done
 59 | ```
 60 | 
 61 | #### See all cluster nodes CPU and Memory requests and limits
 62 | ```shell script
 63 | # With node names
 64 | kubectl describe nodes | grep -A 3 "Name:\|Resource .*Requests .*Limits" | grep -v "Roles:"
 65 | 
 66 | # Just the resources
 67 | kubectl describe nodes | grep -A 3 "Resource .*Requests .*Limits"
 68 | ``` 
 69 | 
 70 | ##### Using kube-capacity
 71 | There is a great CLI for getting a cluster capacity and utilization - [kube-capacity](https://github.com/robscott/kube-capacity).<br>
 72 | Install as described in the [installation](https://github.com/robscott/kube-capacity#installation) section.
 73 | ```shell script
 74 | # Get cluster current capacity
 75 | kube-capacity
 76 | 
 77 | # Get cluster current capacity with pods breakdown
 78 | kube-capacity --pods
 79 | 
 80 | # Get cluster current capacity and utilization
 81 | kube-capacity --util
 82 | 
 83 | # Displaying available resources
 84 | kube-capacity --available
 85 | 
 86 | # Roll over all clusters in your kubectl contexts
 87 | for a in $(kubectl ctx); do echo -e "\n---$a"; kubectl ctx $a; kube-capacity; done
 88 | 
 89 | # Roll over all clusters in your kubectl contexts and get just summary of each cluster
 90 | for a in $(kubectl ctx); do echo -e "\n---$a"; kubectl ctx $a; kube-capacity| grep -B 1 "\*"; done
 91 | ```
 92 | 
 93 | #### Get all labels attached to all pods in a namespace
 94 | ```shell script
 95 | for a in $(kubectl get pods -n namespace1 -o name); do \
 96 |   echo -e "\nPod ${a}"; \
 97 |   kubectl -n namespace1 describe ${a} | awk '/Labels:/,/Annotations/' | sed '/Annotations/d'; \
 98 | done
 99 | ```
100 | 
101 | #### Forward local port to a pod or service
102 | ```shell script
103 | # Forward localhost port 8080 to a specific pod exposing port 8080
104 | kubectl port-forward -n namespace1 web 8080:8080
105 | 
106 | # Forward localhost port 8080 to a specific web service exposing port 80
107 | kubectl port-forward -n namespace1 svc/web 8080:80
108 | ```
109 | 
110 | #### Port forwarding
111 | * A great tool for port forwarding all services in a namespace + adding aliases to `/etc/hosts` is [kubefwd](https://github.com/txn2/kubefwd).
112 | Note that this requires root or sudo to allow temporary editing of `/etc/host`.
113 | ```shell script
114 | # Port forward all service in namespace1
115 | kubefwd svc -n namespace1
116 | ```
117 | 
118 | #### Extract and decode a secret's value
119 | ```shell script
120 | # Get the value of the postgresql password
121 | kubectl get secret -n namespace1 my-postgresql -o jsonpath="{.data.postgres-password}" | base64 --decode
122 | ```
123 | 
124 | #### Copy secret from `namespace1` to `namespace2`
125 | ```shell script
126 | kubectl get secret my-secret --namespace namespace1 -o yaml | sed "/namespace:/d" | kubectl apply --namespace=namespace2 -f -
127 | ```
128 | 
129 | #### Create an Ubuntu pod
130 | A one liner to create an Ubuntu pod that will just wait forever. 
131 | ```shell script
132 | # Create the pod
133 | cat <<ZZZ | kubectl apply -f -
134 | apiVersion: v1
135 | kind: Pod
136 | metadata:
137 |   name: my-ubuntu-pod
138 | spec:
139 |   containers:
140 |   - name: my-ubuntu-container
141 |     image: eldada.jfrog.io/docker/ubuntu:22.04
142 |     command:
143 |     - 'bash'
144 |     - '-c'
145 |     - 'while true; do date; sleep 60; done'
146 | ZZZ
147 | 
148 | # Shell into the pod
149 | kubectl exec -it my-ubuntu-pod bash
150 | 
151 | # Delete the pod once done
152 | kubectl delete pod my-ubuntu-pod
153 | ```
154 | 
155 | #### Start a shell in a temporary pod
156 | Note - Pod will terminate once exited
157 | ```shell script
158 | # Ubuntu
159 | kubectl run my-ubuntu --rm -i -t --restart=Never --image ubuntu -- bash
160 | 
161 | # CentOS
162 | kubectl run my-centos --rm -i -t --restart=Never --image centos:8 -- bash
163 | 
164 | # Alpine
165 | kubectl run my-alpine --rm -i -t --restart=Never --image alpine:3.10 -- sh
166 | 
167 | # Busybox
168 | kubectl run my-busybox --rm -i -t --restart=Never --image busybox -- sh
169 | ```
170 | 
171 | #### Get formatted list of containers and container images
172 | Useful for listing all running containers in your cluster
173 | ```shell script
174 | # Example 1 - just the container names
175 | kubectl get pods -A -o jsonpath='{..containers[*].name}' | tr -s ' ' '\n'
176 | # With sorting and unique names
177 | kubectl get pods -A -o jsonpath='{..containers[*].name}' | tr -s ' ' '\n' | sort | uniq
178 | 
179 | # Example 2 - container images and tags
180 | kubectl get pods -A -o=jsonpath='{..containers[*].image}' | tr -s ' ' '\n'
181 | # With sorting and unique names
182 | kubectl get pods -A -o=jsonpath='{..containers[*].image}' | tr -s ' ' '\n' | sort | uniq
183 | 
184 | # Example 3 - pod and its container images
185 | kubectl get pods -A -o=jsonpath='{range .items[*]}{"\n"}{.metadata.name}{":\n"}{range .spec.containers[*]}{.name},{.image}{"\n"}{end}{end}'
186 | 
187 | # Example 4 - pod and its container images with their resources requests (cpu and memory)
188 | kubectl get pods -A -o=jsonpath='{range .items[*]}{"\n"}{.metadata.name}{":\n"}{range .spec.containers[*]}{.name},{.image}{.resources.requests.cpu},{.resources.requests.memory}{"\n"}{end}{end}'
189 | 
190 | ```
191 | Look into [a few more examples](https://kubernetes.io/docs/tasks/access-application-cluster/list-all-running-container-images) of listing containers
192 | 
193 | #### Get list of pods sorted by restart count
194 | * Option 1 for all pods (Taken from [kubectl cheatsheet](https://kubernetes.io/docs/reference/kubectl/cheatsheet/#viewing-finding-resources))
195 | ```shell script
196 | kubectl get pods -A --sort-by='.status.containerStatuses[0].restartCount'
197 | ```
198 | 
199 | * Option 2 with a filter, and a CSV friendly output
200 | ```shell script
201 | kubectl get pods -A | grep my-app | awk '{print $5 ", " $1 ", " $6}'  | sort -n -r
202 | ```
203 | 
204 | #### Get current replica count on all HPAs (Horizontal Pod Autoscaling)
205 | ```shell script
206 | kubectl get hpa -A -o=custom-columns=NAME:.metadata.name,REPLICAS:.status.currentReplicas | sort -k2 -n -r
207 | ```
208 | 
209 | #### List non-running pods
210 | ```shell script
211 | kubectl get pods -A --no-headers | grep -v Running | grep -v Completed
212 | ```
213 | 
214 | #### Top Pods by CPU or memory usage
215 | ```shell script
216 | # Top 20 pods by highest CPU usage
217 | kubectl top pods -A --sort-by=cpu | head -20
218 | 
219 | # Top 20 pods by highest memory usage
220 | kubectl top pods -A --sort-by=memory | head -20
221 | 
222 | # Roll over all kubectl contexts and get top 20 CPU users
223 | for a in $(kubectl ctx); do echo -e "\n---$a"; kubectl ctx $a; kubectl top pods -A --sort-by=cpu | head -20; done
224 | ```
225 | 
226 | #### Debugging Pods and Nodes
227 | This section is based on [debugging pods using ephemeral containers](https://kubernetes.io/docs/tasks/debug/debug-application/debug-running-pod/#ephemeral-container-example)
228 | and [kubectl node debug](https://kubernetes.io/docs/tasks/debug/debug-cluster/kubectl-node-debug/)
229 | 
230 | ##### Pod Debugging
231 | ```shell script
232 | # Attach an ephemeral container to an existing container in a pod for debugging
233 | kubectl debug -it my-pod --image=ubuntu --target=my-container
234 | ```
235 | 
236 | ##### Node Debugging
237 | ```shell script
238 | # Debug a node with a new pod attached to it
239 | # IMPORTANT to delete the pods after exiting it. It will not be deleted automatically (although it will be in the "Completed" state)
240 | kubectl debug node/<mynode> -it --image=ubuntu
241 | ```
242 | ### Helm
243 | 
244 | #### Helm template
245 | View the templates generated by `helm install`. Useful for seeing the actual templates generated by helm before deploying.<br>
246 | Can also be used for deploying the templates generated when cannot use Tiller
247 | ```shell script
248 | helm template <chart>
249 | ```
250 | 
251 | #### Debug helm install
252 | * Debug a `helm install`. Useful for seeing the actual values resolved by helm before deploying
253 | ```shell script
254 | helm install --debug --dry-run <chart>
255 | ```
256 | 
257 | ### Rolling restarts
258 | Roll a restart across all resources managed by a Deployment, DaemonSet or StatefulSet with **zero downtime**<br>
259 | **IMPORTANT**: For a Deployment or StatefulSet, a zero downtime is possible only if initial replica count is **higher than 1**!
260 | ```shell script
261 | # Deployment
262 | kubectl -n <namespace> rollout restart deployment <deployment-name>
263 | 
264 | # DaemonSet
265 | kubectl -n <namespace> rollout restart daemonset <daemonset-name>
266 | 
267 | # StatefulSet
268 | kubectl -n <namespace> rollout restart statefulsets <statefulset-name>
269 | ```
270 | 
271 | ### Mark Nodes with some roles for visibility (ex. EKS nodes marked with the LifeCycle,NodeType)
272 | * Most use of it can be gained with some GUI client (Lens), still "k get nodes" shows ROLE fields as well
273 | ```shell
274 | for n in $(kubectl get nodes -o 'jsonpath={.items[*].metadata.name}') ; do
275 |   lb=""
276 |   for a in $(kubectl label --list nodes $n | sort | grep -e NodeType -e lifecycle | cut -d= -f 2); do
277 |     lb="${lb}$a"
278 |   done
279 |   kubectl label nodes $n node-role.kubernetes.io/$lb=
280 | done
281 | ```
282 | 
283 | ## A Multi Node Kubernetes cluster in Mac with Kind
284 | To run a multi node Kubernetes cluster in Mac with [Kind](https://kind.sigs.k8s.io/), do the following (assuming Docker Desktop is already installed)
285 | - Install `kind` as described in [kind installation](https://kind.sigs.k8s.io/docs/user/quick-start/#installation)
286 | - Start a local, three worker nodes cluster using the [kind-config.yaml](yaml/kind-config.yaml) config file
287 | ```shell
288 | kind create cluster --config yaml/kind-config.yaml --name demo
289 | ```
290 | 
291 | Delete the cluster with
292 | ```shell
293 | kind delete cluster --name demo
294 | ```
295 | 
296 | ## Metrics Server in Kubernetes on Docker Desktop or Kind for Mac
297 | To get around issue with certificates in your local Docker Desktop or Kind Kubernetes
298 | 
299 | Install a `metrics-server`
300 | ```shell
301 | kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml
302 | ```
303 | 
304 | Patch the `metrics-server` Deployment with the fix
305 | ```shell
306 | kubectl patch deployment metrics-server -n kube-system --patch-file yaml/metrics-server-patch.yaml
307 | ```
308 | 
309 | **OR** Edit the `metrics-server` deployment directly and add `--kubelet-insecure-tls` to the `args` key:
310 | ```yaml
311 | spec:
312 |   containers:
313 |   - args:
314 |     - --cert-dir=/tmp
315 |     - --secure-port=443
316 |     - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
317 |     - --kubelet-use-node-status-port
318 |     - --metric-resolution=15s
319 |     - --kubelet-insecure-tls
320 | ```
321 | 
322 | ### Resources
323 | Most of the code above is self experimenting and reading the docs. Some are copied and modified to my needs from other resources...
324 | * https://kubernetes.io/docs/reference/kubectl/cheatsheet/
325 | * https://medium.com/flant-com/kubectl-commands-and-tips-7b33de0c5476
326 | * https://github.com/robscott/kube-capacity
327 | 


--------------------------------------------------------------------------------
/canIdo.sh:
--------------------------------------------------------------------------------
  1 | #!/bin/bash
  2 | 
  3 | # Go over all resources in the Kubernetes cluster and check if current user can run an action on them
  4 | NAMESPACE=default
  5 | ACTIONS="get list watch create update patch delete"
  6 | RESOURCES=all
  7 | SCRIPT_NAME=$0
  8 | 
  9 | ######### Functions #########
 10 | 
 11 | errorExit () {
 12 |     echo -e "\nERROR: $1\n"
 13 |     exit 1
 14 | }
 15 | 
 16 | usage () {
 17 |     cat << END_USAGE
 18 | 
 19 | ${SCRIPT_NAME} - Check if current user can perform the selected action in a given namespace
 20 | 
 21 | Usage: ${SCRIPT_NAME} <options>
 22 | 
 23 | -n | --namespace <name>                   : Namespace to check.          Default: ${NAMESPACE}
 24 | -a | --actions <comma delimited list>     : List of actions.             Default: (${ACTIONS})
 25 | -r | --resources <comma delimited list>   : List of resources to test.   Default: all
 26 | -h | --help                               : Show this usage.
 27 | 
 28 | Examples:
 29 | ========
 30 | Test all permissions for namespace 'foo':                            $ ${SCRIPT_NAME} --namespace foo
 31 | Test 'get' and 'list' permissions for namespace 'bar':               $ ${SCRIPT_NAME} --namespace bar --actions get,list
 32 | Test 'get' and 'list' permissions on 'pods' and 'secrets':           $ ${SCRIPT_NAME} --resources pods,secrets --actions get,list
 33 | 
 34 | END_USAGE
 35 | 
 36 |     exit 1
 37 | }
 38 | 
 39 | # Process command line options. See usage above for supported options
 40 | processOptions () {
 41 |     while [[ $# -gt 0 ]]; do
 42 |         case "$1" in
 43 |             -n | --namespace)
 44 |                 NAMESPACE="$2"
 45 |                 shift 2
 46 |             ;;
 47 |             -a | --actions)
 48 |                 ACTIONS="$(echo $2 | tr ',' ' ')"
 49 |                 shift 2
 50 |             ;;
 51 |             -r | --resources)
 52 |                 RESOURCES="$(echo $2 | tr ',' ' ')"
 53 |                 shift 2
 54 |             ;;
 55 |             -h | --help)
 56 |                 usage
 57 |             ;;
 58 |             *)
 59 |                 usage
 60 |             ;;
 61 |         esac
 62 |     done
 63 | }
 64 | 
 65 | # Test connection to a cluster by kubectl
 66 | testConnection () {
 67 |     echo "Testing connection to cluster"
 68 |     kubectl cluster-info > /dev/null || errorExit "Connection to cluster failed"
 69 | }
 70 | 
 71 | canIdo () {
 72 |     local resource
 73 |     local action
 74 |     local resource_list
 75 | 
 76 |     if [[ ${RESOURCES} =~ all ]]; then
 77 |         resource_list="$(kubectl api-resources --verbs=list --namespaced -o name)"
 78 |     else
 79 |         resource_list=${RESOURCES}
 80 |     fi
 81 | 
 82 | #    echo "List of resources:"
 83 | #    echo "${resource_list}"
 84 | 
 85 |     for resource in ${resource_list}; do
 86 |         for action in ${ACTIONS}; do
 87 |             echo -n "--- Checking ${action} on '${resource}': "
 88 |             kubectl auth can-i "${action}" "${resource}" --namespace="${NAMESPACE}"
 89 |         done
 90 |     done
 91 | }
 92 | 
 93 | main () {
 94 |     processOptions "$@"
 95 |     testConnection
 96 | 
 97 |     echo "Testing actions [${ACTIONS}] on [${RESOURCES}] resources in namespace ${NAMESPACE}"
 98 | 
 99 |     canIdo
100 | }
101 | 
102 | ######### Main #########
103 | 
104 | main "$@"
105 | 


--------------------------------------------------------------------------------
/countPodsAndContainerPerNodeCSV.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | # Count number of pods and containers per node in a kubernetes cluster and write a CSV formatted output
 4 | 
 5 | errorExit () {
 6 |     echo -e "\nERROR: $1\n"
 7 |     exit 1
 8 | }
 9 | 
10 | # Test connection to cluster
11 | kubectl cluster-info > /dev/null || errorExit "Connection to cluster failed"
12 | 
13 | # Get the list of nodes in the cluster
14 | nodes=$(kubectl get nodes -o jsonpath="{.items[*].metadata.name}")
15 | 
16 | # Header for CSV
17 | echo "Node,Pods count,Containers count"
18 | 
19 | # Loop over the nodes
20 | for node in ${nodes}; do
21 |     # Get the list of pods running on the current node
22 |     pods=$(kubectl get pods -A -o jsonpath="{.items[*].metadata.name}" --field-selector spec.nodeName="${node}")
23 | 
24 |     # Get the list of containers running on the current node
25 |     containers=$(kubectl get pods -A -o jsonpath="{.items[*].spec.containers[*].name}" --field-selector spec.nodeName="${node}")
26 | 
27 |     echo "${node},$(echo "${pods}" | wc -w | tr -d ' '),$(echo "${containers}" | wc -w | tr -d ' ')"
28 | done
29 | 


--------------------------------------------------------------------------------
/findEmptyNamespaces.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | # Go over all namespaces in a cluster and check if they are empty of workloads and controller objects
 4 | 
 5 | # All workload objects to search for
 6 | OBJECTS=pods,jobs,cronjobs,deployments,daemonsets,statefulsets
 7 | 
 8 | # Temp files for the grep later on
 9 | NS_ALL=namespaces-all.txt
10 | NS_WL=namespaces-with-workloads.txt
11 | 
12 | # Get list of all namespaces
13 | kubectl get ns --no-headers -o=custom-columns=NAME:.metadata.name > ${NS_ALL}
14 | 
15 | # Get list of all namespaces with any workload in them
16 | kubectl get ${OBJECTS} --no-headers --all-namespaces -o=custom-columns=NAMESPACE:.metadata.namespace | sort -u > ${NS_WL}
17 | 
18 | # Use grep to find all values in NS_ALL that are not in NS_WL (these are empty namespaces)
19 | grep -vf ${NS_WL} ${NS_ALL}
20 | 
21 | # Cleanup temp files
22 | rm -f ${NS_WL} ${NS_ALL}
23 | 


--------------------------------------------------------------------------------
/getNodesLoadCSV.sh:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env bash
 2 | 
 3 | # Go over all nodes in a cluster and extract the load average from them by running
 4 | #  > cat /proc/loadavg
 5 | # in the kube-proxy pods (since they run as a Daemonset).
 6 | 
 7 | errorExit () {
 8 |     echo -e "\nERROR: $1\n"
 9 |     exit 1
10 | }
11 | 
12 | # Check we have the kube-proxy daemonset, so we can run commands on its pods
13 | kubectl get ns kube-system > /dev/null 2>&1 || errorExit "Namespace kube-system not found, or you don't have permission to get it"
14 | kubectl get ds kube-proxy -n kube-system > /dev/null 2>&1 || errorExit "Daemonset kube-proxy not found in kube-system"
15 | 
16 | pods=$(kubectl get po -n kube-system | grep kube-proxy | awk '{print $1}' | tr '\n' ' ')
17 | 
18 | # Test on the first pod that the "cat /proc/loadavg; echo ' '; nproc" works (blocked on newer versions of kube-proxy)
19 | first_pod="${pods%% *}"
20 | kubectl exec -n kube-system ${first_pod} -c kube-proxy -- sh -c "cat /proc/loadavg; echo ' '; nproc" > /dev/null 2>&1 || errorExit "Failed to run command on kube-proxy pod. This script cannot be used to get load"
21 | 
22 | # Header for the CSV output
23 | echo "Node, Load 1 min, Load 5 min, Load 15 min, CPU, High load"
24 | 
25 | # Go over the pods and extract data
26 | for p in $pods; do
27 |     alert="-"
28 |     node=$(kubectl describe po -n kube-system $p | grep Node: | awk '{print $2}')
29 |     read -r load1 load5 load15 dummy1 dummy2 cpu <<< $(kubectl exec -n kube-system $p -c kube-proxy -- sh -c "cat /proc/loadavg; echo ' '; nproc" 2> /dev/null | tr -d '\n')
30 | 
31 |     # Convert load5 to integer for easier comparison later
32 |     load_int=${load5%.*}
33 | 
34 |     # If load > number of cpu, it should be marked with YES as "High load"
35 |     if [[ $load_int -gt $cpu ]]; then alert="YES"; fi
36 | 
37 |     echo "$node, $load1, $load5, $load15, $cpu, $alert"
38 | done
39 | 
40 | 


--------------------------------------------------------------------------------
/getPodsLoad.sh:
--------------------------------------------------------------------------------
  1 | #!/bin/bash
  2 | 
  3 | # UNCOMMENT this line to enable debugging
  4 | # set -xv
  5 | 
  6 | # Get formatted results of the pods underlying node's load average (using cat /proc/loadavg)
  7 | NAMESPACE=default
  8 | INCLUDE_FILTER=.*
  9 | SCRIPT_NAME=$0
 10 | 
 11 | ######### Functions #########
 12 | 
 13 | errorExit () {
 14 |     echo -e "\nERROR: $1\n"
 15 |     exit 1
 16 | }
 17 | 
 18 | usage () {
 19 |     cat << END_USAGE
 20 | 
 21 | ${SCRIPT_NAME} - Get formatted results of the pods underlying node's load average (using cat /proc/loadavg)
 22 | 
 23 | Usage: ${SCRIPT_NAME} <options>
 24 | 
 25 | -n | --namespace <name>      : Namespace to use. Default: default
 26 | -i | --include <string>      : Include only pods with this string in their name
 27 | -h | --help                  : Show this usage
 28 | 
 29 | Examples:
 30 | ========
 31 | # Get load form pods in namespace bar:
 32 | ${SCRIPT_NAME} --namespace bar
 33 | 
 34 | # Get only specific pods in namespace
 35 | ${SCRIPT_NAME} --namespace bar --include artifactory
 36 | 
 37 | END_USAGE
 38 | 
 39 |     exit 1
 40 | }
 41 | 
 42 | # Process command line options. See usage above for supported options
 43 | processOptions () {
 44 |     while [[ $# -gt 0 ]]; do
 45 |         case "$1" in
 46 |             -n | --namespace)
 47 |                 NAMESPACE="$2"
 48 |                 shift 2
 49 |             ;;
 50 |             -i | --include)
 51 |                 INCLUDE_FILTER="$2"
 52 |                 shift 2
 53 |             ;;
 54 |             -h | --help)
 55 |                 usage
 56 |                 exit 0
 57 |             ;;
 58 |             *)
 59 |                 usage
 60 |             ;;
 61 |         esac
 62 |     done
 63 | 
 64 |     if [[ -z "${INCLUDE_FILTER}" ]]; then
 65 |         INCLUDE_FILTER=.*
 66 |     fi
 67 | }
 68 | 
 69 | # Test connection and that there are pods in the namespace
 70 | testConnection () {
 71 |     kubectl get ns "${NAMESPACE}" > /dev/null || errorExit "Namespace ${NAMESPACE} does not exist"
 72 |     [[ $(kubectl get pods -n "${NAMESPACE}" 2> /dev/null| wc -l | tr -d ' ') == '0' ]] && errorExit "Namespace ${NAMESPACE} has no running pods"
 73 | }
 74 | 
 75 | getPodsLoad () {
 76 |     local load1
 77 |     local load5
 78 |     local load15
 79 |     local dummy
 80 |     local pods
 81 | 
 82 |     # Print header
 83 |     echo "Pod, Load 1, Load 5, Load 15"
 84 | 
 85 |     # Get list of pods
 86 |     pods=$(kubectl get pods -n "${NAMESPACE}" --no-headers -o=custom-columns=NAME:.metadata.name)
 87 | 
 88 |     # Go over the pods and extract data
 89 |     for p in $pods; do
 90 |         if [[ ${p} =~ ${INCLUDE_FILTER} ]]; then
 91 |             read -r load1 load5 load15 dummy <<< $(kubectl exec -n "${NAMESPACE}" "$p" -- sh -c "cat /proc/loadavg" 2> /dev/null)
 92 |             echo "$p, $load1, $load5, $load15"
 93 |         fi
 94 |     done
 95 | }
 96 | 
 97 | main () {
 98 |     processOptions "$@"
 99 |     testConnection
100 |     getPodsLoad
101 | }
102 | 
103 | ######### Main #########
104 | 
105 | main "$@"
106 | 


--------------------------------------------------------------------------------
/getPodsTopCSV.sh:
--------------------------------------------------------------------------------
  1 | #!/bin/bash
  2 | 
  3 | # UNCOMMENT this line to enable debugging
  4 | # set -xv
  5 | 
  6 | ## Get formatted results of kubectl top pod --containers
  7 | 
  8 | OUT=top-$(date +"%Y-%m-%d_%H:%M:%S").csv
  9 | NAMESPACE=--all-namespaces
 10 | QUITE=false
 11 | HEADERS=true
 12 | SCRIPT_NAME=$0
 13 | DURATION=0
 14 | INTERVAL=5
 15 | POD=
 16 | CONTAINERS=
 17 | 
 18 | ######### Functions #########
 19 | 
 20 | errorExit () {
 21 |     echo -e "\nERROR: $1\n"
 22 |     exit 1
 23 | }
 24 | 
 25 | usage () {
 26 |     cat << END_USAGE
 27 | 
 28 | ${SCRIPT_NAME} - Get formatted results of kubectl top pod in a Kubernetes cluster for a selected namespace and pod in a CSV format
 29 | 
 30 | Usage: ${SCRIPT_NAME} <options>
 31 | 
 32 | -n | --namespace <name>                : Namespace to analyse.      Default: default
 33 | -p | --pod <name>                      : Pod to analyse.
 34 | -d | --duration <seconds>              : Duration of sampling.      Default: ${DURATION} (infinite)
 35 | -i | --interval <seconds>              : Interval between samples.  Default: ${INTERVAL}
 36 | -o | --output <name>                   : Output file.               Default: top-<timestamp>.csv
 37 | -c | --containers                      : Output per container.      Default: off
 38 | -q | --quite                           : Don't output to screen.    Default: Output to screen
 39 | -h | --help                            : Show this usage
 40 | --no-headers                           : Don't print headers line
 41 | 
 42 | Examples:
 43 | ========
 44 | Get for pod foo in namespace bar:                                $ ${SCRIPT_NAME} --namespace bar --pod foo
 45 | Get for pod foo in namespace bar and output to file foo.csv :    $ ${SCRIPT_NAME} --namespace bar --pod foo --output foo.csv
 46 | 
 47 | END_USAGE
 48 | 
 49 |     exit 1
 50 | }
 51 | 
 52 | # Process command line options. See usage above for supported options
 53 | processOptions () {
 54 |     while [[ $# -gt 0 ]]; do
 55 |         case "$1" in
 56 |             -n | --namespace)
 57 |                 NAMESPACE="--namespace $2"
 58 |                 shift 2
 59 |             ;;
 60 |             -p | --pod)
 61 |                 POD="$2"
 62 |                 shift 2
 63 |             ;;
 64 |             -d | --duration)
 65 |                 DURATION="$2"
 66 |                 shift 2
 67 |             ;;
 68 |             -i | --interval)
 69 |                 INTERVAL="$2"
 70 |                 shift 2
 71 |             ;;
 72 |             -o | --output)
 73 |                 OUT=$2
 74 |                 shift 2
 75 |             ;;
 76 |             -c | --containers)
 77 |                 CONTAINERS="--containers"
 78 |                 shift 1
 79 |             ;;
 80 |             -q | --quite)
 81 |                 QUITE=true
 82 |                 shift 1
 83 |             ;;
 84 |             --no-headers)
 85 |                 HEADERS=false
 86 |                 shift 1
 87 |             ;;
 88 |             -h | --help)
 89 |                 usage
 90 |                 exit 0
 91 |             ;;
 92 |             *)
 93 |                 usage
 94 |             ;;
 95 |         esac
 96 |     done
 97 | 
 98 |     [ -z "${POD}" ] && errorExit "Must provide pod name (--pod <POD>)"
 99 | }
100 | 
101 | # Test connection to a cluster by kubectl
102 | testConnection () {
103 |     kubectl cluster-info > /dev/null || errorExit "Connection to cluster failed"
104 |     kubectl get pod ${NAMESPACE} "${POD}" > /dev/null || errorExit "Pod ${POD} not found in namespace ${NAMESPACE}"
105 | }
106 | 
107 | formatCpu () {
108 |     local result=$1
109 |     if [[ ${result} =~ m$ ]]; then
110 |         result=$(echo "${result}" | tr -d 'm')
111 |         result=$(awk "BEGIN {print ${result}/1000}")
112 |     fi
113 | 
114 |     echo -n "${result}"
115 | }
116 | 
117 | formatMemory () {
118 |     local result=$1
119 |     if [[ ${result} =~ M ]]; then
120 |         result=$(echo "${result}" | tr -d 'Mi')
121 |         result=$(awk "BEGIN {print ${result}/1000}")
122 |     elif [[ ${result} =~ m ]]; then
123 |         result=$(echo "${result}" | tr -d 'm')
124 |         result=$(awk "BEGIN {print ${result}/1000000000000}")
125 |     elif [[ ${result} =~ G ]]; then
126 |         result=$(echo "${result}" | tr -d 'Gi')
127 |     fi
128 | 
129 |     echo -n "${result}"
130 | }
131 | 
132 | getPodsTop () {
133 |     local condition=true
134 |     local counter=0
135 |     local line=
136 |     local time_stamp=
137 |     local cpu=
138 |     local memory=
139 | 
140 |     if [ "${DURATION}" != 0 ]; then
141 |         condition="[]"
142 |     fi
143 | 
144 |     while ${condition}; do
145 |         line=$(kubectl top pod ${NAMESPACE} "${POD}" --no-headers ${CONTAINERS} --use-protocol-buffers)
146 | #        echo "--- $line"
147 |         local final_line=
148 | 
149 |         # Go over all the containers
150 |         local OLD_IFS=${IFS}
151 |         IFS=$'\n'
152 |         for l in ${line}; do
153 |             local container=""
154 |             local header="Timestamp,Pod,CPU (cores),Memory (GB)"
155 |             time_stamp=$(date +"%Y-%m-%d_%H:%M:%S")
156 |             if [ -n "${CONTAINERS}" ]; then
157 |                 container=$(echo "${l}" | awk '{print $2}')
158 |                 cpu=$(formatCpu "$(echo "${l}" | awk '{print $3}')")
159 |                 memory=$(formatMemory "$(echo "${l}" | awk '{print $4}')")
160 |                 header="Timestamp,Pod,Container,CPU (cores),Memory (GB)"
161 |             else
162 |                 cpu=$(formatCpu "$(echo "${l}" | awk '{print $2}')")
163 |                 memory=$(formatMemory "$(echo "${l}" | awk '{print $3}')")
164 |             fi
165 |             local out_file=${OUT}
166 | 
167 |             # Print header in each file if this is the first line
168 |             if [ ${counter} -eq 0 ]; then
169 |                 if [ "${HEADERS}" == true ]; then
170 |                     echo "${header}" > "${out_file}"
171 |                 else
172 |                     echo -n "" > "${out_file}"
173 |                 fi
174 |             fi
175 | 
176 |             final_line=${time_stamp},${POD},${cpu},${memory}
177 |             if [ -n "${CONTAINERS}" ]; then
178 |                 final_line=${time_stamp},${POD},${container},${cpu},${memory}
179 |             fi
180 | 
181 |             if [ "${QUITE}" == true ]; then
182 |                 echo "${final_line}" >> "${out_file}"
183 |             else
184 |                 echo "${final_line}" | tee -a "${out_file}"
185 |             fi
186 |         done
187 |         IFS=${OLD_IFS}
188 | 
189 |         sleep "${INTERVAL}"
190 | 
191 |         (( counter++ ))
192 |     done
193 | }
194 | 
195 | main () {
196 |     processOptions "$@"
197 |     [ "${QUITE}" == true ] || echo "Getting Pod memory and cpu usage"
198 |     testConnection
199 |     getPodsTop
200 | }
201 | 
202 | ######### Main #########
203 | 
204 | main "$@"
205 | 


--------------------------------------------------------------------------------
/getResourcesCSV.sh:
--------------------------------------------------------------------------------
  1 | #!/bin/bash
  2 | 
  3 | # UNCOMMENT this line to enable debugging
  4 | # set -xv
  5 | 
  6 | ## Get resources requests and limits per container in a Kubernetes cluster.
  7 | 
  8 | OUT=resources.csv
  9 | NAMESPACE=--all-namespaces
 10 | SKIP_USAGE=false
 11 | QUITE=false
 12 | HEADERS=true
 13 | CONSOLE_ONLY=false
 14 | SCRIPT_NAME=$0
 15 | 
 16 | ######### Functions #########
 17 | 
 18 | errorExit () {
 19 |     echo -e "\nERROR: $1\n"
 20 |     exit 1
 21 | }
 22 | 
 23 | usage () {
 24 |     cat << END_USAGE
 25 | 
 26 | ${SCRIPT_NAME} - Extract resource requests and limits in a Kubernetes cluster for a selected namespace or all namespaces in a CSV format
 27 | 
 28 | Usage: ${SCRIPT_NAME} <options>
 29 | 
 30 | -n | --namespace <name>                : Namespace to analyse.    Default: --all-namespaces
 31 | -o | --output <name>                   : Output file.             Default: ${OUT}
 32 | -s | --skip-usage                      : Don't get usage (kubectl top)
 33 | -q | --quite                           : Don't output to stdout.  Default: Output to stdout
 34 | -h | --help                            : Show this usage
 35 | --no-headers                           : Don't print headers line
 36 | --console-only                         : Output to stdout only (don't write to file)
 37 | 
 38 | Examples:
 39 | ========
 40 | Get all:                                                  $ ${SCRIPT_NAME}
 41 | Get for namespace foo:                                    $ ${SCRIPT_NAME} --namespace foo
 42 | Get for namespace foo and use output file bar.csv :       $ ${SCRIPT_NAME} --namespace foo --output bar.csv
 43 | 
 44 | END_USAGE
 45 | 
 46 |     exit 1
 47 | }
 48 | 
 49 | # Process command line options. See usage above for supported options
 50 | processOptions () {
 51 |     while [[ $# -gt 0 ]]; do
 52 |         case "$1" in
 53 |             -n | --namespace)
 54 |                 NAMESPACE="--namespace $2"
 55 |                 shift 2
 56 |             ;;
 57 |             -o | --output)
 58 |                 OUT=$2
 59 |                 shift 2
 60 |             ;;
 61 |             -s | --skip-usage)
 62 |                 SKIP_USAGE=true
 63 |                 shift 1
 64 |             ;;
 65 |             -q | --quite)
 66 |                 QUITE=true
 67 |                 shift 1
 68 |             ;;
 69 |             --no-headers)
 70 |                 HEADERS=false
 71 |                 shift 1
 72 |             ;;
 73 |             --console-only)
 74 |                 CONSOLE_ONLY=true
 75 |                 OUT=/dev/null
 76 |                 shift 1
 77 |             ;;
 78 |             -h | --help)
 79 |                 usage
 80 |                 exit 0
 81 |             ;;
 82 |             *)
 83 |                 usage
 84 |             ;;
 85 |         esac
 86 |     done
 87 | }
 88 | 
 89 | # Test connection to a cluster by kubectl
 90 | testConnection () {
 91 |     kubectl cluster-info > /dev/null || errorExit "Connection to cluster failed"
 92 | }
 93 | 
 94 | formatCpu () {
 95 |     local result=$1
 96 |     if [[ ${result} =~ m$ ]]; then
 97 |         result=$(echo "${result}" | tr -d 'm')
 98 |         result=$(awk "BEGIN {print ${result}/1000}")
 99 |     fi
100 | 
101 |     echo -n "${result}"
102 | }
103 | 
104 | formatMemory () {
105 |     local result=$1
106 |     if [[ ${result} =~ M ]]; then
107 |         result=$(echo "${result}" | tr -d 'Mi')
108 |         result=$(awk "BEGIN {print ${result}/1000}")
109 |     elif [[ ${result} =~ m ]]; then
110 |         result=$(echo "${result}" | tr -d 'm')
111 |         result=$(awk "BEGIN {print ${result}/1000000000000}")
112 |     elif [[ ${result} =~ G ]]; then
113 |         result=$(echo "${result}" | tr -d 'Gi')
114 |     fi
115 | 
116 |     echo -n "${result}"
117 | }
118 | 
119 | getRequestsAndLimits () {
120 |     local data=
121 |     local namespace=
122 |     local pod=
123 |     local container=
124 |     local cpu_request=
125 |     local mem_request=
126 |     local cpu_limit=
127 |     local mem_limit=
128 |     local cpu_usage=
129 |     local memory_usage=
130 |     local line=
131 |     local final_line=
132 | 
133 |     data=$(kubectl get pods ${NAMESPACE} -o json | jq -r '.items[] | .metadata.namespace + "," + .metadata.name + "," + (.spec.containers[] | .name + "," + .resources.requests.cpu + "," + .resources.requests.memory + "," + .resources.limits.cpu + "," + .resources.limits.memory)')
134 | 
135 |     # Backup OUT file if already exists
136 |     [ -f "${OUT}" ] && [ "$CONSOLE_ONLY" == "false" ] && cp -f "${OUT}" "${OUT}.$(date +"%Y-%m-%d_%H:%M:%S")"
137 | 
138 |     # Prepare header for output CSV
139 |     if [ "${HEADERS}" == true ]; then
140 |         echo "Namespace,Pod,Container,CPU request,CPU Usage,Memory request,Memory Usage,CPU limit,Memory limit" | tee "${OUT}"
141 |     else
142 |         echo -n "" > "${OUT}"
143 |     fi
144 | 
145 |     local OLD_IFS=${IFS}
146 |     IFS=$'\n'
147 |     for l in ${data}; do
148 |         namespace=$(echo "${l}" | awk -F, '{print $1}')
149 |         pod=$(echo "${l}" | awk -F, '{print $2}')
150 |         container=$(echo "${l}" | awk -F, '{print $3}')
151 |         cpu_request=$(formatCpu "$(echo "${l}" | awk -F, '{print $4}')")
152 |         mem_request=$(formatMemory "$(echo "${l}" | awk -F, '{print $5}')")
153 |         cpu_limit=$(formatCpu "$(echo "${l}" | awk -F, '{print $6}')")
154 |         mem_limit=$(formatMemory "$(echo "${l}" | awk -F, '{print $7}')")
155 | 
156 |         # Adding pod and container actual usage with pod top data
157 |         if [[ "${SKIP_USAGE}" =~ false ]]; then
158 |             line=$(kubectl top pod -n ${namespace} ${pod} --containers --use-protocol-buffers | grep " ${container} ")
159 | 
160 |             cpu_usage=$(formatCpu "$(echo "${line}" | awk '{print $3}')")
161 |             memory_usage=$(formatMemory "$(echo "${line}" | awk '{print $4}')")
162 |         else
163 |             cpu_usage="-"
164 |             memory_usage="-"
165 |         fi
166 | 
167 |         final_line=${namespace},${pod},${container},${cpu_request},${cpu_usage},${mem_request},${memory_usage},${cpu_limit},${mem_limit}
168 |         if [ "${QUITE}" == true ]; then
169 |             echo "${final_line}" >> "${OUT}"
170 |         else
171 |             echo "${final_line}" | tee -a "${OUT}"
172 |         fi
173 |     done
174 |     IFS=${OLD_IFS}
175 | }
176 | 
177 | main () {
178 |     processOptions "$@"
179 |     [ "${QUITE}" == true ] || echo "Getting pods resource requests and limits"
180 |     testConnection
181 |     getRequestsAndLimits
182 | }
183 | 
184 | ######### Main #########
185 | 
186 | main "$@"
187 | 


--------------------------------------------------------------------------------
/getRestartingPods.sh:
--------------------------------------------------------------------------------
  1 | #!/bin/bash
  2 | 
  3 | # UNCOMMENT this line to enable debugging
  4 | # set -xv
  5 | 
  6 | ## Get all pods with restarting containers sorted as csv output
  7 | 
  8 | NAMESPACE=--all-namespaces
  9 | SCRIPT_NAME=$0
 10 | 
 11 | ######### Functions #########
 12 | 
 13 | errorExit () {
 14 |     echo -e "\nERROR: $1\n"
 15 |     exit 1
 16 | }
 17 | 
 18 | usage () {
 19 |     cat << END_USAGE
 20 | 
 21 | ${SCRIPT_NAME} - Extract resource requests and limits in a Kubernetes cluster for a selected namespace or all namespaces in a CSV format
 22 | 
 23 | Usage: ${SCRIPT_NAME} <options>
 24 | 
 25 | -n | --namespace <name>                : Namespace to analyse.    Default: --all-namespaces
 26 | -h | --help                            : Show this usage
 27 | 
 28 | Examples:
 29 | ========
 30 | Get all:                                                  $ ${SCRIPT_NAME}
 31 | Get for namespace foo:                                    $ ${SCRIPT_NAME} --namespace foo
 32 | 
 33 | END_USAGE
 34 | 
 35 |     exit 1
 36 | }
 37 | 
 38 | # Process command line options. See usage above for supported options
 39 | processOptions () {
 40 |     while [[ $# -gt 0 ]]; do
 41 |         case "$1" in
 42 |             -n | --namespace)
 43 |                 NAMESPACE="--namespace $2"
 44 |                 shift 2
 45 |             ;;
 46 |             -h | --help)
 47 |                 usage
 48 |             ;;
 49 |             *)
 50 |                 usage
 51 |             ;;
 52 |         esac
 53 |     done
 54 | }
 55 | 
 56 | # Test connection to a cluster by kubectl
 57 | testConnection () {
 58 |     kubectl cluster-info > /dev/null || errorExit "Connection to cluster failed"
 59 | }
 60 | 
 61 | addValues () {
 62 |     local values_str=$1
 63 | 
 64 |     [ -z "${values_str}" ] && echo -n 0
 65 | 
 66 |     local values_sum=0
 67 |     OLD_IFS=${IFS}
 68 |     IFS=' '
 69 |     for v in ${values_str}; do
 70 |         values_sum=$(( values_sum + v ))
 71 |     done
 72 |     IFS=${OLD_IFS}
 73 | 
 74 |     echo -n ${values_sum}
 75 | }
 76 | 
 77 | getRestartingPods () {
 78 |     local data=
 79 |     local namespace_and_pod=
 80 |     local restarts=
 81 |     local restart_sum=
 82 |     local out_temp=
 83 | 
 84 |     out_temp=$(mktemp)
 85 | 
 86 |     data=$(kubectl get pod ${NAMESPACE} -o=jsonpath='{range .items[*]}{.metadata.namespace},{.metadata.name} {.status.containerStatuses[*].restartCount}{"\n"}{end}')
 87 | 
 88 |     OLD_IFS=${IFS}
 89 |     IFS=$'\n'
 90 |     for l in ${data}; do
 91 |         # Extract the fields so the restarts can be added up later
 92 |         IFS=${OLD_IFS}; read -r namespace_and_pod restarts <<< "${l}"; OLD_IFS=${IFS}; IFS=$'\n'
 93 | 
 94 |         # Go over container restart and add them up
 95 |         restart_sum=$(addValues "${restarts}")
 96 | 
 97 |         if [ "${restart_sum}" -gt 0 ]; then
 98 |             local final_line=${namespace_and_pod},${restart_sum}
 99 |             echo "${final_line}" >> "${out_temp}"
100 |         fi
101 |     done
102 |     IFS=${OLD_IFS}
103 | 
104 |     # Print sorted output
105 |     echo "Namespace,Pod,Total restarts"
106 |     sort -t , -n -k 3 -r "${out_temp}"
107 | 
108 |     # Cleanup
109 |     rm -f "${out_temp}"
110 | }
111 | 
112 | main () {
113 |     processOptions "$@"
114 |     testConnection
115 |     getRestartingPods
116 | }
117 | 
118 | ######### Main #########
119 | 
120 | main "$@"
121 | 


--------------------------------------------------------------------------------
/podReady.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | # set -x
 3 | 
 4 | ## Simple script to check if pod is really ready.
 5 | # Check containers are ready
 6 | # Return 1 if not ready. Return 0 if ready.
 7 | 
 8 | NAMESPACE=default
 9 | POD=
10 | 
11 | errorExit () {
12 |     echo -e "\nERROR: $1\n"
13 |     exit 1
14 | }
15 | 
16 | usage () {
17 |     cat << END_USAGE
18 | 
19 | ${SCRIPT_NAME} - Get pod readiness (all containers ready).
20 |                  Exit with 0 of all containers are ready. Exit with 1 if not.
21 | 
22 | Usage: ${SCRIPT_NAME} <options>
23 | 
24 | -n | --namespace <name>      : Namespace. Default: default
25 | -p | --pod       <name>      : Pod to check
26 | -h | --help                  : Show this usage
27 | 
28 | Examples:
29 | ========
30 | $ ${SCRIPT_NAME} --namespace test --pod nginx-1234asdf
31 | 
32 | END_USAGE
33 | 
34 |     exit 1
35 | }
36 | 
37 | # Process command line options. See usage above for supported options
38 | processOptions () {
39 |     while [[ $# -gt 0 ]]; do
40 |         case "$1" in
41 |             -n | --namespace)
42 |                 NAMESPACE="$2"
43 |                 shift 2
44 |             ;;
45 |             -p | --pod)
46 |                 POD="$2"
47 |                 shift 2
48 |             ;;
49 |             -h | --help)
50 |                 usage
51 |                 exit 0
52 |             ;;
53 |             *)
54 |                 usage
55 |             ;;
56 |         esac
57 |     done
58 | 
59 |     if [[ -z "${POD}" ]]; then errorExit "Pod not passed"; fi
60 | }
61 | 
62 | checkPod () {
63 |     local ready_status
64 |     ready_status=$(kubectl get pod -n ${NAMESPACE} ${POD} --output=jsonpath='{.status.containerStatuses[*].ready}')
65 | 
66 |     # Leave only unique words
67 |     ready_status=$(echo "$ready_status" | tr ' ' '\n' | sort -u)
68 | 
69 |     [[ $ready_status =~ ^true$ ]] && return 0
70 | 
71 |     return 1
72 | }
73 | 
74 | main () {
75 |     processOptions "$@"
76 | 
77 |     checkPod && exit 0
78 |     exit 1
79 | }
80 | 
81 | ######### Main #########
82 | 
83 | main "$@"
84 | 


--------------------------------------------------------------------------------
/runCommandOnPods.sh:
--------------------------------------------------------------------------------
  1 | #!/bin/bash
  2 | 
  3 | # UNCOMMENT this line to enable debugging
  4 | # set -xv
  5 | 
  6 | ## Execute a command on a list of pods
  7 | 
  8 | EXEC="df -h"
  9 | PODS=
 10 | CONTAINER=
 11 | SCRIPT_NAME=$0
 12 | 
 13 | ######### Functions #########
 14 | 
 15 | errorExit () {
 16 |     echo -e "\nERROR: $1\n"
 17 |     exit 1
 18 | }
 19 | 
 20 | usage () {
 21 |     cat << END_USAGE
 22 | 
 23 | ${SCRIPT_NAME} - Run a given command on all pods matching criteria
 24 | 
 25 | Usage: ${SCRIPT_NAME} <options>
 26 | 
 27 | -n | --namespace <name>               : Namespace to analyse
 28 | -p | --pods <string>                  : Pods to run on (pattern)
 29 | -c | --container <string>             : Container to run on (name)
 30 | -x | --exec <string>                  : Command to execute
 31 | 
 32 | -h | --help                           : Show this usage
 33 | 
 34 | Examples:
 35 | ========
 36 | ${SCRIPT_NAME} --namespace demo -pods artifactory-primary -exec "uptime" -container artifactory-app
 37 | 
 38 | END_USAGE
 39 | 
 40 |     exit 1
 41 | }
 42 | 
 43 | # Process command line options. See usage above for supported options
 44 | processOptions () {
 45 |     while [[ $# -gt 0 ]]; do
 46 |         case "$1" in
 47 |             -n | --namespace)
 48 |                 NAMESPACE="-n $2"
 49 |                 shift 2
 50 |             ;;
 51 |             -x | --exec)
 52 |                 EXEC="$2"
 53 |                 shift 2
 54 |             ;;
 55 |             -p | --pods)
 56 |                 PODS="$2"
 57 |                 shift 2
 58 |             ;;
 59 |             -c | --container)
 60 |                 CONTAINER="$2"
 61 |                 shift 2
 62 |             ;;
 63 |             -h | --help)
 64 |                 usage
 65 |                 exit 0
 66 |             ;;
 67 |             *)
 68 |                 usage
 69 |             ;;
 70 |         esac
 71 |     done
 72 | 
 73 |     [[ -z "${NAMESPACE}" ]] && errorExit "Must provide a namespace"
 74 |     [[ -z "${CONTAINER}" ]] && echo "WARNING: No container set. Kubectl will default to the first container in the pod"
 75 | }
 76 | 
 77 | # Test connection to a cluster by kubectl
 78 | testConnection () {
 79 |     kubectl cluster-info > /dev/null || errorExit "Connection to cluster failed"
 80 | }
 81 | 
 82 | runCommandOnPods () {
 83 |     local list=
 84 | 
 85 |     if [[ -n "${PODS}" ]]; then
 86 |         list=$(kubectl get pods ${NAMESPACE} -o custom-columns=:metadata.name | grep ${PODS})
 87 |     else
 88 |         list=$(kubectl get pods ${NAMESPACE} -o custom-columns=:metadata.name)
 89 |     fi
 90 |     echo "${list}"
 91 | 
 92 |     for l in ${list}; do
 93 |         echo -en "\n---- Pod: $l"
 94 | 
 95 | #        set -x
 96 |         if [[ -n "${CONTAINER}" ]]; then
 97 |             echo " (Container: ${CONTAINER})"
 98 |             kubectl exec ${NAMESPACE} ${l} -c ${CONTAINER} -- ${EXEC}
 99 |         else
100 |             echo
101 |             kubectl exec ${NAMESPACE} ${l} -- ${EXEC}
102 |         fi
103 | #        set +x
104 |     done
105 | }
106 | 
107 | main () {
108 |     processOptions "$@"
109 |     [ "${QUITE}" == true ] || echo "Running command '${EXEC}' on pods"
110 |     testConnection
111 |     runCommandOnPods
112 | }
113 | 
114 | ######### Main #########
115 | 
116 | main "$@"
117 | 


--------------------------------------------------------------------------------
/yaml/kind-config.yaml:
--------------------------------------------------------------------------------
 1 | # Setup for a simple Kind multi node Kubernetes cluster
 2 | # One control plane node
 3 | # Three worker nodes
 4 | kind: Cluster
 5 | apiVersion: kind.x-k8s.io/v1alpha4
 6 | nodes:
 7 |   - role: control-plane
 8 |   - role: worker
 9 |   - role: worker
10 |   - role: worker
11 | 


--------------------------------------------------------------------------------
/yaml/memory.yaml:
--------------------------------------------------------------------------------
 1 | ###########################################################
 2 | ## This file has the configuration for
 3 | ## - A namespace called 0-eldada for isolation
 4 | ## - A pod with an ubuntu container installed with some basic tools like vim, curl.
 5 | ##   This container runs a script that holds a block of memory for a given time with
 6 | ##   an optional gap between memory block runs.
 7 | ##
 8 | ## Deploy: kubectl apply -f memory.yaml
 9 | ## Update (force replacing with new config): kubectl apply -f memory.yaml --force
10 | ## Terminal access: kubectl exec -it pod-memory -- bash
11 | ## Remove: kubectl delete -f memory.yaml
12 | ###########################################################
13 | apiVersion: v1
14 | kind: Namespace
15 | metadata:
16 |   name: 0-eldada
17 | ---
18 | apiVersion: v1
19 | kind: Pod
20 | metadata:
21 |   name: pod-memory
22 |   namespace: 0-eldada
23 | spec:
24 |   securityContext:
25 |     runAsUser: 1000
26 |     fsGroup: 1000
27 |   containers:
28 |     - name: container-memory
29 |       image: eldada.jfrog.io/docker/ubuntu-with-tools:22.10
30 |       command:
31 |         - 'bash'
32 |         - '-c'
33 |         - >
34 |           echo "#######################################";
35 |           echo "### Downloading memory.sh";
36 |           curl -f -s -o ./memory.sh https://raw.githubusercontent.com/eldada/command-examples/master/scripts/memory.sh;
37 |           echo "### Running memory.sh";
38 |           bash ./memory.sh --mb ${MEMORY_TO_USE_MB} --wait ${TIME_TO_HOLD} --restart ${RESTART} --gap ${GAP} || exit 1;
39 |           echo "Sleeping forever...";
40 |           while true; do date; sleep 10; done
41 |       env:
42 |         ## Actual memory used in the container will be MEMORY_TO_USE_MB + other system processes
43 |         - name: MEMORY_TO_USE_MB
44 |           value: "500"
45 |         - name: TIME_TO_HOLD
46 |           value: "60"
47 |         - name: RESTART
48 |           value: "true"
49 |         - name: GAP
50 |           value: "45"
51 | ## WARNING: Need to set a limit twice as much as the required memory (MEMORY_TO_USE_MB) to avoid an OOMKill.
52 | ##          This is due to a spike in memory usage during the actual creation of the memory blob.
53 | #      resources:
54 | #        limits:
55 | #          cpu: "2"
56 | #          memory: 2000Mi
57 | #        requests:
58 | #          cpu: 5m
59 | #          memory: 50Mi
60 | 


--------------------------------------------------------------------------------
/yaml/metrics-server-patch.yaml:
--------------------------------------------------------------------------------
 1 | spec:
 2 |   template:
 3 |     spec:
 4 |       containers:
 5 |         - name: metrics-server
 6 |           args:
 7 |             - --cert-dir=/tmp
 8 |             - --secure-port=4443
 9 |             - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
10 |             - --kubelet-use-node-status-port
11 |             - --kubelet-insecure-tls
12 |             - --metric-resolution=15s
13 | 


--------------------------------------------------------------------------------
/yaml/podWithHostFS.yaml:
--------------------------------------------------------------------------------
  1 | ###########################################################
  2 | ## This file has the configuration for
  3 | ## - A namespace called 0-danger-zone (for easy sort and context)
  4 | ## - A hostPath PersistentVolume
  5 | ## - A PersistentVolumeClaim to use the PersistentVolume
  6 | ## - A pod with host root file system mounted to /mnt/hostfs
  7 | ##
  8 | ## This is useful for exploring the host file system for various
  9 | ## debugging scenarios like examining the kubelet configuration.
 10 | ## USE WITH CAUTION! Changes here can break your node!
 11 | ##
 12 | ## Deploy: kubectl apply -f podWithHostFS.yaml
 13 | ## Terminal access: kubectl exec -n 0-danger-zone -it pod-with-host-fs -- bash
 14 | ## Remove: kubectl delete -f podWithHostFS.yaml
 15 | ###########################################################
 16 | apiVersion: v1
 17 | kind: Namespace
 18 | metadata:
 19 |   name: 0-danger-zone
 20 | ---
 21 | apiVersion: v1
 22 | kind: Pod
 23 | metadata:
 24 |   name: pod-with-host-fs
 25 |   namespace: 0-danger-zone
 26 | spec:
 27 |   containers:
 28 |     - name: container-with-host-fs
 29 |       image: eldada.jfrog.io/docker/ubuntu:22.04
 30 |       command:
 31 |         - 'bash'
 32 |         - '-c'
 33 |         - >
 34 |           echo "############# Setting up some tools...";
 35 |           apt update && apt install vim curl -y;
 36 |           echo -e "\n\n############# Get scripts...";
 37 |           cd /opt;
 38 |           curl -# -O https://raw.githubusercontent.com/eldada/command-examples/master/scripts/top.sh;
 39 |           curl -# -O https://raw.githubusercontent.com/eldada/command-examples/master/scripts/procInfo.sh;
 40 |           chmod +x *.sh;
 41 |           ls -la;
 42 |           echo -e "\n\n############# Going to sleep for 30 days...";
 43 |           sleep 30d
 44 |       resources:
 45 |         limits:
 46 |           cpu: 200m
 47 |           memory: 200Mi
 48 |         requests:
 49 |           cpu: 5m
 50 |           memory: 50Mi
 51 |       securityContext:
 52 |         privileged: true
 53 |       volumeMounts:
 54 |         - mountPath: /mnt/hostfs
 55 |           name: hostpath-privileged
 56 |   securityContext: {}
 57 |   volumes:
 58 |     - name: hostpath-privileged
 59 |       persistentVolumeClaim:
 60 |         claimName: host-pvc-volume
 61 | ## Uncomment to apply node affinity when you need to force the pod to start on a given node
 62 | ## Replace NODE_HOST_NAME with the value of the label "kubernetes.io/hostname" on the node
 63 | #  affinity:
 64 | #    nodeAffinity:
 65 | #      requiredDuringSchedulingIgnoredDuringExecution:
 66 | #        nodeSelectorTerms:
 67 | #          - matchExpressions:
 68 | #              - key: kubernetes.io/hostname
 69 | #                operator: In
 70 | #                values:
 71 | #                  - NODE_HOST_NAME
 72 | 
 73 | ---
 74 | apiVersion: v1
 75 | kind: PersistentVolume
 76 | metadata:
 77 |   name: host-pv-volume
 78 |   labels:
 79 |     type: local
 80 | spec:
 81 |   storageClassName: manual
 82 |   capacity:
 83 |     storage: 5Gi
 84 |   accessModes:
 85 |     - ReadWriteOnce
 86 |   hostPath:
 87 |     path: "/"
 88 | ---
 89 | apiVersion: v1
 90 | kind: PersistentVolumeClaim
 91 | metadata:
 92 |   name: host-pvc-volume
 93 |   namespace: 0-danger-zone
 94 | spec:
 95 |   accessModes:
 96 |     - ReadWriteOnce
 97 |   resources:
 98 |     requests:
 99 |       storage: 1Gi
100 |   storageClassName: manual


--------------------------------------------------------------------------------
/yaml/podWithTools.yaml:
--------------------------------------------------------------------------------
 1 | ###########################################################
 2 | ## This file has the configuration for
 3 | ## - A pod with an ubuntu container installed with some basic tools like vim, curl
 4 | ##
 5 | ## Deploy: kubectl apply -f podWithTools.yaml
 6 | ## Terminal access: kubectl exec -it pod-with-tools -- bash
 7 | ## Remove: kubectl delete -f podWithTools.yaml
 8 | ###########################################################
 9 | apiVersion: v1
10 | kind: Pod
11 | metadata:
12 |   name: pod-with-tools
13 | spec:
14 |   terminationGracePeriodSeconds: 5
15 |   containers:
16 |     - name: ubuntu
17 |       image: ubuntu:24.04
18 |       command:
19 |         - 'bash'
20 |         - '-c'
21 |         - >
22 |           echo "############# Setting up the tools...";
23 |           apt update && apt install vim curl git jq apache2-utils dnsutils traceroute yq -y;
24 |           curl -fL -s https://install-cli.jfrog.io | sh;
25 |           echo "############# Done. Sleeping for 30 days";
26 |           sleep 30d
27 | #      resources:
28 | #        limits:
29 | #          cpu: "2"
30 | #          memory: 100Mi
31 | #        requests:
32 | #          cpu: 5m
33 | #          memory: 50Mi


--------------------------------------------------------------------------------
/yaml/podmanPod.yaml:
--------------------------------------------------------------------------------
 1 | ###########################################################
 2 | ## This file has the configuration for
 3 | ## - A pod with podman installed
 4 | ##
 5 | ## Deploy: kubectl apply -f podmanPod.yaml
 6 | ## Terminal access: kubectl exec -it podman -- bash
 7 | ## Remove: kubectl delete -f podmanPod.yaml
 8 | ## HINT: run alias docker=podman in the container
 9 | ###########################################################
10 | apiVersion: v1
11 | kind: Pod
12 | metadata:
13 |   name: podman
14 | spec:
15 |   containers:
16 |     - name: podman
17 |       image: quay.io/podman/stable
18 |       args:
19 |         - sleep
20 |         - "1d"
21 |       securityContext:
22 |         privileged: true
23 | 


--------------------------------------------------------------------------------
/yaml/superPod.yaml:
--------------------------------------------------------------------------------
 1 | ###########################################################
 2 | ## This file has the configuration for
 3 | ## - A pod with elevated privileges that has access to the host filesystem and processes (using "spec.hostPID=true").
 4 | ##
 5 | ## Deploy: kubectl apply -f superPod.yaml
 6 | ## Terminal access: kubectl exec -it super-pod -- bash
 7 | ## Remove: kubectl delete -f superPod.yaml
 8 | ##
 9 | ## WARNING: This pod has elevated privileges and can access the host filesystem and processes!
10 | ##          This pod is not secure and should not be used in production environments.
11 | ##
12 | ## You can restart the kubelet and access the host /var/lib/kubelet and /var/log filesystems.
13 | ## Restarting kubelet: nsenter -t 1 -m -p -- systemctl restart kubelet
14 | ###########################################################
15 | apiVersion: v1
16 | kind: Pod
17 | metadata:
18 |   name: super-pod
19 | spec:
20 |   automountServiceAccountToken: true
21 |   containers:
22 |     - command:
23 |         - /bin/bash
24 |         - -c
25 |         - sleep 30d
26 |       image: eldada.jfrog.io/docker/alpine-with-tools:3.21.0
27 |       imagePullPolicy: IfNotPresent
28 |       name: super-container
29 |       resources:
30 |         limits:
31 |           cpu: 50m
32 |           memory: 100Mi
33 |         requests:
34 |           cpu: 5m
35 |           memory: 10Mi
36 |       securityContext:
37 |         allowPrivilegeEscalation: true
38 |         privileged: true
39 |         readOnlyRootFilesystem: false
40 |         runAsNonRoot: false
41 |       volumeMounts:
42 |         - mountPath: /var/lib/kubelet
43 |           mountPropagation: None
44 |           name: kubelet
45 |         - mountPath: /var/log
46 |           mountPropagation: None
47 |           name: varlog
48 |   hostPID: true
49 |   volumes:
50 |     - name: kubelet
51 |       hostPath:
52 |         path: /var/lib/kubelet
53 |         type: Directory
54 |     - name: varlog
55 |       hostPath:
56 |         path: /var/log
57 |         type: Directory
58 | ## Optional nodeSelector to run the pod on a specific node
59 | #  nodeSelector:
60 | #    kubernetes.io/hostname: ip-10-100-100-10.us-east-1.compute.internal
61 | 


--------------------------------------------------------------------------------