├── LICENSE ├── check_kubernetes_api.sh ├── check_kube_nodes.sh ├── README.md ├── check_kube_deployments.sh └── check_kube_pods.sh /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2017-2019 Justin Miller https://github.com/colebrooke 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | SOFTWARE. 20 | -------------------------------------------------------------------------------- /check_kubernetes_api.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ######################################################### 4 | # ./check_kubernetes.sh # 5 | # # 6 | # Nagios check script for kubernetes cluster # 7 | # This is a super simple check, with plenty # 8 | # of room for improvements :) # 9 | # Author: Justin Miller # 10 | # Website: https://github.com/colebrooke # 11 | # # 12 | ######################################################### 13 | 14 | function usage { 15 | cat < -c ] [-k ] 17 | 18 | Options: 19 | -t # Required, the endpoint for your Kubernetes API 20 | -c # Required, credentials for your Kubernetes API, in the format outlined below 21 | 22 | Credentials file format: 23 | machine yourEndPointOrTarget login yourUserNameHere password YOURPASSWORDHERE 24 | 25 | EOF 26 | 27 | exit 2 28 | } 29 | 30 | # Comment out if you have SSL enabled on your K8 API 31 | SSL="--insecure" 32 | 33 | while getopts ":t:c:hk:x:" OPTIONS; do 34 | case "${OPTIONS}" in 35 | t) TARGET=${OPTARG} ;; 36 | c) CREDENTIALS_FILE="--netrc-file ${OPTARG}" ;; 37 | h) usage ;; 38 | k) KUBE_CONFIG="--kubeconfig ${OPTARG}" ;; 39 | x) KUBE_CONTEXT="--context ${OPTARG}" ;; 40 | *) usage ;; 41 | esac 42 | done 43 | 44 | if [ -z $TARGET ]; then 45 | type kubectl >/dev/null 2>&1 || { echo >&2 "CRITICAL: The kubectl utility is required for this script to run if no API endpoint target is specified"; exit 3; } 46 | kubectl $KUBE_CONFIG $KUBE_CONTEXT proxy >/dev/null 2>&1 & 47 | PROXY_PID=$! 48 | sleep 1 49 | TARGET="http://127.0.0.1:8001" 50 | fi 51 | 52 | HEALTH=$(curl -sS $SSL $CREDENTIALS_FILE $TARGET/healthz) 53 | BSC_HEALTH=$(curl -sS $SSL $CREDENTIALS_FILE $TARGET/healthz/poststarthook/bootstrap-controller) 54 | EXT_HEALTH=$(curl -sS $SSL $CREDENTIALS_FILE $TARGET/healthz/poststarthook/extensions/third-party-resources) 55 | BSR_HEALTH=$(curl -sS $SSL $CREDENTIALS_FILE $TARGET/healthz/poststarthook/rbac/bootstrap-roles) 56 | 57 | if [ -n "$PROXY_PID" ] 58 | then 59 | kill -15 $PROXY_PID 60 | fi 61 | 62 | case "$HEALTH $BSC_HEALTH $BSR_HEALTH" in 63 | "ok ok ok") echo "OK - Kubernetes API status is OK" && exit 0;; 64 | *) 65 | echo "WARNING - Kubernetes API status is not OK!" 66 | echo "/healthz - $HEALTH" 67 | echo "/healthz/poststarthook/bootstrap-controller - $BSC_HEALTH" 68 | echo "/healthz/poststarthook/extensions/third-party-resources - $EXT_HEALTH" 69 | echo "/healthz/poststarthook/rbac/bootstrap-roles - $BSR_HEALTH" 70 | exit 1 71 | ;; 72 | esac 73 | 74 | 75 | -------------------------------------------------------------------------------- /check_kube_nodes.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ######################################################### 4 | # ./check_kube_nodes.sh # 5 | # # 6 | # Nagios check script for kubernetes cluster # 7 | # nodes. Uses kubectl or API to check status # 8 | # on each k8 node. # 9 | # # 10 | # Author: Justin Miller # 11 | # # 12 | ######################################################### 13 | 14 | type jq >/dev/null 2>&1 || { echo >&2 "CRITICAL: The jq utility is required for this script to run."; exit 2; } 15 | type kubectl >/dev/null 2>&1 || { echo >&2 "CRITICAL: The kubectl utility is required for this script to run."; exit 2; } 16 | 17 | 18 | function usage { 19 | cat < -c ] [-k ] 21 | 22 | Options: 23 | -t # Optional, the endpoint for your Kubernetes API (otherwise will use kubectl) 24 | -c # Required if a API is specified, in the format outlined below 25 | -k # Path to kube config file if using kubectl 26 | 27 | Credentials file format: 28 | machine yourEndPointOrTarget login yourUserNameHere password YOURPASSWORDHERE 29 | 30 | EOF 31 | exit 2 32 | } 33 | 34 | while getopts ":t:c:k:x:h" OPTIONS; do 35 | case "${OPTIONS}" in 36 | t) TARGET=${OPTARG} ;; 37 | c) CREDENTIALS_FILE="--netrc-file ${OPTARG}" ;; 38 | k) KUBE_CONFIG="--kubeconfig ${OPTARG}" ;; 39 | x) KUBE_CONTEXT="--context ${OPTARG}" ;; 40 | h) usage ;; 41 | *) usage ;; 42 | esac 43 | done 44 | 45 | # Comment out if you have SSL enabled on your K8 API 46 | SSL="--insecure" 47 | EXITCODE=0 48 | 49 | if [ -z $TARGET ]; then 50 | # kubectl mode 51 | K8STATUS="$(kubectl $KUBE_CONFIG $KUBE_CONTEXT get nodes -o json)" 52 | if [ $(echo "$K8STATUS" | wc -l) -le 30 ]; then echo "CRITICAL - unable to connect to Kubernetes via kubectl!"; exit 3; fi 53 | else 54 | # k8 API mode 55 | # Make call to Kubernetes API to get the status: 56 | K8STATUS="$(curl -sS $SSL $CREDENTIALS_FILE $TARGET/api/v1/nodes)" 57 | if [ $(echo "$K8STATUS" | wc -l) -le 30 ]; then echo "CRITICAL - unable to connect to Kubernetes API!"; exit 3; fi 58 | fi 59 | 60 | # Derive nodes from the json returned by the API 61 | NODES=$(echo "$K8STATUS" | jq -r '.items[].metadata.name') 62 | 63 | function returnResult () { 64 | CHECKSTATUS="$1" 65 | if [[ "$CHECKSTATUS" == "Critical" ]] && [ $EXITCODE -le 2 ]; then 66 | RESULT=$(echo -e "$CHECKSTATUS: $NODE has condition $CHECK - $STATUS\n$RESULT") 67 | EXITCODE=2 68 | elif [[ "$CHECKSTATUS" == "Warning" ]]; then 69 | RESULT=$(echo -e "$CHECKSTATUS: $NODE has condition $CHECK - $STATUS\n$RESULT") 70 | EXITCODE=1 71 | fi 72 | } 73 | 74 | # Itterate through each node 75 | for NODE in ${NODES[*]}; do 76 | CHECKS=$(echo "$K8STATUS" | jq -r '.items[] | select(.metadata.name=="'$NODE'") | .status.conditions[].type') 77 | # Itterate through each condition for each node 78 | for CHECK in ${CHECKS[*]}; do 79 | STATUS=$(echo "$K8STATUS" | jq '.items[] | select(.metadata.name=="'$NODE'") | .status.conditions[]' | jq -r 'select(.type=="'$CHECK'") .status') 80 | case "$CHECK-$STATUS" in 81 | "OutOfDisk-True") returnResult Warning;; 82 | "MemoryPressure-True") returnResult Critical;; 83 | "DiskPressure-True") returnResult Critical;; 84 | "Ready-False") returnResult Warning;; 85 | "Ready-Unknown") returnResult Warning;; 86 | # Note the API only checks these 4 conditions at present. Others can be added here. 87 | *) returnResult OK;; 88 | esac 89 | done 90 | done 91 | 92 | case $EXITCODE in 93 | 0) printf "OK - Kubernetes nodes all OK\n" ;; 94 | 1) printf "" ;; 95 | 2) printf "" ;; 96 | esac 97 | 98 | echo "$RESULT" 99 | exit $EXITCODE 100 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # kubernetes-nagios 2 | 3 | Some checks for Kubernetes clusters, which can be use with Nagios, Zabbix, Icinga, or any other 4 | monitoring system that can be configured to use an external shell script. 5 | It's been tested on macOS, Ubuntu and Debian with bash. There are relatively few dependencies, but 6 | the "jq" utility for processing json is required. If your Kubernetes API is not exposed, the checks 7 | can use kubectl, in which case this will need to be installed and configured. 8 | 9 | ## check_kube_pods.sh 10 | 11 | ### Usage 12 | 13 | ```bash 14 | ./check_kube_pods.sh [-t -c ] [-k ] [-n ] [-w ] [-C ] 15 | ``` 16 | 17 | ### Options 18 | 19 | ``` 20 | -t -- Optional, the endpoint for your Kubernetes API (otherwise will use kubectl) 21 | -c -- Required if a API is specified, in the format outlined below 22 | -n -- Namespace to check, for example, "kube-system". By default all are checked. 23 | -w -- Warning threshold for number of container restarts [default: 5] 24 | -C -- Critical threshold for number of container restarts [default: 50] 25 | -k -- Path to kube config file if using kubectl 26 | -h -- Show usage / help 27 | -v -- Show verbose output 28 | ``` 29 | 30 | ### Example Output 31 | 32 | ```bash 33 | $ ./check_kube_pods.sh -n kube-system 34 | OK - Kubernetes pods are all OK 35 | OK: Pod: nginx-ingress-controller-v1-zg7gw Container: nginx-ingress-lb Ready: true Restarts: 1 36 | OK: Pod: nginx-ingress-controller-v1-txc1w Container: nginx-ingress-lb Ready: true Restarts: 1 37 | OK: Pod: nginx-ingress-controller-v1-dffl3 Container: nginx-ingress-lb Ready: true Restarts: 1 38 | ``` 39 | 40 | ```bash 41 | $ ./check_kube_pods.sh -n kube-system -w 0 -c 30 42 | WARNING - One or more pods show warning status! 43 | Warning: Pod: nginx-ingress-controller-v1-zg7gw Container: nginx-ingress-lb Ready: true Restarts: 1 44 | Warning: Pod: nginx-ingress-controller-v1-txc1w Container: nginx-ingress-lb Ready: true Restarts: 1 45 | Warning: Pod: nginx-ingress-controller-v1-dffl3 Container: nginx-ingress-lb Ready: true Restarts: 1 46 | ``` 47 | 48 | ## check_kube_deployments.sh 49 | 50 | ### Usage 51 | 52 | ```bash 53 | ./check_kube_deployments.sh [-t -c ] [-k ] 54 | ``` 55 | 56 | ```bash 57 | $ ./check_kube_deployments.sh -t https://api.mykube-cluster.co.uk -c ~/my-credentials 58 | OK - Kubernetes deployments are all OK 59 | OK: kubernetes-dashboard-v1.4.0 has condition Available: True - Deployment has minimum availability. 60 | OK: kubernetes-dashboard has condition Available: True - Deployment has minimum availability. 61 | OK: kube-dns-autoscaler has condition Available: True - Deployment has minimum availability. 62 | OK: kube-dns has condition Available: True - Deployment has minimum availability. 63 | OK: heapster has condition Available: True - Deployment has minimum availability. 64 | OK: dns-controller has condition Available: True - Deployment has minimum availability. 65 | ``` 66 | 67 | ## check_kube_nodes.sh 68 | 69 | This uses the Kubernetes API to check condition statuses across your nodes. 70 | 71 | ### Usage 72 | 73 | ```bash 74 | ./check_kube_nodes.sh [-t -c ] [-k ] 75 | ``` 76 | 77 | ```bash 78 | $ ./kubernetes-nagios ❯❯❯ ./check_kube_nodes.sh -t https://api.mykube-cluster.co.uk -c ~/my-credentials 79 | WARNING - One or more nodes show warning status! 80 | Warning: ip-10-123-81-96.eu-west-1.compute.internal has condition OutOfDisk - True 81 | Warning: ip-10-123-82-87.eu-west-1.compute.internal has condition OutOfDisk - True 82 | ``` 83 | 84 | ## check_kubernetes_api.sh 85 | 86 | This check returns the health status of the overall cluster. 87 | 88 | ### Usage 89 | 90 | ```bash 91 | ./check_kubernetes_api.sh [-t -c ] [-k ] 92 | ``` 93 | 94 | ## Dependencies 95 | 96 | These scripts call the Kubernetes API, so this must be exposed to the machine running the script. 97 | If not, the script will try and use the kubectl utility, which must be installed and configured. 98 | 99 | The jq utility for parsing json is required. 100 | 101 | ## Credentials file format (when connecting to API) 102 | 103 | The credentials must be supplied for the Kubernetes cluster API. It's in a file in the following format, 104 | this is required for all these checks in this project to work correctly. 105 | 106 | ```bash 107 | $ cat my-credentials-file 108 | machine yourEndPointOrTarget login yourUserNameHere password YOURPASSWORDHERE 109 | ``` 110 | -------------------------------------------------------------------------------- /check_kube_deployments.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ######################################################### 4 | # ./check_kube_deployments.sh # 5 | # # 6 | # Nagios check script for kubernetes cluster # 7 | # deployments. Uses API to check status for # 8 | # each deployment. # 9 | # Author: Justin Miller # 10 | # # 11 | ######################################################### 12 | 13 | type jq >/dev/null 2>&1 || { echo >&2 "CRITICAL: The jq utility is required for this script to run."; exit 2; } 14 | 15 | function usage { 16 | cat < -c 18 | 19 | Options: 20 | -t # Required, the endpoint for your Kubernetes API 21 | -c # Required, credentials for your Kubernetes API, in the format outlined below 22 | -n # Namespace to check, for example, "kube-system". By default all are checked. 23 | 24 | Credentials file format: 25 | machine yourEndPointOrTarget login yourUserNameHere password YOURPASSWORDHERE 26 | 27 | EOF 28 | exit 2 29 | } 30 | 31 | while getopts ":t:c:hn:k:x:" OPTIONS; do 32 | case "${OPTIONS}" in 33 | t) TARGET=${OPTARG} ;; 34 | c) CREDENTIALS_FILE=${OPTARG} ;; 35 | n) NAMESPACE_TARGET=${OPTARG} ;; 36 | k) KUBE_CONFIG="--kubeconfig ${OPTARG}" ;; 37 | x) KUBE_CONTEXT="--context ${OPTARG}" ;; 38 | h) usage ;; 39 | *) usage ;; 40 | esac 41 | done 42 | 43 | if [ ! -z $TARGET ] && [ -z $CREDENTIALS_FILE ]; then 44 | echo "Required argument -c missing when specifing -t "; 45 | exit 3; 46 | fi 47 | 48 | # Comment out if you have SSL enabled on your K8 API 49 | SSL="--insecure" 50 | EXITCODE=0 51 | 52 | if [[ -z $TARGET ]]; then 53 | # use kubectl when no API endpoint is specified 54 | type kubectl >/dev/null 2>&1 || { echo >&2 "CRITICAL: The kubectl utility is required for this script to run if no API endpoint is specified"; exit 3; } 55 | if [[ -z $NAMESPACE_TARGET ]]; then 56 | ALL_NAMESPACE_OPTION="true" 57 | # should return all namespaces even when we set namespaces to default 58 | NAMESPACES="default" 59 | else 60 | NAMESPACES="$NAMESPACE_TARGET" 61 | fi 62 | else 63 | # API target has been specified 64 | # Make call to Kubernetes API to get the list of namespaces: 65 | if [[ -z $NAMESPACE_TARGET ]] && [[ ! -z $TARGET ]]; then 66 | NAMESPACES="$(curl -sS $SSL --netrc-file $CREDENTIALS_FILE $TARGET/api/v1/namespaces)" 67 | NAMESPACES=$(echo "$NAMESPACES" | jq -r '.items[].metadata.name') 68 | else 69 | NAMESPACES="$NAMESPACE_TARGET" 70 | fi 71 | fi 72 | 73 | function returnResult () { 74 | CHECKSTATUS="$1" 75 | RESULT=$(echo -e "$CHECKSTATUS: $DEPLOYMENT has condition $TYPE: $STATUS - $REASON\n$RESULT") 76 | if [[ "$CHECKSTATUS" == "Critical" ]] && [ $EXITCODE -le 2 ]; then EXITCODE=2; fi 77 | if [[ "$CHECKSTATUS" == "Warning" ]] && [ $EXITCODE -eq 0 ]; then EXITCODE=1; fi 78 | if [[ "$CHECKSTATUS" == "Unknown" ]] && [ $EXITCODE -eq 0 ]; then EXITCODE=3; fi 79 | } 80 | 81 | # Itterate through each namespace 82 | for NAMESPACE in ${NAMESPACES[*]}; do 83 | # get deployments data for the namespace 84 | if [[ -z $TARGET ]]; then 85 | # kubectl mode 86 | if [[ "$ALL_NAMESPACE_OPTION" == "true" ]]; then 87 | DEPLOYMENTS_STATUS=$(kubectl $KUBE_CONFIG $KUBE_CONTEXT get deployments --all-namespaces -o json) 88 | else 89 | DEPLOYMENTS_STATUS=$(kubectl $KUBE_CONFIG $KUBE_CONTEXT get deployments --namespace $NAMESPACE -o json) 90 | fi 91 | else 92 | # api mode 93 | DEPLOYMENTS_STATUS=$(curl -sS $SSL --netrc-file $CREDENTIALS_FILE $TARGET/apis/extensions/v1beta1/namespaces/$NAMESPACE/deployments/) 94 | fi 95 | 96 | DEPLOYMENTS=$(echo "$DEPLOYMENTS_STATUS" | jq -r '.items[].metadata.name') 97 | # Itterate through each deployment 98 | for DEPLOYMENT in ${DEPLOYMENTS[*]}; do 99 | TYPE=$(echo "$DEPLOYMENTS_STATUS" | jq -r '.items[] | select(.metadata.name=="'$DEPLOYMENT'") | .status.conditions |= sort_by(.lastTransitionTime) | .status.conditions[-1].type' ) 100 | STATUS=$(echo "$DEPLOYMENTS_STATUS" | jq -r '.items[] | select(.metadata.name=="'$DEPLOYMENT'") | .status.conditions |= sort_by(.lastTransitionTime) | .status.conditions[-1].status' ) 101 | REASON=$(echo "$DEPLOYMENTS_STATUS" | jq -r '.items[] | select(.metadata.name=="'$DEPLOYMENT'") | .status.conditions |= sort_by(.lastTransitionTime) | .status.conditions[-1].message' ) 102 | # uncomment the following line to test a failure: 103 | # if [[ "$DEPLOYMENT" == "kubernetes-dashboard" ]]; then TYPE="Available"; STATUS="False"; fi 104 | case "${TYPE}-${STATUS}" in 105 | "Available-True") returnResult OK;; 106 | "Available-False") returnResult Warning;; 107 | *) returnResult Unknown ;; 108 | esac 109 | done 110 | done 111 | 112 | case $EXITCODE in 113 | 0) printf "OK - Kubernetes deployments are all OK\n" ;; 114 | 1) printf "WARNING - One or more deployments show warning status!\n" ;; 115 | 2) printf "CRITICAL - One or more nodes show critical status!\n" ;; 116 | esac 117 | 118 | echo "$RESULT" 119 | exit $EXITCODE 120 | -------------------------------------------------------------------------------- /check_kube_pods.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ######################################################### 4 | # ./check_kube_pods.sh # 5 | # # 6 | # Nagios check script for kubernetes cluster # 7 | # pods. Uses kubectl or API to check status # 8 | # for each pod. # 9 | # # 10 | # Author: Justin Miller # 11 | # # 12 | ######################################################### 13 | 14 | type jq >/dev/null 2>&1 || { echo >&2 "CRITICAL: The jq utility is required for this script to run."; exit 2; } 15 | 16 | function usage { 17 | cat < -c ] [-k ] [-n ] [-w ] [-C # Optional, the endpoint for your Kubernetes API (otherwise will use kubectl) 23 | -c # Required if a API is specified, in the format outlined below 24 | -n # Namespace to check, for example, "kube-system". By default all are checked. 25 | -w # Warning threshold for number of container restarts [default: 200] 26 | -C # Critical threshold for number of container restarts [default: 1000] 27 | -k # Path to kube config file if using kubectl 28 | -p # Search for particular pods only 29 | -e # Expected number of pods in a ready condition 30 | -h # Show usage / help 31 | -v # Show verbose output 32 | 33 | Credentials file format: 34 | machine yourEndPointOrTarget login yourUserNameHere password YOURPASSWORDHERE 35 | 36 | EOF 37 | exit 2 38 | } 39 | 40 | # Comment out if you have SSL enabled on your K8 API 41 | SSL="--insecure" 42 | EXITCODE=0 43 | # Default thresholds for container restarts 44 | WARN_THRESHOLD=1000 45 | CRIT_THRESHOLD=3000 46 | EXPECTED_PODS=0 47 | PODS_READY=0 48 | 49 | while getopts ":t:c:hw:C:n:k:x:p:e:v" OPTIONS; do 50 | case "${OPTIONS}" in 51 | t) TARGET=${OPTARG} ;; 52 | c) CREDENTIALS_FILE="--netrc-file ${OPTARG}" ;; 53 | w) WARN_THRESHOLD=${OPTARG} ;; 54 | C) CRIT_THRESHOLD=${OPTARG} ;; 55 | n) NAMESPACE_TARGET=${OPTARG} ;; 56 | v) VERBOSE="true" ;; 57 | k) KUBE_CONFIG="--kubeconfig ${OPTARG}" ;; 58 | x) KUBE_CONTEXT="--context ${OPTARG}" ;; 59 | p) POD_SEARCH="${OPTARG}" ;; 60 | e) EXPECTED_PODS=${OPTARG} ;; 61 | h) usage ;; 62 | *) usage ;; 63 | esac 64 | done 65 | 66 | 67 | WARN_THRESHOLD=$(($WARN_THRESHOLD + 0)) 68 | CRIT_THRESHOLD=$(($CRIT_THRESHOLD + 0)) 69 | 70 | if [[ -z $TARGET ]]; then 71 | # use kubectl when no API endpoint is specified 72 | type kubectl >/dev/null 2>&1 || { echo >&2 "CRITICAL: The kubectl utility is required for this script to run if no API endpoint is specified"; exit 3; } 73 | if [[ -z $NAMESPACE_TARGET ]]; then 74 | ALL_NAMESPACE_OPTION="true" 75 | # should return all namespaces even when we set namespaces to default 76 | NAMESPACES="default" 77 | else 78 | NAMESPACES="$NAMESPACE_TARGET" 79 | fi 80 | else 81 | # API target has been specified 82 | # Make call to Kubernetes API to get the list of namespaces: 83 | if [[ -z $NAMESPACE_TARGET ]] && [[ ! -z $TARGET ]]; then 84 | NAMESPACES="$(curl -sS $SSL $CREDENTIALS_FILE $TARGET/api/v1/namespaces)" 85 | NAMESPACES=$(echo "$NAMESPACES" | jq -r '.items[].metadata.name') 86 | else 87 | NAMESPACES="$NAMESPACE_TARGET" 88 | fi 89 | fi 90 | 91 | 92 | function returnResult () { 93 | RESULT=$(echo -e "$1: $2\n$RESULT") 94 | if [[ "$1" == "Critical" ]] && [ $EXITCODE -le 2 ]; then EXITCODE=2; fi 95 | if [[ "$1" == "Warning" ]] && [ $EXITCODE -eq 0 ]; then EXITCODE=1; fi 96 | if [[ "$1" == "Unknown" ]] && [ $EXITCODE -eq 0 ]; then EXITCODE=3; fi 97 | } 98 | 99 | # Itterate through each namespace 100 | for NAMESPACE in ${NAMESPACES[*]}; do 101 | # get deployments data for the namespace 102 | if [[ -z $TARGET ]]; then 103 | # kubectl mode 104 | if [[ "$ALL_NAMESPACE_OPTION" == "true" ]]; then 105 | PODS_STATUS=$(kubectl $KUBE_CONFIG $KUBE_CONTEXT get pods --all-namespaces -o json) 106 | else 107 | PODS_STATUS=$(kubectl $KUBE_CONFIG $KUBE_CONTEXT get pods --namespace $NAMESPACE -o json) 108 | fi 109 | 110 | else 111 | # api mode 112 | PODS_STATUS=$(curl -sS $SSL $CREDENTIALS_FILE $TARGET/api/v1/namespaces/$NAMESPACE/pods) 113 | fi 114 | if [ $(echo "$PODS_STATUS" | wc -l) -le 10 ]; then echo "CRITICAL - unable to connect to kubernetes cluster!"; exit 3; fi 115 | 116 | if [ -z $POD_SEARCH ]; then 117 | PODS=$(echo "$PODS_STATUS" | jq -r '.items[].metadata.name') 118 | else 119 | PODS=$(echo "$PODS_STATUS" | jq -r '.items[].metadata.name'| grep "$POD_SEARCH") 120 | fi 121 | 122 | # Itterate through each pod 123 | for POD in ${PODS[*]}; do 124 | POD_STATUS=$(echo "$PODS_STATUS" | jq -r '.items[] | select(.metadata.name | contains("'$POD'"))') 125 | POD_CONDITION_TYPES=$(echo "$POD_STATUS" | jq -r '.status.conditions[] | .type') 126 | # Itterate through each condition type 127 | for TYPE in ${POD_CONDITION_TYPES[*]}; do 128 | TYPE_STATUS=$(echo "$POD_STATUS" | jq -r '.status.conditions[] | select(.type=="'$TYPE'") | .status') 129 | #echo "$TYPE_STATUS" 130 | #echo "-------------" 131 | if [[ "${TYPE_STATUS}" != "True" ]]; then 132 | returnResult OK "Pod: $POD $TYPE: $TYPE_STATUS" 133 | else 134 | if [[ "${TYPE}" == "Ready" ]]; then PODS_READY=$((PODS_READY+1)); fi 135 | if [[ "$VERBOSE" == "true" ]]; then returnResult OK "Pod: $POD $TYPE: $TYPE_STATUS"; fi 136 | fi 137 | done 138 | CONTAINERS=$(echo "$POD_STATUS" | jq -r '.status.containerStatuses[].name') 139 | # Itterate through each container 140 | for CONTAINER in ${CONTAINERS[*]}; do 141 | 142 | CONTAINER_READY=$(echo "$POD_STATUS" | jq -r '.status.containerStatuses[] | select(.name=="'$CONTAINER'") | .ready') 143 | CONTAINER_RESTARTS=$(echo "$POD_STATUS" | jq -r '.status.containerStatuses[] | select(.name=="'$CONTAINER'") | .restartCount') 144 | if (( $CONTAINER_RESTARTS > $WARN_THRESHOLD && $CONTAINER_RESTARTS < $CRIT_THRESHOLD )); then 145 | returnResult Warning "Pod: $POD Container: $CONTAINER Ready: $CONTAINER_READY Restarts: $CONTAINER_RESTARTS" 146 | elif (( $CONTAINER_RESTARTS > $CRIT_THRESHOLD )); then 147 | returnResult Critical "Pod: $POD Container: $CONTAINER Ready: $CONTAINER_READY Restarts: $CONTAINER_RESTARTS" 148 | elif (( $CONTAINER_RESTARTS > 0 )); then 149 | returnResult OK "Pod: $POD Container: $CONTAINER Ready: $CONTAINER_READY Restarts: $CONTAINER_RESTARTS" 150 | fi 151 | done 152 | done 153 | done 154 | 155 | if (( $EXPECTED_PODS > $PODS_READY )); then returnResult Critical "$POD_SEARCH only has $PODS_READY pods ready, expecting $EXPECTED_PODS!"; fi 156 | 157 | case $EXITCODE in 158 | 0) printf "OK - $POD_SEARCH pods are all OK, found $PODS_READY in ready state.\n" ;; 159 | 1) printf "Warning - $POD_SEARCH pods show warning status, $PODS_READY in ready state.\n" ;; 160 | 2) printf "Critical - $POD_SEARCH pods show critical status, $PODS_READY in ready state.\n" ;; 161 | esac 162 | 163 | echo "$RESULT" 164 | exit $EXITCODE 165 | --------------------------------------------------------------------------------