├── aks.sh ├── doks.sh ├── eks.sh ├── gke.sh ├── inspector.sh ├── lke.sh └── readme.md /aks.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | node=$(kubectl get node -o name | head -n 1) 3 | file="/host/etc/default/kubelet" 4 | input=$(kubectl debug "$node" -it --image xxradar/hackon -- cat $file) 5 | node_info=$(kubectl get $node -o json); 6 | 7 | echo 'total capacity:' 8 | echo "cpu="$(echo $node_info | jq -r '.status.capacity.cpu // 0'); 9 | echo "memory="$(echo $node_info | jq -r '.status.capacity.memory // 0'); 10 | echo $'\n'; 11 | 12 | echo 'total allocatable:' 13 | echo "cpu="$(echo $node_info | jq -r '.status.allocatable.cpu // 0'); 14 | echo "memory="$(echo $node_info | jq -r '.status.allocatable.memory // 0'); 15 | echo $'\n'; 16 | 17 | echo 'system reserved:' 18 | echo $input | tr ' ' '\n' | grep 'system-reserved' | cut -d '=' -f 2,3,4,5 | tr ',' '\n'; 19 | echo $'\n'; 20 | 21 | echo 'kubelet reserved:'; 22 | echo $input | tr ' ' '\n' | grep 'kube-reserved' | cut -d '=' -f 2,3,4,5 | tr ',' '\n'; 23 | echo $'\n'; 24 | 25 | echo 'eviction threshold:'; 26 | echo "memory="$(echo $input | tr ' ' '\n' | grep 'eviction-hard' | cut -d '=' -f 2 | cut -d ',' -f 1 | cut -d '<' -f 2) 27 | -------------------------------------------------------------------------------- /doks.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | node=$(kubectl get node -o name | head -n 1) 3 | file="/host/etc/kubernetes/kubelet.conf" 4 | input="./.doks.yaml" 5 | kubectl debug "$node" -it --image xxradar/hackon -- cat $file | tail -n +2 > $input; 6 | node_info=$(kubectl get $node -o json); 7 | 8 | echo 'total capacity:' 9 | echo "cpu="$(echo $node_info | jq -r '.status.capacity.cpu // 0'); 10 | echo "memory="$(echo $node_info | jq -r '.status.capacity.memory // 0'); 11 | echo $'\n'; 12 | 13 | echo 'total allocatable:' 14 | echo "cpu="$(echo $node_info | jq -r '.status.allocatable.cpu // 0'); 15 | echo "memory="$(echo $node_info | jq -r '.status.allocatable.memory // 0'); 16 | echo $'\n'; 17 | 18 | echo 'system reserved:' 19 | echo "cpu="$(cat $input | yq -r '.systemReserved.cpu // 0'); 20 | echo "memory="$(cat $input | yq -r '.systemReserved.memory // 0'); 21 | echo $'\n'; 22 | 23 | echo 'kubelet reserved:'; 24 | echo "cpu="$(cat $input | yq -r '.kubeReserved.cpu // 0'); 25 | echo "memory="$(cat $input | yq -r '.kubeReserved.memory // 0'); 26 | echo $'\n'; 27 | 28 | echo 'eviction threshold:'; 29 | echo "memory="$(cat $input | yq -r '.evictionHard."memory.available" // 0'); 30 | 31 | rm $input; -------------------------------------------------------------------------------- /eks.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | node=$(kubectl get node -o name | head -n 1) 3 | file="/host/etc/kubernetes/kubelet/kubelet-config.json" 4 | input="./.eks.json" 5 | kubectl debug "$node" -it --image xxradar/hackon -- cat $file | tail -n +2 > $input; 6 | node_info=$(kubectl get $node -o json); 7 | 8 | echo 'total capacity:' 9 | echo "cpu="$(echo $node_info | jq -r '.status.capacity.cpu // 0'); 10 | echo "memory="$(echo $node_info | jq -r '.status.capacity.memory // 0'); 11 | echo $'\n'; 12 | 13 | echo 'total allocatable:' 14 | echo "cpu="$(echo $node_info | jq -r '.status.allocatable.cpu // 0'); 15 | echo "memory="$(echo $node_info | jq -r '.status.allocatable.memory // 0'); 16 | echo $'\n'; 17 | 18 | echo 'system reserved:' 19 | echo "cpu="$(cat $input | jq -r '.systemReserved.cpu // 0'); 20 | echo "memory="$(cat $input | jq -r '.systemReserved.memory // 0'); 21 | echo $'\n'; 22 | 23 | echo 'kubelet reserved:'; 24 | echo "cpu="$(cat $input | jq -r '.kubeReserved.cpu // 0'); 25 | echo "memory="$(cat $input | jq -r '.kubeReserved.memory // 0'); 26 | echo $'\n'; 27 | 28 | echo 'eviction threshold:'; 29 | echo "memory="$(cat $input | jq -r '.evictionHard."memory.available" // 0'); 30 | 31 | rm $input; -------------------------------------------------------------------------------- /gke.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | node=$(kubectl get node -o name | head -n 1) 3 | file="/host/home/kubernetes/kubelet-config.yaml" 4 | input="./.gke.yaml" 5 | kubectl debug "$node" -it --image xxradar/hackon -- cat $file | tail -n +2 > $input; 6 | node_info=$(kubectl get $node -o json); 7 | 8 | echo 'total capacity:' 9 | echo "cpu="$(echo $node_info | jq -r '.status.capacity.cpu // 0'); 10 | echo "memory="$(echo $node_info | jq -r '.status.capacity.memory // 0'); 11 | echo $'\n'; 12 | 13 | echo 'total allocatable:' 14 | echo "cpu="$(echo $node_info | jq -r '.status.allocatable.cpu // 0'); 15 | echo "memory="$(echo $node_info | jq -r '.status.allocatable.memory // 0'); 16 | echo $'\n'; 17 | 18 | echo 'system reserved:' 19 | echo "cpu="$(cat $input | yq -r '.systemReserved.cpu // 0'); 20 | echo "memory="$(cat $input | yq -r '.systemReserved.memory // 0'); 21 | echo $'\n'; 22 | 23 | echo 'kubelet reserved:'; 24 | echo "cpu="$(cat $input | yq -r '.kubeReserved.cpu // 0'); 25 | echo "memory="$(cat $input | yq -r '.kubeReserved.memory // 0'); 26 | echo $'\n'; 27 | 28 | echo 'eviction threshold:'; 29 | echo "memory="$(cat $input | yq -r '.evictionHard."memory.available" // 0'); 30 | 31 | rm $input; -------------------------------------------------------------------------------- /inspector.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo "detect the cluster ..."; 4 | 5 | if [[ ! -z $(kubectl cluster-info dump | grep -e 'eks') ]] 6 | then 7 | cluster="EKS"; 8 | elif [[ ! -z $(kubectl cluster-info dump | grep -e 'gke') ]] 9 | then 10 | cluster="GKE"; 11 | elif [[ ! -z $(kubectl cluster-info dump | grep -e 'aks') ]] 12 | then 13 | cluster="AKS"; 14 | elif [[ ! -z $(kubectl cluster-info dump | grep -e 'lke') ]] 15 | then 16 | cluster="LKE"; 17 | elif [[ ! -z $(kubectl cluster-info dump | grep -e 'doks') ]] 18 | then 19 | cluster="DOKS"; 20 | else 21 | echo "this cluster is not supported";exit; 22 | fi 23 | 24 | script=$(echo $cluster | tr '[:upper:]' '[:lower:]')".sh" 25 | 26 | echo "your are running $cluster cluster" 27 | echo "running $script ..." 28 | echo $'\n'; 29 | 30 | /bin/bash $script -------------------------------------------------------------------------------- /lke.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | node=$(kubectl get node -o name | head -n 1) 3 | file="/host/var/lib/kubelet/config.yaml" 4 | input="./.lke.yaml" 5 | kubectl debug "$node" -it --image xxradar/hackon -- cat $file | tail -n +2 > $input; 6 | node_info=$(kubectl get $node -o json); 7 | 8 | echo 'total capacity:' 9 | echo "cpu="$(echo $node_info | jq -r '.status.capacity.cpu // 0'); 10 | echo "memory="$(echo $node_info | jq -r '.status.capacity.memory // 0'); 11 | echo $'\n'; 12 | 13 | echo 'total allocatable:' 14 | echo "cpu="$(echo $node_info | jq -r '.status.allocatable.cpu // 0'); 15 | echo "memory="$(echo $node_info | jq -r '.status.allocatable.memory // 0'); 16 | echo $'\n'; 17 | 18 | echo 'system reserved:' 19 | echo "cpu="$(cat $input | yq -r '.systemReserved.cpu // 0'); 20 | echo "memory="$(cat $input | yq -r '.systemReserved.memory // 0'); 21 | echo $'\n'; 22 | 23 | echo 'kubelet reserved:'; 24 | echo "cpu="$(cat $input | yq -r '.kubeReserved.cpu // 0'); 25 | echo "memory="$(cat $input | yq -r '.kubeReserved.memory // 0'); 26 | echo $'\n'; 27 | 28 | echo 'eviction threshold:'; 29 | echo "memory="$(cat $input | yq -r '.evictionHard."memory.available" // 0'); 30 | 31 | rm $input; -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | ### Scripts for getting the reserved resources 2 | 3 | - This article explain the resource allocation https://learnk8s.io/allocatable-resources 4 | - This is how we can ssh to the managed nodes https://gist.github.com/danielepolencic/b2b40da7c3157f5bb6c291b48279aba1 5 | - We can get the total allocatable resources by describing the node `kubectl describe node` 6 | - Run the script `inspector.sh` and set your k8s cluster type. 7 | 8 | ### GKE (Google Kubernetes Engine) 9 | 10 | - make sure you have the right kubectl context, you cen run `kubectl get node` to verify 11 | - the script use `yq` command make sure to [install it](https://kislyuk.github.io/yq/#installation). 12 | - kubelet service file `/etc/systemd/system/kubelet.service` 13 | ``` 14 | [Unit] 15 | Description=Kubernetes kubelet 16 | Requires=network-online.target 17 | After=network-online.target 18 | 19 | [Service] 20 | Restart=always 21 | RestartSec=10 22 | EnvironmentFile=/etc/default/kubelet 23 | ExecStart=/home/kubernetes/bin/kubelet $KUBELET_OPTS 24 | 25 | [Install] 26 | WantedBy=multi-user.target 27 | ``` 28 | - GKE store the running command for kubelet with all the flags in `/etc/default/kubelet` 29 | ``` 30 | KUBELET_OPTS="--v=2 --experimental-check-node-capabilities-before-mount=true --cloud-provider=gce --experimental-mounter-path=/home/kubernetes/containerized_mounter/mounter --cert-dir=/var/lib/kubelet/pki/ --kubeconfig=/var/lib/kubelet/kubeconfig --cni-bin-dir=/home/kubernetes/bin --image-pull-progress-deadline=5m --max-pods=110 --non-masquerade-cidr=0.0.0.0/0 --network-plugin=kubenet --volume-plugin-dir=/home/kubernetes/flexvolume --node-status-max-images=25 --container-runtime=remote --container-runtime-endpoint=unix:///run/containerd/containerd.sock --runtime-cgroups=/system.slice/containerd.service --registry-qps=10 --registry-burst=20 --config /home/kubernetes/kubelet-config.yaml --pod-sysctls='net.core.somaxconn=1024,net.ipv4.conf.all.accept_redirects=0,net.ipv4.conf.all.forwarding=1,net.ipv4.conf.all.route_localnet=1,net.ipv4.conf.default.forwarding=1,net.ipv4.ip_forward=1,net.ipv4.tcp_fin_timeout=60,net.ipv4.tcp_keepalive_intvl=60,net.ipv4.tcp_keepalive_probes=5,net.ipv4.tcp_keepalive_time=300,net.ipv4.tcp_rmem=4096 87380 6291456,net.ipv4.tcp_syn_retries=6,net.ipv4.tcp_tw_reuse=0,net.ipv4.tcp_wmem=4096 16384 4194304,net.ipv4.udp_rmem_min=4096,net.ipv4.udp_wmem_min=4096,net.ipv6.conf.all.disable_ipv6=1,net.ipv6.conf.default.accept_ra=0,net.ipv6.conf.default.disable_ipv6=1,net.netfilter.nf_conntrack_generic_timeout=600,net.netfilter.nf_conntrack_tcp_be_liberal=1,net.netfilter.nf_conntrack_tcp_timeout_close_wait=3600,net.netfilter.nf_conntrack_tcp_timeout_established=86400'" 31 | KUBE_COVERAGE_FILE="/var/log/kubelet.cov" 32 | ``` 33 | - GKE store the config in the file `/home/kubernetes/kubelet-config.yaml` 34 | ```yaml 35 | apiVersion: kubelet.config.k8s.io/v1beta1 36 | authentication: 37 | anonymous: 38 | enabled: false 39 | webhook: 40 | enabled: true 41 | x509: 42 | clientCAFile: /etc/srv/kubernetes/pki/ca-certificates.crt 43 | authorization: 44 | mode: Webhook 45 | cgroupRoot: / 46 | clusterDNS: 47 | - 10.60.0.10 48 | clusterDomain: cluster.local 49 | enableDebuggingHandlers: true 50 | evictionHard: 51 | memory.available: 100Mi 52 | nodefs.available: 10% 53 | nodefs.inodesFree: 5% 54 | pid.available: 10% 55 | featureGates: 56 | CSIMigrationGCE: true 57 | DynamicKubeletConfig: false 58 | ExecProbeTimeout: false 59 | InTreePluginAWSUnregister: true 60 | InTreePluginAzureDiskUnregister: true 61 | InTreePluginOpenStackUnregister: true 62 | InTreePluginvSphereUnregister: true 63 | RotateKubeletServerCertificate: true 64 | kernelMemcgNotification: true 65 | kind: KubeletConfiguration 66 | kubeReserved: 67 | cpu: 70m 68 | ephemeral-storage: 41Gi 69 | memory: 1736Mi 70 | readOnlyPort: 10255 71 | serverTLSBootstrap: true 72 | staticPodPath: /etc/kubernetes/manifests 73 | ``` 74 | 75 | 76 | ### EKS (Elastic Kubernetes Service) 77 | 78 | - make sure you have the right kubectl context, you cen run `kubectl get node` to verify 79 | - the script use `jq` command make sure to [install it](https://formulae.brew.sh/formula/jq). 80 | - you can customize the reserved resources with [eksctl](https://eksctl.io/usage/customizing-the-kubelet/). 81 | - kubelet service file `/etc/systemd/system/kubelet.service` 82 | ```bash 83 | [Unit] 84 | Description=Kubernetes Kubelet 85 | Documentation=https://github.com/kubernetes/kubernetes 86 | After=docker.service iptables-restore.service 87 | Requires=docker.service 88 | 89 | [Service] 90 | ExecStartPre=/sbin/iptables -P FORWARD ACCEPT -w 5 91 | ExecStart=/usr/bin/kubelet --cloud-provider aws \ 92 | --config /etc/kubernetes/kubelet/kubelet-config.json \ 93 | --kubeconfig /var/lib/kubelet/kubeconfig \ 94 | --container-runtime docker \ 95 | --network-plugin cni $KUBELET_ARGS $KUBELET_EXTRA_ARGS 96 | 97 | Restart=always 98 | RestartSec=5 99 | KillMode=process 100 | 101 | [Install] 102 | WantedBy=multi-user.target 103 | ``` 104 | - EKS store the config in the file `/etc/kubernetes/kubelet/kubelet-config.json` 105 | ```json 106 | { 107 | "kind": "KubeletConfiguration", 108 | "apiVersion": "kubelet.config.k8s.io/v1beta1", 109 | "address": "0.0.0.0", 110 | "authentication": { 111 | "anonymous": { 112 | "enabled": false 113 | }, 114 | "webhook": { 115 | "cacheTTL": "2m0s", 116 | "enabled": true 117 | }, 118 | "x509": { 119 | "clientCAFile": "/etc/kubernetes/pki/ca.crt" 120 | } 121 | }, 122 | "authorization": { 123 | "mode": "Webhook", 124 | "webhook": { 125 | "cacheAuthorizedTTL": "5m0s", 126 | "cacheUnauthorizedTTL": "30s" 127 | } 128 | }, 129 | "clusterDomain": "cluster.local", 130 | "hairpinMode": "hairpin-veth", 131 | "readOnlyPort": 0, 132 | "cgroupDriver": "cgroupfs", 133 | "cgroupRoot": "/", 134 | "featureGates": { 135 | "RotateKubeletServerCertificate": true 136 | }, 137 | "protectKernelDefaults": true, 138 | "serializeImagePulls": false, 139 | "serverTLSBootstrap": true, 140 | "tlsCipherSuites": [ 141 | "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256", 142 | "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", 143 | "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305", 144 | "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", 145 | "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305", 146 | "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384", 147 | "TLS_RSA_WITH_AES_256_GCM_SHA384", 148 | "TLS_RSA_WITH_AES_128_GCM_SHA256" 149 | ], 150 | "clusterDNS": [ 151 | "10.100.0.10" 152 | ], 153 | "evictionHard": { 154 | "memory.available": "100Mi", 155 | "nodefs.available": "10%", 156 | "nodefs.inodesFree": "5%" 157 | }, 158 | "kubeReserved": { 159 | "cpu": "70m", 160 | "ephemeral-storage": "1Gi", 161 | "memory": "574Mi" 162 | } 163 | } 164 | ``` 165 | 166 | ### AKS (Azure Kubernetes Service) 167 | 168 | - make sure you have the right kubectl context, you cen run `kubectl get node` to verify 169 | - kubelet service file `/etc/systemd/system/kubelet.service` 170 | ```bash 171 | [Unit] 172 | Description=Kubelet 173 | ConditionPathExists=/usr/local/bin/kubelet 174 | Wants=network-online.target 175 | After=network-online.target 176 | 177 | [Service] 178 | Restart=always 179 | EnvironmentFile=/etc/default/kubelet 180 | SuccessExitStatus=143 181 | ExecStartPre=/bin/bash /opt/azure/containers/kubelet.sh 182 | ExecStartPre=/bin/mkdir -p /var/lib/kubelet 183 | ExecStartPre=/bin/mkdir -p /var/lib/cni 184 | ExecStartPre=/bin/bash -c "if [ $(mount | grep \"/var/lib/kubelet\" | wc -l) -le 0 ] ; then /bin/mount --bind /var/lib/kubelet /var/lib/kubelet ; fi" 185 | ExecStartPre=/bin/mount --make-shared /var/lib/kubelet 186 | 187 | ExecStartPre=-/sbin/ebtables -t nat --list 188 | ExecStartPre=-/sbin/iptables -t nat --numeric --list 189 | 190 | ExecStart=/usr/local/bin/kubelet \ 191 | --enable-server \ 192 | --node-labels="${KUBELET_NODE_LABELS}" \ 193 | --v=2 \ 194 | --volume-plugin-dir=/etc/kubernetes/volumeplugins \ 195 | $KUBELET_TLS_BOOTSTRAP_FLAGS \ 196 | $KUBELET_CONFIG_FILE_FLAGS \ 197 | $KUBELET_CONTAINERD_FLAGS \ 198 | $KUBELET_FLAGS 199 | 200 | [Install] 201 | WantedBy=multi-user.target 202 | ``` 203 | - AKS use flags (KUBELET_FLAGS) instead of config files, you can find the flags in `/etc/default/kubelet` 204 | ```bash 205 | KUBELET_FLAGS=--address=0.0.0.0 --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --azure-container-registry-config=/etc/kubernetes/azure.json --cgroups-per-qos=true --client-ca-file=/etc/kubernetes/certs/ca.crt --cloud-provider=external --cluster-dns=10.0.0.10 --cluster-domain=cluster.local --container-log-max-size=50M --enforce-node-allocatable=pods --event-qps=0 --eviction-hard=memory.available<750Mi,nodefs.available<10%,nodefs.inodesFree<5% --feature-gates=CSIMigration=true,CSIMigrationAzureDisk=true,CSIMigrationAzureFile=true,DelegateFSGroupToCSIDriver=true,DisableAcceleratorUsageMetrics=false,DynamicKubeletConfig=false --image-gc-high-threshold=85 --image-gc-low-threshold=80 --keep-terminated-pod-volumes=false --kube-reserved=cpu=100m,memory=1843Mi --kubeconfig=/var/lib/kubelet/kubeconfig --max-pods=110 --node-status-update-frequency=10s --pod-infra-container-image=mcr.microsoft.com/oss/kubernetes/pause:3.6 --pod-manifest-path=/etc/kubernetes/manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --rotate-certificates=true --streaming-connection-idle-timeout=4h --tls-cert-file=/etc/kubernetes/certs/kubeletserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --tls-private-key-file=/etc/kubernetes/certs/kubeletserver.key 206 | KUBELET_REGISTER_SCHEDULABLE=true 207 | NETWORK_POLICY= 208 | 209 | KUBELET_NODE_LABELS=kubernetes.azure.com/role=agent,agentpool=agentpool,kubernetes.azure.com/agentpool=agentpool,storageprofile=managed,storagetier=Premium_LRS,kubernetes.azure.com/storageprofile=managed,kubernetes.azure.com/storagetier=Premium_LRS,kubernetes.azure.com/os-sku=Ubuntu,kubernetes.azure.com/cluster=MC_664-a59f9d43-create-an-aks-cluster-in-azure-with-t_k8su_cent,kubernetes.azure.com/kubelet-identity-client-id=,kubernetes.azure.com/mode=system,kubernetes.azure.com/node-image-version=AKSUbuntu-1804gen2containerd-2022.08.10 210 | ``` 211 | 212 | ### LKE (Linode Kubernetes Engine) 213 | 214 | - make sure you have the right kubectl context, you cen run `kubectl get node` to verify 215 | - the script use `yq` command make sure to [install it](https://kislyuk.github.io/yq/#installation). 216 | - kubelet service file `/etc/systemd/system/kubelet.service.d/10-kubeadm.conf` 217 | ```bash 218 | # Note: This dropin only works with kubeadm and kubelet v1.11+ 219 | [Service] 220 | Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf" 221 | Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml" 222 | # This is a file that "kubeadm init" and "kubeadm join" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically 223 | EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env 224 | # This is a file that the user can use for overrides of the kubelet args as a last resort. Preferably, the user should use 225 | # the .NodeRegistration.KubeletExtraArgs object in the configuration files instead. KUBELET_EXTRA_ARGS should be sourced from this file. 226 | EnvironmentFile=-/etc/default/kubelet 227 | ExecStart= 228 | ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS 229 | ``` 230 | - LKE store the config in the file `/var/lib/kubelet/config.yaml` 231 | ```yaml 232 | address: 0.0.0.0 233 | apiVersion: kubelet.config.k8s.io/v1beta1 234 | authentication: 235 | anonymous: 236 | enabled: false 237 | webhook: 238 | cacheTTL: 2m0s 239 | enabled: true 240 | x509: 241 | clientCAFile: /etc/kubernetes/pki/ca.crt 242 | authorization: 243 | mode: Webhook 244 | webhook: 245 | cacheAuthorizedTTL: 5m0s 246 | cacheUnauthorizedTTL: 30s 247 | cgroupDriver: cgroupfs 248 | cgroupsPerQOS: true 249 | clusterDNS: 250 | - 10.128.0.10 251 | clusterDomain: cluster.local 252 | containerLogMaxFiles: 5 253 | containerLogMaxSize: 10Mi 254 | contentType: application/vnd.kubernetes.protobuf 255 | cpuCFSQuota: true 256 | cpuManagerPolicy: none 257 | cpuManagerReconcilePeriod: 10s 258 | enableControllerAttachDetach: true 259 | enableDebuggingHandlers: true 260 | enforceNodeAllocatable: 261 | - pods 262 | eventBurst: 10 263 | eventRecordQPS: 5 264 | evictionHard: 265 | imagefs.available: 15% 266 | memory.available: 100Mi 267 | nodefs.available: 10% 268 | nodefs.inodesFree: 5% 269 | evictionPressureTransitionPeriod: 5m0s 270 | failSwapOn: true 271 | featureGates: 272 | EphemeralContainers: true 273 | GracefulNodeShutdown: true 274 | fileCheckFrequency: 20s 275 | hairpinMode: promiscuous-bridge 276 | healthzBindAddress: 127.0.0.1 277 | healthzPort: 10248 278 | httpCheckFrequency: 20s 279 | imageGCHighThresholdPercent: 85 280 | imageGCLowThresholdPercent: 80 281 | imageMinimumGCAge: 2m0s 282 | iptablesDropBit: 15 283 | iptablesMasqueradeBit: 14 284 | kind: KubeletConfiguration 285 | kubeAPIBurst: 10 286 | kubeAPIQPS: 5 287 | logging: 288 | flushFrequency: 0 289 | options: 290 | json: 291 | infoBufferSize: "0" 292 | verbosity: 0 293 | makeIPTablesUtilChains: true 294 | maxOpenFiles: 1000000 295 | maxPods: 110 296 | memorySwap: {} 297 | nodeStatusReportFrequency: 0s 298 | nodeStatusUpdateFrequency: 10s 299 | oomScoreAdj: -999 300 | podPidsLimit: -1 301 | port: 10250 302 | registryBurst: 10 303 | registryPullQPS: 5 304 | resolvConf: /etc/resolv.conf 305 | rotateCertificates: true 306 | runtimeRequestTimeout: 2m0s 307 | serializeImagePulls: true 308 | shutdownGracePeriod: 30s 309 | shutdownGracePeriodCriticalPods: 10s 310 | staticPodPath: /etc/kubernetes/manifests 311 | streamingConnectionIdleTimeout: 4h0m0s 312 | syncFrequency: 1m0s 313 | volumeStatsAggPeriod: 1m0s 314 | ``` 315 | 316 | ### DOKS (DigitalOcean Kubernetes Service) 317 | 318 | - make sure you have the right kubectl context, you cen run `kubectl get node` to verify 319 | - the script use `yq` command make sure to [install it](https://kislyuk.github.io/yq/#installation). 320 | - kubelet service file `/etc/systemd/system/kubelet.service` 321 | ```bash 322 | [Unit] 323 | Description=Kubernetes Kubelet Server 324 | Documentation=https://kubernetes.io/docs/concepts/overview/components/#kubelet 325 | 326 | [Service] 327 | OOMScoreAdjust=-999 328 | 329 | ExecStart=/usr/bin/kubelet \ 330 | --config=/etc/kubernetes/kubelet.conf \ 331 | --logtostderr=true \ 332 | --kubeconfig=/etc/kubernetes/kubelet.kubeconfig \ 333 | --bootstrap-kubeconfig=/etc/kubernetes/bootstrap.kubeconfig \ 334 | --register-node=true \ 335 | --node-labels="doks.digitalocean.com/node-id=4f2b99f0-9ebd-47a5-a954-efd61779f143,doks.digitalocean.com/node-pool-id=f6e0de05-2336-4a06-834c-d02faec8006c,doks.digitalocean.com/node-pool=pool-1pvhk09r2,doks.digitalocean.com/version=1.23.9-do.0" \ 336 | --root-dir=/var/lib/kubelet \ 337 | --v=2 \ 338 | --cloud-provider=external \ 339 | --node-ip="10.106.0.2" \ 340 | --container-runtime=remote \ 341 | --container-runtime-endpoint=unix:///run/containerd/containerd.sock 342 | 343 | Restart=on-failure 344 | RestartSec=5 345 | 346 | [Install] 347 | WantedBy=multi-user.target 348 | ``` 349 | - DOKS store the config in the file `/etc/kubernetes/kubelet.conf` 350 | ```yaml 351 | kind: KubeletConfiguration 352 | providerID: "digitalocean://314476419" 353 | apiVersion: kubelet.config.k8s.io/v1beta1 354 | staticPodPath: "/etc/kubernetes/manifests" 355 | rotateCertificates: true 356 | authentication: 357 | anonymous: 358 | enabled: false 359 | webhook: 360 | enabled: true 361 | x509: 362 | clientCAFile: "/etc/kubernetes/ca.pem" 363 | authorization: 364 | mode: AlwaysAllow 365 | runtimeRequestTimeout: "15m" 366 | clusterDomain: cluster.local 367 | clusterDNS: 368 | - 10.245.0.10 369 | kubeReserved: 370 | memory: "369432Ki" 371 | cpu: "100m" 372 | pid: "1024" 373 | evictionHard: 374 | memory.available: "51200Ki" 375 | pid.available: 10% 376 | # The nodefs and imagefs reference the same fs in DOKS, but the reason 377 | # for the eviction matters in how its handled. Images will only be cleaned 378 | # if the imagefs limits are hit. 379 | imagefs.inodesFree: 6% 380 | # Note: these are the defaults, but spelled out here for clarity. 381 | nodefs.available: 10% 382 | nodefs.inodesFree: 5% 383 | imagefs.available: 15% 384 | serverTLSBootstrap: true 385 | serializeImagePulls: false 386 | # Delay node shutdown for 30s to give pods time to terminate gracefully. Normal 387 | # pods will be shut down during the first 20s, then critical pods will have 10s. 388 | shutdownGracePeriod: 30s 389 | shutdownGracePeriodCriticalPods: 10s 390 | ``` --------------------------------------------------------------------------------