├── Dockerfile ├── LICENSE ├── README.md ├── docker-clean.sh ├── docker-clean.yml ├── etcd-empty-dir-cleanup.sh ├── etcd-empty-dir-cleanup.yml ├── k8s-clean.sh ├── k8s-clean.yml └── rbac.yml /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine 2 | 3 | ENV ETCD_VERSION 3.1.4 4 | ENV KUBE_VERSION 1.7.8 5 | 6 | RUN apk add --update bash curl docker \ 7 | && rm -rf /var/cache/apk/* 8 | 9 | RUN cd /usr/local/bin \ 10 | && curl -O https://storage.googleapis.com/kubernetes-release/release/v${KUBE_VERSION}/bin/linux/amd64/kubectl \ 11 | && chmod 755 /usr/local/bin/kubectl 12 | 13 | RUN cd /tmp \ 14 | && curl -OL https://github.com/coreos/etcd/releases/download/v${ETCD_VERSION}/etcd-v${ETCD_VERSION}-linux-amd64.tar.gz \ 15 | && tar zxf etcd-v${ETCD_VERSION}-linux-amd64.tar.gz \ 16 | && cp etcd-v${ETCD_VERSION}-linux-amd64/etcdctl /usr/local/bin/etcdctl \ 17 | && rm -rf etcd-v${ETCD_VERSION}-linux-amd64* \ 18 | && chmod +x /usr/local/bin/etcdctl 19 | 20 | COPY docker-clean.sh k8s-clean.sh etcd-empty-dir-cleanup.sh /bin/ 21 | RUN chmod +x /bin/docker-clean.sh /bin/k8s-clean.sh /bin/etcd-empty-dir-cleanup.sh 22 | 23 | ENV DOCKER_CLEAN_INTERVAL 1800 24 | ENV DAYS 7 25 | 26 | CMD ["bash", "/bin/docker-clean.sh"] 27 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2017 Onfido 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## k8s-cleanup 2 | 3 | Here are 3 cleanups you can apply on your kubernetes cluster: 4 | * Cleans up exited containers and dangling images/volumes running as a DaemonSet (`docker-clean.yml`). 5 | * Cleans up old replica sets, finished jobs and unrecycled evicted pods as a CronJob (`k8s-clean.yml`). 6 | * Cleans up empty directory (not used anymore) in etcd as a CronJob (`etcd-empty-dir-cleanup.yml`). 7 | 8 | You must have `batch/v2alpha1` enabled on your k8s API server runtime config in order to run the CronJob. 9 | 10 | ### Env vars 11 | In the DaemonSet (`docker-clean.yml`) you can set `DOCKER_CLEAN_INTERVAL` to modify the interval when it cleans up exited containers and dangling images/volumes; defaults to 30min (1800s). 12 | 13 | In the CronJob (`k8s-clean.yml`) you can set `DAYS` to modify the maximum age of replica sets; defaults to 7 days. 14 | 15 | ### Deployment 16 | 17 | ``` 18 | kubectl --context CONTEXT -n kube-system apply -f rbac.yml 19 | kubectl --context CONTEXT -n kube-system apply -f docker-clean.yml 20 | kubectl --context CONTEXT -n kube-system apply -f k8s-clean.yml 21 | kubectl --context CONTEXT -n kube-system apply -f etcd-empty-dir-cleanup.yml 22 | ``` 23 | -------------------------------------------------------------------------------- /docker-clean.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | while true; do 4 | # Remove exited containers 5 | docker ps -a -q -f status=exited | xargs --no-run-if-empty docker rm -v 6 | # Remove dangling images 7 | docker images -f "dangling=true" -q | xargs --no-run-if-empty docker rmi 8 | # Remove dangling volumes 9 | docker volume ls -qf dangling=true | xargs --no-run-if-empty docker volume rm 10 | 11 | # DOCKER_CLEAN_INTERVAL defaults to 30min 12 | sleep $DOCKER_CLEAN_INTERVAL 13 | done 14 | -------------------------------------------------------------------------------- /docker-clean.yml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: extensions/v1beta1 3 | kind: DaemonSet 4 | metadata: 5 | name: k8s-cleanup 6 | namespace: kube-system 7 | spec: 8 | updateStrategy: 9 | type: RollingUpdate 10 | rollingUpdate: 11 | maxUnavailable: 10% 12 | template: 13 | metadata: 14 | labels: 15 | app: k8s-cleanup 16 | spec: 17 | volumes: 18 | - name: dockersocket 19 | hostPath: 20 | path: /var/run/docker.sock 21 | containers: 22 | - name: k8s-cleanup 23 | image: onfido/k8s-cleanup 24 | imagePullPolicy: Always 25 | resources: 26 | requests: 27 | cpu: 100m 28 | memory: 50Mi 29 | volumeMounts: 30 | - name: dockersocket 31 | mountPath: /var/run/docker.sock 32 | -------------------------------------------------------------------------------- /etcd-empty-dir-cleanup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Copyright 2016 The Kubernetes Authors. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | echo "Starting cleanup..." 18 | echo "Removing empty directories from etcd..." 19 | 20 | cleanup_empty_dirs () { 21 | if [ "$(etcdctl ls $1)" ]; then 22 | for SUBDIR in $(etcdctl ls -p $1 | grep "/$") 23 | do 24 | cleanup_empty_dirs ${SUBDIR} 25 | done 26 | else 27 | echo "Removing empty key $1 ..." 28 | etcdctl rmdir $1 29 | fi 30 | } 31 | 32 | cleanup_empty_dirs "/registry" 33 | echo "Done with cleanup." 34 | -------------------------------------------------------------------------------- /etcd-empty-dir-cleanup.yml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: batch/v2alpha1 3 | kind: CronJob 4 | metadata: 5 | name: etcd-empty-dir-cleanup 6 | namespace: kube-system 7 | spec: 8 | schedule: "0 2 * * *" 9 | concurrencyPolicy: "Forbid" 10 | jobTemplate: 11 | spec: 12 | template: 13 | metadata: 14 | name: etcd-empty-dir-cleanup 15 | spec: 16 | hostNetwork: true 17 | nodeSelector: 18 | kubernetes.io/role: master 19 | tolerations: 20 | - effect: NoSchedule 21 | key: node-role.kubernetes.io/master 22 | restartPolicy: OnFailure 23 | containers: 24 | - name: k8s-cleanup 25 | image: onfido/k8s-cleanup 26 | command: ["/bin/bash", "/bin/etcd-empty-dir-cleanup.sh"] 27 | resources: 28 | requests: 29 | cpu: 100m 30 | memory: 50Mi 31 | -------------------------------------------------------------------------------- /k8s-clean.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # DAYS defaults to 7 4 | max=$DAYS 5 | 6 | # Get rs, filter by days, filter by empty rs and select rs older than $max days 7 | emptyReplicaSets=$(kubectl get rs --all-namespaces | \ 8 | grep -oE '^[a-zA-Z0-9-]+\s+[a-zA-Z0-9-]+(\s+[0-9]+){4}d' | sed 's/d$//g' | \ 9 | grep -E '(0\s+){3}' | \ 10 | awk -v max=$max '$6 > max {print $1 "|" $2}') 11 | 12 | # Loop through empty replica sets and delete them 13 | for rs in $emptyReplicaSets; do 14 | IFS='|' read namespace replicaSet <<< "$rs"; 15 | kubectl -n $namespace delete rs $replicaSet; 16 | sleep 0.25 17 | done 18 | 19 | # Get finished jobs older than 1h 20 | finishedJobs=$(kubectl get jobs --all-namespaces | awk 'IF $4 == 1 && $5 ~ /h|d/ {print $1 "|" $2}') 21 | 22 | # Loop through jobs and delete them 23 | for job in $finishedJobs; do 24 | IFS='|' read namespace oldJob <<< "$job"; 25 | kubectl -n $namespace delete job $oldJob; 26 | sleep 0.25 27 | done 28 | 29 | # Get unrecycled evicted pods older than 1h 30 | evictedPods=$(kubectl get pods --all-namespaces -a | grep 'Evicted' | \ 31 | awk 'IF $6 ~ /h|d/ {print $1 "|" $2}') 32 | 33 | # Loop through evicted pods and delete them 34 | for pod in $evictedPods; do 35 | IFS='|' read namespace evictedPod <<< "$pod"; 36 | kubectl -n $namespace delete pod $evictedPod; 37 | sleep 0.25 38 | done 39 | -------------------------------------------------------------------------------- /k8s-clean.yml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: batch/v2alpha1 3 | kind: CronJob 4 | metadata: 5 | name: k8s-cleanup 6 | namespace: kube-system 7 | spec: 8 | schedule: "0 */3 * * *" 9 | concurrencyPolicy: "Forbid" 10 | jobTemplate: 11 | spec: 12 | template: 13 | metadata: 14 | name: k8s-cleanup 15 | spec: 16 | nodeSelector: 17 | kubernetes.io/role: master 18 | tolerations: 19 | - effect: NoSchedule 20 | key: node-role.kubernetes.io/master 21 | restartPolicy: OnFailure 22 | serviceAccountName: k8s-cleanup 23 | containers: 24 | - name: k8s-cleanup 25 | image: onfido/k8s-cleanup 26 | command: ["/bin/bash", "/bin/k8s-clean.sh"] 27 | imagePullPolicy: Always 28 | resources: 29 | requests: 30 | cpu: 100m 31 | memory: 50Mi 32 | -------------------------------------------------------------------------------- /rbac.yml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: rbac.authorization.k8s.io/v1beta1 3 | kind: ClusterRole 4 | metadata: 5 | name: k8s-cleanup 6 | rules: 7 | - apiGroups: ["*"] 8 | resources: ["replicasets", "jobs", "pods"] 9 | verbs: ["get", "list", "delete", "update"] 10 | --- 11 | apiVersion: v1 12 | kind: ServiceAccount 13 | metadata: 14 | name: k8s-cleanup 15 | namespace: kube-system 16 | --- 17 | apiVersion: rbac.authorization.k8s.io/v1beta1 18 | kind: ClusterRoleBinding 19 | metadata: 20 | name: k8s-cleanup 21 | roleRef: 22 | apiGroup: rbac.authorization.k8s.io 23 | kind: ClusterRole 24 | name: k8s-cleanup 25 | subjects: 26 | - kind: ServiceAccount 27 | name: k8s-cleanup 28 | namespace: kube-system 29 | --------------------------------------------------------------------------------