├── .github └── workflows │ ├── release.yaml │ └── test.yaml ├── Dockerfile ├── LICENSE ├── README.md ├── k8s ├── daemonset.yaml └── deployment.yaml └── script ├── build ├── env └── publish /.github/workflows/release.yaml: -------------------------------------------------------------------------------- 1 | name: release 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | 8 | jobs: 9 | release: 10 | runs-on: ubuntu-latest 11 | 12 | steps: 13 | - name: checkout 14 | uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 # v2.4.0 15 | with: 16 | fetch-depth: 1 17 | 18 | - name: build image 19 | run: docker build --platform linux/amd64 -t ghcr.io/digitalocean-packages/doks-debug:latest . 20 | 21 | - name: log into container registry 22 | run: echo "${{ secrets.GHCRTOKEN }}" | docker login ghcr.io --username "${{ secrets.GHCRUSER }}" --password-stdin 23 | 24 | - name: push image 25 | run: docker push ghcr.io/digitalocean-packages/doks-debug:latest 26 | -------------------------------------------------------------------------------- /.github/workflows/test.yaml: -------------------------------------------------------------------------------- 1 | name: test 2 | 3 | on: [pull_request] 4 | 5 | jobs: 6 | test: 7 | runs-on: ubuntu-latest 8 | 9 | steps: 10 | - name: checkout 11 | uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 # v2.4.0 12 | with: 13 | fetch-depth: 1 14 | 15 | - name: build image 16 | run: docker build --platform linux/amd64 -t doks-debug . 17 | 18 | - name: smoke test 19 | run: docker run --rm doks-debug sleep 1 20 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # match doks-debug version with DOKS worker node image version for kernel 2 | # tooling compatibility reasons 3 | FROM debian:12-slim 4 | 5 | # Specify the version of crictl to install 6 | ARG CRICTL_VERSION="v1.31.1" 7 | 8 | WORKDIR /root 9 | 10 | # use same dpkg path-exclude settings that come by default with ubuntu:focal 11 | # image that we previously used 12 | RUN echo 'path-exclude=/usr/share/locale/*/LC_MESSAGES/*.mo' >> /etc/dpkg/dpkg.cfg.d/excludes 13 | RUN echo 'path-exclude=/usr/share/doc/*' >> /etc/dpkg/dpkg.cfg.d/excludes 14 | RUN echo 'path-include=/usr/share/doc/*/copyright' >> /etc/dpkg/dpkg.cfg.d/excludes 15 | RUN echo 'path-include=/usr/share/doc/*/changelog.Debian.*' >> /etc/dpkg/dpkg.cfg.d/excludes 16 | 17 | RUN apt-get update -qq && \ 18 | apt-get install -y apt-transport-https \ 19 | ca-certificates \ 20 | software-properties-common \ 21 | httping \ 22 | man \ 23 | man-db \ 24 | vim \ 25 | screen \ 26 | curl \ 27 | gnupg \ 28 | atop \ 29 | htop \ 30 | dstat \ 31 | jq \ 32 | dnsutils \ 33 | tcpdump \ 34 | traceroute \ 35 | iputils-ping \ 36 | iptables \ 37 | net-tools \ 38 | ncat \ 39 | iproute2 \ 40 | strace \ 41 | telnet \ 42 | openssl \ 43 | psmisc \ 44 | dsniff \ 45 | mtr-tiny \ 46 | conntrack \ 47 | llvm-13 llvm-13-tools \ 48 | wget \ 49 | bpftool 50 | 51 | # Install crictl 52 | RUN wget https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz && \ 53 | tar zxvf crictl-${CRICTL_VERSION}-linux-amd64.tar.gz -C /usr/local/bin && \ 54 | rm -f crictl-${CRICTL_VERSION}-linux-amd64.tar.gz 55 | 56 | # Specify the default image endpoint for crictl 57 | RUN echo 'runtime-endpoint: unix:///run/containerd/containerd.sock' >> /etc/crictl.yaml 58 | RUN echo 'image-endpoint: unix:///run/containerd/containerd.sock' >> /etc/crictl.yaml 59 | RUN echo 'timeout: 2' >> /etc/crictl.yaml 60 | 61 | CMD [ "/bin/bash" ] 62 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 DigitalOcean 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | A Docker image with Kubernetes manifests for investigation and troubleshooting your cluster. 2 | 3 | ![main build](https://github.com/digitalocean/doks-debug/actions/workflows/test.yaml/badge.svg) ![main release](https://github.com/digitalocean/doks-debug/actions/workflows/release.yaml/badge.svg) 4 | 5 | # Purpose 6 | 7 | The DOKS team provides this image for use as-is and for transparency as the image used when a request to "deploy a debug pod" is made to our customers, which may occur when deeper investigation is needed with direct access to a cluster. 8 | 9 | # Usage 10 | 11 | ```bash 12 | kubectl apply -f k8s/daemonset.yaml 13 | ``` 14 | 15 | This DaemonSet manifest will: 16 | 17 | 1. Ensure a pod with our Docker image is running indefinitely on every node. 18 | 2. Use `hostPID`, `hostIPC`, and `hostNetwork`. 19 | 3. Mount the entire host filesystem to `/host` in the containers. 20 | 4. Mount the `containerd` socket at `/run/containerd/containerd.sock` from the host into the container. 21 | 22 | In order to make use of these workloads, you can exec into a pod of choice by name: 23 | 24 | ```bash 25 | kubectl -n kube-system exec -it my-pod-name bash 26 | ``` 27 | 28 | If you know the specific node name that you're interested in, you can exec into the debug pod on that node with: 29 | 30 | ```bash 31 | NODE_NAME="my-node-name" 32 | POD_NAME=$(kubectl -n kube-system get pods --field-selector spec.nodeName=${NODE_NAME} -ojsonpath='{.items[0].metadata.name}') 33 | kubectl -n kube-system exec -it ${POD_NAME} bash 34 | ``` 35 | 36 | Once you're in, you have access to the set of tools listed in the `Dockerfile`. This includes: 37 | 38 | - [`vim`](https://github.com/vim/vim) - is a greatly improved version of the good old UNIX editor Vi. 39 | - [`screen`](https://www.gnu.org/software/screen/) - is a full-screen window manager that multiplexes a physical terminal between several processes, typically interactive shells. 40 | - [`curl`](https://github.com/curl/curl) - is a command-line tool for transferring data specified with URL syntax. 41 | - [`jq`](https://github.com/stedolan/jq) - is a lightweight and flexible command-line JSON processor. 42 | - [`dnsutils`](https://packages.debian.org/stretch/dnsutils) - includes various client programs related to DNS that are derived from the BIND source tree, specifically [`dig`](https://linux.die.net/man/1/dig), [`nslookup`](https://linux.die.net/man/1/nslookup), and [`nsupdate`](https://linux.die.net/man/8/nsupdate). 43 | - [`iputils-ping`](https://packages.debian.org/stretch/iputils-ping) - includes the [`ping`](https://linux.die.net/man/8/ping) tool that sends ICMP `ECHO_REQUEST` packets to a host in order to test if the host is reachable via the network. 44 | - [`tcpdump`](https://www.tcpdump.org/) - a powerful command-line packet analyzer; and libpcap, a portable C/C++ library for network traffic capture. 45 | - [`traceroute`](https://linux.die.net/man/8/traceroute) - tracks the route packets taken from an IP network on their way to a given host. 46 | - [`net-tools`](https://packages.debian.org/stretch/net-tools) - includes the important tools for controlling the network subsystem of the Linux kernel, specifically [`arp`](http://man7.org/linux/man-pages/man8/arp.8.html), [`ifconfig`](https://linux.die.net/man/8/ifconfig), and [`netstat`](https://linux.die.net/man/8/netstat). 47 | - [`netcat`](https://linux.die.net/man/1/nc) - is a multi-tool for interacting with TCP and UDP; it can open TCP connections, send UDP packets, listen on arbitrary TCP and UDP ports, do port scanning, and deal with both IPv4 and IPv6. 48 | - [`iproute2`](https://wiki.linuxfoundation.org/networking/iproute2) - is a collection of utilities for controlling TCP / IP networking and traffic control in Linux. 49 | - [`strace`](https://github.com/strace/strace) - is a diagnostic, debugging and instructional userspace utility with a traditional command-line interface for Linux. It is used to monitor and tamper with interactions between processes and the Linux kernel, which include system calls, signal deliveries, and changes of process state. 50 | - [`dstat`](http://dag.wiee.rs/home-made/dstat/) - is a versatile replacement for vmstat, iostat, netstat and ifstat. Dstat overcomes some of their limitations and adds some extra features, more counters and flexibility. Dstat is handy for monitoring systems during performance tuning tests, benchmarks or troubleshooting. 51 | - [`htop`](https://hisham.hm/htop/) - is interactive process viewer for Unix systems. 52 | - [`atop`](https://www.atoptool.nl/) - is an advanced interactive monitor for Linux-systems to view the load on system-level and process-level. 53 | - [`wget`](https://www.gnu.org/software/wget/) - for retrieving files using HTTP, HTTPS, FTP and FTPS. 54 | - [`crictl`](https://github.com/kubernetes-sigs/cri-tools/blob/master/docs/crictl.md) - A CLI for CRI endpoints. Configured to use `/run/containerd/containerd.sock` as a default endpoint. 55 | # Tips and Tricks 56 | 57 | ## chroot + systemctl 58 | 59 | ```bash 60 | chroot /host /bin/bash 61 | systemctl status kubelet 62 | journalctl -xe 63 | journalctl -u kubelet 64 | ``` 65 | 66 | # Contributing 67 | 68 | At DigitalOcean we value and love our community! If you have any issues or would like to contribute, feel free to open an issue or PR and cc any of the maintainers. 69 | -------------------------------------------------------------------------------- /k8s/daemonset.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: DaemonSet 3 | metadata: 4 | name: doks-debug 5 | namespace: kube-system 6 | labels: 7 | app: doks-debug 8 | spec: 9 | selector: 10 | matchLabels: 11 | name: doks-debug 12 | template: 13 | metadata: 14 | labels: 15 | name: doks-debug 16 | annotations: 17 | clusterlint.digitalocean.com/disabled-checks: "hostpath-volume" 18 | cluster-autoscaler.kubernetes.io/safe-to-evict: "true" 19 | spec: 20 | dnsPolicy: ClusterFirstWithHostNet 21 | hostPID: true 22 | hostIPC: true 23 | hostNetwork: true 24 | tolerations: 25 | - operator: Exists 26 | containers: 27 | - name: doks-debug 28 | securityContext: 29 | privileged: true 30 | image: ghcr.io/digitalocean-packages/doks-debug:latest 31 | command: [ "sleep", "infinity" ] 32 | resources: 33 | requests: 34 | memory: "0" 35 | cpu: "0" 36 | limits: 37 | memory: "500Mi" 38 | cpu: "500m" 39 | volumeMounts: 40 | - name: host 41 | mountPath: /host 42 | - name: containerd 43 | mountPath: /run/containerd/containerd.sock 44 | terminationGracePeriodSeconds: 0 45 | volumes: 46 | - name: host 47 | hostPath: 48 | path: / 49 | - name: containerd 50 | hostPath: 51 | path: /run/containerd/containerd.sock 52 | type: Socket 53 | updateStrategy: 54 | rollingUpdate: 55 | maxSurge: 0 56 | maxUnavailable: 100% 57 | type: RollingUpdate 58 | -------------------------------------------------------------------------------- /k8s/deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: doks-debug 5 | namespace: kube-system 6 | labels: 7 | app: doks-debug 8 | spec: 9 | replicas: 1 10 | selector: 11 | matchLabels: 12 | name: doks-debug 13 | template: 14 | metadata: 15 | labels: 16 | name: doks-debug 17 | annotations: 18 | clusterlint.digitalocean.com/disabled-checks: "hostpath-volume" 19 | cluster-autoscaler.kubernetes.io/safe-to-evict: "true" 20 | spec: 21 | dnsPolicy: ClusterFirstWithHostNet 22 | hostPID: true 23 | hostIPC: true 24 | hostNetwork: true 25 | tolerations: 26 | - operator: Exists 27 | containers: 28 | - name: doks-debug 29 | securityContext: 30 | privileged: true 31 | image: ghcr.io/digitalocean-packages/doks-debug:latest 32 | command: [ "sleep", "infinity" ] 33 | resources: 34 | requests: 35 | memory: "0" 36 | cpu: "0" 37 | limits: 38 | memory: "500Mi" 39 | cpu: "500m" 40 | volumeMounts: 41 | - name: host 42 | mountPath: /host 43 | - name: containerd 44 | mountPath: /run/containerd/containerd.sock 45 | terminationGracePeriodSeconds: 0 46 | volumes: 47 | - name: host 48 | hostPath: 49 | path: / 50 | - name: containerd 51 | hostPath: 52 | path: /run/containerd/containerd.sock 53 | type: Socket 54 | strategy: 55 | rollingUpdate: 56 | maxSurge: 0 57 | maxUnavailable: 100% 58 | type: RollingUpdate 59 | -------------------------------------------------------------------------------- /script/build: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | . script/env 4 | docker build -t ${DOCKER_REPO} . -------------------------------------------------------------------------------- /script/env: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | export DOCKER_REPO="digitalocean/doks-debug" -------------------------------------------------------------------------------- /script/publish: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | . script/env 4 | script/build 5 | docker push ${DOCKER_REPO} --------------------------------------------------------------------------------