├── requirements.txt ├── img ├── endoscope.png └── graph-ping.png ├── sha256sums ├── .gitlab-ci.yml ├── pause.c ├── example.yml ├── Dockerfile ├── README.md ├── LICENSE-2.0 └── scope /requirements.txt: -------------------------------------------------------------------------------- 1 | kubernetes>=7.0.0,<7.1.0 2 | -------------------------------------------------------------------------------- /img/endoscope.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Agilicus/endoscope/HEAD/img/endoscope.png -------------------------------------------------------------------------------- /img/graph-ping.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Agilicus/endoscope/HEAD/img/graph-ping.png -------------------------------------------------------------------------------- /sha256sums: -------------------------------------------------------------------------------- 1 | dc84268cc3271fc05d0638dc8a50e49a1450c73abbf67cb12ff1dc1e1a9b3a66 get-pip.py 2 | -------------------------------------------------------------------------------- /.gitlab-ci.yml: -------------------------------------------------------------------------------- 1 | --- 2 | stages: 3 | - lint 4 | - build 5 | 6 | include: 7 | - project: tooling/pipelines 8 | ref: master 9 | file: lint-conform.yml 10 | - project: tooling/pipelines 11 | ref: master 12 | file: docker-build.yml 13 | 14 | -------------------------------------------------------------------------------- /pause.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | static void _endme(int sig) 5 | { 6 | _exit(0); 7 | } 8 | int 9 | main(int argc, char **argv) 10 | { 11 | signal(SIGINT, _endme); 12 | signal(SIGTERM, _endme); 13 | pause(); 14 | _exit(0); 15 | } 16 | -------------------------------------------------------------------------------- /example.yml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Pod 4 | metadata: 5 | name: debug 6 | spec: 7 | nodeName: kube-spawn-flannel-worker-fth2z1 8 | hostPID: true 9 | imagePullSecrets: {name: regcred} 10 | containers: 11 | - name: debug 12 | securityContext: 13 | privileged: true 14 | capabilities: 15 | add: ["NET_ADMIN", "SYS_PTRACE"] 16 | image: agilicus/endoscope:latest 17 | volumeMounts: 18 | - mountPath: /var/run/dockershim.sock 19 | name: crisock 20 | - mountPath: /run/docker/netns 21 | name: netns 22 | volumes: 23 | - hostPath: 24 | path: /var/run/dockershim.sock 25 | type: "" 26 | name: crisock 27 | - hostPath: 28 | path: /var/run/docker/netns 29 | type: "" 30 | name: netns 31 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:18.04 as wireshark 2 | LABEL maintainer="don@agilicus.com" 3 | 4 | ENV DEBIAN_FRONTEND noninteractive 5 | 6 | COPY pause.c /pause.c 7 | # Going to build a static-linked dump-cap, rather than 8 | # install wireshark-common in below. Saves 200MB. 9 | # Used github.com mirror rather than https://code.wireshark.org/review/wireshark 10 | # for speed. The 1b3cedbc5fe5b9d8b454a10fcd2046f0d38a9f19 == tags/wireshark-2.6.2 11 | # We do the fetch SHA rather than clone since the repo is big. 12 | RUN echo deb-src http://archive.ubuntu.com/ubuntu/ bionic-updates universe >> /etc/apt/sources.list \ 13 | && apt-get update \ 14 | && apt-get -y install --no-install-recommends git build-essential ca-certificates libncurses5-dev \ 15 | && apt-get -y build-dep wireshark-common \ 16 | && gcc -o /usr/local/bin/pause /pause.c 17 | 18 | RUN git clone https://github.com/donbowman/liboping \ 19 | && cd liboping \ 20 | && git checkout reset-count \ 21 | && ./autogen.sh \ 22 | && ./configure --enable-static --disable-shared --with-ncurses --prefix=/usr \ 23 | && make \ 24 | && make install 25 | 26 | RUN mkdir -p wireshark/build \ 27 | && cd wireshark \ 28 | && git init \ 29 | && git remote add origin https://github.com/wireshark/wireshark \ 30 | && git fetch origin 1b3cedbc5fe5b9d8b454a10fcd2046f0d38a9f19 \ 31 | && git reset --hard FETCH_HEAD 32 | RUN cd wireshark/build \ 33 | && cmake -DENABLE_STATIC=1 -DBUILD_dumpcap=ON \ 34 | -DENABLE_LUA=OFF \ 35 | -DENABLE_GNUTLS=OFF \ 36 | -DENABLE_NGHTTP2=OFF \ 37 | -DENABLE_SMI=OFF \ 38 | -DENABLE_KERBEROS=OFF \ 39 | -DENABLE_SBC=OFF \ 40 | -DENABLE_SPANDSP=OFF \ 41 | -DENABLE_BCG729=OFF \ 42 | -DENABLE_LIBXML2=OFF \ 43 | -DBUILD_wireshark=OFF \ 44 | -DBUILD_tshark=OFF \ 45 | -DBUILD_tfshark=OFF \ 46 | -DBUILD_rawshark=OFF \ 47 | -DBUILD_text2pcap=OFF \ 48 | -DBUILD_mergecap=OFF \ 49 | -DBUILD_reordercap=OFF \ 50 | -DBUILD_editcap=OFF \ 51 | -DBUILD_capinfos=OFF \ 52 | -DBUILD_captype=OFF \ 53 | -DBUILD_randpkt=OFF \ 54 | -DBUILD_dftest=OFF \ 55 | -DBUILD_corbaidl2wrs=OFF \ 56 | -DBUILD_dcerpcidl2wrs=OFF \ 57 | -DBUILD_xxx2deb=OFF \ 58 | -DBUILD_androiddump=OFF \ 59 | -DBUILD_sshdump=OFF \ 60 | -DBUILD_ciscodump=OFF \ 61 | -DBUILD_dpauxmon=OFF \ 62 | -DBUILD_randpktdump=OFF \ 63 | -DBUILD_udpdump=OFF \ 64 | -DBUILD_sharkd=OFF .. \ 65 | && make -j $(getconf _NPROCESSORS_ONLN) dumpcap \ 66 | && cp -r run/dumpcap /usr/local/bin/dumpcap \ 67 | && chmod a=rx /usr/local/bin/dumpcap \ 68 | && strip /usr/local/bin/dumpcap 69 | 70 | FROM golang:1.10-stretch as crictl 71 | RUN mkdir -p /go/bin /go/src/github.com/kubernetes-incubator \ 72 | && cd /go/src/github.com/kubernetes-incubator \ 73 | && git clone https://github.com/kubernetes-incubator/cri-tools \ 74 | && cd cri-tools \ 75 | && git checkout 3df9c005e3e812dfb933867ae31843bc61969f63 \ 76 | && make \ 77 | && make install 78 | 79 | FROM ubuntu:18.04 80 | COPY --from=wireshark /usr/local/bin/dumpcap /usr/local/bin/dumpcap 81 | COPY --from=wireshark /usr/local/bin/pause /usr/local/bin/pause 82 | COPY --from=wireshark /usr/bin/oping /usr/bin/oping 83 | COPY --from=wireshark /usr/bin/noping /usr/bin/noping 84 | 85 | COPY --from=crictl /usr/local/bin/crictl /usr/local/bin/crictl 86 | COPY sha256sums sha256sums 87 | ENV LANG en_CA.UTF-8 88 | ENV LC_ALL en_CA.UTF-8 89 | ENV DEBIAN_FRONTEND noninteractive 90 | 91 | RUN apt-get update \ 92 | && apt-get -y install --no-install-recommends \ 93 | locales util-linux python3 hping3 fping ca-certificates build-essential python3-dev python3-distutils \ 94 | inetutils-ping iproute2 curl tcpdump libpcap0.8 libglib2.0-0 libnl-3-200 libnl-genl-3-200 libpcre3 \ 95 | zlib1g libcap2 gdb strace iptables tcpflow net-tools lsof vim gawk netcat-openbsd \ 96 | && curl -fLs https://bootstrap.pypa.io/3.3/get-pip.py > get-pip.py \ 97 | && sha256sum -c sha256sums \ 98 | && python3 get-pip.py \ 99 | && rm -rf /var/lib/apt/lists/* \ 100 | && locale-gen en_CA.UTF-8 101 | 102 | CMD /usr/local/bin/pause 103 | WORKDIR /root 104 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## container-diagnostic-tools 2 | 3 | ![img](img/endoscope.png) 4 | 5 | Endoscope: snoop around inside your Kubernetes pods. 6 | 7 | - Debug them 8 | - Ping from them to others 9 | - Wireshark from them 10 | - tcpdump 11 | - and more 12 | 13 | At [Agilicus](https://www.agilicus.com) we use this to diagnose 14 | service mesh issues, east-west calls, etc. Its more efficient 15 | than rebuilding your containers to have root and diagnostics tools, 16 | its simpler than redoing your yaml to add a sidecar. 17 | 18 | Endoscope has a companion, [dink](https://www.agilicus.com/snooping-on-your-kubernetes-nodes-containers-without-sshing-to-it-dink/) 19 | which allows access to the CRI (e.g. docker image ls etc) on a node, without ssh'ing to it. 20 | 21 | ## Usage 22 | 23 | This repo contains two components: 24 | 25 | 1. A Python script (scope) which will launch a utility container 26 | into the namespace of a running pod, and then perform commands 27 | such as ping other pods or debug or capture within it. 28 | 2. A utility container (utilities/endoscope) which is a Ubuntu 18.04 29 | image with gdb/tcpdump/dumpcap/ping/hping/curl installed in it. 30 | 31 | Overal usage. This requires a source namespace/pod to attadch to. 32 | By default it launches a single new container, leaving it running 33 | for better interactive performance. To terminate at the end 34 | of each command, use '--terminate'. To cleanup all, use the 'cleanup' 35 | command. 36 | 37 | By default this assumes the utility container comes from a private 38 | registry, and that registry credentials named 'regcred' exist. If 39 | not, use '--regcred ""' to override. 40 | 41 | ``` 42 | kubectl create secret -n NAMESPACE docker-registry regcred --docker-server=SERVER --docker-username="USER" --docker-password="PASS" 43 | ``` 44 | 45 | Use scope --help for up-to-date arguments. Current global arguments: 46 | 47 | ``` 48 | -h, --help show this help message and exit 49 | -i IMAGE, --image IMAGE 50 | Image to scope with 51 | -n NAMESPACE, --namespace NAMESPACE 52 | Source namespace 53 | -p POD, --pod POD Source pod 54 | -c REGCRED, --regcred REGCRED 55 | Registry credentials, if private 56 | -t, --terminate Terminate (do not cache) debug pod (e.g. terminate 57 | each time) 58 | ``` 59 | 60 | ### Ping 61 | 62 | This will ping either a single, or all, pods in a given namespace, 63 | from another pod/namespace combination. The output can be graphical 64 | (-g) or not. 65 | 66 | ``` 67 | -h, --help show this help message and exit 68 | -d DEST_POD, --dest-pod DEST_POD 69 | Destination pod 70 | -N DEST_NAMESPACE, --dest-namespace DEST_NAMESPACE 71 | Destination pod namespace 72 | -c COUNT, --count COUNT 73 | Count of pings to send 74 | -i INTERVAL, --interval INTERVAL 75 | Interval of pings to send 76 | -g, --graph Graph result 77 | -a, --all Ping all in namespace 78 | ``` 79 | 80 | Ping all pods in a namespace, from a given one, in graphical form: 81 | `scope -n NAMESPACE -p SRC-POD ping -c0 -a -g` 82 | ![img](img/graph-ping.png) 83 | 84 | ### Hping 85 | 86 | hping3 a pod in a namespace, from a given one. This allows 87 | TCP, UDP as options instead of just ICMP. 88 | 89 | `scope -n NAMESPACE -shop -p POD hping -N DESTNAMESPACE -d DESTPOD -- -c 1` 90 | 91 | After the --, all arguments are passed to hping3. See hping3(8) 92 | 93 | ### Shell 94 | 95 | Obtain a shell in the network namespace of a given pod: 96 | `scope -n NAMESPACE -p SRC-POD shell` 97 | 98 | Run 'ip address' command in the network namespace of a given pod: 99 | `scope -n NAMESPACE -p SRC-POD shell ip address` 100 | 101 | ### Launch 102 | 103 | Launch a debug pod and leave it running, with access to the PID/network 104 | namespace of a given pod: 105 | `scope -n NAMESPACE -p SRC-POD launch` 106 | 107 | ### Cleanup 108 | 109 | Cleanup a/all the debug-* pods in a namespace. 110 | 111 | `scope -n NAMESPACE -p all cleanup` -- remove all the debug-* 112 | `scope -n NAMESPACE -p POD cleanup` -- remove the debug-POD 113 | 114 | The above commands, unless run with '-t' for terminate, leave the 115 | debug- pod around after starting (for better interactive performance). 116 | 117 | ### Pids 118 | 119 | Obtain the list of pids (in host coordinate terms) for the given 120 | container. This is useful to e.g. attach gdb or strace: 121 | 122 | `scope -n NAMESPACE -p POD pids` 123 | 124 | ### Strace 125 | 126 | strace a pid (default first pid in Pod). 127 | 128 | `scope -n NAMESPACE -p POD strace [-p #] [-e expr]` 129 | 130 | If 'expr' is specified, show only those syscalls (see strace(1)). 131 | If -p # is specified (e.g. as the result of the 'pids' command), use 132 | this pid instead of the first. 133 | 134 | ### gdb 135 | 136 | Debug (gdb) a pid (default first pid in Pod). 137 | 138 | `scope -n NAMESPACE -p POD gdb [-p #]` 139 | 140 | Runs the debugger, privileged, in the pid namespace, attached. 141 | 142 | ## Container 143 | 144 | The container here (agilicus/endoscope) contains tools to 145 | 146 | - cross ping / tcp / udp connectivity check 147 | - capture traffic 148 | - inject traffic 149 | - debug processes 150 | 151 | This is normally run as a privileged container on the 152 | node, mounting the network namespaces. It can then, 153 | given another Pod name, 'enter' it. It works in conjuncton 154 | with the python script 'scope'. 155 | 156 | The motivation is to run this under Kubernetes as: 157 | 158 | ``` 159 | --- 160 | apiVersion: v1 161 | kind: Pod 162 | metadata: 163 | name: debug 164 | spec: 165 | nodeName: XXXX 166 | hostPID: true 167 | imagePullSecrets: 168 | - name: regcred 169 | containers: 170 | - name: debug 171 | securityContext: 172 | privileged: true 173 | image: agilicus/endoscope:latest 174 | volumeMounts: 175 | - mountPath: /var/run/cri.sock 176 | name: crisock 177 | - mountPath: /run/docker/netns 178 | name: netns 179 | volumes: 180 | - hostPath: 181 | path: /var/run/dockershim.sock 182 | type: "" 183 | name: crisock 184 | - hostPath: 185 | path: /var/run/docker/netns 186 | type: "" 187 | name: netns 188 | ``` 189 | 190 | and thus attach to a Pod in XXXX w/ 'nsenter -n -t '. 191 | 192 | ## Use within istio and outbound (egress) firewall 193 | 194 | If you are attaching this tool to a pod which is managed by an istio 195 | sidecar, you may find that outbound network access is blocked. If this 196 | is a problem you can run: 197 | 198 | ``` 199 | iptables -t NAT -D OUTPUT -p tcp -j ISTIO_OUTPUT 200 | ``` 201 | 202 | as a temporary means of enabling output access. 203 | 204 | ## Filesystem access 205 | To find which overlay mount is the guest filesystem (e.g. to find a file), 206 | on the guest run 'ls -i ', which gives you the inode. Then, in 207 | endoscope, run debugfs: 208 | 209 | On debugee: 210 | ``` 211 | bash-4.4$ ls -i usr/local/lib/python3.7/http/client.py 212 | 3601465 usr/local/lib/python3.7/http/client.py 213 | ``` 214 | 215 | On endoscope: 216 | ``` 217 | # df -lh /var/lib/docker/overlay2/ 218 | Filesystem Size Used Avail Use% Mounted on 219 | /dev/sda1 30G 22G 7.2G 76% /var/lib/docker 220 | root@debug-dashboard-superset-d4cd75b78-mswbl:/var/lib# debugfs -R 'ncheck 3601465' /dev/sda1 2>/dev/null 221 | Inode Pathname 222 | 3601465 /var/lib/docker/overlay2/c509876fb71d801e4398f7c11cc9d4458dc30f6f6334301d3e51704037cadd68/diff/usr/local/lib/python3.7/http/client.py 223 | ``` 224 | 225 | ## License 226 | 227 | The container is released under Apache 2.0 license. 228 | The individuals files within it vary. 229 | -------------------------------------------------------------------------------- /LICENSE-2.0: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /scope: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # pip install -r requirements.txt 3 | 4 | # 5 | # We have a new container with some tools (ping et al) 6 | # We run it as a privileged Pod, mounting CRI + netns. 7 | # Inside it we find the pid of the 'source', and from 8 | # this we can do a 'nsenter -n -t ', putting us 9 | # in the same network namespace. 10 | # 11 | # Now we can 'tcpdump to stdout' 12 | # 13 | # See extcap for how to integrate with wireshark. 14 | # e.g. /usr/lib/x86_64-linux-gnu/wireshark/extcap/sshdump 15 | import argparse 16 | import logging 17 | import yaml 18 | import re 19 | import os 20 | import sys 21 | import json 22 | import time 23 | from kubernetes import client, config 24 | from kubernetes.stream import stream 25 | from kubernetes.stream.ws_client import ERROR_CHANNEL 26 | 27 | if os.path.basename(sys.argv[0]) == 'k8scap': 28 | sys.argv.insert(1, '--pod') 29 | sys.argv.insert(2, 'None') 30 | sys.argv.insert(3, 'dumpcap') 31 | 32 | parser = argparse.ArgumentParser() 33 | parser.add_argument('-d', '--debug', help='Debug API stream', default = False, action = 'store_true') 34 | parser.add_argument('-i', '--image', help='Image to scope with', default = 'agilicus/endoscope') 35 | parser.add_argument('-n', '--namespace', help='Source namespace', default = 'default') 36 | parser.add_argument('-p', '--pod', help='Source pod', default = '', required = True) 37 | parser.add_argument('-c', '--regcred', help='Registry credentials, if private', default = '') 38 | parser.add_argument('-t', '--terminate', help='Terminate (do not cache) debug pod (e.g. terminate each time)', action = 'store_true', default = False) 39 | subparsers = parser.add_subparsers(help='command',dest='command') 40 | ## Ping 41 | parser_ping = subparsers.add_parser('ping', help='ping help') 42 | parser_ping.add_argument('-d', '--dest-pod', help='Destination pod') 43 | parser_ping.add_argument('-N', '--dest-namespace', help='Destination pod namespace', default = 'default') 44 | parser_ping.add_argument('-c', '--count', help='Count of pings to send', default = '1') 45 | parser_ping.add_argument('-C', '--loopcount', help='Count of pings to send before new flow/ICMP ID', default = '5') 46 | parser_ping.add_argument('-i', '--interval', help='Interval of pings to send', default = '0.1') 47 | parser_ping.add_argument('-g', '--graph', help='Graph result', action = 'store_true', default = False) 48 | parser_ping.add_argument('-a', '--all', help='Ping all in namespace (pod ip)', action = 'store_true', default = False) 49 | parser_ping.add_argument('-H', '--host', help='Ping all in namespace (host ip)', action = 'store_true', default = False) 50 | ## Shell 51 | parser_shell = subparsers.add_parser('shell', help='shell help') 52 | parser_shell.add_argument('args', nargs=argparse.REMAINDER) 53 | ## Launch 54 | parser_launch = subparsers.add_parser('launch', help='launch help') 55 | ## Cleanup 56 | parser_cleanup = subparsers.add_parser('cleanup', help='cleanup. If source-pod == all, delete all debug- in namespace.') 57 | ## pids 58 | parser_pids = subparsers.add_parser('pids', help='Show pids in namespace') 59 | ## strace 60 | parser_strace = subparsers.add_parser('strace', help='Show syscalls of pid in pod') 61 | parser_strace.add_argument('-p', '--pid', help='Override first pid (see pids commmand)', default = '') 62 | parser_strace.add_argument('-e', '--expr', help='Set the strace filter (-e) expression, e.g. -e file', default = '') 63 | ## gdb 64 | parser_gdb = subparsers.add_parser('gdb', help='Attach gdb to pid') 65 | parser_gdb.add_argument('-p', '--pid', help='Override first pid (see pids commmand)', default = '') 66 | ## hping 67 | parser_hping = subparsers.add_parser('hping', help='hping help') 68 | parser_hping.add_argument('-d', '--dest-pod', help='Destination pod') 69 | parser_hping.add_argument('-N', '--dest-namespace', help='Destination pod namespace', default = 'default') 70 | parser_hping.add_argument('args', nargs=argparse.REMAINDER) 71 | ## dumpcap 72 | parser_hping = subparsers.add_parser('dumpcap', help='dumpcap help') 73 | parser_hping.add_argument('--extcap-interfaces', help='extcap interfaces', action = 'store_true') 74 | parser_hping.add_argument('--extcap-interface', help='extcap interface') 75 | parser_hping.add_argument('--extcap-capture-filter', help='extcap-capture-filter') 76 | parser_hping.add_argument('--extcap-dlts-interface', help='extcap dlts', action = 'store_true') 77 | parser_hping.add_argument('--extcap-config', help='extcap-config', action = 'store_true') 78 | parser_hping.add_argument('--capture-config', help='config', action = 'store_true') 79 | parser_hping.add_argument('--capture', help='capture', action = 'store_true') 80 | parser_hping.add_argument('--filter', help='filter', default='') 81 | parser_hping.add_argument('--fifo', help='fifo') 82 | parser_hping.add_argument('args', nargs=argparse.REMAINDER) 83 | 84 | args = parser.parse_args() 85 | 86 | template = """ 87 | apiVersion: v1 88 | metadata: 89 | name: {name} 90 | spec: 91 | nodeName: {node_name} 92 | hostPID: true 93 | restartPolicy: Never 94 | {imagePullSecrets} 95 | containers: 96 | - name: {name} 97 | imagePullPolicy: Always 98 | securityContext: 99 | privileged: true 100 | capabilities: 101 | add: ["SYS_PTRACE", "NET_ADMIN"] 102 | image: {image} 103 | volumeMounts: 104 | - mountPath: /var/run/dockershim.sock 105 | name: crisock 106 | - mountPath: /run/docker/netns 107 | name: netns 108 | - mountPath: /var/lib/docker 109 | name: dockerlib 110 | env: 111 | - name: "LINES" 112 | value: "{lines}" 113 | - name: "COLUMNS" 114 | value: "{columns}" 115 | - name: "TERM" 116 | value: "{term}" 117 | volumes: 118 | - hostPath: 119 | path: /var/run/dockershim.sock 120 | type: "" 121 | name: crisock 122 | - hostPath: 123 | path: /var/run/docker/netns 124 | type: "" 125 | name: netns 126 | - hostPath: 127 | path: /var/lib/docker 128 | type: "" 129 | name: dockerlib 130 | """ 131 | 132 | def findFirstPid(name, args, spod, api): 133 | # This is a bit gross, I'm not sure a better way. We have the cgroup-name 134 | # from 'container_statuses': [{'container_id': 'docker://5f5f3135d5ff2300ea478704a774ed758b5c476888ff881bfa178436693ea410'...] 135 | # and, this is references in /cgroup. There could be more than 1 pid in that cgroup, 136 | # but we just care about the first since we use it to enter the netns. 137 | # So, we run grep -l CGROUP /proc/*/cgroup, and then split the result. 138 | docker_name = spod.status.container_statuses[0].container_id 139 | cgroup = re.sub("^docker://","", docker_name) 140 | cmd = ['/bin/sh', '-c','/bin/grep -l pids:/.*%s /proc/*/cgroup' % cgroup] 141 | line = stream(api.connect_get_namespaced_pod_exec, name=name, container=name, namespace=spod.metadata.namespace, command=cmd, stderr=True, stdin=False, stdout=True, tty=False) 142 | _pids = list(filter(None, str.split(line, '\n'))) 143 | pidlist = [] 144 | for pid in _pids: 145 | pid = re.sub("[^[0-9]*","", pid) 146 | if pid != None: 147 | pidlist.append(pid) 148 | pidlist.sort(key=int) 149 | return pidlist[0] 150 | 151 | def runit(name, args, spod, cmd): 152 | pid = findFirstPid(name, args, spod, api) 153 | ns_command = ['/usr/bin/nsenter', '-n', '-t', pid] + cmd 154 | resp = stream(api.connect_get_namespaced_pod_exec, container=name, name=name, namespace=spod.metadata.namespace, command=ns_command, stderr=True, stdin=False, stdout=True, tty=False, _preload_content=False) 155 | 156 | rc = -1 157 | while resp.is_open(): 158 | resp.update(timeout=1) 159 | if resp.peek_stdout(): 160 | print(resp.read_stdout(), end='') 161 | if resp.peek_stderr(): 162 | print(resp.read_stderr(), file=sys.stderr, end='') 163 | if resp.peek_channel(ERROR_CHANNEL): 164 | status = json.loads(resp.read_channel(ERROR_CHANNEL)) 165 | if status['status'] == 'Success': 166 | rc = 0 167 | else: 168 | rc = int(status['details']['causes'][0]['message']) 169 | resp.close() 170 | return rc 171 | 172 | def launch(name, args, spod, api): 173 | try: 174 | columns, lines = os.get_terminal_size() 175 | except: 176 | # e.g. not a tty 177 | columns = 80 178 | lines = 24 179 | imagePullSecrets = "imagePullSecrets:\n - name: %s" % args.regcred if len(args.regcred) else "" 180 | ym = template.format(name=name, node_name=spod.spec.node_name, image=args.image, lines=lines, columns=columns, term=os.getenv('TERM'), imagePullSecrets=imagePullSecrets) 181 | sm = yaml.load(ym) 182 | try: 183 | api.create_namespaced_pod(namespace=spod.metadata.namespace, body=sm) 184 | while True: 185 | resp = api.read_namespaced_pod(namespace=spod.metadata.namespace, name=name) 186 | if resp.status.phase != 'Pending': 187 | break 188 | time.sleep(0.1) 189 | except client.rest.ApiException as e: 190 | if e.status == 409: 191 | # Conflict, e.g. exists, we'll use it 192 | pass 193 | else: 194 | raise e 195 | return 0 196 | 197 | def shell(name, args, spod, api): 198 | launch(name, args, spod, api) 199 | # As a special case, if args is empty, make an interactive shell 200 | # if we are a tty 201 | if len(args.args) == 0 and os.isatty(0): 202 | pid = findFirstPid(name, args, spod, api) 203 | rc = os.system("kubectl -n %s exec -it %s -- /usr/bin/nsenter -n -t %s bash" % (spod.metadata.namespace, name, pid)) 204 | rc = (rc & 0xff00) >> 8 205 | else: 206 | cmd = args.args 207 | rc = runit(name, args, spod, cmd) 208 | if args.terminate == True: 209 | body = client.V1DeleteOptions() 210 | api.delete_namespaced_pod(namespace=spod.metadata.namespace, name=name, body=body) 211 | return rc 212 | 213 | def ping(name, args, spod, api): 214 | launch(name, args, spod, api) 215 | ips = [] 216 | cmd = ['/usr/bin/noping' if args.graph else '/usr/bin/oping', '-c', args.count, '-C', args.loopcount, '-i', args.interval] 217 | if args.all: 218 | spods = api.list_namespaced_pod(watch=False, namespace=args.dest_namespace) 219 | for _spod in spods.items: 220 | cmd.append(_spod.status.pod_ip) if not args.host else cmd.append(_spod.status.host_ip) 221 | else: 222 | spods = api.list_namespaced_pod(watch=False, namespace=args.dest_namespace, field_selector='metadata.name=%s' % args.dest_pod ) 223 | for _spod in spods.items: 224 | cmd.append(_spod.status.pod_ip) 225 | rc = runit(name, args,spod,cmd) 226 | if args.terminate == True: 227 | body = client.V1DeleteOptions() 228 | api.delete_namespaced_pod(namespace=spod.metadata.namespace, name=name, body=body) 229 | return rc 230 | 231 | def cleanup(name, args, spod, api): 232 | if spod == None: 233 | dpods = api.list_namespaced_pod(watch=False, namespace=args.namespace).items 234 | else: 235 | dpods = api.list_namespaced_pod(watch=False, namespace=args.namespace, field_selector = 'metadata.name=debug-%s' % spod.metadata.name).items 236 | for pod in dpods: 237 | body = client.V1DeleteOptions() 238 | if re.match("^debug-", pod.metadata.name): 239 | print("Delete %s/%s" % (pod.metadata.namespace, pod.metadata.name)) 240 | api.delete_namespaced_pod(namespace=pod.metadata.namespace, name=pod.metadata.name, body=body) 241 | return 0 242 | 243 | def pids(name, args, spod, api): 244 | launch(name, args, spod, api) 245 | docker_name = spod.status.container_statuses[0].container_id 246 | cgroup = re.sub("^docker://","", docker_name) 247 | cmd = ['/bin/sh', '-c','/bin/grep pids:/.*%s /proc/*/cgroup' % cgroup] 248 | 249 | line = stream(api.connect_get_namespaced_pod_exec, container=name, name=name, namespace=spod.metadata.namespace, command=cmd, stderr=True, stdin=False, stdout=True, tty=False) 250 | _pids = list(filter(None, str.split(line, '\n'))) 251 | pids = [] 252 | for pid in _pids: 253 | pid = re.sub("/cgroup.*","", pid) 254 | pid = re.sub("[^[0-9]*","", pid) 255 | pids.append(pid) 256 | print(' '.join(pids)) 257 | return 0 258 | 259 | def strace(name, args, spod, api): 260 | launch(name, args, spod, api) 261 | if len(args.pid) == 0: 262 | args.pid = findFirstPid(name, args, spod, api) 263 | cmd = ['/usr/bin/strace', '-p', args.pid] 264 | if len(args.expr): 265 | cmd.append ( '-e' ) 266 | cmd.append ( args.expr ) 267 | return runit(name, args, spod, cmd) 268 | 269 | def gdb(name, args, spod, api): 270 | launch(name, args, spod, api) 271 | if len(args.pid) == 0: 272 | args.pid = findFirstPid(name, args, spod, api) 273 | return os.system("kubectl -n %s exec -it %s -- /usr/bin/nsenter -p -t %s /usr/bin/gdb -p %s" % (spod.metadata.namespace, name, args.pid, args.pid)) 274 | 275 | def hping(name, args, spod, api): 276 | launch(name, args, spod, api) 277 | 278 | if len(args.args) and args.args[0] == '--': 279 | args.args = args.args[1:] 280 | 281 | cmd = ['/usr/sbin/hping3'] + args.args 282 | 283 | spods = api.list_namespaced_pod(watch=False, namespace=args.dest_namespace, field_selector='metadata.name=%s' % args.dest_pod ) 284 | for _spod in spods.items: 285 | cmd.append(_spod.status.pod_ip) 286 | 287 | rc = runit(name, args,spod,cmd) 288 | if args.terminate == True: 289 | body = client.V1DeleteOptions() 290 | api.delete_namespaced_pod(namespace=spod.metadata.namespace, name=name, body=body) 291 | return rc 292 | 293 | # 1. k8scap --extcap-interfaces 294 | # 2. k8scap --extcap-config --extcap-interface k8scap 295 | # 3. k8scap --extcap-dlts --extcap-interface k8scap 296 | # 4. k8scap --extcap-config --extcap-interface k8scap 297 | # 5. k8scap --capture --extcap-interface k8scap 298 | # dumpcap -n -i /tmp/xxx -Z none 299 | def dumpcap(name, args, spod, api): 300 | #["python3", "/usr/lib/x86_64-linux-gnu/wireshark/extcap/k8scap", "--capture", "--extcap-interface", "default/ingress-nginx-ingress-controller-7b66cb4878-x7wwb", "--fifo", "/tmp/wireshark_extcap_default-ingress-nginx-ingress-controller-7b66cb4878-x7wwb_20180813162059_RCnP40"], 0x7ffd92b919e0 /* 59 vars */) = 0 301 | 302 | if args.capture: 303 | namespace = args.extcap_interface.split('/')[0] 304 | pod = args.extcap_interface.split('/')[1] 305 | spods = api.list_namespaced_pod(watch=False, namespace=namespace, field_selector='metadata.name=%s' % pod) 306 | spod = spods.items[0] 307 | name = 'debug-%s' % spod.metadata.name 308 | launch(name, args, spod, api) 309 | pid = findFirstPid(name, args, spod, api) 310 | command = ['/bin/sh', '-c', '/usr/bin/nsenter -n -t %s /usr/local/bin/dumpcap -f \'%s\' -w - q 2>/dev/null' % (pid,args.filter)] 311 | 312 | resp = stream(api.connect_get_namespaced_pod_exec, container=name, name=name, namespace=spod.metadata.namespace, command=command, stderr=True, stdin=False, stdout=True, tty=False, _preload_content=False) 313 | 314 | fd = open(args.fifo, "wb") 315 | 316 | while resp.is_open(): 317 | resp.update(timeout=10) 318 | if resp.peek_stdout(): 319 | ln = resp.read_stdout() 320 | fd.write(ln) 321 | if resp.peek_stderr(): 322 | print(resp.read_stderr(), file=sys.stderr, end='') 323 | if resp.peek_channel(ERROR_CHANNEL): 324 | status = json.loads(resp.read_channel(ERROR_CHANNEL)) 325 | if status['status'] == 'Success': 326 | rc = 0 327 | else: 328 | rc = int(status['details']['causes'][0]['message']) 329 | fd.close() 330 | resp.close() 331 | elif args.extcap_config and len(args.extcap_interface): 332 | print("arg {number=0}{call=--filter}{type=string}{default=''}{display=Capture filter}") 333 | elif args.extcap_config: 334 | print("arg {number=0}{call=--namespace}{display=namespace}{type=string}{tooltip='*|all|namespace'}") 335 | print("arg {number=1}{call=--pod}{display=pod}{type=string}{default='*'}{tooltip='*|pod'}") 336 | print("arg {number=2}{call=dumpcap}{display=dumpcap}") 337 | print("arg {number=3}{call=--remote-filter}{display=remote filter}") 338 | elif args.extcap_interfaces: 339 | print("k8scap {version=0.1.0}{help=https://git.agilicus.com/utilities/endoscope}") 340 | spods = api.list_pod_for_all_namespaces(watch=False) 341 | for spod in spods.items: 342 | print("interface {value=%s/%s}{display=Capture on ns:%s, pod:%s}" % (spod.metadata.namespace, spod.metadata.name, spod.metadata.namespace,spod.metadata.name)) 343 | 344 | ####### 345 | 346 | config.debug = True 347 | config.load_kube_config() 348 | api = client.CoreV1Api() 349 | 350 | if args.debug: 351 | import websocket 352 | kslogger = logging.getLogger('kubernetes') 353 | urllogger = logging.getLogger('urllib3') 354 | clientlogger = logging.getLogger('client') 355 | websocketlogger = logging.getLogger('websocket') 356 | console_h= logging.StreamHandler() 357 | kslogger.addHandler(console_h) 358 | kslogger.setLevel(logging.DEBUG) 359 | clientlogger.setLevel(logging.DEBUG) 360 | websocketlogger.setLevel(logging.DEBUG) 361 | websocket.enableTrace(True, console_h) 362 | 363 | # Cleanup can accept 'all', which doesn't match 364 | if args.command == 'dumpcap' or (args.command == 'cleanup' and args.pod == 'all'): 365 | spod = None 366 | name = None 367 | else: 368 | spods = api.list_namespaced_pod(watch=False, namespace=args.namespace, field_selector='metadata.name=%s' % args.pod) 369 | if len(spods.items) == 0: 370 | print("Error: %s/%s does not specify a valid pod" % (args.namespace, args.pod), file=sys.stderr) 371 | sys.exit(1) 372 | spod = spods.items[0] 373 | name = 'debug-%s' % spod.metadata.name 374 | 375 | r = locals()[args.command](name, args, spod, api) 376 | sys.exit(r) 377 | --------------------------------------------------------------------------------