├── .github └── workflows │ └── docker-publish.yml ├── LICENSE ├── README.md ├── docker ├── Dockerfile └── proxy.py ├── docker_swarm_proxy.py └── service-exec ├── Dockerfile └── service_exec.py /.github/workflows/docker-publish.yml: -------------------------------------------------------------------------------- 1 | name: Docker 2 | 3 | # This workflow uses actions that are not certified by GitHub. 4 | # They are provided by a third-party and are governed by 5 | # separate terms of service, privacy policy, and support 6 | # documentation. 7 | 8 | on: 9 | schedule: 10 | - cron: '0 0 * * *' 11 | push: 12 | branches: [ "master" ] 13 | # Publish semver tags as releases. 14 | tags: [ '*.*.*' ] 15 | pull_request: 16 | branches: [ "master" ] 17 | 18 | env: 19 | # Use docker.io for Docker Hub if empty 20 | REGISTRY: ghcr.io 21 | 22 | 23 | jobs: 24 | build: 25 | strategy: 26 | fail-fast: false 27 | matrix: 28 | program: 29 | - docker 30 | - service-exec 31 | 32 | 33 | runs-on: ubuntu-latest 34 | permissions: 35 | contents: read 36 | packages: write 37 | # This is used to complete the identity challenge 38 | # with sigstore/fulcio when running outside of PRs. 39 | id-token: write 40 | 41 | steps: 42 | - name: Checkout repository 43 | uses: actions/checkout@v3 44 | 45 | - name: Set up QEMU 46 | uses: docker/setup-qemu-action@v2 47 | 48 | # Install the cosign tool except on PR 49 | # https://github.com/sigstore/cosign-installer 50 | - name: Install cosign 51 | if: github.event_name != 'pull_request' 52 | uses: sigstore/cosign-installer@f3c664df7af409cb4873aa5068053ba9d61a57b6 #v2.6.0 53 | with: 54 | cosign-release: 'v1.11.0' 55 | 56 | 57 | # Workaround: https://github.com/docker/build-push-action/issues/461 58 | - name: Setup Docker buildx 59 | uses: docker/setup-buildx-action@79abd3f86f79a9d68a23c75a09a9a85889262adf 60 | 61 | # Login against a Docker registry except on PR 62 | # https://github.com/docker/login-action 63 | - name: Log into registry ${{ env.REGISTRY }} 64 | if: github.event_name != 'pull_request' 65 | uses: docker/login-action@28218f9b04b4f3f62068d7b6ce6ca5b26e35336c 66 | with: 67 | registry: ${{ env.REGISTRY }} 68 | username: ${{ github.actor }} 69 | password: ${{ secrets.GITHUB_TOKEN }} 70 | 71 | # Extract metadata (tags, labels) for Docker 72 | # https://github.com/docker/metadata-action 73 | - name: Extract Docker metadata 74 | id: meta 75 | uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 76 | with: 77 | images: ${{ env.REGISTRY }}/${{ github.repository }}/${{ matrix.program }} 78 | 79 | # Build and push Docker image with Buildx (don't push on PR) 80 | # https://github.com/docker/build-push-action 81 | - name: Build and push Docker image 82 | id: build-and-push 83 | uses: docker/build-push-action@ac9327eae2b366085ac7f6a2d02df8aa8ead720a 84 | with: 85 | context: ./${{ matrix.program }} 86 | platforms: linux/amd64,linux/arm/v6,linux/arm64 87 | push: ${{ github.event_name != 'pull_request' }} 88 | tags: ${{ steps.meta.outputs.tags }} 89 | labels: ${{ steps.meta.outputs.labels }} 90 | cache-from: type=gha 91 | cache-to: type=gha,mode=max 92 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 NeuroForge GmbH & Co. KG 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # docker-swarm-proxy 2 | 3 | What if you wanted a docker exec, but for Docker swarm? 4 | 5 | A problem with Docker Swarm and automation with it has been that you can't directly exec into any service from the command line. There exist some workarounds to achieve this behaviour, but in the end you want something similar and as convenient as `docker exec` but for services. 6 | 7 | ![grafik](https://github.com/neuroforgede/docker-swarm-proxy/assets/719760/e40aae96-1b0f-4193-8f7e-1054b5db6a6e) 8 | 9 | ## Installation 10 | 11 | ### Prerequisites 12 | 13 | Install docker-py and click: 14 | 15 | ```bash 16 | pip3 install docker 17 | pip3 install click 18 | ``` 19 | 20 | Install the plugin your docker cli (from github) 21 | 22 | ```bash 23 | rm ~/.docker/cli-plugins/docker-swarmproxy 24 | curl -L https://raw.githubusercontent.com/neuroforgede/docker-swarm-proxy/master/docker_swarm_proxy.py -o ~/.docker/cli-plugins/docker-swarmproxy 25 | chmod +x ~/.docker/cli-plugins/docker-swarmproxy 26 | ``` 27 | 28 | Or copy from a local copy of this repo: 29 | 30 | ```bash 31 | cp docker_swarm_proxy.py ~/.docker/cli-plugins/docker-swarmproxy 32 | chmod +x ~/.docker/cli-plugins/docker-swarmproxy 33 | ``` 34 | 35 | ## Usage 36 | 37 | NOTE: For remote clusters, only usage of the DOCKER_HOST environment variable is supported. Usage of Docker Contexts for switching environments is not supported. For remote clusters we strongly advise against exposing the TCP socket directly. Instead use the SSH tunneling support of docker cli as described [here](https://docs.docker.com/engine/security/protect-access/). 38 | 39 | ### Exec into a running service 40 | 41 | ```bash 42 | docker swarmproxy service exec -it vibrant_bell bash 43 | ``` 44 | 45 | See all available options: 46 | 47 | ```bash 48 | docker swarmproxy service exec --help 49 | ``` 50 | 51 | ### Use `-` in the command 52 | 53 | Since swarmproxy uses click under the hood for argument parsing, you have to use `--` before any exec command. 54 | 55 | This will not work: 56 | 57 | ```bash 58 | docker swarmproxy service exec -it vibrant_bell bash -c 'echo hello' 59 | ``` 60 | 61 | This will work: 62 | 63 | ```bash 64 | docker swarmproxy service exec -it vibrant_bell -- bash -c 'echo hello' 65 | ``` 66 | -------------------------------------------------------------------------------- /docker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM --platform=${BUILDPLATFORM:-linux/amd64} docker:24.0.2-cli 2 | RUN apk add python3 py-pip 3 | WORKDIR /code 4 | COPY proxy.py proxy.py 5 | RUN pip3 install docker 6 | RUN pip3 install dnspython 7 | ENTRYPOINT ["python3", "proxy.py"] 8 | -------------------------------------------------------------------------------- /docker/proxy.py: -------------------------------------------------------------------------------- 1 | import docker 2 | import dns.resolver 3 | import os 4 | import sys 5 | 6 | TARGET_HOST = os.environ['TARGET_HOST'] 7 | 8 | answer = dns.resolver.resolve('tasks.docker_swarm_proxy', 'A') 9 | for rdata in answer: 10 | client = docker.DockerClient(base_url=f'tcp://{rdata.address}:2375') 11 | info = client.info() 12 | 13 | if info['Name'] == TARGET_HOST: 14 | new_argv = [*sys.argv] 15 | new_argv[0] = '/usr/local/bin/docker' 16 | os.execvpe('/usr/local/bin/docker', sys.argv, { 17 | 'DOCKER_HOST': f'tcp://{rdata.address}:2375' 18 | }) 19 | -------------------------------------------------------------------------------- /docker_swarm_proxy.py: -------------------------------------------------------------------------------- 1 | #!/bin/python3 2 | import random 3 | import string 4 | import subprocess 5 | import os 6 | import docker 7 | import time 8 | import sys 9 | import click 10 | from typing import List 11 | 12 | if len(sys.argv) >= 2: 13 | if sys.argv[1] == 'docker-cli-plugin-metadata': 14 | print(""" 15 | { 16 | "SchemaVersion": "0.1.0", 17 | "Vendor": "Martin Braun", 18 | "Version": "0.0.1", 19 | "ShortDescription": "Docker Swarm Proxy" 20 | } 21 | """) 22 | exit(0) 23 | 24 | if 'swarmproxy' in sys.argv and len(sys.argv) >= 2: 25 | # we need to strip the first agument for click to work 26 | # if we run as a docker cli plugin 27 | sys.argv = sys.argv[1:] 28 | 29 | def get_random_string(length): 30 | return ''.join(random.choice(string.ascii_letters) for i in range(length)) 31 | 32 | random_str = get_random_string(32) 33 | 34 | stack_name = f"docker_swarm_proxy_{random_str}" 35 | network_name = stack_name 36 | proxy_service_name = "srv" 37 | proxy_shell_container_name = f"proxy_shell_{random_str}" 38 | 39 | TEMPLATE = f""" 40 | version: "3.8" 41 | 42 | services: 43 | {proxy_service_name}: 44 | image: tecnativa/docker-socket-proxy 45 | volumes: 46 | - /var/run/docker.sock:/var/run/docker.sock 47 | networks: 48 | docker_swarm_proxy: 49 | environment: 50 | CONTAINERS: 1 51 | SERVICES: 1 52 | SWARM: 1 53 | NODES: 1 54 | NETWORKS: 1 55 | TASKS: 1 56 | VERSION: 1 57 | 58 | AUTH: 1 59 | SECRETS: 1 60 | POST: 1 61 | BUILD: 1 62 | COMMIT: 1 63 | CONFIGS: 1 64 | DISTRIBUTION: 1 65 | EXEC: 1 66 | GRPC: 1 67 | IMAGES: 1 68 | INFO: 1 69 | PLUGINS: 1 70 | SESSION: 1 71 | SYSTEM: 1 72 | VOLUMES: 1 73 | deploy: 74 | mode: global 75 | 76 | networks: 77 | docker_swarm_proxy: 78 | driver: overlay 79 | attachable: true 80 | name: {network_name} 81 | driver_opts: 82 | encrypted: "" 83 | com.docker.network.driver.mtu: "1350" 84 | """ 85 | 86 | 87 | if os.path.isfile("/bin/docker"): 88 | docker_binary = "/bin/docker" 89 | elif os.path.isfile("/usr/bin/docker"): 90 | docker_binary = "/usr/bin/docker" 91 | 92 | @click.group() 93 | def cli() -> None: 94 | pass 95 | 96 | @click.group() 97 | def service() -> None: 98 | """ 99 | Docker Swarm Service Utilities 100 | """ 101 | pass 102 | 103 | @service.command('exec') 104 | @click.option('-i', '--interactive', is_flag=True, show_default=True, default=False, help='Keep STDIN open even if not attached') 105 | @click.option('-t', '--tty', is_flag=True, show_default=True, default=False, help='Allocate a pseudo-TTY') 106 | @click.option('-u', '--user', help='Username or UID (format: "[:]")') 107 | @click.argument('service') 108 | @click.argument('command') 109 | @click.argument('arg', nargs=-1) 110 | def service_exec( 111 | interactive: bool, 112 | tty: bool, 113 | user: string, 114 | service: string, 115 | command: string, 116 | arg: List[str] 117 | ): 118 | """ 119 | Exec into a running service task. 120 | By default chooses the first task. 121 | """ 122 | def get_running_tasks(service): 123 | return [ 124 | task 125 | for task in service.tasks() 126 | if "Spec" in task 127 | and "DesiredState" in task 128 | and task["DesiredState"] == "running" 129 | and "Status" in task 130 | and "State" in task["Status"] 131 | and task["Status"]["State"] == "running" 132 | ] 133 | 134 | needs_cleanup = False 135 | 136 | try: 137 | # force usage of the regular SSH client 138 | # to be able to pick up DOCKER_HOST env var automatically 139 | from_env = docker.from_env(use_ssh_client=True) 140 | 141 | def get_service(name): 142 | # get all service with similar name 143 | services = from_env.services.list(filters={"name": name}) 144 | # exact match required 145 | services = [service for service in services if service.attrs["Spec"]["Name"] == name] 146 | if len(services) != 1: 147 | raise AssertionError(f'did not find exactly one service with name {name}') 148 | return services[0] 149 | 150 | service = get_service(service) 151 | 152 | running_tasks = get_running_tasks(service) 153 | if len(running_tasks) == 0: 154 | raise AssertionError(f"didn't find running task for service {service}") 155 | 156 | running_task = running_tasks[0] 157 | node_id_running_task = running_task["NodeID"] 158 | container_id = running_task["Status"]["ContainerStatus"]["ContainerID"] 159 | 160 | # TODO: dont deploy the service to all nodes, but instead only 161 | # to the one we care about that is running the task 162 | needs_cleanup = True 163 | subprocess.run( 164 | [docker_binary, "stack", "deploy", "-c", "-", stack_name], 165 | env={ 166 | **os.environ, 167 | }, 168 | cwd=os.getcwd(), 169 | input=TEMPLATE.encode('utf-8'), 170 | check=True 171 | ) 172 | 173 | while True: 174 | # wait for proxy service to be there 175 | service = get_service(f'{stack_name}_{proxy_service_name}') 176 | all_tasks = service.tasks() 177 | desired_running = [ 178 | task 179 | for task in all_tasks 180 | if "Spec" in task 181 | and "DesiredState" in task 182 | and task["DesiredState"] == "running" 183 | ] 184 | actually_running = [ 185 | task 186 | for task in desired_running 187 | if "Status" in task 188 | and "State" in task["Status"] 189 | and task["Status"]["State"] == "running" 190 | ] 191 | if len(desired_running) != len(actually_running): 192 | time.sleep(1) 193 | else: 194 | break 195 | 196 | interactive_str = '-i' if interactive else '' 197 | tty_str = '-t' if tty else '' 198 | user_str = user or '' 199 | 200 | docker_flags = [elem for elem in [interactive_str, tty_str] if elem != ''] 201 | 202 | subprocess.run( 203 | [ 204 | docker_binary, 205 | "run", 206 | "--env", f"PROXY_SERVICE_NAME={proxy_service_name}", 207 | "--env", f"CONTAINER_ID={container_id}", 208 | "--env", f"USER_FLAG={user_str}", 209 | "--env", f"IS_TTY={tty_str}", 210 | "--env", f"IS_INTERACTIVE={interactive_str}", 211 | "--env", f"NODE_ID_RUNNING_TASK={node_id_running_task}", 212 | "--name", proxy_shell_container_name, 213 | "--network", network_name, 214 | "--pull", "always", 215 | "--rm", 216 | "--entrypoint", "python3", 217 | *docker_flags, 218 | "ghcr.io/neuroforgede/docker-swarm-proxy/service-exec:master", 219 | "service_exec.py", 220 | command, 221 | *arg 222 | ], 223 | env={ 224 | **os.environ, 225 | }, 226 | cwd=os.getcwd(), 227 | check=True 228 | ) 229 | finally: 230 | if needs_cleanup: 231 | # hack, ensure that the proxy container is dead and removed 232 | subprocess.run( 233 | [docker_binary, "kill", proxy_shell_container_name], 234 | env={ 235 | **os.environ, 236 | }, 237 | cwd=os.getcwd(), 238 | stdout=None, 239 | stderr=None, 240 | check=False, 241 | capture_output=True 242 | ) 243 | # hack, ensure that the proxy container is dead and removed 244 | subprocess.run( 245 | [docker_binary, "rm", proxy_shell_container_name], 246 | env={ 247 | **os.environ, 248 | }, 249 | cwd=os.getcwd(), 250 | stdout=None, 251 | stderr=None, 252 | check=False, 253 | capture_output=True 254 | ) 255 | 256 | # do the proper cleanup of the stack 257 | subprocess.run( 258 | [docker_binary, "stack", "rm", stack_name], 259 | env={ 260 | **os.environ, 261 | }, 262 | cwd=os.getcwd(), 263 | check=True 264 | ) 265 | 266 | 267 | cli.add_command(service) 268 | if __name__ == '__main__': 269 | cli() -------------------------------------------------------------------------------- /service-exec/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM --platform=${BUILDPLATFORM:-linux/amd64} docker:24.0.2-cli 2 | RUN apk add python3 py-pip 3 | WORKDIR /code 4 | COPY service_exec.py service_exec.py 5 | RUN pip3 install docker 6 | RUN pip3 install dnspython 7 | ENTRYPOINT ["/usr/bin/python3", "service_exec.py"] 8 | -------------------------------------------------------------------------------- /service-exec/service_exec.py: -------------------------------------------------------------------------------- 1 | import docker 2 | import dns.resolver 3 | import os 4 | import sys 5 | 6 | PROXY_SERVICE_NAME = os.environ['PROXY_SERVICE_NAME'] 7 | CONTAINER_ID = os.environ['CONTAINER_ID'] 8 | NODE_ID_RUNNING_TASK = os.environ['NODE_ID_RUNNING_TASK'] 9 | 10 | USER_FLAG = os.getenv('USER_FLAG', '') 11 | IS_TTY = os.getenv('IS_TTY', '') 12 | IS_INTERACTIVE = os.getenv('IS_INTERACTIVE') 13 | 14 | FLAGS = [] 15 | 16 | if USER_FLAG != '': 17 | FLAGS.append('-u') 18 | FLAGS.append(USER_FLAG) 19 | 20 | if IS_TTY != '': 21 | FLAGS.append('-t') 22 | 23 | if IS_INTERACTIVE != '': 24 | FLAGS.append('-i') 25 | 26 | answer = dns.resolver.resolve(f'tasks.{PROXY_SERVICE_NAME}', 'A') 27 | for rdata in answer: 28 | client = docker.DockerClient(base_url=f'tcp://{rdata.address}:2375') 29 | info = client.info() 30 | 31 | node_id = info["Swarm"]["NodeID"] 32 | if node_id == NODE_ID_RUNNING_TASK: 33 | os.execvpe('/usr/local/bin/docker', ['/usr/local/bin/docker', 'exec', *FLAGS, CONTAINER_ID, *sys.argv[1:]], env={ 34 | 'DOCKER_HOST': 'tcp://' + str(rdata.address) + ':2375' 35 | }) 36 | 37 | print("did not find node " + NODE_ID_RUNNING_TASK, file=sys.stderr) 38 | exit(1) --------------------------------------------------------------------------------