├── .bin └── .gitkeep ├── .github ├── dependabot.yml └── workflows │ └── go.yml ├── .gitignore ├── .golangci-lint.yaml ├── .makefiles ├── config │ └── golangci-lint.yaml ├── go.mk ├── help.mk └── scripts │ ├── docker.bash │ └── tags.bash ├── .nomad.hcl ├── .wgnet.conflist ├── CONTRIBUTING.md ├── Dockerfile ├── Makefile ├── README.md ├── Vagrantfile ├── bash-poc ├── README.md ├── c-c-vagrant │ ├── README.md │ ├── container-create.bash │ ├── container-teardown.bash │ ├── host-create.bash │ ├── peer-container0.conf │ ├── peer-wg0.conf │ ├── scratch │ ├── server-container0.conf │ └── server-wg0.conf ├── container-vagrant-disallow-rules │ ├── README.md │ ├── cni │ │ └── cni.conf │ ├── container-create.bash │ ├── container-teardown.bash │ ├── exec-plugins.sh │ ├── host-create.bash │ ├── peer-container0.conf │ ├── peer-wg0.conf │ ├── priv-net-run.sh │ ├── scratch │ ├── server-container0.conf │ └── server-wg0.conf ├── container-vagrant │ ├── README.md │ ├── cni │ │ └── cni.conf │ ├── container-create.bash │ ├── container-teardown.bash │ ├── exec-plugins.sh │ ├── host-create.bash │ ├── peer-container0.conf │ ├── peer-wg0.conf │ ├── priv-net-run.sh │ ├── scratch │ ├── server-container0.conf │ └── server-wg0.conf ├── experiments │ ├── create-ns.bash │ ├── create.bash │ ├── host │ │ ├── host.key │ │ └── host.pub │ ├── lib.bash │ ├── ns1 │ │ ├── ns1.key │ │ └── ns1.pub │ ├── ns2 │ │ ├── ns2.key │ │ └── ns2.pub │ ├── ns3 │ │ ├── ns3.key │ │ └── ns3.pub │ ├── teardown.bash │ └── veth.bash ├── static-vagrant │ ├── create.bash │ ├── peer-wg0.conf │ └── server-wg0.conf └── wg-cni │ ├── README.md │ ├── cni │ └── cni.conf │ ├── container-create.bash │ ├── container-teardown.bash │ ├── dev.sh │ ├── exec-plugins.sh │ ├── host-create.bash │ ├── input │ └── priv-net-run.sh ├── buf.gen.yaml ├── buf.yaml ├── cmd ├── cluster-manager │ └── main.go ├── cni │ ├── main.go │ ├── sample_linux_test.go │ ├── sample_suite_test.go │ └── wireguard.go ├── demo │ ├── README.md │ └── main.go └── node-manager │ └── main.go ├── config ├── client.hcl ├── cni.conf └── nomad-systemd.service ├── docs └── spike │ └── container-to-container-communication.md ├── entrypoint.bash ├── gen └── wgcni │ ├── ipam │ └── v1 │ │ ├── ipam.pb.go │ │ └── ipamv1connect │ │ ├── ipam.connect.go │ │ ├── mock_IPAMServiceClient.go │ │ └── mock_IPAMServiceHandler.go │ └── wireguard │ └── v1 │ ├── wireguard.pb.go │ └── wireguardv1connect │ ├── mock_WireguardServiceClient.go │ ├── mock_WireguardServiceHandler.go │ └── wireguard.connect.go ├── go.mod ├── go.sum ├── nomad ├── cluster-manager.hcl ├── countdash.hcl ├── node-manager.hcl └── sleep.hcl ├── pkg ├── ipam │ ├── ipam.go │ ├── ipam_test.go │ ├── provider.go │ └── testdata │ │ └── ipam.json ├── node-manager │ ├── hack │ │ └── .gitkeep │ ├── runners.go │ ├── runners_test.go │ └── svr.go ├── server │ ├── mock_MapDbOpt.go │ ├── mock_newServerOpt.go │ ├── mock_stringerFunc.go │ ├── server.go │ ├── server_alloc.go │ ├── server_alloc_test.go │ ├── server_test.go │ ├── testdata │ │ └── ipam.json │ ├── wireguard.go │ └── wireguard_test.go └── wireguard │ ├── config.go │ ├── config_test.go │ ├── hack │ ├── EmptyPeers.conf │ ├── NilPeers.conf │ ├── OnePeer.conf │ └── TwoPeer.conf │ ├── manager.go │ ├── manager_test.go │ ├── mock_Peers.go │ ├── mock_WGClient.go │ ├── mock_WGOption.go │ ├── mock_WireguardManager.go │ ├── option.go │ ├── tmpl │ └── wireguard.conf.tmpl │ └── varz.go ├── run-dev.sh ├── scripts ├── docker-run.sh ├── exec-plugins.sh ├── priv-net-run.sh ├── release.sh └── wg-client.sh └── wgcni ├── ipam └── v1 │ └── ipam.proto └── wireguard └── v1 └── wireguard.proto /.bin/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/clly/wireguard-cni/32f47886a49c3771c004c7b80264d63b8fbe0257/.bin/.gitkeep -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "github-actions" 4 | directory: "/" 5 | open-pull-requests-limit: 1 6 | schedule: 7 | interval: "daily" 8 | - package-ecosystem: "gomod" 9 | directory: "/" 10 | open-pull-requests-limit: 1 11 | schedule: 12 | interval: "daily" 13 | - package-ecosystem: "gitsubmodule" 14 | directory: "/" 15 | open-pull-requests-limit: 1 16 | schedule: 17 | interval: "daily" 18 | -------------------------------------------------------------------------------- /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | name: Go 2 | 3 | on: 4 | pull_request: 5 | branches: [ main ] 6 | workflow_call: 7 | 8 | jobs: 9 | build: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v4 13 | 14 | - name: Set up Go 15 | uses: actions/setup-go@v5 16 | with: 17 | go-version: 1.21 18 | 19 | - name: Install deps 20 | run: make lint/install 21 | 22 | - name: Test 23 | run: make test 24 | 25 | - name: Build 26 | run: make build 27 | 28 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Standard 2 | **/*.swp 3 | 4 | # Environment 5 | .vagrant/* 6 | 7 | # Compilation 8 | wireguard-cni 9 | cmd/demo/pkey 10 | cmd/demo/demo 11 | 12 | # Local binary dependencies 13 | .bin/** 14 | 15 | # Local output binaries 16 | bin 17 | 18 | # Vendor 19 | vendor 20 | -------------------------------------------------------------------------------- /.makefiles/go.mk: -------------------------------------------------------------------------------- 1 | .PHONY: deps 2 | 3 | BINS=$(wildcard cmd/*) 4 | deps: ## download go modules 5 | go mod download 6 | 7 | .PHONY: vendor 8 | vendor: ## download and vendor dependencies 9 | go mod vendor 10 | 11 | .PHONY: fmt 12 | fmt: lint/check ## ensure consistent code style 13 | golangci-lint run --fix > /dev/null 2>&1 || true 14 | go mod tidy 15 | 16 | .PHONY: lint/check 17 | lint/check: lint/install 18 | @if ! golangci-lint --version > /dev/null 2>&1; then \ 19 | echo -e "golangci-lint is not installed: run \`make lint/install\` or install it from https://golangci-lint.run"; \ 20 | exit 1; \ 21 | fi 22 | 23 | .PHONY: lint/install 24 | lint/install: ## installs golangci-lint to the go bin dir 25 | @if ! golangci-lint --version > /dev/null 2>&1; then \ 26 | echo "Installing golangci-lint"; \ 27 | curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(BIN_DIR) v1.54.1; \ 28 | fi 29 | 30 | .PHONY: lint 31 | lint: lint/check ## run golangci-lint 32 | @if [[ -f .golangci-lint.yaml ]]; then golangci-lint --config .golangci-lint.yaml run ; else golangci-lint --config .makefiles/config/golangci-lint.yaml run; fi 33 | 34 | .PHONY: test 35 | test: lint ## run go tests 36 | go test ./... -race -timeout 1m 37 | 38 | .PHONY: build 39 | build: ## compile and build artifact 40 | @for i in cmd/*; do \ 41 | echo "building $$i"; \ 42 | go build -v -o bin/$$i ./$$i; \ 43 | done 44 | 45 | .PHONY: docker/build 46 | docker/build: ## compile and build binaries in a docker contianer 47 | @docker run --rm \ 48 | -v $$PWD:/app --workdir /app \ 49 | --user $$(id -u) \ 50 | -e GOCACHE=/tmp \ 51 | golang:1.21 make build 52 | 53 | 54 | .PHONY: build/cmd 55 | build/cmd: 56 | go build ./cmd/... 57 | -------------------------------------------------------------------------------- /.makefiles/help.mk: -------------------------------------------------------------------------------- 1 | 2 | 3 | help: ## displays this help message 4 | @perl -n -e'(/(^[a-zA-Z_\/-]+):.*##(.*)/ && printf "\033[34m%-12s\033[0m %s\n", $$1, $$2) || /(^[a-zA-Z_\/-]+):/ && printf "\033[34m%-12s\033[0m %s\n", $$1' $(MAKEFILE_LIST) | \ 5 | sort | \ 6 | grep -v '#' 7 | -------------------------------------------------------------------------------- /.makefiles/scripts/docker.bash: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -euo pipefail 4 | 5 | tag=local-$(date +%s) 6 | 7 | docker build -t wireguard-cni:$tag . 8 | 9 | echo "Finished building $tag" 10 | -------------------------------------------------------------------------------- /.makefiles/scripts/tags.bash: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -eou pipefail 4 | # shellcheck disable=SC2154 5 | trap 's=${?:-1}; echo >&2 "$0: Error on line "$LINENO": $BASH_COMMAND"; exit $s' ERR 6 | 7 | 8 | gitCommit=$(git rev-parse HEAD) 9 | gitBranch=$(git rev-parse --abbrev-ref HEAD) 10 | gitTagRef=$(git name-rev --name-only --tags "${gitCommit}") 11 | gitTag=${gitTagRef#tags/} 12 | case $(uname -m) in 13 | i386) architecture="386" ;; 14 | i686) architecture="386" ;; 15 | x86_64) architecture="amd64" ;; 16 | aarch64) architecture="arm64" ;; 17 | esac 18 | 19 | if grep -q -s -P "^ID=\S+$" /etc/os-release; then 20 | os=$(grep -P "^ID=\S+$" /etc/os-release|cut -f2 -d=) 21 | else 22 | os="" 23 | fi 24 | 25 | 26 | echo "os: $os" 27 | echo "gitCommit: $gitCommit" 28 | echo "gitTag: $gitTag" 29 | echo "architecture: $architecture" 30 | echo "gitBranch: $gitBranch" 31 | -------------------------------------------------------------------------------- /.nomad.hcl: -------------------------------------------------------------------------------- 1 | plugin "docker" { 2 | config { 3 | allow_caps = ["audit_write", "chown", "dac_override", "fowner", "fsetid", "kill", "mknod", "net_bind_service", "setfcap", "setgid", "setpcap", "setuid", "sys_chroot", "net_admin","sys_module","net_raw"] 4 | allow_privileged = true 5 | } 6 | } 7 | 8 | bind_addr = "0.0.0.0" 9 | 10 | advertise { 11 | rpc = "{{ GetPrivateInterfaces | include \"network\" \"192.168.56.0/24\" | attr \"address\" }}" 12 | serf = "{{ GetPrivateInterfaces | include \"network\" \"192.168.56.0/24\" | attr \"address\" }}" 13 | } 14 | 15 | client { 16 | cni_config_dir = "/opt/cni/config" 17 | network_interface = "{{ GetPrivateInterfaces | include \"network\" \"192.168.56.0/24\" | attr \"name\" }}" 18 | host_network "public" { 19 | cidr = "10.0.2.15/24" 20 | } 21 | } 22 | 23 | -------------------------------------------------------------------------------- /.wgnet.conflist: -------------------------------------------------------------------------------- 1 | { 2 | "cniVersion": "0.4.0", 3 | "name": "wgnet", 4 | "plugins": [ 5 | { 6 | "type": "bridge", 7 | "bridge": "cni0", 8 | "keyA": ["some more", "plugin specific", "configuration"], 9 | "isGateway": true, 10 | "ipMasq": false, 11 | "ipam": { 12 | "type": "host-local", 13 | "subnet": "192.168.0.0/24", 14 | "gateway": "192.168.0.1", 15 | "routes": [ 16 | {"dst": "0.0.0.0/0"} 17 | ] 18 | }, 19 | "dns": { 20 | "nameservers": [ "10.1.0.1" ] 21 | } 22 | }, 23 | { 24 | "type": "wireguard", 25 | "nodeManagerAddr": "http://localhost:5000" 26 | } 27 | ] 28 | } 29 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | Wireugard-CNI Codebase Documentation 2 | === 3 | 4 | This directory contains some documentation about the wireguard-cni codebase, 5 | aimed at readers who are interested in making code contributions. 6 | 7 | If you're looking for information on _using_ wireguard-cni, please refer 8 | to the main README instead. 9 | 10 | * cmd - CNI, cluster-manager, and node-manager runtimes 11 | * gen - generated code 12 | * pkg/server - wireguard and ipam server implementations 13 | * pkg/wireguard - creating, syncing, and destroying wireguard interfaces 14 | * wgcni - proto definitions 15 | 16 | ## Creating new APIs 17 | All APIs are defined in proto files and generated using buf. The dependencies can be installed using `make deps`. Modify 18 | the definitions in wgcni and then generate the new code using `make proto`. Then implement the changes in the 19 | pkg/server. You can validate that the implementation still passes tests using `make test` 20 | 21 | ## Testing 22 | * Unit tests can be executed using `make test` 23 | 24 | ## Developing with Vagrant 25 | * Install [Virtualbox](https://www.virtualbox.org/) 26 | * Install dependencies with `make extra-deps` 27 | * Build binaries with `make` 28 | * ssh to vagrant machine 29 | ** `vagrant ssh server` 30 | * Execute binaries directly 31 | ** `/vagrant/bin/cmd/cluster-manager` 32 | ** `/vagrant/bin/cmd/node-manager` 33 | ** `cd /vagrant/bash-poc/wg-cni && sudo ./container-create.bash server` 34 | ** Use logs and other output to examine outputs 35 | 36 | ## Developing with Vagrant and Nomad 37 | 38 | * Install [Virtualbox](https://www.virtualbox.org/) 39 | * Install dependencies with `make extra-deps` 40 | * Build binaries with `make` 41 | * Build local docker container 42 | * Set nomad configuration with the local docker tag 43 | * start nomad jobs with `nomad run cluster-manager.hcl`, `nomad run host-manager.hcl`, `nomad run sleep.hcl` 44 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.21 as build 2 | 3 | WORKDIR /build 4 | COPY . ./ 5 | RUN make build 6 | 7 | FROM ubuntu:jammy 8 | 9 | WORKDIR /opt 10 | COPY --from=build /build/bin/cmd/ ./ 11 | COPY entrypoint.bash /entrypoint.bash 12 | 13 | 14 | RUN apt-get update && apt-get install -y \ 15 | wireguard-tools jq iproute2 iptables \ 16 | && rm -rf /var/lib/apt/lists/* 17 | 18 | USER nobody 19 | ENTRYPOINT ["/entrypoint.bash"] 20 | 21 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | MAKEFLAGS += --warn-undefined-variables 2 | CWD := $(abspath $(patsubst %/,%,$(dir $(abspath $(lastword $(MAKEFILE_LIST)))))) 3 | SHELL := bash 4 | .SHELLFLAGS := -euo pipefail -c 5 | .DEFAULT_GOAL := all 6 | APPNAME := "wireguard-cni" 7 | 8 | # Build dependencies go in here 9 | BIN=.bin 10 | BUF=./.bin/buf 11 | 12 | # Respect $GOBIN if set in environment or via $GOENV file. 13 | GOBIN_DIR ?= $(shell go env GOBIN) 14 | BIN_DIR ?= $(shell go env GOPATH)/bin 15 | export PATH := ${CWD}/${BIN}:$(PATH):$(BIN_DIR) 16 | 17 | ifneq ("$(wildcard .makefiles/*.mk)","") 18 | include .makefiles/*.mk 19 | else 20 | $(info "no makefiles to load") 21 | endif 22 | 23 | .PHONY: all 24 | all: proto mocks test build 25 | 26 | .PHONY: proto 27 | proto: buf/lint deps 28 | @$(BIN)/buf generate 29 | 30 | .PHONY: buf/lint 31 | buf/lint: deps 32 | @$(BIN)/buf lint 33 | 34 | .PHONY: deps 35 | deps: ./.bin/buf ./.bin/protoc-gen-go ./.bin/protoc-gen-connect-go ## deps: installs build time dependencies 36 | 37 | .PHONY: extra-deps 38 | extra-deps: ./.bin/hc-install ./.bin/nomad ./.bin/vagrant ## extra-deps: installs other tools like nomad and vagrant 39 | 40 | .PHONY: mocks 41 | mocks: ./.bin/mockery 42 | @./.bin/mockery --all --inpackage --log-level info 43 | 44 | ./.bin/mockery: 45 | GOBIN=${CWD}/${BIN} go install github.com/vektra/mockery/v2@v2.40.1 46 | 47 | ./.bin/buf: 48 | GOBIN=${CWD}/${BIN} go install github.com/bufbuild/buf/cmd/buf@v1.5.0 49 | 50 | ./.bin/protoc-gen-go: 51 | GOBIN=${CWD}/${BIN} go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.28.0 52 | 53 | ./.bin/protoc-gen-connect-go: 54 | GOBIN=${CWD}/${BIN} go install connectrpc.com/connect/cmd/protoc-gen-connect-go@v1.12.0 55 | 56 | ./.bin/hc-install: 57 | GOBIN=${CWD}/${BIN} go install github.com/hashicorp/hc-install/cmd/hc-install@main 58 | 59 | ./.bin/nomad: ./.bin/hc-install 60 | hc-install install -version 1.3.2 -path ./.bin nomad && chmod 755 ./.bin/nomad 61 | 62 | ./.bin/vagrant: ./.bin/hc-install 63 | hc-install install -version 2.2.19 -path ./.bin vagrant && chmod 755 ./.bin/vagrant 64 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Wireguard CNI 2 | 3 | Wireguard CNI is a plugin implementation of the Container Networking Interface. It is composed of 3 components - a Cluster Manager, Node Manager, and the Containers themselves. It will create a secure wireguard mesh between components and manage IP management between all nodes in a _cluster_. A cluster is any set of nodes that share a cluster-manager. 4 | 5 | # Getting Started 6 | We use protobuf to generate our APIs and types. All code is checked in so building the project can be done without any code generation dependencies installed. A list of available make targets can be found using 7 | ``` 8 | make help 9 | ``` 10 | 11 | ## Quick Start 12 | 13 | ### Using Vagrant 14 | Install dependencies and start vagrant 15 | ``` 16 | make extra-deps && .bin/vagrant up 17 | ``` 18 | 19 | Install dependencies 20 | ``` 21 | # install wireguard-tools package for your OS 22 | ## install dependencies 23 | make deps && make extra-deps && ./.bin/nomad & 24 | ## Run jobs in nomad 25 | ./.bin/nomad run nomad/cluster-manager.hcl 26 | ./.bin/nomad run nomad/node-manager.hcl 27 | ./.bin/nomad run nomad/sleep.hcl 28 | ## see that wireguard is running 29 | sudo wg 30 | ## kill and cleanup nomad 31 | kill %1 32 | ## I've seen the node-manager leave the wg0 interface behind. Still need to dig into that 33 | sudo ip link del wg0 34 | ``` 35 | 36 | ## Building 37 | 38 | Building the initial software requires no dependencies other than Go itself. 39 | ``` 40 | make build 41 | ``` 42 | 43 | There's also a make target to build inside a docker container using 44 | ``` 45 | make docker/build 46 | ``` 47 | 48 | ## Code Generation 49 | buf, protoc-gen-go and protoc-gen-connect-go are both required to generate the APIs and types. All of them are easily installed using 50 | ``` 51 | make deps 52 | ``` 53 | 54 | And code can be generated using 55 | ``` 56 | make proto 57 | ``` 58 | 59 | ## Running the project 60 | There are vagrant and nomad configuration available. Both binaries are installable using `make extra-deps` 61 | 62 | ## Components 63 | 64 | ### Cluster Manager 65 | The cluster manager handles IP Management for a set of nodes. It hands out subnets to nodes to manage. It also performs Wireguard Public Key coordination and shares the list of peers and subnets between nodes. This enables encrypted node <-> node communication over the wireguard mesh. 66 | 67 | ### Node Manager 68 | Each "node" gets it's own IP space to manage from the cluster manager and hand out to network namespaces or wireguard peers. The node-manager performs the same responsibility as the Cluster Manager but at a per-namespace level. It also creates peers and routing between itself and the network namespaces running on the node and between other nodes. 69 | 70 | #### Dependencies 71 | * Cluster Manager 72 | * wireguard-tools 73 | * iptables 74 | 75 | ### CNI 76 | The cni creates the wireguard interface and sets the default route from the container to go through the wireguard interface 77 | 78 | #### Dependencies 79 | * Node Manager 80 | * wireguard-tools 81 | * iptables 82 | 83 | ### Component Sequence Diagram 84 | 85 | ```mermaid 86 | sequenceDiagram 87 | participant Cluster Manager 88 | participant Node Manager 89 | participant CNI 90 | Node Manager->>Cluster Manager: Allocate Subnet 91 | Node Manager->>Cluster Manager: Register WG Public Key 92 | Node Manager->>Cluster Manager: Request Peers 93 | Cluster Manager->>Node Manager: Return All other cluster-peers (other node managers _or_ external clients dialing into the wireguard network) 94 | loop Manage Routes 95 | Node Manager->>Node Manager: Setup Routes for Cluster Peers and CNI Peers 96 | end 97 | CNI->>Node Manager: Allocate IP Address 98 | CNI->>Node Manager: Register Public Key 99 | CNI->>Node Manager: Request Peers 100 | Node Manager->>CNI: Return Node-Manager Peer (include other allocs if CNI <-> CNI communication is enabled) 101 | ``` 102 | ## Network Communication Model 103 | 104 | The network communication model is a hub and spoke. Containers connect to nodes. Nodes connect to other nodes. Traffic routes from a container to a node to another node to a container. 105 | 106 | ### Diagram 107 | ```mermaid 108 | graph LR 109 | A[Container] --> B(Node) 110 | C[Container] --> B 111 | B --> D[Node] 112 | D --> E[Container] 113 | D --> F[Container] 114 | D --> B 115 | ``` 116 | 117 | # Initial Proof of Concept 118 | 119 | Initial proof of concepts exist as static configuration and bash scripts in the bash-poc folder. These exist to prove out the initial networking model and configuration. Read README's there for more instructions. All POC's should be started 120 | by running `vagrant up` 121 | -------------------------------------------------------------------------------- /Vagrantfile: -------------------------------------------------------------------------------- 1 | # -*- mode: ruby -*- 2 | # vi: set ft=ruby : 3 | 4 | # All Vagrant configuration is done below. The "2" in Vagrant.configure 5 | # configures the configuration version (we support older styles for 6 | # backwards compatibility). Please don't change it unless you know what 7 | # you're doing. 8 | Vagrant.configure("2") do |config| 9 | # The most common configuration options are documented and commented below. 10 | # For a complete reference, please see the online documentation at 11 | # https://docs.vagrantup.com. 12 | 13 | # Every Vagrant development environment requires a box. You can search for 14 | # boxes at https://vagrantcloud.com/search. 15 | #config.env.enable # enable .env support plugin (it will let us easily enable cloud_init support) 16 | 17 | # URL used as a source for the vm.box defined above 18 | config.vm.box_url = "https://cloud-images.ubuntu.com/jammy/current/jammy-server-cloudimg-amd64-vagrant.box" 19 | 20 | config.vm.define "server" do |server| 21 | server.vm.box = "wg-server" 22 | server.vm.hostname = "wg-server" 23 | server.vm.network "private_network", ip: "192.168.56.11" 24 | server.vm.network "forwarded_port", guest: 4646, host: 14646, host_ip: "127.0.0.1" 25 | server.vm.network "forwarded_port", guest: 9002, host: 19002, host_ip: "127.0.0.1" 26 | end 27 | 28 | config.vm.define "peer" do |peer| 29 | peer.vm.box = "wg-peer" 30 | peer.vm.hostname = "wg-peer" 31 | peer.vm.network "private_network", ip: "192.168.56.10" 32 | peer.vm.network "forwarded_port", guest: 9002, host: 29002, host_ip: "127.0.0.1" 33 | end 34 | # Disable automatic box update checking. If you disable this, then 35 | # boxes will only be checked for updates when the user runs 36 | # `vagrant box outdated`. This is not recommended. 37 | # config.vm.box_check_update = false 38 | 39 | # Create a forwarded port mapping which allows access to a specific port 40 | # within the machine from a port on the host machine. In the example below, 41 | # accessing "localhost:8080" will access port 80 on the guest machine. 42 | # NOTE: This will enable public access to the opened port 43 | # config.vm.network "forwarded_port", guest: 80, host: 8080 44 | 45 | # Create a forwarded port mapping which allows access to a specific port 46 | # within the machine from a port on the host machine and only allow access 47 | # via 127.0.0.1 to disable public access 48 | 49 | 50 | # Create a private network, which allows host-only access to the machine 51 | # using a specific IP. 52 | 53 | 54 | # Create a public network, which generally matched to bridged network. 55 | # Bridged networks make the machine appear as another physical device on 56 | # your network. 57 | # config.vm.network "public_network" 58 | 59 | # Share an additional folder to the guest VM. The first argument is 60 | # the path on the host to the actual folder. The second argument is 61 | # the path on the guest to mount the folder. And the optional third 62 | # argument is a set of non-required options. 63 | # config.vm.synced_folder "../data", "/vagrant_data" 64 | 65 | # Provider-specific configuration so you can fine-tune various 66 | # backing providers for Vagrant. These expose provider-specific options. 67 | # Example for VirtualBox: 68 | # 69 | # config.vm.provider "virtualbox" do |vb| 70 | # # Display the VirtualBox GUI when booting the machine 71 | # vb.gui = true 72 | # 73 | # # Customize the amount of memory on the VM: 74 | # vb.memory = "1024" 75 | # end 76 | # 77 | # View the documentation for the provider you are using for more 78 | # information on available options. 79 | 80 | # config.vm.cloud_init do |cloud_init| 81 | # cloud_init.content_type = "text/cloud-config" 82 | # cloud_init.path = "../digitalocean/config/cloud-init" 83 | # end 84 | #config.vm.cloud_init do |cloud_init| 85 | # cloud_init.content_type = "text/cloud-config" 86 | # cloud_init.path = "terraform/user-data.yml" 87 | #end 88 | 89 | #config.vm.network "forwarded_port", guest: 4646, host: 4646 90 | # Enable provisioning with a shell script. Additional provisioners such as 91 | # Puppet, Chef, Ansible, Salt, and Docker are also available. Please see the 92 | # documentation for more information about their specific syntax and use. 93 | config.vm.provision "shell", inline: <<-SHELL 94 | apt-get update && apt-get install -y git wireguard-tools docker.io jq containernetworking-plugins make && apt-get upgrade -y 95 | cd /tmp && wget https://go.dev/dl/go1.21.6.linux-amd64.tar.gz && tar -C /usr/local -xzf go1.21.6.linux-amd64.tar.gz 96 | cd /vagrant 97 | PATH=/usr/local/go/bin:$PATH make extra-deps build 98 | ls -l /opt /opt/cni 99 | mkdir -p /opt/cni/config 100 | ln -snf /usr/lib/cni /opt/cni/bin 101 | cp bin/cmd/cni /opt/cni/bin/wireguard 102 | cp .wgnet.conflist /opt/cni/config/wgnet.conflist 103 | cp -f .bin/nomad /usr/local/bin 104 | mkdir -p /etc/nomad.d 105 | cp .nomad.hcl /etc/nomad.d/nomad.hcl 106 | cp config/nomad-systemd.service /etc/systemd/system/nomad.service 107 | 108 | mkdir -p /opt/nomad 109 | chmod +x /usr/local/bin/nomad 110 | if [[ $(hostnamectl --static) == "wg-peer" ]]; then 111 | cp config/client.hcl /etc/nomad.d/ 112 | cat config/nomad-systemd.service | sed "s/-dev//" > /etc/systemd/system/nomad.service 113 | echo 192.168.56.11 wg-server | sudo tee -a /etc/hosts 114 | else 115 | echo 192.168.56.10 wg-peer | sudo tee -a /etc/hosts 116 | fi 117 | systemctl daemon-reload && systemctl start nomad 118 | SHELL 119 | end 120 | -------------------------------------------------------------------------------- /bash-poc/README.md: -------------------------------------------------------------------------------- 1 | # Proof of Concepts 2 | 3 | These are all bash based proof of concepts. 4 | 5 | * experiments - These are all single host based container experiments. I could never route out of the container and 6 | needed a bridge and I didn't know how to do that manually so I switched to using CNI 7 | * static-vagrant - This runs in a pair of hosts. I used Vagrant. Basically making sure I got wireguard working between hosts 8 | * container-vagrant - This is the meat of the poc. This validates that I can create containers and then route between the containers and hosts over wireguard 9 | -------------------------------------------------------------------------------- /bash-poc/c-c-vagrant/README.md: -------------------------------------------------------------------------------- 1 | # Analysis 2 | 3 | Wireguard interfaces are namespace aware. After creation, they keep the 4 | socket in the originating namespace (host namespace in this case) and then 5 | forward traffic to the interface in the container namespace. 6 | 7 | Listening UDP Ports 8 | ```commandline 9 | UNCONN 0 0 0.0.0.0:51820 0.0.0.0:* 10 | UNCONN 0 0 0.0.0.0:51821 0.0.0.0:* 11 | UNCONN 0 0 [::]:51820 [::]:* 12 | UNCONN 0 0 [::]:51821 [::]:* 13 | ``` 14 | 15 | ```commandline 16 | root@wg-server:/vagrant/bash-poc/c-c-vagrant# wg 17 | interface: wg0 18 | public key: v2XPnF7Yk7KEugBRVyCdD6mssm2Tsc4edDUApwzFoBk= 19 | private key: (hidden) 20 | listening port: 51820 21 | 22 | peer: X7qQfHNDtiUS5+qg2H5T25IwhcWdVbyoFjGEDTFJ1WU= 23 | endpoint: 192.168.56.11:51821 24 | allowed ips: 10.0.10.3/32 25 | latest handshake: 56 seconds ago 26 | transfer: 532 B received, 4.75 KiB sent 27 | persistent keepalive: every 10 seconds 28 | ``` 29 | 30 | 31 | With multiple listen ports we can have each container on a different UDP 32 | port and then configuration Peers for each `address:port` combination for each 33 | wireguard interface. This will slightly tighten the security boundary 34 | because there is no longer a bridge to pass traffic in clear text. With the 35 | current firewall rules we will still forward traffic if a user knows the 36 | Wireguard network space and where a wireguard interface lives. 37 | 38 | We can not use wg-quick with this model because it creates, configures and 39 | brings up the interface at the same time. Once the interface is live it 40 | cannot be moved from one namespace to another. 41 | 42 | ## Convergence notes 43 | 44 | ### When are endpoints required 45 | While being lazy during setup I've discovered configuring actual dialing and 46 | routing does _not seem_ to be required. After one party performs any request 47 | against any other party in the mesh that is properly configured as a peer. The 48 | peer can then respond back over the wireguard tunnel. Only one side 49 | _requires_ an endpoint which feels like it allows a level of control and 50 | protection around which clients can start a connection. Unsure if those ever 51 | expire. 52 | 53 | This was confirmed with tcpdump to ensure that packets were not passing 54 | through the host accidentally. If the route didn't exist would the host pass 55 | traffic? If the peer exists and the route doesn't it appears that traffic 56 | won't pass? 57 | 58 | ### Hosts can still pass traffic but require a route 59 | 60 | When a subnet is configured in AllowedIPs and a route is configured and IP 61 | forwarding is configured, the peer will still pass traffic between nodes. As 62 | long as we configure routes properly on Hosts first new network namespaces 63 | should be able to communicate between other network namespaces quickly. 64 | 65 | This was confirmed using traceroute. 66 | 67 | ### iptables changes 68 | 69 | A minor iptables change _was_ required to ensure that we don't NAT traffic 70 | intended for containers. If a container starts the request then the host can 71 | pass traffic back to the container. Why didn't we find this out earlier? 72 | Unknown. Maybe I started the request from inside the container so things 73 | just sort of worked or traffic could more easily get passed into the 74 | container because of the connection from the bridge to the host-local IP 75 | address for the wireguard dial. 76 | 77 | ### When will a container pass traffic using a host. 78 | When fully configured a container will _not_ attempt to pass traffic through 79 | it's host unless it does not have an AllowedIPs for the address set on the 80 | interface. 81 | 82 | ## Testing 83 | * open two terminals 84 | * `vagrant ssh peer` and `vagrant ssh server` 85 | * `./host-create.sh ` 86 | * `./container-create.sh ` 87 | 88 | Then ping or curl or whatever between them. Using something like echo-server would make it easier to validate -------------------------------------------------------------------------------- /bash-poc/c-c-vagrant/container-create.bash: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | 3 | set -euof pipefail 4 | 5 | N=$1 6 | 7 | case $N in 8 | server) 9 | ADDRESS=10.0.10.3 10 | ROUTES="10.0.0.0/24 " 11 | ;; 12 | peer) 13 | ADDRESS=10.0.0.3 14 | ;; 15 | *) 16 | echo "unsupported" 17 | exit 1 18 | ;; 19 | esac 20 | 21 | netnspath=/var/run/netns/container 22 | #function catch() { 23 | # NETCONFPATH=./cni CNI_PATH=/usr/lib/cni ./exec-plugins.sh del container $netnspath 24 | # ip netns del container 25 | #} 26 | 27 | function catch() { 28 | ip netns exec container ip link del container0 29 | } 30 | 31 | trap 'catch' ERR 32 | 33 | ip netns add container || true 34 | 35 | #NETCONFPATH=./cni CNI_PATH=/usr/lib/cni ./exec-plugins.sh add container $netnspath 36 | set -x 37 | ip link add container0 type wireguard 38 | ip link set container0 netns container 39 | cp $N-container0.conf /etc/wireguard/container0.conf 40 | ip netns exec container wg setconf container0 /etc/wireguard/container0.conf 41 | ip netns exec container ip -4 address add "${ADDRESS}" dev container0 42 | ip netns exec container ip link set mtu 1420 up dev container0 43 | #ip netns exec container wg set container0 fwmark 51820 44 | ip netns exec container ip -4 route add 0.0.0.0/0 dev container0 45 | #ip netns exec container ip -4 rule add not fwmark 51820 table 51820 46 | #ip netns exec container ip -4 rule add table main suppress_prefixlength 0 47 | ##sysctl -q net.ipv4.conf.all.src_valid_mark=1 48 | ip netns exec container ip link set lo up 49 | #ip link up container0 50 | #ip netns exec container wg-quick up container0 51 | #] ip link add container0 type wireguard 52 | #] wg setconf container0 /dev/fd/63 53 | #] ip -4 address add 10.0.10.3 dev container0 54 | #] ip link set mtu 1420 up dev container0 55 | #] wg set container0 fwmark 51820 56 | #] ip -4 route add 0.0.0.0/0 dev container0 table 51820 57 | #] ip -4 rule add not fwmark 51820 table 51820 58 | #] ip -4 rule add table main suppress_prefixlength 0 59 | #] sysctl -q net.ipv4.conf.all.src_valid_mark=1 60 | #] iptables-restore -n 61 | #] ip link set lo up 62 | 63 | -------------------------------------------------------------------------------- /bash-poc/c-c-vagrant/container-teardown.bash: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | 3 | set -euof pipefail 4 | 5 | #N=$1 6 | 7 | 8 | netnspath=/var/run/netns/container 9 | 10 | ip netns del container 11 | #ip link add container0 type wireguard 12 | #ip link set container0 netns container 13 | -------------------------------------------------------------------------------- /bash-poc/c-c-vagrant/host-create.bash: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | N=$1 6 | cp $N-wg0.conf /etc/wireguard/wg0.conf 7 | sysctl -w net.ipv4.ip_forward=1 8 | wg-quick up wg0 9 | -------------------------------------------------------------------------------- /bash-poc/c-c-vagrant/peer-container0.conf: -------------------------------------------------------------------------------- 1 | # container 0 - peer host 2 | [Interface] 3 | #Address = 10.0.0.3 4 | ListenPort = 51821 5 | PrivateKey = EK1DWZkUjrXjQJUnWsbgI5WnOOEWVJDRKBCBNKX3yXI= 6 | #PostUp = ip link set lo up 7 | 8 | # wg0 - server host 9 | [Peer] 10 | Endpoint = 192.168.56.11:51820 11 | PublicKey = v2XPnF7Yk7KEugBRVyCdD6mssm2Tsc4edDUApwzFoBk= 12 | AllowedIPs = 10.0.10.1/24 13 | 14 | # wg0 - peer host 15 | [Peer] 16 | Endpoint = 192.168.56.10:51820 17 | PublicKey = WzEKDzUqj4XKiyNwCjWYCuFTHCTZxQGjumj/v+Iytxg= 18 | AllowedIPs = 0.0.0.0/0 19 | 20 | # container 0 - server host 21 | [Peer] 22 | Endpoint = 192.168.56.11:51821 23 | PublicKey = X7qQfHNDtiUS5+qg2H5T25IwhcWdVbyoFjGEDTFJ1WU= 24 | AllowedIps = 10.0.10.3/32 25 | -------------------------------------------------------------------------------- /bash-poc/c-c-vagrant/peer-wg0.conf: -------------------------------------------------------------------------------- 1 | # wg0 on peer 2 | [Interface] 3 | Address = 10.0.0.1/24 4 | PrivateKey = sLIO8JBk7bOnSVPVlBIYht9fw3w61ZazKNIW9B5CDlw= 5 | ListenPort = 51820 6 | PostUp = iptables -A FORWARD -i %i -j ACCEPT; iptables -A FORWARD -o %i -j ACCEPT; iptables -t nat -A POSTROUTING -s 10.0.10.0/24 ! -o wg0 -j MASQUERADE 7 | PostDown = iptables -D FORWARD -i %i -j ACCEPT; iptables -D FORWARD -o %i -j ACCEPT; iptables -t nat -D POSTROUTING -s 10.0.10.0/24 ! -o wg0 -j MASQUERADE 8 | 9 | # wg0 on server 10 | [Peer] 11 | Endpoint = 192.168.56.11:51820 12 | PublicKey = v2XPnF7Yk7KEugBRVyCdD6mssm2Tsc4edDUApwzFoBk= 13 | AllowedIPs = 10.0.10.0/24 14 | PersistentKeepAlive = 10 15 | 16 | # container on peer 17 | [Peer] 18 | Endpoint = 192.168.56.11:51821 19 | PublicKey = j6Dgd8syAcNM/Uu4yGR/1t+dBbDos7utcjzBcXiZIQc= 20 | AllowedIPs = 10.0.10.3/32 21 | 22 | # container on server 23 | [Peer] 24 | Endpoint = 192.168.56.10:51821 25 | PersistentKeepAlive = 10 26 | PublicKey = X7qQfHNDtiUS5+qg2H5T25IwhcWdVbyoFjGEDTFJ1WU= 27 | AllowedIPs = 10.0.0.3/32 28 | -------------------------------------------------------------------------------- /bash-poc/c-c-vagrant/scratch: -------------------------------------------------------------------------------- 1 | host subnet address privatekey pubkey 2 | server-wg0 10.0.10.0/24 10.0.10.1 CH9Z7veOrPfbKP9anOLEmKDs8PVhWtOtADLSBIktHH8= v2XPnF7Yk7KEugBRVyCdD6mssm2Tsc4edDUApwzFoBk= 3 | peer-wg0 10.0.0.0/24 10.0.0.1 sLIO8JBk7bOnSVPVlBIYht9fw3w61ZazKNIW9B5CDlw= WzEKDzUqj4XKiyNwCjWYCuFTHCTZxQGjumj/v+Iytxg= 4 | container-server 10.0.10.0/24 10.0.10.3 iBDdm9fXmcy+wAl26Ef6zZcnc0Q01oGi8KIVQoYtAFg= X7qQfHNDtiUS5+qg2H5T25IwhcWdVbyoFjGEDTFJ1WU= 5 | container-peer 10.0.0.0/24 10.0.0.3 EK1DWZkUjrXjQJUnWsbgI5WnOOEWVJDRKBCBNKX3yXI= j6Dgd8syAcNM/Uu4yGR/1t+dBbDos7utcjzBcXiZIQc= 6 | -------------------------------------------------------------------------------- /bash-poc/c-c-vagrant/server-container0.conf: -------------------------------------------------------------------------------- 1 | # container0 on server host 2 | [Interface] 3 | ListenPort = 51821 4 | #Address = 10.0.10.3 5 | PrivateKey = iBDdm9fXmcy+wAl26Ef6zZcnc0Q01oGi8KIVQoYtAFg= 6 | #PostUp = ip link set lo up 7 | 8 | # wg0 on server 9 | [Peer] 10 | Endpoint = 192.168.56.11:51820 11 | PublicKey = v2XPnF7Yk7KEugBRVyCdD6mssm2Tsc4edDUApwzFoBk= 12 | AllowedIPs = 0.0.0.0/0 13 | 14 | # wg0 on peer 15 | [Peer] 16 | Endpoint = 192.168.56.10:51820 17 | PublicKey = WzEKDzUqj4XKiyNwCjWYCuFTHCTZxQGjumj/v+Iytxg= 18 | AllowedIPs = 10.0.0.0/24 19 | 20 | # container0 on peer 21 | [Peer] 22 | PublicKey = j6Dgd8syAcNM/Uu4yGR/1t+dBbDos7utcjzBcXiZIQc= 23 | AllowedIPs = 10.0.0.3/32 24 | -------------------------------------------------------------------------------- /bash-poc/c-c-vagrant/server-wg0.conf: -------------------------------------------------------------------------------- 1 | # server - wg0 2 | [Interface] 3 | Address = 10.0.10.1/24 4 | PrivateKey = CH9Z7veOrPfbKP9anOLEmKDs8PVhWtOtADLSBIktHH8= 5 | ListenPort = 51820 6 | PostUp = iptables -A FORWARD -i %i -j ACCEPT; iptables -A FORWARD -o %i -j ACCEPT; iptables -t nat -A POSTROUTING -s 10.0.10.0/24 ! -o wg0 -j MASQUERADE 7 | PostDown = iptables -D FORWARD -i %i -j ACCEPT; iptables -D FORWARD -o %i -j ACCEPT; iptables -t nat -D POSTROUTING -s 10.0.10.0/24 ! -o wg0 -j MASQUERADE 8 | 9 | # WzEKDzUqj4XKiyNwCjWYCuFTHCTZxQGjumj/v+Iytxg= - peer wg0 10 | # subnet 10.0.0.0/24 11 | [Peer] 12 | Endpoint = 192.168.56.10:51820 13 | PublicKey = WzEKDzUqj4XKiyNwCjWYCuFTHCTZxQGjumj/v+Iytxg= 14 | AllowedIPs = 10.0.0.0/24 15 | PersistentKeepAlive = 10 16 | 17 | # j6Dgd8syAcNM/Uu4yGR/1t+dBbDos7utcjzBcXiZIQc= - container0 on peer 18 | [Peer] 19 | PublicKey = j6Dgd8syAcNM/Uu4yGR/1t+dBbDos7utcjzBcXiZIQc= 20 | AllowedIPs = 10.0.0.3/32 21 | 22 | # container0 on server 23 | [Peer] 24 | Endpoint = 192.168.56.11:51821 25 | PublicKey = X7qQfHNDtiUS5+qg2H5T25IwhcWdVbyoFjGEDTFJ1WU= 26 | AllowedIPs = 10.0.10.3/32 27 | PersistentKeepAlive = 10 28 | -------------------------------------------------------------------------------- /bash-poc/container-vagrant-disallow-rules/README.md: -------------------------------------------------------------------------------- 1 | # Instructions 2 | 3 | * open two terminals 4 | * `vagrant ssh peer` and `vagrant ssh server` 5 | * `./host-create.sh ` 6 | * `./container-create.sh ` 7 | 8 | Then ping or curl or whatever between them. Using something like echo-server would make it easier to validate. 9 | 10 | This example disallows `container0` on `server` (10.0.10.3) from talking to `container0` on `peer` (10.0.0.3) -------------------------------------------------------------------------------- /bash-poc/container-vagrant-disallow-rules/cni/cni.conf: -------------------------------------------------------------------------------- 1 | { 2 | "cniVersion": "0.4.0", 3 | "name": "wgnet", 4 | "plugins": [ 5 | { 6 | "type": "bridge", 7 | "bridge": "cni0", 8 | "keyA": ["some more", "plugin specific", "configuration"], 9 | "isGateway": true, 10 | "ipam": { 11 | "type": "host-local", 12 | "subnet": "192.168.0.0/24", 13 | "gateway": "192.168.0.1", 14 | "routes": [ 15 | {"dst": "0.0.0.0/0"} 16 | ] 17 | }, 18 | "dns": { 19 | "nameservers": [ "10.1.0.1" ] 20 | } 21 | }, 22 | { 23 | "type": "portmap", 24 | "capabilities": {"portMappings": true} 25 | } 26 | ] 27 | } 28 | -------------------------------------------------------------------------------- /bash-poc/container-vagrant-disallow-rules/container-create.bash: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | 3 | set -euof pipefail 4 | 5 | N=$1 6 | 7 | 8 | netnspath=/var/run/netns/container 9 | function catch() { 10 | NETCONFPATH=./cni CNI_PATH=/usr/lib/cni ./exec-plugins.sh del container $netnspath 11 | ip netns del container 12 | } 13 | 14 | trap 'catch' ERR 15 | 16 | ip netns add container || true 17 | 18 | NETCONFPATH=./cni CNI_PATH=/usr/lib/cni ./exec-plugins.sh add container $netnspath 19 | set -x 20 | #ip link add container0 type wireguard 21 | #ip link set container0 netns container 22 | cp $N-container0.conf /etc/wireguard/container0.conf 23 | ip netns exec container wg-quick up container0 24 | -------------------------------------------------------------------------------- /bash-poc/container-vagrant-disallow-rules/container-teardown.bash: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | 3 | set -euof pipefail 4 | 5 | N=$1 6 | 7 | 8 | netnspath=/var/run/netns/container 9 | function catch() { 10 | NETCONFPATH=./cni CNI_PATH=/usr/lib/cni ./exec-plugins.sh del container $netnspath 11 | ip netns del container 12 | } 13 | 14 | trap 'catch' ERR 15 | 16 | 17 | NETCONFPATH=./cni CNI_PATH=/usr/lib/cni ./exec-plugins.sh del container $netnspath 18 | ip netns del container 19 | #ip link add container0 type wireguard 20 | #ip link set container0 netns container 21 | -------------------------------------------------------------------------------- /bash-poc/container-vagrant-disallow-rules/exec-plugins.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | if [[ ${DEBUG} -gt 0 ]]; then set -x; fi 4 | 5 | NETCONFPATH="${NETCONFPATH-/etc/cni/net.d}" 6 | 7 | function exec_list() { 8 | plist="$1" 9 | name="$2" 10 | cniVersion="$3" 11 | echo "$plist" | jq -c '.[]' | while read -r conf; do 12 | plugin_bin="$(echo "$conf" | jq -r '.type')" 13 | conf="$(echo "$conf" | jq -r ".name = \"$name\" | .cniVersion = \"$cniVersion\"")" 14 | if [ -n "$res" ]; then 15 | conf="$(echo "$conf" | jq -r ".prevResult=$res")" 16 | fi 17 | if ! res=$(echo "$conf" | $plugin_bin); then 18 | error "$name" "$res" 19 | elif [[ ${DEBUG} -gt 0 ]]; then 20 | echo "${res}" | jq -r . 21 | fi 22 | done 23 | } 24 | 25 | function error () { 26 | name="$1" 27 | res="$2" 28 | err_msg=$(echo "$res" | jq -r '.msg') 29 | if [ -z "$errmsg" ]; then 30 | err_msg=$res 31 | fi 32 | echo "${name} : error executing $CNI_COMMAND: $err_msg" 33 | exit 1 34 | } 35 | 36 | function exec_plugins() { 37 | i=0 38 | contid=$2 39 | netns=$3 40 | export CNI_COMMAND=$(echo $1 | tr '[:lower:]' '[:upper:]') 41 | export PATH=$CNI_PATH:$PATH 42 | export CNI_CONTAINERID=$contid 43 | export CNI_NETNS=$netns 44 | 45 | for netconf in $(echo "$NETCONFPATH"/*.conf | sort); do 46 | export CNI_IFNAME=$(printf eth%d $i) 47 | name=$(jq -r '.name' <"$netconf") 48 | cniVersion=$(jq -r '.cniVersion' <"$netconf") 49 | plist=$(jq '.plugins | select(.!=null)' <"$netconf") 50 | if [ -n "$plist" ]; then 51 | exec_list "$plist" "$name" "$cniVersion" 52 | else 53 | plugin=$(jq -r '.type' <"$netconf") 54 | 55 | if ! res=$($plugin <"$netconf"); then 56 | error "$name" "$res" 57 | elif [[ ${DEBUG} -gt 0 ]]; then 58 | echo "${res}" | jq -r . 59 | fi 60 | fi 61 | 62 | (( i++ )) || true 63 | done 64 | } 65 | 66 | if [ $# -ne 3 ]; then 67 | echo "Usage: $0 add|del CONTAINER-ID NETNS-PATH" 68 | echo " Adds or deletes the container specified by NETNS-PATH to the networks" 69 | echo " specified in \$NETCONFPATH directory" 70 | exit 1 71 | fi 72 | 73 | exec_plugins $1 $2 $3 74 | -------------------------------------------------------------------------------- /bash-poc/container-vagrant-disallow-rules/host-create.bash: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | N=$1 6 | cp $N-wg0.conf /etc/wireguard/wg0.conf 7 | sysctl -w net.ipv4.ip_forward=1 8 | wg-quick up wg0 9 | -------------------------------------------------------------------------------- /bash-poc/container-vagrant-disallow-rules/peer-container0.conf: -------------------------------------------------------------------------------- 1 | # container 0 - peer host 2 | [Interface] 3 | Address = 10.0.0.3 4 | PrivateKey = EK1DWZkUjrXjQJUnWsbgI5WnOOEWVJDRKBCBNKX3yXI= 5 | PostUp = ip link set lo up 6 | PostUp = ip route add blackhole 10.0.10.3 7 | # wg0 - server host 8 | #[Peer] 9 | #PublicKey = v2XPnF7Yk7KEugBRVyCdD6mssm2Tsc4edDUApwzFoBk= 10 | #AllowedIPs = 10.0.10.1/32 11 | 12 | # wg0 - peer host 13 | [Peer] 14 | Endpoint = 192.168.56.10:51820 15 | PublicKey = WzEKDzUqj4XKiyNwCjWYCuFTHCTZxQGjumj/v+Iytxg= 16 | AllowedIPs = 0.0.0.0/0 17 | 18 | # container 0 - server host 19 | #[Peer] 20 | #PublicKey = X7qQfHNDtiUS5+qg2H5T25IwhcWdVbyoFjGEDTFJ1WU= 21 | #AllowedIps = 10.0.10.3 22 | -------------------------------------------------------------------------------- /bash-poc/container-vagrant-disallow-rules/peer-wg0.conf: -------------------------------------------------------------------------------- 1 | # wg0 on peer 2 | [Interface] 3 | Address = 10.0.0.1 4 | PrivateKey = sLIO8JBk7bOnSVPVlBIYht9fw3w61ZazKNIW9B5CDlw= 5 | ListenPort = 51820 6 | PostUp = iptables -A FORWARD -i %i -j ACCEPT; iptables -A FORWARD -o %i -j ACCEPT; iptables -t nat -A POSTROUTING -j MASQUERADE -s 10.0.0.0/24 7 | PostDown = iptables -D FORWARD -i %i -j ACCEPT; iptables -D FORWARD -o %i -j ACCEPT; iptables -t nat -D POSTROUTING -s 10.0.0.0/24 -j MASQUERADE 8 | 9 | # wg0 on server 10 | [Peer] 11 | Endpoint = 192.168.56.11:51820 12 | PublicKey = v2XPnF7Yk7KEugBRVyCdD6mssm2Tsc4edDUApwzFoBk= 13 | AllowedIPs = 10.0.10.0/24 14 | 15 | # container on peer 16 | [Peer] 17 | PublicKey = j6Dgd8syAcNM/Uu4yGR/1t+dBbDos7utcjzBcXiZIQc= 18 | AllowedIPs = 10.0.0.3/32 19 | 20 | # container on server 21 | #[Peer] 22 | #PublicKey = X7qQfHNDtiUS5+qg2H5T25IwhcWdVbyoFjGEDTFJ1WU= 23 | #AllowedIPs = 10.0.10.3/32 24 | -------------------------------------------------------------------------------- /bash-poc/container-vagrant-disallow-rules/priv-net-run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | if [[ ${DEBUG} -gt 0 ]]; then set -x; fi 4 | 5 | # Run a command in a private network namespace 6 | # set up by CNI plugins 7 | contid=$(printf '%x%x%x%x' $RANDOM $RANDOM $RANDOM $RANDOM) 8 | netnspath=/var/run/netns/$contid 9 | 10 | ip netns add $contid 11 | ./exec-plugins.sh add $contid $netnspath 12 | 13 | 14 | function cleanup() { 15 | ./exec-plugins.sh del $contid $netnspath 16 | ip netns delete $contid 17 | } 18 | trap cleanup EXIT 19 | 20 | ip netns exec $contid "$@" 21 | -------------------------------------------------------------------------------- /bash-poc/container-vagrant-disallow-rules/scratch: -------------------------------------------------------------------------------- 1 | host subnet address privatekey pubkey 2 | server-wg0 10.0.10.0/24 10.0.10.1 CH9Z7veOrPfbKP9anOLEmKDs8PVhWtOtADLSBIktHH8= v2XPnF7Yk7KEugBRVyCdD6mssm2Tsc4edDUApwzFoBk= 3 | peer-wg0 10.0.0.0/24 10.0.0.1 sLIO8JBk7bOnSVPVlBIYht9fw3w61ZazKNIW9B5CDlw= WzEKDzUqj4XKiyNwCjWYCuFTHCTZxQGjumj/v+Iytxg= 4 | container-server 10.0.10.0/24 10.0.10.3 iBDdm9fXmcy+wAl26Ef6zZcnc0Q01oGi8KIVQoYtAFg= X7qQfHNDtiUS5+qg2H5T25IwhcWdVbyoFjGEDTFJ1WU= 5 | container-peer 10.0.0.0/24 10.0.0.3 EK1DWZkUjrXjQJUnWsbgI5WnOOEWVJDRKBCBNKX3yXI= j6Dgd8syAcNM/Uu4yGR/1t+dBbDos7utcjzBcXiZIQc= 6 | -------------------------------------------------------------------------------- /bash-poc/container-vagrant-disallow-rules/server-container0.conf: -------------------------------------------------------------------------------- 1 | # container0 on server host 2 | [Interface] 3 | Address = 10.0.10.3 4 | PrivateKey = iBDdm9fXmcy+wAl26Ef6zZcnc0Q01oGi8KIVQoYtAFg= 5 | PostUp = ip link set lo up 6 | 7 | # wg0 on server 8 | [Peer] 9 | Endpoint = 192.168.56.11:51820 10 | PublicKey = v2XPnF7Yk7KEugBRVyCdD6mssm2Tsc4edDUApwzFoBk= 11 | AllowedIPs = 0.0.0.0/0 12 | 13 | # wg0 on peer 14 | #[Peer] 15 | #PublicKey = WzEKDzUqj4XKiyNwCjWYCuFTHCTZxQGjumj/v+Iytxg= 16 | #AllowedIPs = 10.0.0.1/32 17 | 18 | # container0 on peer 19 | #[Peer] 20 | #PublicKey = j6Dgd8syAcNM/Uu4yGR/1t+dBbDos7utcjzBcXiZIQc= 21 | #AllowedIPs = 10.0.0.3/32 22 | -------------------------------------------------------------------------------- /bash-poc/container-vagrant-disallow-rules/server-wg0.conf: -------------------------------------------------------------------------------- 1 | # server - wg0 2 | [Interface] 3 | Address = 10.0.10.1 4 | PrivateKey = CH9Z7veOrPfbKP9anOLEmKDs8PVhWtOtADLSBIktHH8= 5 | ListenPort = 51820 6 | PostUp = iptables -A FORWARD -i %i -j ACCEPT; iptables -A FORWARD -o %i -j ACCEPT; iptables -t nat -A POSTROUTING -s 10.0.10.0/24 -j MASQUERADE 7 | PostDown = iptables -D FORWARD -i %i -j ACCEPT; iptables -D FORWARD -o %i -j ACCEPT; iptables -t nat -D POSTROUTING -s 10.0.10.0/24 -j MASQUERADE 8 | 9 | # WzEKDzUqj4XKiyNwCjWYCuFTHCTZxQGjumj/v+Iytxg= - peer wg0 10 | # subnet 10.0.0.0/24 11 | [Peer] 12 | Endpoint = 192.168.56.10:51820 13 | PublicKey = WzEKDzUqj4XKiyNwCjWYCuFTHCTZxQGjumj/v+Iytxg= 14 | AllowedIPs = 10.0.0.0/24 15 | 16 | # j6Dgd8syAcNM/Uu4yGR/1t+dBbDos7utcjzBcXiZIQc= - container0 on peer 17 | #[Peer] 18 | #PublicKey = j6Dgd8syAcNM/Uu4yGR/1t+dBbDos7utcjzBcXiZIQc= 19 | #AllowedIPs = 10.0.0.3/32 20 | 21 | # container0 on server 22 | [Peer] 23 | PublicKey = X7qQfHNDtiUS5+qg2H5T25IwhcWdVbyoFjGEDTFJ1WU= 24 | AllowedIPs = 10.0.10.3/32 25 | -------------------------------------------------------------------------------- /bash-poc/container-vagrant/README.md: -------------------------------------------------------------------------------- 1 | # Instructions 2 | 3 | * open two terminals 4 | * `vagrant ssh peer` and `vagrant ssh server` 5 | * `./host-create.sh ` 6 | * `./container-create.sh ` 7 | 8 | Then ping or curl or whatever between them. Using something like echo-server would make it easier to validate -------------------------------------------------------------------------------- /bash-poc/container-vagrant/cni/cni.conf: -------------------------------------------------------------------------------- 1 | { 2 | "cniVersion": "0.4.0", 3 | "name": "wgnet", 4 | "plugins": [ 5 | { 6 | "type": "bridge", 7 | "bridge": "cni0", 8 | "keyA": ["some more", "plugin specific", "configuration"], 9 | "isGateway": true, 10 | "ipam": { 11 | "type": "host-local", 12 | "subnet": "192.168.0.0/24", 13 | "gateway": "192.168.0.1", 14 | "routes": [ 15 | {"dst": "0.0.0.0/0"} 16 | ] 17 | }, 18 | "dns": { 19 | "nameservers": [ "10.1.0.1" ] 20 | } 21 | }, 22 | { 23 | "type": "portmap", 24 | "capabilities": {"portMappings": true} 25 | } 26 | ] 27 | } 28 | -------------------------------------------------------------------------------- /bash-poc/container-vagrant/container-create.bash: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | 3 | set -euof pipefail 4 | 5 | N=$1 6 | 7 | 8 | netnspath=/var/run/netns/container 9 | function catch() { 10 | NETCONFPATH=./cni CNI_PATH=/usr/lib/cni ./exec-plugins.sh del container $netnspath 11 | ip netns del container 12 | } 13 | 14 | trap 'catch' ERR 15 | 16 | ip netns add container || true 17 | 18 | NETCONFPATH=./cni CNI_PATH=/usr/lib/cni ./exec-plugins.sh add container $netnspath 19 | set -x 20 | #ip link add container0 type wireguard 21 | #ip link set container0 netns container 22 | cp $N-container0.conf /etc/wireguard/container0.conf 23 | ip netns exec container wg-quick up container0 24 | -------------------------------------------------------------------------------- /bash-poc/container-vagrant/container-teardown.bash: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | 3 | set -euof pipefail 4 | 5 | N=$1 6 | 7 | 8 | netnspath=/var/run/netns/container 9 | function catch() { 10 | NETCONFPATH=./cni CNI_PATH=/usr/lib/cni ./exec-plugins.sh del container $netnspath 11 | ip netns del container 12 | } 13 | 14 | trap 'catch' ERR 15 | 16 | 17 | NETCONFPATH=./cni CNI_PATH=/usr/lib/cni ./exec-plugins.sh del container $netnspath 18 | ip netns del container 19 | #ip link add container0 type wireguard 20 | #ip link set container0 netns container 21 | -------------------------------------------------------------------------------- /bash-poc/container-vagrant/exec-plugins.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | if [[ ${DEBUG} -gt 0 ]]; then set -x; fi 4 | 5 | NETCONFPATH="${NETCONFPATH-/etc/cni/net.d}" 6 | 7 | function exec_list() { 8 | plist="$1" 9 | name="$2" 10 | cniVersion="$3" 11 | echo "$plist" | jq -c '.[]' | while read -r conf; do 12 | plugin_bin="$(echo "$conf" | jq -r '.type')" 13 | conf="$(echo "$conf" | jq -r ".name = \"$name\" | .cniVersion = \"$cniVersion\"")" 14 | if [ -n "$res" ]; then 15 | conf="$(echo "$conf" | jq -r ".prevResult=$res")" 16 | fi 17 | if ! res=$(echo "$conf" | $plugin_bin); then 18 | error "$name" "$res" 19 | elif [[ ${DEBUG} -gt 0 ]]; then 20 | echo "${res}" | jq -r . 21 | fi 22 | done 23 | } 24 | 25 | function error () { 26 | name="$1" 27 | res="$2" 28 | err_msg=$(echo "$res" | jq -r '.msg') 29 | if [ -z "$errmsg" ]; then 30 | err_msg=$res 31 | fi 32 | echo "${name} : error executing $CNI_COMMAND: $err_msg" 33 | exit 1 34 | } 35 | 36 | function exec_plugins() { 37 | i=0 38 | contid=$2 39 | netns=$3 40 | export CNI_COMMAND=$(echo $1 | tr '[:lower:]' '[:upper:]') 41 | export PATH=$CNI_PATH:$PATH 42 | export CNI_CONTAINERID=$contid 43 | export CNI_NETNS=$netns 44 | 45 | for netconf in $(echo "$NETCONFPATH"/*.conf | sort); do 46 | export CNI_IFNAME=$(printf eth%d $i) 47 | name=$(jq -r '.name' <"$netconf") 48 | cniVersion=$(jq -r '.cniVersion' <"$netconf") 49 | plist=$(jq '.plugins | select(.!=null)' <"$netconf") 50 | if [ -n "$plist" ]; then 51 | exec_list "$plist" "$name" "$cniVersion" 52 | else 53 | plugin=$(jq -r '.type' <"$netconf") 54 | 55 | if ! res=$($plugin <"$netconf"); then 56 | error "$name" "$res" 57 | elif [[ ${DEBUG} -gt 0 ]]; then 58 | echo "${res}" | jq -r . 59 | fi 60 | fi 61 | 62 | (( i++ )) || true 63 | done 64 | } 65 | 66 | if [ $# -ne 3 ]; then 67 | echo "Usage: $0 add|del CONTAINER-ID NETNS-PATH" 68 | echo " Adds or deletes the container specified by NETNS-PATH to the networks" 69 | echo " specified in \$NETCONFPATH directory" 70 | exit 1 71 | fi 72 | 73 | exec_plugins $1 $2 $3 74 | -------------------------------------------------------------------------------- /bash-poc/container-vagrant/host-create.bash: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | N=$1 6 | cp $N-wg0.conf /etc/wireguard/wg0.conf 7 | sysctl -w net.ipv4.ip_forward=1 8 | wg-quick up wg0 9 | -------------------------------------------------------------------------------- /bash-poc/container-vagrant/peer-container0.conf: -------------------------------------------------------------------------------- 1 | # container 0 - peer host 2 | [Interface] 3 | Address = 10.0.0.3 4 | PrivateKey = EK1DWZkUjrXjQJUnWsbgI5WnOOEWVJDRKBCBNKX3yXI= 5 | PostUp = ip link set lo up 6 | 7 | # wg0 - server host 8 | #[Peer] 9 | #PublicKey = v2XPnF7Yk7KEugBRVyCdD6mssm2Tsc4edDUApwzFoBk= 10 | #AllowedIPs = 10.0.10.1/32 11 | 12 | # wg0 - peer host 13 | [Peer] 14 | Endpoint = 192.168.56.10:51820 15 | PublicKey = WzEKDzUqj4XKiyNwCjWYCuFTHCTZxQGjumj/v+Iytxg= 16 | AllowedIPs = 0.0.0.0/0 17 | 18 | # container 0 - server host 19 | #[Peer] 20 | #PublicKey = X7qQfHNDtiUS5+qg2H5T25IwhcWdVbyoFjGEDTFJ1WU= 21 | #AllowedIps = 10.0.10.3 22 | -------------------------------------------------------------------------------- /bash-poc/container-vagrant/peer-wg0.conf: -------------------------------------------------------------------------------- 1 | # wg0 on peer 2 | [Interface] 3 | Address = 10.0.0.1 4 | PrivateKey = sLIO8JBk7bOnSVPVlBIYht9fw3w61ZazKNIW9B5CDlw= 5 | ListenPort = 51820 6 | PostUp = iptables -A FORWARD -i %i -j ACCEPT; iptables -A FORWARD -o %i -j ACCEPT; iptables -t nat -A POSTROUTING -j MASQUERADE -s 10.0.0.0/24 7 | PostDown = iptables -D FORWARD -i %i -j ACCEPT; iptables -D FORWARD -o %i -j ACCEPT; iptables -t nat -D POSTROUTING -s 10.0.0.0/24 -j MASQUERADE 8 | 9 | # wg0 on server 10 | [Peer] 11 | Endpoint = 192.168.56.11:51820 12 | PublicKey = v2XPnF7Yk7KEugBRVyCdD6mssm2Tsc4edDUApwzFoBk= 13 | AllowedIPs = 10.0.10.0/24 14 | 15 | # container on peer 16 | [Peer] 17 | PublicKey = j6Dgd8syAcNM/Uu4yGR/1t+dBbDos7utcjzBcXiZIQc= 18 | AllowedIPs = 10.0.0.3/32 19 | 20 | # container on server 21 | #[Peer] 22 | #PublicKey = X7qQfHNDtiUS5+qg2H5T25IwhcWdVbyoFjGEDTFJ1WU= 23 | #AllowedIPs = 10.0.10.3/32 24 | -------------------------------------------------------------------------------- /bash-poc/container-vagrant/priv-net-run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | if [[ ${DEBUG} -gt 0 ]]; then set -x; fi 4 | 5 | # Run a command in a private network namespace 6 | # set up by CNI plugins 7 | contid=$(printf '%x%x%x%x' $RANDOM $RANDOM $RANDOM $RANDOM) 8 | netnspath=/var/run/netns/$contid 9 | 10 | ip netns add $contid 11 | ./exec-plugins.sh add $contid $netnspath 12 | 13 | 14 | function cleanup() { 15 | ./exec-plugins.sh del $contid $netnspath 16 | ip netns delete $contid 17 | } 18 | trap cleanup EXIT 19 | 20 | ip netns exec $contid "$@" 21 | -------------------------------------------------------------------------------- /bash-poc/container-vagrant/scratch: -------------------------------------------------------------------------------- 1 | host subnet address privatekey pubkey 2 | server-wg0 10.0.10.0/24 10.0.10.1 CH9Z7veOrPfbKP9anOLEmKDs8PVhWtOtADLSBIktHH8= v2XPnF7Yk7KEugBRVyCdD6mssm2Tsc4edDUApwzFoBk= 3 | peer-wg0 10.0.0.0/24 10.0.0.1 sLIO8JBk7bOnSVPVlBIYht9fw3w61ZazKNIW9B5CDlw= WzEKDzUqj4XKiyNwCjWYCuFTHCTZxQGjumj/v+Iytxg= 4 | container-server 10.0.10.0/24 10.0.10.3 iBDdm9fXmcy+wAl26Ef6zZcnc0Q01oGi8KIVQoYtAFg= X7qQfHNDtiUS5+qg2H5T25IwhcWdVbyoFjGEDTFJ1WU= 5 | container-peer 10.0.0.0/24 10.0.0.3 EK1DWZkUjrXjQJUnWsbgI5WnOOEWVJDRKBCBNKX3yXI= j6Dgd8syAcNM/Uu4yGR/1t+dBbDos7utcjzBcXiZIQc= 6 | -------------------------------------------------------------------------------- /bash-poc/container-vagrant/server-container0.conf: -------------------------------------------------------------------------------- 1 | # container0 on server host 2 | [Interface] 3 | Address = 10.0.10.3 4 | PrivateKey = iBDdm9fXmcy+wAl26Ef6zZcnc0Q01oGi8KIVQoYtAFg= 5 | PostUp = ip link set lo up 6 | 7 | # wg0 on server 8 | [Peer] 9 | Endpoint = 192.168.56.11:51820 10 | PublicKey = v2XPnF7Yk7KEugBRVyCdD6mssm2Tsc4edDUApwzFoBk= 11 | AllowedIPs = 0.0.0.0/0 12 | 13 | # wg0 on peer 14 | #[Peer] 15 | #PublicKey = WzEKDzUqj4XKiyNwCjWYCuFTHCTZxQGjumj/v+Iytxg= 16 | #AllowedIPs = 10.0.0.1/32 17 | 18 | # container0 on peer 19 | #[Peer] 20 | #PublicKey = j6Dgd8syAcNM/Uu4yGR/1t+dBbDos7utcjzBcXiZIQc= 21 | #AllowedIPs = 10.0.0.3/32 22 | -------------------------------------------------------------------------------- /bash-poc/container-vagrant/server-wg0.conf: -------------------------------------------------------------------------------- 1 | # server - wg0 2 | [Interface] 3 | Address = 10.0.10.1 4 | PrivateKey = CH9Z7veOrPfbKP9anOLEmKDs8PVhWtOtADLSBIktHH8= 5 | ListenPort = 51820 6 | PostUp = iptables -A FORWARD -i %i -j ACCEPT; iptables -A FORWARD -o %i -j ACCEPT; iptables -t nat -A POSTROUTING -s 10.0.10.0/24 -j MASQUERADE 7 | PostDown = iptables -D FORWARD -i %i -j ACCEPT; iptables -D FORWARD -o %i -j ACCEPT; iptables -t nat -D POSTROUTING -s 10.0.10.0/24 -j MASQUERADE 8 | 9 | # WzEKDzUqj4XKiyNwCjWYCuFTHCTZxQGjumj/v+Iytxg= - peer wg0 10 | # subnet 10.0.0.0/24 11 | [Peer] 12 | Endpoint = 192.168.56.10:51820 13 | PublicKey = WzEKDzUqj4XKiyNwCjWYCuFTHCTZxQGjumj/v+Iytxg= 14 | AllowedIPs = 10.0.0.0/24 15 | 16 | # j6Dgd8syAcNM/Uu4yGR/1t+dBbDos7utcjzBcXiZIQc= - container0 on peer 17 | #[Peer] 18 | #PublicKey = j6Dgd8syAcNM/Uu4yGR/1t+dBbDos7utcjzBcXiZIQc= 19 | #AllowedIPs = 10.0.0.3/32 20 | 21 | # container0 on server 22 | [Peer] 23 | PublicKey = X7qQfHNDtiUS5+qg2H5T25IwhcWdVbyoFjGEDTFJ1WU= 24 | AllowedIPs = 10.0.10.3/32 25 | -------------------------------------------------------------------------------- /bash-poc/experiments/create-ns.bash: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -eof pipefail 4 | 5 | 6 | 7 | sudo ./create.bash ns1 10.0.0.1 8 | sudo ./create.bash ns2 10.0.0.2 9 | sudo ./create.bash ns3 10.0.0.3 10 | 11 | 12 | for i in $(ls -d --color=never ns*/*.pub); do 13 | sudo ip netns exec ns1 wg set peer $i 14 | sudo ip netns exec ns2 wg set peer $i 15 | sudo ip netns exec ns3 wg set peer $i 16 | done 17 | #for-each-peer ip netns exec wg set peer $(cat $i/$i.pub) 18 | #set -x 19 | #for i in $(ls -d --color=never *); do 20 | # if [[ -d $i ]]; then 21 | # ip netns exec $i wg set $i peer $(cat $i/$i.pub) 22 | # fi 23 | #done 24 | 25 | 26 | -------------------------------------------------------------------------------- /bash-poc/experiments/create.bash: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -euo pipefail 4 | set -x 5 | [[ $UID == 0 ]] || { echo "You must be root to run this."; exit 1; } 6 | NAME=$1 7 | ADDRESS=$2 8 | mkdir -p $NAME 9 | wg genkey > $NAME/$NAME.key 10 | wg pubkey <$NAME/$NAME.key > $NAME/$NAME.pub 11 | set -x 12 | ip netns add $NAME || true 13 | ip link del dev $NAME 2>/dev/null || true 14 | ip -link add dev $NAME type wireguard 15 | ip link set $NAME netns $NAME 16 | 17 | 18 | ip netns exec $NAME wg set $NAME private-key $NAME/$NAME.key listen-port 0 19 | ip -n $NAME address add "${ADDRESS}/24" dev $NAME 20 | ip -n $NAME link set $NAME up 21 | ip -n $NAME link set lo up 22 | #ip -n $NAME route add default dev ns1 23 | listen_port=$(ip netns exec $NAME wg show $NAME listen-port) 24 | cat << /etc/wireguard/$NAME.conf 25 | [Interface] 26 | PrivateKey = $(cat $NAME/$NAME.key) 27 | Address = 10.8.0.1/24 28 | ListenPort = $listen_port 29 | EOF 30 | -------------------------------------------------------------------------------- /bash-poc/experiments/host/host.key: -------------------------------------------------------------------------------- 1 | oMK7hXtInDk3XkVd4stIR6lQh7qToVBcSSDOR2Aff2o= 2 | -------------------------------------------------------------------------------- /bash-poc/experiments/host/host.pub: -------------------------------------------------------------------------------- 1 | txmzk0NvEPQzGL0EBnKobXphWnJ8TfU0sILqhT3qaAk= 2 | -------------------------------------------------------------------------------- /bash-poc/experiments/lib.bash: -------------------------------------------------------------------------------- 1 | 2 | function for-each-peer() { 3 | for i in $(ls -d --color=never *); do 4 | if [[ -d $i ]]; then 5 | echo ip netns exec $i $@ 6 | fi 7 | done 8 | } 9 | -------------------------------------------------------------------------------- /bash-poc/experiments/ns1/ns1.key: -------------------------------------------------------------------------------- 1 | oB+UUeqgzlACwuGP2KTouTaWg3kxi410J3UtXkvi0WM= 2 | -------------------------------------------------------------------------------- /bash-poc/experiments/ns1/ns1.pub: -------------------------------------------------------------------------------- 1 | cZtu2Qja4DIOhrnrt/4aMRjHuoC8KWyXyQIndFGPblo= 2 | -------------------------------------------------------------------------------- /bash-poc/experiments/ns2/ns2.key: -------------------------------------------------------------------------------- 1 | eIfNtvIouAEzEaYIOn3qG92btQ0rafLkD0dqHWTdJV8= 2 | -------------------------------------------------------------------------------- /bash-poc/experiments/ns2/ns2.pub: -------------------------------------------------------------------------------- 1 | mi77Bg+pQfoY62l8+m7CvU/35yE4jpePHme8nlb0Ugo= 2 | -------------------------------------------------------------------------------- /bash-poc/experiments/ns3/ns3.key: -------------------------------------------------------------------------------- 1 | 8N9yHH853nyrGx/U+5nPbNqCipResbhUQAySExON2E4= 2 | -------------------------------------------------------------------------------- /bash-poc/experiments/ns3/ns3.pub: -------------------------------------------------------------------------------- 1 | NbF7OfqshmKADgqJt6nx2/9fLwt7XebYvRPqTODt3UM= 2 | -------------------------------------------------------------------------------- /bash-poc/experiments/teardown.bash: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | for i in $(ls -d --color=never *); do 4 | if [[ -d $i ]]; then 5 | ip -n $i link set $i down 6 | ip link del $i 7 | ip -n $i link del $i 8 | for ns in $(ip netns list|cut -f1 -d ' '); do 9 | echo $ns 10 | ip netns delete $ns 11 | done 12 | fi 13 | done 14 | -------------------------------------------------------------------------------- /bash-poc/experiments/veth.bash: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | N=$1 4 | ADDR=$2 5 | ip netns add vnet$N 6 | ip link add veth$N type veth peer name vpeer$N 7 | ip link set veth$N netns vnet$N 8 | ip -n vnet$N addr add $ADDR/24 dev veth$N 9 | ip -n vnet$N link set veth$N up 10 | ip -n vnet$N link set lo up 11 | ip -n vnet$N addr show 12 | 13 | 14 | -------------------------------------------------------------------------------- /bash-poc/static-vagrant/create.bash: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | N=$1 6 | cp $N-wg0.conf /etc/wireguard/wg0.conf 7 | wg-quick up wg0 8 | -------------------------------------------------------------------------------- /bash-poc/static-vagrant/peer-wg0.conf: -------------------------------------------------------------------------------- 1 | [Interface] 2 | Address = 10.0.0.2 3 | PrivateKey = sLIO8JBk7bOnSVPVlBIYht9fw3w61ZazKNIW9B5CDlw= 4 | ListenPort = 51820 5 | 6 | [Peer] 7 | Endpoint = 192.168.56.11:51820 8 | PublicKey = v2XPnF7Yk7KEugBRVyCdD6mssm2Tsc4edDUApwzFoBk= 9 | AllowedIPs = 10.0.0.0/8 10 | -------------------------------------------------------------------------------- /bash-poc/static-vagrant/server-wg0.conf: -------------------------------------------------------------------------------- 1 | [Interface] 2 | Address = 10.0.0.1 3 | PrivateKey = CH9Z7veOrPfbKP9anOLEmKDs8PVhWtOtADLSBIktHH8= 4 | ListenPort = 51820 5 | 6 | [Peer] 7 | Endpoint = 192.168.56.10:51820 8 | PublicKey = WzEKDzUqj4XKiyNwCjWYCuFTHCTZxQGjumj/v+Iytxg= 9 | AllowedIPs = 10.0.0.0/8 10 | -------------------------------------------------------------------------------- /bash-poc/wg-cni/README.md: -------------------------------------------------------------------------------- 1 | # Instructions 2 | 3 | * open two terminals 4 | * `vagrant ssh peer` and `vagrant ssh server` 5 | * `./host-create.sh ` 6 | * `./container-create.sh ` 7 | 8 | Then ping or curl or whatever between them. Using something like echo-server would make it easier to validate -------------------------------------------------------------------------------- /bash-poc/wg-cni/cni/cni.conf: -------------------------------------------------------------------------------- 1 | { 2 | "cniVersion": "0.4.0", 3 | "name": "wgnet", 4 | "plugins": [ 5 | { 6 | "type": "bridge", 7 | "bridge": "cni0", 8 | "keyA": ["some more", "plugin specific", "configuration"], 9 | "isGateway": true, 10 | "ipMasq": false, 11 | "ipam": { 12 | "type": "host-local", 13 | "subnet": "192.168.0.0/24", 14 | "gateway": "192.168.0.1", 15 | "routes": [ 16 | {"dst": "0.0.0.0/0"} 17 | ] 18 | }, 19 | "dns": { 20 | "nameservers": [ "10.1.0.1" ] 21 | } 22 | }, 23 | { 24 | "type": "wireguard", 25 | "nodeManagerAddr": "http://localhost:5000" 26 | } 27 | ] 28 | } 29 | -------------------------------------------------------------------------------- /bash-poc/wg-cni/container-create.bash: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | 3 | set -euof pipefail 4 | 5 | netnspath=/var/run/netns/container 6 | function catch() { 7 | NETCONFPATH=./cni CNI_PATH=/usr/lib/cni ./exec-plugins.sh del container $netnspath 8 | ip netns del container 9 | } 10 | 11 | trap 'catch' ERR 12 | 13 | ip netns add container || true 14 | 15 | set -x 16 | NETCONFPATH=./cni CNI_PATH=/usr/lib/cni ./exec-plugins.sh add container $netnspath 17 | 18 | #ip link add container0 type wireguard 19 | #ip link set container0 netns container 20 | #cp $N-container0.conf /etc/wireguard/container0.conf 21 | #ip netns exec container wg-quick up container0 22 | -------------------------------------------------------------------------------- /bash-poc/wg-cni/container-teardown.bash: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | 3 | set -euof pipefail 4 | 5 | N=$1 6 | 7 | 8 | netnspath=/var/run/netns/container 9 | function catch() { 10 | NETCONFPATH=./cni CNI_PATH=/usr/lib/cni ./exec-plugins.sh del container $netnspath 11 | ip netns del container 12 | } 13 | 14 | trap 'catch' ERR 15 | 16 | 17 | NETCONFPATH=./cni CNI_PATH=/usr/lib/cni ./exec-plugins.sh del container $netnspath 18 | ip netns del container 19 | #ip link add container0 type wireguard 20 | #ip link set container0 netns container 21 | -------------------------------------------------------------------------------- /bash-poc/wg-cni/dev.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -euf pipefail 4 | 5 | cp ../../bin/cmd/cni /usr/lib/cni/wireguard 6 | ./container-teardown.bash container || true 7 | ./container-create.bash container 8 | -------------------------------------------------------------------------------- /bash-poc/wg-cni/exec-plugins.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | if [[ ${DEBUG} -gt 0 ]]; then set -x; fi 4 | 5 | NETCONFPATH="${NETCONFPATH-/etc/cni/net.d}" 6 | 7 | function exec_list() { 8 | plist="$1" 9 | name="$2" 10 | cniVersion="$3" 11 | echo "$plist" | jq -c '.[]' | while read -r conf; do 12 | plugin_bin="$(echo "$conf" | jq -r '.type')" 13 | conf="$(echo "$conf" | jq -r ".name = \"$name\" | .cniVersion = \"$cniVersion\"")" 14 | if [ -n "$res" ]; then 15 | conf="$(echo "$conf" | jq -r ".prevResult=$res")" 16 | fi 17 | if ! res=$(echo "$conf" | $plugin_bin); then 18 | echo "========" 19 | echo "input: $conf | $plugin_bin" 20 | echo "result: $res" 21 | echo "========" 22 | error "$name" "$res" 23 | elif [[ ${DEBUG} -gt 0 ]]; then 24 | echo "${res}" | jq -r . 25 | fi 26 | done 27 | } 28 | 29 | function error () { 30 | name="$1" 31 | res="$2" 32 | err_msg=$(echo "$res" | jq -r '.msg') 33 | if [ -z "$errmsg" ]; then 34 | err_msg=$res 35 | fi 36 | echo "${name} : error executing $CNI_COMMAND: $err_msg" 37 | exit 1 38 | } 39 | 40 | function exec_plugins() { 41 | i=0 42 | contid=$2 43 | netns=$3 44 | export CNI_COMMAND=$(echo $1 | tr '[:lower:]' '[:upper:]') 45 | export PATH=$CNI_PATH:$PATH 46 | export CNI_CONTAINERID=$contid 47 | export CNI_NETNS=$netns 48 | 49 | for netconf in $(echo "$NETCONFPATH"/*.conf | sort); do 50 | export CNI_IFNAME=$(printf eth%d $i) 51 | name=$(jq -r '.name' <"$netconf") 52 | cniVersion=$(jq -r '.cniVersion' <"$netconf") 53 | plist=$(jq '.plugins | select(.!=null)' <"$netconf") 54 | if [ -n "$plist" ]; then 55 | exec_list "$plist" "$name" "$cniVersion" 56 | else 57 | plugin=$(jq -r '.type' <"$netconf") 58 | 59 | if ! res=$($plugin <"$netconf"); then 60 | error "$name" "$res" 61 | elif [[ ${DEBUG} -gt 0 ]]; then 62 | echo "${res}" | jq -r . 63 | fi 64 | fi 65 | 66 | (( i++ )) || true 67 | done 68 | } 69 | 70 | if [ $# -ne 3 ]; then 71 | echo "Usage: $0 add|del CONTAINER-ID NETNS-PATH" 72 | echo " Adds or deletes the container specified by NETNS-PATH to the networks" 73 | echo " specified in \$NETCONFPATH directory" 74 | exit 1 75 | fi 76 | 77 | exec_plugins $1 $2 $3 78 | -------------------------------------------------------------------------------- /bash-poc/wg-cni/host-create.bash: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | N=$1 6 | cp $N-wg0.conf /etc/wireguard/wg0.conf 7 | sysctl -w net.ipv4.ip_forward=1 8 | wg-quick up wg0 9 | -------------------------------------------------------------------------------- /bash-poc/wg-cni/input: -------------------------------------------------------------------------------- 1 | { 2 | "type": "wireguard", 3 | "nodeManagerAddr": "http://localhost:5000", 4 | "name": "wgnet", 5 | "cniVersion": "0.4.0", 6 | "prevResult": { 7 | "cniVersion": "0.4.0", 8 | "interfaces": [ 9 | { 10 | "name": "cni0", 11 | "mac": "fa:1d:89:9b:3c:ab" 12 | }, 13 | { 14 | "name": "vethb60ff325", 15 | "mac": "a2:46:f8:76:b5:93" 16 | }, 17 | { 18 | "name": "eth0", 19 | "mac": "c6:11:77:f8:9d:40", 20 | "sandbox": "/var/run/netns/container" 21 | } 22 | ], 23 | "ips": [ 24 | { 25 | "version": "4", 26 | "interface": 2, 27 | "address": "192.168.0.27/24", 28 | "gateway": "192.168.0.1" 29 | } 30 | ], 31 | "routes": [ 32 | { 33 | "dst": "0.0.0.0/0" 34 | } 35 | ], 36 | "dns": { 37 | "nameservers": [ 38 | "10.1.0.1" 39 | ] 40 | } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /bash-poc/wg-cni/priv-net-run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | if [[ ${DEBUG} -gt 0 ]]; then set -x; fi 4 | 5 | # Run a command in a private network namespace 6 | # set up by CNI plugins 7 | contid=$(printf '%x%x%x%x' $RANDOM $RANDOM $RANDOM $RANDOM) 8 | netnspath=/var/run/netns/$contid 9 | 10 | ip netns add $contid 11 | ./exec-plugins.sh add $contid $netnspath 12 | 13 | 14 | function cleanup() { 15 | ./exec-plugins.sh del $contid $netnspath 16 | ip netns delete $contid 17 | } 18 | trap cleanup EXIT 19 | 20 | ip netns exec $contid "$@" 21 | -------------------------------------------------------------------------------- /buf.gen.yaml: -------------------------------------------------------------------------------- 1 | version: v1 2 | plugins: 3 | - name: go 4 | out: gen 5 | opt: paths=source_relative 6 | - name: connect-go 7 | out: gen 8 | opt: paths=source_relative -------------------------------------------------------------------------------- /buf.yaml: -------------------------------------------------------------------------------- 1 | version: v1 2 | breaking: 3 | use: 4 | - FILE 5 | lint: 6 | use: 7 | - DEFAULT 8 | - COMMENT_RPC 9 | -------------------------------------------------------------------------------- /cmd/cluster-manager/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "expvar" 6 | "flag" 7 | "log" 8 | "net/http" 9 | "os" 10 | "path" 11 | 12 | "golang.org/x/net/http2" 13 | "golang.org/x/net/http2/h2c" 14 | 15 | "github.com/clly/wireguard-cni/gen/wgcni/ipam/v1/ipamv1connect" 16 | "github.com/clly/wireguard-cni/gen/wgcni/wireguard/v1/wireguardv1connect" 17 | "github.com/clly/wireguard-cni/pkg/ipam" 18 | "github.com/clly/wireguard-cni/pkg/server" 19 | ) 20 | 21 | var ( 22 | _ ipamv1connect.IPAMServiceHandler = &server.Server{} 23 | _ wireguardv1connect.WireguardServiceHandler = &server.Server{} 24 | ) 25 | 26 | type ClusterManagerConfig struct { 27 | prefix string 28 | dataDirectory string 29 | } 30 | 31 | func main() { 32 | log.SetFlags(log.Lshortfile | log.Ldate | log.Ltime) 33 | log.Println("initializing cluster-manager") 34 | c := config() 35 | log.Println("initializing server") 36 | 37 | wd, err := os.Getwd() 38 | if err != nil { 39 | log.Fatal(err) 40 | } 41 | if c.dataDirectory == "" { 42 | c.dataDirectory = path.Join(wd, "cluster-manager") 43 | } 44 | 45 | clusterIpam, err := ipam.New(context.TODO(), c.dataDirectory, c.prefix) 46 | if err != nil { 47 | log.Fatal(err) 48 | } 49 | 50 | s, err := server.NewServer(clusterIpam, server.WithDataDir(c.dataDirectory)) 51 | 52 | if err != nil { 53 | log.Fatal(err) 54 | } 55 | log.Println("initializing serve mux") 56 | mux := http.NewServeMux() 57 | 58 | path, handler := ipamv1connect.NewIPAMServiceHandler(s) 59 | log.Println("Registering IPAM Handler on", path) 60 | mux.Handle(path, handler) 61 | 62 | path, handler = wireguardv1connect.NewWireguardServiceHandler(s) 63 | log.Println("Registering Wireguard Handler on ", path) 64 | mux.Handle(path, handler) 65 | 66 | mux.Handle("/debug/varz", expvar.Handler()) 67 | log.Println("listening :8080 ...") 68 | log.Fatal(http.ListenAndServe(":8080", h2c.NewHandler(mux, &http2.Server{}))) 69 | } 70 | 71 | func config() ClusterManagerConfig { 72 | cidrPrefix := flag.String("cidr-prefix", "10.0.0.0/8", "IPAM CIDR prefix") 73 | dataDir := flag.String("data-dir", "", "Data directory to store ipam and wireguard files in") 74 | flag.Parse() 75 | return ClusterManagerConfig{ 76 | prefix: *cidrPrefix, 77 | dataDirectory: *dataDir, 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /cmd/cni/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 CNI authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // This is a sample chained plugin that supports multiple CNI versions. It 16 | // parses prevResult according to the cniVersion 17 | package main 18 | 19 | import ( 20 | "context" 21 | "encoding/json" 22 | "fmt" 23 | "os" 24 | "path/filepath" 25 | "time" 26 | 27 | "github.com/containernetworking/cni/pkg/skel" 28 | "github.com/containernetworking/cni/pkg/types" 29 | current "github.com/containernetworking/cni/pkg/types/100" 30 | "github.com/containernetworking/cni/pkg/version" 31 | "github.com/containernetworking/plugins/pkg/ns" 32 | bv "github.com/containernetworking/plugins/pkg/utils/buildversion" 33 | ) 34 | 35 | // PluginConf is whatever you expect your configuration json to be. This is whatever 36 | // is passed in on stdin. Your plugin may wish to expose its functionality via 37 | // runtime args, see CONVENTIONS.md in the CNI spec. 38 | type PluginConf struct { 39 | // This embeds the standard NetConf structure which allows your plugin 40 | // to more easily parse standard fields like Name, Type, CNIVersion, 41 | // and PrevResult. 42 | types.NetConf 43 | 44 | RuntimeConfig *struct { 45 | SampleConfig map[string]interface{} `json:"sample"` 46 | } `json:"runtimeConfig"` 47 | 48 | // Add plugin-specifc flags here 49 | NodeManagerAddr string `json:"nodeManagerAddr"` 50 | } 51 | 52 | // parseConfig parses the supplied configuration (and prevResult) from stdin. 53 | func parseConfig(stdin []byte) (*PluginConf, error) { 54 | conf := PluginConf{} 55 | 56 | if err := json.Unmarshal(stdin, &conf); err != nil { 57 | return nil, fmt.Errorf("failed to parse network configuration: %v", err) 58 | } 59 | 60 | // fmt.Fprintf(os.Stderr, "%s", stdin) 61 | // fmt.Fprintf(os.Stderr, "%#v", conf) 62 | // Parse previous result. This will parse, validate, and place the 63 | // previous result object into conf.PrevResult. If you need to modify 64 | // or inspect the PrevResult you will need to convert it to a concrete 65 | // versioned Result struct. 66 | if err := version.ParsePrevResult(&conf.NetConf); err != nil { 67 | return nil, fmt.Errorf("could not parse prevResult: %v", err) 68 | } 69 | // End previous result parsing 70 | 71 | // Do any validation here 72 | if conf.NodeManagerAddr != "" { 73 | conf.NodeManagerAddr = "http://localhost:5242" 74 | } 75 | 76 | return &conf, nil 77 | } 78 | 79 | // cmdAdd is called for ADD requests 80 | func cmdAdd(args *skel.CmdArgs) error { 81 | conf, err := parseConfig(args.StdinData) 82 | if err != nil { 83 | return err 84 | } 85 | 86 | // A plugin can be either an "originating" plugin or a "chained" plugin. 87 | // Originating plugins perform initial sandbox setup and do not require 88 | // any result from a previous plugin in the chain. A chained plugin 89 | // modifies sandbox configuration that was previously set up by an 90 | // originating plugin and may optionally require a PrevResult from 91 | // earlier plugins in the chain. 92 | 93 | // START chained plugin code 94 | if conf.PrevResult == nil { 95 | return fmt.Errorf("must be called as chained plugin") 96 | } 97 | 98 | // Convert the PrevResult to a concrete Result type that can be modified. 99 | prevResult, err := current.GetResult(conf.PrevResult) 100 | if err != nil { 101 | return fmt.Errorf("failed to convert prevResult: %v", err) 102 | } 103 | 104 | if len(prevResult.IPs) == 0 { 105 | return fmt.Errorf("got no container IPs") 106 | } 107 | 108 | // Pass the prevResult through this plugin to the next one 109 | result := prevResult 110 | 111 | // We're going to hard code this because I don't have time to really mess around with this 112 | 113 | // END chained plugin code 114 | 115 | netns, err := ns.GetNS(args.Netns) 116 | if err != nil { 117 | return err 118 | } 119 | 120 | var device string 121 | ctx := context.Background() 122 | err = netns.Do(func(netns ns.NetNS) error { 123 | fmt.Fprintln(os.Stderr, "netns: ", args.Netns) 124 | fmt.Fprintln(os.Stderr, "netns name: ", filepath.Base(args.Netns)) 125 | time.Sleep(1 * time.Second) 126 | device, err = addWgInterface(ctx, *conf, result, netns) 127 | 128 | ip, iface, err := getResult(device, args.Netns) 129 | 130 | if err != nil { 131 | return fmt.Errorf("failed to get result %w", err) 132 | } 133 | 134 | fmt.Fprintln(os.Stderr, "IP: ", ip) 135 | fmt.Fprintln(os.Stderr, "Interface: ", iface) 136 | ip.Interface = ptr(0) 137 | ifaces := make([]*current.Interface, len(result.Interfaces)+1) 138 | ifaces[0] = &iface 139 | // append our interfaces to the new slice 140 | copy(ifaces[1:], result.Interfaces) 141 | result.Interfaces = ifaces 142 | result.IPs = append(result.IPs, &ip) 143 | 144 | return err 145 | }) 146 | if err != nil { 147 | return fmt.Errorf("failed to add interface %w", err) 148 | } 149 | 150 | // START originating plugin code 151 | // if conf.PrevResult != nil { 152 | // return fmt.Errorf("must be called as the first plugin") 153 | // } 154 | 155 | // Generate some fake container IPs and add to the result 156 | // result := ¤t.Result{CNIVersion: current.ImplementedSpecVersion} 157 | // result.Interfaces = []*current.Interface{ 158 | // { 159 | // Name: "intf0", 160 | // Sandbox: args.Netns, 161 | // Mac: "00:11:22:33:44:55", 162 | // }, 163 | // } 164 | // result.IPs = []*current.IPConfig{ 165 | // { 166 | // Address: "1.2.3.4/24", 167 | // Gateway: "1.2.3.1", 168 | // // Interface is an index into the Interfaces array 169 | // // of the Interface element this IP applies to 170 | // Interface: current.Int(0), 171 | // } 172 | // } 173 | // END originating plugin code 174 | 175 | // Implement your plugin here 176 | 177 | // Pass through the result for the next plugin 178 | return types.PrintResult(result, conf.CNIVersion) 179 | } 180 | 181 | // cmdDel is called for DELETE requests 182 | func cmdDel(args *skel.CmdArgs) error { 183 | conf, err := parseConfig(args.StdinData) 184 | if err != nil { 185 | return err 186 | } 187 | _ = conf 188 | 189 | // Do your delete here 190 | 191 | return nil 192 | } 193 | 194 | func main() { 195 | // replace TODO with your plugin name 196 | skel.PluginMain(cmdAdd, cmdCheck, cmdDel, version.All, bv.BuildString("TODO")) 197 | } 198 | 199 | func cmdCheck(args *skel.CmdArgs) error { 200 | // TODO: implement 201 | return fmt.Errorf("not implemented") 202 | } 203 | 204 | func ptr[a any](val a) *a { 205 | return &val 206 | } 207 | -------------------------------------------------------------------------------- /cmd/cni/sample_linux_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 CNI authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "fmt" 19 | 20 | "github.com/containernetworking/cni/pkg/skel" 21 | "github.com/containernetworking/plugins/pkg/ns" 22 | "github.com/containernetworking/plugins/pkg/testutils" 23 | . "github.com/onsi/ginkgo" 24 | . "github.com/onsi/gomega" 25 | ) 26 | 27 | var _ = Describe("sample test", func() { 28 | var targetNs ns.NetNS 29 | 30 | BeforeEach(func() { 31 | var err error 32 | targetNs, err = testutils.NewNS() 33 | Expect(err).NotTo(HaveOccurred()) 34 | }) 35 | 36 | AfterEach(func() { 37 | targetNs.Close() 38 | }) 39 | 40 | It("Works with a 0.3.0 config", func() { 41 | ifname := "eth0" 42 | conf := `{ 43 | "cniVersion": "0.3.0", 44 | "name": "cni-plugin-sample-test", 45 | "type": "sample", 46 | "anotherAwesomeArg": "awesome", 47 | "prevResult": { 48 | "cniVersion": "0.3.0", 49 | "interfaces": [ 50 | { 51 | "name": "%s", 52 | "sandbox": "%s" 53 | } 54 | ], 55 | "ips": [ 56 | { 57 | "version": "4", 58 | "address": "10.0.0.2/24", 59 | "gateway": "10.0.0.1", 60 | "interface": 0 61 | } 62 | ], 63 | "routes": [] 64 | } 65 | }` 66 | conf = fmt.Sprintf(conf, ifname, targetNs.Path()) 67 | args := &skel.CmdArgs{ 68 | ContainerID: "dummy", 69 | Netns: targetNs.Path(), 70 | IfName: ifname, 71 | StdinData: []byte(conf), 72 | } 73 | _, _, err := testutils.CmdAddWithArgs(args, func() error { return cmdAdd(args) }) 74 | Expect(err).NotTo(HaveOccurred()) 75 | }) 76 | 77 | It("fails an invalid config", func() { 78 | conf := `{ 79 | "cniVersion": "0.3.0", 80 | "name": "cni-plugin-sample-test", 81 | "type": "sample", 82 | "prevResult": { 83 | "interfaces": [ 84 | { 85 | "name": "eth0", 86 | "sandbox": "/var/run/netns/test" 87 | } 88 | ], 89 | "ips": [ 90 | { 91 | "version": "4", 92 | "address": "10.0.0.2/24", 93 | "gateway": "10.0.0.1", 94 | "interface": 0 95 | } 96 | ], 97 | "routes": [] 98 | } 99 | }` 100 | 101 | args := &skel.CmdArgs{ 102 | ContainerID: "dummy", 103 | Netns: targetNs.Path(), 104 | IfName: "eth0", 105 | StdinData: []byte(conf), 106 | } 107 | _, _, err := testutils.CmdAddWithArgs(args, func() error { return cmdAdd(args) }) 108 | Expect(err).To(MatchError("anotherAwesomeArg must be specified")) 109 | }) 110 | 111 | It("fails with CNI spec versions that don't support plugin chaining", func() { 112 | conf := `{ 113 | "cniVersion": "0.2.0", 114 | "name": "cni-plugin-sample-test", 115 | "type": "sample", 116 | "anotherAwesomeArg": "foo" 117 | }` 118 | 119 | args := &skel.CmdArgs{ 120 | ContainerID: "dummy", 121 | Netns: targetNs.Path(), 122 | IfName: "eth0", 123 | StdinData: []byte(conf), 124 | } 125 | _, _, err := testutils.CmdAddWithArgs(args, func() error { return cmdAdd(args) }) 126 | Expect(err).To(MatchError("must be called as chained plugin")) 127 | }) 128 | 129 | }) 130 | -------------------------------------------------------------------------------- /cmd/cni/sample_suite_test.go: -------------------------------------------------------------------------------- 1 | // The boilerplate needed for Ginkgo 2 | 3 | package main 4 | 5 | import ( 6 | . "github.com/onsi/ginkgo" 7 | . "github.com/onsi/gomega" 8 | 9 | "testing" 10 | ) 11 | 12 | func TestSample(t *testing.T) { 13 | t.Skip() 14 | RegisterFailHandler(Fail) 15 | RunSpecs(t, "plugins/sample") 16 | } 17 | -------------------------------------------------------------------------------- /cmd/cni/wireguard.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "encoding/hex" 6 | "fmt" 7 | "io" 8 | "io/ioutil" 9 | "log" 10 | "math/rand" 11 | "net" 12 | "os" 13 | "path/filepath" 14 | "time" 15 | 16 | "connectrpc.com/connect" 17 | ipamv1 "github.com/clly/wireguard-cni/gen/wgcni/ipam/v1" 18 | "github.com/clly/wireguard-cni/gen/wgcni/ipam/v1/ipamv1connect" 19 | "github.com/clly/wireguard-cni/gen/wgcni/wireguard/v1/wireguardv1connect" 20 | "github.com/clly/wireguard-cni/pkg/wireguard" 21 | current "github.com/containernetworking/cni/pkg/types/100" 22 | "github.com/containernetworking/plugins/pkg/ns" 23 | "github.com/hashicorp/go-cleanhttp" 24 | "golang.zx2c4.com/wireguard/wgctrl" 25 | ) 26 | 27 | // type wgResult struct {} 28 | 29 | func addWgInterface(ctx context.Context, cfg PluginConf, result *current.Result, netns ns.NetNS) (string, error) { 30 | 31 | f, err := ioutil.TempFile("/tmp", "wireguard") 32 | if err != nil { 33 | f.Close() 34 | } 35 | 36 | var device string 37 | err = netns.Do(func(nn ns.NetNS) error { 38 | ip := result.IPs[0].Address.IP.String() 39 | 40 | ipamClient := ipamv1connect.NewIPAMServiceClient(cleanhttp.DefaultClient(), cfg.NodeManagerAddr) 41 | wireguardClient := wireguardv1connect.NewWireguardServiceClient(cleanhttp.DefaultClient(), cfg.NodeManagerAddr) 42 | 43 | resp, err := ipamClient.Alloc(ctx, connect.NewRequest(&ipamv1.AllocRequest{})) 44 | if err != nil { 45 | fmt.Fprintln(os.Stderr, "failed to request alloc", err) 46 | return err 47 | } 48 | 49 | addr := net.JoinHostPort(ip, "51820") 50 | // log.Println("Using", ip, "as wireguard endpoint") 51 | // log.Println("Using", addr, "as wireguard interface address") 52 | 53 | fmt.Fprintf(os.Stderr, "Namespace Path: %#v\n", nn.Path()) 54 | 55 | wgAddr := resp.Msg.Alloc.Address 56 | 57 | cidr := fmt.Sprintf("%s/%s", resp.Msg.Alloc.Address, resp.Msg.Alloc.Netmask) 58 | 59 | device = fmt.Sprintf("wg%s", randomString()) 60 | wgConf := wireguard.Config{ 61 | Address: wgAddr, 62 | Endpoint: addr, 63 | Route: cidr, 64 | Namespace: nn.Path(), 65 | Device: device, 66 | } 67 | 68 | fmt.Fprintln(os.Stderr, wgConf) 69 | 70 | wgclient, err := wgctrl.New() 71 | if err != nil { 72 | return err 73 | } 74 | 75 | wgMgr, err := wireguard.New(ctx, wgConf, wgclient, wireguardClient, wireguard.WithOutput(os.Stderr)) 76 | if err != nil { 77 | log.Println("failed to create wireguard manager") 78 | return err 79 | } 80 | 81 | readCl, err := openConfig(device) 82 | if err != nil { 83 | fmt.Fprintln(os.Stderr, "failed to open configuration file for interface", device) 84 | return err 85 | } 86 | if err = wgMgr.Config(readCl); err != nil { 87 | fmt.Fprintln(os.Stderr, "failed to write config") 88 | return err 89 | } 90 | 91 | fmt.Fprintln(os.Stderr, "Bringing up device", device) 92 | return wgMgr.Up(device) 93 | }) 94 | 95 | return device, err 96 | } 97 | 98 | func randomString() string { 99 | b := make([]byte, 4) 100 | 101 | rand.New(rand.NewSource(time.Now().UnixNano())).Read(b) 102 | return hex.EncodeToString(b) 103 | } 104 | 105 | func openConfig(device string) (io.WriteCloser, error) { 106 | filename := fmt.Sprintf("%s.conf", device) 107 | f := filepath.Join("/etc", "wireguard", filename) 108 | return os.OpenFile(f, os.O_CREATE|os.O_TRUNC|os.O_SYNC|os.O_RDWR, 0644) 109 | } 110 | 111 | func getResult(device string, sandbox string) (current.IPConfig, current.Interface, error) { 112 | intf, err := net.InterfaceByName(device) 113 | if err != nil { 114 | return current.IPConfig{}, current.Interface{}, fmt.Errorf("could not retrieve interface by name: %s %w", device, err) 115 | } 116 | 117 | currentInterface := current.Interface{ 118 | Name: device, 119 | Mac: intf.HardwareAddr.String(), 120 | Sandbox: sandbox, 121 | } 122 | addrs, err := intf.Addrs() 123 | if err != nil { 124 | return current.IPConfig{}, current.Interface{}, fmt.Errorf("failed to get addrs %w", err) 125 | } 126 | _, ipNet, err := net.ParseCIDR(addrs[0].String()) 127 | if err != nil { 128 | return current.IPConfig{}, current.Interface{}, fmt.Errorf("failed to parse cidr %w", err) 129 | } 130 | 131 | ipCfg := current.IPConfig{ 132 | Interface: nil, 133 | Address: *ipNet, 134 | Gateway: ipNet.IP, 135 | } 136 | return ipCfg, currentInterface, nil 137 | } 138 | -------------------------------------------------------------------------------- /cmd/demo/README.md: -------------------------------------------------------------------------------- 1 | # demo 2 | 3 | A simple wiregaurd client demo. 4 | 5 | ### Build 6 | 7 | ```shell 8 | go build 9 | ``` 10 | 11 | ### Usage 12 | 13 | Create `wg0` interface 14 | 15 | 16 | ``` 17 | sudo ./demo up 18 | ``` 19 | 20 | Destroy `wg0` interface 21 | 22 | ``` 23 | sudo ./demo down 24 | ``` 25 | 26 | ### Visual 27 | 28 | When the interface exists, you can visit demo.wireguard.com and 29 | view the output of your own connected device! 30 | -------------------------------------------------------------------------------- /cmd/demo/main.go: -------------------------------------------------------------------------------- 1 | // Command demo provides a simple Go CLI for creating and removing a wiregaurd 2 | // interface for connecting to demo.wiregaurd.com. 3 | // 4 | // Usage: demo [up|down] 5 | // 6 | // Once the interface is created (e.g. 'ip addr' will show wg0) you should be 7 | // able to visit demo.wiregaurd.com and see your device listed in the output(s). 8 | package main 9 | 10 | import ( 11 | "errors" 12 | "fmt" 13 | "io/ioutil" 14 | "log" 15 | "net" 16 | "os" 17 | "os/exec" 18 | "strings" 19 | "unicode" 20 | ) 21 | 22 | func up() error { 23 | pkey, err := shell("wg", "genkey") 24 | if err != nil { 25 | return err 26 | } 27 | 28 | cmd := exec.Command("wg", "pubkey") 29 | cmd.Stdin = strings.NewReader(pkey) 30 | pubKey, err := cmd.Output() 31 | if err != nil { 32 | return err 33 | } 34 | 35 | w, err := demoPublicKey(string(pubKey)) 36 | if err != nil { 37 | return err 38 | } 39 | 40 | log.Println("wg status", w.status) 41 | log.Println("wg pubkey", w.server_pubkey) 42 | log.Println("wg server port", w.server_port) 43 | internalIP := strings.TrimRightFunc(w.internal_ip, unicode.IsControl) 44 | log.Println("wg internal ip", string(internalIP[0:])) 45 | 46 | if err := run("ip", "link", "add", "dev", "wg0", "type", "wireguard"); err != nil { 47 | return err 48 | } 49 | 50 | if err := ioutil.WriteFile("pkey", []byte(pkey), 0444); err != nil { 51 | return err 52 | } 53 | 54 | if err := run( 55 | "wg", "set", "wg0", "private-key", "pkey", "peer", w.server_pubkey, 56 | "allowed-ips", "0.0.0.0/0", "endpoint", fmt.Sprintf("demo.wireguard.com:%s", w.server_port), 57 | "persistent-keepalive", "25", 58 | ); err != nil { 59 | return err 60 | } 61 | 62 | if err := run("ip", "address", "add", fmt.Sprintf("%s/24", internalIP), "dev", "wg0"); err != nil { 63 | return err 64 | } 65 | 66 | if err := run("ip", "link", "set", "up", "wg0"); err != nil { 67 | return err 68 | } 69 | 70 | return nil 71 | } 72 | 73 | func down() error { 74 | return run("ip", "link", "del", "dev", "wg0") 75 | } 76 | 77 | func main() { 78 | mode := "up" 79 | if len(os.Args) == 2 { 80 | mode = os.Args[1] 81 | } 82 | 83 | var err error 84 | 85 | switch mode { 86 | case "up": 87 | err = up() 88 | case "down": 89 | err = down() 90 | default: 91 | err = errors.New("usage: demo [up|down]") 92 | } 93 | 94 | if err != nil { 95 | log.Fatal("failed:", err) 96 | } 97 | } 98 | 99 | type wg struct { 100 | status string 101 | server_pubkey string 102 | server_port string 103 | internal_ip string 104 | } 105 | 106 | func (w wg) String() string { 107 | return fmt.Sprintf("(%s %s %s %s)", w.status, w.server_pubkey, w.server_port, w.internal_ip) 108 | } 109 | 110 | func demoPublicKey(publicKey string) (*wg, error) { 111 | endpoint := "demo.wireguard.com:42912" 112 | conn, err := net.Dial("tcp4", endpoint) 113 | if err != nil { 114 | return nil, err 115 | } 116 | conn.Write([]byte(publicKey)) 117 | b := make([]byte, 1024) 118 | _, err = conn.Read(b) 119 | if err != nil { 120 | return nil, err 121 | } 122 | 123 | pieces := strings.Split(strings.TrimSpace(string(b)), ":") 124 | return &wg{ 125 | status: pieces[0], 126 | server_pubkey: pieces[1], 127 | server_port: pieces[2], 128 | internal_ip: pieces[3], 129 | }, nil 130 | } 131 | 132 | func shell(cmd string, args ...string) (string, error) { 133 | c := exec.Command(cmd, args...) 134 | b, err := c.CombinedOutput() 135 | return string(b), err 136 | } 137 | 138 | func run(cmd string, args ...string) error { 139 | output, err := shell(cmd, args...) 140 | log.Println("run", fmt.Sprintf("[%s %s]", cmd, strings.Join(args, " "))) 141 | if len(output) > 0 { 142 | fmt.Println(output) 143 | } 144 | return err 145 | } 146 | -------------------------------------------------------------------------------- /cmd/node-manager/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "expvar" 7 | "flag" 8 | "fmt" 9 | "log" 10 | "net" 11 | "net/http" 12 | "os" 13 | "os/signal" 14 | 15 | "github.com/hashicorp/go-sockaddr" 16 | socktemplate "github.com/hashicorp/go-sockaddr/template" 17 | "golang.org/x/net/http2" 18 | "golang.org/x/net/http2/h2c" 19 | 20 | "github.com/clly/wireguard-cni/gen/wgcni/ipam/v1/ipamv1connect" 21 | "github.com/clly/wireguard-cni/gen/wgcni/wireguard/v1/wireguardv1connect" 22 | "github.com/clly/wireguard-cni/pkg/node-manager" 23 | ) 24 | 25 | func main() { 26 | ctx := context.Background() 27 | log.Println("initializing node-manager") 28 | cfg := config() 29 | 30 | b, err := json.MarshalIndent(cfg, "==>", " ") 31 | if err != nil { 32 | log.Fatal(err) 33 | } 34 | log.Println(string(b)) 35 | 36 | log.Println("initializing server") 37 | log.Println("initializing client ipam cidr") 38 | svr, err := nodemanager.NewNodeManagerServer(ctx, cfg) 39 | if err != nil { 40 | log.Fatal(err) 41 | } 42 | 43 | quit(svr, cfg.Wireguard.InterfaceName) 44 | 45 | log.Println("initializing serve mux") 46 | mux := http.NewServeMux() 47 | 48 | path, handler := ipamv1connect.NewIPAMServiceHandler(svr) 49 | log.Println("Registering IPAM Handler on", path) 50 | mux.Handle(path, handler) 51 | 52 | path, handler = wireguardv1connect.NewWireguardServiceHandler(svr) 53 | log.Println("Registering Wireguard Handler on ", path) 54 | mux.Handle(path, handler) 55 | 56 | mux.Handle("/debug/varz", expvar.Handler()) 57 | log.Println("listening", cfg.ListenAddr, "...") 58 | log.Fatal(http.ListenAndServe("localhost:5242", h2c.NewHandler(mux, &http2.Server{}))) 59 | 60 | } 61 | 62 | const clusterMgrEnvKey = "CLUSTER_MANAGER_ADDR" 63 | const clusterMgrDefault = "http://localhost:8080" 64 | 65 | func config() nodemanager.NodeConfig { 66 | log.SetFlags(log.Lshortfile | log.Ldate | log.Ltime) 67 | ip, err := sockaddr.GetPrivateIP() 68 | addr := net.JoinHostPort(ip, "51820") 69 | if err != nil { 70 | log.Println("failed to discover default address") 71 | addr = "" 72 | } 73 | clusterMgrAddr := flag.String("cluster-manager-url", "", "CNI Cluster Manager address") 74 | wireguardEndpoint := flag.String("wireguard-endpoint", addr, "endpoint:port for the wireguard socket") 75 | wireguardParse := flag.String("wireguard-sockaddr-network", "", "use sockaddr to parse the subnet on the interface for the wireguard endpoint") 76 | interfaceName := flag.String("wireguard-interface", "wg0", "wireguard interface name") 77 | configDirectory := flag.String("wireguard-config-directory", "/etc/wireguard", "Wireguard configuration directory") 78 | listenAddr := flag.String("addr", "localhost:5242", "node manager listen address") 79 | dataDir := flag.String("data-dir", "", "Data directory to store ipam and wireguard files in") 80 | 81 | flag.Parse() 82 | addr = *wireguardEndpoint 83 | 84 | if *wireguardParse != "" { 85 | log.Println(*wireguardParse) 86 | socktmpl := fmt.Sprintf("{{ GetPrivateInterfaces|include \"network\" \"%s\" | attr \"address\" }}", *wireguardParse) 87 | log.Println("parsing wireguard endpoint using sockaddr", socktmpl) 88 | addr, err = socktemplate.Parse(socktmpl) 89 | if err != nil { 90 | log.Fatal("failed to parse sockaddr tempalte") 91 | } 92 | addr = net.JoinHostPort(addr, "51820") 93 | } 94 | 95 | // later we can use flag.Visit to see if the clusterMgrAddr was visited 96 | clusterMgr := os.ExpandEnv(first(*clusterMgrAddr, os.Getenv(clusterMgrEnvKey), clusterMgrDefault)) 97 | 98 | return nodemanager.NodeConfig{ 99 | ClusterManagerAddr: clusterMgr, 100 | ConfigDirectory: *configDirectory, 101 | ListenAddr: *listenAddr, 102 | DataDirectory: *dataDir, 103 | Wireguard: nodemanager.WireguardNodeConfig{ 104 | Endpoint: addr, 105 | InterfaceName: *interfaceName, 106 | }, 107 | } 108 | } 109 | 110 | func first(s ...string) string { 111 | for _, v := range s { 112 | fmt.Fprintln(os.Stderr, v) 113 | if v != "" { 114 | return v 115 | } 116 | } 117 | return "" 118 | } 119 | 120 | func quit(mgr *nodemanager.NodeManagerServer, device string) { 121 | c := make(chan os.Signal, 1) 122 | signal.Notify(c, os.Interrupt) 123 | 124 | go func() { 125 | <-c 126 | log.Println("shutting down...") 127 | logOnErr(mgr.Down(device)) 128 | os.Exit(1) 129 | }() 130 | } 131 | 132 | func logOnErr(err error) { 133 | if err != nil { 134 | log.Println(err) 135 | } 136 | } 137 | -------------------------------------------------------------------------------- /config/client.hcl: -------------------------------------------------------------------------------- 1 | data_dir = "/opt/nomad" 2 | client { 3 | enabled = true 4 | server_join { 5 | retry_join = [ "192.168.56.11" ] 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /config/cni.conf: -------------------------------------------------------------------------------- 1 | { 2 | "cniVersion": "0.4.0", 3 | "name": "dbnet", 4 | "plugins": [ 5 | { 6 | "type": "bridge", 7 | "bridge": "cni0", 8 | "keyA": ["some more", "plugin specific", "configuration"], 9 | "ipam": { 10 | "type": "host-local", 11 | "subnet": "10.1.0.0/16", 12 | "gateway": "10.1.0.1", 13 | "routes": [ 14 | {"dst": "0.0.0.0/0"} 15 | ] 16 | }, 17 | "dns": { 18 | "nameservers": [ "10.1.0.1" ] 19 | } 20 | }, 21 | { 22 | "type": "wireguard-cni", 23 | "nodeManagerAddr": "127.0.0.1:5000" 24 | }, 25 | { 26 | "type": "portmap", 27 | "capabilities": {"portMappings": true} 28 | } 29 | ] 30 | } 31 | -------------------------------------------------------------------------------- /config/nomad-systemd.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Nomad 3 | Documentation=https://www.nomadproject.io/docs/ 4 | Wants=network-online.target 5 | After=network-online.target 6 | 7 | # When using Nomad with Consul it is not necessary to start Consul first. These 8 | # lines start Consul before Nomad as an optimization to avoid Nomad logging 9 | # that Consul is unavailable at startup. 10 | #Wants=consul.service 11 | #After=consul.service 12 | 13 | [Service] 14 | 15 | # Nomad server should be run as the nomad user. Nomad clients 16 | # should be run as root 17 | User=root 18 | Group=root 19 | 20 | ExecReload=/bin/kill -HUP $MAINPID 21 | ExecStart=/usr/local/bin/nomad agent -dev -config /etc/nomad.d 22 | KillMode=process 23 | KillSignal=SIGINT 24 | LimitNOFILE=65536 25 | LimitNPROC=infinity 26 | Restart=on-failure 27 | RestartSec=2 28 | RuntimeDirectory=nomad 29 | 30 | ## Configure unit start rate limiting. Units which are started more than 31 | ## *burst* times within an *interval* time span are not permitted to start any 32 | ## more. Use `StartLimitIntervalSec` or `StartLimitInterval` (depending on 33 | ## systemd version) to configure the checking interval and `StartLimitBurst` 34 | ## to configure how many starts per interval are allowed. The values in the 35 | ## commented lines are defaults. 36 | 37 | # StartLimitBurst = 5 38 | 39 | ## StartLimitIntervalSec is used for systemd versions >= 230 40 | # StartLimitIntervalSec = 10s 41 | 42 | ## StartLimitInterval is used for systemd versions < 230 43 | # StartLimitInterval = 10s 44 | 45 | TasksMax=infinity 46 | OOMScoreAdjust=-1000 47 | 48 | [Install] 49 | WantedBy=multi-user.target 50 | 51 | -------------------------------------------------------------------------------- /docs/spike/container-to-container-communication.md: -------------------------------------------------------------------------------- 1 | # Analysis 2 | 3 | Wireguard interfaces are namespace aware. After creation, they keep the 4 | socket in the originating namespace (host namespace in this case) and then 5 | forward traffic to the interface in the container namespace. 6 | 7 | Wireguard UDP Listen Ports in the host namespace. 8 | ```commandline 9 | UNCONN 0 0 0.0.0.0:51820 0.0.0.0:* 10 | UNCONN 0 0 0.0.0.0:51821 0.0.0.0:* 11 | UNCONN 0 0 [::]:51820 [::]:* 12 | UNCONN 0 0 [::]:51821 [::]:* 13 | ``` 14 | 15 | ```commandline 16 | root@wg-server:/vagrant/bash-poc/c-c-vagrant# wg 17 | interface: wg0 18 | public key: v2XPnF7Yk7KEugBRVyCdD6mssm2Tsc4edDUApwzFoBk= 19 | private key: (hidden) 20 | listening port: 51820 21 | 22 | peer: X7qQfHNDtiUS5+qg2H5T25IwhcWdVbyoFjGEDTFJ1WU= 23 | endpoint: 192.168.56.11:51821 24 | allowed ips: 10.0.10.3/32 25 | latest handshake: 56 seconds ago 26 | transfer: 532 B received, 4.75 KiB sent 27 | persistent keepalive: every 10 seconds 28 | ``` 29 | 30 | 31 | With multiple listen ports we can have each container on a different UDP port. Those containers ultimately communicate 32 | with each other through those UDP listen ports. Traffic that is received at the wireguard listen address in the host 33 | namespace goes to the wireguard interface in the container namespace. We give each container Peer configuration that 34 | points their Peers at the `address:port` combincation in the host namespace. This will slightly tighten the security 35 | boundary because there is no longer a bridge to pass traffic in clear text. With the current firewall rules we will 36 | still forward traffic if a user knows the Wireguard network space and where a wireguard interface lives. 37 | 38 | We can not use wg-quick with this model because it creates, configures and brings up the interface in the container 39 | namespace. Once the interface is live it cannot be moved from one namespace to another so we would not be able to bring 40 | the interface up in the host namespace and move it to the container namespace. 41 | 42 | ## Convergence notes 43 | 44 | ### When are endpoints required 45 | While being lazy during setup I've discovered configuring actual dialing and routing does _not seem_ to be required. 46 | After one party performs any request against any other party in the mesh that is properly configured as a peer. The peer 47 | can then respond back over the wireguard tunnel. Only one side _requires_ an endpoint which feels like it allows a level 48 | of control and protection around which clients can start a connection. Unsure if those ever expire. 49 | 50 | This was confirmed with tcpdump to ensure that packets were not passing through the host accidentally. If the route 51 | didn't exist would the host pass traffic? If the peer exists and the route doesn't it appears that traffic won't pass? 52 | 53 | ### Hosts can still pass traffic but require a route 54 | 55 | When a subnet is configured in AllowedIPs and a route is configured and IP forwarding is configured, the peer will still 56 | pass traffic between nodes. As long as we configure routes properly on Hosts first new network namespaces should be able 57 | to communicate between other network namespaces quickly. 58 | 59 | This was confirmed using traceroute. 60 | 61 | ### iptables changes 62 | 63 | A minor iptables change _was_ required to ensure that we don't NAT traffic intended for containers. If a container 64 | starts the request then the host can pass traffic back to the container. Why didn't we find this out earlier? Unknown. 65 | Maybe I started the request from inside the container so things just sort of worked or traffic could more easily get 66 | passed into the container because of the connection from the bridge to the host-local IP address for the wireguard dial. 67 | 68 | ### When will a container pass traffic using a host. 69 | When fully configured a container will _not_ attempt to pass traffic through it's host unless it does not have an 70 | AllowedIPs for the address set on the interface. 71 | 72 | ## Testing 73 | * open two terminals 74 | * `vagrant ssh peer` and `vagrant ssh server` 75 | * `./host-create.sh ` 76 | * `./container-create.sh ` 77 | 78 | Then ping or curl or whatever between them. Using something like echo-server would make it easier to validate -------------------------------------------------------------------------------- /entrypoint.bash: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -euo pipefail 4 | 5 | bin=/opt/$1 6 | shift 7 | 8 | if [[ -f $bin ]]; then 9 | echo "Starting $bin" >&2 10 | echo "whoami: $(whoami)" 11 | exec $bin $@ 12 | fi 13 | 14 | echo "failed to start ${bin}" 15 | exit 1 16 | -------------------------------------------------------------------------------- /gen/wgcni/ipam/v1/ipamv1connect/ipam.connect.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-connect-go. DO NOT EDIT. 2 | // 3 | // Source: wgcni/ipam/v1/ipam.proto 4 | 5 | package ipamv1connect 6 | 7 | import ( 8 | connect "connectrpc.com/connect" 9 | context "context" 10 | errors "errors" 11 | v1 "github.com/clly/wireguard-cni/gen/wgcni/ipam/v1" 12 | http "net/http" 13 | strings "strings" 14 | ) 15 | 16 | // This is a compile-time assertion to ensure that this generated file and the connect package are 17 | // compatible. If you get a compiler error that this constant is not defined, this code was 18 | // generated with a version of connect newer than the one compiled into your binary. You can fix the 19 | // problem by either regenerating this code with an older version of connect or updating the connect 20 | // version compiled into your binary. 21 | const _ = connect.IsAtLeastVersion0_1_0 22 | 23 | const ( 24 | // IPAMServiceName is the fully-qualified name of the IPAMService service. 25 | IPAMServiceName = "wgcni.ipam.v1.IPAMService" 26 | ) 27 | 28 | // These constants are the fully-qualified names of the RPCs defined in this package. They're 29 | // exposed at runtime as Spec.Procedure and as the final two segments of the HTTP route. 30 | // 31 | // Note that these are different from the fully-qualified method names used by 32 | // google.golang.org/protobuf/reflect/protoreflect. To convert from these constants to 33 | // reflection-formatted method names, remove the leading slash and convert the remaining slash to a 34 | // period. 35 | const ( 36 | // IPAMServiceAllocProcedure is the fully-qualified name of the IPAMService's Alloc RPC. 37 | IPAMServiceAllocProcedure = "/wgcni.ipam.v1.IPAMService/Alloc" 38 | ) 39 | 40 | // IPAMServiceClient is a client for the wgcni.ipam.v1.IPAMService service. 41 | type IPAMServiceClient interface { 42 | // Alloc requests a IP address or subnet from the ipam server 43 | Alloc(context.Context, *connect.Request[v1.AllocRequest]) (*connect.Response[v1.AllocResponse], error) 44 | } 45 | 46 | // NewIPAMServiceClient constructs a client for the wgcni.ipam.v1.IPAMService service. By default, 47 | // it uses the Connect protocol with the binary Protobuf Codec, asks for gzipped responses, and 48 | // sends uncompressed requests. To use the gRPC or gRPC-Web protocols, supply the connect.WithGRPC() 49 | // or connect.WithGRPCWeb() options. 50 | // 51 | // The URL supplied here should be the base URL for the Connect or gRPC server (for example, 52 | // http://api.acme.com or https://acme.com/grpc). 53 | func NewIPAMServiceClient(httpClient connect.HTTPClient, baseURL string, opts ...connect.ClientOption) IPAMServiceClient { 54 | baseURL = strings.TrimRight(baseURL, "/") 55 | return &iPAMServiceClient{ 56 | alloc: connect.NewClient[v1.AllocRequest, v1.AllocResponse]( 57 | httpClient, 58 | baseURL+IPAMServiceAllocProcedure, 59 | opts..., 60 | ), 61 | } 62 | } 63 | 64 | // iPAMServiceClient implements IPAMServiceClient. 65 | type iPAMServiceClient struct { 66 | alloc *connect.Client[v1.AllocRequest, v1.AllocResponse] 67 | } 68 | 69 | // Alloc calls wgcni.ipam.v1.IPAMService.Alloc. 70 | func (c *iPAMServiceClient) Alloc(ctx context.Context, req *connect.Request[v1.AllocRequest]) (*connect.Response[v1.AllocResponse], error) { 71 | return c.alloc.CallUnary(ctx, req) 72 | } 73 | 74 | // IPAMServiceHandler is an implementation of the wgcni.ipam.v1.IPAMService service. 75 | type IPAMServiceHandler interface { 76 | // Alloc requests a IP address or subnet from the ipam server 77 | Alloc(context.Context, *connect.Request[v1.AllocRequest]) (*connect.Response[v1.AllocResponse], error) 78 | } 79 | 80 | // NewIPAMServiceHandler builds an HTTP handler from the service implementation. It returns the path 81 | // on which to mount the handler and the handler itself. 82 | // 83 | // By default, handlers support the Connect, gRPC, and gRPC-Web protocols with the binary Protobuf 84 | // and JSON codecs. They also support gzip compression. 85 | func NewIPAMServiceHandler(svc IPAMServiceHandler, opts ...connect.HandlerOption) (string, http.Handler) { 86 | iPAMServiceAllocHandler := connect.NewUnaryHandler( 87 | IPAMServiceAllocProcedure, 88 | svc.Alloc, 89 | opts..., 90 | ) 91 | return "/wgcni.ipam.v1.IPAMService/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 92 | switch r.URL.Path { 93 | case IPAMServiceAllocProcedure: 94 | iPAMServiceAllocHandler.ServeHTTP(w, r) 95 | default: 96 | http.NotFound(w, r) 97 | } 98 | }) 99 | } 100 | 101 | // UnimplementedIPAMServiceHandler returns CodeUnimplemented from all methods. 102 | type UnimplementedIPAMServiceHandler struct{} 103 | 104 | func (UnimplementedIPAMServiceHandler) Alloc(context.Context, *connect.Request[v1.AllocRequest]) (*connect.Response[v1.AllocResponse], error) { 105 | return nil, connect.NewError(connect.CodeUnimplemented, errors.New("wgcni.ipam.v1.IPAMService.Alloc is not implemented")) 106 | } 107 | -------------------------------------------------------------------------------- /gen/wgcni/ipam/v1/ipamv1connect/mock_IPAMServiceClient.go: -------------------------------------------------------------------------------- 1 | // Code generated by mockery v2.40.1. DO NOT EDIT. 2 | 3 | package ipamv1connect 4 | 5 | import ( 6 | context "context" 7 | 8 | connect "connectrpc.com/connect" 9 | 10 | ipamv1 "github.com/clly/wireguard-cni/gen/wgcni/ipam/v1" 11 | 12 | mock "github.com/stretchr/testify/mock" 13 | ) 14 | 15 | // MockIPAMServiceClient is an autogenerated mock type for the IPAMServiceClient type 16 | type MockIPAMServiceClient struct { 17 | mock.Mock 18 | } 19 | 20 | // Alloc provides a mock function with given fields: _a0, _a1 21 | func (_m *MockIPAMServiceClient) Alloc(_a0 context.Context, _a1 *connect.Request[ipamv1.AllocRequest]) (*connect.Response[ipamv1.AllocResponse], error) { 22 | ret := _m.Called(_a0, _a1) 23 | 24 | if len(ret) == 0 { 25 | panic("no return value specified for Alloc") 26 | } 27 | 28 | var r0 *connect.Response[ipamv1.AllocResponse] 29 | var r1 error 30 | if rf, ok := ret.Get(0).(func(context.Context, *connect.Request[ipamv1.AllocRequest]) (*connect.Response[ipamv1.AllocResponse], error)); ok { 31 | return rf(_a0, _a1) 32 | } 33 | if rf, ok := ret.Get(0).(func(context.Context, *connect.Request[ipamv1.AllocRequest]) *connect.Response[ipamv1.AllocResponse]); ok { 34 | r0 = rf(_a0, _a1) 35 | } else { 36 | if ret.Get(0) != nil { 37 | r0 = ret.Get(0).(*connect.Response[ipamv1.AllocResponse]) 38 | } 39 | } 40 | 41 | if rf, ok := ret.Get(1).(func(context.Context, *connect.Request[ipamv1.AllocRequest]) error); ok { 42 | r1 = rf(_a0, _a1) 43 | } else { 44 | r1 = ret.Error(1) 45 | } 46 | 47 | return r0, r1 48 | } 49 | 50 | // NewMockIPAMServiceClient creates a new instance of MockIPAMServiceClient. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. 51 | // The first argument is typically a *testing.T value. 52 | func NewMockIPAMServiceClient(t interface { 53 | mock.TestingT 54 | Cleanup(func()) 55 | }) *MockIPAMServiceClient { 56 | mock := &MockIPAMServiceClient{} 57 | mock.Mock.Test(t) 58 | 59 | t.Cleanup(func() { mock.AssertExpectations(t) }) 60 | 61 | return mock 62 | } 63 | -------------------------------------------------------------------------------- /gen/wgcni/ipam/v1/ipamv1connect/mock_IPAMServiceHandler.go: -------------------------------------------------------------------------------- 1 | // Code generated by mockery v2.40.1. DO NOT EDIT. 2 | 3 | package ipamv1connect 4 | 5 | import ( 6 | context "context" 7 | 8 | connect "connectrpc.com/connect" 9 | 10 | ipamv1 "github.com/clly/wireguard-cni/gen/wgcni/ipam/v1" 11 | 12 | mock "github.com/stretchr/testify/mock" 13 | ) 14 | 15 | // MockIPAMServiceHandler is an autogenerated mock type for the IPAMServiceHandler type 16 | type MockIPAMServiceHandler struct { 17 | mock.Mock 18 | } 19 | 20 | // Alloc provides a mock function with given fields: _a0, _a1 21 | func (_m *MockIPAMServiceHandler) Alloc(_a0 context.Context, _a1 *connect.Request[ipamv1.AllocRequest]) (*connect.Response[ipamv1.AllocResponse], error) { 22 | ret := _m.Called(_a0, _a1) 23 | 24 | if len(ret) == 0 { 25 | panic("no return value specified for Alloc") 26 | } 27 | 28 | var r0 *connect.Response[ipamv1.AllocResponse] 29 | var r1 error 30 | if rf, ok := ret.Get(0).(func(context.Context, *connect.Request[ipamv1.AllocRequest]) (*connect.Response[ipamv1.AllocResponse], error)); ok { 31 | return rf(_a0, _a1) 32 | } 33 | if rf, ok := ret.Get(0).(func(context.Context, *connect.Request[ipamv1.AllocRequest]) *connect.Response[ipamv1.AllocResponse]); ok { 34 | r0 = rf(_a0, _a1) 35 | } else { 36 | if ret.Get(0) != nil { 37 | r0 = ret.Get(0).(*connect.Response[ipamv1.AllocResponse]) 38 | } 39 | } 40 | 41 | if rf, ok := ret.Get(1).(func(context.Context, *connect.Request[ipamv1.AllocRequest]) error); ok { 42 | r1 = rf(_a0, _a1) 43 | } else { 44 | r1 = ret.Error(1) 45 | } 46 | 47 | return r0, r1 48 | } 49 | 50 | // NewMockIPAMServiceHandler creates a new instance of MockIPAMServiceHandler. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. 51 | // The first argument is typically a *testing.T value. 52 | func NewMockIPAMServiceHandler(t interface { 53 | mock.TestingT 54 | Cleanup(func()) 55 | }) *MockIPAMServiceHandler { 56 | mock := &MockIPAMServiceHandler{} 57 | mock.Mock.Test(t) 58 | 59 | t.Cleanup(func() { mock.AssertExpectations(t) }) 60 | 61 | return mock 62 | } 63 | -------------------------------------------------------------------------------- /gen/wgcni/wireguard/v1/wireguardv1connect/mock_WireguardServiceClient.go: -------------------------------------------------------------------------------- 1 | // Code generated by mockery v2.40.1. DO NOT EDIT. 2 | 3 | package wireguardv1connect 4 | 5 | import ( 6 | context "context" 7 | 8 | connect "connectrpc.com/connect" 9 | 10 | mock "github.com/stretchr/testify/mock" 11 | 12 | wireguardv1 "github.com/clly/wireguard-cni/gen/wgcni/wireguard/v1" 13 | ) 14 | 15 | // MockWireguardServiceClient is an autogenerated mock type for the WireguardServiceClient type 16 | type MockWireguardServiceClient struct { 17 | mock.Mock 18 | } 19 | 20 | // Peers provides a mock function with given fields: _a0, _a1 21 | func (_m *MockWireguardServiceClient) Peers(_a0 context.Context, _a1 *connect.Request[wireguardv1.PeersRequest]) (*connect.Response[wireguardv1.PeersResponse], error) { 22 | ret := _m.Called(_a0, _a1) 23 | 24 | if len(ret) == 0 { 25 | panic("no return value specified for Peers") 26 | } 27 | 28 | var r0 *connect.Response[wireguardv1.PeersResponse] 29 | var r1 error 30 | if rf, ok := ret.Get(0).(func(context.Context, *connect.Request[wireguardv1.PeersRequest]) (*connect.Response[wireguardv1.PeersResponse], error)); ok { 31 | return rf(_a0, _a1) 32 | } 33 | if rf, ok := ret.Get(0).(func(context.Context, *connect.Request[wireguardv1.PeersRequest]) *connect.Response[wireguardv1.PeersResponse]); ok { 34 | r0 = rf(_a0, _a1) 35 | } else { 36 | if ret.Get(0) != nil { 37 | r0 = ret.Get(0).(*connect.Response[wireguardv1.PeersResponse]) 38 | } 39 | } 40 | 41 | if rf, ok := ret.Get(1).(func(context.Context, *connect.Request[wireguardv1.PeersRequest]) error); ok { 42 | r1 = rf(_a0, _a1) 43 | } else { 44 | r1 = ret.Error(1) 45 | } 46 | 47 | return r0, r1 48 | } 49 | 50 | // Register provides a mock function with given fields: _a0, _a1 51 | func (_m *MockWireguardServiceClient) Register(_a0 context.Context, _a1 *connect.Request[wireguardv1.RegisterRequest]) (*connect.Response[wireguardv1.RegisterResponse], error) { 52 | ret := _m.Called(_a0, _a1) 53 | 54 | if len(ret) == 0 { 55 | panic("no return value specified for Register") 56 | } 57 | 58 | var r0 *connect.Response[wireguardv1.RegisterResponse] 59 | var r1 error 60 | if rf, ok := ret.Get(0).(func(context.Context, *connect.Request[wireguardv1.RegisterRequest]) (*connect.Response[wireguardv1.RegisterResponse], error)); ok { 61 | return rf(_a0, _a1) 62 | } 63 | if rf, ok := ret.Get(0).(func(context.Context, *connect.Request[wireguardv1.RegisterRequest]) *connect.Response[wireguardv1.RegisterResponse]); ok { 64 | r0 = rf(_a0, _a1) 65 | } else { 66 | if ret.Get(0) != nil { 67 | r0 = ret.Get(0).(*connect.Response[wireguardv1.RegisterResponse]) 68 | } 69 | } 70 | 71 | if rf, ok := ret.Get(1).(func(context.Context, *connect.Request[wireguardv1.RegisterRequest]) error); ok { 72 | r1 = rf(_a0, _a1) 73 | } else { 74 | r1 = ret.Error(1) 75 | } 76 | 77 | return r0, r1 78 | } 79 | 80 | // NewMockWireguardServiceClient creates a new instance of MockWireguardServiceClient. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. 81 | // The first argument is typically a *testing.T value. 82 | func NewMockWireguardServiceClient(t interface { 83 | mock.TestingT 84 | Cleanup(func()) 85 | }) *MockWireguardServiceClient { 86 | mock := &MockWireguardServiceClient{} 87 | mock.Mock.Test(t) 88 | 89 | t.Cleanup(func() { mock.AssertExpectations(t) }) 90 | 91 | return mock 92 | } 93 | -------------------------------------------------------------------------------- /gen/wgcni/wireguard/v1/wireguardv1connect/mock_WireguardServiceHandler.go: -------------------------------------------------------------------------------- 1 | // Code generated by mockery v2.40.1. DO NOT EDIT. 2 | 3 | package wireguardv1connect 4 | 5 | import ( 6 | context "context" 7 | 8 | connect "connectrpc.com/connect" 9 | 10 | mock "github.com/stretchr/testify/mock" 11 | 12 | wireguardv1 "github.com/clly/wireguard-cni/gen/wgcni/wireguard/v1" 13 | ) 14 | 15 | // MockWireguardServiceHandler is an autogenerated mock type for the WireguardServiceHandler type 16 | type MockWireguardServiceHandler struct { 17 | mock.Mock 18 | } 19 | 20 | // Peers provides a mock function with given fields: _a0, _a1 21 | func (_m *MockWireguardServiceHandler) Peers(_a0 context.Context, _a1 *connect.Request[wireguardv1.PeersRequest]) (*connect.Response[wireguardv1.PeersResponse], error) { 22 | ret := _m.Called(_a0, _a1) 23 | 24 | if len(ret) == 0 { 25 | panic("no return value specified for Peers") 26 | } 27 | 28 | var r0 *connect.Response[wireguardv1.PeersResponse] 29 | var r1 error 30 | if rf, ok := ret.Get(0).(func(context.Context, *connect.Request[wireguardv1.PeersRequest]) (*connect.Response[wireguardv1.PeersResponse], error)); ok { 31 | return rf(_a0, _a1) 32 | } 33 | if rf, ok := ret.Get(0).(func(context.Context, *connect.Request[wireguardv1.PeersRequest]) *connect.Response[wireguardv1.PeersResponse]); ok { 34 | r0 = rf(_a0, _a1) 35 | } else { 36 | if ret.Get(0) != nil { 37 | r0 = ret.Get(0).(*connect.Response[wireguardv1.PeersResponse]) 38 | } 39 | } 40 | 41 | if rf, ok := ret.Get(1).(func(context.Context, *connect.Request[wireguardv1.PeersRequest]) error); ok { 42 | r1 = rf(_a0, _a1) 43 | } else { 44 | r1 = ret.Error(1) 45 | } 46 | 47 | return r0, r1 48 | } 49 | 50 | // Register provides a mock function with given fields: _a0, _a1 51 | func (_m *MockWireguardServiceHandler) Register(_a0 context.Context, _a1 *connect.Request[wireguardv1.RegisterRequest]) (*connect.Response[wireguardv1.RegisterResponse], error) { 52 | ret := _m.Called(_a0, _a1) 53 | 54 | if len(ret) == 0 { 55 | panic("no return value specified for Register") 56 | } 57 | 58 | var r0 *connect.Response[wireguardv1.RegisterResponse] 59 | var r1 error 60 | if rf, ok := ret.Get(0).(func(context.Context, *connect.Request[wireguardv1.RegisterRequest]) (*connect.Response[wireguardv1.RegisterResponse], error)); ok { 61 | return rf(_a0, _a1) 62 | } 63 | if rf, ok := ret.Get(0).(func(context.Context, *connect.Request[wireguardv1.RegisterRequest]) *connect.Response[wireguardv1.RegisterResponse]); ok { 64 | r0 = rf(_a0, _a1) 65 | } else { 66 | if ret.Get(0) != nil { 67 | r0 = ret.Get(0).(*connect.Response[wireguardv1.RegisterResponse]) 68 | } 69 | } 70 | 71 | if rf, ok := ret.Get(1).(func(context.Context, *connect.Request[wireguardv1.RegisterRequest]) error); ok { 72 | r1 = rf(_a0, _a1) 73 | } else { 74 | r1 = ret.Error(1) 75 | } 76 | 77 | return r0, r1 78 | } 79 | 80 | // NewMockWireguardServiceHandler creates a new instance of MockWireguardServiceHandler. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. 81 | // The first argument is typically a *testing.T value. 82 | func NewMockWireguardServiceHandler(t interface { 83 | mock.TestingT 84 | Cleanup(func()) 85 | }) *MockWireguardServiceHandler { 86 | mock := &MockWireguardServiceHandler{} 87 | mock.Mock.Test(t) 88 | 89 | t.Cleanup(func() { mock.AssertExpectations(t) }) 90 | 91 | return mock 92 | } 93 | -------------------------------------------------------------------------------- /gen/wgcni/wireguard/v1/wireguardv1connect/wireguard.connect.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-connect-go. DO NOT EDIT. 2 | // 3 | // Source: wgcni/wireguard/v1/wireguard.proto 4 | 5 | package wireguardv1connect 6 | 7 | import ( 8 | connect "connectrpc.com/connect" 9 | context "context" 10 | errors "errors" 11 | v1 "github.com/clly/wireguard-cni/gen/wgcni/wireguard/v1" 12 | http "net/http" 13 | strings "strings" 14 | ) 15 | 16 | // This is a compile-time assertion to ensure that this generated file and the connect package are 17 | // compatible. If you get a compiler error that this constant is not defined, this code was 18 | // generated with a version of connect newer than the one compiled into your binary. You can fix the 19 | // problem by either regenerating this code with an older version of connect or updating the connect 20 | // version compiled into your binary. 21 | const _ = connect.IsAtLeastVersion0_1_0 22 | 23 | const ( 24 | // WireguardServiceName is the fully-qualified name of the WireguardService service. 25 | WireguardServiceName = "wgcni.wireguard.v1.WireguardService" 26 | ) 27 | 28 | // These constants are the fully-qualified names of the RPCs defined in this package. They're 29 | // exposed at runtime as Spec.Procedure and as the final two segments of the HTTP route. 30 | // 31 | // Note that these are different from the fully-qualified method names used by 32 | // google.golang.org/protobuf/reflect/protoreflect. To convert from these constants to 33 | // reflection-formatted method names, remove the leading slash and convert the remaining slash to a 34 | // period. 35 | const ( 36 | // WireguardServiceRegisterProcedure is the fully-qualified name of the WireguardService's Register 37 | // RPC. 38 | WireguardServiceRegisterProcedure = "/wgcni.wireguard.v1.WireguardService/Register" 39 | // WireguardServicePeersProcedure is the fully-qualified name of the WireguardService's Peers RPC. 40 | WireguardServicePeersProcedure = "/wgcni.wireguard.v1.WireguardService/Peers" 41 | ) 42 | 43 | // WireguardServiceClient is a client for the wgcni.wireguard.v1.WireguardService service. 44 | type WireguardServiceClient interface { 45 | // Register will register a wireguard peer 46 | Register(context.Context, *connect.Request[v1.RegisterRequest]) (*connect.Response[v1.RegisterResponse], error) 47 | // Peers rpc will return a list of all wireguard peers 48 | Peers(context.Context, *connect.Request[v1.PeersRequest]) (*connect.Response[v1.PeersResponse], error) 49 | } 50 | 51 | // NewWireguardServiceClient constructs a client for the wgcni.wireguard.v1.WireguardService 52 | // service. By default, it uses the Connect protocol with the binary Protobuf Codec, asks for 53 | // gzipped responses, and sends uncompressed requests. To use the gRPC or gRPC-Web protocols, supply 54 | // the connect.WithGRPC() or connect.WithGRPCWeb() options. 55 | // 56 | // The URL supplied here should be the base URL for the Connect or gRPC server (for example, 57 | // http://api.acme.com or https://acme.com/grpc). 58 | func NewWireguardServiceClient(httpClient connect.HTTPClient, baseURL string, opts ...connect.ClientOption) WireguardServiceClient { 59 | baseURL = strings.TrimRight(baseURL, "/") 60 | return &wireguardServiceClient{ 61 | register: connect.NewClient[v1.RegisterRequest, v1.RegisterResponse]( 62 | httpClient, 63 | baseURL+WireguardServiceRegisterProcedure, 64 | opts..., 65 | ), 66 | peers: connect.NewClient[v1.PeersRequest, v1.PeersResponse]( 67 | httpClient, 68 | baseURL+WireguardServicePeersProcedure, 69 | opts..., 70 | ), 71 | } 72 | } 73 | 74 | // wireguardServiceClient implements WireguardServiceClient. 75 | type wireguardServiceClient struct { 76 | register *connect.Client[v1.RegisterRequest, v1.RegisterResponse] 77 | peers *connect.Client[v1.PeersRequest, v1.PeersResponse] 78 | } 79 | 80 | // Register calls wgcni.wireguard.v1.WireguardService.Register. 81 | func (c *wireguardServiceClient) Register(ctx context.Context, req *connect.Request[v1.RegisterRequest]) (*connect.Response[v1.RegisterResponse], error) { 82 | return c.register.CallUnary(ctx, req) 83 | } 84 | 85 | // Peers calls wgcni.wireguard.v1.WireguardService.Peers. 86 | func (c *wireguardServiceClient) Peers(ctx context.Context, req *connect.Request[v1.PeersRequest]) (*connect.Response[v1.PeersResponse], error) { 87 | return c.peers.CallUnary(ctx, req) 88 | } 89 | 90 | // WireguardServiceHandler is an implementation of the wgcni.wireguard.v1.WireguardService service. 91 | type WireguardServiceHandler interface { 92 | // Register will register a wireguard peer 93 | Register(context.Context, *connect.Request[v1.RegisterRequest]) (*connect.Response[v1.RegisterResponse], error) 94 | // Peers rpc will return a list of all wireguard peers 95 | Peers(context.Context, *connect.Request[v1.PeersRequest]) (*connect.Response[v1.PeersResponse], error) 96 | } 97 | 98 | // NewWireguardServiceHandler builds an HTTP handler from the service implementation. It returns the 99 | // path on which to mount the handler and the handler itself. 100 | // 101 | // By default, handlers support the Connect, gRPC, and gRPC-Web protocols with the binary Protobuf 102 | // and JSON codecs. They also support gzip compression. 103 | func NewWireguardServiceHandler(svc WireguardServiceHandler, opts ...connect.HandlerOption) (string, http.Handler) { 104 | wireguardServiceRegisterHandler := connect.NewUnaryHandler( 105 | WireguardServiceRegisterProcedure, 106 | svc.Register, 107 | opts..., 108 | ) 109 | wireguardServicePeersHandler := connect.NewUnaryHandler( 110 | WireguardServicePeersProcedure, 111 | svc.Peers, 112 | opts..., 113 | ) 114 | return "/wgcni.wireguard.v1.WireguardService/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 115 | switch r.URL.Path { 116 | case WireguardServiceRegisterProcedure: 117 | wireguardServiceRegisterHandler.ServeHTTP(w, r) 118 | case WireguardServicePeersProcedure: 119 | wireguardServicePeersHandler.ServeHTTP(w, r) 120 | default: 121 | http.NotFound(w, r) 122 | } 123 | }) 124 | } 125 | 126 | // UnimplementedWireguardServiceHandler returns CodeUnimplemented from all methods. 127 | type UnimplementedWireguardServiceHandler struct{} 128 | 129 | func (UnimplementedWireguardServiceHandler) Register(context.Context, *connect.Request[v1.RegisterRequest]) (*connect.Response[v1.RegisterResponse], error) { 130 | return nil, connect.NewError(connect.CodeUnimplemented, errors.New("wgcni.wireguard.v1.WireguardService.Register is not implemented")) 131 | } 132 | 133 | func (UnimplementedWireguardServiceHandler) Peers(context.Context, *connect.Request[v1.PeersRequest]) (*connect.Response[v1.PeersResponse], error) { 134 | return nil, connect.NewError(connect.CodeUnimplemented, errors.New("wgcni.wireguard.v1.WireguardService.Peers is not implemented")) 135 | } 136 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/clly/wireguard-cni 2 | 3 | go 1.18 4 | 5 | require ( 6 | connectrpc.com/connect v1.12.0 7 | github.com/containernetworking/cni v1.1.2 8 | github.com/containernetworking/plugins v1.1.1 9 | github.com/go-ozzo/ozzo-validation v3.6.0+incompatible 10 | github.com/hashicorp/go-cleanhttp v0.5.2 11 | github.com/hashicorp/go-sockaddr v1.0.2 12 | github.com/hashicorp/go-uuid v1.0.3 13 | github.com/metal-stack/go-ipam v1.11.6 14 | github.com/onsi/ginkgo v1.16.5 15 | github.com/onsi/gomega v1.30.0 16 | github.com/stretchr/testify v1.8.4 17 | golang.org/x/net v0.18.0 18 | golang.zx2c4.com/wireguard/wgctrl v0.0.0-20220504211119-3d4a969bb56b 19 | google.golang.org/protobuf v1.31.0 20 | ) 21 | 22 | require ( 23 | github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect 24 | github.com/avast/retry-go/v4 v4.3.3 // indirect 25 | github.com/cespare/xxhash/v2 v2.2.0 // indirect 26 | github.com/coreos/go-semver v0.3.1 // indirect 27 | github.com/coreos/go-systemd/v22 v22.5.0 // indirect 28 | github.com/davecgh/go-spew v1.1.1 // indirect 29 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect 30 | github.com/fsnotify/fsnotify v1.4.9 // indirect 31 | github.com/gogo/protobuf v1.3.2 // indirect 32 | github.com/golang/protobuf v1.5.3 // indirect 33 | github.com/golang/snappy v0.0.4 // indirect 34 | github.com/google/go-cmp v0.6.0 // indirect 35 | github.com/hashicorp/errwrap v1.1.0 // indirect 36 | github.com/jmoiron/sqlx v1.3.5 // indirect 37 | github.com/josharian/native v1.0.0 // indirect 38 | github.com/klauspost/compress v1.15.15 // indirect 39 | github.com/kr/pretty v0.3.0 // indirect 40 | github.com/lib/pq v1.10.7 // indirect 41 | github.com/mdlayher/genetlink v1.2.0 // indirect 42 | github.com/mdlayher/netlink v1.6.0 // indirect 43 | github.com/mdlayher/socket v0.2.3 // indirect 44 | github.com/montanaflynn/stats v0.7.0 // indirect 45 | github.com/nxadm/tail v1.4.8 // indirect 46 | github.com/pkg/errors v0.9.1 // indirect 47 | github.com/pmezard/go-difflib v1.0.0 // indirect 48 | github.com/redis/go-redis/v9 v9.0.2 // indirect 49 | github.com/stretchr/objx v0.5.0 // indirect 50 | github.com/xdg-go/pbkdf2 v1.0.0 // indirect 51 | github.com/xdg-go/scram v1.1.2 // indirect 52 | github.com/xdg-go/stringprep v1.0.4 // indirect 53 | github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a // indirect 54 | go.etcd.io/etcd/api/v3 v3.5.7 // indirect 55 | go.etcd.io/etcd/client/pkg/v3 v3.5.7 // indirect 56 | go.etcd.io/etcd/client/v3 v3.5.7 // indirect 57 | go.mongodb.org/mongo-driver v1.11.2 // indirect 58 | go.uber.org/atomic v1.10.0 // indirect 59 | go.uber.org/multierr v1.9.0 // indirect 60 | go.uber.org/zap v1.24.0 // indirect 61 | go4.org/netipx v0.0.0-20230125063823-8449b0a6169f // indirect 62 | golang.org/x/crypto v0.17.0 // indirect 63 | golang.org/x/sync v0.1.0 // indirect 64 | golang.org/x/sys v0.15.0 // indirect 65 | golang.org/x/text v0.14.0 // indirect 66 | golang.zx2c4.com/wireguard v0.0.0-20220407013110-ef5c587f782d // indirect 67 | google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect 68 | google.golang.org/grpc v1.56.3 // indirect 69 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect 70 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect 71 | gopkg.in/yaml.v3 v3.0.1 // indirect 72 | ) 73 | -------------------------------------------------------------------------------- /nomad/cluster-manager.hcl: -------------------------------------------------------------------------------- 1 | job "cluster-manager" { 2 | type = "service" 3 | 4 | datacenters = ["dc1"] 5 | 6 | group "cluster-manager" { 7 | count = 1 8 | 9 | network { 10 | port "http" { 11 | static = "8080" 12 | } 13 | mode = "host" 14 | } 15 | service { 16 | name = "wireguard-cluster-manager" 17 | provider = "nomad" 18 | port = "http" 19 | task = "cluster-manager" 20 | } 21 | task "cluster-manager" { 22 | driver = "docker" 23 | 24 | config { 25 | network_mode = "host" 26 | image = "clly/wireguard-cni:v0.0.4" 27 | args = ["cluster-manager","-cidr-prefix","172.16.0.0/12"] 28 | } 29 | } 30 | } 31 | } 32 | 33 | -------------------------------------------------------------------------------- /nomad/countdash.hcl: -------------------------------------------------------------------------------- 1 | job "countdash" { 2 | datacenters = ["dc1"] 3 | 4 | group "api" { 5 | network { 6 | mode = "cni/wgnet" 7 | } 8 | 9 | service { 10 | provider = "nomad" 11 | address_mode = "alloc" 12 | name = "count-api" 13 | 14 | task = "web" 15 | port = "9001" 16 | } 17 | 18 | task "web" { 19 | driver = "docker" 20 | 21 | config { 22 | image = "hashicorpdev/counter-api:v3" 23 | } 24 | } 25 | task "debug" { 26 | driver = "docker" 27 | resources { 28 | memory = 10 29 | } 30 | config { 31 | image = "clly/debug" 32 | cap_add = ["net_raw"] // for pings 33 | args = ["sleep","3600"] 34 | } 35 | } 36 | } 37 | 38 | group "dashboard" { 39 | network { 40 | mode = "bridge" 41 | 42 | port "http" { 43 | static = 9002 44 | to = 9002 45 | host_network = "public" 46 | } 47 | } 48 | 49 | service { 50 | name = "count-dashboard" 51 | port = "http" 52 | provider = "nomad" 53 | address_mode = "alloc" 54 | task = "dashboard" 55 | 56 | } 57 | 58 | task "dashboard" { 59 | driver = "docker" 60 | 61 | template { 62 | data = < 0 { 55 | fmt.Fprintln(w, output) 56 | } 57 | return err 58 | } 59 | 60 | //go:embed tmpl/wireguard.conf.tmpl 61 | var wgConfigTemplate string 62 | 63 | type wgConfig struct { 64 | Address string 65 | PrivateKey string 66 | Port string 67 | PostUp *string 68 | PostDown *string 69 | Peers []Peer 70 | } 71 | 72 | type Peer struct { 73 | Endpoint string 74 | PublicKey string 75 | AllowedIPs string 76 | } 77 | 78 | func (w *WGQuickManager) Config(writer io.Writer) error { 79 | t, err := template.New("wg-config").Parse(wgConfigTemplate) 80 | if err != nil { 81 | return err 82 | } 83 | 84 | // We need to feed the context in for tracing in the future 85 | 86 | // This could also be cached and updated in the background in the future or we can add streaming which would 87 | // probably be more efficient and everyone could get updated at the same time 88 | peers, err := w.client.Peers(context.Background(), connect.NewRequest(&wireguardv1.PeersRequest{})) 89 | if err != nil { 90 | return err 91 | } 92 | 93 | // get the peers that are connected to ourself. Maybe this should be feed differently but this seems easiest right now 94 | var selfPbPeers = []*wireguardv1.Peer{} 95 | if w.peerRegistry != nil { 96 | registryPeers, err := w.peerRegistry.ListPeers() 97 | if err != nil { 98 | return err 99 | } 100 | selfPbPeers = registryPeers 101 | } 102 | 103 | cfgPeer := fromPeerSlice(selfPbPeers, w.self()) 104 | 105 | _, port, err := net.SplitHostPort(w.endpoint) 106 | if err != nil { 107 | return err 108 | } 109 | 110 | cfgPeer = append(cfgPeer, fromPeerSlice(peers.Msg.GetPeers(), w.self())...) 111 | 112 | sort.SliceStable(cfgPeer, func(i, j int) bool { 113 | return cfgPeer[i].AllowedIPs < cfgPeer[j].AllowedIPs 114 | }) 115 | 116 | cfg := wgConfig{ 117 | Address: w.addr, 118 | PrivateKey: w.key.String(), 119 | Port: port, 120 | PostUp: w.postup, 121 | PostDown: w.postdown, 122 | Peers: cfgPeer, 123 | } 124 | 125 | return t.Execute(writer, cfg) 126 | } 127 | 128 | // we could preconstruct this to lower allocs 129 | func (w *WGQuickManager) self() Peer { 130 | return Peer{ 131 | Endpoint: w.endpoint, 132 | PublicKey: w.key.PublicKey().String(), 133 | } 134 | } 135 | 136 | func fromPeerSlice(pbPeers []*wireguardv1.Peer, self Peer) []Peer { 137 | peers := make([]Peer, 0, len(pbPeers)) 138 | for _, pbPeer := range pbPeers { 139 | peer := fromPeer(pbPeer) 140 | if peer.PublicKey == self.PublicKey { 141 | continue 142 | } 143 | peers = append(peers, peer) 144 | } 145 | return peers 146 | } 147 | 148 | func fromPeer(p *wireguardv1.Peer) Peer { 149 | return Peer{ 150 | Endpoint: p.GetEndpoint(), 151 | PublicKey: p.GetPublicKey(), 152 | AllowedIPs: p.GetRoute(), 153 | } 154 | } 155 | -------------------------------------------------------------------------------- /pkg/wireguard/manager_test.go: -------------------------------------------------------------------------------- 1 | package wireguard 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io/ioutil" 7 | "strings" 8 | "testing" 9 | 10 | "connectrpc.com/connect" 11 | "github.com/stretchr/testify/mock" 12 | "github.com/stretchr/testify/require" 13 | 14 | wireguardv1 "github.com/clly/wireguard-cni/gen/wgcni/wireguard/v1" 15 | "github.com/clly/wireguard-cni/gen/wgcni/wireguard/v1/wireguardv1connect" 16 | ) 17 | 18 | func Test_WGQuick_Config(t *testing.T) { 19 | testcases := []struct { 20 | name string 21 | peers []*wireguardv1.Peer 22 | }{ 23 | { 24 | name: "NilPeers", 25 | peers: nil, 26 | }, 27 | { 28 | name: "EmptyPeers", 29 | peers: []*wireguardv1.Peer{}, 30 | }, 31 | { 32 | name: "OnePeer", 33 | peers: []*wireguardv1.Peer{ 34 | { 35 | PublicKey: "abc=", 36 | Endpoint: "192.168.1.2:51820", 37 | Route: "10.0.0.0/24", 38 | }, 39 | }, 40 | }, 41 | { 42 | name: "TwoPeer", 43 | peers: []*wireguardv1.Peer{ 44 | { 45 | PublicKey: "abc=", 46 | Endpoint: "192.168.1.2:51820", 47 | Route: "10.0.0.0/24", 48 | }, 49 | { 50 | PublicKey: "def=", 51 | Endpoint: "192.168.1.3:51820", 52 | Route: "10.0.0.1/24", 53 | }, 54 | }, 55 | }, 56 | } 57 | 58 | for _, testcase := range testcases { 59 | t.Run(testcase.name, func(t *testing.T) { 60 | r := require.New(t) 61 | wireguardM := &wireguardv1connect.MockWireguardServiceClient{} 62 | wgManager := &WGQuickManager{ 63 | client: wireguardM, 64 | key: [32]byte{}, 65 | endpoint: "192.168.1.1:51820", 66 | addr: "10.0.0.1", 67 | } 68 | 69 | pbPeer := &wireguardv1.Peer{ 70 | PublicKey: wgManager.self().PublicKey, 71 | Endpoint: wgManager.endpoint, 72 | Route: "", 73 | } 74 | testcase.peers = append(testcase.peers, pbPeer) 75 | 76 | wireguardM.On("Peers", mock.Anything, mock.Anything). 77 | Once(). 78 | Return(connect.NewResponse(&wireguardv1.PeersResponse{ 79 | Peers: testcase.peers, 80 | }), nil) 81 | 82 | b := bytes.NewBuffer(make([]byte, 0, 1024)) 83 | err := wgManager.Config(b) 84 | r.NoError(err) 85 | golden, err := ioutil.ReadFile(fmt.Sprintf("%s/%s.conf", "hack", testcase.name)) 86 | r.NoError(err) 87 | r.Equal(b.String(), strings.TrimSpace(string(golden))) 88 | }) 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /pkg/wireguard/mock_Peers.go: -------------------------------------------------------------------------------- 1 | // Code generated by mockery v2.40.1. DO NOT EDIT. 2 | 3 | package wireguard 4 | 5 | import ( 6 | wireguardv1 "github.com/clly/wireguard-cni/gen/wgcni/wireguard/v1" 7 | mock "github.com/stretchr/testify/mock" 8 | ) 9 | 10 | // MockPeers is an autogenerated mock type for the Peers type 11 | type MockPeers struct { 12 | mock.Mock 13 | } 14 | 15 | // ListPeers provides a mock function with given fields: 16 | func (_m *MockPeers) ListPeers() ([]*wireguardv1.Peer, error) { 17 | ret := _m.Called() 18 | 19 | if len(ret) == 0 { 20 | panic("no return value specified for ListPeers") 21 | } 22 | 23 | var r0 []*wireguardv1.Peer 24 | var r1 error 25 | if rf, ok := ret.Get(0).(func() ([]*wireguardv1.Peer, error)); ok { 26 | return rf() 27 | } 28 | if rf, ok := ret.Get(0).(func() []*wireguardv1.Peer); ok { 29 | r0 = rf() 30 | } else { 31 | if ret.Get(0) != nil { 32 | r0 = ret.Get(0).([]*wireguardv1.Peer) 33 | } 34 | } 35 | 36 | if rf, ok := ret.Get(1).(func() error); ok { 37 | r1 = rf() 38 | } else { 39 | r1 = ret.Error(1) 40 | } 41 | 42 | return r0, r1 43 | } 44 | 45 | // NewMockPeers creates a new instance of MockPeers. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. 46 | // The first argument is typically a *testing.T value. 47 | func NewMockPeers(t interface { 48 | mock.TestingT 49 | Cleanup(func()) 50 | }) *MockPeers { 51 | mock := &MockPeers{} 52 | mock.Mock.Test(t) 53 | 54 | t.Cleanup(func() { mock.AssertExpectations(t) }) 55 | 56 | return mock 57 | } 58 | -------------------------------------------------------------------------------- /pkg/wireguard/mock_WGClient.go: -------------------------------------------------------------------------------- 1 | // Code generated by mockery v2.40.1. DO NOT EDIT. 2 | 3 | package wireguard 4 | 5 | import ( 6 | mock "github.com/stretchr/testify/mock" 7 | wgtypes "golang.zx2c4.com/wireguard/wgctrl/wgtypes" 8 | ) 9 | 10 | // MockWGClient is an autogenerated mock type for the WGClient type 11 | type MockWGClient struct { 12 | mock.Mock 13 | } 14 | 15 | // Close provides a mock function with given fields: 16 | func (_m *MockWGClient) Close() error { 17 | ret := _m.Called() 18 | 19 | if len(ret) == 0 { 20 | panic("no return value specified for Close") 21 | } 22 | 23 | var r0 error 24 | if rf, ok := ret.Get(0).(func() error); ok { 25 | r0 = rf() 26 | } else { 27 | r0 = ret.Error(0) 28 | } 29 | 30 | return r0 31 | } 32 | 33 | // ConfigureDevice provides a mock function with given fields: name, cfg 34 | func (_m *MockWGClient) ConfigureDevice(name string, cfg wgtypes.Config) error { 35 | ret := _m.Called(name, cfg) 36 | 37 | if len(ret) == 0 { 38 | panic("no return value specified for ConfigureDevice") 39 | } 40 | 41 | var r0 error 42 | if rf, ok := ret.Get(0).(func(string, wgtypes.Config) error); ok { 43 | r0 = rf(name, cfg) 44 | } else { 45 | r0 = ret.Error(0) 46 | } 47 | 48 | return r0 49 | } 50 | 51 | // Device provides a mock function with given fields: name 52 | func (_m *MockWGClient) Device(name string) (*wgtypes.Device, error) { 53 | ret := _m.Called(name) 54 | 55 | if len(ret) == 0 { 56 | panic("no return value specified for Device") 57 | } 58 | 59 | var r0 *wgtypes.Device 60 | var r1 error 61 | if rf, ok := ret.Get(0).(func(string) (*wgtypes.Device, error)); ok { 62 | return rf(name) 63 | } 64 | if rf, ok := ret.Get(0).(func(string) *wgtypes.Device); ok { 65 | r0 = rf(name) 66 | } else { 67 | if ret.Get(0) != nil { 68 | r0 = ret.Get(0).(*wgtypes.Device) 69 | } 70 | } 71 | 72 | if rf, ok := ret.Get(1).(func(string) error); ok { 73 | r1 = rf(name) 74 | } else { 75 | r1 = ret.Error(1) 76 | } 77 | 78 | return r0, r1 79 | } 80 | 81 | // Devices provides a mock function with given fields: 82 | func (_m *MockWGClient) Devices() ([]*wgtypes.Device, error) { 83 | ret := _m.Called() 84 | 85 | if len(ret) == 0 { 86 | panic("no return value specified for Devices") 87 | } 88 | 89 | var r0 []*wgtypes.Device 90 | var r1 error 91 | if rf, ok := ret.Get(0).(func() ([]*wgtypes.Device, error)); ok { 92 | return rf() 93 | } 94 | if rf, ok := ret.Get(0).(func() []*wgtypes.Device); ok { 95 | r0 = rf() 96 | } else { 97 | if ret.Get(0) != nil { 98 | r0 = ret.Get(0).([]*wgtypes.Device) 99 | } 100 | } 101 | 102 | if rf, ok := ret.Get(1).(func() error); ok { 103 | r1 = rf() 104 | } else { 105 | r1 = ret.Error(1) 106 | } 107 | 108 | return r0, r1 109 | } 110 | 111 | // NewMockWGClient creates a new instance of MockWGClient. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. 112 | // The first argument is typically a *testing.T value. 113 | func NewMockWGClient(t interface { 114 | mock.TestingT 115 | Cleanup(func()) 116 | }) *MockWGClient { 117 | mock := &MockWGClient{} 118 | mock.Mock.Test(t) 119 | 120 | t.Cleanup(func() { mock.AssertExpectations(t) }) 121 | 122 | return mock 123 | } 124 | -------------------------------------------------------------------------------- /pkg/wireguard/mock_WGOption.go: -------------------------------------------------------------------------------- 1 | // Code generated by mockery v2.40.1. DO NOT EDIT. 2 | 3 | package wireguard 4 | 5 | import mock "github.com/stretchr/testify/mock" 6 | 7 | // MockWGOption is an autogenerated mock type for the WGOption type 8 | type MockWGOption struct { 9 | mock.Mock 10 | } 11 | 12 | // Execute provides a mock function with given fields: _a0 13 | func (_m *MockWGOption) Execute(_a0 *WGQuickManager) { 14 | _m.Called(_a0) 15 | } 16 | 17 | // NewMockWGOption creates a new instance of MockWGOption. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. 18 | // The first argument is typically a *testing.T value. 19 | func NewMockWGOption(t interface { 20 | mock.TestingT 21 | Cleanup(func()) 22 | }) *MockWGOption { 23 | mock := &MockWGOption{} 24 | mock.Mock.Test(t) 25 | 26 | t.Cleanup(func() { mock.AssertExpectations(t) }) 27 | 28 | return mock 29 | } 30 | -------------------------------------------------------------------------------- /pkg/wireguard/mock_WireguardManager.go: -------------------------------------------------------------------------------- 1 | // Code generated by mockery v2.40.1. DO NOT EDIT. 2 | 3 | package wireguard 4 | 5 | import ( 6 | io "io" 7 | 8 | mock "github.com/stretchr/testify/mock" 9 | wgtypes "golang.zx2c4.com/wireguard/wgctrl/wgtypes" 10 | ) 11 | 12 | // MockWireguardManager is an autogenerated mock type for the WireguardManager type 13 | type MockWireguardManager struct { 14 | mock.Mock 15 | } 16 | 17 | // Config provides a mock function with given fields: w 18 | func (_m *MockWireguardManager) Config(w io.Writer) error { 19 | ret := _m.Called(w) 20 | 21 | if len(ret) == 0 { 22 | panic("no return value specified for Config") 23 | } 24 | 25 | var r0 error 26 | if rf, ok := ret.Get(0).(func(io.Writer) error); ok { 27 | r0 = rf(w) 28 | } else { 29 | r0 = ret.Error(0) 30 | } 31 | 32 | return r0 33 | } 34 | 35 | // Device provides a mock function with given fields: 36 | func (_m *MockWireguardManager) Device() *wgtypes.Device { 37 | ret := _m.Called() 38 | 39 | if len(ret) == 0 { 40 | panic("no return value specified for Device") 41 | } 42 | 43 | var r0 *wgtypes.Device 44 | if rf, ok := ret.Get(0).(func() *wgtypes.Device); ok { 45 | r0 = rf() 46 | } else { 47 | if ret.Get(0) != nil { 48 | r0 = ret.Get(0).(*wgtypes.Device) 49 | } 50 | } 51 | 52 | return r0 53 | } 54 | 55 | // Down provides a mock function with given fields: device 56 | func (_m *MockWireguardManager) Down(device string) error { 57 | ret := _m.Called(device) 58 | 59 | if len(ret) == 0 { 60 | panic("no return value specified for Down") 61 | } 62 | 63 | var r0 error 64 | if rf, ok := ret.Get(0).(func(string) error); ok { 65 | r0 = rf(device) 66 | } else { 67 | r0 = ret.Error(0) 68 | } 69 | 70 | return r0 71 | } 72 | 73 | // SetPeers provides a mock function with given fields: device, peers 74 | func (_m *MockWireguardManager) SetPeers(device string, peers []*Peer) error { 75 | ret := _m.Called(device, peers) 76 | 77 | if len(ret) == 0 { 78 | panic("no return value specified for SetPeers") 79 | } 80 | 81 | var r0 error 82 | if rf, ok := ret.Get(0).(func(string, []*Peer) error); ok { 83 | r0 = rf(device, peers) 84 | } else { 85 | r0 = ret.Error(0) 86 | } 87 | 88 | return r0 89 | } 90 | 91 | // Up provides a mock function with given fields: device 92 | func (_m *MockWireguardManager) Up(device string) error { 93 | ret := _m.Called(device) 94 | 95 | if len(ret) == 0 { 96 | panic("no return value specified for Up") 97 | } 98 | 99 | var r0 error 100 | if rf, ok := ret.Get(0).(func(string) error); ok { 101 | r0 = rf(device) 102 | } else { 103 | r0 = ret.Error(0) 104 | } 105 | 106 | return r0 107 | } 108 | 109 | // NewMockWireguardManager creates a new instance of MockWireguardManager. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. 110 | // The first argument is typically a *testing.T value. 111 | func NewMockWireguardManager(t interface { 112 | mock.TestingT 113 | Cleanup(func()) 114 | }) *MockWireguardManager { 115 | mock := &MockWireguardManager{} 116 | mock.Mock.Test(t) 117 | 118 | t.Cleanup(func() { mock.AssertExpectations(t) }) 119 | 120 | return mock 121 | } 122 | -------------------------------------------------------------------------------- /pkg/wireguard/option.go: -------------------------------------------------------------------------------- 1 | package wireguard 2 | 3 | import "io" 4 | 5 | type WGOption func(*WGQuickManager) 6 | 7 | func WithOutput(w io.Writer) WGOption { 8 | return func(wm *WGQuickManager) { 9 | wm.logOutput = w 10 | } 11 | } 12 | 13 | func WithPost(postUp, postDown string) WGOption { 14 | return func(wm *WGQuickManager) { 15 | wm.postup = &postUp 16 | wm.postdown = &postDown 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /pkg/wireguard/tmpl/wireguard.conf.tmpl: -------------------------------------------------------------------------------- 1 | [Interface] 2 | Address = {{ .Address }} 3 | PrivateKey = {{ .PrivateKey }} 4 | ListenPort = {{ .Port }} 5 | {{- if .PostUp }} 6 | PostUp = {{ .PostUp }} 7 | {{- end }} 8 | {{- if .PostDown }} 9 | PostDown = {{ .PostDown }} 10 | {{- end }} 11 | 12 | {{- range $index, $peer := .Peers }} 13 | 14 | [Peer] 15 | Endpoint = {{ $peer.Endpoint }} 16 | PublicKey = {{ $peer.PublicKey }} 17 | AllowedIPs = {{ $peer.AllowedIPs }} 18 | {{- end }} -------------------------------------------------------------------------------- /pkg/wireguard/varz.go: -------------------------------------------------------------------------------- 1 | package wireguard 2 | 3 | import "expvar" 4 | 5 | var ( 6 | expWireguardPublicKey = &expvar.String{} 7 | ) 8 | 9 | func init() { 10 | expvar.Publish("wireguard-public-key", expWireguardPublicKey) 11 | } 12 | -------------------------------------------------------------------------------- /run-dev.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -euo pipefail 4 | 5 | PACKAGED_CNI_PATH=/opt/cni/bin/ 6 | NETCONFPATH=$PWD/config 7 | 8 | cnipath=$(mktemp -d) 9 | trap "rm -rf $cnipath" EXIT 10 | 11 | rsync -a $PACKAGED_CNI_PATH $cnipath/ 12 | go build . 13 | cp wireguard-cni $cnipath 14 | ( 15 | cd scripts 16 | sudo id 17 | sudo NETCONFPATH=$NETCONFPATH CNI_PATH=$cnipath ./priv-net-run.sh ip addr && ip link 18 | ) 19 | -------------------------------------------------------------------------------- /scripts/docker-run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Run a docker container with network namespace set up by the 4 | # CNI plugins. 5 | 6 | # Example usage: ./docker-run.sh --rm busybox /sbin/ifconfig 7 | 8 | contid=$(docker run -d --net=none busybox:latest /bin/sleep 10000000) 9 | pid=$(docker inspect -f '{{ .State.Pid }}' $contid) 10 | netnspath=/proc/$pid/ns/net 11 | 12 | ./exec-plugins.sh add $contid $netnspath 13 | 14 | function cleanup() { 15 | ./exec-plugins.sh del $contid $netnspath 16 | docker rm -f $contid >/dev/null 17 | } 18 | trap cleanup EXIT 19 | 20 | docker run --net=container:$contid $@ 21 | 22 | -------------------------------------------------------------------------------- /scripts/exec-plugins.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | if [[ ${DEBUG} -gt 0 ]]; then set -x; fi 4 | 5 | NETCONFPATH="${NETCONFPATH-/etc/cni/net.d}" 6 | 7 | function exec_list() { 8 | plist="$1" 9 | name="$2" 10 | cniVersion="$3" 11 | echo "$plist" | jq -c '.[]' | while read -r conf; do 12 | plugin_bin="$(echo "$conf" | jq -r '.type')" 13 | conf="$(echo "$conf" | jq -r ".name = \"$name\" | .cniVersion = \"$cniVersion\"")" 14 | if [ -n "$res" ]; then 15 | conf="$(echo "$conf" | jq -r ".prevResult=$res")" 16 | fi 17 | if ! res=$(echo "$conf" | $plugin_bin); then 18 | error "$name" "$res" 19 | elif [[ ${DEBUG} -gt 0 ]]; then 20 | echo "${res}" | jq -r . 21 | fi 22 | done 23 | } 24 | 25 | function error () { 26 | name="$1" 27 | res="$2" 28 | err_msg=$(echo "$res" | jq -r '.msg') 29 | if [ -z "$errmsg" ]; then 30 | err_msg=$res 31 | fi 32 | echo "${name} : error executing $CNI_COMMAND: $err_msg" 33 | exit 1 34 | } 35 | 36 | function exec_plugins() { 37 | i=0 38 | contid=$2 39 | netns=$3 40 | export CNI_COMMAND=$(echo $1 | tr '[:lower:]' '[:upper:]') 41 | export PATH=$CNI_PATH:$PATH 42 | export CNI_CONTAINERID=$contid 43 | export CNI_NETNS=$netns 44 | 45 | for netconf in $(echo "$NETCONFPATH"/*.conf | sort); do 46 | export CNI_IFNAME=$(printf eth%d $i) 47 | name=$(jq -r '.name' <"$netconf") 48 | cniVersion=$(jq -r '.cniVersion' <"$netconf") 49 | plist=$(jq '.plugins | select(.!=null)' <"$netconf") 50 | if [ -n "$plist" ]; then 51 | exec_list "$plist" "$name" "$cniVersion" 52 | else 53 | plugin=$(jq -r '.type' <"$netconf") 54 | 55 | if ! res=$($plugin <"$netconf"); then 56 | error "$name" "$res" 57 | elif [[ ${DEBUG} -gt 0 ]]; then 58 | echo "${res}" | jq -r . 59 | fi 60 | fi 61 | 62 | (( i++ )) || true 63 | done 64 | } 65 | 66 | if [ $# -ne 3 ]; then 67 | echo "Usage: $0 add|del CONTAINER-ID NETNS-PATH" 68 | echo " Adds or deletes the container specified by NETNS-PATH to the networks" 69 | echo " specified in \$NETCONFPATH directory" 70 | exit 1 71 | fi 72 | 73 | exec_plugins $1 $2 $3 74 | -------------------------------------------------------------------------------- /scripts/priv-net-run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | if [[ ${DEBUG} -gt 0 ]]; then set -x; fi 4 | 5 | # Run a command in a private network namespace 6 | # set up by CNI plugins 7 | contid=$(printf '%x%x%x%x' $RANDOM $RANDOM $RANDOM $RANDOM) 8 | netnspath=/var/run/netns/$contid 9 | 10 | ip netns add $contid 11 | ./exec-plugins.sh add $contid $netnspath 12 | 13 | 14 | function cleanup() { 15 | ./exec-plugins.sh del $contid $netnspath 16 | ip netns delete $contid 17 | } 18 | trap cleanup EXIT 19 | 20 | ip netns exec $contid "$@" 21 | -------------------------------------------------------------------------------- /scripts/release.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -xe 3 | 4 | SRC_DIR="${SRC_DIR:-$PWD}" 5 | BUILDFLAGS="-a --ldflags '-extldflags \"-static\"'" 6 | 7 | TAG=$(git describe --tags --dirty) 8 | RELEASE_DIR=release-${TAG} 9 | 10 | OUTPUT_DIR=bin 11 | 12 | # Always clean first 13 | rm -Rf ${SRC_DIR}/${RELEASE_DIR} 14 | mkdir -p ${SRC_DIR}/${RELEASE_DIR} 15 | mkdir -p ${OUTPUT_DIR} 16 | 17 | docker run -i -v ${SRC_DIR}:/opt/src --rm golang:1.14-alpine \ 18 | /bin/sh -xe -c "\ 19 | apk --no-cache add bash tar; 20 | cd /opt/src; umask 0022; 21 | for arch in amd64 arm arm64 ppc64le s390x; do \ 22 | rm -f ${OUTPUT_DIR}/* ; \ 23 | CGO_ENABLED=0 GOARCH=\$arch ./build.sh ${BUILDFLAGS}; \ 24 | for format in tgz; do \ 25 | FILENAME=cni-\$arch-${TAG}.\$format; \ 26 | FILEPATH=${RELEASE_DIR}/\$FILENAME; \ 27 | tar -C ${OUTPUT_DIR} --owner=0 --group=0 -caf \$FILEPATH .; \ 28 | if [ \"\$arch\" == \"amd64\" ]; then \ 29 | cp \$FILEPATH ${RELEASE_DIR}/cni-${TAG}.\$format; \ 30 | fi; \ 31 | done; \ 32 | done; 33 | cd ${RELEASE_DIR}; 34 | for f in *.tgz; do sha1sum \$f > \$f.sha1; done; 35 | for f in *.tgz; do sha256sum \$f > \$f.sha256; done; 36 | for f in *.tgz; do sha512sum \$f > \$f.sha512; done; 37 | cd .. 38 | chown -R ${UID} ${OUTPUT_DIR} ${RELEASE_DIR}" 39 | -------------------------------------------------------------------------------- /scripts/wg-client.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # SPDX-License-Identifier: GPL-2.0 3 | # 4 | # Copyright (C) 2015-2020 Jason A. Donenfeld . All Rights Reserved. 5 | 6 | set -e 7 | [[ $UID == 0 ]] || { echo "You must be root to run this."; exit 1; } 8 | exec 3<>/dev/tcp/demo.wireguard.com/42912 9 | privatekey="$(wg genkey)" 10 | wg pubkey <<<"$privatekey" >&3 11 | IFS=: read -r status server_pubkey server_port internal_ip <&3 12 | [[ $status == OK ]] 13 | ip link del dev wg0 2>/dev/null || true 14 | ip link add dev wg0 type wireguard 15 | wg set wg0 private-key <(echo "$privatekey") peer "$server_pubkey" allowed-ips 0.0.0.0/0 endpoint "demo.wireguard.com:$server_port" persistent-keepalive 25 16 | ip address add "$internal_ip"/24 dev wg0 17 | ip link set up dev wg0 18 | if [ "$1" == "default-route" ]; then 19 | host="$(wg show wg0 endpoints | sed -n 's/.*\t\(.*\):.*/\1/p')" 20 | ip route add $(ip route get $host | sed '/ via [0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}/{s/^\(.* via [0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\).*/\1/}' | head -n 1) 2>/dev/null || true 21 | ip route add 0/1 dev wg0 22 | ip route add 128/1 dev wg0 23 | fi 24 | -------------------------------------------------------------------------------- /wgcni/ipam/v1/ipam.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package wgcni.ipam.v1; 4 | 5 | option go_package = "github.com/clly/wireguard-cni/gen/wgcni/ipam/v1;ipamv1"; 6 | 7 | service IPAMService { 8 | // Alloc requests a IP address or subnet from the ipam server 9 | rpc Alloc(AllocRequest) returns (AllocResponse) {} 10 | } 11 | 12 | enum IPVersion { 13 | IP_VERSION_UNSPECIFIED = 0; 14 | IP_VERSION_V4 = 1; 15 | IP_VERSION_V6 = 2; 16 | } 17 | 18 | message IPAlloc { 19 | // address is the IP address of the allocation (192.168.1.0, 192.168.3.5) 20 | string address = 1; 21 | // netmask is the subnet mask in / notatation (/24, /32) 22 | string netmask = 2; 23 | IPVersion version = 3; 24 | } 25 | 26 | message AllocRequest {} 27 | 28 | message AllocResponse { 29 | IPAlloc alloc = 1; 30 | } 31 | -------------------------------------------------------------------------------- /wgcni/wireguard/v1/wireguard.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package wgcni.wireguard.v1; 4 | 5 | option go_package = "github.com/clly/wireguard-cni/gen/wgcni/wireguard/v1;wireguardv1"; 6 | 7 | service WireguardService { 8 | // Register will register a wireguard peer 9 | rpc Register(RegisterRequest) returns (RegisterResponse) {} 10 | 11 | // Peers rpc will return a list of all wireguard peers 12 | rpc Peers(PeersRequest) returns (PeersResponse) {} 13 | } 14 | 15 | message RegisterRequest { 16 | // public_key is the wireguard public key 17 | string public_key = 1; 18 | // endpoint is the address:port clients should dial to connect to this wireguard peer 19 | string endpoint = 2; 20 | // route is the addresses that can be found on this endpoint 21 | string route = 3; 22 | // namespace is the path to the network namespace that this peer belongs to 23 | string namespace = 4; 24 | } 25 | 26 | message RegisterResponse {} 27 | 28 | message PeersRequest {} 29 | 30 | message PeersResponse { 31 | repeated Peer peers = 1; 32 | } 33 | 34 | message Peer { 35 | string public_key = 1; 36 | string endpoint = 2; 37 | string route = 3; 38 | } 39 | --------------------------------------------------------------------------------