├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ └── issue.md └── workflows │ ├── ci.yaml │ └── release.yaml ├── .gitignore ├── CHANGELOG.md ├── LICENSE ├── README.md ├── RELEASE.md ├── docs ├── cmd-coil-egress-controller.md ├── cmd-coil-egress.md ├── cmd-coil-installer.md ├── cmd-coil-ipam-controller.md ├── cmd-coil-router.md ├── cmd-coil.md ├── cmd-coild.md ├── cni-grpc.md ├── design.md ├── img │ └── dashboard.png ├── setup.md └── usage.md └── v2 ├── .dockerignore ├── .gitignore ├── .vscode └── settings.json ├── Dockerfile ├── Makefile ├── PROJECT ├── api └── v2 │ ├── addressblock_types.go │ ├── addresspool_types.go │ ├── addresspool_types_test.go │ ├── addresspool_webhook.go │ ├── addresspool_webhook_test.go │ ├── blockrequest_types.go │ ├── blockrequest_types_test.go │ ├── egress_types.go │ ├── egress_webhook.go │ ├── egress_webhook_test.go │ ├── groupversion_info.go │ ├── suite_test.go │ └── zz_generated.deepcopy.go ├── cmd ├── coil-egress-controller │ ├── main.go │ └── sub │ │ ├── root.go │ │ └── run.go ├── coil-egress │ ├── main.go │ └── sub │ │ ├── root.go │ │ └── run.go ├── coil-installer │ ├── main.go │ └── sub │ │ ├── install.go │ │ └── root.go ├── coil-ipam-controller │ ├── main.go │ └── sub │ │ ├── root.go │ │ └── run.go ├── coil-router │ ├── main.go │ └── sub │ │ ├── root.go │ │ └── run.go ├── coil │ ├── main.go │ ├── rpc.go │ ├── types.go │ └── types_test.go ├── coild │ ├── main.go │ └── sub │ │ ├── root.go │ │ └── run.go └── gencert │ └── main.go ├── config ├── cke │ ├── kustomization.yaml │ ├── webhook-secret.yaml │ └── webhook_manifests_patch.yaml ├── crd │ ├── bases │ │ ├── coil.cybozu.com_addressblocks.yaml │ │ ├── coil.cybozu.com_addresspools.yaml │ │ ├── coil.cybozu.com_blockrequests.yaml │ │ └── coil.cybozu.com_egresses.yaml │ ├── egress │ │ └── kustomization.yaml │ ├── kustomization.yaml │ ├── kustomizeconfig.yaml │ └── patches │ │ ├── cainjection_in_addressblocks.yaml │ │ ├── cainjection_in_addresspools.yaml │ │ ├── cainjection_in_blockrequests.yaml │ │ ├── cainjection_in_egresses.yaml │ │ ├── egress │ │ └── remove_status.yaml │ │ ├── remove_status.yaml │ │ ├── webhook_in_addressblocks.yaml │ │ ├── webhook_in_addresspools.yaml │ │ ├── webhook_in_blockrequests.yaml │ │ └── webhook_in_egresses.yaml ├── default │ ├── .gitignore │ ├── egress │ │ ├── v4 │ │ │ └── kustomization.yaml │ │ ├── v6 │ │ │ └── kustomization.yaml │ │ └── webhook_manifests_patch.yaml.tmpl │ ├── ipam │ │ └── webhook_manifests_patch.yaml.tmpl │ └── kustomization.yaml ├── pod │ ├── coil-egress-controller.yaml │ ├── coil-ipam-controller.yaml │ ├── coil-router.yaml │ ├── coild.yaml │ ├── compat_calico.yaml │ ├── egress │ │ ├── v4 │ │ │ ├── coild.yaml │ │ │ └── kustomization.yaml │ │ └── v6 │ │ │ ├── coild.yaml │ │ │ └── kustomization.yaml │ ├── generate_certs.yaml │ └── kustomization.yaml ├── rbac │ ├── addressblock_viewer_role.yaml │ ├── addresspool_viewer_role.yaml │ ├── blockrequest_viewer_role.yaml │ ├── coil-egress-controller-certs_role.yaml │ ├── coil-egress-controller_role.yaml │ ├── coil-egress_role.yaml │ ├── coil-ipam-controller-certs_role.yaml │ ├── coil-ipam-controller_role.yaml │ ├── coil-router_role.yaml │ ├── coild_role.yaml │ ├── egress │ │ └── kustomization.yaml │ ├── egress_viewer_role.yaml │ ├── kustomization.yaml │ ├── leader_election_role.yaml │ ├── leader_election_role_binding.yaml │ ├── role_binding.yaml │ └── serviceaccount.yaml ├── samples │ ├── coil_v2_addresspool.yaml │ └── coil_v2_egress.yaml └── webhook │ ├── egress │ ├── kustomization.yaml │ ├── manifests.yaml │ └── service.yaml │ ├── ipam │ ├── kustomization.yaml │ ├── manifests.yaml │ └── service.yaml │ └── kustomizeconfig.yaml ├── controllers ├── addressblock_controller.go ├── addressblock_controller_test.go ├── addresspool_controller.go ├── addresspool_controller_test.go ├── blockrequest_controller.go ├── blockrequest_controller_test.go ├── blockrequest_watcher.go ├── blockrequest_watcher_test.go ├── clusterrolebinding_controller.go ├── egress_controller.go ├── egress_controller_test.go ├── egress_watcher.go ├── egress_watcher_test.go ├── mock_test.go ├── pod_watcher.go ├── pod_watcher_test.go └── suite_test.go ├── dashboard └── coil.json ├── e2e ├── Makefile ├── README.md ├── coil-ipam-controller_patch.yaml ├── coil_test.go ├── configs │ └── egress │ │ ├── dualstack │ │ └── kustomization.yaml │ │ ├── v4 │ │ └── kustomization.yaml │ │ └── v6 │ │ └── kustomization.yaml ├── controller_test.go ├── daemon.json ├── echo-server │ └── main.go ├── kind-config.yaml ├── kind-config_dualstack.yaml ├── kind-config_dualstack_kindnet.yaml ├── kind-config_dualstack_v6.yaml ├── kind-config_dualstack_v6_kindnet.yaml ├── kind-config_kindnet.yaml ├── kind-config_kindnet_v6.yaml ├── kind-config_v6.yaml ├── kindnet-configurer │ └── main.go ├── kustomization.yaml ├── manifests │ ├── another_httpd.yaml │ ├── default_pool.yaml │ ├── default_pool_dualstack.yaml │ ├── default_pool_v6.yaml │ ├── dummy_pod.yaml │ ├── dummy_pool.yaml │ ├── egress-sport-auto.yaml │ ├── egress-updated.yaml │ ├── egress.yaml │ ├── httpd.yaml │ ├── invalid_pool.yaml │ ├── nat-client-sport-auto.yaml │ ├── nat-client.yaml │ ├── orphaned.yaml │ └── ubuntu.yaml ├── netconf │ ├── netconf-kindnet-dualstack.json │ ├── netconf-kindnet-v4.json │ └── netconf-kindnet-v6.json └── suite_test.go ├── go.mod ├── go.sum ├── hack └── boilerplate.go.txt ├── kustomization.yaml ├── netconf.json ├── pkg ├── cert │ └── cert.go ├── cnirpc │ ├── cni.pb.go │ ├── cni.proto │ ├── cni_grpc.pb.go │ └── mock_test.go ├── config │ └── config.go ├── constants │ ├── constants.go │ └── paths.go ├── founat │ ├── client.go │ ├── client_test.go │ ├── egress.go │ ├── egress_test.go │ ├── errors.go │ ├── fou.go │ ├── fou_test.go │ ├── nat_test.go │ └── setup_test.go ├── indexing │ └── indexing.go ├── ipam │ ├── allocator.go │ ├── allocator_test.go │ ├── node.go │ ├── node_test.go │ ├── pool.go │ ├── pool_test.go │ └── suite_test.go ├── metrics │ ├── collector.go │ └── egress.go ├── nodenet │ ├── pod.go │ ├── pod_test.go │ ├── route_exporter.go │ ├── route_exporter_test.go │ ├── route_syncer.go │ └── route_syncer_test.go └── test │ └── ip_matcher.go ├── runners ├── coild_server.go ├── coild_server_test.go ├── garbage_collector.go ├── garbage_collector_test.go ├── router.go ├── router_test.go └── suite_test.go └── version.go /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: bug 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **Environments** 14 | - Version: 15 | - OS: 16 | 17 | **To Reproduce** 18 | Steps to reproduce the behavior: 19 | 1. Go to '...' 20 | 2. Click on '....' 21 | 3. Scroll down to '....' 22 | 4. See error 23 | 24 | **Expected behavior** 25 | A clear and concise description of what you expected to happen. 26 | 27 | **Additional context** 28 | Add any other context about the problem here. 29 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/issue.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Task 3 | about: Describe this issue 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | ## What 11 | 12 | Describe what this issue should address. 13 | 14 | ## How 15 | 16 | Describe how to address the issue. 17 | 18 | ## Checklist 19 | 20 | - [ ] Finish implentation of the issue 21 | - [ ] Test all functions 22 | - [ ] Have enough logs to trace activities 23 | - [ ] Notify developers of necessary actions 24 | -------------------------------------------------------------------------------- /.github/workflows/release.yaml: -------------------------------------------------------------------------------- 1 | name: Release 2 | on: 3 | push: 4 | tags: 5 | - 'v*' 6 | defaults: 7 | run: 8 | working-directory: v2 9 | env: 10 | go-version: "1.24" 11 | cache-version: 1 12 | jobs: 13 | image: 14 | name: Push container image 15 | runs-on: ubuntu-24.04 16 | steps: 17 | - uses: actions/checkout@v4 18 | - uses: actions/setup-go@v5 19 | with: 20 | go-version: ${{ env.go-version }} 21 | - name: Set up QEMU 22 | uses: docker/setup-qemu-action@v3 23 | with: 24 | platforms: linux/amd64,linux/arm64/v8 25 | - name: Set up Docker Buildx 26 | uses: docker/setup-buildx-action@v3 27 | - name: Cache tools 28 | id: cache-tools 29 | uses: actions/cache@v4 30 | with: 31 | path: | 32 | v2/bin 33 | v2/include 34 | key: cache-${{ env.cache-version }}-go-${{ env.go-version }}-${{ hashFiles('v2/Makefile') }} 35 | - run: make setup 36 | if: steps.cache-tools.outputs.cache-hit != 'true' 37 | - name: Login to GitHub Container Registry 38 | uses: docker/login-action@v3 39 | with: 40 | registry: ghcr.io 41 | username: ${{ github.actor }} 42 | password: ${{ secrets.GITHUB_TOKEN }} 43 | - name: Set Tag 44 | id: set-tag 45 | run: echo "RELEASE_TAG=${GITHUB_REF#refs/tags/v}" >> $GITHUB_OUTPUT # Remove "v" prefix. 46 | - name: Build and push 47 | uses: docker/build-push-action@v6 48 | with: 49 | context: ./v2 50 | platforms: linux/amd64,linux/arm64/v8 51 | push: true 52 | tags: ghcr.io/cybozu-go/coil:${{ steps.set-tag.outputs.RELEASE_TAG }} 53 | release: 54 | name: Release on GitHub 55 | needs: image 56 | runs-on: ubuntu-24.04 57 | steps: 58 | - uses: actions/checkout@v4 59 | - name: Create release 60 | env: 61 | GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} 62 | run: | 63 | VERSION=${GITHUB_REF#refs/tags/} # Don't remove "v" prefix. 64 | if echo ${VERSION} | grep -q -e '-'; then PRERELEASE_FLAG=-p; fi 65 | gh release create $VERSION $PRERELEASE_FLAG \ 66 | -t "Release $VERSION" \ 67 | -n "See [CHANGELOG.md](./CHANGELOG.md) for details." 68 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Test binary, build with `go test -c` 2 | *.test 3 | 4 | # Output of the go coverage tool, specifically when used with LiteIDE 5 | *.out 6 | 7 | # Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736 8 | .glide/ 9 | 10 | # Editors 11 | *~ 12 | .*.swp 13 | .#* 14 | \#*# 15 | /.vscode 16 | -------------------------------------------------------------------------------- /RELEASE.md: -------------------------------------------------------------------------------- 1 | Release procedure 2 | ================= 3 | 4 | This document describes how to release a new version of Coil. 5 | 6 | ## Versioning 7 | 8 | Follow [semantic versioning 2.0.0][semver] to choose the new version number. 9 | 10 | ## Prepare change log entries 11 | 12 | Add notable changes since the last release to [CHANGELOG.md](CHANGELOG.md). 13 | It should look like: 14 | 15 | ```markdown 16 | (snip) 17 | ## [Unreleased] 18 | 19 | ### Added 20 | - Implement ... (#35) 21 | 22 | ### Changed 23 | - Fix a bug in ... (#33) 24 | 25 | ### Removed 26 | - Deprecated `-option` is removed ... (#39) 27 | 28 | (snip) 29 | ``` 30 | 31 | ## Adding and removing supported Kubernetes versions 32 | 33 | - Edit [`.github/workflows/ci.yaml`](.github/workflows/ci.yaml) and edit `kindtest-node` field values. 34 | - Edit Kubernetes versions in [`README.md`](README.md). 35 | - Make sure that the changes pass CI. 36 | 37 | You should also update `sigs.k8s.io/controller-runtime` Go package periodically. 38 | 39 | ## Bump version 40 | 41 | 1. Determine a new version number. Then set `VERSION` variable. 42 | 43 | ```console 44 | # Set VERSION and confirm it. It should not have "v" prefix. 45 | $ VERSION=x.y.z 46 | $ echo $VERSION 47 | ``` 48 | 49 | 2. Make a branch to release 50 | 51 | ```console 52 | $ git switch main 53 | $ git pull origin main 54 | $ git switch -c "bump-$VERSION" 55 | ``` 56 | 57 | 3. Edit `CHANGELOG.md` for the new version ([example][]). 58 | 4. Edit `v2/version.go` for the new version. 59 | 5. Edit `v2/kustomization.yaml` and update `newTag` value for the new version. 60 | 6. Commit the changes and push it. 61 | 62 | ```console 63 | $ git commit -a -m "Bump version to $VERSION" 64 | $ gh pr create --fill 65 | ``` 66 | 67 | 7. Merge this branch. 68 | 8. Add a git tag to the main HEAD, then push it. 69 | 70 | ```console 71 | # Set VERSION again. 72 | $ VERSION=x.y.z 73 | $ echo $VERSION 74 | 75 | $ git checkout main 76 | $ git pull 77 | $ git tag -a -m "Release v$VERSION" "v$VERSION" 78 | 79 | # Make sure the release tag exists. 80 | $ git tag -ln | grep $VERSION 81 | 82 | $ git push origin "v$VERSION" 83 | ``` 84 | 85 | GitHub actions will build and push artifacts such as container images and 86 | create a new GitHub release. 87 | 88 | [semver]: https://semver.org/spec/v2.0.0.html 89 | [example]: https://github.com/cybozu-go/etcdpasswd/commit/77d95384ac6c97e7f48281eaf23cb94f68867f79 90 | -------------------------------------------------------------------------------- /docs/cmd-coil-egress-controller.md: -------------------------------------------------------------------------------- 1 | coil-egress-controller 2 | =============== 3 | 4 | `coil-egress-controller` is a Kubernetes controller for Coil custom resources related to on-demand NAT egress. 5 | It is intended to be run as a Pod in `kube-system` namespace. 6 | 7 | 8 | ## Egress 9 | 10 | `coil-egress-controller` creates **Deployment** and **Service** for each Egress. 11 | 12 | It also creates `coil-egress` **ServiceAccount** in the namespace of Egress, 13 | and binds it to the **ClusterRoles** for `coil-egress`. 14 | 15 | ## Command-line flags 16 | 17 | ``` 18 | Flags: 19 | --cert-dir string directory to locate TLS certs for webhook (default "/certs") 20 | --egress-port int32 UDP port number used by coil-egress (default 5555) 21 | --health-addr string bind address of health/readiness probes (default ":9387") 22 | -h, --help help for coil-egress-controller 23 | --metrics-addr string bind address of metrics endpoint (default ":9386") 24 | -v, --version version for coil-egress-controller 25 | --webhook-addr string bind address of admission webhook (default ":9443") 26 | --enable-cert-rotation enables webhook's certificate generation 27 | --enable-restart-on-cert-refresh enables pod's restart on webhook certificate refresh 28 | ``` 29 | -------------------------------------------------------------------------------- /docs/cmd-coil-egress.md: -------------------------------------------------------------------------------- 1 | coil-egress 2 | =========== 3 | 4 | `coil-egress` is a program to be run in Egress pod. 5 | 6 | It watches client Pods and creates or deletes Foo-over-UDP tunnels. 7 | 8 | ## Environment variables 9 | 10 | `coil-egress` references the following environment variables: 11 | 12 | | Name | Required | Description | 13 | | -------------------- | -------- | ------------------------------------------------------ | 14 | | `COIL_POD_ADDRESSES` | YES | `status.podIPs` field value of the Pod. | 15 | | `COIL_POD_NAMESPACE` | YES | `metadata.namespace` field value of the parent Egress. | 16 | | `COIL_EGRESS_NAME` | YES | `metadata.name` field value of the parent Egress. | 17 | 18 | ## Command-line flags 19 | 20 | ``` 21 | Flags: 22 | --fou-port int port number for foo-over-udp tunnels (default 5555) 23 | --enable-sport-auto enable automatic source port assignment (default false) 24 | --health-addr string bind address of health/readiness probes (default ":8081") 25 | -h, --help help for coil-egress 26 | --metrics-addr string bind address of metrics endpoint (default ":8080") 27 | -v, --version version for coil-egress 28 | ``` 29 | 30 | ## Prometheus metrics 31 | 32 | ### `coil_egress_client_pod_count` 33 | 34 | This is the number of client pods which use the egress. 35 | 36 | | Label | Description | 37 | | ----------- | ----------------------------- | 38 | | `namespace` | The egress resource namespace | 39 | | `egress` | The egress resource name | 40 | 41 | ### `coil_egress_client_pod_info` 42 | 43 | This is the client pod information. 44 | 45 | | Label | Description | 46 | | -------------------| ----------------------------- | 47 | | `namespace` | The pod resource namespace | 48 | | `pod` | The pod name | 49 | | `pod_ip` | The pod's IP address | 50 | | `interface` | The interface for the pod | 51 | | `egress` | The egress resource name | 52 | | `egress_namespace` | The egress resource namespace | 53 | 54 | ### `coil_egress_nf_conntrack_entries_limit` 55 | 56 | This is the limit of conntrack entries in the kernel. 57 | This value is from `/proc/sys/net/netfilter/nf_conntrack_max`. 58 | 59 | | Label | Description | 60 | | ----------- | ----------------------------- | 61 | | `namespace` | The egress resource namespace | 62 | | `egress` | The egress resource name | 63 | | `pod` | The pod name | 64 | 65 | 66 | ### `coil_egress_nf_conntrack_entries` 67 | 68 | This is the number of conntrack entries in the kernel. 69 | This value is from `/proc/sys/net/netfilter/nf_conntrack_count`. 70 | 71 | | Label | Description | 72 | | ----------- | ----------------------------- | 73 | | `namespace` | The egress resource namespace | 74 | | `egress` | The egress resource name | 75 | | `pod` | The pod name | 76 | 77 | ### `coil_egress_nftables_masqueraded_packets_total` 78 | 79 | This is the total number of packets masqueraded by iptables in a egress NAT pod. 80 | This value is from the result of `iptables -t nat -L POSTROUTING -vn`. 81 | 82 | | Label | Description | 83 | | ----------- | ----------------------------- | 84 | | `namespace` | The egress resource namespace | 85 | | `egress` | The egress resource name | 86 | | `pod` | The pod name | 87 | 88 | ### `coil_egress_nftables_masqueraded_bytes_total` 89 | 90 | This is the total bytes of masqueraded packets by iptables in a egress NAT pod. 91 | This value is from the result of `iptables -t nat -L POSTROUTING -vn`. 92 | 93 | | Label | Description | 94 | | ----------- | ----------------------------- | 95 | | `namespace` | The egress resource namespace | 96 | | `egress` | The egress resource name | 97 | | `pod` | The pod name | 98 | -------------------------------------------------------------------------------- /docs/cmd-coil-installer.md: -------------------------------------------------------------------------------- 1 | coil-installer 2 | ============== 3 | 4 | `coil-installer` is a program intended to be run as an init container of 5 | `coild` DaemonSet. 6 | 7 | It installs `coil` CNI binary and network configuration file into the host OS. 8 | 9 | ## Environment variables 10 | 11 | The installer references the following environment variables: 12 | 13 | | Name | Default | Description | 14 | | ------------------ | --------------------- | -------------------------------------------------- | 15 | | `CNI_CONF_NAME` | `10-coil.conflist` | The filename of the CNI configuration file. | 16 | | `CNI_ETC_DIR` | `/host/etc/cni/net.d` | Installation directory for CNI configuration file. | 17 | | `CNI_BIN_DIR` | `/host/opt/cni/bin` | Installation directory for CNI plugin. | 18 | | `COIL_PATH` | `/coil` | Path to `coil`. | 19 | | `CNI_NETCONF_FILE` | | Path to CNI configuration file. | 20 | | `CNI_NETCONF` | | CNI configuration file contents. | 21 | -------------------------------------------------------------------------------- /docs/cmd-coil-ipam-controller.md: -------------------------------------------------------------------------------- 1 | coil-ipam-controller 2 | =============== 3 | 4 | `coil-ipam-controller` is a Kubernetes controller for Coil IPAM related custom resources. 5 | It is intended to be run as a Pod in `kube-system` namespace. 6 | 7 | ## AddressPool and AddressBlock 8 | 9 | `coil-ipam-controller` has an in-memory database of address pools and 10 | address blocks to allocate address blocks quickly. 11 | 12 | ## BlockRequest 13 | 14 | `coil-ipam-controller` watches newly created block requests and carve out 15 | address blocks from the requested pool. 16 | 17 | ## Garbage collection 18 | 19 | `coil-ipam-controller` periodically checks orphaned address blocks and deletes them. 20 | 21 | ## Command-line flags 22 | 23 | ``` 24 | Flags: 25 | --cert-dir string directory to locate TLS certs for webhook (default "/certs") 26 | --gc-interval duration garbage collection interval (default 1h0m0s) 27 | --health-addr string bind address of health/readiness probes (default ":9387") 28 | -h, --help help for coil-ipam-controller 29 | --metrics-addr string bind address of metrics endpoint (default ":9386") 30 | -v, --version version for coil-ipam-controller 31 | --webhook-addr string bind address of admission webhook (default ":9443") 32 | --enable-cert-rotation enables webhook's certificate generation 33 | --enable-restart-on-cert-refresh enables pod's restart on webhook certificate refresh 34 | ``` 35 | 36 | ## Prometheus metrics 37 | 38 | ### `coil_controller_max_blocks` 39 | 40 | This is a gauge of the maximum number of allocatable address blocks of a pool. 41 | 42 | | Label | Description | 43 | | ------ | ------------- | 44 | | `pool` | The pool name | 45 | 46 | ### `coil_controller_allocated_blocks` 47 | 48 | This is a gauge of the number of currently allocated address blocks. 49 | 50 | | Label | Description | 51 | | ------ | ------------- | 52 | | `pool` | The pool name | 53 | -------------------------------------------------------------------------------- /docs/cmd-coil-router.md: -------------------------------------------------------------------------------- 1 | coil-router 2 | =========== 3 | 4 | `coil-router` is an _optional_ program to setup the kernel routing table 5 | to route Pod packets between Nodes. `coil-router` can be used only when 6 | all the nodes are in a flat layer-2 network. 7 | 8 | ## How it works 9 | 10 | Coil allocates address blocks to Nodes. Therefore, each node should receive 11 | packets to the addresses in the address blocks it owns. 12 | 13 | `coil-router` retrieves address block allocation information and configures 14 | the kernel routing table so that each address block are routed to their 15 | owning node. 16 | 17 | This behavior assumes that all the nodes are directly connected in a flat 18 | layer-2 network. 19 | 20 | ## Environment variables 21 | 22 | `coil-router` references the following environment variables: 23 | 24 | | Name | Required | Description | 25 | | ---------------- | -------- | ---------------------------------------- | 26 | | `COIL_NODE_NAME` | YES | Kubernetes node name of the running node | 27 | 28 | ## Command-line flags 29 | 30 | **CAVEAT**: `--protocol-id` value must be different from the value of `coild`. 31 | 32 | ``` 33 | Flags: 34 | --health-addr string bind address of health/readiness probes (default ":9389") 35 | -h, --help help for coil-router 36 | --metrics-addr string bind address of metrics endpoint (default ":9388") 37 | --protocol-id int route author ID (default 31) 38 | --update-interval duration interval for forced route update (default 10m0s) 39 | -v, --version version for coil-router 40 | ``` 41 | 42 | ## Prometheus metrics 43 | 44 | ### `coil_router_syncs_total` 45 | 46 | This is a counter of the total number of route synchronizations. 47 | 48 | | Label | Description | 49 | | ------ | ---------------------- | 50 | | `node` | The node resource name | 51 | 52 | ### `coil_router_routes_synced` 53 | 54 | This is a gauge of the number of routes last synchronized to the kernel. 55 | 56 | | Label | Description | 57 | | ------ | ---------------------- | 58 | | `node` | The node resource name | 59 | -------------------------------------------------------------------------------- /docs/cmd-coil.md: -------------------------------------------------------------------------------- 1 | coil 2 | ==== 3 | 4 | `coil` command is a [CNI plugin](https://github.com/containernetworking/cni/blob/spec-v0.4.0/SPEC.md#cni-plugin). 5 | 6 | It delegates all requests except for `VERSION` to `coild` through gRPC over UNIX domain socket. 7 | The default socket path is `/run/coild.sock`, but it can be changed with `socket` parameter in the network configuration as follows: 8 | 9 | ```json 10 | { 11 | "cniVersion": "0.4.0", 12 | "name": "k8s", 13 | "type": "coil", 14 | "socket": "/tmp/coild.sock" 15 | } 16 | ``` 17 | -------------------------------------------------------------------------------- /docs/cmd-coild.md: -------------------------------------------------------------------------------- 1 | coild 2 | ===== 3 | 4 | `coild` is a gRPC server running on each node. 5 | 6 | ## gRPC server 7 | 8 | `coild` listens on a UNIX domain socket and accepts requests from `coil` 9 | over gRPC protocol. The default socket path is `/run/coild.sock`. 10 | 11 | The gRPC server provides following additional features: 12 | 13 | - [gRPC Server Reflection](https://github.com/grpc/grpc-go/blob/master/Documentation/server-reflection-tutorial.md) 14 | - [gRPC metrics](https://github.com/grpc-ecosystem/go-grpc-prometheus#metrics) 15 | - Access logging 16 | 17 | ## Pod routes 18 | 19 | `coild` registers the routes to local Pods into a kernel routing table. 20 | The default routing table ID is **116**. 21 | 22 | This routing table is looked up by a routing rule inserted by `coild`. 23 | The default rule priority is **2000**. 24 | 25 | ## Route export 26 | 27 | `coild` exports address blocks owned by the running node to a kernel 28 | routing table. The default routing table ID is **119**. 29 | 30 | The routes are created in that table with a specific author (protocol) ID. 31 | The default protocol ID is **30**. 32 | 33 | ## Compatibility with Calico 34 | 35 | `coild` optionally can make veth interface names compatible with Calico. 36 | If you want to use Calico for network policy together with Coil, enable 37 | this feature with `--compat-calico` flag. 38 | 39 | Calico needs to be configured to set [`FELIX_INTERFACEPREFIX`](https://github.com/projectcalico/calico/blob/c0fe9f811ea8721007df9362d63af6697b42f6f3/reference/felix/configuration.md#bare-metal-specific-configuration) to `veth`. 40 | 41 | ## Environment variables 42 | 43 | `coild` references the following environment variables: 44 | 45 | | Name | Required | Description | 46 | | ---------------- | -------- | ---------------------------------------- | 47 | | `COIL_NODE_NAME` | YES | Kubernetes node name of the running node | 48 | 49 | ## Command-line flags 50 | 51 | ``` 52 | Flags: 53 | --compat-calico make veth name compatible with Calico 54 | --egress-port int UDP port number for egress NAT (default 5555) 55 | --enable-egress enable Egress related features (default true) 56 | --enable-ipam enable IPAM related features (default true) 57 | --export-table-id int routing table ID to which coild exports routes (default 119) 58 | --health-addr string bind address of health/readiness probes (default ":9385") 59 | -h, --help help for coild 60 | --metrics-addr string bind address of metrics endpoint (default ":9384") 61 | --pod-rule-prio int priority with which the rule for Pod table is inserted (default 2000) 62 | --pod-table-id int routing table ID to which coild registers routes for Pods (default 116) 63 | --protocol-id int route author ID (default 30) 64 | --register-from-main help migration from Coil 2.0.1 65 | --socket string UNIX domain socket path (default "/run/coild.sock") 66 | -v, --version version for coild 67 | ``` 68 | -------------------------------------------------------------------------------- /docs/img/dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cybozu-go/coil/314fc3a333147b456b0e84a6348c49edb5d94579/docs/img/dashboard.png -------------------------------------------------------------------------------- /v2/.dockerignore: -------------------------------------------------------------------------------- 1 | /bin 2 | /.vscode 3 | /config 4 | /dashboard 5 | /e2e 6 | /hack 7 | -------------------------------------------------------------------------------- /v2/.gitignore: -------------------------------------------------------------------------------- 1 | /bin 2 | /config/default/ipam/webhook_manifests_patch.yaml 3 | /config/default/egress/webhook_manifests_patch.yaml 4 | /include 5 | /testbin 6 | /tmp 7 | /work 8 | /e2e/tmp 9 | /e2e/kindnet-conf -------------------------------------------------------------------------------- /v2/.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "files.watcherExclude": { 3 | "tmp/**": true 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /v2/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM --platform=$BUILDPLATFORM ghcr.io/cybozu/golang:1.24-noble as build-env 2 | 3 | ARG TARGETARCH 4 | 5 | COPY . /workdir 6 | WORKDIR /workdir 7 | 8 | RUN make build GOARCH=${TARGETARCH} 9 | 10 | FROM --platform=$TARGETPLATFORM ghcr.io/cybozu/ubuntu:24.04 11 | 12 | # https://docs.github.com/en/packages/managing-container-images-with-github-container-registry/connecting-a-repository-to-a-container-image#connecting-a-repository-to-a-container-image-on-the-command-line 13 | LABEL org.opencontainers.image.source https://github.com/cybozu-go/coil 14 | 15 | RUN apt-get update \ 16 | && apt-get install -y --no-install-recommends netbase kmod iptables iproute2 conntrack \ 17 | && rm -rf /var/lib/apt/lists/* 18 | 19 | COPY --from=build-env /workdir/work /usr/local/coil 20 | 21 | ENV PATH /usr/local/coil:$PATH 22 | -------------------------------------------------------------------------------- /v2/PROJECT: -------------------------------------------------------------------------------- 1 | domain: cybozu.com 2 | layout: 3 | - go.kubebuilder.io/v3 4 | projectName: coil 5 | repo: github.com/cybozu-go/coil/v2 6 | resources: 7 | - api: 8 | crdVersion: v1 9 | controller: true 10 | domain: cybozu.com 11 | group: coil 12 | kind: AddressPool 13 | path: github.com/cybozu-go/coil/v2/api/v2 14 | version: v2 15 | webhooks: 16 | defaulting: true 17 | validation: true 18 | webhookVersion: v1 19 | - api: 20 | crdVersion: v1 21 | controller: true 22 | domain: cybozu.com 23 | group: coil 24 | kind: AddressBlock 25 | path: github.com/cybozu-go/coil/v2/api/v2 26 | version: v2 27 | - api: 28 | crdVersion: v1 29 | controller: true 30 | domain: cybozu.com 31 | group: coil 32 | kind: BlockRequest 33 | path: github.com/cybozu-go/coil/v2/api/v2 34 | version: v2 35 | - api: 36 | crdVersion: v1 37 | namespaced: true 38 | controller: true 39 | domain: cybozu.com 40 | group: coil 41 | kind: Egress 42 | path: github.com/cybozu-go/coil/v2/api/v2 43 | version: v2 44 | webhooks: 45 | defaulting: true 46 | validation: true 47 | webhookVersion: v1 48 | version: "3" 49 | -------------------------------------------------------------------------------- /v2/api/v2/addressblock_types.go: -------------------------------------------------------------------------------- 1 | package v2 2 | 3 | import ( 4 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 5 | ) 6 | 7 | // EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! 8 | // NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. 9 | 10 | // +kubebuilder:object:root=true 11 | // +kubebuilder:resource:scope=Cluster 12 | // +kubebuilder:printcolumn:JSONPath=`.metadata.labels['coil\.cybozu\.com/node']`,name=Node,type=string 13 | // +kubebuilder:printcolumn:JSONPath=`.metadata.labels['coil\.cybozu\.com/pool']`,name=Pool,type=string 14 | // +kubebuilder:printcolumn:JSONPath=.ipv4,name=IPv4,type=string 15 | // +kubebuilder:printcolumn:JSONPath=.ipv6,name=IPv6,type=string 16 | 17 | // AddressBlock is the Schema for the addressblocks API 18 | // 19 | // The ownerReferences field contains the AddressPool where the block is carved from. 20 | type AddressBlock struct { 21 | metav1.TypeMeta `json:",inline"` 22 | metav1.ObjectMeta `json:"metadata,omitempty"` 23 | 24 | // Index indicates the index of this block from the origin pool 25 | // +kubebuilder:validation:Minimum=0 26 | Index int32 `json:"index"` 27 | 28 | // IPv4 is an IPv4 subnet address 29 | IPv4 *string `json:"ipv4,omitempty"` 30 | 31 | // IPv6 is an IPv6 subnet address 32 | IPv6 *string `json:"ipv6,omitempty"` 33 | } 34 | 35 | // +kubebuilder:object:root=true 36 | 37 | // AddressBlockList contains a list of AddressBlock 38 | type AddressBlockList struct { 39 | metav1.TypeMeta `json:",inline"` 40 | metav1.ListMeta `json:"metadata,omitempty"` 41 | Items []AddressBlock `json:"items"` 42 | } 43 | 44 | func init() { 45 | SchemeBuilder.Register(&AddressBlock{}, &AddressBlockList{}) 46 | } 47 | -------------------------------------------------------------------------------- /v2/api/v2/addresspool_webhook.go: -------------------------------------------------------------------------------- 1 | package v2 2 | 3 | import ( 4 | "github.com/cybozu-go/coil/v2/pkg/constants" 5 | apierrors "k8s.io/apimachinery/pkg/api/errors" 6 | "k8s.io/apimachinery/pkg/runtime" 7 | "k8s.io/apimachinery/pkg/runtime/schema" 8 | ctrl "sigs.k8s.io/controller-runtime" 9 | "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" 10 | "sigs.k8s.io/controller-runtime/pkg/webhook" 11 | "sigs.k8s.io/controller-runtime/pkg/webhook/admission" 12 | ) 13 | 14 | // SetupWebhookWithManager registers webhooks for AddressPool 15 | func (r *AddressPool) SetupWebhookWithManager(mgr ctrl.Manager) error { 16 | return ctrl.NewWebhookManagedBy(mgr). 17 | For(r). 18 | Complete() 19 | } 20 | 21 | // EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! 22 | 23 | //+kubebuilder:webhook:path=/mutate-coil-cybozu-com-v2-addresspool,mutating=true,failurePolicy=fail,sideEffects=None,groups=coil.cybozu.com,resources=addresspools,verbs=create,versions=v2,name=maddresspool.kb.io,admissionReviewVersions={v1,v1beta1} 24 | 25 | var _ webhook.Defaulter = &AddressPool{} 26 | 27 | // Default implements webhook.Defaulter so a webhook will be registered for the type 28 | func (r *AddressPool) Default() { 29 | controllerutil.AddFinalizer(r, constants.FinCoil) 30 | } 31 | 32 | // +kubebuilder:webhook:path=/validate-coil-cybozu-com-v2-addresspool,mutating=false,failurePolicy=fail,sideEffects=None,groups=coil.cybozu.com,resources=addresspools,verbs=create;update,versions=v2,name=vaddresspool.kb.io,admissionReviewVersions={v1,v1beta1} 33 | 34 | var _ webhook.Validator = &AddressPool{} 35 | 36 | // ValidateCreate implements webhook.Validator so a webhook will be registered for the type 37 | func (r *AddressPool) ValidateCreate() (warnings admission.Warnings, err error) { 38 | errs := r.Spec.validate() 39 | if len(errs) == 0 { 40 | return nil, nil 41 | } 42 | 43 | return nil, apierrors.NewInvalid(schema.GroupKind{Group: GroupVersion.Group, Kind: "AddressPool"}, r.Name, errs) 44 | } 45 | 46 | // ValidateUpdate implements webhook.Validator so a webhook will be registered for the type 47 | func (r *AddressPool) ValidateUpdate(old runtime.Object) (warnings admission.Warnings, err error) { 48 | errs := r.Spec.validateUpdate(old.(*AddressPool).Spec) 49 | if len(errs) == 0 { 50 | return nil, nil 51 | } 52 | 53 | return nil, apierrors.NewInvalid(schema.GroupKind{Group: GroupVersion.Group, Kind: "AddressPool"}, r.Name, errs) 54 | } 55 | 56 | // ValidateDelete implements webhook.Validator so a webhook will be registered for the type 57 | func (r *AddressPool) ValidateDelete() (warnings admission.Warnings, err error) { 58 | return nil, nil 59 | } 60 | -------------------------------------------------------------------------------- /v2/api/v2/blockrequest_types_test.go: -------------------------------------------------------------------------------- 1 | package v2 2 | 3 | import ( 4 | "testing" 5 | 6 | corev1 "k8s.io/api/core/v1" 7 | ) 8 | 9 | func TestBlockRequestGetResult(t *testing.T) { 10 | t.Parallel() 11 | 12 | cases := []struct { 13 | name string 14 | status BlockRequestStatus 15 | expectErr bool 16 | blockName string 17 | }{ 18 | { 19 | "completed", 20 | BlockRequestStatus{ 21 | AddressBlockName: "foo", 22 | Conditions: []BlockRequestCondition{ 23 | {Type: BlockRequestComplete, Status: corev1.ConditionTrue}, 24 | }, 25 | }, 26 | false, 27 | "foo", 28 | }, 29 | { 30 | "completed-false-failure", 31 | BlockRequestStatus{ 32 | AddressBlockName: "foo", 33 | Conditions: []BlockRequestCondition{ 34 | {Type: BlockRequestFailed, Status: corev1.ConditionFalse}, 35 | {Type: BlockRequestComplete, Status: corev1.ConditionTrue}, 36 | }, 37 | }, 38 | false, 39 | "foo", 40 | }, 41 | { 42 | "completed-failure", 43 | BlockRequestStatus{ 44 | AddressBlockName: "foo", 45 | Conditions: []BlockRequestCondition{ 46 | {Type: BlockRequestComplete, Status: corev1.ConditionTrue}, 47 | {Type: BlockRequestFailed, Status: corev1.ConditionTrue}, 48 | }, 49 | }, 50 | true, 51 | "foo", 52 | }, 53 | { 54 | "not-completed", 55 | BlockRequestStatus{ 56 | AddressBlockName: "foo", 57 | Conditions: []BlockRequestCondition{ 58 | {Type: BlockRequestComplete, Status: corev1.ConditionFalse}, 59 | }, 60 | }, 61 | true, 62 | "foo", 63 | }, 64 | { 65 | "no-conditions", 66 | BlockRequestStatus{ 67 | AddressBlockName: "foo", 68 | }, 69 | true, 70 | "foo", 71 | }, 72 | { 73 | "failed", 74 | BlockRequestStatus{ 75 | AddressBlockName: "foo", 76 | Conditions: []BlockRequestCondition{ 77 | {Type: BlockRequestFailed, Status: corev1.ConditionTrue}, 78 | }, 79 | }, 80 | true, 81 | "foo", 82 | }, 83 | } 84 | 85 | for _, tc := range cases { 86 | t.Run(tc.name, func(t *testing.T) { 87 | actual, err := tc.status.getResult() 88 | if err != nil { 89 | if !tc.expectErr { 90 | t.Error("unexpected error", err) 91 | } 92 | return 93 | } 94 | if actual != tc.blockName { 95 | t.Error("unexpected block name", actual) 96 | } 97 | }) 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /v2/api/v2/egress_webhook.go: -------------------------------------------------------------------------------- 1 | package v2 2 | 3 | import ( 4 | corev1 "k8s.io/api/core/v1" 5 | apierrors "k8s.io/apimachinery/pkg/api/errors" 6 | "k8s.io/apimachinery/pkg/runtime" 7 | "k8s.io/apimachinery/pkg/runtime/schema" 8 | ctrl "sigs.k8s.io/controller-runtime" 9 | "sigs.k8s.io/controller-runtime/pkg/webhook" 10 | "sigs.k8s.io/controller-runtime/pkg/webhook/admission" 11 | ) 12 | 13 | // SetupWebhookWithManager setups the webhook for Egress 14 | func (r *Egress) SetupWebhookWithManager(mgr ctrl.Manager) error { 15 | return ctrl.NewWebhookManagedBy(mgr). 16 | For(r). 17 | Complete() 18 | } 19 | 20 | // EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! 21 | 22 | // +kubebuilder:webhook:path=/mutate-coil-cybozu-com-v2-egress,mutating=true,failurePolicy=fail,sideEffects=None,groups=coil.cybozu.com,resources=egresses,verbs=create,versions=v2,name=megress.kb.io,admissionReviewVersions={v1,v1beta1} 23 | 24 | var _ webhook.Defaulter = &Egress{} 25 | 26 | // Default implements webhook.Defaulter so a webhook will be registered for the type 27 | func (r *Egress) Default() { 28 | tmpl := r.Spec.Template 29 | if tmpl == nil { 30 | return 31 | } 32 | 33 | if len(tmpl.Spec.Containers) == 0 { 34 | tmpl.Spec.Containers = []corev1.Container{ 35 | { 36 | Name: "egress", 37 | }, 38 | } 39 | } 40 | } 41 | 42 | // +kubebuilder:webhook:path=/validate-coil-cybozu-com-v2-egress,mutating=false,failurePolicy=fail,sideEffects=None,groups=coil.cybozu.com,resources=egresses,verbs=create;update,versions=v2,name=vegress.kb.io,admissionReviewVersions={v1,v1beta1} 43 | 44 | var _ webhook.Validator = &Egress{} 45 | 46 | // ValidateCreate implements webhook.Validator so a webhook will be registered for the type 47 | func (r *Egress) ValidateCreate() (warnings admission.Warnings, err error) { 48 | errs := r.Spec.validate() 49 | if len(errs) == 0 { 50 | return nil, nil 51 | } 52 | 53 | return nil, apierrors.NewInvalid(schema.GroupKind{Group: GroupVersion.Group, Kind: "Egress"}, r.Name, errs) 54 | } 55 | 56 | // ValidateUpdate implements webhook.Validator so a webhook will be registered for the type 57 | func (r *Egress) ValidateUpdate(old runtime.Object) (warnings admission.Warnings, err error) { 58 | errs := r.Spec.validateUpdate() 59 | if len(errs) == 0 { 60 | return nil, nil 61 | } 62 | 63 | return nil, apierrors.NewInvalid(schema.GroupKind{Group: GroupVersion.Group, Kind: "Egress"}, r.Name, errs) 64 | } 65 | 66 | // ValidateDelete implements webhook.Validator so a webhook will be registered for the type 67 | func (r *Egress) ValidateDelete() (warnings admission.Warnings, err error) { 68 | return nil, nil 69 | } 70 | -------------------------------------------------------------------------------- /v2/api/v2/groupversion_info.go: -------------------------------------------------------------------------------- 1 | // Package v2 contains API Schema definitions for the coil v2 API group 2 | // +kubebuilder:object:generate=true 3 | // +groupName=coil.cybozu.com 4 | package v2 5 | 6 | import ( 7 | "k8s.io/apimachinery/pkg/runtime/schema" 8 | "sigs.k8s.io/controller-runtime/pkg/scheme" 9 | ) 10 | 11 | var ( 12 | // GroupVersion is group version used to register these objects 13 | GroupVersion = schema.GroupVersion{Group: "coil.cybozu.com", Version: "v2"} 14 | 15 | // SchemeBuilder is used to add go types to the GroupVersionKind scheme 16 | SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} 17 | 18 | // AddToScheme adds the types in this group-version to the given scheme. 19 | AddToScheme = SchemeBuilder.AddToScheme 20 | ) 21 | -------------------------------------------------------------------------------- /v2/api/v2/suite_test.go: -------------------------------------------------------------------------------- 1 | package v2 2 | 3 | import ( 4 | "context" 5 | "crypto/tls" 6 | "fmt" 7 | "net" 8 | "path/filepath" 9 | "testing" 10 | "time" 11 | 12 | . "github.com/onsi/ginkgo/v2" 13 | . "github.com/onsi/gomega" 14 | admissionv1 "k8s.io/api/admission/v1" 15 | clientgoscheme "k8s.io/client-go/kubernetes/scheme" 16 | 17 | //+kubebuilder:scaffold:imports 18 | "k8s.io/apimachinery/pkg/runtime" 19 | "k8s.io/client-go/rest" 20 | ctrl "sigs.k8s.io/controller-runtime" 21 | "sigs.k8s.io/controller-runtime/pkg/client" 22 | "sigs.k8s.io/controller-runtime/pkg/envtest" 23 | logf "sigs.k8s.io/controller-runtime/pkg/log" 24 | "sigs.k8s.io/controller-runtime/pkg/log/zap" 25 | "sigs.k8s.io/controller-runtime/pkg/metrics/server" 26 | "sigs.k8s.io/controller-runtime/pkg/webhook" 27 | ) 28 | 29 | // These tests use Ginkgo (BDD-style Go testing framework). Refer to 30 | // http://onsi.github.io/ginkgo/ to learn more about Ginkgo. 31 | 32 | var cfg *rest.Config 33 | var k8sClient client.Client 34 | var testEnv *envtest.Environment 35 | var ctx context.Context 36 | var cancel context.CancelFunc 37 | 38 | func TestAPIs(t *testing.T) { 39 | RegisterFailHandler(Fail) 40 | 41 | RunSpecs(t, "Coil v2 Suite") 42 | } 43 | 44 | var _ = BeforeSuite(func() { 45 | logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) 46 | 47 | ctx, cancel = context.WithCancel(context.TODO()) 48 | 49 | By("bootstrapping test environment") 50 | testEnv = &envtest.Environment{ 51 | CRDDirectoryPaths: []string{filepath.Join("..", "..", "config", "crd", "bases")}, 52 | WebhookInstallOptions: envtest.WebhookInstallOptions{ 53 | Paths: []string{ 54 | filepath.Join("..", "..", "config", "webhook", "ipam"), 55 | filepath.Join("..", "..", "config", "webhook", "egress"), 56 | }, 57 | }, 58 | ErrorIfCRDPathMissing: true, 59 | } 60 | 61 | var err error 62 | cfg, err = testEnv.Start() 63 | Expect(err).NotTo(HaveOccurred()) 64 | Expect(cfg).NotTo(BeNil()) 65 | 66 | scheme := runtime.NewScheme() 67 | err = AddToScheme(scheme) 68 | Expect(err).NotTo(HaveOccurred()) 69 | err = clientgoscheme.AddToScheme(scheme) 70 | Expect(err).NotTo(HaveOccurred()) 71 | err = admissionv1.AddToScheme(scheme) 72 | Expect(err).NotTo(HaveOccurred()) 73 | 74 | // +kubebuilder:scaffold:scheme 75 | 76 | k8sClient, err = client.New(cfg, client.Options{Scheme: scheme}) 77 | Expect(err).NotTo(HaveOccurred()) 78 | Expect(k8sClient).NotTo(BeNil()) 79 | 80 | // start webhook server using Manager 81 | webhookInstallOptions := &testEnv.WebhookInstallOptions 82 | mgr, err := ctrl.NewManager(cfg, ctrl.Options{ 83 | Scheme: scheme, 84 | WebhookServer: webhook.NewServer(webhook.Options{ 85 | Host: webhookInstallOptions.LocalServingHost, 86 | Port: webhookInstallOptions.LocalServingPort, 87 | CertDir: webhookInstallOptions.LocalServingCertDir, 88 | }), 89 | LeaderElection: false, 90 | Metrics: server.Options{BindAddress: "0"}, 91 | }) 92 | Expect(err).NotTo(HaveOccurred()) 93 | 94 | err = (&AddressPool{}).SetupWebhookWithManager(mgr) 95 | Expect(err).NotTo(HaveOccurred()) 96 | err = (&Egress{}).SetupWebhookWithManager(mgr) 97 | Expect(err).NotTo(HaveOccurred()) 98 | 99 | //+kubebuilder:scaffold:webhook 100 | 101 | go func() { 102 | err = mgr.Start(ctx) 103 | if err != nil { 104 | Expect(err).NotTo(HaveOccurred()) 105 | } 106 | }() 107 | 108 | // wait for the webhook server to get ready 109 | dialer := &net.Dialer{Timeout: time.Second} 110 | addrPort := fmt.Sprintf("%s:%d", webhookInstallOptions.LocalServingHost, webhookInstallOptions.LocalServingPort) 111 | Eventually(func() error { 112 | conn, err := tls.DialWithDialer(dialer, "tcp", addrPort, &tls.Config{InsecureSkipVerify: true}) 113 | if err != nil { 114 | return err 115 | } 116 | conn.Close() 117 | return nil 118 | }).Should(Succeed()) 119 | 120 | }) 121 | 122 | var _ = AfterSuite(func() { 123 | cancel() 124 | By("tearing down the test environment") 125 | err := testEnv.Stop() 126 | Expect(err).NotTo(HaveOccurred()) 127 | }) 128 | -------------------------------------------------------------------------------- /v2/cmd/coil-egress-controller/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "github.com/cybozu-go/coil/v2/cmd/coil-egress-controller/sub" 4 | 5 | func main() { 6 | sub.Execute() 7 | } 8 | -------------------------------------------------------------------------------- /v2/cmd/coil-egress-controller/sub/root.go: -------------------------------------------------------------------------------- 1 | package sub 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "os" 7 | 8 | v2 "github.com/cybozu-go/coil/v2" 9 | "github.com/cybozu-go/coil/v2/pkg/constants" 10 | "github.com/spf13/cobra" 11 | "k8s.io/klog/v2" 12 | "sigs.k8s.io/controller-runtime/pkg/log/zap" 13 | ) 14 | 15 | var config struct { 16 | metricsAddr string 17 | healthAddr string 18 | webhookAddr string 19 | certDir string 20 | egressPort int32 21 | zapOpts zap.Options 22 | 23 | enableCertRotation bool 24 | enableRestartOnCertRefresh bool 25 | } 26 | 27 | var rootCmd = &cobra.Command{ 28 | Use: "coil-egress-controller", 29 | Short: "controller for coil egress related custom resources", 30 | Long: `coil-egress-controller is a Kubernetes controller for coil egress related custom resources.`, 31 | Version: v2.Version(), 32 | RunE: func(cmd *cobra.Command, _ []string) error { 33 | cmd.SilenceUsage = true 34 | return subMain() 35 | }, 36 | } 37 | 38 | // Execute adds all child commands to the root command and sets flags appropriately. 39 | // This is called by main.main(). It only needs to happen once to the rootCmd. 40 | func Execute() { 41 | if err := rootCmd.Execute(); err != nil { 42 | fmt.Println(err) 43 | os.Exit(1) 44 | } 45 | } 46 | 47 | func init() { 48 | pf := rootCmd.PersistentFlags() 49 | pf.StringVar(&config.metricsAddr, "metrics-addr", ":9396", "bind address of metrics endpoint") 50 | pf.StringVar(&config.healthAddr, "health-addr", ":9397", "bind address of health/readiness probes") 51 | pf.StringVar(&config.webhookAddr, "webhook-addr", ":9444", "bind address of admission webhook") 52 | pf.StringVar(&config.certDir, "cert-dir", "/certs", "directory to locate TLS certs for webhook") 53 | pf.Int32Var(&config.egressPort, "egress-port", 5555, "UDP port number used by coil-egress") 54 | pf.BoolVar(&config.enableCertRotation, "enable-cert-rotation", constants.DefaultEnableCertRotation, "enables webhook's certificate generation") 55 | pf.BoolVar(&config.enableRestartOnCertRefresh, "enable-restart-on-cert-refresh", constants.DefaultEnableRestartOnCertRefresh, "enables pod's restart on webhook certificate refresh") 56 | 57 | goflags := flag.NewFlagSet("klog", flag.ExitOnError) 58 | klog.InitFlags(goflags) 59 | config.zapOpts.BindFlags(goflags) 60 | 61 | pf.AddGoFlagSet(goflags) 62 | } 63 | -------------------------------------------------------------------------------- /v2/cmd/coil-egress/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "github.com/cybozu-go/coil/v2/cmd/coil-egress/sub" 4 | 5 | func main() { 6 | sub.Execute() 7 | } 8 | -------------------------------------------------------------------------------- /v2/cmd/coil-egress/sub/root.go: -------------------------------------------------------------------------------- 1 | package sub 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "os" 7 | 8 | v2 "github.com/cybozu-go/coil/v2" 9 | "github.com/spf13/cobra" 10 | "k8s.io/klog/v2" 11 | "sigs.k8s.io/controller-runtime/pkg/log/zap" 12 | ) 13 | 14 | var config struct { 15 | metricsAddr string 16 | healthAddr string 17 | port int 18 | enableSportAuto bool 19 | zapOpts zap.Options 20 | } 21 | 22 | var rootCmd = &cobra.Command{ 23 | Use: "coil-egress", 24 | Short: "manage foo-over-udp tunnels in egress pods", 25 | Long: `coil-egress manages Foo-over-UDP tunnels in pods created by Egress.`, 26 | Version: v2.Version(), 27 | RunE: func(cmd *cobra.Command, _ []string) error { 28 | cmd.SilenceUsage = true 29 | return subMain() 30 | }, 31 | } 32 | 33 | // Execute adds all child commands to the root command and sets flags appropriately. 34 | // This is called by main.main(). It only needs to happen once to the rootCmd. 35 | func Execute() { 36 | if err := rootCmd.Execute(); err != nil { 37 | fmt.Println(err) 38 | os.Exit(1) 39 | } 40 | } 41 | 42 | func init() { 43 | pf := rootCmd.PersistentFlags() 44 | pf.StringVar(&config.metricsAddr, "metrics-addr", ":8080", "bind address of metrics endpoint") 45 | pf.StringVar(&config.healthAddr, "health-addr", ":8081", "bind address of health/readiness probes") 46 | pf.IntVar(&config.port, "fou-port", 5555, "port number for foo-over-udp tunnels") 47 | pf.BoolVar(&config.enableSportAuto, "enable-sport-auto", false, "enable automatic source port assignment") 48 | 49 | goflags := flag.NewFlagSet("klog", flag.ExitOnError) 50 | klog.InitFlags(goflags) 51 | config.zapOpts.BindFlags(goflags) 52 | 53 | pf.AddGoFlagSet(goflags) 54 | } 55 | -------------------------------------------------------------------------------- /v2/cmd/coil-installer/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "github.com/cybozu-go/coil/v2/cmd/coil-installer/sub" 4 | 5 | func main() { 6 | sub.Execute() 7 | } 8 | -------------------------------------------------------------------------------- /v2/cmd/coil-installer/sub/install.go: -------------------------------------------------------------------------------- 1 | package sub 2 | 3 | import ( 4 | "io" 5 | "os" 6 | "path/filepath" 7 | "strings" 8 | ) 9 | 10 | func installCniConf(cniConfName, cniEtcDir, cniNetConf, cniNetConfFile string) error { 11 | data := []byte(cniNetConf) 12 | if cniNetConf == "" { 13 | bData, err := os.ReadFile(cniNetConfFile) 14 | if err != nil { 15 | return err 16 | } 17 | data = bData 18 | } 19 | 20 | err := os.MkdirAll(cniEtcDir, 0755) 21 | if err != nil { 22 | return err 23 | } 24 | 25 | files, err := os.ReadDir(cniEtcDir) 26 | if err != nil { 27 | return err 28 | } 29 | for _, fi := range files { 30 | if fi.IsDir() { 31 | continue 32 | } 33 | if strings.Contains(fi.Name(), "conflist") { 34 | err := os.Remove(filepath.Join(cniEtcDir, fi.Name())) 35 | if err != nil { 36 | return err 37 | } 38 | } 39 | } 40 | 41 | f, err := os.Create(filepath.Join(cniEtcDir, cniConfName)) 42 | if err != nil { 43 | return err 44 | } 45 | defer f.Close() 46 | 47 | err = f.Chmod(0644) 48 | if err != nil { 49 | return err 50 | } 51 | 52 | _, err = f.Write(data) 53 | if err != nil { 54 | return err 55 | } 56 | 57 | return f.Sync() 58 | } 59 | 60 | func installCoil(coilPath, cniBinDir string) error { 61 | f, err := os.Open(coilPath) 62 | if err != nil { 63 | return err 64 | } 65 | defer f.Close() 66 | 67 | err = os.MkdirAll(cniBinDir, 0755) 68 | if err != nil { 69 | return err 70 | } 71 | 72 | g, err := os.CreateTemp(cniBinDir, ".tmp") 73 | if err != nil { 74 | return err 75 | } 76 | defer func() { 77 | g.Close() 78 | os.Remove(g.Name()) 79 | }() 80 | 81 | _, err = io.Copy(g, f) 82 | if err != nil { 83 | return err 84 | } 85 | 86 | err = g.Chmod(0755) 87 | if err != nil { 88 | return err 89 | } 90 | 91 | err = g.Sync() 92 | if err != nil { 93 | return err 94 | } 95 | 96 | return os.Rename(g.Name(), filepath.Join(cniBinDir, "coil")) 97 | } 98 | -------------------------------------------------------------------------------- /v2/cmd/coil-installer/sub/root.go: -------------------------------------------------------------------------------- 1 | package sub 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "time" 7 | 8 | v2 "github.com/cybozu-go/coil/v2" 9 | "github.com/spf13/cobra" 10 | "github.com/spf13/viper" 11 | ) 12 | 13 | const ( 14 | defaultCniConfName = "10-coil.conflist" 15 | defaultCniEtcDir = "/host/etc/cni/net.d" 16 | defaultCniBinDir = "/host/opt/cni/bin" 17 | defaultCoilPath = "/usr/local/coil/coil" 18 | ) 19 | 20 | var rootCmd = &cobra.Command{ 21 | Use: "coil-installer", 22 | Short: "install coil CNI binary and configuration files", 23 | Long: `coil-installer setup coil on each node by installing CNI binary and config files.`, 24 | Version: v2.Version(), 25 | RunE: func(cmd *cobra.Command, _ []string) error { 26 | cniConfName := viper.GetString("CNI_CONF_NAME") 27 | cniEtcDir := viper.GetString("CNI_ETC_DIR") 28 | cniBinDir := viper.GetString("CNI_BIN_DIR") 29 | coilPath := viper.GetString("COIL_PATH") 30 | cniNetConf := viper.GetString("CNI_NETCONF") 31 | cniNetConfFile := viper.GetString("CNI_NETCONF_FILE") 32 | 33 | err := installCniConf(cniConfName, cniEtcDir, cniNetConf, cniNetConfFile) 34 | if err != nil { 35 | return err 36 | } 37 | 38 | err = installCoil(coilPath, cniBinDir) 39 | if err != nil { 40 | return err 41 | } 42 | 43 | // Because kubelet scans /etc/cni/net.d for CNI config files every 5 seconds, 44 | // the installer need to sleep at least 5 seconds before finish. 45 | // ref: https://github.com/kubernetes/kubernetes/blob/3d9c6eb9e6e1759683d2c6cda726aad8a0e85604/pkg/kubelet/kubelet.go#L1416 46 | time.Sleep(6 * time.Second) 47 | return nil 48 | }, 49 | } 50 | 51 | // Execute adds all child commands to the root command and sets flags appropriately. 52 | // This is called by main.main(). It only needs to happen once to the rootCmd. 53 | func Execute() { 54 | if err := rootCmd.Execute(); err != nil { 55 | fmt.Println(err) 56 | os.Exit(1) 57 | } 58 | } 59 | 60 | func init() { 61 | cobra.OnInitialize(initConfig) 62 | } 63 | 64 | // initConfig reads in config file and ENV variables if set. 65 | func initConfig() { 66 | viper.BindEnv("CNI_CONF_NAME") 67 | viper.BindEnv("CNI_ETC_DIR") 68 | viper.BindEnv("CNI_BIN_DIR") 69 | viper.BindEnv("COIL_PATH") 70 | viper.BindEnv("CNI_NETCONF_FILE") 71 | viper.BindEnv("CNI_NETCONF") 72 | viper.BindEnv("COIL_BOOT_TAINT") 73 | 74 | viper.SetDefault("CNI_CONF_NAME", defaultCniConfName) 75 | viper.SetDefault("CNI_ETC_DIR", defaultCniEtcDir) 76 | viper.SetDefault("CNI_BIN_DIR", defaultCniBinDir) 77 | viper.SetDefault("COIL_PATH", defaultCoilPath) 78 | } 79 | -------------------------------------------------------------------------------- /v2/cmd/coil-ipam-controller/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "github.com/cybozu-go/coil/v2/cmd/coil-ipam-controller/sub" 4 | 5 | func main() { 6 | sub.Execute() 7 | } 8 | -------------------------------------------------------------------------------- /v2/cmd/coil-ipam-controller/sub/root.go: -------------------------------------------------------------------------------- 1 | package sub 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "os" 7 | "time" 8 | 9 | v2 "github.com/cybozu-go/coil/v2" 10 | "github.com/cybozu-go/coil/v2/pkg/constants" 11 | "github.com/spf13/cobra" 12 | "k8s.io/klog/v2" 13 | "sigs.k8s.io/controller-runtime/pkg/log/zap" 14 | ) 15 | 16 | var config struct { 17 | metricsAddr string 18 | healthAddr string 19 | webhookAddr string 20 | certDir string 21 | gcInterval time.Duration 22 | zapOpts zap.Options 23 | 24 | enableCertRotation bool 25 | enableRestartOnCertRefresh bool 26 | } 27 | 28 | var rootCmd = &cobra.Command{ 29 | Use: "coil-ipam-controller", 30 | Short: "controller for coil ipam related custom resources", 31 | Long: `coil-ipam-controller is a Kubernetes controller for coil ipam related custom resources.`, 32 | Version: v2.Version(), 33 | RunE: func(cmd *cobra.Command, _ []string) error { 34 | cmd.SilenceUsage = true 35 | return subMain() 36 | }, 37 | } 38 | 39 | // Execute adds all child commands to the root command and sets flags appropriately. 40 | // This is called by main.main(). It only needs to happen once to the rootCmd. 41 | func Execute() { 42 | if err := rootCmd.Execute(); err != nil { 43 | fmt.Println(err) 44 | os.Exit(1) 45 | } 46 | } 47 | 48 | func init() { 49 | pf := rootCmd.PersistentFlags() 50 | pf.StringVar(&config.metricsAddr, "metrics-addr", ":9386", "bind address of metrics endpoint") 51 | pf.StringVar(&config.healthAddr, "health-addr", ":9387", "bind address of health/readiness probes") 52 | pf.StringVar(&config.webhookAddr, "webhook-addr", ":9443", "bind address of admission webhook") 53 | pf.StringVar(&config.certDir, "cert-dir", "/certs", "directory to locate TLS certs for webhook") 54 | pf.DurationVar(&config.gcInterval, "gc-interval", 1*time.Hour, "garbage collection interval") 55 | pf.BoolVar(&config.enableCertRotation, "enable-cert-rotation", constants.DefaultEnableCertRotation, "enables webhook's certificate generation") 56 | pf.BoolVar(&config.enableRestartOnCertRefresh, "enable-restart-on-cert-refresh", constants.DefaultEnableRestartOnCertRefresh, "enables pod's restart on webhook certificate refresh") 57 | 58 | goflags := flag.NewFlagSet("klog", flag.ExitOnError) 59 | klog.InitFlags(goflags) 60 | config.zapOpts.BindFlags(goflags) 61 | 62 | pf.AddGoFlagSet(goflags) 63 | } 64 | -------------------------------------------------------------------------------- /v2/cmd/coil-router/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "github.com/cybozu-go/coil/v2/cmd/coil-router/sub" 4 | 5 | func main() { 6 | sub.Execute() 7 | } 8 | -------------------------------------------------------------------------------- /v2/cmd/coil-router/sub/root.go: -------------------------------------------------------------------------------- 1 | package sub 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "os" 7 | "time" 8 | 9 | v2 "github.com/cybozu-go/coil/v2" 10 | "github.com/spf13/cobra" 11 | "k8s.io/klog/v2" 12 | "sigs.k8s.io/controller-runtime/pkg/log/zap" 13 | ) 14 | 15 | var config struct { 16 | metricsAddr string 17 | healthAddr string 18 | protocolId int 19 | updateInterval time.Duration 20 | zapOpts zap.Options 21 | } 22 | 23 | var rootCmd = &cobra.Command{ 24 | Use: "coil-router", 25 | Short: "a simple routing program for Coil", 26 | Long: `coil-router programs Linux kernel routing table to route 27 | Pod packets between nodes. 28 | 29 | coil-router does not speak any routing protocol such as BGP. 30 | Instead, it directly insert routes corresponding to AddressBlocks 31 | owned by other Nodes. This means that coil-router can be used 32 | only for clusters where all the nodes are in a flat L2 network.`, 33 | Version: v2.Version(), 34 | RunE: func(cmd *cobra.Command, _ []string) error { 35 | cmd.SilenceUsage = true 36 | return subMain() 37 | }, 38 | } 39 | 40 | // Execute adds all child commands to the root command and sets flags appropriately. 41 | // This is called by main.main(). It only needs to happen once to the rootCmd. 42 | func Execute() { 43 | if err := rootCmd.Execute(); err != nil { 44 | fmt.Println(err) 45 | os.Exit(1) 46 | } 47 | } 48 | 49 | func init() { 50 | pf := rootCmd.PersistentFlags() 51 | pf.StringVar(&config.metricsAddr, "metrics-addr", ":9388", "bind address of metrics endpoint") 52 | pf.StringVar(&config.healthAddr, "health-addr", ":9389", "bind address of health/readiness probes") 53 | pf.IntVar(&config.protocolId, "protocol-id", 31, "route author ID") 54 | pf.DurationVar(&config.updateInterval, "update-interval", 10*time.Minute, "interval for forced route update") 55 | 56 | goflags := flag.NewFlagSet("klog", flag.ExitOnError) 57 | klog.InitFlags(goflags) 58 | config.zapOpts.BindFlags(goflags) 59 | 60 | pf.AddGoFlagSet(goflags) 61 | } 62 | -------------------------------------------------------------------------------- /v2/cmd/coil-router/sub/run.go: -------------------------------------------------------------------------------- 1 | package sub 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "os" 7 | "time" 8 | 9 | v2 "github.com/cybozu-go/coil/v2" 10 | coilv2 "github.com/cybozu-go/coil/v2/api/v2" 11 | "github.com/cybozu-go/coil/v2/controllers" 12 | "github.com/cybozu-go/coil/v2/pkg/constants" 13 | "github.com/cybozu-go/coil/v2/pkg/nodenet" 14 | "github.com/cybozu-go/coil/v2/runners" 15 | "k8s.io/apimachinery/pkg/runtime" 16 | utilruntime "k8s.io/apimachinery/pkg/util/runtime" 17 | clientgoscheme "k8s.io/client-go/kubernetes/scheme" 18 | ctrl "sigs.k8s.io/controller-runtime" 19 | "sigs.k8s.io/controller-runtime/pkg/healthz" 20 | "sigs.k8s.io/controller-runtime/pkg/log/zap" 21 | metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" 22 | ) 23 | 24 | const ( 25 | gracefulTimeout = 5 * time.Second 26 | ) 27 | 28 | var ( 29 | scheme = runtime.NewScheme() 30 | setupLog = ctrl.Log.WithName("setup") 31 | ) 32 | 33 | func init() { 34 | utilruntime.Must(clientgoscheme.AddToScheme(scheme)) 35 | utilruntime.Must(coilv2.AddToScheme(scheme)) 36 | 37 | // +kubebuilder:scaffold:scheme 38 | } 39 | 40 | func subMain() error { 41 | ctrl.SetLogger(zap.New(zap.UseFlagOptions(&config.zapOpts))) 42 | 43 | nodeName := os.Getenv(constants.EnvNode) 44 | if nodeName == "" { 45 | return errors.New(constants.EnvNode + " environment variable must be set") 46 | } 47 | 48 | timeout := gracefulTimeout 49 | mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ 50 | Scheme: scheme, 51 | LeaderElection: false, 52 | Metrics: metricsserver.Options{ 53 | BindAddress: config.metricsAddr, 54 | }, 55 | GracefulShutdownTimeout: &timeout, 56 | HealthProbeBindAddress: config.healthAddr, 57 | }) 58 | if err != nil { 59 | return err 60 | } 61 | 62 | if err := mgr.AddHealthzCheck("ping", healthz.Ping); err != nil { 63 | return err 64 | } 65 | if err := mgr.AddReadyzCheck("ping", healthz.Ping); err != nil { 66 | return err 67 | } 68 | 69 | notifyCh := make(chan struct{}, 1) 70 | abr := &controllers.AddressBlockReconciler{Notify: notifyCh} 71 | if err := abr.SetupWithManager(mgr); err != nil { 72 | return err 73 | } 74 | 75 | syncer := nodenet.NewRouteSyncer(config.protocolId, ctrl.Log.WithName("route-syncer")) 76 | router := runners.NewRouter(mgr, ctrl.Log.WithName("router"), nodeName, notifyCh, syncer, config.updateInterval) 77 | if err := mgr.Add(router); err != nil { 78 | return err 79 | } 80 | 81 | setupLog.Info(fmt.Sprintf("starting manager (version: %s)", v2.Version())) 82 | if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { 83 | setupLog.Error(err, "problem running manager") 84 | return err 85 | } 86 | 87 | return nil 88 | } 89 | -------------------------------------------------------------------------------- /v2/cmd/coil/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "time" 7 | 8 | "github.com/containernetworking/cni/pkg/skel" 9 | "github.com/containernetworking/cni/pkg/types" 10 | current "github.com/containernetworking/cni/pkg/types/100" 11 | "github.com/containernetworking/cni/pkg/version" 12 | v2 "github.com/cybozu-go/coil/v2" 13 | "github.com/cybozu-go/coil/v2/pkg/cnirpc" 14 | ) 15 | 16 | const ( 17 | rpcTimeout = 1 * time.Minute 18 | ) 19 | 20 | func cmdAdd(args *skel.CmdArgs) error { 21 | conf, err := parseConfig(args.StdinData) 22 | if err != nil { 23 | return err 24 | } 25 | 26 | cniArgs, err := makeCNIArgs(args, conf) 27 | if err != nil { 28 | return err 29 | } 30 | 31 | conn, err := connect(conf.Socket) 32 | if err != nil { 33 | return err 34 | } 35 | defer conn.Close() 36 | 37 | client := cnirpc.NewCNIClient(conn) 38 | ctx, cancel := context.WithTimeout(context.Background(), rpcTimeout) 39 | defer cancel() 40 | 41 | resp, err := client.Add(ctx, cniArgs) 42 | if err != nil { 43 | return convertError(err) 44 | } 45 | 46 | var result types.Result 47 | if conf.PrevResult != nil { 48 | result, err = current.NewResultFromResult(conf.PrevResult) 49 | } else { 50 | result, err = current.NewResult(resp.Result) 51 | } 52 | 53 | if err != nil { 54 | return types.NewError(types.ErrDecodingFailure, "failed to unmarshal result", err.Error()) 55 | } 56 | 57 | return types.PrintResult(result, conf.CNIVersion) 58 | } 59 | 60 | func cmdDel(args *skel.CmdArgs) error { 61 | conf, err := parseConfig(args.StdinData) 62 | if err != nil { 63 | return err 64 | } 65 | 66 | cniArgs, err := makeCNIArgs(args, conf) 67 | if err != nil { 68 | return err 69 | } 70 | 71 | conn, err := connect(conf.Socket) 72 | if err != nil { 73 | return err 74 | } 75 | defer conn.Close() 76 | 77 | client := cnirpc.NewCNIClient(conn) 78 | ctx, cancel := context.WithTimeout(context.Background(), rpcTimeout) 79 | defer cancel() 80 | 81 | if _, err = client.Del(ctx, cniArgs); err != nil { 82 | return convertError(err) 83 | } 84 | 85 | return nil 86 | } 87 | 88 | func cmdCheck(args *skel.CmdArgs) error { 89 | conf, err := parseConfig(args.StdinData) 90 | if err != nil { 91 | return err 92 | } 93 | 94 | cniArgs, err := makeCNIArgs(args, conf) 95 | if err != nil { 96 | return err 97 | } 98 | 99 | conn, err := connect(conf.Socket) 100 | if err != nil { 101 | return err 102 | } 103 | defer conn.Close() 104 | 105 | client := cnirpc.NewCNIClient(conn) 106 | ctx, cancel := context.WithTimeout(context.Background(), rpcTimeout) 107 | defer cancel() 108 | 109 | if _, err = client.Check(ctx, cniArgs); err != nil { 110 | return convertError(err) 111 | } 112 | 113 | return nil 114 | } 115 | 116 | func main() { 117 | skel.PluginMainFuncs(skel.CNIFuncs{Add: cmdAdd, Del: cmdDel, Check: cmdCheck, GC: nil, Status: nil}, version.PluginSupports("0.3.1", "0.4.0", "1.0.0"), fmt.Sprintf("coil %s", v2.Version())) 118 | } 119 | -------------------------------------------------------------------------------- /v2/cmd/coil/rpc.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net" 7 | "strconv" 8 | 9 | "github.com/containernetworking/cni/pkg/skel" 10 | "github.com/containernetworking/cni/pkg/types" 11 | current "github.com/containernetworking/cni/pkg/types/100" 12 | "github.com/cybozu-go/coil/v2/pkg/cnirpc" 13 | "github.com/cybozu-go/coil/v2/pkg/constants" 14 | "google.golang.org/grpc" 15 | "google.golang.org/grpc/credentials/insecure" 16 | "google.golang.org/grpc/resolver" 17 | "google.golang.org/grpc/status" 18 | ) 19 | 20 | // makeCNIArgs creates *CNIArgs. 21 | func makeCNIArgs(args *skel.CmdArgs, conf *PluginConf) (*cnirpc.CNIArgs, error) { 22 | env := &PluginEnvArgs{} 23 | if err := types.LoadArgs(args.Args, env); err != nil { 24 | return nil, types.NewError(types.ErrInvalidEnvironmentVariables, "failed to load CNI_ARGS", err.Error()) 25 | } 26 | 27 | argsData := env.Map() 28 | argsData[constants.IsChained] = strconv.FormatBool(conf.PrevResult != nil) 29 | 30 | ips := []string{} 31 | interfaces := map[string]bool{} 32 | if conf.PrevResult != nil { 33 | prevResult, err := current.GetResult(conf.PrevResult) 34 | if err != nil { 35 | return nil, fmt.Errorf("error getting previous CNI result: %w", err) 36 | } 37 | for _, ip := range prevResult.IPs { 38 | ips = append(ips, ip.Address.IP.String()) 39 | } 40 | for _, intf := range prevResult.Interfaces { 41 | interfaces[intf.Name] = intf.Sandbox != "" 42 | } 43 | } 44 | 45 | cniArgs := &cnirpc.CNIArgs{ 46 | ContainerId: args.ContainerID, 47 | Netns: args.Netns, 48 | Ifname: args.IfName, 49 | Args: argsData, 50 | Path: args.Path, 51 | StdinData: args.StdinData, 52 | Ips: ips, 53 | Interfaces: interfaces, 54 | } 55 | 56 | return cniArgs, nil 57 | } 58 | 59 | // connect connects to coild 60 | func connect(sock string) (*grpc.ClientConn, error) { 61 | dialer := &net.Dialer{} 62 | dialFunc := func(ctx context.Context, a string) (net.Conn, error) { 63 | return dialer.DialContext(ctx, "unix", a) 64 | } 65 | resolver.SetDefaultScheme("passthrough") 66 | 67 | conn, err := grpc.NewClient(sock, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithContextDialer(dialFunc)) 68 | if err != nil { 69 | return nil, types.NewError(types.ErrTryAgainLater, "failed to connect to "+sock, err.Error()) 70 | } 71 | return conn, nil 72 | } 73 | 74 | // convertError turns err returned from gRPC library into CNI's types.Error 75 | func convertError(err error) error { 76 | st := status.Convert(err) 77 | details := st.Details() 78 | if len(details) != 1 { 79 | return types.NewError(types.ErrInternal, st.Message(), err.Error()) 80 | } 81 | 82 | cniErr, ok := details[0].(*cnirpc.CNIError) 83 | if !ok { 84 | types.NewError(types.ErrInternal, st.Message(), err.Error()) 85 | } 86 | 87 | return types.NewError(uint(cniErr.Code), cniErr.Msg, cniErr.Details) 88 | } 89 | -------------------------------------------------------------------------------- /v2/cmd/coil/types.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | 7 | "github.com/containernetworking/cni/pkg/types" 8 | "github.com/containernetworking/cni/pkg/version" 9 | "github.com/cybozu-go/coil/v2/pkg/constants" 10 | ) 11 | 12 | // PluginEnvArgs represents CNI_ARG 13 | type PluginEnvArgs struct { 14 | types.CommonArgs 15 | K8S_POD_NAMESPACE types.UnmarshallableString 16 | K8S_POD_NAME types.UnmarshallableString 17 | K8S_POD_INFRA_CONTAINER_ID types.UnmarshallableString 18 | } 19 | 20 | // Map returns a map[string]string 21 | func (e PluginEnvArgs) Map() map[string]string { 22 | return map[string]string{ 23 | constants.PodNamespaceKey: string(e.K8S_POD_NAMESPACE), 24 | constants.PodNameKey: string(e.K8S_POD_NAME), 25 | constants.PodContainerKey: string(e.K8S_POD_INFRA_CONTAINER_ID), 26 | } 27 | } 28 | 29 | // PluginConf represents JSON netconf for Coil. 30 | type PluginConf struct { 31 | types.NetConf 32 | 33 | // Coil specific flags 34 | Socket string `json:"socket"` 35 | } 36 | 37 | func parseConfig(stdin []byte) (*PluginConf, error) { 38 | conf := &PluginConf{ 39 | Socket: constants.DefaultSocketPath, 40 | } 41 | 42 | if err := json.Unmarshal(stdin, conf); err != nil { 43 | return nil, fmt.Errorf("failed to parse network configuration: %w", err) 44 | } 45 | 46 | if err := version.ParsePrevResult(&conf.NetConf); err != nil { 47 | return nil, fmt.Errorf("failed to parse prev result: %w", err) 48 | } 49 | 50 | return conf, nil 51 | } 52 | -------------------------------------------------------------------------------- /v2/cmd/coil/types_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/containernetworking/cni/pkg/types" 7 | "github.com/cybozu-go/coil/v2/pkg/constants" 8 | ) 9 | 10 | func TestPluginEnvArgs(t *testing.T) { 11 | env := &PluginEnvArgs{} 12 | 13 | args := "IgnoreUnknown=1;K8S_POD_NAMESPACE=test;K8S_POD_NAME=testhttpd-host1;K8S_POD_INFRA_CONTAINER_ID=c8f4a9c50c85b36eff718aab2ac39209e541a4551420488c33d9216cf1795b3a" 14 | if err := types.LoadArgs(args, env); err != nil { 15 | t.Fatal(err) 16 | } 17 | 18 | if env.K8S_POD_NAMESPACE != "test" { 19 | t.Error(`env.K8S_POD_NAMESPACE != "test"`, env.K8S_POD_NAMESPACE) 20 | } 21 | 22 | if env.K8S_POD_NAME != "testhttpd-host1" { 23 | t.Error(`env.K8S_POD_NAME != "testhttpd-host1"`, env.K8S_POD_NAME) 24 | } 25 | 26 | if env.K8S_POD_INFRA_CONTAINER_ID != "c8f4a9c50c85b36eff718aab2ac39209e541a4551420488c33d9216cf1795b3a" { 27 | t.Error(`env.K8S_POD_INFRA_CONTAINER_ID != "c8f4a9c50c85b36eff718aab2ac39209e541a4551420488c33d9216cf1795b3a"`, env.K8S_POD_INFRA_CONTAINER_ID) 28 | } 29 | } 30 | 31 | func TestParseConfig(t *testing.T) { 32 | conf := []byte(` 33 | { 34 | "cniVersion": "0.4.0", 35 | "name": "k8s", 36 | "type": "coil" 37 | } 38 | `) 39 | 40 | pc, err := parseConfig(conf) 41 | if err != nil { 42 | t.Fatal(err) 43 | } 44 | if pc.CNIVersion != "0.4.0" { 45 | t.Error(`pc.CNIVersion != "0.4.0"`) 46 | } 47 | if pc.Name != "k8s" { 48 | t.Error(`pc.Name != "k8s"`) 49 | } 50 | if pc.Type != "coil" { 51 | t.Error(`pc.Type != "coil"`) 52 | } 53 | if pc.PrevResult != nil { 54 | t.Error("pc.Result should be nil") 55 | } 56 | if pc.Socket != constants.DefaultSocketPath { 57 | t.Error(`pc.Socket != constants.DefaultSocketPath`) 58 | } 59 | 60 | conf = []byte(` 61 | { 62 | "cniVersion": "0.4.0", 63 | "name": "k8s", 64 | "type": "coil", 65 | "socket": "/tmp/coild.sock" 66 | } 67 | `) 68 | pc, err = parseConfig(conf) 69 | if err != nil { 70 | t.Fatal(err) 71 | } 72 | if pc.Socket != "/tmp/coild.sock" { 73 | t.Error(`pc.Socket != "/tmp/coild.sock"`) 74 | } 75 | 76 | conf = []byte(` 77 | { 78 | "cniVersion": "0.4.0", 79 | "name": "k8s", 80 | "type": "coil", 81 | "prevResult": { 82 | "ips": [ 83 | { 84 | "version": "4", 85 | "address": "10.0.0.5/32", 86 | "interface": 2 87 | } 88 | ], 89 | "interfaces": [ 90 | { 91 | "name": "cni0", 92 | "mac": "00:11:22:33:44:55" 93 | }, 94 | { 95 | "name": "veth3243", 96 | "mac": "55:44:33:22:11:11" 97 | }, 98 | { 99 | "name": "eth0", 100 | "mac": "99:88:77:66:55:44", 101 | "sandbox": "/var/run/netns/blue" 102 | } 103 | ], 104 | "dns": { 105 | "nameservers": [ "10.1.0.1" ] 106 | } 107 | } 108 | }`) 109 | 110 | pc, err = parseConfig(conf) 111 | if err != nil { 112 | t.Fatal(err) 113 | } 114 | if pc.PrevResult == nil { 115 | t.Error("pc.Result should not be nil") 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /v2/cmd/coild/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "github.com/cybozu-go/coil/v2/cmd/coild/sub" 4 | 5 | func main() { 6 | sub.Execute() 7 | } 8 | -------------------------------------------------------------------------------- /v2/cmd/coild/sub/root.go: -------------------------------------------------------------------------------- 1 | package sub 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | v2 "github.com/cybozu-go/coil/v2" 8 | "github.com/cybozu-go/coil/v2/pkg/config" 9 | "github.com/spf13/cobra" 10 | ) 11 | 12 | var rootCmd = &cobra.Command{ 13 | Use: "coild", 14 | Short: "gRPC server running on each node", 15 | Long: `coild is a gRPC server running on each node. 16 | 17 | It listens on a UNIX domain socket and accepts requests from 18 | coil CNI plugin.`, 19 | Version: v2.Version(), 20 | RunE: func(cmd *cobra.Command, _ []string) error { 21 | cmd.SilenceUsage = true 22 | return subMain() 23 | }, 24 | } 25 | 26 | var cfg *config.Config 27 | 28 | // Execute adds all child commands to the root command and sets flags appropriately. 29 | // This is called by main.main(). It only needs to happen once to the rootCmd. 30 | func Execute() { 31 | cfg = config.Parse(rootCmd) 32 | if err := rootCmd.Execute(); err != nil { 33 | fmt.Println(err) 34 | os.Exit(1) 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /v2/config/cke/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - ../crd 3 | - ../rbac 4 | - ../pod 5 | - ../webhook/egress 6 | - ../webhook/ipam 7 | - ./webhook-secret.yaml 8 | 9 | patchesStrategicMerge: 10 | - ./webhook_manifests_patch.yaml 11 | -------------------------------------------------------------------------------- /v2/config/cke/webhook-secret.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Secret 3 | metadata: 4 | name: coilv2-ipam-webhook-server-cert 5 | namespace: system 6 | annotations: 7 | cke.cybozu.com/issue-cert: coilv2-ipam-webhook-service 8 | --- 9 | apiVersion: v1 10 | kind: Secret 11 | metadata: 12 | name: coilv2-egress-webhook-server-cert 13 | namespace: system 14 | annotations: 15 | cke.cybozu.com/issue-cert: coilv2-egress-webhook-service 16 | -------------------------------------------------------------------------------- /v2/config/cke/webhook_manifests_patch.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: admissionregistration.k8s.io/v1 2 | kind: MutatingWebhookConfiguration 3 | metadata: 4 | name: mutating-ipam-webhook-configuration 5 | annotations: 6 | cke.cybozu.com/inject-cacert: "true" 7 | --- 8 | apiVersion: admissionregistration.k8s.io/v1 9 | kind: ValidatingWebhookConfiguration 10 | metadata: 11 | name: validating-ipam-webhook-configuration 12 | annotations: 13 | cke.cybozu.com/inject-cacert: "true" 14 | --- 15 | apiVersion: admissionregistration.k8s.io/v1 16 | kind: MutatingWebhookConfiguration 17 | metadata: 18 | name: mutating-egress-webhook-configuration 19 | annotations: 20 | cke.cybozu.com/inject-cacert: "true" 21 | --- 22 | apiVersion: admissionregistration.k8s.io/v1 23 | kind: ValidatingWebhookConfiguration 24 | metadata: 25 | name: validating-egress-webhook-configuration 26 | annotations: 27 | cke.cybozu.com/inject-cacert: "true" 28 | --- 29 | -------------------------------------------------------------------------------- /v2/config/crd/bases/coil.cybozu.com_addressblocks.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apiextensions.k8s.io/v1 3 | kind: CustomResourceDefinition 4 | metadata: 5 | annotations: 6 | controller-gen.kubebuilder.io/version: v0.16.4 7 | name: addressblocks.coil.cybozu.com 8 | spec: 9 | group: coil.cybozu.com 10 | names: 11 | kind: AddressBlock 12 | listKind: AddressBlockList 13 | plural: addressblocks 14 | singular: addressblock 15 | scope: Cluster 16 | versions: 17 | - additionalPrinterColumns: 18 | - jsonPath: .metadata.labels['coil\.cybozu\.com/node'] 19 | name: Node 20 | type: string 21 | - jsonPath: .metadata.labels['coil\.cybozu\.com/pool'] 22 | name: Pool 23 | type: string 24 | - jsonPath: .ipv4 25 | name: IPv4 26 | type: string 27 | - jsonPath: .ipv6 28 | name: IPv6 29 | type: string 30 | name: v2 31 | schema: 32 | openAPIV3Schema: 33 | description: |- 34 | AddressBlock is the Schema for the addressblocks API 35 | 36 | The ownerReferences field contains the AddressPool where the block is carved from. 37 | properties: 38 | apiVersion: 39 | description: |- 40 | APIVersion defines the versioned schema of this representation of an object. 41 | Servers should convert recognized schemas to the latest internal value, and 42 | may reject unrecognized values. 43 | More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources 44 | type: string 45 | index: 46 | description: Index indicates the index of this block from the origin pool 47 | format: int32 48 | minimum: 0 49 | type: integer 50 | ipv4: 51 | description: IPv4 is an IPv4 subnet address 52 | type: string 53 | ipv6: 54 | description: IPv6 is an IPv6 subnet address 55 | type: string 56 | kind: 57 | description: |- 58 | Kind is a string value representing the REST resource this object represents. 59 | Servers may infer this from the endpoint the client submits requests to. 60 | Cannot be updated. 61 | In CamelCase. 62 | More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds 63 | type: string 64 | metadata: 65 | type: object 66 | required: 67 | - index 68 | type: object 69 | served: true 70 | storage: true 71 | subresources: {} 72 | -------------------------------------------------------------------------------- /v2/config/crd/bases/coil.cybozu.com_addresspools.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apiextensions.k8s.io/v1 3 | kind: CustomResourceDefinition 4 | metadata: 5 | annotations: 6 | controller-gen.kubebuilder.io/version: v0.16.4 7 | name: addresspools.coil.cybozu.com 8 | spec: 9 | group: coil.cybozu.com 10 | names: 11 | kind: AddressPool 12 | listKind: AddressPoolList 13 | plural: addresspools 14 | singular: addresspool 15 | scope: Cluster 16 | versions: 17 | - additionalPrinterColumns: 18 | - jsonPath: .spec.blockSizeBits 19 | name: BlockSize Bits 20 | type: integer 21 | name: v2 22 | schema: 23 | openAPIV3Schema: 24 | description: AddressPool is the Schema for the addresspools API 25 | properties: 26 | apiVersion: 27 | description: |- 28 | APIVersion defines the versioned schema of this representation of an object. 29 | Servers should convert recognized schemas to the latest internal value, and 30 | may reject unrecognized values. 31 | More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources 32 | type: string 33 | kind: 34 | description: |- 35 | Kind is a string value representing the REST resource this object represents. 36 | Servers may infer this from the endpoint the client submits requests to. 37 | Cannot be updated. 38 | In CamelCase. 39 | More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds 40 | type: string 41 | metadata: 42 | type: object 43 | spec: 44 | description: AddressPoolSpec defines the desired state of AddressPool 45 | properties: 46 | blockSizeBits: 47 | default: 5 48 | description: |- 49 | BlockSizeBits specifies the size of the address blocks carved from this pool. 50 | If this is 5, a block will have 2^5 = 32 addresses. Default is 5. 51 | format: int32 52 | minimum: 0 53 | type: integer 54 | subnets: 55 | description: |- 56 | Subnets is a list of IPv4, or IPv6, or dual stack IPv4/IPv6 subnets in this pool. 57 | All items in the list should be consistent to have the same set of subnets. 58 | For example, if the first item is an IPv4 subnet, the other items must also be 59 | an IPv4 subnet. 60 | 61 | This field can be updated only by adding subnets to the list. 62 | items: 63 | description: |- 64 | SubnetSet defines a IPv4-only or IPv6-only or IPv4/v6 dual stack subnet 65 | A dual stack subnet must has the same size subnet of IPv4 and IPv6. 66 | properties: 67 | ipv4: 68 | description: IPv4 is an IPv4 subnet like "10.2.0.0/16" 69 | type: string 70 | ipv6: 71 | description: IPv6 is an IPv6 subnet like "fd00:0200::/112" 72 | type: string 73 | type: object 74 | minItems: 1 75 | type: array 76 | required: 77 | - subnets 78 | type: object 79 | type: object 80 | served: true 81 | storage: true 82 | subresources: {} 83 | -------------------------------------------------------------------------------- /v2/config/crd/egress/kustomization.yaml: -------------------------------------------------------------------------------- 1 | # This kustomization.yaml is not intended to be run by itself, 2 | # since it depends on service name and namespace that are out of this kustomize package. 3 | # It should be run by config/default 4 | resources: 5 | # [EGRESS] Following files should be uncommented to enable Egress NAT features. 6 | - ../bases/coil.cybozu.com_egresses.yaml 7 | # +kubebuilder:scaffold:crdkustomizeresource 8 | 9 | patchesStrategicMerge: 10 | - ../patches/egress/remove_status.yaml 11 | # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix. 12 | # patches here are for enabling the conversion webhook for each CRD 13 | # [EGRESS] Following files should be uncommented to enable Egress NAT features. 14 | #- patches/webhook_in_egresses.yaml 15 | # +kubebuilder:scaffold:crdkustomizewebhookpatch 16 | 17 | # [CERTMANAGER] To enable webhook, uncomment all the sections with [CERTMANAGER] prefix. 18 | # patches here are for enabling the CA injection for each CRD 19 | # [EGRESS] Following files should be uncommented to enable Egress NAT features. 20 | #- patches/cainjection_in_egresses.yaml 21 | # +kubebuilder:scaffold:crdkustomizecainjectionpatch 22 | 23 | # the following config is for teaching kustomize how to do kustomization for CRDs. 24 | configurations: 25 | - ../kustomizeconfig.yaml 26 | -------------------------------------------------------------------------------- /v2/config/crd/kustomization.yaml: -------------------------------------------------------------------------------- 1 | # This kustomization.yaml is not intended to be run by itself, 2 | # since it depends on service name and namespace that are out of this kustomize package. 3 | # It should be run by config/default 4 | resources: 5 | # [IPAM] Following files should be uncommented to enable IPAM features. 6 | - bases/coil.cybozu.com_addresspools.yaml 7 | - bases/coil.cybozu.com_addressblocks.yaml 8 | - bases/coil.cybozu.com_blockrequests.yaml 9 | # [EGRESS] Following files should be uncommented to enable Egress NAT features. 10 | - bases/coil.cybozu.com_egresses.yaml 11 | # +kubebuilder:scaffold:crdkustomizeresource 12 | 13 | patchesStrategicMerge: 14 | - patches/remove_status.yaml 15 | # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix. 16 | # patches here are for enabling the conversion webhook for each CRD 17 | # [IPAM] Following files should be uncommented to enable IPAM features. 18 | #- patches/webhook_in_addresspools.yaml 19 | #- patches/webhook_in_addressblocks.yaml 20 | #- patches/webhook_in_blockrequests.yaml 21 | # [EGRESS] Following files should be uncommented to enable Egress NAT features. 22 | #- patches/webhook_in_egresses.yaml 23 | # +kubebuilder:scaffold:crdkustomizewebhookpatch 24 | 25 | # [CERTMANAGER] To enable webhook, uncomment all the sections with [CERTMANAGER] prefix. 26 | # patches here are for enabling the CA injection for each CRD 27 | # [IPAM] Following files should be uncommented to enable IPAM features. 28 | #- patches/cainjection_in_addresspools.yaml 29 | #- patches/cainjection_in_addressblocks.yaml 30 | #- patches/cainjection_in_blockrequests.yaml 31 | # [EGRESS] Following files should be uncommented to enable Egress NAT features. 32 | #- patches/cainjection_in_egresses.yaml 33 | # +kubebuilder:scaffold:crdkustomizecainjectionpatch 34 | 35 | # the following config is for teaching kustomize how to do kustomization for CRDs. 36 | configurations: 37 | - kustomizeconfig.yaml 38 | -------------------------------------------------------------------------------- /v2/config/crd/kustomizeconfig.yaml: -------------------------------------------------------------------------------- 1 | # This file is for teaching kustomize how to substitute name and namespace reference in CRD 2 | nameReference: 3 | - kind: Service 4 | version: v1 5 | fieldSpecs: 6 | - kind: CustomResourceDefinition 7 | group: apiextensions.k8s.io 8 | path: spec/conversion/webhookClientConfig/service/name 9 | 10 | namespace: 11 | - kind: CustomResourceDefinition 12 | group: apiextensions.k8s.io 13 | path: spec/conversion/webhookClientConfig/service/namespace 14 | create: false 15 | 16 | varReference: 17 | - path: metadata/annotations 18 | -------------------------------------------------------------------------------- /v2/config/crd/patches/cainjection_in_addressblocks.yaml: -------------------------------------------------------------------------------- 1 | # The following patch adds a directive for certmanager to inject CA into the CRD 2 | # CRD conversion requires k8s 1.13 or later. 3 | apiVersion: apiextensions.k8s.io/v1beta1 4 | kind: CustomResourceDefinition 5 | metadata: 6 | annotations: 7 | cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) 8 | name: addressblocks.coil.cybozu.com 9 | -------------------------------------------------------------------------------- /v2/config/crd/patches/cainjection_in_addresspools.yaml: -------------------------------------------------------------------------------- 1 | # The following patch adds a directive for certmanager to inject CA into the CRD 2 | # CRD conversion requires k8s 1.13 or later. 3 | apiVersion: apiextensions.k8s.io/v1beta1 4 | kind: CustomResourceDefinition 5 | metadata: 6 | annotations: 7 | cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) 8 | name: addresspools.coil.cybozu.com 9 | -------------------------------------------------------------------------------- /v2/config/crd/patches/cainjection_in_blockrequests.yaml: -------------------------------------------------------------------------------- 1 | # The following patch adds a directive for certmanager to inject CA into the CRD 2 | # CRD conversion requires k8s 1.13 or later. 3 | apiVersion: apiextensions.k8s.io/v1beta1 4 | kind: CustomResourceDefinition 5 | metadata: 6 | annotations: 7 | cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) 8 | name: blockrequests.coil.cybozu.com 9 | -------------------------------------------------------------------------------- /v2/config/crd/patches/cainjection_in_egresses.yaml: -------------------------------------------------------------------------------- 1 | # The following patch adds a directive for certmanager to inject CA into the CRD 2 | # CRD conversion requires k8s 1.13 or later. 3 | apiVersion: apiextensions.k8s.io/v1beta1 4 | kind: CustomResourceDefinition 5 | metadata: 6 | annotations: 7 | cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) 8 | name: egresses.coil.cybozu.com 9 | -------------------------------------------------------------------------------- /v2/config/crd/patches/egress/remove_status.yaml: -------------------------------------------------------------------------------- 1 | # [EGRESS] Following resources be uncommented to enable Egress NAT features. 2 | apiVersion: apiextensions.k8s.io/v1 3 | kind: CustomResourceDefinition 4 | metadata: 5 | name: egresses.coil.cybozu.com 6 | status: null 7 | --- 8 | -------------------------------------------------------------------------------- /v2/config/crd/patches/remove_status.yaml: -------------------------------------------------------------------------------- 1 | # [IPAM] Following resources should be uncommented to enable IPAM features. 2 | # apiVersion: apiextensions.k8s.io/v1 3 | # kind: CustomResourceDefinition 4 | # metadata: 5 | # name: addressblocks.coil.cybozu.com 6 | # status: null 7 | # --- 8 | # apiVersion: apiextensions.k8s.io/v1 9 | # kind: CustomResourceDefinition 10 | # metadata: 11 | # name: addresspools.coil.cybozu.com 12 | # status: null 13 | # --- 14 | # apiVersion: apiextensions.k8s.io/v1 15 | # kind: CustomResourceDefinition 16 | # metadata: 17 | # name: blockrequests.coil.cybozu.com 18 | # status: null 19 | # --- 20 | # [EGRESS] Following resources be uncommented to enable Egress NAT features. 21 | apiVersion: apiextensions.k8s.io/v1 22 | kind: CustomResourceDefinition 23 | metadata: 24 | name: egresses.coil.cybozu.com 25 | status: null 26 | --- 27 | -------------------------------------------------------------------------------- /v2/config/crd/patches/webhook_in_addressblocks.yaml: -------------------------------------------------------------------------------- 1 | # The following patch enables conversion webhook for CRD 2 | # CRD conversion requires k8s 1.13 or later. 3 | apiVersion: apiextensions.k8s.io/v1beta1 4 | kind: CustomResourceDefinition 5 | metadata: 6 | name: addressblocks.coil.cybozu.com 7 | spec: 8 | conversion: 9 | strategy: Webhook 10 | webhookClientConfig: 11 | # this is "\n" used as a placeholder, otherwise it will be rejected by the apiserver for being blank, 12 | # but we're going to set it later using the cert-manager (or potentially a patch if not using cert-manager) 13 | caBundle: Cg== 14 | service: 15 | namespace: system 16 | name: ipam-webhook-service 17 | path: /convert 18 | -------------------------------------------------------------------------------- /v2/config/crd/patches/webhook_in_addresspools.yaml: -------------------------------------------------------------------------------- 1 | # The following patch enables conversion webhook for CRD 2 | # CRD conversion requires k8s 1.13 or later. 3 | apiVersion: apiextensions.k8s.io/v1beta1 4 | kind: CustomResourceDefinition 5 | metadata: 6 | name: addresspools.coil.cybozu.com 7 | spec: 8 | conversion: 9 | strategy: Webhook 10 | webhookClientConfig: 11 | # this is "\n" used as a placeholder, otherwise it will be rejected by the apiserver for being blank, 12 | # but we're going to set it later using the cert-manager (or potentially a patch if not using cert-manager) 13 | caBundle: Cg== 14 | service: 15 | namespace: system 16 | name: ipam-webhook-service 17 | path: /convert 18 | -------------------------------------------------------------------------------- /v2/config/crd/patches/webhook_in_blockrequests.yaml: -------------------------------------------------------------------------------- 1 | # The following patch enables conversion webhook for CRD 2 | # CRD conversion requires k8s 1.13 or later. 3 | apiVersion: apiextensions.k8s.io/v1beta1 4 | kind: CustomResourceDefinition 5 | metadata: 6 | name: blockrequests.coil.cybozu.com 7 | spec: 8 | conversion: 9 | strategy: Webhook 10 | webhookClientConfig: 11 | # this is "\n" used as a placeholder, otherwise it will be rejected by the apiserver for being blank, 12 | # but we're going to set it later using the cert-manager (or potentially a patch if not using cert-manager) 13 | caBundle: Cg== 14 | service: 15 | namespace: system 16 | name: ipam-webhook-service 17 | path: /convert 18 | -------------------------------------------------------------------------------- /v2/config/crd/patches/webhook_in_egresses.yaml: -------------------------------------------------------------------------------- 1 | # The following patch enables conversion webhook for CRD 2 | # CRD conversion requires k8s 1.13 or later. 3 | apiVersion: apiextensions.k8s.io/v1beta1 4 | kind: CustomResourceDefinition 5 | metadata: 6 | name: egresses.coil.cybozu.com 7 | spec: 8 | conversion: 9 | strategy: Webhook 10 | webhookClientConfig: 11 | # this is "\n" used as a placeholder, otherwise it will be rejected by the apiserver for being blank, 12 | # but we're going to set it later using the cert-manager (or potentially a patch if not using cert-manager) 13 | caBundle: Cg== 14 | service: 15 | namespace: system 16 | name: egress-webhook-service 17 | path: /convert 18 | -------------------------------------------------------------------------------- /v2/config/default/.gitignore: -------------------------------------------------------------------------------- 1 | cert.pem 2 | key.pem 3 | ipam-cert.pem 4 | ipam-key.pem 5 | egress-cert.pem 6 | egress-key.pem -------------------------------------------------------------------------------- /v2/config/default/egress/v4/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - ../../../crd/egress 3 | - ../../../rbac/egress 4 | - ../../../pod/egress/v4 5 | - ../../../webhook/egress 6 | 7 | # [CERTS] Following lines should be commented if automatic cert generation is used. 8 | patchesStrategicMerge: 9 | - ../webhook_manifests_patch.yaml 10 | 11 | generatorOptions: 12 | disableNameSuffixHash: true 13 | 14 | secretGenerator: 15 | # [EGRESS] Following lines be uncommented to enable Egress NAT features. 16 | - name: coilv2-egress-webhook-server-cert 17 | files: 18 | - ca.crt=../../cert.pem 19 | - tls.crt=../../egress-cert.pem 20 | - tls.key=../../egress-key.pem 21 | type: "kubernetes.io/tls" 22 | -------------------------------------------------------------------------------- /v2/config/default/egress/v6/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - ../../../crd/egress 3 | - ../../../rbac/egress 4 | - ../../../pod/egress/v6 5 | - ../../../webhook/egress 6 | 7 | # [CERTS] Following lines should be commented if automatic cert generation is used. 8 | patchesStrategicMerge: 9 | - ../webhook_manifests_patch.yaml 10 | 11 | generatorOptions: 12 | disableNameSuffixHash: true 13 | 14 | secretGenerator: 15 | # [EGRESS] Following lines be uncommented to enable Egress NAT features. 16 | - name: coilv2-egress-webhook-server-cert 17 | files: 18 | - ca.crt=../../cert.pem 19 | - tls.crt=../../egress-cert.pem 20 | - tls.key=../../egress-key.pem 21 | type: "kubernetes.io/tls" 22 | -------------------------------------------------------------------------------- /v2/config/default/egress/webhook_manifests_patch.yaml.tmpl: -------------------------------------------------------------------------------- 1 | apiVersion: admissionregistration.k8s.io/v1 2 | kind: MutatingWebhookConfiguration 3 | metadata: 4 | name: coilv2-mutating-egress-webhook-configuration 5 | webhooks: 6 | - name: megress.kb.io 7 | clientConfig: 8 | caBundle: "%CACERT%" 9 | --- 10 | apiVersion: admissionregistration.k8s.io/v1 11 | kind: ValidatingWebhookConfiguration 12 | metadata: 13 | name: coilv2-validating-egress-webhook-configuration 14 | webhooks: 15 | - name: vegress.kb.io 16 | clientConfig: 17 | caBundle: "%CACERT%" -------------------------------------------------------------------------------- /v2/config/default/ipam/webhook_manifests_patch.yaml.tmpl: -------------------------------------------------------------------------------- 1 | apiVersion: admissionregistration.k8s.io/v1 2 | kind: MutatingWebhookConfiguration 3 | metadata: 4 | name: coilv2-mutating-ipam-webhook-configuration 5 | webhooks: 6 | - name: maddresspool.kb.io 7 | clientConfig: 8 | caBundle: "%CACERT%" 9 | --- 10 | apiVersion: admissionregistration.k8s.io/v1 11 | kind: ValidatingWebhookConfiguration 12 | metadata: 13 | name: coilv2-validating-ipam-webhook-configuration 14 | webhooks: 15 | - name: vaddresspool.kb.io 16 | clientConfig: 17 | caBundle: "%CACERT%" 18 | -------------------------------------------------------------------------------- /v2/config/default/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - ../crd 3 | - ../rbac 4 | - ../pod 5 | - ../webhook/ipam 6 | - ../webhook/egress 7 | 8 | # [CERTS] Following lines should be commented if automatic cert generation is used. 9 | patchesStrategicMerge: 10 | - egress/webhook_manifests_patch.yaml 11 | - ipam/webhook_manifests_patch.yaml 12 | 13 | generatorOptions: 14 | disableNameSuffixHash: true 15 | 16 | secretGenerator: 17 | # [IPAM] Following lines should be uncommented to enable IPAM features. 18 | - name: coilv2-ipam-webhook-server-cert 19 | # [CERTS] Following lines should be commented if automatic cert generation is used. 20 | files: 21 | - ca.crt=./cert.pem 22 | - tls.crt=./ipam-cert.pem 23 | - tls.key=./ipam-key.pem 24 | type: "kubernetes.io/tls" 25 | # [EGRESS] Following lines be uncommented to enable Egress NAT features. 26 | - name: coilv2-egress-webhook-server-cert 27 | # [CERTS] Following lines should be commented if automatic cert generation is used. 28 | files: 29 | - ca.crt=./cert.pem 30 | - tls.crt=./egress-cert.pem 31 | - tls.key=./egress-key.pem 32 | type: "kubernetes.io/tls" 33 | -------------------------------------------------------------------------------- /v2/config/pod/coil-egress-controller.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: coil-egress-controller 5 | namespace: system 6 | labels: 7 | app.kubernetes.io/component: coil-egress-controller 8 | spec: 9 | selector: 10 | matchLabels: 11 | app.kubernetes.io/component: coil-egress-controller 12 | replicas: 2 13 | template: 14 | metadata: 15 | labels: 16 | app.kubernetes.io/component: coil-egress-controller 17 | spec: 18 | hostNetwork: true 19 | priorityClassName: system-cluster-critical 20 | tolerations: 21 | - key: node-role.kubernetes.io/master 22 | effect: NoSchedule 23 | - key: node-role.kubernetes.io/control-plane 24 | effect: NoSchedule 25 | - key: node.kubernetes.io/not-ready 26 | effect: NoSchedule 27 | affinity: 28 | podAntiAffinity: 29 | preferredDuringSchedulingIgnoredDuringExecution: 30 | - weight: 100 31 | podAffinityTerm: 32 | labelSelector: 33 | matchExpressions: 34 | - key: app.kubernetes.io/component 35 | operator: In 36 | values: ["coil-egress-controller"] 37 | topologyKey: kubernetes.io/hostname 38 | securityContext: 39 | runAsUser: 10000 40 | runAsGroup: 10000 41 | serviceAccountName: coil-egress-controller 42 | terminationGracePeriodSeconds: 10 43 | containers: 44 | - name: coil-egress-controller 45 | image: coil:dev 46 | command: ["coil-egress-controller"] 47 | args: 48 | - --zap-stacktrace-level=panic 49 | env: 50 | - name: "COIL_POD_NAMESPACE" 51 | valueFrom: 52 | fieldRef: 53 | fieldPath: metadata.namespace 54 | - name: "COIL_POD_NAME" 55 | valueFrom: 56 | fieldRef: 57 | fieldPath: metadata.name 58 | ports: 59 | - name: metrics 60 | containerPort: 9396 61 | protocol: TCP 62 | - name: health 63 | containerPort: 9397 64 | protocol: TCP 65 | - name: webhook-server 66 | containerPort: 9444 67 | protocol: TCP 68 | resources: 69 | requests: 70 | cpu: 100m 71 | memory: 200Mi 72 | readinessProbe: 73 | httpGet: 74 | path: /readyz 75 | port: health 76 | livenessProbe: 77 | httpGet: 78 | path: /healthz 79 | port: health 80 | volumeMounts: 81 | - mountPath: /certs 82 | name: cert 83 | readOnly: true 84 | volumes: 85 | - name: cert 86 | secret: 87 | defaultMode: 420 88 | secretName: coilv2-egress-webhook-server-cert 89 | -------------------------------------------------------------------------------- /v2/config/pod/coil-ipam-controller.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: coil-ipam-controller 5 | namespace: system 6 | labels: 7 | app.kubernetes.io/component: coil-ipam-controller 8 | spec: 9 | selector: 10 | matchLabels: 11 | app.kubernetes.io/component: coil-ipam-controller 12 | replicas: 2 13 | template: 14 | metadata: 15 | labels: 16 | app.kubernetes.io/component: coil-ipam-controller 17 | spec: 18 | hostNetwork: true 19 | priorityClassName: system-cluster-critical 20 | tolerations: 21 | - key: node-role.kubernetes.io/master 22 | effect: NoSchedule 23 | - key: node-role.kubernetes.io/control-plane 24 | effect: NoSchedule 25 | - key: node.kubernetes.io/not-ready 26 | effect: NoSchedule 27 | affinity: 28 | podAntiAffinity: 29 | preferredDuringSchedulingIgnoredDuringExecution: 30 | - weight: 100 31 | podAffinityTerm: 32 | labelSelector: 33 | matchExpressions: 34 | - key: app.kubernetes.io/component 35 | operator: In 36 | values: ["coil-ipam-controller"] 37 | topologyKey: kubernetes.io/hostname 38 | securityContext: 39 | runAsUser: 10000 40 | runAsGroup: 10000 41 | serviceAccountName: coil-ipam-controller 42 | terminationGracePeriodSeconds: 10 43 | containers: 44 | - name: coil-ipam-controller 45 | image: coil:dev 46 | command: ["coil-ipam-controller"] 47 | args: 48 | - --zap-stacktrace-level=panic 49 | env: 50 | - name: "COIL_POD_NAMESPACE" 51 | valueFrom: 52 | fieldRef: 53 | fieldPath: metadata.namespace 54 | - name: "COIL_POD_NAME" 55 | valueFrom: 56 | fieldRef: 57 | fieldPath: metadata.name 58 | ports: 59 | - name: metrics 60 | containerPort: 9386 61 | protocol: TCP 62 | - name: health 63 | containerPort: 9387 64 | protocol: TCP 65 | - name: webhook-server 66 | containerPort: 9443 67 | protocol: TCP 68 | resources: 69 | requests: 70 | cpu: 100m 71 | memory: 200Mi 72 | readinessProbe: 73 | httpGet: 74 | path: /readyz 75 | port: health 76 | host: localhost 77 | livenessProbe: 78 | httpGet: 79 | path: /healthz 80 | port: health 81 | host: localhost 82 | volumeMounts: 83 | - mountPath: /certs 84 | name: cert 85 | readOnly: true 86 | volumes: 87 | - name: cert 88 | secret: 89 | defaultMode: 420 90 | secretName: coilv2-ipam-webhook-server-cert 91 | -------------------------------------------------------------------------------- /v2/config/pod/coil-router.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: DaemonSet 3 | metadata: 4 | name: coil-router 5 | namespace: system 6 | labels: 7 | app.kubernetes.io/component: coil-router 8 | spec: 9 | selector: 10 | matchLabels: 11 | app.kubernetes.io/component: coil-router 12 | template: 13 | metadata: 14 | labels: 15 | app.kubernetes.io/component: coil-router 16 | spec: 17 | hostNetwork: true 18 | priorityClassName: system-node-critical 19 | tolerations: 20 | - effect: NoSchedule 21 | operator: Exists 22 | - effect: NoExecute 23 | operator: Exists 24 | serviceAccountName: coil-router 25 | terminationGracePeriodSeconds: 1 26 | containers: 27 | - name: coil-router 28 | image: coil:dev 29 | command: ["coil-router"] 30 | args: 31 | - --zap-stacktrace-level=panic 32 | env: 33 | - name: COIL_NODE_NAME 34 | valueFrom: 35 | fieldRef: 36 | fieldPath: spec.nodeName 37 | securityContext: 38 | capabilities: 39 | add: ["NET_ADMIN"] 40 | ports: 41 | - name: metrics 42 | containerPort: 9388 43 | protocol: TCP 44 | - name: health 45 | containerPort: 9389 46 | protocol: TCP 47 | resources: 48 | requests: 49 | cpu: 100m 50 | memory: 200Mi 51 | readinessProbe: 52 | httpGet: 53 | path: /readyz 54 | port: health 55 | host: localhost 56 | livenessProbe: 57 | httpGet: 58 | path: /healthz 59 | port: health 60 | host: localhost 61 | -------------------------------------------------------------------------------- /v2/config/pod/coild.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: DaemonSet 3 | metadata: 4 | name: coild 5 | namespace: system 6 | labels: 7 | app.kubernetes.io/component: coild 8 | spec: 9 | selector: 10 | matchLabels: 11 | app.kubernetes.io/component: coild 12 | template: 13 | metadata: 14 | labels: 15 | app.kubernetes.io/component: coild 16 | spec: 17 | hostNetwork: true 18 | hostPID: true # to see netns file under /proc 19 | priorityClassName: system-node-critical 20 | tolerations: 21 | - effect: NoSchedule 22 | operator: Exists 23 | - effect: NoExecute 24 | operator: Exists 25 | serviceAccountName: coild 26 | terminationGracePeriodSeconds: 1 27 | containers: 28 | - name: coild 29 | image: coil:dev 30 | command: ["coild"] 31 | args: 32 | - --zap-stacktrace-level=panic 33 | - --enable-ipam=true 34 | - --enable-egress=true 35 | env: 36 | - name: COIL_NODE_NAME 37 | valueFrom: 38 | fieldRef: 39 | fieldPath: spec.nodeName 40 | securityContext: 41 | privileged: true 42 | ports: 43 | - name: metrics 44 | containerPort: 9384 45 | protocol: TCP 46 | - name: health 47 | containerPort: 9385 48 | protocol: TCP 49 | resources: 50 | requests: 51 | cpu: 100m 52 | memory: 200Mi 53 | readinessProbe: 54 | httpGet: 55 | path: /readyz 56 | port: health 57 | host: localhost 58 | livenessProbe: 59 | httpGet: 60 | path: /healthz 61 | port: health 62 | host: localhost 63 | volumeMounts: 64 | - mountPath: /run 65 | name: run 66 | mountPropagation: HostToContainer # to see bind mount netns file under /run/netns 67 | - mountPath: /lib/modules 68 | name: modules 69 | readOnly: true 70 | initContainers: 71 | - name: coil-installer 72 | image: coil:dev 73 | command: ["coil-installer"] 74 | env: 75 | - name: CNI_NETCONF 76 | valueFrom: 77 | configMapKeyRef: 78 | name: coil-config 79 | key: cni_netconf 80 | securityContext: 81 | privileged: true 82 | volumeMounts: 83 | - mountPath: /host/opt/cni/bin 84 | name: cni-bin-dir 85 | - mountPath: /host/etc/cni/net.d 86 | name: cni-net-dir 87 | volumes: 88 | - name: run 89 | hostPath: 90 | path: /run 91 | - name: modules 92 | hostPath: 93 | path: /lib/modules 94 | - name: cni-bin-dir 95 | hostPath: 96 | path: /opt/cni/bin 97 | - name: cni-net-dir 98 | hostPath: 99 | path: /etc/cni/net.d 100 | -------------------------------------------------------------------------------- /v2/config/pod/compat_calico.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: DaemonSet 3 | metadata: 4 | name: coild 5 | namespace: system 6 | spec: 7 | template: 8 | spec: 9 | containers: 10 | - name: coild 11 | args: ["--compat-calico"] 12 | -------------------------------------------------------------------------------- /v2/config/pod/egress/v4/coild.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: DaemonSet 3 | metadata: 4 | name: coild 5 | namespace: system 6 | labels: 7 | app.kubernetes.io/component: coild 8 | spec: 9 | selector: 10 | matchLabels: 11 | app.kubernetes.io/component: coild 12 | template: 13 | metadata: 14 | labels: 15 | app.kubernetes.io/component: coild 16 | spec: 17 | hostNetwork: true 18 | hostPID: true # to see netns file under /proc 19 | priorityClassName: system-node-critical 20 | tolerations: 21 | - effect: NoSchedule 22 | operator: Exists 23 | - effect: NoExecute 24 | operator: Exists 25 | serviceAccountName: coild 26 | terminationGracePeriodSeconds: 1 27 | containers: 28 | - name: coild 29 | image: coil:dev 30 | command: ["coild"] 31 | args: 32 | - --zap-stacktrace-level=panic 33 | - --enable-ipam=false 34 | - --enable-egress=true 35 | - --pod-table-id=0 36 | - --protocol-id=2 37 | env: 38 | - name: COIL_NODE_NAME 39 | valueFrom: 40 | fieldRef: 41 | fieldPath: spec.nodeName 42 | securityContext: 43 | privileged: true 44 | ports: 45 | - name: metrics 46 | containerPort: 9384 47 | protocol: TCP 48 | - name: health 49 | containerPort: 9385 50 | protocol: TCP 51 | resources: 52 | requests: 53 | cpu: 100m 54 | memory: 200Mi 55 | readinessProbe: 56 | httpGet: 57 | path: /readyz 58 | port: health 59 | host: localhost 60 | livenessProbe: 61 | httpGet: 62 | path: /healthz 63 | port: health 64 | host: localhost 65 | volumeMounts: 66 | - mountPath: /run 67 | name: run 68 | mountPropagation: HostToContainer # to see bind mount netns file under /run/netns 69 | - mountPath: /lib/modules 70 | name: modules 71 | readOnly: true 72 | initContainers: 73 | - name: coil-installer 74 | image: coil:dev 75 | command: ["coil-installer"] 76 | env: 77 | - name: CNI_NETCONF 78 | valueFrom: 79 | configMapKeyRef: 80 | name: coil-config 81 | key: cni_netconf 82 | - name: CNI_CONF_NAME 83 | value: "10-coil.conflist" 84 | securityContext: 85 | privileged: true 86 | volumeMounts: 87 | - mountPath: /host/opt/cni/bin 88 | name: cni-bin-dir 89 | - mountPath: /host/etc/cni/net.d 90 | name: cni-net-dir 91 | volumes: 92 | - name: run 93 | hostPath: 94 | path: /run 95 | - name: modules 96 | hostPath: 97 | path: /lib/modules 98 | - name: cni-bin-dir 99 | hostPath: 100 | path: /opt/cni/bin 101 | - name: cni-net-dir 102 | hostPath: 103 | path: /etc/cni/net.d 104 | -------------------------------------------------------------------------------- /v2/config/pod/egress/v4/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | # [EGRESS] Following line should be uncommented to enable Egress NAT features. 3 | - ../../coil-egress-controller.yaml 4 | - coild.yaml 5 | -------------------------------------------------------------------------------- /v2/config/pod/egress/v6/coild.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: DaemonSet 3 | metadata: 4 | name: coild 5 | namespace: system 6 | labels: 7 | app.kubernetes.io/component: coild 8 | spec: 9 | selector: 10 | matchLabels: 11 | app.kubernetes.io/component: coild 12 | template: 13 | metadata: 14 | labels: 15 | app.kubernetes.io/component: coild 16 | spec: 17 | hostNetwork: true 18 | hostPID: true # to see netns file under /proc 19 | priorityClassName: system-node-critical 20 | tolerations: 21 | - effect: NoSchedule 22 | operator: Exists 23 | - effect: NoExecute 24 | operator: Exists 25 | serviceAccountName: coild 26 | terminationGracePeriodSeconds: 1 27 | containers: 28 | - name: coild 29 | image: coil:dev 30 | command: ["coild"] 31 | args: 32 | - --zap-stacktrace-level=panic 33 | - --enable-ipam=false 34 | - --enable-egress=true 35 | - --pod-table-id=255 36 | - --protocol-id=2 37 | env: 38 | - name: COIL_NODE_NAME 39 | valueFrom: 40 | fieldRef: 41 | fieldPath: spec.nodeName 42 | securityContext: 43 | privileged: true 44 | ports: 45 | - name: metrics 46 | containerPort: 9384 47 | protocol: TCP 48 | - name: health 49 | containerPort: 9385 50 | protocol: TCP 51 | resources: 52 | requests: 53 | cpu: 100m 54 | memory: 200Mi 55 | readinessProbe: 56 | httpGet: 57 | path: /readyz 58 | port: health 59 | host: localhost 60 | livenessProbe: 61 | httpGet: 62 | path: /healthz 63 | port: health 64 | host: localhost 65 | volumeMounts: 66 | - mountPath: /run 67 | name: run 68 | mountPropagation: HostToContainer # to see bind mount netns file under /run/netns 69 | - mountPath: /lib/modules 70 | name: modules 71 | readOnly: true 72 | initContainers: 73 | - name: coil-installer 74 | image: coil:dev 75 | command: ["coil-installer"] 76 | env: 77 | - name: CNI_NETCONF 78 | valueFrom: 79 | configMapKeyRef: 80 | name: coil-config 81 | key: cni_netconf 82 | - name: CNI_CONF_NAME 83 | value: "10-coil.conflist" 84 | securityContext: 85 | privileged: true 86 | volumeMounts: 87 | - mountPath: /host/opt/cni/bin 88 | name: cni-bin-dir 89 | - mountPath: /host/etc/cni/net.d 90 | name: cni-net-dir 91 | volumes: 92 | - name: run 93 | hostPath: 94 | path: /run 95 | - name: modules 96 | hostPath: 97 | path: /lib/modules 98 | - name: cni-bin-dir 99 | hostPath: 100 | path: /opt/cni/bin 101 | - name: cni-net-dir 102 | hostPath: 103 | path: /etc/cni/net.d 104 | -------------------------------------------------------------------------------- /v2/config/pod/egress/v6/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | # [EGRESS] Following line should be uncommented to enable Egress NAT features. 3 | - ../../coil-egress-controller.yaml 4 | - coild.yaml 5 | -------------------------------------------------------------------------------- /v2/config/pod/generate_certs.yaml: -------------------------------------------------------------------------------- 1 | - op: add 2 | path: /spec/template/spec/containers/0/args/- 3 | value: --enable-cert-rotation=true 4 | -------------------------------------------------------------------------------- /v2/config/pod/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | # [IPAM] Next file should be uncommented to enable IPAM features. 3 | - coil-ipam-controller.yaml 4 | # [EGRESS] Following line should be uncommented to enable Egress NAT features. 5 | - coil-egress-controller.yaml 6 | - coild.yaml 7 | -------------------------------------------------------------------------------- /v2/config/rbac/addressblock_viewer_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to view addressblocks. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: coilv2-addressblock-viewer-role 6 | labels: 7 | rbac.authorization.k8s.io/aggregate-to-admin: "true" 8 | rbac.authorization.k8s.io/aggregate-to-edit: "true" 9 | rbac.authorization.k8s.io/aggregate-to-view: "true" 10 | rules: 11 | - apiGroups: 12 | - coil.cybozu.com 13 | resources: 14 | - addressblocks 15 | verbs: 16 | - get 17 | - list 18 | - watch 19 | - apiGroups: 20 | - coil.cybozu.com 21 | resources: 22 | - addressblocks/status 23 | verbs: 24 | - get 25 | -------------------------------------------------------------------------------- /v2/config/rbac/addresspool_viewer_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to view addresspools. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: coilv2-addresspool-viewer-role 6 | labels: 7 | rbac.authorization.k8s.io/aggregate-to-admin: "true" 8 | rbac.authorization.k8s.io/aggregate-to-edit: "true" 9 | rbac.authorization.k8s.io/aggregate-to-view: "true" 10 | rules: 11 | - apiGroups: 12 | - coil.cybozu.com 13 | resources: 14 | - addresspools 15 | verbs: 16 | - get 17 | - list 18 | - watch 19 | - apiGroups: 20 | - coil.cybozu.com 21 | resources: 22 | - addresspools/status 23 | verbs: 24 | - get 25 | -------------------------------------------------------------------------------- /v2/config/rbac/blockrequest_viewer_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to view blockrequests. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: coilv2-blockrequest-viewer-role 6 | labels: 7 | rbac.authorization.k8s.io/aggregate-to-admin: "true" 8 | rbac.authorization.k8s.io/aggregate-to-edit: "true" 9 | rbac.authorization.k8s.io/aggregate-to-view: "true" 10 | rules: 11 | - apiGroups: 12 | - coil.cybozu.com 13 | resources: 14 | - blockrequests 15 | verbs: 16 | - get 17 | - list 18 | - watch 19 | - apiGroups: 20 | - coil.cybozu.com 21 | resources: 22 | - blockrequests/status 23 | verbs: 24 | - get 25 | -------------------------------------------------------------------------------- /v2/config/rbac/coil-egress-controller-certs_role.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: coil-egress-controller 6 | rules: 7 | - apiGroups: 8 | - "" 9 | resources: 10 | - pods 11 | verbs: 12 | - get 13 | - list 14 | - watch 15 | - apiGroups: 16 | - "" 17 | resources: 18 | - secrets 19 | verbs: 20 | - get 21 | - list 22 | - update 23 | - watch 24 | - apiGroups: 25 | - "" 26 | resources: 27 | - serviceaccounts 28 | - services 29 | verbs: 30 | - create 31 | - get 32 | - list 33 | - patch 34 | - update 35 | - watch 36 | - apiGroups: 37 | - admissionregistration.k8s.io 38 | resources: 39 | - mutatingwebhookconfigurations 40 | - validatingwebhookconfigurations 41 | verbs: 42 | - get 43 | - list 44 | - update 45 | - watch 46 | - apiGroups: 47 | - apps 48 | resources: 49 | - deployments 50 | verbs: 51 | - create 52 | - get 53 | - list 54 | - patch 55 | - update 56 | - watch 57 | - apiGroups: 58 | - coil.cybozu.com 59 | resources: 60 | - egresses 61 | verbs: 62 | - get 63 | - list 64 | - watch 65 | - apiGroups: 66 | - coil.cybozu.com 67 | resources: 68 | - egresses/status 69 | verbs: 70 | - get 71 | - patch 72 | - update 73 | - apiGroups: 74 | - policy 75 | resources: 76 | - poddisruptionbudgets 77 | verbs: 78 | - create 79 | - delete 80 | - get 81 | - list 82 | - patch 83 | - update 84 | - watch 85 | - apiGroups: 86 | - rbac.authorization.k8s.io 87 | resources: 88 | - clusterrolebindings 89 | verbs: 90 | - get 91 | - list 92 | - patch 93 | - update 94 | - watch 95 | -------------------------------------------------------------------------------- /v2/config/rbac/coil-egress-controller_role.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: coil-egress-controller 6 | rules: 7 | - apiGroups: 8 | - "" 9 | resources: 10 | - pods 11 | verbs: 12 | - get 13 | - list 14 | - watch 15 | - apiGroups: 16 | - "" 17 | resources: 18 | - serviceaccounts 19 | - services 20 | verbs: 21 | - create 22 | - get 23 | - list 24 | - patch 25 | - update 26 | - watch 27 | - apiGroups: 28 | - apps 29 | resources: 30 | - deployments 31 | verbs: 32 | - create 33 | - get 34 | - list 35 | - patch 36 | - update 37 | - watch 38 | - apiGroups: 39 | - coil.cybozu.com 40 | resources: 41 | - egresses 42 | verbs: 43 | - get 44 | - list 45 | - watch 46 | - apiGroups: 47 | - coil.cybozu.com 48 | resources: 49 | - egresses/status 50 | verbs: 51 | - get 52 | - patch 53 | - update 54 | - apiGroups: 55 | - policy 56 | resources: 57 | - poddisruptionbudgets 58 | verbs: 59 | - create 60 | - delete 61 | - get 62 | - list 63 | - patch 64 | - update 65 | - watch 66 | - apiGroups: 67 | - rbac.authorization.k8s.io 68 | resources: 69 | - clusterrolebindings 70 | verbs: 71 | - get 72 | - list 73 | - patch 74 | - update 75 | - watch 76 | -------------------------------------------------------------------------------- /v2/config/rbac/coil-egress_role.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: coil-egress 6 | rules: 7 | - apiGroups: 8 | - "" 9 | resources: 10 | - pods 11 | verbs: 12 | - get 13 | - list 14 | - watch 15 | -------------------------------------------------------------------------------- /v2/config/rbac/coil-ipam-controller-certs_role.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: coil-ipam-controller 6 | rules: 7 | - apiGroups: 8 | - "" 9 | resources: 10 | - nodes 11 | verbs: 12 | - get 13 | - list 14 | - apiGroups: 15 | - "" 16 | resources: 17 | - secrets 18 | verbs: 19 | - get 20 | - list 21 | - update 22 | - watch 23 | - apiGroups: 24 | - admissionregistration.k8s.io 25 | resources: 26 | - mutatingwebhookconfigurations 27 | - validatingwebhookconfigurations 28 | verbs: 29 | - get 30 | - list 31 | - update 32 | - watch 33 | - apiGroups: 34 | - coil.cybozu.com 35 | resources: 36 | - addressblocks 37 | verbs: 38 | - create 39 | - delete 40 | - get 41 | - list 42 | - patch 43 | - update 44 | - watch 45 | - apiGroups: 46 | - coil.cybozu.com 47 | resources: 48 | - addresspools 49 | verbs: 50 | - get 51 | - list 52 | - patch 53 | - update 54 | - watch 55 | - apiGroups: 56 | - coil.cybozu.com 57 | resources: 58 | - blockrequests 59 | verbs: 60 | - get 61 | - list 62 | - watch 63 | - apiGroups: 64 | - coil.cybozu.com 65 | resources: 66 | - blockrequests/status 67 | verbs: 68 | - get 69 | - patch 70 | - update 71 | -------------------------------------------------------------------------------- /v2/config/rbac/coil-ipam-controller_role.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: coil-ipam-controller 6 | rules: 7 | - apiGroups: 8 | - "" 9 | resources: 10 | - nodes 11 | verbs: 12 | - get 13 | - list 14 | - apiGroups: 15 | - coil.cybozu.com 16 | resources: 17 | - addressblocks 18 | verbs: 19 | - create 20 | - delete 21 | - get 22 | - list 23 | - patch 24 | - update 25 | - watch 26 | - apiGroups: 27 | - coil.cybozu.com 28 | resources: 29 | - addresspools 30 | verbs: 31 | - get 32 | - list 33 | - patch 34 | - update 35 | - watch 36 | - apiGroups: 37 | - coil.cybozu.com 38 | resources: 39 | - blockrequests 40 | verbs: 41 | - get 42 | - list 43 | - watch 44 | - apiGroups: 45 | - coil.cybozu.com 46 | resources: 47 | - blockrequests/status 48 | verbs: 49 | - get 50 | - patch 51 | - update 52 | -------------------------------------------------------------------------------- /v2/config/rbac/coil-router_role.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: coil-router 6 | rules: 7 | - apiGroups: 8 | - "" 9 | resources: 10 | - nodes 11 | verbs: 12 | - list 13 | - apiGroups: 14 | - coil.cybozu.com 15 | resources: 16 | - addressblocks 17 | verbs: 18 | - get 19 | - list 20 | - watch 21 | -------------------------------------------------------------------------------- /v2/config/rbac/coild_role.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: coild 6 | rules: 7 | - apiGroups: 8 | - "" 9 | resources: 10 | - namespaces 11 | - pods 12 | - services 13 | verbs: 14 | - get 15 | - list 16 | - watch 17 | - apiGroups: 18 | - "" 19 | resources: 20 | - nodes 21 | verbs: 22 | - get 23 | - apiGroups: 24 | - coil.cybozu.com 25 | resources: 26 | - addressblocks 27 | verbs: 28 | - delete 29 | - get 30 | - list 31 | - patch 32 | - update 33 | - apiGroups: 34 | - coil.cybozu.com 35 | resources: 36 | - blockrequests 37 | verbs: 38 | - create 39 | - delete 40 | - get 41 | - list 42 | - watch 43 | - apiGroups: 44 | - coil.cybozu.com 45 | resources: 46 | - blockrequests/status 47 | verbs: 48 | - get 49 | - apiGroups: 50 | - coil.cybozu.com 51 | resources: 52 | - egresses 53 | verbs: 54 | - get 55 | - list 56 | - watch 57 | -------------------------------------------------------------------------------- /v2/config/rbac/egress/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - ../serviceaccount.yaml 3 | - ../role_binding.yaml 4 | - ../leader_election_role.yaml 5 | - ../leader_election_role_binding.yaml 6 | - ../coild_role.yaml 7 | # [EGRESS] Following files should be uncommented to enable Egress NAT features. 8 | # [CERTS] Please uncomment 'coil-egress-controller-certs_role.yaml' and 9 | # comment 'coil-egress-controller_role.yaml' if automatic cert generation is being used. 10 | - ../coil-egress-controller_role.yaml 11 | # - ../coil-egress-controller-certs_role.yaml 12 | - ../coil-egress_role.yaml 13 | - ../egress_viewer_role.yaml 14 | -------------------------------------------------------------------------------- /v2/config/rbac/egress_viewer_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to view egresses. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: coilv2-egress-viewer-role 6 | labels: 7 | rbac.authorization.k8s.io/aggregate-to-admin: "true" 8 | rbac.authorization.k8s.io/aggregate-to-edit: "true" 9 | rbac.authorization.k8s.io/aggregate-to-view: "true" 10 | rules: 11 | - apiGroups: 12 | - coil.cybozu.com 13 | resources: 14 | - egresses 15 | verbs: 16 | - get 17 | - list 18 | - watch 19 | - apiGroups: 20 | - coil.cybozu.com 21 | resources: 22 | - egresses/status 23 | verbs: 24 | - get 25 | -------------------------------------------------------------------------------- /v2/config/rbac/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - serviceaccount.yaml 3 | - role_binding.yaml 4 | - leader_election_role.yaml 5 | - leader_election_role_binding.yaml 6 | - coild_role.yaml 7 | 8 | # [IPAM] Following files should be uncommented to enable IPAM features. 9 | # [CERTS] Please uncomment 'coil-egress-controller-certs_role.yaml' and 10 | # comment 'coil-egress-controller_role.yaml' if automatic cert generation is being used. 11 | - coil-ipam-controller_role.yaml 12 | # - coil-ipam-controller-certs_role.yaml 13 | - coil-router_role.yaml 14 | - addressblock_viewer_role.yaml 15 | - addresspool_viewer_role.yaml 16 | - blockrequest_viewer_role.yaml 17 | 18 | # [EGRESS] Following files should be uncommented to enable Egress NAT features. 19 | # [CERTS] Please uncomment 'coil-egress-controller-certs_role.yaml' and 20 | # comment 'coil-egress-controller_role.yaml' if automatic cert generation is being used. 21 | - coil-egress-controller_role.yaml 22 | # - coil-egress-controller-certs_role.yaml 23 | - coil-egress_role.yaml 24 | - egress_viewer_role.yaml 25 | -------------------------------------------------------------------------------- /v2/config/rbac/leader_election_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions to do leader election. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: Role 4 | metadata: 5 | name: coil-leader-election 6 | rules: 7 | - apiGroups: 8 | - "" 9 | - coordination.k8s.io 10 | resources: 11 | - configmaps 12 | - leases 13 | verbs: 14 | - get 15 | - list 16 | - watch 17 | - create 18 | - update 19 | - patch 20 | - delete 21 | - apiGroups: 22 | - "" 23 | resources: 24 | - events 25 | verbs: 26 | - create 27 | - patch 28 | -------------------------------------------------------------------------------- /v2/config/rbac/leader_election_role_binding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: RoleBinding 3 | metadata: 4 | name: coil-leader-election 5 | roleRef: 6 | apiGroup: rbac.authorization.k8s.io 7 | kind: Role 8 | name: coil-leader-election 9 | subjects: 10 | - kind: ServiceAccount 11 | name: coil-ipam-controller 12 | namespace: system 13 | - kind: ServiceAccount 14 | name: coil-egress-controller 15 | namespace: system -------------------------------------------------------------------------------- /v2/config/rbac/role_binding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRoleBinding 3 | metadata: 4 | name: coil-ipam-controller 5 | roleRef: 6 | apiGroup: rbac.authorization.k8s.io 7 | kind: ClusterRole 8 | name: coil-ipam-controller 9 | subjects: 10 | - kind: ServiceAccount 11 | name: coil-ipam-controller 12 | namespace: system 13 | --- 14 | apiVersion: rbac.authorization.k8s.io/v1 15 | kind: ClusterRoleBinding 16 | metadata: 17 | name: coil-egress-controller 18 | roleRef: 19 | apiGroup: rbac.authorization.k8s.io 20 | kind: ClusterRole 21 | name: coil-egress-controller 22 | subjects: 23 | - kind: ServiceAccount 24 | name: coil-egress-controller 25 | namespace: system 26 | --- 27 | apiVersion: rbac.authorization.k8s.io/v1 28 | kind: ClusterRoleBinding 29 | metadata: 30 | name: coild 31 | roleRef: 32 | apiGroup: rbac.authorization.k8s.io 33 | kind: ClusterRole 34 | name: coild 35 | subjects: 36 | - kind: ServiceAccount 37 | name: coild 38 | namespace: system 39 | --- 40 | apiVersion: rbac.authorization.k8s.io/v1 41 | kind: ClusterRoleBinding 42 | metadata: 43 | name: coil-router 44 | roleRef: 45 | apiGroup: rbac.authorization.k8s.io 46 | kind: ClusterRole 47 | name: coil-router 48 | subjects: 49 | - kind: ServiceAccount 50 | name: coil-router 51 | namespace: system 52 | --- 53 | apiVersion: rbac.authorization.k8s.io/v1 54 | kind: ClusterRoleBinding 55 | metadata: 56 | name: coil-egress 57 | roleRef: 58 | apiGroup: rbac.authorization.k8s.io 59 | kind: ClusterRole 60 | name: coil-egress 61 | -------------------------------------------------------------------------------- /v2/config/rbac/serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: coil-ipam-controller 5 | namespace: system 6 | --- 7 | apiVersion: v1 8 | kind: ServiceAccount 9 | metadata: 10 | name: coil-egress-controller 11 | namespace: system 12 | --- 13 | apiVersion: v1 14 | kind: ServiceAccount 15 | metadata: 16 | name: coild 17 | namespace: system 18 | --- 19 | apiVersion: v1 20 | kind: ServiceAccount 21 | metadata: 22 | name: coil-router 23 | namespace: system 24 | --- 25 | -------------------------------------------------------------------------------- /v2/config/samples/coil_v2_addresspool.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: coil.cybozu.com/v2 2 | kind: AddressPool 3 | metadata: 4 | name: addresspool-sample 5 | spec: 6 | # Add fields here 7 | foo: bar 8 | -------------------------------------------------------------------------------- /v2/config/samples/coil_v2_egress.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: coil.cybozu.com/v2 2 | kind: Egress 3 | metadata: 4 | name: egress-sample 5 | spec: 6 | # Add fields here 7 | foo: bar 8 | -------------------------------------------------------------------------------- /v2/config/webhook/egress/kustomization.yaml: -------------------------------------------------------------------------------- 1 | namePrefix: coilv2- 2 | 3 | resources: 4 | - manifests.yaml 5 | - service.yaml 6 | 7 | configurations: 8 | - ../kustomizeconfig.yaml 9 | -------------------------------------------------------------------------------- /v2/config/webhook/egress/manifests.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: admissionregistration.k8s.io/v1 3 | kind: MutatingWebhookConfiguration 4 | metadata: 5 | name: mutating-egress-webhook-configuration 6 | webhooks: 7 | - admissionReviewVersions: 8 | - v1 9 | - v1beta1 10 | clientConfig: 11 | service: 12 | name: egress-webhook-service 13 | namespace: system 14 | path: /mutate-coil-cybozu-com-v2-egress 15 | failurePolicy: Fail 16 | name: megress.kb.io 17 | rules: 18 | - apiGroups: 19 | - coil.cybozu.com 20 | apiVersions: 21 | - v2 22 | operations: 23 | - CREATE 24 | resources: 25 | - egresses 26 | sideEffects: None 27 | --- 28 | apiVersion: admissionregistration.k8s.io/v1 29 | kind: ValidatingWebhookConfiguration 30 | metadata: 31 | name: validating-egress-webhook-configuration 32 | webhooks: 33 | - admissionReviewVersions: 34 | - v1 35 | - v1beta1 36 | clientConfig: 37 | service: 38 | name: egress-webhook-service 39 | namespace: system 40 | path: /validate-coil-cybozu-com-v2-egress 41 | failurePolicy: Fail 42 | name: vegress.kb.io 43 | rules: 44 | - apiGroups: 45 | - coil.cybozu.com 46 | apiVersions: 47 | - v2 48 | operations: 49 | - CREATE 50 | - UPDATE 51 | resources: 52 | - egresses 53 | sideEffects: None 54 | -------------------------------------------------------------------------------- /v2/config/webhook/egress/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: egress-webhook-service 5 | namespace: system 6 | spec: 7 | ports: 8 | - port: 443 9 | targetPort: 9444 10 | protocol: TCP 11 | selector: 12 | app.kubernetes.io/component: coil-egress-controller 13 | -------------------------------------------------------------------------------- /v2/config/webhook/ipam/kustomization.yaml: -------------------------------------------------------------------------------- 1 | namePrefix: coilv2- 2 | 3 | resources: 4 | - manifests.yaml 5 | - service.yaml 6 | 7 | configurations: 8 | - ../kustomizeconfig.yaml 9 | -------------------------------------------------------------------------------- /v2/config/webhook/ipam/manifests.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: admissionregistration.k8s.io/v1 3 | kind: MutatingWebhookConfiguration 4 | metadata: 5 | name: mutating-ipam-webhook-configuration 6 | webhooks: 7 | - admissionReviewVersions: 8 | - v1 9 | - v1beta1 10 | clientConfig: 11 | service: 12 | name: ipam-webhook-service 13 | namespace: system 14 | path: /mutate-coil-cybozu-com-v2-addresspool 15 | failurePolicy: Fail 16 | name: maddresspool.kb.io 17 | rules: 18 | - apiGroups: 19 | - coil.cybozu.com 20 | apiVersions: 21 | - v2 22 | operations: 23 | - CREATE 24 | resources: 25 | - addresspools 26 | sideEffects: None 27 | --- 28 | apiVersion: admissionregistration.k8s.io/v1 29 | kind: ValidatingWebhookConfiguration 30 | metadata: 31 | name: validating-ipam-webhook-configuration 32 | webhooks: 33 | - admissionReviewVersions: 34 | - v1 35 | - v1beta1 36 | clientConfig: 37 | service: 38 | name: ipam-webhook-service 39 | namespace: system 40 | path: /validate-coil-cybozu-com-v2-addresspool 41 | failurePolicy: Fail 42 | name: vaddresspool.kb.io 43 | rules: 44 | - apiGroups: 45 | - coil.cybozu.com 46 | apiVersions: 47 | - v2 48 | operations: 49 | - CREATE 50 | - UPDATE 51 | resources: 52 | - addresspools 53 | sideEffects: None 54 | -------------------------------------------------------------------------------- /v2/config/webhook/ipam/service.yaml: -------------------------------------------------------------------------------- 1 | 2 | apiVersion: v1 3 | kind: Service 4 | metadata: 5 | name: ipam-webhook-service 6 | namespace: system 7 | spec: 8 | ports: 9 | - port: 443 10 | targetPort: 9443 11 | protocol: TCP 12 | selector: 13 | app.kubernetes.io/component: coil-ipam-controller 14 | -------------------------------------------------------------------------------- /v2/config/webhook/kustomizeconfig.yaml: -------------------------------------------------------------------------------- 1 | # the following config is for teaching kustomize where to look at when substituting vars. 2 | # It requires kustomize v2.1.0 or newer to work properly. 3 | nameReference: 4 | - kind: Service 5 | version: v1 6 | fieldSpecs: 7 | - kind: MutatingWebhookConfiguration 8 | group: admissionregistration.k8s.io 9 | path: webhooks/clientConfig/service/name 10 | - kind: ValidatingWebhookConfiguration 11 | group: admissionregistration.k8s.io 12 | path: webhooks/clientConfig/service/name 13 | 14 | namespace: 15 | - kind: MutatingWebhookConfiguration 16 | group: admissionregistration.k8s.io 17 | path: webhooks/clientConfig/service/namespace 18 | create: true 19 | - kind: ValidatingWebhookConfiguration 20 | group: admissionregistration.k8s.io 21 | path: webhooks/clientConfig/service/namespace 22 | create: true 23 | -------------------------------------------------------------------------------- /v2/controllers/addressblock_controller.go: -------------------------------------------------------------------------------- 1 | package controllers 2 | 3 | import ( 4 | "context" 5 | 6 | coilv2 "github.com/cybozu-go/coil/v2/api/v2" 7 | ctrl "sigs.k8s.io/controller-runtime" 8 | "sigs.k8s.io/controller-runtime/pkg/reconcile" 9 | ) 10 | 11 | // AddressBlockReconciler watches AddressBlocks and notifies a channel 12 | type AddressBlockReconciler struct { 13 | Notify chan<- struct{} 14 | } 15 | 16 | var _ reconcile.Reconciler = &AddressBlockReconciler{} 17 | 18 | // +kubebuilder:rbac:groups=coil.cybozu.com,resources=addressblocks,verbs=get;list;watch 19 | 20 | // Reconcile implements Reconciler interface. 21 | func (r *AddressBlockReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { 22 | select { 23 | case r.Notify <- struct{}{}: 24 | default: 25 | } 26 | return ctrl.Result{}, nil 27 | } 28 | 29 | // SetupWithManager registers this with the manager. 30 | func (r *AddressBlockReconciler) SetupWithManager(mgr ctrl.Manager) error { 31 | return ctrl.NewControllerManagedBy(mgr). 32 | For(&coilv2.AddressBlock{}). 33 | Complete(r) 34 | } 35 | -------------------------------------------------------------------------------- /v2/controllers/addressblock_controller_test.go: -------------------------------------------------------------------------------- 1 | package controllers 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "time" 7 | 8 | coilv2 "github.com/cybozu-go/coil/v2/api/v2" 9 | . "github.com/onsi/ginkgo/v2" 10 | . "github.com/onsi/gomega" 11 | ctrl "sigs.k8s.io/controller-runtime" 12 | metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" 13 | ) 14 | 15 | var _ = Describe("AddressBlock reconciler", func() { 16 | ctx := context.Background() 17 | var cancel context.CancelFunc 18 | notifyCh := make(chan struct{}, 100) 19 | 20 | BeforeEach(func() { 21 | ctx, cancel = context.WithCancel(context.TODO()) 22 | mgr, err := ctrl.NewManager(cfg, ctrl.Options{ 23 | Scheme: scheme, 24 | LeaderElection: false, 25 | Metrics: metricsserver.Options{ 26 | BindAddress: "0", 27 | }, 28 | }) 29 | Expect(err).ToNot(HaveOccurred()) 30 | 31 | apr := AddressBlockReconciler{Notify: notifyCh} 32 | err = apr.SetupWithManager(mgr) 33 | Expect(err).ToNot(HaveOccurred()) 34 | 35 | go func() { 36 | err := mgr.Start(ctx) 37 | if err != nil { 38 | panic(err) 39 | } 40 | }() 41 | time.Sleep(100 * time.Millisecond) 42 | }) 43 | 44 | AfterEach(func() { 45 | cancel() 46 | time.Sleep(10 * time.Millisecond) 47 | }) 48 | 49 | It("works as expected", func() { 50 | By("checking the notification upon AddressBlock creation") 51 | b := &coilv2.AddressBlock{} 52 | b.Name = "notify-0" 53 | err := k8sClient.Create(ctx, b) 54 | Expect(err).To(Succeed()) 55 | 56 | Eventually(func() error { 57 | select { 58 | case <-notifyCh: 59 | return nil 60 | default: 61 | time.Sleep(1 * time.Millisecond) 62 | return errors.New("not yet notified") 63 | } 64 | }).Should(Succeed()) 65 | }) 66 | }) 67 | -------------------------------------------------------------------------------- /v2/controllers/addresspool_controller.go: -------------------------------------------------------------------------------- 1 | package controllers 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | apierrors "k8s.io/apimachinery/pkg/api/errors" 8 | "k8s.io/apimachinery/pkg/runtime" 9 | ctrl "sigs.k8s.io/controller-runtime" 10 | "sigs.k8s.io/controller-runtime/pkg/builder" 11 | "sigs.k8s.io/controller-runtime/pkg/client" 12 | "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" 13 | "sigs.k8s.io/controller-runtime/pkg/event" 14 | "sigs.k8s.io/controller-runtime/pkg/log" 15 | "sigs.k8s.io/controller-runtime/pkg/predicate" 16 | "sigs.k8s.io/controller-runtime/pkg/reconcile" 17 | 18 | coilv2 "github.com/cybozu-go/coil/v2/api/v2" 19 | "github.com/cybozu-go/coil/v2/pkg/constants" 20 | "github.com/cybozu-go/coil/v2/pkg/ipam" 21 | ) 22 | 23 | // AddressPoolReconciler watches child AddressBlocks and pool itself for deletion. 24 | type AddressPoolReconciler struct { 25 | client.Client 26 | Scheme *runtime.Scheme 27 | Manager ipam.PoolManager 28 | } 29 | 30 | var _ reconcile.Reconciler = &AddressPoolReconciler{} 31 | 32 | // +kubebuilder:rbac:groups=coil.cybozu.com,resources=addresspools,verbs=get;list;watch;update;patch 33 | // +kubebuilder:rbac:groups=coil.cybozu.com,resources=addressblocks,verbs=get;list;watch 34 | 35 | // Reconcile implements Reconciler interface. 36 | // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.8.3/pkg/reconcile?tab=doc#Reconciler 37 | func (r *AddressPoolReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { 38 | logger := log.FromContext(ctx) 39 | ap := &coilv2.AddressPool{} 40 | err := r.Client.Get(ctx, req.NamespacedName, ap) 41 | 42 | if apierrors.IsNotFound(err) { 43 | logger.Info("dropping address pool from manager") 44 | r.Manager.DropPool(req.Name) 45 | return ctrl.Result{}, nil 46 | } 47 | if err != nil { 48 | return ctrl.Result{}, fmt.Errorf("failed to get address pool: %w", err) 49 | } 50 | 51 | if err := r.Manager.SyncPool(ctx, req.Name); err != nil { 52 | return ctrl.Result{}, fmt.Errorf("SyncPool failed: %w", err) 53 | } 54 | 55 | logger.Info("synchronized") 56 | 57 | if ap.DeletionTimestamp == nil { 58 | return ctrl.Result{}, nil 59 | } 60 | 61 | if !controllerutil.ContainsFinalizer(ap, constants.FinCoil) { 62 | return ctrl.Result{}, nil 63 | } 64 | 65 | used, err := r.Manager.IsUsed(ctx, req.Name) 66 | if err != nil { 67 | return ctrl.Result{}, fmt.Errorf("IsUsed failed: %w", err) 68 | } 69 | if used { 70 | return ctrl.Result{}, nil 71 | } 72 | 73 | controllerutil.RemoveFinalizer(ap, constants.FinCoil) 74 | if err := r.Update(ctx, ap); err != nil { 75 | return ctrl.Result{}, fmt.Errorf("failed to remove finalizer from address pool: %w", err) 76 | } 77 | return ctrl.Result{}, nil 78 | } 79 | 80 | // SetupWithManager registers this with the manager. 81 | func (r *AddressPoolReconciler) SetupWithManager(mgr ctrl.Manager) error { 82 | return ctrl.NewControllerManagedBy(mgr). 83 | For(&coilv2.AddressPool{}). 84 | Owns(&coilv2.AddressBlock{}, builder.WithPredicates(predicate.Funcs{ 85 | // predicate.Funcs returns true by default 86 | CreateFunc: func(event.CreateEvent) bool { 87 | return false 88 | }, 89 | UpdateFunc: func(event.UpdateEvent) bool { 90 | return false 91 | }, 92 | GenericFunc: func(event.GenericEvent) bool { 93 | return false 94 | }, 95 | })). 96 | Complete(r) 97 | } 98 | -------------------------------------------------------------------------------- /v2/controllers/blockrequest_watcher.go: -------------------------------------------------------------------------------- 1 | package controllers 2 | 3 | import ( 4 | "context" 5 | 6 | ctrl "sigs.k8s.io/controller-runtime" 7 | "sigs.k8s.io/controller-runtime/pkg/builder" 8 | "sigs.k8s.io/controller-runtime/pkg/client" 9 | "sigs.k8s.io/controller-runtime/pkg/event" 10 | "sigs.k8s.io/controller-runtime/pkg/log" 11 | "sigs.k8s.io/controller-runtime/pkg/predicate" 12 | 13 | coilv2 "github.com/cybozu-go/coil/v2/api/v2" 14 | "github.com/cybozu-go/coil/v2/pkg/ipam" 15 | ) 16 | 17 | // BlockRequestWatcher watches BlockRequest status on each node. 18 | type BlockRequestWatcher struct { 19 | client.Client 20 | NodeIPAM ipam.NodeIPAM 21 | NodeName string 22 | } 23 | 24 | // +kubebuilder:rbac:groups=coil.cybozu.com,resources=blockrequests,verbs=get;list;watch 25 | // +kubebuilder:rbac:groups=coil.cybozu.com,resources=blockrequests/status,verbs=get 26 | 27 | // Reconcile implements Reconcile interface. 28 | // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.8.3/pkg/reconcile?tab=doc#Watcher 29 | func (r *BlockRequestWatcher) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { 30 | logger := log.FromContext(ctx) 31 | 32 | br := &coilv2.BlockRequest{} 33 | err := r.Client.Get(ctx, req.NamespacedName, br) 34 | 35 | if err != nil { 36 | // as Delete event is ignored, this is unlikely to happen. 37 | logger.Error(err, "failed to get") 38 | return ctrl.Result{}, client.IgnoreNotFound(err) 39 | } 40 | 41 | // The following conditions have been checked in the event filter. 42 | // These are just safeguards. 43 | if br.Spec.NodeName != r.NodeName { 44 | return ctrl.Result{}, nil 45 | } 46 | if len(br.Status.Conditions) == 0 { 47 | return ctrl.Result{}, nil 48 | } 49 | 50 | r.NodeIPAM.Notify(br) 51 | return ctrl.Result{}, nil 52 | } 53 | 54 | // SetupWithManager registers this with the manager. 55 | func (r *BlockRequestWatcher) SetupWithManager(mgr ctrl.Manager) error { 56 | return ctrl.NewControllerManagedBy(mgr). 57 | For(&coilv2.BlockRequest{}, builder.WithPredicates(predicate.Funcs{ 58 | // predicate.Funcs returns true by default 59 | CreateFunc: func(ev event.CreateEvent) bool { 60 | // This needs to be the same as UpdateFunc because 61 | // sometimes updates can be merged into a create event. 62 | req := ev.Object.(*coilv2.BlockRequest) 63 | if req.Spec.NodeName != r.NodeName { 64 | return false 65 | } 66 | return len(req.Status.Conditions) > 0 67 | }, 68 | UpdateFunc: func(ev event.UpdateEvent) bool { 69 | req := ev.ObjectNew.(*coilv2.BlockRequest) 70 | if req.Spec.NodeName != r.NodeName { 71 | return false 72 | } 73 | return len(req.Status.Conditions) > 0 74 | }, 75 | DeleteFunc: func(event.DeleteEvent) bool { 76 | return false 77 | }, 78 | })). 79 | Complete(r) 80 | } 81 | -------------------------------------------------------------------------------- /v2/controllers/blockrequest_watcher_test.go: -------------------------------------------------------------------------------- 1 | package controllers 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | coilv2 "github.com/cybozu-go/coil/v2/api/v2" 8 | . "github.com/onsi/ginkgo/v2" 9 | . "github.com/onsi/gomega" 10 | corev1 "k8s.io/api/core/v1" 11 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 12 | "k8s.io/utils/ptr" 13 | ctrl "sigs.k8s.io/controller-runtime" 14 | "sigs.k8s.io/controller-runtime/pkg/config" 15 | metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" 16 | ) 17 | 18 | var _ = Describe("BlockRequest watcher", func() { 19 | ctx := context.Background() 20 | var cancel context.CancelFunc 21 | var nodeIPAM *mockNodeIPAM 22 | 23 | BeforeEach(func() { 24 | ctx, cancel = context.WithCancel(context.TODO()) 25 | nodeIPAM = &mockNodeIPAM{} 26 | mgr, err := ctrl.NewManager(cfg, ctrl.Options{ 27 | Scheme: scheme, 28 | LeaderElection: false, 29 | Metrics: metricsserver.Options{ 30 | BindAddress: "0", 31 | }, 32 | Controller: config.Controller{ 33 | SkipNameValidation: ptr.To(true), 34 | }, 35 | }) 36 | Expect(err).ToNot(HaveOccurred()) 37 | 38 | brw := &BlockRequestWatcher{ 39 | Client: mgr.GetClient(), 40 | NodeIPAM: nodeIPAM, 41 | NodeName: "node2", 42 | } 43 | err = brw.SetupWithManager(mgr) 44 | Expect(err).ToNot(HaveOccurred()) 45 | 46 | go func() { 47 | err := mgr.Start(ctx) 48 | if err != nil { 49 | panic(err) 50 | } 51 | }() 52 | time.Sleep(100 * time.Millisecond) 53 | }) 54 | 55 | AfterEach(func() { 56 | cancel() 57 | err := k8sClient.DeleteAllOf(context.Background(), &coilv2.BlockRequest{}) 58 | Expect(err).To(Succeed()) 59 | time.Sleep(10 * time.Millisecond) 60 | }) 61 | 62 | It("should notify requests", func() { 63 | By("ignoring a new request") 64 | br := &coilv2.BlockRequest{} 65 | br.Name = "br-2" 66 | br.Spec.NodeName = "node2" 67 | br.Spec.PoolName = "default" 68 | err := k8sClient.Create(ctx, br) 69 | Expect(err).To(Succeed()) 70 | time.Sleep(10 * time.Millisecond) 71 | Expect(nodeIPAM.GetNotified()).To(Equal(0)) 72 | 73 | By("updating the request status and check it notifies quickly") 74 | br.Status.Conditions = []coilv2.BlockRequestCondition{ 75 | { 76 | Type: coilv2.BlockRequestComplete, 77 | Status: corev1.ConditionTrue, 78 | LastProbeTime: metav1.Now(), 79 | LastTransitionTime: metav1.Now(), 80 | }, 81 | } 82 | br.Status.AddressBlockName = "default-0" 83 | err = k8sClient.Status().Update(ctx, br) 84 | Expect(err).To(Succeed()) 85 | 86 | Eventually(func() int { 87 | return nodeIPAM.GetNotified() 88 | }).Should(Equal(1)) 89 | }) 90 | 91 | It("should ignore requests of other nodes", func() { 92 | br := &coilv2.BlockRequest{} 93 | br.Name = "br-2" 94 | br.Spec.NodeName = "node1" 95 | br.Spec.PoolName = "default" 96 | err := k8sClient.Create(ctx, br) 97 | Expect(err).To(Succeed()) 98 | br.Status.Conditions = []coilv2.BlockRequestCondition{ 99 | { 100 | Type: coilv2.BlockRequestComplete, 101 | Status: corev1.ConditionTrue, 102 | LastProbeTime: metav1.Now(), 103 | LastTransitionTime: metav1.Now(), 104 | }, 105 | } 106 | br.Status.AddressBlockName = "default-1" 107 | err = k8sClient.Status().Update(ctx, br) 108 | Expect(err).To(Succeed()) 109 | 110 | Consistently(func() int { 111 | return nodeIPAM.GetNotified() 112 | }).Should(Equal(0)) 113 | }) 114 | }) 115 | -------------------------------------------------------------------------------- /v2/controllers/clusterrolebinding_controller.go: -------------------------------------------------------------------------------- 1 | package controllers 2 | 3 | import ( 4 | "context" 5 | "sort" 6 | 7 | coilv2 "github.com/cybozu-go/coil/v2/api/v2" 8 | "github.com/cybozu-go/coil/v2/pkg/constants" 9 | "github.com/go-logr/logr" 10 | rbacv1 "k8s.io/api/rbac/v1" 11 | "k8s.io/apimachinery/pkg/api/equality" 12 | apierrors "k8s.io/apimachinery/pkg/api/errors" 13 | ctrl "sigs.k8s.io/controller-runtime" 14 | "sigs.k8s.io/controller-runtime/pkg/client" 15 | "sigs.k8s.io/controller-runtime/pkg/log" 16 | "sigs.k8s.io/controller-runtime/pkg/manager" 17 | "sigs.k8s.io/controller-runtime/pkg/predicate" 18 | ) 19 | 20 | // +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=clusterrolebindings,verbs=get;list;watch;update;patch 21 | // +kubebuilder:rbac:groups=coil.cybozu.com,resources=egresses,verbs=get;list;watch 22 | 23 | // SetupCRBReconciler setups ClusterResourceBinding reconciler for coil-ipam-controller and coil-egress-controller. 24 | func SetupCRBReconciler(mgr manager.Manager) error { 25 | r := &crbReconciler{ 26 | Client: mgr.GetClient(), 27 | } 28 | 29 | return ctrl.NewControllerManagedBy(mgr). 30 | For(&rbacv1.ClusterRoleBinding{}). 31 | WithEventFilter(predicate.NewPredicateFuncs(func(object client.Object) bool { 32 | switch object.GetName() { 33 | case constants.CRBEgress, constants.CRBEgressPSP: 34 | return true 35 | } 36 | return false 37 | })). 38 | Complete(r) 39 | } 40 | 41 | type crbReconciler struct { 42 | client.Client 43 | } 44 | 45 | func (r *crbReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { 46 | switch req.Name { 47 | case constants.CRBEgress, constants.CRBEgressPSP: 48 | default: 49 | return ctrl.Result{}, nil 50 | } 51 | 52 | logger := log.FromContext(ctx) 53 | 54 | if err := reconcileCRB(ctx, r.Client, logger, req.Name); err != nil { 55 | logger.Error(err, "failed to reconcile cluster role binding") 56 | return ctrl.Result{}, err 57 | } 58 | 59 | return ctrl.Result{}, nil 60 | } 61 | 62 | func reconcileCRB(ctx context.Context, cl client.Client, log logr.Logger, name string) error { 63 | ignoreNotFound := name == constants.CRBEgressPSP 64 | 65 | crb := &rbacv1.ClusterRoleBinding{} 66 | if err := cl.Get(ctx, client.ObjectKey{Name: name}, crb); err != nil { 67 | if apierrors.IsNotFound(err) && ignoreNotFound { 68 | // PSP resources have not been applied 69 | return nil 70 | } 71 | return err 72 | } 73 | 74 | egresses := &coilv2.EgressList{} 75 | if err := cl.List(ctx, egresses); err != nil { 76 | return err 77 | } 78 | 79 | nsMap := make(map[string]struct{}) 80 | for _, eg := range egresses.Items { 81 | nsMap[eg.Namespace] = struct{}{} 82 | } 83 | 84 | namespaces := make([]string, 0, len(nsMap)) 85 | for k := range nsMap { 86 | namespaces = append(namespaces, k) 87 | } 88 | sort.Strings(namespaces) 89 | 90 | subjects := make([]rbacv1.Subject, len(namespaces)) 91 | for i, n := range namespaces { 92 | subjects[i] = rbacv1.Subject{ 93 | APIGroup: "", 94 | Kind: "ServiceAccount", 95 | Name: constants.SAEgress, 96 | Namespace: n, 97 | } 98 | } 99 | 100 | if equality.Semantic.DeepDerivative(subjects, crb.Subjects) { 101 | return nil 102 | } 103 | 104 | log.Info("updating cluster role binding") 105 | crb.Subjects = subjects 106 | return cl.Update(ctx, crb) 107 | } 108 | -------------------------------------------------------------------------------- /v2/controllers/suite_test.go: -------------------------------------------------------------------------------- 1 | package controllers 2 | 3 | import ( 4 | "context" 5 | "path/filepath" 6 | "testing" 7 | "time" 8 | 9 | . "github.com/onsi/ginkgo/v2" 10 | . "github.com/onsi/gomega" 11 | corev1 "k8s.io/api/core/v1" 12 | rbacv1 "k8s.io/api/rbac/v1" 13 | "k8s.io/apimachinery/pkg/runtime" 14 | clientgoscheme "k8s.io/client-go/kubernetes/scheme" 15 | "k8s.io/client-go/rest" 16 | "sigs.k8s.io/controller-runtime/pkg/client" 17 | "sigs.k8s.io/controller-runtime/pkg/envtest" 18 | logf "sigs.k8s.io/controller-runtime/pkg/log" 19 | "sigs.k8s.io/controller-runtime/pkg/log/zap" 20 | 21 | coilv2 "github.com/cybozu-go/coil/v2/api/v2" 22 | // +kubebuilder:scaffold:imports 23 | ) 24 | 25 | func strPtr(s string) *string { 26 | return &s 27 | } 28 | 29 | // These tests use Ginkgo (BDD-style Go testing framework). Refer to 30 | // http://onsi.github.io/ginkgo/ to learn more about Ginkgo. 31 | 32 | var cfg *rest.Config 33 | var k8sClient client.Client 34 | var testEnv *envtest.Environment 35 | var scheme = runtime.NewScheme() 36 | 37 | func TestAPIs(t *testing.T) { 38 | RegisterFailHandler(Fail) 39 | 40 | SetDefaultEventuallyTimeout(20 * time.Second) 41 | 42 | RunSpecs(t, "Controller Suite") 43 | } 44 | 45 | var _ = BeforeSuite(func() { 46 | logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) 47 | 48 | By("bootstrapping test environment") 49 | testEnv = &envtest.Environment{ 50 | CRDDirectoryPaths: []string{filepath.Join("..", "config", "crd", "bases")}, 51 | ErrorIfCRDPathMissing: true, 52 | } 53 | 54 | var err error 55 | cfg, err = testEnv.Start() 56 | Expect(err).NotTo(HaveOccurred()) 57 | Expect(cfg).NotTo(BeNil()) 58 | 59 | err = clientgoscheme.AddToScheme(scheme) 60 | Expect(err).NotTo(HaveOccurred()) 61 | err = coilv2.AddToScheme(scheme) 62 | Expect(err).NotTo(HaveOccurred()) 63 | 64 | // +kubebuilder:scaffold:scheme 65 | 66 | k8sClient, err = client.New(cfg, client.Options{Scheme: scheme}) 67 | Expect(err).ToNot(HaveOccurred()) 68 | Expect(k8sClient).ToNot(BeNil()) 69 | 70 | // prepare resources 71 | ctx := context.Background() 72 | ap := &coilv2.AddressPool{} 73 | ap.Name = "default" 74 | ap.Spec.BlockSizeBits = 1 75 | ap.Spec.Subnets = []coilv2.SubnetSet{ 76 | {IPv4: strPtr("10.2.0.0/29"), IPv6: strPtr("fd02::0200/125")}, 77 | {IPv4: strPtr("10.3.0.0/30"), IPv6: strPtr("fd02::0300/126")}, 78 | } 79 | err = k8sClient.Create(ctx, ap) 80 | Expect(err).ToNot(HaveOccurred()) 81 | 82 | ap = &coilv2.AddressPool{} 83 | ap.Name = "v4" 84 | ap.Spec.BlockSizeBits = 2 85 | ap.Spec.Subnets = []coilv2.SubnetSet{ 86 | {IPv4: strPtr("10.4.0.0/29")}, 87 | } 88 | err = k8sClient.Create(ctx, ap) 89 | Expect(err).ToNot(HaveOccurred()) 90 | 91 | node1 := &corev1.Node{} 92 | node1.Name = "node1" 93 | err = k8sClient.Create(ctx, node1) 94 | Expect(err).ToNot(HaveOccurred()) 95 | 96 | cr := &rbacv1.ClusterRole{} 97 | cr.Name = "coil-egress" 98 | err = k8sClient.Create(ctx, cr) 99 | Expect(err).ToNot(HaveOccurred()) 100 | 101 | crb := &rbacv1.ClusterRoleBinding{} 102 | crb.Name = "coil-egress" 103 | crb.RoleRef.APIGroup = rbacv1.SchemeGroupVersion.Group 104 | crb.RoleRef.Kind = "ClusterRole" 105 | crb.RoleRef.Name = "coil-egress" 106 | err = k8sClient.Create(ctx, crb) 107 | Expect(err).ToNot(HaveOccurred()) 108 | 109 | ns := &corev1.Namespace{} 110 | ns.Name = "egtest" 111 | err = k8sClient.Create(ctx, ns) 112 | Expect(err).ToNot(HaveOccurred()) 113 | 114 | }) 115 | 116 | var _ = AfterSuite(func() { 117 | By("tearing down the test environment") 118 | err := testEnv.Stop() 119 | Expect(err).ToNot(HaveOccurred()) 120 | }) 121 | -------------------------------------------------------------------------------- /v2/e2e/README.md: -------------------------------------------------------------------------------- 1 | End-to-end test suites 2 | ====================== 3 | 4 | This document describes the strategy, analysis, and implementation of 5 | end-to-end (e2e) tests for Coil. 6 | 7 | - [Strategy](#strategy) 8 | - [Analysis](#analysis) 9 | - [Manifests](#manifests) 10 | - [`coil-ipam-controller`](#coil-ipam-controller) 11 | - [`coil-egress-controller`](#coil-egress-controller) 12 | - [`coild`](#coild) 13 | - [`coil-router`](#coil-router) 14 | - [`coil-egress`](#coil-egress) 15 | - [How to test](#how-to-test) 16 | - [Implementation](#implementation) 17 | 18 | ## Strategy 19 | 20 | Almost all the functions of Coil are well-tested in unit tests and 21 | integration tests. The exceptions are: 22 | 23 | - YAML manifests 24 | - `main` function of each program 25 | - inter-node communication 26 | - Egress NAT over Kubernetes `Service` 27 | 28 | Therefore, it is enough to cover these functions in e2e tests. 29 | 30 | ## Analysis 31 | 32 | ### Manifests 33 | 34 | RBAC should carefully be examined. 35 | The other manifests are mostly tested together with other tests. 36 | 37 | ### `coil-ipam-controller` 38 | 39 | What the `main` function implements are: 40 | 41 | - Leader election 42 | - Admission webhook 43 | - Health probe server 44 | - Metrics server 45 | - Reconciler for BlockRequest 46 | - Garbage collector for orphaned AddressBlock 47 | 48 | ### `coil-egress-controller` 49 | 50 | What the `main` function implements are: 51 | 52 | - Leader election 53 | - Admission webhook 54 | - Health probe server 55 | - Metrics server 56 | - Reconciler for Egress 57 | 58 | ### `coild` 59 | 60 | What the `main` function implements are: 61 | 62 | - Health probe server 63 | - Metrics server 64 | - gRPC server for `coil` 65 | - Route exporter 66 | - Persisting IPAM status between restarts 67 | - Setup egress NAT clients 68 | 69 | ### `coil-router` 70 | 71 | What the `main` function implements are: 72 | 73 | - Health probe server 74 | - Metrics server 75 | - Routing table setup for inter-node communication 76 | 77 | ### `coil-egress` 78 | 79 | - Watcher for client pods 80 | 81 | ## How to test 82 | 83 | Health probe servers can be tested by checking Pod readiness. 84 | 85 | Reconciler for BlockRequest in `coil-ipam-controller`, gRPC server in `coild`, 86 | and routing table setup in `coil-router` can be tested together by 87 | checking if Pods on different nodes can communicate each other. 88 | 89 | Admission webhook can be tested by trying to create an invalid 90 | AddressPool that cannot be checked by OpenAPI validations. 91 | A too narrow subnet is such an example. 92 | 93 | Garbage collector in `coil-ipam-controller` can be tested by creating 94 | orphaned AddressBlock manually. 95 | 96 | Persisting IPAM status in `coild` can be tested by restarting `coild` Pods 97 | and examine the allocation status. 98 | 99 | Egress NAT feature can be tested by running Egress pods on a specific 100 | node and assigning a fake global IP address such as `9.9.9.9/32` to a dummy 101 | interface of the node. That fake address is reachable only from the pods 102 | running on the node because such pods route all the packets to the node. 103 | 104 | Then, run a NAT client pod on another node. If the NAT client can reach 105 | the fake IP address, we can prove that the Egress NAT feature is working. 106 | 107 | Other functions can be tested straightforwardly. 108 | 109 | ## Implementation 110 | 111 | The end-to-end tests are run on [kind, or Kubernetes IN Docker][kind]. 112 | kind can create mutli-node clusters without installing CNI plugin. 113 | 114 | To run e2e tests, prepare Docker and run the following commands. 115 | 116 | ```console 117 | $ make start 118 | $ make install-coil 119 | $ make test 120 | ``` 121 | 122 | You may change the default Kubernetes image for kind with `IMAGE` option. 123 | 124 | ```console 125 | $ make start KUBERNETES_VERSION=1.25.3 126 | ``` 127 | 128 | To stop the cluster, run `make stop`. 129 | 130 | [kind]: https://github.com/kubernetes-sigs/kind 131 | -------------------------------------------------------------------------------- /v2/e2e/coil-ipam-controller_patch.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: coil-ipam-controller 5 | namespace: system 6 | spec: 7 | template: 8 | spec: 9 | containers: 10 | - name: coil-ipam-controller 11 | args: 12 | - "--gc-interval=10s" 13 | -------------------------------------------------------------------------------- /v2/e2e/configs/egress/dualstack/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - ../../../../config/default/egress/v6 3 | 4 | configMapGenerator: 5 | - name: coil-config 6 | namespace: system 7 | files: 8 | - cni_netconf=../../../netconf/netconf-kindnet-dualstack.json 9 | 10 | # Adds namespace to all resources. 11 | namespace: kube-system 12 | 13 | # Labels to add to all resources and selectors. 14 | commonLabels: 15 | app.kubernetes.io/name: coil 16 | 17 | # [CERTS] Following patches should be uncommented if automatic cert generation is used. 18 | # patches: 19 | # - path: ../../../../config/pod/generate_certs.yaml 20 | # target: 21 | # group: apps 22 | # version: v1 23 | # kind: Deployment 24 | # name: coil-egress-controller 25 | -------------------------------------------------------------------------------- /v2/e2e/configs/egress/v4/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - ../../../../config/default/egress/v4 3 | 4 | configMapGenerator: 5 | - name: coil-config 6 | namespace: system 7 | files: 8 | - cni_netconf=../../../netconf/netconf-kindnet-v4.json 9 | 10 | # Adds namespace to all resources. 11 | namespace: kube-system 12 | 13 | # Labels to add to all resources and selectors. 14 | commonLabels: 15 | app.kubernetes.io/name: coil 16 | 17 | # [CERTS] Following patches should be uncommented if automatic cert generation is used. 18 | # patches: 19 | # - path: ../../../../config/pod/generate_certs.yaml 20 | # target: 21 | # group: apps 22 | # version: v1 23 | # kind: Deployment 24 | # name: coil-egress-controller 25 | -------------------------------------------------------------------------------- /v2/e2e/configs/egress/v6/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - ../../../../config/default/egress/v6 3 | 4 | configMapGenerator: 5 | - name: coil-config 6 | namespace: system 7 | files: 8 | - cni_netconf=../../../netconf/netconf-kindnet-v6.json 9 | 10 | # Adds namespace to all resources. 11 | namespace: kube-system 12 | 13 | # Labels to add to all resources and selectors. 14 | commonLabels: 15 | app.kubernetes.io/name: coil 16 | 17 | # [CERTS] Following patches should be uncommented if automatic cert generation is used. 18 | # patches: 19 | # - path: ../../../../config/pod/generate_certs.yaml 20 | # target: 21 | # group: apps 22 | # version: v1 23 | # kind: Deployment 24 | # name: coil-egress-controller 25 | -------------------------------------------------------------------------------- /v2/e2e/controller_test.go: -------------------------------------------------------------------------------- 1 | package e2e 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "net" 7 | 8 | coilv2 "github.com/cybozu-go/coil/v2/api/v2" 9 | . "github.com/onsi/ginkgo/v2" 10 | . "github.com/onsi/gomega" 11 | "github.com/prometheus/common/expfmt" 12 | corev1 "k8s.io/api/core/v1" 13 | ) 14 | 15 | const ( 16 | testIPv6Key = "TEST_IPV6" 17 | testIPv4Key = "TEST_IPV4" 18 | testIPAMKey = "TEST_IPAM" 19 | testEgressKey = "TEST_EGRESS" 20 | ) 21 | 22 | var _ = Describe("coil controllers", func() { 23 | ParseEnv() 24 | if enableIPAMTests { 25 | Context("when the IPAM features are enabled", Ordered, testCoilIPAMController) 26 | } 27 | if enableEgressTests { 28 | Context("when egress feature is enabled", Ordered, testCoilEgressController) 29 | } 30 | }) 31 | 32 | func testCoilIPAMController() { 33 | It("should elect a leader instance of coil-ipam-controller", func() { 34 | kubectlSafe(nil, "-n", "kube-system", "get", "leases", "coil-ipam-leader") 35 | }) 36 | 37 | It("should run the admission webhook", func() { 38 | By("trying to create an invalid AddressPool") 39 | _, err := kubectl(nil, "apply", "-f", "manifests/invalid_pool.yaml") 40 | Expect(err).Should(HaveOccurred()) 41 | }) 42 | 43 | It("should export metrics", func() { 44 | pods := &corev1.PodList{} 45 | getResourceSafe("kube-system", "pods", "", "app.kubernetes.io/component=coil-ipam-controller", pods) 46 | Expect(pods.Items).Should(HaveLen(2)) 47 | 48 | node := pods.Items[0].Spec.NodeName 49 | out, err := runOnNode(node, "curl", "-sf", "http://localhost:9386/metrics") 50 | Expect(err).ShouldNot(HaveOccurred()) 51 | 52 | mfs, err := (&expfmt.TextParser{}).TextToMetricFamilies(bytes.NewReader(out)) 53 | Expect(err).NotTo(HaveOccurred()) 54 | Expect(mfs).NotTo(BeEmpty()) 55 | }) 56 | 57 | It("should run garbage collector", func() { 58 | By("creating an orphaned AddressBlock") 59 | kubectlSafe(nil, "apply", "-f", "manifests/orphaned.yaml") 60 | 61 | By("checking the AddressBlock gets removed") 62 | Eventually(func() interface{} { 63 | abl := &coilv2.AddressBlockList{} 64 | err := getResource("", "addressblocks", "", "coil.cybozu.com/node=coil-worker100", abl) 65 | if err != nil { 66 | return err 67 | } 68 | return abl.Items 69 | }, 20).Should(BeEmpty()) 70 | }) 71 | } 72 | 73 | func testCoilEgressController() { 74 | Context("when the egress features are enabled", func() { 75 | It("should elect a leader instance of coil-egress-controller", func() { 76 | kubectlSafe(nil, "-n", "kube-system", "get", "leases", "coil-egress-leader") 77 | }) 78 | 79 | It("should export metrics", func() { 80 | pods := &corev1.PodList{} 81 | getResourceSafe("kube-system", "pods", "", "app.kubernetes.io/component=coil-egress-controller", pods) 82 | Expect(pods.Items).Should(HaveLen(2)) 83 | 84 | node := pods.Items[0].Spec.NodeName 85 | 86 | for _, ip := range pods.Items[0].Status.PodIPs { 87 | ipAddr := "" 88 | if enableIPv6Tests && net.ParseIP(ip.IP).To16() != nil { 89 | ipAddr = fmt.Sprintf("[%s]", ip.IP) 90 | } 91 | if enableIPv4Tests && net.ParseIP(ip.IP).To4() != nil { 92 | ipAddr = ip.IP 93 | } 94 | if ipAddr != "" { 95 | address := fmt.Sprintf("http://%s:9396/metrics", ipAddr) 96 | 97 | out, err := runOnNode(node, "curl", "-sf", address) 98 | Expect(err).ShouldNot(HaveOccurred()) 99 | 100 | mfs, err := (&expfmt.TextParser{}).TextToMetricFamilies(bytes.NewReader(out)) 101 | Expect(err).NotTo(HaveOccurred()) 102 | Expect(mfs).NotTo(BeEmpty()) 103 | } 104 | } 105 | }) 106 | }) 107 | } 108 | -------------------------------------------------------------------------------- /v2/e2e/daemon.json: -------------------------------------------------------------------------------- 1 | { 2 | "ipv6": true, 3 | "fixed-cidr-v6": "2001:db8:1::/64" 4 | } 5 | -------------------------------------------------------------------------------- /v2/e2e/echo-server/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "io" 6 | "net" 7 | "net/http" 8 | ) 9 | 10 | type echoHandler struct { 11 | withRemoteAddrReply bool 12 | } 13 | 14 | func (h echoHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { 15 | body, err := io.ReadAll(req.Body) 16 | if err != nil { 17 | http.Error(w, err.Error(), http.StatusBadRequest) 18 | return 19 | } 20 | 21 | w.Header().Set("content-type", "application/octet-stream") 22 | 23 | if h.withRemoteAddrReply { 24 | remote, _, err := net.SplitHostPort(req.RemoteAddr) 25 | if err != nil { 26 | http.Error(w, err.Error(), http.StatusInternalServerError) 27 | return 28 | } 29 | remote += "|" 30 | w.Write([]byte(remote)) 31 | } 32 | w.Write(body) 33 | } 34 | 35 | func main() { 36 | var withRemoteAddress bool 37 | flag.BoolVar(&withRemoteAddress, "reply-remote", false, "if set, echo-server will reply with remote host address (default: false)") 38 | flag.Parse() 39 | 40 | s := &http.Server{ 41 | Handler: echoHandler{ 42 | withRemoteAddrReply: withRemoteAddress, 43 | }, 44 | } 45 | s.ListenAndServe() 46 | } 47 | -------------------------------------------------------------------------------- /v2/e2e/kind-config.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kind.x-k8s.io/v1alpha4 2 | kind: Cluster 3 | networking: 4 | disableDefaultCNI: true 5 | nodes: 6 | - role: control-plane 7 | - role: worker 8 | - role: worker 9 | - role: worker 10 | -------------------------------------------------------------------------------- /v2/e2e/kind-config_dualstack.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kind.x-k8s.io/v1alpha4 2 | kind: Cluster 3 | networking: 4 | ipFamily: dual 5 | disableDefaultCNI: true 6 | nodes: 7 | - role: control-plane 8 | - role: worker 9 | - role: worker 10 | - role: worker 11 | -------------------------------------------------------------------------------- /v2/e2e/kind-config_dualstack_kindnet.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kind.x-k8s.io/v1alpha4 2 | kind: Cluster 3 | networking: 4 | ipFamily: dual 5 | nodes: 6 | - role: control-plane 7 | - role: worker 8 | - role: worker 9 | - role: worker 10 | -------------------------------------------------------------------------------- /v2/e2e/kind-config_dualstack_v6.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kind.x-k8s.io/v1alpha4 2 | kind: Cluster 3 | networking: 4 | ipFamily: dual 5 | podSubnet: "fd00:10:244::/56,10.244.0.0/16" 6 | serviceSubnet: "fd00:10:96::/112,10.96.0.0/16" 7 | disableDefaultCNI: true 8 | nodes: 9 | - role: control-plane 10 | - role: worker 11 | - role: worker 12 | - role: worker 13 | -------------------------------------------------------------------------------- /v2/e2e/kind-config_dualstack_v6_kindnet.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kind.x-k8s.io/v1alpha4 2 | kind: Cluster 3 | networking: 4 | ipFamily: dual 5 | podSubnet: "fd00:10:244::/56,10.244.0.0/16" 6 | serviceSubnet: "fd00:10:96::/112,10.96.0.0/16" 7 | nodes: 8 | - role: control-plane 9 | - role: worker 10 | - role: worker 11 | - role: worker 12 | -------------------------------------------------------------------------------- /v2/e2e/kind-config_kindnet.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kind.x-k8s.io/v1alpha4 2 | kind: Cluster 3 | nodes: 4 | - role: control-plane 5 | - role: worker 6 | - role: worker 7 | - role: worker 8 | -------------------------------------------------------------------------------- /v2/e2e/kind-config_kindnet_v6.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kind.x-k8s.io/v1alpha4 2 | kind: Cluster 3 | networking: 4 | ipFamily: ipv6 5 | nodes: 6 | - role: control-plane 7 | - role: worker 8 | - role: worker 9 | - role: worker 10 | -------------------------------------------------------------------------------- /v2/e2e/kind-config_v6.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kind.x-k8s.io/v1alpha4 2 | kind: Cluster 3 | networking: 4 | ipFamily: ipv6 5 | disableDefaultCNI: true 6 | nodes: 7 | - role: control-plane 8 | - role: worker 9 | - role: worker 10 | - role: worker 11 | -------------------------------------------------------------------------------- /v2/e2e/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - ../config/default 3 | - ../config/pod/coil-router.yaml 4 | 5 | patchesStrategicMerge: 6 | - coil-ipam-controller_patch.yaml 7 | 8 | # [CERTS] Following patches should be uncommented if automatic cert generation is used. 9 | # patches: 10 | # - path: ../config/pod/generate_certs.yaml 11 | # target: 12 | # group: apps 13 | # version: v1 14 | # kind: Deployment 15 | # name: coil-ipam-controller 16 | # - path: ../config/pod/generate_certs.yaml 17 | # target: 18 | # group: apps 19 | # version: v1 20 | # kind: Deployment 21 | # name: coil-egress-controller 22 | 23 | configMapGenerator: 24 | - name: coil-config 25 | namespace: system 26 | files: 27 | - cni_netconf=../netconf.json 28 | 29 | # Adds namespace to all resources. 30 | namespace: kube-system 31 | 32 | # Labels to add to all resources and selectors. 33 | commonLabels: 34 | app.kubernetes.io/name: coil 35 | -------------------------------------------------------------------------------- /v2/e2e/manifests/another_httpd.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: another-httpd 5 | namespace: default 6 | labels: 7 | name: httpd 8 | spec: 9 | tolerations: 10 | - key: test 11 | operator: Exists 12 | nodeSelector: 13 | test: coil 14 | containers: 15 | - name: httpd 16 | image: ghcr.io/cybozu/testhttpd:0 17 | -------------------------------------------------------------------------------- /v2/e2e/manifests/default_pool.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: coil.cybozu.com/v2 2 | kind: AddressPool 3 | metadata: 4 | name: default 5 | spec: 6 | blockSizeBits: 0 7 | subnets: 8 | - ipv4: 10.244.0.0/24 9 | -------------------------------------------------------------------------------- /v2/e2e/manifests/default_pool_dualstack.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: coil.cybozu.com/v2 2 | kind: AddressPool 3 | metadata: 4 | name: default 5 | spec: 6 | blockSizeBits: 0 7 | subnets: 8 | - ipv4: 10.244.0.0/24 9 | ipv6: fd00:10:244::/120 10 | -------------------------------------------------------------------------------- /v2/e2e/manifests/default_pool_v6.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: coil.cybozu.com/v2 2 | kind: AddressPool 3 | metadata: 4 | name: default 5 | spec: 6 | blockSizeBits: 0 7 | subnets: 8 | - ipv6: fd00:10:244::/120 9 | -------------------------------------------------------------------------------- /v2/e2e/manifests/dummy_pod.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: dummy 5 | namespace: default 6 | annotations: 7 | egress.coil.cybozu.com/internet: egress-sport-auto 8 | spec: 9 | tolerations: 10 | - key: test 11 | operator: Exists 12 | nodeSelector: 13 | test: coil 14 | kubernetes.io/hostname: coil-worker 15 | containers: 16 | - name: ubuntu 17 | image: ghcr.io/cybozu/ubuntu-debug:22.04 18 | command: ["pause"] 19 | -------------------------------------------------------------------------------- /v2/e2e/manifests/dummy_pool.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: coil.cybozu.com/v2 2 | kind: AddressPool 3 | metadata: 4 | name: dummy 5 | spec: 6 | blockSizeBits: 0 7 | subnets: 8 | - ipv4: 10.225.0.0/30 9 | -------------------------------------------------------------------------------- /v2/e2e/manifests/egress-sport-auto.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: internet 5 | --- 6 | apiVersion: coil.cybozu.com/v2 7 | kind: Egress 8 | metadata: 9 | name: egress-sport-auto 10 | namespace: internet 11 | spec: 12 | replicas: 2 13 | destinations: 14 | - 0.0.0.0/0 15 | - ::/0 16 | fouSourcePortAuto: true 17 | template: 18 | spec: 19 | nodeSelector: 20 | kubernetes.io/hostname: coil-control-plane 21 | tolerations: 22 | - effect: NoSchedule 23 | operator: Exists 24 | containers: 25 | - name: egress 26 | -------------------------------------------------------------------------------- /v2/e2e/manifests/egress-updated.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: internet 5 | --- 6 | apiVersion: coil.cybozu.com/v2 7 | kind: Egress 8 | metadata: 9 | name: egress 10 | namespace: internet 11 | spec: 12 | replicas: 2 13 | destinations: 14 | - 9.9.9.9/32 15 | - 2606:4700:4700::9999/128 16 | fouSourcePortAuto: true 17 | template: 18 | spec: 19 | nodeSelector: 20 | kubernetes.io/hostname: coil-control-plane 21 | tolerations: 22 | - effect: NoSchedule 23 | operator: Exists 24 | containers: 25 | - name: egress 26 | -------------------------------------------------------------------------------- /v2/e2e/manifests/egress.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: internet 5 | --- 6 | apiVersion: coil.cybozu.com/v2 7 | kind: Egress 8 | metadata: 9 | name: egress 10 | namespace: internet 11 | spec: 12 | replicas: 2 13 | destinations: 14 | - 0.0.0.0/0 15 | - ::/0 16 | template: 17 | spec: 18 | nodeSelector: 19 | kubernetes.io/hostname: coil-control-plane 20 | tolerations: 21 | - effect: NoSchedule 22 | operator: Exists 23 | containers: 24 | - name: egress 25 | podDisruptionBudget: 26 | maxUnavailable: 1 27 | -------------------------------------------------------------------------------- /v2/e2e/manifests/httpd.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: httpd 5 | namespace: default 6 | labels: 7 | name: httpd 8 | spec: 9 | tolerations: 10 | - key: test 11 | operator: Exists 12 | nodeSelector: 13 | test: coil 14 | containers: 15 | - name: httpd 16 | image: ghcr.io/cybozu/testhttpd:0 17 | -------------------------------------------------------------------------------- /v2/e2e/manifests/invalid_pool.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: coil.cybozu.com/v2 2 | kind: AddressPool 3 | metadata: 4 | name: invalid 5 | spec: 6 | blockSizeBits: 5 7 | subnets: 8 | - ipv4: 10.224.0.0/30 9 | -------------------------------------------------------------------------------- /v2/e2e/manifests/nat-client-sport-auto.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: nat-client-sport-auto 5 | namespace: default 6 | annotations: 7 | egress.coil.cybozu.com/internet: egress-sport-auto 8 | spec: 9 | tolerations: 10 | - key: test 11 | operator: Exists 12 | nodeSelector: 13 | test: coil 14 | kubernetes.io/hostname: coil-worker2 15 | containers: 16 | - name: ubuntu 17 | image: ghcr.io/cybozu/ubuntu-debug:22.04 18 | command: ["pause"] 19 | -------------------------------------------------------------------------------- /v2/e2e/manifests/nat-client.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: nat-client 5 | namespace: default 6 | annotations: 7 | egress.coil.cybozu.com/internet: egress 8 | spec: 9 | tolerations: 10 | - key: test 11 | operator: Exists 12 | nodeSelector: 13 | test: coil 14 | kubernetes.io/hostname: coil-worker 15 | containers: 16 | - name: ubuntu 17 | image: ghcr.io/cybozu/ubuntu-debug:22.04 18 | command: ["pause"] 19 | -------------------------------------------------------------------------------- /v2/e2e/manifests/orphaned.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: coil.cybozu.com/v2 2 | kind: AddressBlock 3 | metadata: 4 | name: default-100 5 | finalizers: ["coil.cybozu.com"] 6 | labels: 7 | coil.cybozu.com/node: coil-worker100 8 | coil.cybozu.com/pool: default 9 | index: 100 10 | ipv4: 10.224.0.100/32 11 | -------------------------------------------------------------------------------- /v2/e2e/manifests/ubuntu.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: ubuntu 5 | namespace: default 6 | labels: 7 | name: ubuntu 8 | spec: 9 | tolerations: 10 | - key: test 11 | operator: Exists 12 | nodeSelector: 13 | test: coil 14 | affinity: 15 | podAntiAffinity: 16 | requiredDuringSchedulingIgnoredDuringExecution: 17 | - labelSelector: 18 | matchExpressions: 19 | - key: name 20 | operator: In 21 | values: ["httpd"] 22 | topologyKey: kubernetes.io/hostname 23 | containers: 24 | - name: ubuntu 25 | image: ghcr.io/cybozu/ubuntu-debug:22.04 26 | command: ["/usr/local/bin/pause"] 27 | -------------------------------------------------------------------------------- /v2/e2e/netconf/netconf-kindnet-dualstack.json: -------------------------------------------------------------------------------- 1 | { 2 | "cniVersion": "0.3.1", 3 | "name": "kindnet", 4 | "plugins": [ 5 | { 6 | "type": "ptp", 7 | "ipMasq": false, 8 | "ipam": { 9 | "type": "host-local", 10 | "dataDir": "/run/cni-ipam-state", 11 | "routes": [ 12 | { "dst": "0.0.0.0/0" } 13 | , 14 | { "dst": "::/0" } 15 | ], 16 | "ranges": [ 17 | [ { "subnet": "10.244.0.0/24" } ] 18 | , 19 | [ { "subnet": "fd00:10:244::/64" } ] 20 | ] 21 | } 22 | , 23 | "mtu": 1500 24 | 25 | }, 26 | { 27 | "type": "coil", 28 | "socket": "/run/coild.sock" 29 | }, 30 | { 31 | "type": "portmap", 32 | "capabilities": { 33 | "portMappings": true 34 | } 35 | } 36 | ] 37 | } 38 | -------------------------------------------------------------------------------- /v2/e2e/netconf/netconf-kindnet-v4.json: -------------------------------------------------------------------------------- 1 | { 2 | "cniVersion": "0.3.1", 3 | "name": "kindnet", 4 | "plugins": [ 5 | { 6 | "type": "ptp", 7 | "ipMasq": false, 8 | "ipam": { 9 | "type": "host-local", 10 | "dataDir": "/run/cni-ipam-state", 11 | "routes": [ 12 | { "dst": "0.0.0.0/0" } 13 | ], 14 | "ranges": [ 15 | [ { "subnet": "10.244.0.0/24" } ] 16 | ] 17 | } 18 | , 19 | "mtu": 1500 20 | 21 | }, 22 | { 23 | "type": "coil", 24 | "socket": "/run/coild.sock" 25 | }, 26 | { 27 | "type": "portmap", 28 | "capabilities": { 29 | "portMappings": true 30 | } 31 | } 32 | ] 33 | } -------------------------------------------------------------------------------- /v2/e2e/netconf/netconf-kindnet-v6.json: -------------------------------------------------------------------------------- 1 | { 2 | "cniVersion": "0.3.1", 3 | "name": "kindnet", 4 | "plugins": [ 5 | { 6 | "type": "ptp", 7 | "ipMasq": false, 8 | "ipam": { 9 | "type": "host-local", 10 | "dataDir": "/run/cni-ipam-state", 11 | "routes": [ 12 | { "dst": "::/0" } 13 | ], 14 | "ranges": [ 15 | [ { "subnet": "fd00:10:244::/64" } ] 16 | ] 17 | }, 18 | "mtu": 1500 19 | }, 20 | { 21 | "type": "coil", 22 | "socket": "/run/coild.sock" 23 | }, 24 | { 25 | "type": "portmap", 26 | "capabilities": { 27 | "portMappings": true 28 | } 29 | } 30 | ] 31 | } 32 | -------------------------------------------------------------------------------- /v2/e2e/suite_test.go: -------------------------------------------------------------------------------- 1 | package e2e 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "os" 7 | "os/exec" 8 | "testing" 9 | "time" 10 | 11 | . "github.com/onsi/ginkgo/v2" 12 | . "github.com/onsi/gomega" 13 | ) 14 | 15 | func TestE2e(t *testing.T) { 16 | if kubectlCmd == "" { 17 | t.Skip("Use make to run e2e tests") 18 | } 19 | RegisterFailHandler(Fail) 20 | SetDefaultEventuallyTimeout(5 * time.Minute) 21 | SetDefaultEventuallyPollingInterval(1 * time.Second) 22 | RunSpecs(t, "E2e Suite") 23 | } 24 | 25 | var kubectlCmd = os.Getenv("KUBECTL") 26 | 27 | func kubectl(input []byte, args ...string) (stdout []byte, err error) { 28 | if kubectlCmd == "" { 29 | panic("KUBECTL environment variable must be set") 30 | } 31 | 32 | cmd := exec.Command(kubectlCmd, args...) 33 | if input != nil { 34 | cmd.Stdin = bytes.NewReader(input) 35 | } 36 | 37 | return cmd.Output() 38 | } 39 | 40 | func kubectlSafe(input []byte, args ...string) []byte { 41 | stdout, err := kubectl(input, args...) 42 | ExpectWithOffset(1, err).ShouldNot(HaveOccurred()) 43 | return stdout 44 | } 45 | 46 | // ns, name, label are optional. If name is empty, obj must be a list type. 47 | func getResource(ns, resource, name, label string, obj interface{}) error { 48 | var args []string 49 | if ns != "" { 50 | args = append(args, "-n", ns) 51 | } 52 | args = append(args, "get", resource) 53 | if name != "" { 54 | args = append(args, name) 55 | } 56 | if label != "" { 57 | args = append(args, "-l", label) 58 | } 59 | args = append(args, "-o", "json") 60 | data, err := kubectl(nil, args...) 61 | if err != nil { 62 | return err 63 | } 64 | return json.Unmarshal(data, obj) 65 | } 66 | 67 | // ns, name, label are optional. If name is empty, obj must be a list type. 68 | func getResourceSafe(ns, resource, name, label string, obj interface{}) { 69 | err := getResource(ns, resource, name, label, obj) 70 | ExpectWithOffset(1, err).ShouldNot(HaveOccurred()) 71 | } 72 | 73 | func runOnNode(node, cmd string, args ...string) (stdout []byte, err error) { 74 | dockerArgs := append([]string{"exec", node, cmd}, args...) 75 | return exec.Command("docker", dockerArgs...).Output() 76 | } 77 | -------------------------------------------------------------------------------- /v2/hack/boilerplate.go.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cybozu-go/coil/314fc3a333147b456b0e84a6348c49edb5d94579/v2/hack/boilerplate.go.txt -------------------------------------------------------------------------------- /v2/kustomization.yaml: -------------------------------------------------------------------------------- 1 | images: 2 | - name: coil 3 | newTag: 2.10.1 4 | newName: ghcr.io/cybozu-go/coil 5 | 6 | resources: 7 | - config/default 8 | # If you are using CKE (github.com/cybozu-go/cke) and want to use 9 | # its webhook installation feature, comment the above line and 10 | # uncomment the below line. 11 | #- config/cke 12 | 13 | # If you want to enable coil-router, uncomment the following line. 14 | # Note that coil-router can work only for clusters where all the 15 | # nodes are in a flat L2 network. 16 | #- config/pod/coil-router.yaml 17 | 18 | patchesStrategicMerge: 19 | # Uncomment the following if you want to run Coil with Calico network policy. 20 | # - config/pod/compat_calico.yaml 21 | 22 | # [CERTS] Following patches should be uncommented if automatic cert generation is used. 23 | # patches: 24 | # - path: config/pod/generate_certs.yaml 25 | # target: 26 | # group: apps 27 | # version: v1 28 | # kind: Deployment 29 | # name: coil-ipam-controller 30 | # - path: config/pod/generate_certs.yaml 31 | # target: 32 | # group: apps 33 | # version: v1 34 | # kind: Deployment 35 | # name: coil-egress-controller 36 | 37 | # Edit netconf.json to customize CNI configurations 38 | configMapGenerator: 39 | - name: coil-config 40 | namespace: system 41 | files: 42 | - cni_netconf=./netconf.json 43 | 44 | # Adds namespace to all resources. 45 | namespace: kube-system 46 | 47 | # Labels to add to all resources and selectors. 48 | commonLabels: 49 | app.kubernetes.io/name: coil 50 | -------------------------------------------------------------------------------- /v2/netconf.json: -------------------------------------------------------------------------------- 1 | { 2 | "cniVersion": "0.4.0", 3 | "name": "k8s-pod-network", 4 | "plugins": [ 5 | { 6 | "type": "coil", 7 | "socket": "/run/coild.sock" 8 | }, 9 | { 10 | "type": "portmap", 11 | "capabilities": { 12 | "portMappings": true 13 | } 14 | } 15 | ] 16 | } 17 | -------------------------------------------------------------------------------- /v2/pkg/cert/cert.go: -------------------------------------------------------------------------------- 1 | package cert 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/open-policy-agent/cert-controller/pkg/rotator" 8 | "k8s.io/apimachinery/pkg/types" 9 | ctrl "sigs.k8s.io/controller-runtime" 10 | ) 11 | 12 | // +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;update 13 | // +kubebuilder:rbac:groups=admissionregistration.k8s.io,resources=mutatingwebhookconfigurations;validatingwebhookconfigurations,verbs=get;list;watch;update 14 | 15 | func SetupRotator(mgr ctrl.Manager, objectType string, enableRestartOnCertRefresh bool, setupFinished chan struct{}) (chan struct{}, error) { 16 | webhooks := []rotator.WebhookInfo{ 17 | { 18 | Name: fmt.Sprintf("coilv2-validating-%s-webhook-configuration", objectType), 19 | Type: rotator.Validating, 20 | }, 21 | { 22 | Name: fmt.Sprintf("coilv2-mutating-%s-webhook-configuration", objectType), 23 | Type: rotator.Mutating, 24 | }, 25 | } 26 | 27 | podNamespace := "kube-system" 28 | serviceName := fmt.Sprintf("coilv2-%s-webhook-service", objectType) 29 | secretName := fmt.Sprintf("coilv2-%s-webhook-server-cert", objectType) 30 | 31 | if err := rotator.AddRotator(mgr, &rotator.CertRotator{ 32 | SecretKey: types.NamespacedName{ 33 | Namespace: podNamespace, 34 | Name: secretName, 35 | }, 36 | CertDir: "/certs", 37 | CAName: fmt.Sprintf("coil-%s-ca", objectType), 38 | CAOrganization: "coil", 39 | DNSName: fmt.Sprintf("%s.%s.svc", serviceName, podNamespace), 40 | ExtraDNSNames: []string{fmt.Sprintf("%s.%s.svc.cluster.local", serviceName, podNamespace)}, 41 | IsReady: setupFinished, 42 | RequireLeaderElection: true, 43 | Webhooks: webhooks, 44 | RestartOnSecretRefresh: enableRestartOnCertRefresh, 45 | }); err != nil { 46 | return nil, fmt.Errorf("unable to set up cert rotation: %w", err) 47 | } 48 | 49 | return setupFinished, nil 50 | } 51 | 52 | func WaitForExit(setupErr, mgrErr chan error, cancel context.CancelFunc) error { 53 | logger := ctrl.Log.WithName("coil") 54 | for { 55 | select { 56 | case err := <-setupErr: 57 | if err != nil { 58 | logger.Error(err, "unable to setup reconcilers") 59 | cancel() // if error occurred during setup cancel manager's context 60 | } 61 | case err := <-mgrErr: 62 | if err != nil { 63 | return fmt.Errorf("manager error: %w", err) 64 | } 65 | return nil 66 | } 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /v2/pkg/cnirpc/cni.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package pkg.cnirpc; 3 | 4 | import "google/protobuf/empty.proto"; 5 | 6 | option go_package = "github.com/cybozu-go/coil/v2/pkg/cnirpc"; 7 | 8 | // CNIArgs is a mirror of cni.pkg.skel.CmdArgs struct. 9 | // https://pkg.go.dev/github.com/containernetworking/cni@v0.8.0/pkg/skel?tab=doc#CmdArgs 10 | message CNIArgs { 11 | string container_id = 1; 12 | string netns = 2; 13 | string ifname = 3; 14 | map args = 4; // Key-Value pairs parsed from CNI_ARGS 15 | string path = 5; 16 | bytes stdin_data = 6; 17 | repeated string ips = 7; 18 | map interfaces = 8; 19 | } 20 | 21 | // ErrorCode enumerates errors for CNIError 22 | enum ErrorCode { 23 | UNKNOWN = 0; 24 | INCOMPATIBLE_CNI_VERSION = 1; 25 | UNSUPPORTED_FIELD = 2; 26 | UNKNOWN_CONTAINER = 3; 27 | INVALID_ENVIRONMENT_VARIABLES = 4; 28 | IO_FAILURE = 5; 29 | DECODING_FAILURE = 6; 30 | INVALID_NETWORK_CONFIG = 7; 31 | TRY_AGAIN_LATER = 11; 32 | INTERNAL = 999; 33 | } 34 | 35 | // CNIError is a mirror of cin.pkg.types.Error struct. 36 | // https://pkg.go.dev/github.com/containernetworking/cni@v0.8.0/pkg/types?tab=doc#Error 37 | // 38 | // This should be added to *grpc.Status by WithDetails() 39 | // https://pkg.go.dev/google.golang.org/grpc@v1.31.0/internal/status?tab=doc#Status.WithDetails 40 | message CNIError { 41 | ErrorCode code = 1; 42 | string msg = 2; 43 | string details = 3; 44 | } 45 | 46 | // AddResponse represents the response for ADD command. 47 | // 48 | // `result` is a types.current.Result serialized into JSON. 49 | // https://pkg.go.dev/github.com/containernetworking/cni@v0.8.0/pkg/types/current?tab=doc#Result 50 | message AddResponse { 51 | bytes result = 1; 52 | } 53 | 54 | // CNI implements CNI commands over gRPC. 55 | service CNI { 56 | rpc Add(CNIArgs) returns (AddResponse); 57 | rpc Del(CNIArgs) returns (google.protobuf.Empty); 58 | rpc Check(CNIArgs) returns (google.protobuf.Empty); 59 | } 60 | -------------------------------------------------------------------------------- /v2/pkg/cnirpc/mock_test.go: -------------------------------------------------------------------------------- 1 | package cnirpc 2 | 3 | import ( 4 | "context" 5 | "net" 6 | "os" 7 | "testing" 8 | "time" 9 | 10 | "google.golang.org/grpc" 11 | "google.golang.org/grpc/codes" 12 | "google.golang.org/grpc/credentials/insecure" 13 | "google.golang.org/grpc/resolver" 14 | "google.golang.org/grpc/status" 15 | "google.golang.org/protobuf/types/known/emptypb" 16 | ) 17 | 18 | type mockServer struct { 19 | UnimplementedCNIServer 20 | 21 | sockName string 22 | } 23 | 24 | func (m *mockServer) Check(context.Context, *CNIArgs) (*emptypb.Empty, error) { 25 | st := status.New(codes.Internal, "aaa") 26 | st, err := st.WithDetails(&CNIError{ 27 | Code: ErrorCode_TRY_AGAIN_LATER, 28 | Msg: "abc", 29 | Details: "detail", 30 | }) 31 | if err != nil { 32 | panic(err) 33 | } 34 | 35 | return nil, st.Err() 36 | } 37 | 38 | func (m *mockServer) Start(ctx context.Context) error { 39 | f, err := os.CreateTemp("", "coild-mock") 40 | if err != nil { 41 | return err 42 | } 43 | sockName := f.Name() 44 | f.Close() 45 | os.Remove(sockName) 46 | 47 | s, err := net.Listen("unix", sockName) 48 | if err != nil { 49 | return err 50 | } 51 | m.sockName = sockName 52 | grpcServer := grpc.NewServer() 53 | RegisterCNIServer(grpcServer, m) 54 | 55 | go func() { 56 | err := grpcServer.Serve(s) 57 | if err != nil { 58 | panic(err) 59 | } 60 | }() 61 | go func(ctx context.Context) { 62 | <-ctx.Done() 63 | grpcServer.Stop() 64 | os.Remove(sockName) 65 | }(ctx) 66 | 67 | return nil 68 | } 69 | 70 | func TestCNIWithMock(t *testing.T) { 71 | s := &mockServer{} 72 | ctx, cancel := context.WithCancel(context.Background()) 73 | defer func() { 74 | cancel() 75 | time.Sleep(100 * time.Millisecond) 76 | }() 77 | 78 | err := s.Start(ctx) 79 | if err != nil { 80 | t.Fatal(err) 81 | } 82 | 83 | dialer := &net.Dialer{} 84 | dialFunc := func(ctx context.Context, a string) (net.Conn, error) { 85 | return dialer.DialContext(ctx, "unix", a) 86 | } 87 | resolver.SetDefaultScheme("passthrough") 88 | conn, err := grpc.NewClient(s.sockName, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithContextDialer(dialFunc)) 89 | if err != nil { 90 | t.Fatal(err) 91 | } 92 | defer conn.Close() 93 | 94 | client := NewCNIClient(conn) 95 | _, err = client.Check(context.Background(), &CNIArgs{}) 96 | if err == nil { 97 | t.Fatal("err is expected") 98 | } 99 | t.Log(err) 100 | 101 | st := status.Convert(err) 102 | if st.Code() != codes.Internal { 103 | t.Error(`st.Code() != codes.Internal`) 104 | } 105 | if st.Message() != "aaa" { 106 | t.Error(`st.Message() != "aaa"`) 107 | } 108 | 109 | details := st.Details() 110 | if len(details) != 1 { 111 | t.Fatal(`len(details) != 1`, len(details)) 112 | } 113 | 114 | cniErr, ok := details[0].(*CNIError) 115 | if !ok { 116 | t.Fatal(`not a CNIError`) 117 | } 118 | 119 | if cniErr.Code != ErrorCode_TRY_AGAIN_LATER { 120 | t.Error(`cniErr.Code != CNIError_ERR_TRY_AGAIN_LATER`, cniErr.Code) 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /v2/pkg/config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "flag" 5 | "time" 6 | 7 | "github.com/cybozu-go/coil/v2/pkg/constants" 8 | "github.com/spf13/cobra" 9 | "k8s.io/klog/v2" 10 | "sigs.k8s.io/controller-runtime/pkg/log/zap" 11 | ) 12 | 13 | type Config struct { 14 | MetricsAddr string 15 | HealthAddr string 16 | PodTableId int 17 | PodRulePrio int 18 | ExportTableId int 19 | ProtocolId int 20 | SocketPath string 21 | CompatCalico bool 22 | EgressPort int 23 | RegisterFromMain bool 24 | ZapOpts zap.Options 25 | EnableIPAM bool 26 | EnableEgress bool 27 | AddressBlockGCInterval time.Duration 28 | } 29 | 30 | func Parse(rootCmd *cobra.Command) *Config { 31 | config := &Config{} 32 | pf := rootCmd.PersistentFlags() 33 | pf.StringVar(&config.MetricsAddr, "metrics-addr", constants.DefautlMetricsAddr, "bind address of metrics endpoint") 34 | pf.StringVar(&config.HealthAddr, "health-addr", constants.DefautlHealthAddr, "bind address of health/readiness probes") 35 | pf.IntVar(&config.PodTableId, "pod-table-id", constants.DefautlPodTableId, "routing table ID to which coild registers routes for Pods") 36 | pf.IntVar(&config.PodRulePrio, "pod-rule-prio", constants.DefautlPodRulePrio, "priority with which the rule for Pod table is inserted") 37 | pf.IntVar(&config.ExportTableId, "export-table-id", constants.DefautlExportTableId, "routing table ID to which coild exports routes") 38 | pf.IntVar(&config.ProtocolId, "protocol-id", constants.DefautlProtocolId, "route author ID") 39 | pf.StringVar(&config.SocketPath, "socket", constants.DefaultSocketPath, "UNIX domain socket path") 40 | pf.BoolVar(&config.CompatCalico, "compat-calico", constants.DefaultCompatCalico, "make veth name compatible with Calico") 41 | pf.IntVar(&config.EgressPort, "egress-port", constants.DefaultEgressPort, "UDP port number for egress NAT") 42 | pf.BoolVar(&config.RegisterFromMain, "register-from-main", constants.DefaultRegisterFromMain, "help migration from Coil 2.0.1") 43 | pf.BoolVar(&config.EnableIPAM, "enable-ipam", constants.DefaultEnableIPAM, "enable IPAM related features") 44 | pf.BoolVar(&config.EnableEgress, "enable-egress", constants.DefaultEnableEgress, "enable Egress related features") 45 | pf.DurationVar(&config.AddressBlockGCInterval, "addressblock-gc-interval", constants.DefaultAddressBlockGCInterval, "interval for address block GC") 46 | 47 | goflags := flag.NewFlagSet("klog", flag.ExitOnError) 48 | klog.InitFlags(goflags) 49 | config.ZapOpts.BindFlags(goflags) 50 | 51 | pf.AddGoFlagSet(goflags) 52 | 53 | return config 54 | } 55 | -------------------------------------------------------------------------------- /v2/pkg/constants/constants.go: -------------------------------------------------------------------------------- 1 | package constants 2 | 3 | import "time" 4 | 5 | // annotation keys 6 | const ( 7 | AnnPool = "coil.cybozu.com/pool" 8 | AnnEgressPrefix = "egress.coil.cybozu.com/" 9 | ) 10 | 11 | // Label keys 12 | const ( 13 | LabelPool = "coil.cybozu.com/pool" 14 | LabelNode = "coil.cybozu.com/node" 15 | LabelRequest = "coil.cybozu.com/request" 16 | LabelReserved = "coil.cybozu.com/reserved" 17 | 18 | LabelAppName = "app.kubernetes.io/name" 19 | LabelAppInstance = "app.kubernetes.io/instance" 20 | LabelAppComponent = "app.kubernetes.io/component" 21 | ) 22 | 23 | // Index keys 24 | const ( 25 | AddressBlockRequestKey = "address-block.request" 26 | PodNodeNameKey = "pod.node-name" 27 | ) 28 | 29 | // Finalizers 30 | const ( 31 | FinCoil = "coil.cybozu.com" 32 | ) 33 | 34 | // Keys in CNI_ARGS 35 | const ( 36 | PodNameKey = "K8S_POD_NAME" 37 | PodNamespaceKey = "K8S_POD_NAMESPACE" 38 | PodContainerKey = "K8S_POD_INFRA_CONTAINER_ID" 39 | ) 40 | 41 | // RBAC resource names 42 | const ( 43 | // SAEgress is the name of the ServiceAccount for coil-egress 44 | SAEgress = "coil-egress" 45 | 46 | // CRBEgress is the name of the ClusterRoleBinding for coil-egress 47 | CRBEgress = "coil-egress" 48 | 49 | // CRBEgressPSP is the name of the ClusterRoleBinding for coil-egress PSP. 50 | CRBEgressPSP = "psp-coil-egress" 51 | ) 52 | 53 | // Environment variables 54 | const ( 55 | EnvNode = "COIL_NODE_NAME" 56 | EnvAddresses = "COIL_POD_ADDRESSES" 57 | EnvPodNamespace = "COIL_POD_NAMESPACE" 58 | EnvPodName = "COIL_POD_NAME" 59 | EnvEgressName = "COIL_EGRESS_NAME" 60 | ) 61 | 62 | // Config flags 63 | const ( 64 | IsChained = "IS_CHAINED" 65 | ) 66 | 67 | // Default config values 68 | const ( 69 | DefautlMetricsAddr = ":9384" 70 | DefautlHealthAddr = ":9385" 71 | DefautlPodTableId = 116 72 | DefautlPodRulePrio = 2000 73 | DefautlExportTableId = 119 74 | DefautlProtocolId = 30 75 | DefaultCompatCalico = false 76 | DefaultEgressPort = 5555 77 | DefaultRegisterFromMain = false 78 | DefaultEnableIPAM = true 79 | DefaultEnableEgress = true 80 | DefaultAddressBlockGCInterval = 5 * time.Minute 81 | 82 | DefaultEnableCertRotation = false 83 | DefaultEnableRestartOnCertRefresh = false 84 | ) 85 | 86 | // MetricsNS is the namespace for Prometheus metrics 87 | const MetricsNS = "coil" 88 | 89 | // Misc 90 | const ( 91 | DefaultPool = "default" 92 | ) 93 | -------------------------------------------------------------------------------- /v2/pkg/constants/paths.go: -------------------------------------------------------------------------------- 1 | package constants 2 | 3 | // DefaultSocketPath is the default UNIX domain socket filename 4 | // for gRPC between coil and coild. 5 | const DefaultSocketPath = "/run/coild.sock" 6 | -------------------------------------------------------------------------------- /v2/pkg/founat/errors.go: -------------------------------------------------------------------------------- 1 | package founat 2 | 3 | import "errors" 4 | 5 | // ErrIPFamilyMismatch is the sentinel error to indicate that FoUTunnel or Egress 6 | // cannot handle the given address because it is not setup for the address family. 7 | var ErrIPFamilyMismatch = errors.New("no matching IP family") 8 | -------------------------------------------------------------------------------- /v2/pkg/founat/nat_test.go: -------------------------------------------------------------------------------- 1 | package founat 2 | 3 | import ( 4 | "fmt" 5 | "net" 6 | "net/http" 7 | "os/exec" 8 | "testing" 9 | "time" 10 | 11 | "github.com/containernetworking/plugins/pkg/ns" 12 | ) 13 | 14 | func TestNAT(t *testing.T) { 15 | t.Parallel() 16 | cNS := getNS(nsClient) 17 | defer cNS.Close() 18 | eNS := getNS(nsEgress) 19 | defer eNS.Close() 20 | targetNS := getNS(nsTarget) 21 | defer targetNS.Close() 22 | 23 | err := cNS.Do(func(ns.NetNS) error { 24 | ft := NewFoUTunnel(5555, net.ParseIP("10.1.1.2"), net.ParseIP("fd01::102"), nil) 25 | if err := ft.Init(); err != nil { 26 | return fmt.Errorf("ft.Init on client failed: %w", err) 27 | } 28 | 29 | nc := NewNatClient(net.ParseIP("10.1.1.2"), net.ParseIP("fd01::102"), nil, nil) 30 | if err := nc.Init(); err != nil { 31 | return fmt.Errorf("nc.Init failed: %w", err) 32 | } 33 | 34 | link4, err := ft.AddPeer(net.ParseIP("10.1.2.2"), true) 35 | if err != nil { 36 | return fmt.Errorf("ft.AddPeer failed for 10.1.2.2: %w", err) 37 | } 38 | link6, err := ft.AddPeer(net.ParseIP("fd01::202"), true) 39 | if err != nil { 40 | return fmt.Errorf("ft.AddPeer failed for fd01::202: %w", err) 41 | } 42 | 43 | err = nc.AddEgress(link4, []*net.IPNet{{IP: net.ParseIP("10.1.3.0"), Mask: net.CIDRMask(24, 32)}}) 44 | if err != nil { 45 | return fmt.Errorf("nc.AddEgress failed for 10.1.3.0/24: %w", err) 46 | } 47 | err = nc.AddEgress(link6, []*net.IPNet{{IP: net.ParseIP("fd01::300"), Mask: net.CIDRMask(120, 128)}}) 48 | if err != nil { 49 | return fmt.Errorf("nc.AddEgress failed for fd01::300/120: %w", err) 50 | } 51 | 52 | return nil 53 | }) 54 | if err != nil { 55 | t.Fatal(err) 56 | } 57 | 58 | err = eNS.Do(func(ns.NetNS) error { 59 | ft := NewFoUTunnel(5555, net.ParseIP("10.1.2.2"), net.ParseIP("fd01::202"), nil) 60 | if err := ft.Init(); err != nil { 61 | return fmt.Errorf("ft.Init on egress failed: %w", err) 62 | } 63 | 64 | egress := NewEgress("eth1", net.ParseIP("10.1.2.2"), net.ParseIP("fd01::202")) 65 | if err := egress.Init(); err != nil { 66 | return fmt.Errorf("egress.Init failed: %w", err) 67 | } 68 | 69 | link4, err := ft.AddPeer(net.ParseIP("10.1.1.2"), true) 70 | if err != nil { 71 | return fmt.Errorf("ft.AddPeer failed for 10.1.1.2: %w", err) 72 | } 73 | link6, err := ft.AddPeer(net.ParseIP("fd01::102"), true) 74 | if err != nil { 75 | return fmt.Errorf("ft.AddPeer failed for fd01::102: %w", err) 76 | } 77 | 78 | if err := egress.AddClient(net.ParseIP("10.1.1.2"), link4); err != nil { 79 | return fmt.Errorf("egress.AddClient failed for 10.1.1.2: %w", err) 80 | } 81 | if err := egress.AddClient(net.ParseIP("fd01::102"), link6); err != nil { 82 | return fmt.Errorf("egress.AddClient failed for 10.1.1.2: %w", err) 83 | } 84 | 85 | return nil 86 | }) 87 | if err != nil { 88 | t.Fatal(err) 89 | } 90 | 91 | go targetNS.Do(func(_ ns.NetNS) error { 92 | s := &http.Server{} 93 | t.Log("httpd running in the target network namespace") 94 | s.ListenAndServe() 95 | return nil 96 | }) 97 | time.Sleep(100 * time.Millisecond) 98 | 99 | err = cNS.Do(func(ns.NetNS) error { 100 | out, err := exec.Command("curl", "http://10.1.3.1").CombinedOutput() 101 | if err != nil { 102 | return fmt.Errorf("curl over fou IPv4 failed: %s, %w", string(out), err) 103 | } 104 | out, err = exec.Command("curl", "http://[fd01::301]").CombinedOutput() 105 | if err != nil { 106 | return fmt.Errorf("curl over fou IPv6 failed: %s, %w", string(out), err) 107 | } 108 | 109 | return nil 110 | }) 111 | if err != nil { 112 | t.Error(err) 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /v2/pkg/founat/setup_test.go: -------------------------------------------------------------------------------- 1 | package founat 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "os/exec" 7 | "path/filepath" 8 | "testing" 9 | 10 | "github.com/containernetworking/plugins/pkg/ns" 11 | ) 12 | 13 | func TestMain(m *testing.M) { 14 | if os.Getuid() != 0 { 15 | os.Exit(0) 16 | } 17 | 18 | setup() 19 | os.Exit(m.Run()) 20 | } 21 | 22 | const ( 23 | nsClient = "nat-client" 24 | nsRouter = "nat-router" 25 | nsEgress = "nat-egress" 26 | nsTarget = "nat-target" 27 | ) 28 | 29 | func getNS(name string) ns.NetNS { 30 | a, err := ns.GetNS(filepath.Join("/run/netns", name)) 31 | if err != nil { 32 | panic(err) 33 | } 34 | return a 35 | } 36 | 37 | func runIP(args ...string) { 38 | out, err := exec.Command("ip", args...).CombinedOutput() 39 | if err != nil { 40 | panic(fmt.Sprintf("%s: %v", string(out), err)) 41 | } 42 | } 43 | 44 | func netnsExec(nsName string, args ...string) { 45 | nargs := append([]string{"netns", "exec", nsName}, args...) 46 | runIP(nargs...) 47 | } 48 | 49 | func setup() { 50 | // setup network as follows: 51 | // client eth0 <-> eth0 router eth1 <-> eth0 egress eth1 <-> eth0 target 52 | runIP("link", "add", "veth-coil", "type", "veth", "peer", "name", "veth-coil2") 53 | runIP("link", "set", "veth-coil", "netns", nsClient, "name", "eth0", "up") 54 | runIP("link", "set", "veth-coil2", "netns", nsRouter, "name", "eth0", "up") 55 | runIP("link", "add", "veth-coil", "type", "veth", "peer", "name", "veth-coil2") 56 | runIP("link", "set", "veth-coil", "netns", nsRouter, "name", "eth1", "up") 57 | runIP("link", "set", "veth-coil2", "netns", nsEgress, "name", "eth0", "up") 58 | runIP("link", "add", "veth-coil", "type", "veth", "peer", "name", "veth-coil2") 59 | runIP("link", "set", "veth-coil", "netns", nsEgress, "name", "eth1", "up") 60 | runIP("link", "set", "veth-coil2", "netns", nsTarget, "name", "eth0", "up") 61 | 62 | // assign IP addresses 63 | // 10.1.1.0/24,fd01::100/120 for client-router 64 | // 10.1.2.0/24,fd01::200/120 for router-egress 65 | // 10.1.3.0/24,fd01::300/120 for egress-target 66 | netnsExec(nsClient, "ip", "a", "add", "10.1.1.2/24", "dev", "eth0") 67 | netnsExec(nsClient, "ip", "a", "add", "fd01::102/120", "dev", "eth0", "nodad") 68 | netnsExec(nsRouter, "ip", "a", "add", "10.1.1.1/24", "dev", "eth0") 69 | netnsExec(nsRouter, "ip", "a", "add", "fd01::101/120", "dev", "eth0", "nodad") 70 | netnsExec(nsRouter, "ip", "a", "add", "10.1.2.1/24", "dev", "eth1") 71 | netnsExec(nsRouter, "ip", "a", "add", "fd01::201/120", "dev", "eth1", "nodad") 72 | netnsExec(nsEgress, "ip", "a", "add", "10.1.2.2/24", "dev", "eth0") 73 | netnsExec(nsEgress, "ip", "a", "add", "fd01::202/120", "dev", "eth0", "nodad") 74 | netnsExec(nsEgress, "ip", "a", "add", "10.1.3.2/24", "dev", "eth1") 75 | netnsExec(nsEgress, "ip", "a", "add", "fd01::302/120", "dev", "eth1", "nodad") 76 | netnsExec(nsTarget, "ip", "a", "add", "10.1.3.1/24", "dev", "eth0") 77 | netnsExec(nsTarget, "ip", "a", "add", "fd01::301/120", "dev", "eth0", "nodad") 78 | 79 | // setup routing 80 | netnsExec(nsRouter, "sysctl", "-w", "net.ipv4.ip_forward=1") 81 | netnsExec(nsRouter, "sysctl", "-w", "net.ipv6.conf.all.forwarding=1") 82 | netnsExec(nsClient, "ip", "route", "add", "default", "via", "10.1.1.1") 83 | netnsExec(nsClient, "ip", "-6", "route", "add", "default", "via", "fd01::101") 84 | netnsExec(nsEgress, "ip", "route", "add", "default", "via", "10.1.2.1") 85 | netnsExec(nsEgress, "ip", "-6", "route", "add", "default", "via", "fd01::201") 86 | netnsExec(nsClient, "ping", "-4", "-c", "1", "10.1.2.2") 87 | netnsExec(nsClient, "ping", "-6", "-c", "1", "fd01::202") 88 | } 89 | -------------------------------------------------------------------------------- /v2/pkg/indexing/indexing.go: -------------------------------------------------------------------------------- 1 | package indexing 2 | 3 | import ( 4 | "context" 5 | coilv2 "github.com/cybozu-go/coil/v2/api/v2" 6 | "github.com/cybozu-go/coil/v2/pkg/constants" 7 | corev1 "k8s.io/api/core/v1" 8 | "sigs.k8s.io/controller-runtime/pkg/client" 9 | "sigs.k8s.io/controller-runtime/pkg/manager" 10 | ) 11 | 12 | // SetupIndexForAddressBlock sets up an indexer for addressBlock. 13 | func SetupIndexForAddressBlock(ctx context.Context, mgr manager.Manager) error { 14 | return mgr.GetFieldIndexer().IndexField(ctx, &coilv2.AddressBlock{}, constants.AddressBlockRequestKey, func(rawObj client.Object) []string { 15 | val := rawObj.GetLabels()[constants.LabelRequest] 16 | if val == "" { 17 | return nil 18 | } 19 | return []string{val} 20 | }) 21 | } 22 | 23 | // SetupIndexForPodByNodeName sets up an indexer for Pod. 24 | func SetupIndexForPodByNodeName(ctx context.Context, mgr manager.Manager) error { 25 | return mgr.GetFieldIndexer().IndexField(ctx, &corev1.Pod{}, constants.PodNodeNameKey, func(rawObj client.Object) []string { 26 | return []string{rawObj.(*corev1.Pod).Spec.NodeName} 27 | }) 28 | } 29 | -------------------------------------------------------------------------------- /v2/pkg/ipam/allocator.go: -------------------------------------------------------------------------------- 1 | package ipam 2 | 3 | import ( 4 | "fmt" 5 | "net" 6 | 7 | "github.com/bits-and-blooms/bitset" 8 | "github.com/cybozu-go/netutil" 9 | ) 10 | 11 | type allocator struct { 12 | ipv4 *net.IPNet 13 | ipv6 *net.IPNet 14 | usage *bitset.BitSet 15 | lastAllocIdx int64 16 | } 17 | 18 | func newAllocator(ipv4, ipv6 *string) *allocator { 19 | a := &allocator{ 20 | lastAllocIdx: -1, 21 | } 22 | if ipv4 != nil { 23 | ip, n, _ := net.ParseCIDR(*ipv4) 24 | if ip.To4() == nil { 25 | panic("ipv4 must be an IPv4 subnet") 26 | } 27 | a.ipv4 = n 28 | ones, bits := n.Mask.Size() 29 | a.usage = bitset.New(uint(1) << (bits - ones)) 30 | } 31 | if ipv6 != nil { 32 | _, n, _ := net.ParseCIDR(*ipv6) 33 | a.ipv6 = n 34 | if a.usage == nil { 35 | ones, bits := n.Mask.Size() 36 | a.usage = bitset.New(uint(1) << (bits - ones)) 37 | } 38 | } 39 | return a 40 | } 41 | 42 | func (a *allocator) isFull() bool { 43 | return a.usage.All() 44 | } 45 | 46 | func (a *allocator) isEmpty() bool { 47 | return a.usage.None() 48 | } 49 | 50 | func (a *allocator) fill() { 51 | for i := uint(0); i < a.usage.Len(); i++ { 52 | a.usage.Set(i) 53 | } 54 | a.lastAllocIdx = int64(a.usage.Len() - 1) 55 | } 56 | 57 | func (a *allocator) register(ipv4, ipv6 net.IP) (uint, bool) { 58 | if a.ipv4 != nil && a.ipv4.Contains(ipv4) { 59 | offset := netutil.IPDiff(a.ipv4.IP, ipv4) 60 | if offset < 0 { 61 | panic(fmt.Sprintf("ip: %v, base: %v, offset: %v", ipv4, a.ipv4.IP, offset)) 62 | } 63 | a.usage.Set(uint(offset)) 64 | a.lastAllocIdx = int64(offset) 65 | return uint(offset), true 66 | } 67 | if a.ipv6 != nil && a.ipv6.Contains(ipv6) { 68 | offset := netutil.IPDiff(a.ipv6.IP, ipv6) 69 | if offset < 0 { 70 | panic(fmt.Sprintf("ip: %v, base: %v, offset: %v", ipv6, a.ipv6.IP, offset)) 71 | } 72 | a.usage.Set(uint(offset)) 73 | a.lastAllocIdx = int64(offset) 74 | return uint(offset), true 75 | } 76 | return 0, false 77 | } 78 | 79 | func (a *allocator) allocate() (ipv4, ipv6 net.IP, idx uint, ok bool) { 80 | // try to get an usable index from the last allocated index 81 | idx, ok = a.usage.NextClear(uint(a.lastAllocIdx + 1)) 82 | if !ok { 83 | // if an usable index is not found, try to get from index 0 84 | if idx, ok = a.usage.NextClear(0); !ok { 85 | return nil, nil, 0, false 86 | } 87 | } 88 | 89 | if a.ipv4 != nil { 90 | ipv4 = netutil.IPAdd(a.ipv4.IP, int64(idx)) 91 | } 92 | if a.ipv6 != nil { 93 | ipv6 = netutil.IPAdd(a.ipv6.IP, int64(idx)) 94 | } 95 | a.usage.Set(idx) 96 | a.lastAllocIdx = int64(idx) 97 | return 98 | } 99 | 100 | func (a *allocator) free(idx uint) { 101 | a.usage.Clear(idx) 102 | } 103 | -------------------------------------------------------------------------------- /v2/pkg/metrics/collector.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "context" 5 | "sync" 6 | "time" 7 | 8 | "sigs.k8s.io/controller-runtime/pkg/log" 9 | ) 10 | 11 | type Collector interface { 12 | Update() error 13 | Name() string 14 | } 15 | 16 | type Runner struct { 17 | collectors []Collector 18 | interval time.Duration 19 | } 20 | 21 | func NewRunner() *Runner { 22 | return &Runner{ 23 | collectors: []Collector{}, 24 | interval: time.Second * 30, 25 | } 26 | } 27 | 28 | func (r *Runner) Register(collector Collector) { 29 | r.collectors = append(r.collectors, collector) 30 | } 31 | 32 | func (r *Runner) Run(ctx context.Context) { 33 | ticker := time.NewTicker(r.interval) 34 | 35 | for { 36 | <-ticker.C 37 | r.collect(ctx) 38 | } 39 | } 40 | 41 | func (r *Runner) collect(ctx context.Context) { 42 | logger := log.FromContext(ctx) 43 | wg := sync.WaitGroup{} 44 | wg.Add(len(r.collectors)) 45 | for _, c := range r.collectors { 46 | go func(c Collector) { 47 | if err := c.Update(); err != nil { 48 | logger.Error(err, "failed to collect metrics", "name", c.Name()) 49 | } 50 | wg.Done() 51 | }(c) 52 | } 53 | 54 | wg.Wait() 55 | } 56 | -------------------------------------------------------------------------------- /v2/pkg/nodenet/route_exporter.go: -------------------------------------------------------------------------------- 1 | package nodenet 2 | 3 | import ( 4 | "fmt" 5 | "net" 6 | "sync" 7 | 8 | "github.com/go-logr/logr" 9 | "github.com/vishvananda/netlink" 10 | ) 11 | 12 | // RouteExporter exports subnets to a Linux kernel routing table. 13 | type RouteExporter interface { 14 | Sync([]*net.IPNet) error 15 | } 16 | 17 | // NewRouteExporter creates a new RouteExporter 18 | func NewRouteExporter(tableId, protocolId int, log logr.Logger) RouteExporter { 19 | return &routeExporter{ 20 | tableId: tableId, 21 | protocolId: netlink.RouteProtocol(protocolId), 22 | log: log, 23 | } 24 | } 25 | 26 | type routeExporter struct { 27 | tableId int 28 | protocolId netlink.RouteProtocol 29 | log logr.Logger 30 | 31 | mu sync.Mutex 32 | } 33 | 34 | func (r *routeExporter) Sync(nets []*net.IPNet) error { 35 | r.mu.Lock() 36 | defer r.mu.Unlock() 37 | 38 | r.log.Info("synchronizing routing table", "table-id", r.tableId) 39 | 40 | h, err := netlink.NewHandle() 41 | if err != nil { 42 | r.log.Error(err, "netlink: failed to open handle") 43 | return fmt.Errorf("netlink: failed to open handle: %w", err) 44 | } 45 | defer h.Close() 46 | 47 | lo, err := h.LinkByName("lo") 48 | if err != nil { 49 | r.log.Error(err, "netlink: failed to get link lo") 50 | return fmt.Errorf("netlink: failed to get link lo: %w", err) 51 | } 52 | 53 | filter := &netlink.Route{Table: r.tableId} 54 | routes, err := h.RouteListFiltered(0, filter, netlink.RT_FILTER_TABLE) 55 | if err != nil { 56 | r.log.Error(err, "netlink: failed to list routes") 57 | return fmt.Errorf("netlink: failed to list routes: %w", err) 58 | } 59 | routeHash := make(map[string]bool) 60 | for _, r := range routes { 61 | if r.Dst != nil { 62 | routeHash[r.Dst.String()] = true 63 | } 64 | } 65 | 66 | // add routes 67 | netHash := make(map[string]bool) 68 | for _, n := range nets { 69 | key := n.String() 70 | netHash[key] = true 71 | if routeHash[key] { 72 | continue 73 | } 74 | 75 | err := h.RouteAdd(&netlink.Route{ 76 | Scope: netlink.SCOPE_UNIVERSE, 77 | Dst: n, 78 | Table: r.tableId, 79 | LinkIndex: lo.Attrs().Index, 80 | Protocol: r.protocolId, 81 | }) 82 | if err != nil { 83 | r.log.Error(err, "netlink: failed to add route", "network", key) 84 | return fmt.Errorf("netlink: failed to add route to %s: %w", key, err) 85 | } 86 | } 87 | 88 | // remove routes 89 | for _, route := range routes { 90 | if route.Dst == nil { 91 | continue 92 | } 93 | key := route.Dst.String() 94 | if netHash[key] { 95 | continue 96 | } 97 | 98 | err := h.RouteDel(&route) 99 | if err != nil { 100 | r.log.Error(err, "netlink: failed to delete route", "route", key) 101 | return fmt.Errorf("netlink: failed to delete route to %s: %w", key, err) 102 | } 103 | } 104 | return nil 105 | } 106 | -------------------------------------------------------------------------------- /v2/pkg/nodenet/route_exporter_test.go: -------------------------------------------------------------------------------- 1 | package nodenet 2 | 3 | import ( 4 | "net" 5 | "os" 6 | "testing" 7 | 8 | "github.com/google/go-cmp/cmp" 9 | "github.com/vishvananda/netlink" 10 | ctrl "sigs.k8s.io/controller-runtime" 11 | ) 12 | 13 | const ( 14 | testTable = 133 15 | testProtocol = 99 16 | ) 17 | 18 | func getRoutes(t *testing.T) map[string]bool { 19 | h, err := netlink.NewHandle() 20 | if err != nil { 21 | t.Fatal(err) 22 | } 23 | defer h.Close() 24 | 25 | filter := &netlink.Route{Table: testTable} 26 | routes, err := h.RouteListFiltered(0, filter, netlink.RT_FILTER_TABLE) 27 | if err != nil { 28 | t.Fatal(err) 29 | } 30 | 31 | result := make(map[string]bool) 32 | for _, route := range routes { 33 | if route.Dst == nil { 34 | continue 35 | } 36 | result[route.Dst.String()] = true 37 | } 38 | return result 39 | } 40 | 41 | func TestRouteExporter(t *testing.T) { 42 | if os.Getuid() != 0 { 43 | t.Skip("need root privilege") 44 | } 45 | 46 | _, n1, _ := net.ParseCIDR("10.2.0.0/27") 47 | _, n2, _ := net.ParseCIDR("10.3.0.0/31") 48 | _, n3, _ := net.ParseCIDR("fd02::0200/123") 49 | _, n4, _ := net.ParseCIDR("fd02::0300/127") 50 | 51 | exporter := NewRouteExporter(testTable, testProtocol, ctrl.Log.WithName("exporter")) 52 | err := exporter.Sync([]*net.IPNet{n1, n2, n3, n4}) 53 | if err != nil { 54 | t.Fatal(err) 55 | } 56 | 57 | routes := getRoutes(t) 58 | if !cmp.Equal(routes, map[string]bool{ 59 | "10.2.0.0/27": true, 60 | "10.3.0.0/31": true, 61 | "fd02::200/123": true, 62 | "fd02::300/127": true, 63 | }) { 64 | t.Error("mismatch1", routes) 65 | } 66 | 67 | err = exporter.Sync([]*net.IPNet{n1, n3}) 68 | if err != nil { 69 | t.Fatal(err) 70 | } 71 | 72 | routes = getRoutes(t) 73 | if !cmp.Equal(routes, map[string]bool{ 74 | "10.2.0.0/27": true, 75 | "fd02::200/123": true, 76 | }) { 77 | t.Error("mismatch2", routes) 78 | } 79 | 80 | err = exporter.Sync([]*net.IPNet{n1, n2, n4}) 81 | if err != nil { 82 | t.Fatal(err) 83 | } 84 | 85 | routes = getRoutes(t) 86 | if !cmp.Equal(routes, map[string]bool{ 87 | "10.2.0.0/27": true, 88 | "10.3.0.0/31": true, 89 | "fd02::300/127": true, 90 | }) { 91 | t.Error("mismatch3", routes) 92 | } 93 | 94 | err = exporter.Sync(nil) 95 | if err != nil { 96 | t.Fatal(err) 97 | } 98 | 99 | routes = getRoutes(t) 100 | if len(routes) != 0 { 101 | t.Error("could not clear routing table") 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /v2/pkg/nodenet/route_syncer.go: -------------------------------------------------------------------------------- 1 | package nodenet 2 | 3 | import ( 4 | "fmt" 5 | "net" 6 | "sync" 7 | 8 | "github.com/go-logr/logr" 9 | "github.com/vishvananda/netlink" 10 | ) 11 | 12 | // GatewayInfo is a set of destination networks for a gateway. 13 | type GatewayInfo struct { 14 | Gateway net.IP 15 | Networks []*net.IPNet 16 | } 17 | 18 | // RouteSyncer is the interface to program direct routing. 19 | type RouteSyncer interface { 20 | // Sync synchronizes the kernel routing table with the given routes. 21 | Sync([]GatewayInfo) error 22 | } 23 | 24 | // NewRouteSyncer creates a DirectRouter that marks routes with protocolId. 25 | // 26 | // protocolId must be different from the ID for NewPodNetwork. 27 | func NewRouteSyncer(protocolId int, log logr.Logger) RouteSyncer { 28 | return &routeSyncer{ 29 | protocolId: netlink.RouteProtocol(protocolId), 30 | log: log, 31 | } 32 | } 33 | 34 | type routeSyncer struct { 35 | protocolId netlink.RouteProtocol 36 | log logr.Logger 37 | 38 | mu sync.Mutex 39 | } 40 | 41 | func (d *routeSyncer) Sync(gis []GatewayInfo) error { 42 | d.mu.Lock() 43 | defer d.mu.Unlock() 44 | 45 | d.log.Info("synchronizing the main routing table", "gateways", len(gis)) 46 | routes, err := netlink.RouteListFiltered(0, &netlink.Route{Protocol: d.protocolId}, netlink.RT_FILTER_PROTOCOL) 47 | if err != nil { 48 | return fmt.Errorf("netlink: failed to list routes: %w", err) 49 | } 50 | 51 | routeMap := make(map[string]*netlink.Route) 52 | for _, gi := range gis { 53 | strIP := gi.Gateway.String() 54 | for _, n := range gi.Networks { 55 | routeMap[strIP+" "+n.String()] = &netlink.Route{ 56 | Dst: n, 57 | Gw: gi.Gateway, 58 | Scope: netlink.SCOPE_UNIVERSE, 59 | Protocol: d.protocolId, 60 | } 61 | } 62 | } 63 | 64 | currentMap := make(map[string]bool) 65 | for _, r := range routes { 66 | key := r.Gw.String() + " " + r.Dst.String() 67 | if _, ok := routeMap[key]; !ok { 68 | if err := netlink.RouteDel(&r); err != nil { 69 | return fmt.Errorf("netlink: failed to delete route: %w", err) 70 | } 71 | continue 72 | } 73 | currentMap[key] = true 74 | } 75 | 76 | for k, v := range routeMap { 77 | if !currentMap[k] { 78 | if err := netlink.RouteAdd(v); err != nil { 79 | return fmt.Errorf("netlink: failed to add route to %s: %w", k, err) 80 | } 81 | d.log.Info("added", "dst", v.Dst.String()) 82 | } 83 | } 84 | 85 | return nil 86 | } 87 | -------------------------------------------------------------------------------- /v2/pkg/test/ip_matcher.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "net" 7 | 8 | "github.com/onsi/gomega/types" 9 | ) 10 | 11 | type equalIP struct { 12 | expected net.IP 13 | } 14 | 15 | // EqualIP is a custom mather of Gomega to assert IP equivalence 16 | func EqualIP(ip net.IP) types.GomegaMatcher { 17 | return equalIP{expected: ip} 18 | } 19 | 20 | func (m equalIP) Match(actual interface{}) (success bool, err error) { 21 | ip, ok := actual.(net.IP) 22 | if !ok { 23 | return false, errors.New("EqualIP matcher expects an net.IP") 24 | } 25 | 26 | return ip.Equal(m.expected), nil 27 | } 28 | 29 | func (m equalIP) FailureMessage(actual interface{}) (message string) { 30 | return fmt.Sprintf(`Expected 31 | %s 32 | to be the same as 33 | %s`, actual, m.expected) 34 | } 35 | 36 | func (m equalIP) NegatedFailureMessage(actual interface{}) (message string) { 37 | return fmt.Sprintf(`Expected 38 | %s 39 | not equal to 40 | %s`, actual, m.expected) 41 | } 42 | -------------------------------------------------------------------------------- /v2/runners/garbage_collector.go: -------------------------------------------------------------------------------- 1 | package runners 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "time" 7 | 8 | coilv2 "github.com/cybozu-go/coil/v2/api/v2" 9 | "github.com/cybozu-go/coil/v2/pkg/constants" 10 | "github.com/go-logr/logr" 11 | corev1 "k8s.io/api/core/v1" 12 | "k8s.io/client-go/util/retry" 13 | "sigs.k8s.io/controller-runtime/pkg/client" 14 | "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" 15 | "sigs.k8s.io/controller-runtime/pkg/manager" 16 | ) 17 | 18 | // NewGarbageCollector creates a manager.Runnable to collect 19 | // orphaned AddressBlocks of deleted nodes. 20 | func NewGarbageCollector(mgr manager.Manager, log logr.Logger, interval time.Duration) manager.Runnable { 21 | return &garbageCollector{ 22 | Client: mgr.GetClient(), 23 | apiReader: mgr.GetAPIReader(), 24 | log: log, 25 | interval: interval, 26 | } 27 | } 28 | 29 | type garbageCollector struct { 30 | client.Client 31 | apiReader client.Reader 32 | log logr.Logger 33 | interval time.Duration 34 | } 35 | 36 | // +kubebuilder:rbac:groups=coil.cybozu.com,resources=addressblocks,verbs=get;list;watch;update;patch;delete 37 | // +kubebuilder:rbac:groups="",resources=nodes,verbs=get;list 38 | 39 | var _ manager.LeaderElectionRunnable = &garbageCollector{} 40 | 41 | // NeedLeaderElection implements manager.LeaderElectionRunnable 42 | func (*garbageCollector) NeedLeaderElection() bool { 43 | return true 44 | } 45 | 46 | // Start starts this runner. This implements manager.Runnable 47 | func (gc *garbageCollector) Start(ctx context.Context) error { 48 | tick := time.NewTicker(gc.interval) 49 | defer tick.Stop() 50 | 51 | for { 52 | select { 53 | case <-ctx.Done(): 54 | return nil 55 | case <-tick.C: 56 | if err := gc.do(context.Background()); err != nil { 57 | return err 58 | } 59 | } 60 | } 61 | } 62 | 63 | func (gc *garbageCollector) do(ctx context.Context) error { 64 | gc.log.Info("start garbage collection") 65 | 66 | blocks := &coilv2.AddressBlockList{} 67 | if err := gc.Client.List(ctx, blocks); err != nil { 68 | return fmt.Errorf("failed to list address blocks: %w", err) 69 | } 70 | 71 | nodes := &corev1.NodeList{} 72 | if err := gc.apiReader.List(ctx, nodes); err != nil { 73 | return fmt.Errorf("failed to list nodes: %w", err) 74 | } 75 | 76 | nodeNames := make(map[string]bool) 77 | for _, n := range nodes.Items { 78 | nodeNames[n.Name] = true 79 | } 80 | 81 | for _, b := range blocks.Items { 82 | n := b.Labels[constants.LabelNode] 83 | if nodeNames[n] { 84 | continue 85 | } 86 | 87 | err := gc.deleteBlock(ctx, b.Name) 88 | if err != nil { 89 | return fmt.Errorf("failed to delete a block: %w", err) 90 | } 91 | 92 | gc.log.Info("deleted an orphan block", "block", b.Name, "node", n) 93 | } 94 | 95 | return nil 96 | } 97 | 98 | func (gc *garbageCollector) deleteBlock(ctx context.Context, name string) error { 99 | // remove finalizer 100 | err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { 101 | b := &coilv2.AddressBlock{} 102 | err := gc.apiReader.Get(ctx, client.ObjectKey{Name: name}, b) 103 | if err != nil { 104 | return client.IgnoreNotFound(err) 105 | } 106 | if !controllerutil.ContainsFinalizer(b, constants.FinCoil) { 107 | return nil 108 | } 109 | controllerutil.RemoveFinalizer(b, constants.FinCoil) 110 | return gc.Client.Update(ctx, b) 111 | }) 112 | if err != nil { 113 | return fmt.Errorf("failed to remove finalizer from %s: %w", name, err) 114 | } 115 | 116 | // delete ignoring notfound error. 117 | b := &coilv2.AddressBlock{} 118 | b.Name = name 119 | return client.IgnoreNotFound(gc.Client.Delete(ctx, b)) 120 | } 121 | -------------------------------------------------------------------------------- /v2/runners/garbage_collector_test.go: -------------------------------------------------------------------------------- 1 | package runners 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "time" 7 | 8 | coilv2 "github.com/cybozu-go/coil/v2/api/v2" 9 | "github.com/cybozu-go/coil/v2/pkg/constants" 10 | . "github.com/onsi/ginkgo/v2" 11 | . "github.com/onsi/gomega" 12 | ctrl "sigs.k8s.io/controller-runtime" 13 | metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" 14 | ) 15 | 16 | var _ = Describe("Garbage collector", func() { 17 | ctx := context.Background() 18 | var cancel context.CancelFunc 19 | 20 | BeforeEach(func() { 21 | ctx, cancel = context.WithCancel(context.TODO()) 22 | mgr, err := ctrl.NewManager(cfg, ctrl.Options{ 23 | Scheme: scheme, 24 | LeaderElection: false, 25 | Metrics: metricsserver.Options{ 26 | BindAddress: "0", 27 | }, 28 | }) 29 | Expect(err).ToNot(HaveOccurred()) 30 | 31 | gc := NewGarbageCollector(mgr, ctrl.Log.WithName("garbage collector"), 3*time.Second) 32 | err = mgr.Add(gc) 33 | Expect(err).ToNot(HaveOccurred()) 34 | 35 | go func() { 36 | err := mgr.Start(ctx) 37 | if err != nil { 38 | panic(err) 39 | } 40 | }() 41 | }) 42 | 43 | AfterEach(func() { 44 | deleteAllAddressBlocks() 45 | cancel() 46 | err := k8sClient.DeleteAllOf(context.Background(), &coilv2.BlockRequest{}) 47 | Expect(err).To(Succeed()) 48 | time.Sleep(10 * time.Millisecond) 49 | }) 50 | 51 | It("should collect orphaned blocks", func() { 52 | block := &coilv2.AddressBlock{} 53 | block.Name = "default-0" 54 | block.Index = 0 55 | block.Labels = map[string]string{ 56 | constants.LabelPool: "default", 57 | constants.LabelNode: "node1", 58 | } 59 | err := k8sClient.Create(ctx, block) 60 | Expect(err).To(Succeed()) 61 | 62 | block = &coilv2.AddressBlock{} 63 | block.Name = "default-1" 64 | block.Index = 1 65 | block.Labels = map[string]string{ 66 | constants.LabelPool: "default", 67 | constants.LabelNode: "node2", 68 | } 69 | err = k8sClient.Create(ctx, block) 70 | Expect(err).To(Succeed()) 71 | 72 | block = &coilv2.AddressBlock{} 73 | block.Name = "v4-0" 74 | block.Index = 0 75 | block.Labels = map[string]string{ 76 | constants.LabelPool: "v4", 77 | constants.LabelNode: "node3", 78 | } 79 | err = k8sClient.Create(ctx, block) 80 | Expect(err).To(Succeed()) 81 | 82 | block = &coilv2.AddressBlock{} 83 | block.Name = "v4-1" 84 | block.Index = 1 85 | block.Labels = map[string]string{ 86 | constants.LabelPool: "v4", 87 | constants.LabelNode: "node1", 88 | } 89 | err = k8sClient.Create(ctx, block) 90 | Expect(err).To(Succeed()) 91 | 92 | Eventually(func() error { 93 | blocks := &coilv2.AddressBlockList{} 94 | err := k8sClient.List(ctx, blocks) 95 | if err != nil { 96 | return err 97 | } 98 | count := 0 99 | for _, b := range blocks.Items { 100 | if b.Labels[constants.LabelNode] != "node1" { 101 | return fmt.Errorf("block for node %s still exists", b.Labels[constants.LabelNode]) 102 | } 103 | count++ 104 | } 105 | 106 | if count != 2 { 107 | return fmt.Errorf("unexpected count of node1 blocks: %d", count) 108 | } 109 | 110 | return nil 111 | }, 5).Should(Succeed()) 112 | }) 113 | }) 114 | -------------------------------------------------------------------------------- /v2/version.go: -------------------------------------------------------------------------------- 1 | package v2 2 | 3 | import ( 4 | "runtime/debug" 5 | "strings" 6 | ) 7 | 8 | const version = "2.10.1" 9 | 10 | // Version returns the semantic versioning string of Coil. 11 | func Version() string { 12 | // Once https://github.com/golang/go/issues/37475 is resolved, 13 | // we can use debug.ReadBuildInfo. 14 | if false { 15 | info, ok := debug.ReadBuildInfo() 16 | if !ok || !strings.HasPrefix(info.Main.Version, "v") { 17 | return "(devel)" 18 | } 19 | return info.Main.Version[1:] 20 | } 21 | 22 | return version 23 | } 24 | --------------------------------------------------------------------------------