├── .dockerignore ├── .github ├── release.yaml └── workflows │ ├── e2e-test.yaml │ ├── release.yaml │ └── unit-test.yaml ├── .gitignore ├── CHANGELOG.md ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── docs ├── bgp.md ├── cni.md ├── design.md ├── fib.md ├── img │ ├── cni-internal.drawio.svg │ ├── kubernetes-cni-model.drawio.svg │ ├── kubernetes-model.drawio.svg │ ├── model.drawio.svg │ └── rib-model.drawio.svg ├── kubernetes.md ├── setup-kubernetes.md └── usage.md ├── e2e ├── Makefile ├── Makefile.versions ├── README.md ├── bgp_test.go ├── fixture.go ├── go.mod ├── go.sum ├── img │ ├── kubernetes-cni-compact.drawio.svg │ ├── kubernetes-cni.drawio.svg │ └── kubernetes.drawio.svg ├── kind-config-disable-cni.yaml ├── kind-config.yaml ├── kubernetes_cni_test.go ├── kubernetes_test.go ├── suite_test.go └── topology │ ├── circle-ibgp.yaml │ ├── circle.yaml │ ├── configs │ ├── circle-ibgp │ │ ├── gobgp0.toml │ │ └── gobgp1.toml │ ├── circle │ │ ├── gobgp0.toml │ │ └── gobgp1.toml │ └── gobgp-basic.toml │ ├── figures │ └── circle.drawio.svg │ ├── frr.yaml │ ├── generator │ ├── go.mod │ ├── go.sum │ └── main.go │ ├── gobgp.yaml │ ├── ibgp.yaml │ ├── kubernetes-cni-compact.yaml.tmpl │ ├── kubernetes-cni.yaml.tmpl │ └── kubernetes.yaml ├── kustomization.yaml ├── manifests ├── base │ ├── crd │ │ └── kustomization.yaml │ ├── kustomization.yaml │ ├── rbac │ │ ├── kustomization.yaml │ │ ├── role.yaml │ │ ├── role_binding.yaml │ │ └── serviceaccount.yaml │ ├── webhook │ │ ├── admission_webhook.yaml │ │ ├── admission_webhook_patch.yaml │ │ ├── admission_webhook_patch.yaml.tmpl │ │ ├── kustomization.yaml │ │ ├── kustomizeconfig.yaml │ │ └── service.yaml │ └── workloads │ │ ├── agent.yaml │ │ ├── controller.yaml │ │ ├── kustomization.yaml │ │ └── speaker.yaml ├── cni │ ├── agent-patch.yaml │ ├── configmap.yaml │ ├── controller-patch.yaml │ ├── kustomization.yaml │ ├── netconf.json │ ├── sample │ │ ├── bgp_peer.yaml │ │ ├── client.yaml │ │ ├── cluster_bgp_spine0.yaml │ │ ├── cluster_bgp_spine1.yaml │ │ ├── deployment.yaml │ │ ├── kustomization.yaml │ │ ├── namespace.yaml │ │ ├── peer_template.yaml │ │ ├── pool.yaml │ │ ├── test_pod.yaml │ │ ├── test_pod2.yaml │ │ ├── test_pod_another_pool.yaml │ │ └── test_pod_in_namespace.yaml │ └── speaker-patch.yaml ├── crd │ └── sart.yaml ├── dual │ ├── agent-patch.yaml │ ├── controller-patch.yaml │ └── kustomization.yaml └── lb │ ├── agent-patch.yaml │ ├── controller-patch.yaml │ ├── kustomization.yaml │ └── sample │ ├── bgp_peer.yaml │ ├── cluster_bgp_a.yaml │ ├── cluster_bgp_b.yaml │ ├── cluster_bgp_c.yaml │ ├── kustomization.yaml │ ├── lb.yaml │ ├── lb_address_pool.yaml │ ├── lb_another.yaml │ └── peer_template.yaml ├── proto ├── bgp.proto ├── cni.proto ├── fib.proto └── fib_manager.proto ├── sart ├── Cargo.lock ├── Cargo.toml ├── build.rs └── src │ ├── bgp.rs │ ├── bgp │ ├── cmd.rs │ ├── global.rs │ ├── neighbor.rs │ └── rib.rs │ ├── cmd.rs │ ├── data.rs │ ├── data │ └── bgp.rs │ ├── error.rs │ ├── fib.rs │ ├── fib │ ├── channel.rs │ ├── cmd.rs │ └── route.rs │ ├── main.rs │ ├── proto │ ├── google.protobuf.rs │ ├── mod.rs │ └── sart.v1.rs │ ├── rpc.rs │ └── util.rs ├── sartcni ├── Cargo.lock ├── Cargo.toml ├── build.rs ├── config │ └── .cargo └── src │ ├── cmd.rs │ ├── cmd │ ├── add.rs │ ├── check.rs │ └── del.rs │ ├── config.rs │ ├── error.rs │ ├── main.rs │ ├── mock.rs │ ├── proto │ ├── google.protobuf.rs │ ├── mod.rs │ └── sart.v1.rs │ └── version.rs └── sartd ├── .cargo └── config ├── .rustfmt.toml ├── Cargo.lock ├── Cargo.toml ├── src ├── bgp │ ├── Cargo.lock │ ├── Cargo.toml │ └── src │ │ ├── api_server.rs │ │ ├── capability.rs │ │ ├── config.rs │ │ ├── error.rs │ │ ├── event.rs │ │ ├── family.rs │ │ ├── lib.rs │ │ ├── packet.rs │ │ ├── packet │ │ ├── attribute.rs │ │ ├── capability.rs │ │ ├── codec.rs │ │ ├── message.rs │ │ ├── mock.rs │ │ └── prefix.rs │ │ ├── path.rs │ │ ├── peer.rs │ │ ├── peer │ │ ├── fsm.rs │ │ ├── neighbor.rs │ │ └── peer.rs │ │ ├── rib.rs │ │ └── server.rs ├── bin │ ├── certgen.rs │ ├── cni-installer.rs │ ├── crdgen.rs │ └── sartd.rs ├── cert │ ├── Cargo.lock │ ├── Cargo.toml │ └── src │ │ ├── constants.rs │ │ ├── lib.rs │ │ └── util.rs ├── cmd │ ├── Cargo.lock │ ├── Cargo.toml │ └── src │ │ ├── agent.rs │ │ ├── bgp.rs │ │ ├── cmd.rs │ │ ├── controller.rs │ │ ├── fib.rs │ │ └── lib.rs ├── fib │ ├── Cargo.lock │ ├── Cargo.toml │ └── src │ │ ├── bgp.rs │ │ ├── channel.rs │ │ ├── config.rs │ │ ├── error.rs │ │ ├── kernel.rs │ │ ├── lib.rs │ │ ├── rib.rs │ │ ├── route.rs │ │ └── server.rs ├── ipam │ ├── Cargo.lock │ ├── Cargo.toml │ └── src │ │ ├── allocator.rs │ │ ├── bitset.rs │ │ ├── block_allocator.rs │ │ ├── error.rs │ │ ├── lib.rs │ │ └── manager.rs ├── kubernetes │ ├── Cargo.lock │ ├── Cargo.toml │ ├── config │ │ └── .cargo │ ├── src │ │ ├── agent.rs │ │ ├── agent │ │ │ ├── bgp.rs │ │ │ ├── bgp │ │ │ │ ├── rpc.rs │ │ │ │ └── speaker.rs │ │ │ ├── cni.rs │ │ │ ├── cni │ │ │ │ ├── error.rs │ │ │ │ ├── gc.rs │ │ │ │ ├── netlink.rs │ │ │ │ ├── netns.rs │ │ │ │ ├── pod.rs │ │ │ │ └── server.rs │ │ │ ├── config.rs │ │ │ ├── context.rs │ │ │ ├── error.rs │ │ │ ├── metrics.rs │ │ │ ├── reconciler.rs │ │ │ ├── reconciler │ │ │ │ ├── address_block.rs │ │ │ │ ├── bgp_advertisement.rs │ │ │ │ ├── bgp_peer.rs │ │ │ │ ├── bgp_peer_watcher.rs │ │ │ │ └── node_bgp.rs │ │ │ └── server.rs │ │ ├── config.rs │ │ ├── context.rs │ │ ├── controller.rs │ │ ├── controller │ │ │ ├── config.rs │ │ │ ├── context.rs │ │ │ ├── error.rs │ │ │ ├── metrics.rs │ │ │ ├── reconciler.rs │ │ │ ├── reconciler │ │ │ │ ├── address_block.rs │ │ │ │ ├── address_pool.rs │ │ │ │ ├── bgp_advertisement.rs │ │ │ │ ├── block_request.rs │ │ │ │ ├── cluster_bgp.rs │ │ │ │ ├── endpointslice_watcher.rs │ │ │ │ ├── node_watcher.rs │ │ │ │ └── service_watcher.rs │ │ │ ├── server.rs │ │ │ ├── webhook.rs │ │ │ └── webhook │ │ │ │ ├── address_block.rs │ │ │ │ ├── address_pool.rs │ │ │ │ ├── bgp_advertisement.rs │ │ │ │ ├── bgp_peer.rs │ │ │ │ └── service.rs │ │ ├── crd.rs │ │ ├── crd │ │ │ ├── address_block.rs │ │ │ ├── address_pool.rs │ │ │ ├── bgp_advertisement.rs │ │ │ ├── bgp_peer.rs │ │ │ ├── bgp_peer_template.rs │ │ │ ├── block_request.rs │ │ │ ├── cluster_bgp.rs │ │ │ ├── error.rs │ │ │ └── node_bgp.rs │ │ ├── error.rs │ │ ├── fixture.rs │ │ ├── lib.rs │ │ ├── metrics.rs │ │ └── util.rs │ └── tests │ │ ├── agent_address_block_test.rs │ │ ├── agent_bgp_advertisement_test.rs │ │ ├── agent_bgp_peer_test.rs │ │ ├── agent_cni_server_test.rs │ │ ├── agent_node_bgp_test.rs │ │ ├── common │ │ └── mod.rs │ │ ├── config │ │ ├── .cargo │ │ ├── config.yaml │ │ └── dummy_kubeconfig │ │ ├── controller_address_block_test.rs │ │ ├── controller_address_pool_pod_test.rs │ │ ├── controller_address_pool_service_test.rs │ │ ├── controller_bgp_advertisement_test.rs │ │ ├── controller_block_request_test.rs │ │ ├── controller_cluster_bgp_test.rs │ │ ├── controller_endpointslice_watcher_test.rs │ │ ├── controller_service_watcher_test.rs │ │ └── node_watcher_test.rs ├── mock │ ├── Cargo.lock │ ├── Cargo.toml │ └── src │ │ ├── bgp.rs │ │ └── lib.rs ├── proto │ ├── Cargo.lock │ ├── Cargo.toml │ ├── build.rs │ └── src │ │ ├── google.protobuf.rs │ │ ├── lib.rs │ │ └── sart.v1.rs ├── trace │ ├── Cargo.lock │ ├── Cargo.toml │ └── src │ │ ├── error.rs │ │ ├── init.rs │ │ ├── lib.rs │ │ ├── metrics.rs │ │ └── telemetry.rs └── util │ ├── Cargo.lock │ ├── Cargo.toml │ └── src │ └── lib.rs └── testdata ├── config.yaml ├── fib_config.yaml └── messages ├── frr-ibgp-fail ├── keepalive ├── notification-bad-as-peer ├── open-2bytes-asn ├── open-4bytes-asn ├── open-bad-message-length ├── open-graceful-restart ├── open-ipv6 ├── open-optional-parameters ├── route-refresh ├── update-as-set ├── update-as4-path ├── update-as4-path-aggregator ├── update-ipv6-mp-reach-nlri ├── update-mp-reach-nlri └── update-nlri /.dockerignore: -------------------------------------------------------------------------------- 1 | **/target 2 | .vscode 3 | */bin 4 | e2e/ 5 | manifests/ 6 | -------------------------------------------------------------------------------- /.github/release.yaml: -------------------------------------------------------------------------------- 1 | # .github/release.yml 2 | 3 | changelog: 4 | exclude: 5 | labels: 6 | - ignore-for-release 7 | categories: 8 | - title: Breaking Changes 🛠 9 | labels: 10 | - Semver-Major 11 | - breaking-change 12 | - title: New Features 🎉 13 | labels: 14 | - Semver-Minor 15 | - enhancement 16 | - title: Other Changes 17 | labels: 18 | - "*" 19 | -------------------------------------------------------------------------------- /.github/workflows/release.yaml: -------------------------------------------------------------------------------- 1 | name: Release 2 | on: 3 | push: 4 | tags: 5 | - "v*" 6 | defaults: 7 | run: 8 | working-directory: . 9 | env: 10 | cache-version: 1 11 | jobs: 12 | sart-image: 13 | name: Push sart container image 14 | runs-on: ubuntu-22.04 15 | steps: 16 | - uses: actions/checkout@v4 17 | - name: Cache Docker layers 18 | uses: actions/cache@v3 19 | with: 20 | path: /tmp/.buildx-cache 21 | key: ${{ github.ref }}-${{ github.sha }} 22 | restore-keys: | 23 | ${{ github.ref }}-${{ github.sha }} 24 | ${{ github.ref }} 25 | refs/head/main 26 | - name: Set up Docker Buildx 27 | uses: docker/setup-buildx-action@v3 28 | - name: Login to GitHub Container Registry 29 | uses: docker/login-action@v2 30 | with: 31 | registry: ghcr.io 32 | username: ${{ github.actor }} 33 | password: ${{ secrets.GITHUB_TOKEN }} 34 | - name: Set Tag 35 | id: set-tag 36 | run: echo "RELEASE_TAG=${GITHUB_REF#refs/tags/v}" >> $GITHUB_OUTPUT # Remove "v" prefix. 37 | - name: Build and Push sartd 38 | uses: docker/build-push-action@v5 39 | with: 40 | context: . 41 | platforms: linux/amd64,linux/arm64 42 | push: true 43 | tags: ghcr.io/terassyi/sart:${{ steps.set-tag.outputs.RELEASE_TAG }} 44 | -------------------------------------------------------------------------------- /.github/workflows/unit-test.yaml: -------------------------------------------------------------------------------- 1 | name: unit test 2 | on: [push] 3 | 4 | env: 5 | CARGO_TERM_COLOR: always 6 | 7 | jobs: 8 | unit-test: 9 | name: Unit Test 10 | runs-on: ubuntu-22.04 11 | steps: 12 | - uses: actions/checkout@v4 13 | - name: Setup Build Dependencies 14 | run: make setup-grpc 15 | - name: Fmt 16 | run: make fmt 17 | - name: Run Unit Test 18 | run: make unit-test 19 | integration-test: 20 | name: Integration Test 21 | runs-on: ubuntu-22.04 22 | steps: 23 | - uses: actions/checkout@v4 24 | - name: Setup Build Dependencies 25 | run: make setup-grpc 26 | - name: Generate CRD manifests 27 | run: make crd 28 | - name: Install Kubernetes Dependencies 29 | run: |- 30 | make -C e2e setup 31 | - name: Run Integration Test 32 | run: make integration-test 33 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | */target 2 | *.profraw 3 | *.pcap 4 | *.pcaping 5 | .vscode 6 | *.bin 7 | /bin 8 | /e2e/bin 9 | **/*.pem 10 | **/*.cert 11 | **/*.key 12 | logs* 13 | **/target 14 | sart.yaml 15 | 16 | **/e2e.test 17 | e2e/topology/.*.yaml.bak 18 | e2e/topology/kubernetes-cni*.yaml 19 | e2e/topology/kubernetes-cni-compact.yaml 20 | e2e/clab-sart 21 | 22 | manifests/base/certs/* 23 | manifests/webhook/admission_webhook_patch.yaml 24 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terassyi/sart/82d8bbeb0224e43cc80a03e6198a98226d2ede47/CHANGELOG.md -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | ARG RUST_VERSION=1.76.0 2 | 3 | # BUILDPLATFORM = linux/amd64 4 | 5 | FROM --platform=$BUILDPLATFORM rust:${RUST_VERSION} as builder 6 | 7 | 8 | RUN apt update -y && \ 9 | apt install -y protobuf-compiler libprotobuf-dev clang llvm mold gcc-multilib 10 | 11 | ENV CC_aarch64_unknown_linux_musl=clang 12 | ENV AR_aarch64_unknown_linux_musl=llvm-ar 13 | ENV CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_RUSTFLAGS="-Clink-self-contained=yes -Clinker=rust-lld" 14 | 15 | ENV CC_x86_64_unknown_linux_musl=clang 16 | ENV AR_x86_64_unknown_linux_musl=llvm-ar 17 | ENV CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_RUSTFLAGS="-Clink-self-contained=yes -Clinker=rust-lld" 18 | 19 | ARG TARGETPLATFORM 20 | RUN case "$TARGETPLATFORM" in \ 21 | "linux/arm64") echo aarch64-unknown-linux-musl > /rust_target.txt ;; \ 22 | "linux/amd64") echo x86_64-unknown-linux-musl > /rust_target.txt ;; \ 23 | *) exit 1 ;; \ 24 | esac 25 | 26 | RUN rustup target add $(cat /rust_target.txt) 27 | 28 | WORKDIR /home 29 | COPY ./sartd /home/sartd 30 | COPY ./sart /home/sart 31 | COPY ./sartcni /home/sartcni 32 | COPY ./proto /home/proto 33 | 34 | RUN cd sartd; cargo build --release --target $(cat /rust_target.txt) && \ 35 | cp /home/sartd/target/$(cat /rust_target.txt)/release/sartd /usr/local/bin/sartd && \ 36 | cargo build --release --bin cni-installer --target $(cat /rust_target.txt) && \ 37 | cp /home/sartd/target/$(cat /rust_target.txt)/release/cni-installer /usr/local/bin/cni-installer 38 | RUN cd sart; cargo build --release --target $(cat /rust_target.txt) && \ 39 | cp /home/sart/target/$(cat /rust_target.txt)/release/sart /usr/local/bin/sart 40 | RUN cd sartcni; cargo build --release --target $(cat /rust_target.txt) && \ 41 | cp /home/sartcni/target/$(cat /rust_target.txt)/release/sart-cni /usr/local/bin/sart-cni 42 | 43 | FROM debian:stable 44 | 45 | RUN apt update -y && \ 46 | apt install -y iproute2 47 | 48 | COPY --from=builder /usr/local/bin/sartd /usr/local/bin/sartd 49 | COPY --from=builder /usr/local/bin/sart /usr/local/bin/sart 50 | COPY --from=builder /usr/local/bin/cni-installer /usr/local/bin/cni-installer 51 | 52 | COPY --from=builder /usr/local/bin/sart-cni /host/opt/cni/bin/sart-cni 53 | -------------------------------------------------------------------------------- /docs/bgp.md: -------------------------------------------------------------------------------- 1 | # How to Use 2 | 3 | Sart has two CLI interfaces, sartd is for daemon and sart-cli is for controlling sartd. 4 | 5 | ### sartd bgp 6 | 7 | We can run the BGP daemon with `sartd bgp` command. 8 | And we can specify some parameters such as AS number and router id. 9 | To integrate Fib manager, we have to give the path to the gRPC endpoint for fib manager. 10 | 11 | ```console 12 | Usage: sartd bgp [OPTIONS] 13 | 14 | Options: 15 | -f, --file Config file path for BGP daemon 16 | -a, --as Local AS Number 17 | -d, --format Log display format [default: plain] [possible values: plain, json] 18 | -r, --router-id Local router id(must be ipv4 format) 19 | --fib Fib endpoint url(gRPC) exp) localhost:5001 20 | --table-id Target fib table id(default is main(254)) 21 | --exporter Exporter endpoint url 22 | -l, --level Log level(trace, debug, info, warn, error) [default: info] 23 | -h, --help Print help 24 | ``` 25 | 26 | We also can configure BGP with a configuration file written by Yaml. 27 | This is the example of config files. 28 | 29 | ```yaml 30 | asn: 65000 31 | router_id: 10.0.0.2 32 | multi_path: true 33 | neighbors: 34 | - asn: 65010 35 | router_id: 10.0.0.3 36 | address: 10.0.0.3 37 | - asn: 65020 38 | router_id: 10.0.1.3 39 | address: 10.0.1.3 40 | ``` 41 | 42 | 43 | ## sart Commands 44 | 45 | To control running daemons, we can use sart-cli commands. 46 | Now we support following subcommands 47 | 48 | - bgp 49 | 50 | ### sart bgp 51 | 52 | `sart bgp` command accepts `global` level and `neighbor` level subcommands. 53 | 54 | `global` level can get and set AS number or router id of the local daemon. 55 | And it also can configure RIB(Loc-RIB) information. 56 | 57 | ```console 58 | root@233eff855e37:/# sart bgp global rib -h 59 | Usage: sart bgp global rib [OPTIONS] 60 | 61 | Commands: 62 | get 63 | add 64 | del 65 | help Print this message or the help of the given subcommand(s) 66 | 67 | Options: 68 | -d, --format Display format [default: plain] [possible values: plain, json] 69 | -e, --endpoint Endpoint to API server [default: localhost:5000] 70 | -a, --afi Address Family Information [default: ipv4] [possible values: ipv4, ipv6] 71 | -s, --safi Sub Address Family Information [default: unicast] [possible values: unicast, multicast] 72 | -h, --help Print help 73 | ``` 74 | 75 | For example, to add a prefix to running BGP daemon, run this command. 76 | ```console 77 | root@233eff855e37:/# sart bgp global rib add 10.0.10.0/24 -a ipv4 -t origin=igp 78 | ``` 79 | 80 | `neighbor` level can get and set neighbor information. 81 | 82 | **Some commands are not implemented yet.** 83 | 84 | ```console 85 | root@233eff855e37:/# sart bgp neighbor -h 86 | Usage: sart bgp neighbor [OPTIONS] 87 | 88 | Commands: 89 | get 90 | list 91 | add 92 | del 93 | rib 94 | policy 95 | help Print this message or the help of the given subcommand(s) 96 | ``` 97 | 98 | For example, to add a neighbor, run this. 99 | 100 | ```console 101 | root@233eff855e37:/# sart bgp neighbor add 10.10.0.1 65000 102 | ``` 103 | 104 | ## gRPC Interfaces 105 | 106 | Sartd has gRPC interfaces. 107 | Sart-cli calls this interface internally. 108 | 109 | For detail, please see [proto/](https://github.com/teassyi/sart/blob/main/proto). 110 | 111 | ## Integrate with FIB Manager 112 | 113 | To install paths received from other peers, we need to a FIB manager. 114 | For example, [FRR](https://frrouting.org/) needs to run [zebrad](https://docs.frrouting.org/en/latest/zebra.html) to install paths bgpd gets. 115 | 116 | For `sartd-bgp`, we can run `sartd-fib` as the FIB manager. 117 | `Sartd-fib` has gRPC interface to interact with other component. 118 | 119 | > [!NOTE] 120 | >`Sartd-fib` is in the PoC phase. 121 | 122 | To integrate with `sartd-fib`, we have to specify the gRPC endpoint to FIB manager as a parameter like below. 123 | 124 | ```console 125 | $ sartd bgp --fib localhost:5010 126 | ``` 127 | -------------------------------------------------------------------------------- /docs/cni.md: -------------------------------------------------------------------------------- 1 | # How to use as Kubernetes CNI plugin 2 | 3 | To run as Kubernetes CNI plugin, we need to run four components: `sartd controller`, `sartd agent`, `sartd bgp` and `sartd fib`. 4 | 5 | `controller` runs as `Deployment`. 6 | And `agent`, `bgp` and `fib` run as `DaemonSet`. 7 | 8 | To perform as CNI plugin, all components must run in host network namespace. 9 | 10 | To know sart's CRDs, please see [design #Kubernetes](./design.md#kubernetes) and [design #CNI](./design.md#cni-for-kubernetes). 11 | 12 | To work CNI well, we need to configure BGP speakers on nodes. 13 | Please see [Kubernetes BGP configurations](./kubernetes.md#bgp-configurations). 14 | 15 | ## Address pools 16 | 17 | 18 | To assign IP addresses to pods, we have to create address pools. 19 | We can create multiple pools in a cluster. 20 | 21 | The `type` field specifies for which use the address pool is to be used. 22 | For pods, we have to set `type: pod`. 23 | 24 | ```yaml 25 | apiVersion: sart.terassyi.net/v1alpha2 26 | kind: AddressPool 27 | metadata: 28 | name: default-pod-pool 29 | spec: 30 | cidr: 10.1.0.0/24 31 | type: pod 32 | allocType: bit 33 | blockSize: 29 34 | autoAssign: true 35 | ``` 36 | 37 | `cidr` is the subnet of assignable addresses for pods. 38 | 39 | `allocType` specifies a method how to pick an address from a pool. 40 | We can only use `bit`. 41 | 42 | Bit allocator type choose the address with the lowest number. 43 | For example, `10.0.0.1` and `10.0.0.10` are assignable, the bit allocator always chooses `10.0.0.1`. 44 | 45 | `blockSize` is used to divide the address pool into `AddressBlock`s. 46 | `AddressBlock` for pods is associated with a node and it is requested to create via a `BlockRequest` resource. 47 | Each block requests its cidr based on this value. 48 | 49 | In this example, the cidr of one address block is `10.1.0.0/29` and another is `10.1.0.8/29`. 50 | 51 | `autoAssign` specifies its pool is used as default pool. 52 | If `autoAssign` is true, we can omit specifying the name of the pool in the annotation(described below). 53 | We cannot create multiple auto assignable pools in each `type`. 54 | 55 | Please note that we cannot change `AddressPool`'s spec fields except for `autoAssign` once it is created. 56 | 57 | ### Basic usage 58 | 59 | Once creating `AddressPool`s, we can create some pods same as using other CNI plugins. 60 | When allocating from the auto assignable pool, there is nothing to do any other special procedure. 61 | 62 | ### Choosing AddressPool 63 | 64 | Sart CNI can create multiple pools for pods. 65 | To use the pool that is not auto assignable pool, we should add an annotation named `sart.terassyi.net/addresspool` to the namespace in which its pod exist or to the pod directly. 66 | 67 | -------------------------------------------------------------------------------- /docs/fib.md: -------------------------------------------------------------------------------- 1 | # How to Use 2 | 3 | Sart has two CLI interfaces, sartd is for daemon and sart-cli is for controlling sartd. 4 | 5 | ### sartd fib 6 | 7 | We can run the Fib manager to install or uninstall routing information in kernel with `sartd fib`. 8 | 9 | ```console 10 | root@233eff855e37:/# sartd fib -h 11 | Usage: sartd fib [OPTIONS] 12 | 13 | Options: 14 | -e, --endpoint Fib manager running endpoint url [default: 127.0.0.1:5001] 15 | -l, --level Log level(trace, debug, info, warn, error) [default: info] 16 | -d, --format Log display format [default: plain] [possible values: plain, json] 17 | -h, --help Print help 18 | ``` 19 | -------------------------------------------------------------------------------- /docs/setup-kubernetes.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terassyi/sart/82d8bbeb0224e43cc80a03e6198a98226d2ede47/docs/setup-kubernetes.md -------------------------------------------------------------------------------- /docs/usage.md: -------------------------------------------------------------------------------- 1 | # User Manual 2 | 3 | This document describes how to use Sart. 4 | 5 | Please refer to following links for details of each feature. 6 | 7 | - [bgp](./bgp.md) 8 | - [fib](./fib.md) 9 | - [kubernetes](./kubernetes.md) 10 | 11 | ## Metrics 12 | 13 | ### Controller 14 | 15 | |Name|Type|Description| 16 | |---|---|---| 17 | |sart_controller_reconciliation_total|Counter|Total count of reconciliations| 18 | |sart_controller_reconciliation_errors_total|Counter|Total count of reconciliation errors| 19 | |sart_controller_max_blocks|Gauge|The number of maximum allocatable address blocks| 20 | |sart_controller_allocated_blocks|Gauge|The number of allocated address blocks| 21 | |sart_controller_bgp_advertisements|Gauge|The number of BGP Advertisement| 22 | |sart_controller_bgp_advertisement_status|Gauge|BGP Advertisement status| 23 | |sart_controller_bgp_advertisement_backoff_count|Counter|The number of back off count of BGP Advertisement| 24 | 25 | ### Agent 26 | 27 | |Name|Type|Description| 28 | |---|---|---| 29 | |sart_agent_reconciliation_total|Counter|Total count of reconciliations| 30 | |sart_agent_reconciliation_errors_total|Counter|Total count of reconciliation errors| 31 | |sart_agent_cni_call_total|Counter|Total count of CNI call| 32 | |sart_agent_cni_call_errors_total|Counter|Total count of CNI call error| 33 | |sart_agent_bgp_peer_status|Gauge|BGP peer status| 34 | |sart_agent_node_bgp_status|Gauge|Node BGP status| 35 | |sart_agent_node_bgp_backoff_count_total|Counter|NodeBGP backoff count| 36 | -------------------------------------------------------------------------------- /e2e/Makefile.versions: -------------------------------------------------------------------------------- 1 | KIND_VERSION := 0.22.0 2 | KUBERNETES_VERSION := 1.29.2 3 | KUSTOMIZE_VERSION := 5.3.0 4 | CONTAINERLAB_VERSION := 0.51.3 5 | HELM_VERSION := 3.14.2 6 | CERT_MANAGER_VERSION := 1.14.4 7 | -------------------------------------------------------------------------------- /e2e/README.md: -------------------------------------------------------------------------------- 1 | # End to End Tests 2 | 3 | End-to-End tests is written by Golang using [Ginkgo](https://github.com/onsi/ginkgo). 4 | To run this tests, we need Docker and Golang. 5 | 6 | Before running tests, we need following command in the `e2e` directory. 7 | 8 | ```console 9 | $ make setup 10 | ``` 11 | 12 | ## BGP 13 | 14 | In this test, we prepare testing topologies by [Containerlab](https://containerlab.dev/). 15 | 16 | To run the bgp e2e test, please execute the following command. 17 | ```console 18 | $ make bgp-e2e 19 | ``` 20 | 21 | We confirm following points in this test. 22 | 23 | - Establish peers with other routing softwares. 24 | - FRR 25 | - GoBGP 26 | - Establish iBGP and eBGP peers. 27 | - Receive paths from iBGP and eBGP peers. 28 | - Advertise paths to iBGP and eBGP peers. 29 | 30 | ## Kubernetes 31 | 32 | We need to run additional commands to setup a kubernetes cluster and external routes and install Sart related resources. 33 | 34 | ```console 35 | $ make kubernetes 36 | $ make install-sart 37 | ``` 38 | 39 | After that, we can run the test. 40 | 41 | ```console 42 | make kubernetes-e2e 43 | ``` 44 | 45 | This is the topology for kubernetes e2e test. 46 | 47 | ![kubernetes.drawio.svg](./img/kubernetes.drawio.svg) 48 | 49 | This tests confirm that following points. 50 | 51 | - Establish BGP peers via Kubernetes custom resources. 52 | - Assign external ip addresses to LoadBalancer services. 53 | - Specify an address pool. 54 | - Request a specific address. 55 | - Specify multiple address pool. 56 | - Change the address pool. 57 | - Communicate with an external client via LoadBalancer. 58 | - externalTrafficPolicy=Local 59 | - externalTrafficPolicy=Cluster 60 | - Change the externalTrafficPolicy 61 | - Restart 62 | - sartd-agent 63 | - sartd-bgp 64 | - sart-controller 65 | 66 | ## CNI 67 | 68 | To set up a CNI e2e test environment, we need following commands. 69 | 70 | ```console 71 | $ make kubernetes MODE=cni 72 | $ make install-sart MODE=cni 73 | ``` 74 | 75 | After that, we can run the test. 76 | 77 | ```console 78 | $ make cni-e2e 79 | ``` 80 | 81 | The following figure shows the topology for CNI e2e test. 82 | 83 | ![kubernetes-cni.drawio.svg](./img/kubernetes-cni.drawio.svg) 84 | 85 | This test confirm that following points. 86 | 87 | - Establish BGP peers via Kubernetes custom resources. 88 | - Assign an IP address to pods from the auto assignable pool 89 | - Assign an IP address to pods from the non auto assignable pool 90 | - Connectivity of pods in the cluster 91 | - Restart 92 | - sartd-agent 93 | - sartd-bgp 94 | - sart-controller 95 | -------------------------------------------------------------------------------- /e2e/fixture.go: -------------------------------------------------------------------------------- 1 | package e2e 2 | 3 | import ( 4 | "bytes" 5 | "os" 6 | "os/exec" 7 | "path" 8 | 9 | "k8s.io/client-go/dynamic" 10 | "sigs.k8s.io/controller-runtime/pkg/client/config" 11 | ) 12 | 13 | const ( 14 | BIN string = "bin/" 15 | CONTAINERLAB_BIN string = BIN + "containerlab" 16 | TOPOLOGY_DIR string = "topology/" 17 | ) 18 | 19 | func deployContainerlab(file string) error { 20 | path := path.Join(TOPOLOGY_DIR, file) 21 | return exec.Command(CONTAINERLAB_BIN, "-t", path, "deploy").Run() 22 | } 23 | 24 | func destroyContainerlab(file string) error { 25 | path := path.Join(TOPOLOGY_DIR, file) 26 | return exec.Command(CONTAINERLAB_BIN, "-t", path, "destroy").Run() 27 | } 28 | 29 | func execInContainer(target string, input []byte, cmd string, args ...string) ([]byte, error) { 30 | execArgs := []string{"exec", target, cmd} 31 | execArgs = append(execArgs, args...) 32 | c := exec.Command("docker", execArgs...) 33 | if input != nil { 34 | c.Stdin = bytes.NewReader(input) 35 | } 36 | return c.Output() 37 | } 38 | 39 | func kubectl(input []byte, args ...string) ([]byte, error) { 40 | homeDir := os.Getenv("HOME") 41 | kubeConfigPath := path.Join(homeDir, ".kube/config") 42 | cmdArgs := []string{"--kubeconfig", kubeConfigPath} 43 | cmdArgs = append(cmdArgs, args...) 44 | c := exec.Command(path.Join(BIN, "kubectl"), cmdArgs...) 45 | if input != nil { 46 | c.Stdin = bytes.NewReader(input) 47 | } 48 | c.Stderr = os.Stdout 49 | return c.Output() 50 | } 51 | 52 | func kubectlExec(name, namespace string, input []byte, args ...string) ([]byte, error) { 53 | cmdArgs := []string{"-n", namespace, "exec", name, "--"} 54 | cmdArgs = append(cmdArgs, args...) 55 | return kubectl(input, cmdArgs...) 56 | } 57 | 58 | func getDynamicClient() (*dynamic.DynamicClient, error) { 59 | conf, err := config.GetConfig() 60 | if err != nil { 61 | return nil, err 62 | } 63 | 64 | return dynamic.NewForConfig(conf) 65 | } 66 | -------------------------------------------------------------------------------- /e2e/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/terassyi/sart/e2e 2 | 3 | go 1.21 4 | 5 | require ( 6 | github.com/onsi/ginkgo/v2 v2.13.2 7 | github.com/onsi/gomega v1.30.0 8 | k8s.io/api v0.28.4 9 | k8s.io/apimachinery v0.28.4 10 | k8s.io/client-go v0.28.4 11 | sigs.k8s.io/controller-runtime v0.16.3 12 | ) 13 | 14 | require ( 15 | github.com/davecgh/go-spew v1.1.1 // indirect 16 | github.com/go-logr/logr v1.3.0 // indirect 17 | github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect 18 | github.com/gogo/protobuf v1.3.2 // indirect 19 | github.com/golang/protobuf v1.5.3 // indirect 20 | github.com/google/go-cmp v0.6.0 // indirect 21 | github.com/google/gofuzz v1.2.0 // indirect 22 | github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect 23 | github.com/imdario/mergo v0.3.6 // indirect 24 | github.com/json-iterator/go v1.1.12 // indirect 25 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 26 | github.com/modern-go/reflect2 v1.0.2 // indirect 27 | github.com/spf13/pflag v1.0.5 // indirect 28 | golang.org/x/net v0.17.0 // indirect 29 | golang.org/x/oauth2 v0.8.0 // indirect 30 | golang.org/x/sys v0.14.0 // indirect 31 | golang.org/x/term v0.13.0 // indirect 32 | golang.org/x/text v0.13.0 // indirect 33 | golang.org/x/time v0.3.0 // indirect 34 | golang.org/x/tools v0.14.0 // indirect 35 | google.golang.org/appengine v1.6.7 // indirect 36 | google.golang.org/protobuf v1.31.0 // indirect 37 | gopkg.in/inf.v0 v0.9.1 // indirect 38 | gopkg.in/yaml.v2 v2.4.0 // indirect 39 | gopkg.in/yaml.v3 v3.0.1 // indirect 40 | k8s.io/klog/v2 v2.100.1 // indirect 41 | k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 // indirect 42 | sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect 43 | sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect 44 | sigs.k8s.io/yaml v1.3.0 // indirect 45 | ) 46 | -------------------------------------------------------------------------------- /e2e/kind-config-disable-cni.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kind.x-k8s.io/v1alpha4 2 | kind: Cluster 3 | networking: 4 | disableDefaultCNI: true 5 | serviceSubnet: "10.101.0.0/16" 6 | nodes: 7 | - role: control-plane 8 | - role: worker 9 | - role: worker 10 | - role: worker 11 | -------------------------------------------------------------------------------- /e2e/kind-config.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kind.x-k8s.io/v1alpha4 2 | kind: Cluster 3 | networking: 4 | podSubnet: "10.100.0.0/16" 5 | serviceSubnet: "10.101.0.0/16" 6 | nodes: 7 | - role: control-plane 8 | - role: worker 9 | - role: worker 10 | - role: worker 11 | -------------------------------------------------------------------------------- /e2e/suite_test.go: -------------------------------------------------------------------------------- 1 | package e2e 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "testing" 7 | "time" 8 | 9 | . "github.com/onsi/ginkgo/v2" 10 | . "github.com/onsi/gomega" 11 | ) 12 | 13 | func TestSart(t *testing.T) { 14 | RegisterFailHandler(Fail) 15 | RunSpecs(t, "Sart e2e test") 16 | } 17 | 18 | var _ = BeforeSuite(func() { 19 | fmt.Println("Preparing...") 20 | 21 | SetDefaultEventuallyPollingInterval(time.Second) 22 | SetDefaultEventuallyTimeout(5 * time.Minute) 23 | }) 24 | 25 | var _ = Describe("End to End test for Sart", func() { 26 | BeforeEach(func() { 27 | fmt.Printf("START: %s\n", time.Now().Format(time.RFC3339)) 28 | }) 29 | AfterEach(func() { 30 | fmt.Printf("END: %s\n", time.Now().Format(time.RFC3339)) 31 | }) 32 | 33 | testTarget := os.Getenv("TARGET") 34 | 35 | switch testTarget { 36 | case "bgp": 37 | testBgp() 38 | case "kubernetes": 39 | testKubernetes() 40 | case "cni": 41 | testKubernetesCNI() 42 | default: 43 | fmt.Println("target not set") 44 | os.Exit(1) 45 | } 46 | }) 47 | 48 | func testBgp() { 49 | Context("frr", testEstablishPeerWithFrr) 50 | Context("gobgp", testEstablishPeerWithGoBGP) 51 | Context("ibgp", testEstablishPeerWithIBGP) 52 | Context("cicle", testInCircleTopology) 53 | Context("cicle-ibgp", testInCircleTopologyWithIBGP) 54 | Context("multipath", testInCircleTopologyWithMultiPath) 55 | } 56 | 57 | func testKubernetes() { 58 | Context("workloads", testControllerWorkloads) 59 | Context("bgp=a", testClusterBGPA) 60 | Context("bgp=b", testClusterBGPB) 61 | Context("create address pool", testCreateLBAddressPool) 62 | Context("create load-balancer", testCreatingLoadBalancer) 63 | Context("load-balancer connectivity", testLoadBalancerConnectivity) 64 | Context("address pool", testAddressPool) 65 | Context("externalTrafficPolicy", testExternalTrafficPolicy) 66 | Context("bgp change", testBGPLabelChange) 67 | Context("delete cluster bgp", testDeleteClusterBGP) 68 | Context("delete bgp peer", testDeleteBGPPeer) 69 | Context("bgp2=c", testClusterBGPC) 70 | Context("restart agent", testRestartAgent) 71 | Context("restart controller", testRestartController) 72 | Context("restart bgp", testRestartBGP) 73 | } 74 | 75 | func testKubernetesCNI() { 76 | Context("workloads", testControllerWorkloads) 77 | Context("prepare BGP", testClusterBGPForCNI) 78 | Context("create address pools for pod", testPodAddressPool) 79 | Context("create pods", testCreatePods) 80 | Context("delete pod", testDeletePod) 81 | Context("create pod with non default pool", testNonDefaultPool) 82 | Context("create pods in test-non-default namespace", testNonDefaultPoolInNamespace) 83 | Context("release unused address block", testReleaseAddressBlock) 84 | Context("recover from restart", testRecoverAllocationsAfterRestart) 85 | Context("switch the mode", testSwitchModeToDual) 86 | Context("create LB address pool", testCreateLBAddressPool) 87 | Context("create load-balancer", testCreatingLoadBalancer) 88 | Context("lb connectivity", testLBConnectivityWithDualMode) 89 | } 90 | -------------------------------------------------------------------------------- /e2e/topology/circle-ibgp.yaml: -------------------------------------------------------------------------------- 1 | name: sart 2 | topology: 3 | kinds: 4 | linux: 5 | cmd: bash 6 | nodes: 7 | gobgp0: 8 | kind: linux 9 | image: ghcr.io/terassyi/terakoya:0.1.2 10 | binds: 11 | - configs/circle-ibgp/gobgp0.toml:/etc/gobgp.conf 12 | cmd: gobgpd -f /etc/gobgp.conf 13 | network-mode: container:clab-sart-gobgp0-init 14 | gobgp0-init: 15 | kind: linux 16 | image: nicolaka/netshoot:latest 17 | exec: 18 | - ip addr add 169.254.0.1/24 dev net0 19 | - ip addr add 169.254.1.1/24 dev net1 20 | gobgp1: 21 | kind: linux 22 | image: ghcr.io/terassyi/terakoya:0.1.2 23 | binds: 24 | - configs/circle-ibgp/gobgp1.toml:/etc/gobgp.conf 25 | cmd: gobgpd -f /etc/gobgp.conf 26 | network-mode: container:clab-sart-gobgp1-init 27 | gobgp1-init: 28 | kind: linux 29 | image: nicolaka/netshoot:latest 30 | exec: 31 | - ip addr add 169.254.1.2/24 dev net0 32 | - ip addr add 169.254.2.2/24 dev net1 33 | sart: 34 | kind: linux 35 | image: sart:dev 36 | exec: 37 | - ip addr add 169.254.0.2/24 dev net0 38 | - ip addr add 169.254.2.1/24 dev net1 39 | cmd: sartd bgp 40 | sart-debug: 41 | kind: linux 42 | image: nicolaka/netshoot:latest 43 | network-mode: container:clab-sart-sart 44 | links: 45 | - endpoints: ["gobgp0:net0", "sart:net0"] 46 | - endpoints: ["gobgp0:net1", "gobgp1:net0"] 47 | - endpoints: ["sart:net1", "gobgp1:net1"] 48 | -------------------------------------------------------------------------------- /e2e/topology/circle.yaml: -------------------------------------------------------------------------------- 1 | name: sart 2 | topology: 3 | kinds: 4 | linux: 5 | cmd: bash 6 | nodes: 7 | gobgp0: 8 | kind: linux 9 | image: ghcr.io/terassyi/terakoya:0.1.2 10 | binds: 11 | - configs/circle/gobgp0.toml:/etc/gobgp.conf 12 | cmd: gobgpd -f /etc/gobgp.conf 13 | network-mode: container:clab-sart-gobgp0-init 14 | gobgp0-init: 15 | kind: linux 16 | image: nicolaka/netshoot:latest 17 | exec: 18 | - ip addr add 169.254.0.1/24 dev net0 19 | - ip addr add 169.254.1.1/24 dev net1 20 | gobgp1: 21 | kind: linux 22 | image: ghcr.io/terassyi/terakoya:0.1.2 23 | binds: 24 | - configs/circle/gobgp1.toml:/etc/gobgp.conf 25 | cmd: gobgpd -f /etc/gobgp.conf 26 | network-mode: container:clab-sart-gobgp1-init 27 | gobgp1-init: 28 | kind: linux 29 | image: nicolaka/netshoot:latest 30 | exec: 31 | - ip addr add 169.254.1.2/24 dev net0 32 | - ip addr add 169.254.2.2/24 dev net1 33 | sart: 34 | kind: linux 35 | image: sart:dev 36 | exec: 37 | - ip addr add 169.254.0.2/24 dev net0 38 | - ip addr add 169.254.2.1/24 dev net1 39 | cmd: sartd bgp 40 | sart-debug: 41 | kind: linux 42 | image: nicolaka/netshoot:latest 43 | network-mode: container:clab-sart-sart 44 | links: 45 | - endpoints: ["gobgp0:net0", "sart:net0"] 46 | - endpoints: ["gobgp0:net1", "gobgp1:net0"] 47 | - endpoints: ["sart:net1", "gobgp1:net1"] 48 | -------------------------------------------------------------------------------- /e2e/topology/configs/circle-ibgp/gobgp0.toml: -------------------------------------------------------------------------------- 1 | [global.config] 2 | as = 65000 3 | router-id = "169.254.0.1" 4 | [[neighbors]] 5 | [neighbors.config] 6 | neighbor-address = "169.254.0.2" 7 | peer-as = 65000 8 | [[neighbors]] 9 | [neighbors.config] 10 | neighbor-address = "169.254.1.2" 11 | peer-as = 65000 12 | -------------------------------------------------------------------------------- /e2e/topology/configs/circle-ibgp/gobgp1.toml: -------------------------------------------------------------------------------- 1 | [global.config] 2 | as = 65000 3 | router-id = "169.254.1.2" 4 | [[neighbors]] 5 | [neighbors.config] 6 | neighbor-address = "169.254.1.1" 7 | peer-as = 65000 8 | [[neighbors]] 9 | [neighbors.config] 10 | neighbor-address = "169.254.2.1" 11 | peer-as = 65000 12 | -------------------------------------------------------------------------------- /e2e/topology/configs/circle/gobgp0.toml: -------------------------------------------------------------------------------- 1 | [global.config] 2 | as = 65000 3 | router-id = "169.254.0.1" 4 | [[neighbors]] 5 | [neighbors.config] 6 | neighbor-address = "169.254.0.2" 7 | peer-as = 65001 8 | [[neighbors]] 9 | [neighbors.config] 10 | neighbor-address = "169.254.1.2" 11 | peer-as = 65002 12 | -------------------------------------------------------------------------------- /e2e/topology/configs/circle/gobgp1.toml: -------------------------------------------------------------------------------- 1 | [global.config] 2 | as = 65002 3 | router-id = "169.254.1.2" 4 | [[neighbors]] 5 | [neighbors.config] 6 | neighbor-address = "169.254.1.1" 7 | peer-as = 65000 8 | [[neighbors]] 9 | [neighbors.config] 10 | neighbor-address = "169.254.2.1" 11 | peer-as = 65001 12 | -------------------------------------------------------------------------------- /e2e/topology/configs/gobgp-basic.toml: -------------------------------------------------------------------------------- 1 | [global.config] 2 | as = 65000 3 | router-id = "169.254.0.1" 4 | [[neighbors]] 5 | [neighbors.config] 6 | neighbor-address = "169.254.0.2" 7 | peer-as = 65001 8 | -------------------------------------------------------------------------------- /e2e/topology/frr.yaml: -------------------------------------------------------------------------------- 1 | name: sart 2 | topology: 3 | kinds: 4 | linux: 5 | cmd: bash 6 | nodes: 7 | frr: 8 | kind: linux 9 | image: frrouting/frr:v8.4.0 10 | exec: 11 | - ip addr add 169.254.0.1/24 dev net0 12 | - touch /etc/frr/vtysh.conf 13 | - sed -i -e 's/bgpd=no/bgpd=yes/g' /etc/frr/daemons 14 | - /usr/lib/frr/frrinit.sh start 15 | # FRR configuration 16 | - >- 17 | vtysh -c 'conf t' 18 | -c 'router bgp 65000' 19 | -c ' bgp router-id 169.254.0.1' 20 | -c ' bgp bestpath as-path multipath-relax' 21 | -c ' neighbor 169.254.0.2 remote-as 65001' 22 | -c ' neighbor 169.254.0.2 route-map ACCEPT_ALL in' 23 | -c ' address-family ipv4 unicast' 24 | -c ' network 10.0.0.0/24' 25 | -c ' exit-address-family' 26 | -c 'route-map ACCEPT_ALL permit 10' 27 | -c ' match ip address prefix-list ALL_ROUTES' 28 | -c 'ip prefix-list ALL_ROUTES seq 5 permit 0.0.0.0/0 le 32' 29 | frr-debug: 30 | kind: linux 31 | image: nicolaka/netshoot:latest 32 | network-mode: container:clab-sart-frr 33 | sart: 34 | kind: linux 35 | image: sart:dev 36 | exec: 37 | - ip addr add 169.254.0.2/24 dev net0 38 | cmd: sartd bgp 39 | sart-debug: 40 | kind: linux 41 | image: nicolaka/netshoot:latest 42 | network-mode: container:clab-sart-sart 43 | links: 44 | - endpoints: ["frr:net0", "sart:net0"] 45 | -------------------------------------------------------------------------------- /e2e/topology/generator/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/terassyi/sart/e2e/topology/generator 2 | 3 | go 1.22.0 4 | 5 | require ( 6 | k8s.io/apimachinery v0.29.2 7 | k8s.io/client-go v0.29.2 8 | ) 9 | 10 | require ( 11 | github.com/davecgh/go-spew v1.1.1 // indirect 12 | github.com/emicklei/go-restful/v3 v3.11.0 // indirect 13 | github.com/go-logr/logr v1.3.0 // indirect 14 | github.com/go-openapi/jsonpointer v0.19.6 // indirect 15 | github.com/go-openapi/jsonreference v0.20.2 // indirect 16 | github.com/go-openapi/swag v0.22.3 // indirect 17 | github.com/gogo/protobuf v1.3.2 // indirect 18 | github.com/golang/protobuf v1.5.3 // indirect 19 | github.com/google/gnostic-models v0.6.8 // indirect 20 | github.com/google/gofuzz v1.2.0 // indirect 21 | github.com/google/uuid v1.3.0 // indirect 22 | github.com/imdario/mergo v0.3.6 // indirect 23 | github.com/josharian/intern v1.0.0 // indirect 24 | github.com/json-iterator/go v1.1.12 // indirect 25 | github.com/mailru/easyjson v0.7.7 // indirect 26 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 27 | github.com/modern-go/reflect2 v1.0.2 // indirect 28 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 29 | github.com/spf13/pflag v1.0.5 // indirect 30 | golang.org/x/net v0.19.0 // indirect 31 | golang.org/x/oauth2 v0.10.0 // indirect 32 | golang.org/x/sys v0.15.0 // indirect 33 | golang.org/x/term v0.15.0 // indirect 34 | golang.org/x/text v0.14.0 // indirect 35 | golang.org/x/time v0.3.0 // indirect 36 | google.golang.org/appengine v1.6.7 // indirect 37 | google.golang.org/protobuf v1.31.0 // indirect 38 | gopkg.in/inf.v0 v0.9.1 // indirect 39 | gopkg.in/yaml.v2 v2.4.0 // indirect 40 | gopkg.in/yaml.v3 v3.0.1 // indirect 41 | k8s.io/api v0.29.2 // indirect 42 | k8s.io/klog/v2 v2.110.1 // indirect 43 | k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect 44 | k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect 45 | sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect 46 | sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect 47 | sigs.k8s.io/yaml v1.3.0 // indirect 48 | ) 49 | -------------------------------------------------------------------------------- /e2e/topology/generator/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "io" 7 | "os" 8 | "path/filepath" 9 | "text/template" 10 | 11 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 12 | "k8s.io/client-go/kubernetes" 13 | "k8s.io/client-go/tools/clientcmd" 14 | ) 15 | 16 | const ( 17 | TOPOLOGY_CONFIG_TMPL string = "kubernetes.yaml.tmpl" 18 | TOPOLOGY_CONFIG string = "kubernetes.yaml" 19 | ) 20 | 21 | type NodeAddrs struct { 22 | ControlPlane string 23 | Worker string 24 | Worker2 string 25 | Worker3 string 26 | ControlPlanePodCIDR string 27 | WorkerPodCIDR string 28 | Worker2PodCIDR string 29 | Worker3PodCIDR string 30 | } 31 | 32 | var ( 33 | tmplPath = flag.String("template-path", TOPOLOGY_CONFIG_TMPL, "path to template") 34 | outputPath = flag.String("output-path", TOPOLOGY_CONFIG, "path to output") 35 | ) 36 | 37 | func main() { 38 | 39 | flag.Parse() 40 | 41 | homeDir := os.Getenv("HOME") 42 | kubeConfigPath := filepath.Join(homeDir, ".kube/config") 43 | config, err := clientcmd.BuildConfigFromFlags("", kubeConfigPath) 44 | if err != nil { 45 | panic(err) 46 | } 47 | clientset, err := kubernetes.NewForConfig(config) 48 | if err != nil { 49 | panic(err) 50 | } 51 | ctx := context.Background() 52 | nodes, err := clientset.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) 53 | if err != nil { 54 | panic(err) 55 | } 56 | var nodeAddrs NodeAddrs 57 | for _, node := range nodes.Items { 58 | switch node.GetName() { 59 | case "sart-control-plane": 60 | for _, statusAddrs := range node.Status.Addresses { 61 | if statusAddrs.Type == "InternalIP" { 62 | nodeAddrs.ControlPlane = statusAddrs.Address 63 | } 64 | } 65 | nodeAddrs.ControlPlanePodCIDR = node.Spec.PodCIDR 66 | case "sart-worker": 67 | for _, statusAddrs := range node.Status.Addresses { 68 | if statusAddrs.Type == "InternalIP" { 69 | nodeAddrs.Worker = statusAddrs.Address 70 | } 71 | } 72 | nodeAddrs.WorkerPodCIDR = node.Spec.PodCIDR 73 | case "sart-worker2": 74 | for _, statusAddrs := range node.Status.Addresses { 75 | if statusAddrs.Type == "InternalIP" { 76 | nodeAddrs.Worker2 = statusAddrs.Address 77 | } 78 | } 79 | nodeAddrs.Worker2PodCIDR = node.Spec.PodCIDR 80 | case "sart-worker3": 81 | for _, statusAddrs := range node.Status.Addresses { 82 | if statusAddrs.Type == "InternalIP" { 83 | nodeAddrs.Worker3 = statusAddrs.Address 84 | } 85 | } 86 | nodeAddrs.Worker3PodCIDR = node.Spec.PodCIDR 87 | default: 88 | } 89 | } 90 | 91 | if !nodeAddrs.check() { 92 | panic("Failed to get Node Address") 93 | } 94 | 95 | file, err := os.Open(*tmplPath) 96 | if err != nil { 97 | panic(err) 98 | } 99 | defer file.Close() 100 | 101 | tmplData, err := io.ReadAll(file) 102 | if err != nil { 103 | panic(err) 104 | } 105 | 106 | tmpl, err := template.New(TOPOLOGY_CONFIG_TMPL).Parse(string(tmplData)) 107 | if err != nil { 108 | panic(err) 109 | } 110 | 111 | outFile, err := os.Create(*outputPath) 112 | if err != nil { 113 | panic(err) 114 | } 115 | defer outFile.Close() 116 | 117 | if err := tmpl.Execute(outFile, nodeAddrs); err != nil { 118 | panic(err) 119 | } 120 | 121 | } 122 | 123 | func (n *NodeAddrs) check() bool { 124 | return !(n.ControlPlane == "" || n.Worker == "" || n.Worker2 == "" || n.Worker3 == "") 125 | } 126 | -------------------------------------------------------------------------------- /e2e/topology/gobgp.yaml: -------------------------------------------------------------------------------- 1 | name: sart 2 | topology: 3 | kinds: 4 | linux: 5 | cmd: bash 6 | nodes: 7 | frr: 8 | kind: linux 9 | image: ghcr.io/terassyi/terakoya:0.1.2 10 | exec: 11 | - ip addr add 169.254.0.1/24 dev net0 12 | - touch /etc/frr/vtysh.conf 13 | - /usr/lib/frr/frrinit.sh start 14 | frr-debug: 15 | kind: linux 16 | image: nicolaka/netshoot:latest 17 | network-mode: container:clab-sart-frr 18 | gobgp: 19 | kind: linux 20 | image: ghcr.io/terassyi/terakoya:0.1.2 21 | network-mode: container:clab-sart-frr 22 | binds: 23 | - configs/gobgp-basic.toml:/etc/gobgp.conf 24 | cmd: gobgpd -f /etc/gobgp.conf 25 | sart: 26 | kind: linux 27 | image: sart:dev 28 | exec: 29 | - ip addr add 169.254.0.2/24 dev net0 30 | cmd: sartd bgp 31 | sart-debug: 32 | kind: linux 33 | image: nicolaka/netshoot:latest 34 | network-mode: container:clab-sart-sart 35 | links: 36 | - endpoints: ["frr:net0", "sart:net0"] 37 | -------------------------------------------------------------------------------- /e2e/topology/ibgp.yaml: -------------------------------------------------------------------------------- 1 | name: sart 2 | topology: 3 | kinds: 4 | linux: 5 | cmd: bash 6 | nodes: 7 | frr: 8 | kind: linux 9 | image: frrouting/frr:v8.4.0 10 | exec: 11 | - ip addr add 169.254.0.1/24 dev net0 12 | - touch /etc/frr/vtysh.conf 13 | - sed -i -e 's/bgpd=no/bgpd=yes/g' /etc/frr/daemons 14 | - /usr/lib/frr/frrinit.sh start 15 | # FRR configuration 16 | - >- 17 | vtysh -c 'conf t' 18 | -c 'router bgp 65000' 19 | -c ' bgp router-id 169.254.0.1' 20 | -c ' bgp bestpath as-path multipath-relax' 21 | -c ' neighbor 169.254.0.2 remote-as internal' 22 | -c ' address-family ipv4 unicast' 23 | -c ' network 10.0.0.0/24' 24 | -c ' exit-address-family' 25 | frr-debug: 26 | kind: linux 27 | image: nicolaka/netshoot:latest 28 | network-mode: container:clab-sart-frr 29 | sart: 30 | kind: linux 31 | image: sart:dev 32 | exec: 33 | - ip addr add 169.254.0.2/24 dev net0 34 | cmd: sartd bgp 35 | sart-debug: 36 | kind: linux 37 | image: nicolaka/netshoot:latest 38 | network-mode: container:clab-sart-sart 39 | links: 40 | - endpoints: ["frr:net0", "sart:net0"] 41 | -------------------------------------------------------------------------------- /kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - manifests 3 | images: 4 | - name: sart:dev 5 | newName: ghcr.io/terassyi/sart 6 | newTag: 0.1.0 7 | -------------------------------------------------------------------------------- /manifests/base/crd/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - sart.yaml 3 | -------------------------------------------------------------------------------- /manifests/base/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - crd 3 | - rbac 4 | - webhook 5 | - workloads 6 | 7 | images: 8 | - name: sart:dev 9 | newName: sart 10 | newTag: dev 11 | 12 | generatorOptions: 13 | disableNameSuffixHash: true 14 | 15 | secretGenerator: 16 | - name: sart-webhook-server-cert 17 | namespace: kube-system 18 | files: 19 | - ca.crt=./certs/tls.cert 20 | - tls.crt=./certs/tls.cert 21 | - tls.key=./certs/tls.key 22 | type: "kubernetes.io/tls" 23 | -------------------------------------------------------------------------------- /manifests/base/rbac/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - serviceaccount.yaml 3 | - role.yaml 4 | - role_binding.yaml 5 | namespace: kube-system 6 | -------------------------------------------------------------------------------- /manifests/base/rbac/role_binding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRoleBinding 3 | metadata: 4 | name: sart 5 | roleRef: 6 | apiGroup: rbac.authorization.k8s.io 7 | kind: ClusterRole 8 | name: sart 9 | subjects: 10 | - kind: ServiceAccount 11 | name: sart 12 | namespace: system 13 | -------------------------------------------------------------------------------- /manifests/base/rbac/serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: sart 5 | namespace: system 6 | automountServiceAccountToken: true 7 | -------------------------------------------------------------------------------- /manifests/base/webhook/admission_webhook.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: admissionregistration.k8s.io/v1 2 | kind: ValidatingWebhookConfiguration 3 | metadata: 4 | creationTimestamp: null 5 | name: sart-validating-webhook-configuration 6 | webhooks: 7 | - admissionReviewVersions: 8 | - v1 9 | clientConfig: 10 | service: 11 | name: sart-webhook-service 12 | namespace: system 13 | path: /validate-sart-terassyi-net-v1alpha2-bgpadvertisement 14 | failurePolicy: Fail 15 | name: vbgpadvertisement.kb.io 16 | rules: 17 | - apiGroups: 18 | - sart.terassyi.net 19 | apiVersions: 20 | - v1alpha2 21 | operations: 22 | - CREATE 23 | - UPDATE 24 | resources: 25 | - bgpadvertisements 26 | sideEffects: None 27 | - admissionReviewVersions: 28 | - v1 29 | clientConfig: 30 | service: 31 | name: sart-webhook-service 32 | namespace: system 33 | port: 443 34 | path: /validate-sart-terassyi-net-v1alpha2-bgppeer 35 | failurePolicy: Fail 36 | name: vbgppeer.kb.io 37 | rules: 38 | - apiGroups: 39 | - sart.terassyi.net 40 | apiVersions: 41 | - v1alpha2 42 | operations: 43 | - CREATE 44 | - UPDATE 45 | resources: 46 | - bgppeers 47 | sideEffects: None 48 | - admissionReviewVersions: 49 | - v1 50 | clientConfig: 51 | service: 52 | name: sart-webhook-service 53 | namespace: system 54 | path: /validate-sart-terassyi-net-v1alpha2-addresspool 55 | failurePolicy: Fail 56 | name: vaddresspool.kb.io 57 | rules: 58 | - apiGroups: 59 | - sart.terassyi.net 60 | apiVersions: 61 | - v1alpha2 62 | operations: 63 | - CREATE 64 | - UPDATE 65 | resources: 66 | - addresspools 67 | sideEffects: None 68 | --- 69 | apiVersion: admissionregistration.k8s.io/v1 70 | kind: MutatingWebhookConfiguration 71 | metadata: 72 | creationTimestamp: null 73 | name: sart-mutating-webhook-configuration 74 | webhooks: 75 | - admissionReviewVersions: 76 | - v1 77 | clientConfig: 78 | service: 79 | name: sart-webhook-service 80 | namespace: system 81 | path: /mutate-sart-terassyi-net-v1alpha2-bgppeer 82 | failurePolicy: Fail 83 | name: mbgppeer.kb.io 84 | rules: 85 | - apiGroups: 86 | - sart.terassyi.net 87 | apiVersions: 88 | - v1alpha2 89 | operations: 90 | - CREATE 91 | - UPDATE 92 | resources: 93 | - bgppeers 94 | sideEffects: None 95 | - admissionReviewVersions: 96 | - v1 97 | clientConfig: 98 | service: 99 | name: sart-webhook-service 100 | namespace: system 101 | path: /mutate-sart-terassyi-net-v1alpha2-addressblock 102 | failurePolicy: Fail 103 | name: maddressblock.kb.io 104 | rules: 105 | - apiGroups: 106 | - sart.terassyi.net 107 | apiVersions: 108 | - v1alpha2 109 | operations: 110 | - CREATE 111 | - UPDATE 112 | resources: 113 | - addressblocks 114 | sideEffects: None 115 | - admissionReviewVersions: 116 | - v1 117 | clientConfig: 118 | service: 119 | name: sart-webhook-service 120 | namespace: system 121 | path: /mutate-v1-service 122 | failurePolicy: Fail 123 | name: mservice.kb.io 124 | rules: 125 | - apiGroups: 126 | - "" 127 | apiVersions: 128 | - v1 129 | operations: 130 | - CREATE 131 | - UPDATE 132 | resources: 133 | - services 134 | sideEffects: None 135 | -------------------------------------------------------------------------------- /manifests/base/webhook/admission_webhook_patch.yaml.tmpl: -------------------------------------------------------------------------------- 1 | apiVersion: admissionregistration.k8s.io/v1 2 | kind: ValidatingWebhookConfiguration 3 | metadata: 4 | name: sart-validating-webhook-configuration 5 | webhooks: 6 | - name: vbgppeer.kb.io 7 | clientConfig: 8 | caBundle: "%CACERT%" 9 | - name: vbgpadvertisement.kb.io 10 | clientConfig: 11 | caBundle: "%CACERT%" 12 | - name: vaddresspool.kb.io 13 | clientConfig: 14 | caBundle: "%CACERT%" 15 | --- 16 | apiVersion: admissionregistration.k8s.io/v1 17 | kind: MutatingWebhookConfiguration 18 | metadata: 19 | name: sart-mutating-webhook-configuration 20 | webhooks: 21 | - name: mbgppeer.kb.io 22 | clientConfig: 23 | caBundle: "%CACERT%" 24 | - name: maddressblock.kb.io 25 | clientConfig: 26 | caBundle: "%CACERT%" 27 | - name: mservice.kb.io 28 | clientConfig: 29 | caBundle: "%CACERT%" 30 | -------------------------------------------------------------------------------- /manifests/base/webhook/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - admission_webhook.yaml 3 | - service.yaml 4 | patchesStrategicMerge: 5 | - admission_webhook_patch.yaml 6 | 7 | namespace: kube-system 8 | 9 | configurations: 10 | - kustomizeconfig.yaml 11 | -------------------------------------------------------------------------------- /manifests/base/webhook/kustomizeconfig.yaml: -------------------------------------------------------------------------------- 1 | # the following config is for teaching kustomize where to look at when substituting vars. 2 | # It requires kustomize v2.1.0 or newer to work properly. 3 | nameReference: 4 | - kind: Service 5 | version: v1 6 | fieldSpecs: 7 | - kind: MutatingWebhookConfiguration 8 | group: admissionregistration.k8s.io 9 | path: webhooks/clientConfig/service/name 10 | - kind: ValidatingWebhookConfiguration 11 | group: admissionregistration.k8s.io 12 | path: webhooks/clientConfig/service/name 13 | 14 | namespace: 15 | - kind: MutatingWebhookConfiguration 16 | group: admissionregistration.k8s.io 17 | path: webhooks/clientConfig/service/namespace 18 | create: true 19 | - kind: ValidatingWebhookConfiguration 20 | group: admissionregistration.k8s.io 21 | path: webhooks/clientConfig/service/namespace 22 | create: true 23 | 24 | varReference: 25 | - path: metadata/annotations 26 | -------------------------------------------------------------------------------- /manifests/base/webhook/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: sart-webhook-service 5 | namespace: system 6 | spec: 7 | ports: 8 | - port: 443 9 | protocol: TCP 10 | targetPort: 8443 11 | selector: 12 | control-plane: sart-controller 13 | -------------------------------------------------------------------------------- /manifests/base/workloads/agent.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: DaemonSet 3 | metadata: 4 | labels: 5 | app: sart-agent 6 | component: agent 7 | name: sartd-agent 8 | namespace: system 9 | spec: 10 | selector: 11 | matchLabels: 12 | app: sart-agent 13 | component: agent 14 | template: 15 | metadata: 16 | annotations: 17 | prometheus.io/port: "7472" 18 | prometheus.io/scrape: "true" 19 | labels: 20 | app: sart-agent 21 | component: agent 22 | spec: 23 | containers: 24 | - command: ["sartd"] 25 | args: 26 | - agent 27 | env: 28 | - name: SARTD_BGP_NODE_IP 29 | valueFrom: 30 | fieldRef: 31 | fieldPath: status.hostIP 32 | image: sart:dev 33 | imagePullPolicy: IfNotPresent 34 | name: agent 35 | securityContext: 36 | privileged: true 37 | livenessProbe: 38 | httpGet: 39 | path: /healthz 40 | port: 8000 41 | initialDelaySeconds: 15 42 | periodSeconds: 20 43 | readinessProbe: 44 | httpGet: 45 | path: /readyz 46 | port: 8000 47 | initialDelaySeconds: 5 48 | periodSeconds: 10 49 | resources: 50 | limits: 51 | cpu: 100m 52 | memory: 1Gi 53 | requests: 54 | cpu: 10m 55 | memory: 128Mi 56 | volumeMounts: 57 | - mountPath: /etc/sartd/cert 58 | name: cert 59 | readOnly: true 60 | hostNetwork: true 61 | hostPID: true 62 | nodeSelector: 63 | kubernetes.io/os: linux 64 | terminationGracePeriodSeconds: 2 65 | tolerations: 66 | - effect: NoSchedule 67 | operator: Exists 68 | - effect: NoSchedule 69 | operator: Exists 70 | volumes: 71 | - name: cert 72 | secret: 73 | defaultMode: 420 74 | secretName: sart-webhook-server-cert 75 | serviceAccountName: sart 76 | -------------------------------------------------------------------------------- /manifests/base/workloads/controller.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: sart-controller 5 | namespace: system 6 | labels: 7 | app.kubernetes.io/name: sart-controller 8 | spec: 9 | selector: 10 | matchLabels: 11 | control-plane: sart-controller 12 | replicas: 1 13 | template: 14 | metadata: 15 | labels: 16 | control-plane: sart-controller 17 | spec: 18 | securityContext: 19 | runAsNonRoot: false 20 | containers: 21 | - name: controller 22 | command: 23 | - sartd 24 | args: 25 | - controller 26 | image: sart:dev 27 | securityContext: 28 | allowPrivilegeEscalation: false 29 | capabilities: 30 | drop: 31 | - "ALL" 32 | livenessProbe: 33 | httpGet: 34 | path: /healthz 35 | port: 8080 36 | initialDelaySeconds: 15 37 | periodSeconds: 20 38 | readinessProbe: 39 | httpGet: 40 | path: /readyz 41 | port: 8080 42 | initialDelaySeconds: 5 43 | periodSeconds: 10 44 | # TODO(user): Configure the resources accordingly based on the project requirements. 45 | # More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ 46 | resources: 47 | limits: 48 | cpu: 100m 49 | memory: 1Gi 50 | requests: 51 | cpu: 10m 52 | memory: 128Mi 53 | volumeMounts: 54 | - mountPath: /etc/sartd/cert 55 | name: cert 56 | readOnly: true 57 | affinity: 58 | podAntiAffinity: 59 | requiredDuringSchedulingIgnoredDuringExecution: 60 | - labelSelector: 61 | matchExpressions: 62 | - key: control-plane 63 | operator: In 64 | values: 65 | - sart-controller 66 | topologyKey: "kubernetes.io/hostname" 67 | topologySpreadConstraints: 68 | - maxSkew: 1 69 | topologyKey: kubernetes.io/hostname 70 | whenUnsatisfiable: DoNotSchedule 71 | labelSelector: 72 | matchLabels: 73 | control-plane: sart-controller 74 | volumes: 75 | - name: cert 76 | secret: 77 | defaultMode: 420 78 | secretName: sart-webhook-server-cert 79 | serviceAccountName: sart 80 | terminationGracePeriodSeconds: 10 81 | -------------------------------------------------------------------------------- /manifests/base/workloads/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - controller.yaml 3 | - agent.yaml 4 | - speaker.yaml 5 | 6 | namespace: kube-system 7 | -------------------------------------------------------------------------------- /manifests/base/workloads/speaker.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: DaemonSet 3 | metadata: 4 | labels: 5 | app: sartd 6 | component: bgp 7 | name: sartd-bgp 8 | namespace: system 9 | spec: 10 | selector: 11 | matchLabels: 12 | app: sartd 13 | component: bgp 14 | template: 15 | metadata: 16 | annotations: 17 | prometheus.io/port: "7472" 18 | prometheus.io/scrape: "true" 19 | labels: 20 | app: sartd 21 | component: bgp 22 | spec: 23 | containers: 24 | - command: ["sartd"] 25 | args: 26 | - bgp 27 | - --exporter=127.0.0.1:5003 28 | env: 29 | - name: SARTD_BGP_NODE_IP 30 | valueFrom: 31 | fieldRef: 32 | fieldPath: status.hostIP 33 | image: sart:dev 34 | imagePullPolicy: IfNotPresent 35 | name: bgp 36 | securityContext: 37 | allowPrivilegeEscalation: false 38 | capabilities: 39 | add: 40 | - NET_RAW 41 | drop: 42 | - ALL 43 | readOnlyRootFilesystem: true 44 | hostNetwork: true 45 | nodeSelector: 46 | kubernetes.io/os: linux 47 | # serviceAccountName: sartd 48 | terminationGracePeriodSeconds: 2 49 | tolerations: 50 | - effect: NoSchedule 51 | operator: Exists 52 | - effect: NoSchedule 53 | operator: Exists 54 | -------------------------------------------------------------------------------- /manifests/cni/agent-patch.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: DaemonSet 3 | metadata: 4 | name: sartd-agent 5 | namespace: kube-system 6 | spec: 7 | template: 8 | spec: 9 | containers: 10 | - name: agent 11 | args: 12 | - agent 13 | - --mode=cni 14 | - --cni-endpoint=0.0.0.0:6000 15 | volumeMounts: 16 | - mountPath: /var/run 17 | name: run 18 | mountPropagation: HostToContainer # to see bind mount netns file under /run/netns 19 | - mountPath: /run 20 | name: run2 21 | mountPropagation: HostToContainer # to see bind mount netns file under /run/netns 22 | initContainers: 23 | - name: installer 24 | image: sart:dev 25 | command: ["cni-installer"] 26 | securityContext: 27 | privileged: true 28 | env: 29 | - name: CNI_NETCONF 30 | valueFrom: 31 | configMapKeyRef: 32 | name: cni-conf 33 | key: netconf.json 34 | volumeMounts: 35 | - mountPath: /opt/cni/bin 36 | name: cni-bin 37 | - mountPath: /etc/cni/net.d 38 | name: cni-conf 39 | volumes: 40 | - name: run 41 | hostPath: 42 | path: /var/run 43 | - name: run2 44 | hostPath: 45 | path: /run 46 | - name: cni-bin 47 | hostPath: 48 | path: /opt/cni/bin 49 | - name: cni-conf 50 | hostPath: 51 | path: /etc/cni/net.d 52 | -------------------------------------------------------------------------------- /manifests/cni/configmap.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: sartd-fib-config 5 | namespace: kube-system 6 | data: 7 | fib-config.yaml: | 8 | endpoint: 127.0.0.1:5001 9 | channels: 10 | - name: bgp-to-kernel 11 | ip_version: ipv4 12 | subscribers: 13 | - protocol: bgp 14 | endpoint: 127.0.0.1:5010 15 | publishers: 16 | - protocol: kernel 17 | tables: 18 | - 254 19 | -------------------------------------------------------------------------------- /manifests/cni/controller-patch.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: sart-controller 5 | namespace: kube-system 6 | spec: 7 | template: 8 | spec: 9 | containers: 10 | - name: controller 11 | args: 12 | - controller 13 | - --mode=cni 14 | hostNetwork: true 15 | priorityClassName: system-cluster-critical 16 | tolerations: 17 | - key: node-role.kubernetes.io/master 18 | effect: NoSchedule 19 | - key: node-role.kubernetes.io/control-plane 20 | effect: NoSchedule 21 | - key: node.kubernetes.io/not-ready 22 | effect: NoSchedule 23 | -------------------------------------------------------------------------------- /manifests/cni/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - ../base 3 | - configmap.yaml 4 | 5 | patchesStrategicMerge: 6 | - agent-patch.yaml 7 | - controller-patch.yaml 8 | - speaker-patch.yaml 9 | 10 | configMapGenerator: 11 | - name: cni-conf 12 | files: 13 | - ./netconf.json 14 | namespace: kube-system 15 | options: 16 | disableNameSuffixHash: true 17 | -------------------------------------------------------------------------------- /manifests/cni/netconf.json: -------------------------------------------------------------------------------- 1 | { 2 | "cniVersion": "1.0.0", 3 | "name": "sart-network", 4 | "plugins": [ 5 | { 6 | "type": "sart-cni", 7 | "endpoint": "http://localhost:6000", 8 | "timeout": 60 9 | }, 10 | { 11 | "type": "portmap", 12 | "capabilities": { 13 | "portMappings": true 14 | } 15 | } 16 | ] 17 | } 18 | -------------------------------------------------------------------------------- /manifests/cni/sample/bgp_peer.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: sart.terassyi.net/v1alpha2 2 | kind: BGPPeer 3 | metadata: 4 | labels: 5 | bgp: b 6 | bgppeer.sart.terassyi.net/node: sart-control-plane 7 | name: bgppeer-sart-cp-spine0 8 | spec: 9 | addr: 9.9.9.9 10 | asn: 65001 11 | groups: 12 | - to-spine0 13 | nodeBGPRef: sart-control-plane 14 | speaker: 15 | path: 127.0.0.1:5000 16 | --- 17 | apiVersion: sart.terassyi.net/v1alpha2 18 | kind: BGPPeer 19 | metadata: 20 | labels: 21 | bgp: b 22 | bgppeer.sart.terassyi.net/node: sart-control-plane 23 | name: bgppeer-sart-cp-spine1 24 | spec: 25 | addr: 7.7.7.7 26 | asn: 65002 27 | groups: 28 | - to-spine1 29 | nodeBGPRef: sart-control-plane 30 | speaker: 31 | path: 127.0.0.1:5000 32 | -------------------------------------------------------------------------------- /manifests/cni/sample/client.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: client 5 | namespace: test 6 | spec: 7 | containers: 8 | - name: client 9 | image: ghcr.io/terassyi/test-server:0.1.2 10 | -------------------------------------------------------------------------------- /manifests/cni/sample/cluster_bgp_spine0.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: sart.terassyi.net/v1alpha2 2 | kind: ClusterBGP 3 | metadata: 4 | name: clusterbgp-spine0 5 | spec: 6 | nodeSelector: 7 | bgp: a 8 | asnSelector: 9 | from: label 10 | routerIdSelector: 11 | from: internalAddress 12 | speaker: 13 | path: 127.0.0.1:5000 14 | multipath: true 15 | peers: 16 | - peerTemplateRef: bgppeertemplate-spine0 17 | nodeBGPSelector: 18 | bgp: a 19 | -------------------------------------------------------------------------------- /manifests/cni/sample/cluster_bgp_spine1.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: sart.terassyi.net/v1alpha2 2 | kind: ClusterBGP 3 | metadata: 4 | name: clusterbgp-spine1 5 | spec: 6 | nodeSelector: 7 | bgp: a 8 | asnSelector: 9 | from: label 10 | routerIdSelector: 11 | from: internalAddress 12 | speaker: 13 | path: 127.0.0.1:5000 14 | multipath: true 15 | peers: 16 | - peerTemplateRef: bgppeertemplate-spine1 17 | nodeBGPSelector: 18 | bgp: a 19 | -------------------------------------------------------------------------------- /manifests/cni/sample/deployment.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: Deployment 4 | metadata: 5 | name: app 6 | namespace: test 7 | spec: 8 | replicas: 3 9 | selector: 10 | matchLabels: 11 | app: app 12 | template: 13 | metadata: 14 | labels: 15 | app: app 16 | spec: 17 | containers: 18 | - name: app 19 | image: ghcr.io/terassyi/test-server:0.1.2 20 | --- 21 | # LoadBalancer Service 22 | apiVersion: v1 23 | kind: Service 24 | metadata: 25 | name: app-svc 26 | namespace: test 27 | spec: 28 | selector: 29 | app: app 30 | ports: 31 | - name: http 32 | port: 80 33 | targetPort: 8080 34 | -------------------------------------------------------------------------------- /manifests/cni/sample/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - cluster_bgp_spine0.yaml 3 | - cluster_bgp_spine1.yaml 4 | - peer_template.yaml 5 | - bgp_peer.yaml 6 | - namespace.yaml 7 | - pool.yaml 8 | - deployment.yaml 9 | - client.yaml 10 | -------------------------------------------------------------------------------- /manifests/cni/sample/namespace.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: test 5 | labels: 6 | name: test 7 | -------------------------------------------------------------------------------- /manifests/cni/sample/peer_template.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: sart.terassyi.net/v1alpha2 2 | kind: BGPPeerTemplate 3 | metadata: 4 | name: bgppeertemplate-spine0 5 | spec: 6 | asn: 65001 7 | addr: 9.9.9.9 8 | groups: 9 | - to-spine0 10 | --- 11 | apiVersion: sart.terassyi.net/v1alpha2 12 | kind: BGPPeerTemplate 13 | metadata: 14 | name: bgppeertemplate-spine1 15 | spec: 16 | asn: 65002 17 | addr: 7.7.7.7 18 | groups: 19 | - to-spine1 20 | -------------------------------------------------------------------------------- /manifests/cni/sample/pool.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: sart.terassyi.net/v1alpha2 2 | kind: AddressPool 3 | metadata: 4 | name: default-pod-pool 5 | spec: 6 | cidr: 10.1.0.0/24 7 | type: pod 8 | allocType: bit 9 | blockSize: 29 10 | autoAssign: true 11 | --- 12 | apiVersion: sart.terassyi.net/v1alpha2 13 | kind: AddressPool 14 | metadata: 15 | name: non-default-pod-pool 16 | spec: 17 | cidr: 10.10.0.0/29 18 | type: pod 19 | allocType: bit 20 | blockSize: 32 21 | autoAssign: false 22 | -------------------------------------------------------------------------------- /manifests/cni/sample/test_pod.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: test-cp 5 | namespace: test 6 | spec: 7 | containers: 8 | - name: test 9 | image: ghcr.io/terassyi/test-server:0.1.2 10 | nodeSelector: 11 | kubernetes.io/hostname: sart-control-plane 12 | --- 13 | apiVersion: v1 14 | kind: Pod 15 | metadata: 16 | name: test-worker 17 | namespace: test 18 | spec: 19 | containers: 20 | - name: test 21 | image: ghcr.io/terassyi/test-server:0.1.2 22 | nodeSelector: 23 | kubernetes.io/hostname: sart-worker 24 | --- 25 | apiVersion: v1 26 | kind: Pod 27 | metadata: 28 | name: test-worker2 29 | namespace: test 30 | spec: 31 | containers: 32 | - name: test 33 | image: ghcr.io/terassyi/test-server:0.1.2 34 | nodeSelector: 35 | kubernetes.io/hostname: sart-worker2 36 | --- 37 | apiVersion: v1 38 | kind: Pod 39 | metadata: 40 | name: test-worker3 41 | namespace: test 42 | spec: 43 | containers: 44 | - name: test 45 | image: ghcr.io/terassyi/test-server:0.1.2 46 | nodeSelector: 47 | kubernetes.io/hostname: sart-worker3 48 | --- 49 | apiVersion: v1 50 | kind: Pod 51 | metadata: 52 | name: test-worker3-2 53 | namespace: test 54 | spec: 55 | containers: 56 | - name: test 57 | image: ghcr.io/terassyi/test-server:0.1.2 58 | nodeSelector: 59 | kubernetes.io/hostname: sart-worker3 60 | -------------------------------------------------------------------------------- /manifests/cni/sample/test_pod2.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: test-worker3-3 5 | namespace: test 6 | spec: 7 | containers: 8 | - name: test 9 | image: ghcr.io/terassyi/test-server:0.1.2 10 | nodeSelector: 11 | kubernetes.io/hostname: sart-worker3 12 | -------------------------------------------------------------------------------- /manifests/cni/sample/test_pod_another_pool.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: test-worker-non-default 5 | namespace: test 6 | annotations: 7 | sart.terassyi.net/addresspool: non-default-pod-pool 8 | spec: 9 | containers: 10 | - name: test 11 | image: ghcr.io/terassyi/test-server:0.1.2 12 | nodeSelector: 13 | kubernetes.io/hostname: sart-worker 14 | -------------------------------------------------------------------------------- /manifests/cni/sample/test_pod_in_namespace.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: test-non-default 5 | labels: 6 | name: test 7 | annotations: 8 | sart.terassyi.net/addresspool: non-default-pod-pool 9 | --- 10 | apiVersion: v1 11 | kind: Pod 12 | metadata: 13 | name: test-worker2-non-default 14 | namespace: test-non-default 15 | spec: 16 | containers: 17 | - name: test-non-default 18 | image: ghcr.io/terassyi/test-server:0.1.2 19 | nodeSelector: 20 | kubernetes.io/hostname: sart-worker2 21 | --- 22 | apiVersion: v1 23 | kind: Pod 24 | metadata: 25 | name: test-worker3-non-default 26 | namespace: test-non-default 27 | spec: 28 | containers: 29 | - name: test 30 | image: ghcr.io/terassyi/test-server:0.1.2 31 | nodeSelector: 32 | kubernetes.io/hostname: sart-worker3 33 | -------------------------------------------------------------------------------- /manifests/cni/speaker-patch.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: DaemonSet 3 | metadata: 4 | name: sartd-bgp 5 | namespace: system 6 | spec: 7 | template: 8 | spec: 9 | containers: 10 | - name: bgp 11 | command: ["sartd"] 12 | args: 13 | - bgp 14 | - --fib 15 | - localhost:5010 16 | - --exporter=127.0.0.1:5003 17 | - name: fib 18 | image: sart:dev 19 | imagePullPolicy: IfNotPresent 20 | command: ["sartd"] 21 | args: 22 | - fib 23 | - -f 24 | - /etc/sart/fib-config.yaml 25 | securityContext: 26 | privileged: true 27 | volumeMounts: 28 | - name: sartd-fib-config 29 | mountPath: /etc/sart/fib-config.yaml 30 | subPath: fib-config.yaml 31 | volumes: 32 | - name: sartd-fib-config 33 | configMap: 34 | name: sartd-fib-config 35 | items: 36 | - key: fib-config.yaml 37 | path: fib-config.yaml 38 | -------------------------------------------------------------------------------- /manifests/dual/agent-patch.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: DaemonSet 3 | metadata: 4 | name: sartd-agent 5 | namespace: kube-system 6 | spec: 7 | template: 8 | spec: 9 | containers: 10 | - name: agent 11 | args: 12 | - agent 13 | - --mode=dual 14 | - --cni-endpoint=0.0.0.0:6000 15 | -------------------------------------------------------------------------------- /manifests/dual/controller-patch.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: sart-controller 5 | namespace: kube-system 6 | spec: 7 | template: 8 | spec: 9 | containers: 10 | - name: controller 11 | args: 12 | - controller 13 | - --mode=dual 14 | -------------------------------------------------------------------------------- /manifests/dual/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - ../cni 3 | 4 | patchesStrategicMerge: 5 | - agent-patch.yaml 6 | - controller-patch.yaml 7 | -------------------------------------------------------------------------------- /manifests/lb/agent-patch.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: DaemonSet 3 | metadata: 4 | name: sartd-agent 5 | namespace: kube-system 6 | spec: 7 | template: 8 | spec: 9 | containers: 10 | - name: agent 11 | args: 12 | - agent 13 | - --mode=lb 14 | -------------------------------------------------------------------------------- /manifests/lb/controller-patch.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: sart-controller 5 | namespace: kube-system 6 | spec: 7 | template: 8 | spec: 9 | containers: 10 | - name: controller 11 | args: 12 | - controller 13 | - --mode=lb 14 | -------------------------------------------------------------------------------- /manifests/lb/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - ../base 3 | 4 | patchesStrategicMerge: 5 | - agent-patch.yaml 6 | - controller-patch.yaml 7 | -------------------------------------------------------------------------------- /manifests/lb/sample/bgp_peer.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: sart.terassyi.net/v1alpha2 2 | kind: BGPPeer 3 | metadata: 4 | labels: 5 | bgp: b 6 | bgppeer.sart.terassyi.net/node: sart-control-plane 7 | name: bgppeer-sample-sart-cp 8 | spec: 9 | addr: 9.9.9.9 10 | asn: 65000 11 | groups: 12 | - to-router0 13 | nodeBGPRef: sart-control-plane 14 | speaker: 15 | path: 127.0.0.1:5000 16 | -------------------------------------------------------------------------------- /manifests/lb/sample/cluster_bgp_a.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: sart.terassyi.net/v1alpha2 2 | kind: ClusterBGP 3 | metadata: 4 | name: clusterbgp-sample-a 5 | spec: 6 | nodeSelector: 7 | bgp: a 8 | asnSelector: 9 | from: label 10 | routerIdSelector: 11 | from: internalAddress 12 | speaker: 13 | path: 127.0.0.1:5000 14 | multipath: true 15 | peers: 16 | - peerTemplateRef: bgppeertemplate-sample 17 | nodeBGPSelector: 18 | bgp: a 19 | -------------------------------------------------------------------------------- /manifests/lb/sample/cluster_bgp_b.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: sart.terassyi.net/v1alpha2 2 | kind: ClusterBGP 3 | metadata: 4 | name: clusterbgp-sample-b 5 | spec: 6 | nodeSelector: 7 | bgp: b 8 | asnSelector: 9 | from: label 10 | routerIdSelector: 11 | from: internalAddress 12 | speaker: 13 | path: 127.0.0.1:5000 14 | multipath: true 15 | -------------------------------------------------------------------------------- /manifests/lb/sample/cluster_bgp_c.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: sart.terassyi.net/v1alpha2 2 | kind: ClusterBGP 3 | metadata: 4 | name: clusterbgp-sample-c 5 | spec: 6 | nodeSelector: 7 | bgp2: c 8 | asnSelector: 9 | from: label 10 | routerIdSelector: 11 | from: internalAddress 12 | speaker: 13 | path: 127.0.0.1:5000 14 | multipath: true 15 | peers: 16 | - peerConfig: 17 | asn: 65000 18 | addr: 7.7.7.7 19 | groups: 20 | - to-router1 21 | nodeBGPSelector: 22 | bgp2: c 23 | -------------------------------------------------------------------------------- /manifests/lb/sample/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - cluster_bgp_a.yaml 3 | - cluster_bgp_c.yaml 4 | - peer_template.yaml 5 | - lb_address_pool.yaml 6 | - lb.yaml 7 | - lb_another.yaml 8 | -------------------------------------------------------------------------------- /manifests/lb/sample/lb.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: test 5 | labels: 6 | name: test 7 | --- 8 | apiVersion: apps/v1 9 | kind: Deployment 10 | metadata: 11 | name: app-cluster 12 | namespace: test 13 | spec: 14 | replicas: 2 15 | selector: 16 | matchLabels: 17 | app: app-cluster 18 | template: 19 | metadata: 20 | labels: 21 | app: app-cluster 22 | spec: 23 | containers: 24 | - name: app 25 | image: ghcr.io/terassyi/test-server:0.1.2 26 | ports: 27 | - containerPort: 80 28 | --- 29 | apiVersion: apps/v1 30 | kind: Deployment 31 | metadata: 32 | name: app-local 33 | namespace: test 34 | spec: 35 | replicas: 2 36 | selector: 37 | matchLabels: 38 | app: app-local 39 | template: 40 | metadata: 41 | labels: 42 | app: app-local 43 | spec: 44 | containers: 45 | - name: app 46 | image: ghcr.io/terassyi/test-server:0.1.2 47 | ports: 48 | - containerPort: 80 49 | --- 50 | apiVersion: apps/v1 51 | kind: Deployment 52 | metadata: 53 | name: app-cluster2 54 | namespace: test 55 | spec: 56 | replicas: 2 57 | selector: 58 | matchLabels: 59 | app: app-cluster2 60 | template: 61 | metadata: 62 | labels: 63 | app: app-cluster2 64 | spec: 65 | containers: 66 | - name: app 67 | image: ghcr.io/terassyi/test-server:0.1.2 68 | ports: 69 | - containerPort: 80 70 | --- 71 | # LoadBalancer Service 72 | apiVersion: v1 73 | kind: Service 74 | metadata: 75 | name: app-svc-cluster 76 | namespace: test 77 | annotations: 78 | spec: 79 | type: LoadBalancer 80 | externalTrafficPolicy: Cluster 81 | selector: 82 | app: app-cluster 83 | ports: 84 | - name: http 85 | port: 80 86 | targetPort: 8080 87 | --- 88 | # LoadBalancer Service 89 | apiVersion: v1 90 | kind: Service 91 | metadata: 92 | name: app-svc-local 93 | namespace: test 94 | annotations: 95 | sart.terassyi.net/addresspool: default-lb-pool 96 | spec: 97 | type: LoadBalancer 98 | externalTrafficPolicy: Local 99 | selector: 100 | app: app-local 101 | ports: 102 | - name: http 103 | port: 80 104 | targetPort: 8080 105 | --- 106 | # LoadBalancer Service 107 | apiVersion: v1 108 | kind: Service 109 | metadata: 110 | name: app-svc-cluster2 111 | namespace: test 112 | annotations: 113 | sart.terassyi.net/addresspool: non-default-lb-pool 114 | sart.terassyi.net/loadBalancerIPs: "10.0.100.20" 115 | spec: 116 | type: LoadBalancer 117 | externalTrafficPolicy: Cluster 118 | selector: 119 | app: app-cluster2 120 | ports: 121 | - name: http 122 | port: 80 123 | targetPort: 8080 124 | --- 125 | apiVersion: v1 126 | kind: Pod 127 | metadata: 128 | name: curl 129 | spec: 130 | containers: 131 | - name: curl 132 | image: curlimages/curl 133 | command: ["sleep", "infinity"] 134 | -------------------------------------------------------------------------------- /manifests/lb/sample/lb_address_pool.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: sart.terassyi.net/v1alpha2 2 | kind: AddressPool 3 | metadata: 4 | name: default-lb-pool 5 | spec: 6 | cidr: 10.0.1.0/24 7 | type: service 8 | allocType: bit 9 | autoAssign: true 10 | --- 11 | apiVersion: sart.terassyi.net/v1alpha2 12 | kind: AddressPool 13 | metadata: 14 | name: non-default-lb-pool 15 | spec: 16 | cidr: 10.0.100.0/24 17 | type: service 18 | allocType: bit 19 | blockSize: 24 20 | autoAssign: false 21 | -------------------------------------------------------------------------------- /manifests/lb/sample/lb_another.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: app-cluster3 5 | namespace: test 6 | spec: 7 | replicas: 2 8 | selector: 9 | matchLabels: 10 | app: app-cluster3 11 | template: 12 | metadata: 13 | labels: 14 | app: app-cluster3 15 | spec: 16 | containers: 17 | - name: app 18 | image: nginx:latest 19 | ports: 20 | - containerPort: 80 21 | --- 22 | apiVersion: apps/v1 23 | kind: Deployment 24 | metadata: 25 | name: app-local2 26 | namespace: test 27 | spec: 28 | replicas: 2 29 | selector: 30 | matchLabels: 31 | app: app-local2 32 | template: 33 | metadata: 34 | labels: 35 | app: app-local2 36 | spec: 37 | containers: 38 | - name: app 39 | image: nginx:latest 40 | ports: 41 | - containerPort: 80 42 | --- 43 | # LoadBalancer Service 44 | apiVersion: v1 45 | kind: Service 46 | metadata: 47 | name: app-svc-cluster3 48 | namespace: test 49 | annotations: 50 | spec: 51 | type: LoadBalancer 52 | externalTrafficPolicy: Cluster 53 | selector: 54 | app: app-cluster3 55 | ports: 56 | - name: http 57 | port: 80 58 | targetPort: 80 59 | --- 60 | # LoadBalancer Service 61 | apiVersion: v1 62 | kind: Service 63 | metadata: 64 | name: app-svc-local2 65 | namespace: test 66 | annotations: 67 | sart.terassyi.net/addresspool: non-default-lb-pool 68 | spec: 69 | type: LoadBalancer 70 | externalTrafficPolicy: Local 71 | selector: 72 | app: app-local2 73 | ports: 74 | - name: http 75 | port: 80 76 | targetPort: 80 77 | -------------------------------------------------------------------------------- /manifests/lb/sample/peer_template.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: sart.terassyi.net/v1alpha2 2 | kind: BGPPeerTemplate 3 | metadata: 4 | name: bgppeertemplate-sample 5 | spec: 6 | asn: 65000 7 | addr: 9.9.9.9 8 | groups: 9 | - to-router0 10 | -------------------------------------------------------------------------------- /proto/cni.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package sart.v1; 4 | 5 | import "google/protobuf/empty.proto"; 6 | 7 | service CNIApi { 8 | rpc Add(Args) returns (CNIResult); 9 | rpc Del(Args) returns (CNIResult); 10 | rpc Check(Args) returns (CNIResult); 11 | } 12 | 13 | message Args { 14 | string container_id = 1; 15 | string netns = 2; 16 | string ifname = 3; 17 | repeated string path = 4; 18 | string args = 5; 19 | CNIResult prev_result = 6; 20 | SartConf conf = 7; 21 | string data = 8; 22 | } 23 | 24 | message CNIResult { 25 | repeated Interface interfaces = 1; 26 | repeated IPConf ips = 2; 27 | repeated RouteConf routes = 3; 28 | Dns dns = 4; 29 | } 30 | 31 | message SartConf { 32 | 33 | } 34 | 35 | message Interface { 36 | string name = 1; 37 | string mac = 2; 38 | string sandbox = 3; 39 | } 40 | 41 | message IPConf { 42 | uint32 interface = 1; 43 | string address = 2; 44 | string gateway = 3; 45 | } 46 | 47 | message RouteConf { 48 | string dst = 1; 49 | string gw = 2; 50 | int32 mtu = 3; 51 | int32 advmss = 5; 52 | } 53 | 54 | message Dns { 55 | repeated string nameservers = 1; 56 | string domain = 2; 57 | repeated string search = 3; 58 | repeated string options = 4; 59 | } 60 | 61 | enum ErrorCode { 62 | Unknown = 0; 63 | IncompatibleVersion = 1; 64 | UnsupportedNetworkConfiguration = 2; 65 | NotExist = 3; 66 | InvalidEnvValue = 4; 67 | IOFailure = 5; 68 | FailedToDecode = 6; 69 | InvalidNetworkConfig = 7; 70 | TryAgainLater = 11; 71 | 72 | AlreadyAdded = 120; 73 | } 74 | 75 | message ErrorResult { 76 | ErrorCode code = 1; 77 | string msg = 2; 78 | string details = 3; 79 | } 80 | -------------------------------------------------------------------------------- /proto/fib.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package sart.v1; 4 | 5 | option go_package = "."; 6 | 7 | import "google/protobuf/empty.proto"; 8 | 9 | service FibApi { 10 | rpc GetRoute(GetRouteRequest) returns (GetRouteResponse); 11 | rpc ListRoutes(ListRoutesRequest) returns (ListRoutesResponse); 12 | rpc AddRoute(AddRouteRequest) returns (google.protobuf.Empty); 13 | rpc DeleteRoute(DeleteRouteRequest) returns (google.protobuf.Empty); 14 | rpc AddMultiPathRoute(AddMultiPathRouteRequest) returns (google.protobuf.Empty); 15 | rpc DeleteMultiPathRoute(DeleteMultiPathRouteRequest) returns (google.protobuf.Empty); 16 | } 17 | 18 | message GetRouteRequest { 19 | uint32 table = 1; 20 | IpVersion version = 2; 21 | string destination = 3; 22 | } 23 | 24 | message GetRouteResponse { 25 | Route route = 1; 26 | } 27 | 28 | message ListRoutesRequest { 29 | uint32 table = 1; 30 | IpVersion version = 2; 31 | } 32 | 33 | message ListRoutesResponse { 34 | repeated Route routes = 1; 35 | } 36 | 37 | message AddRouteRequest { 38 | uint32 table = 1; 39 | IpVersion version = 2; 40 | Route route = 3; 41 | bool replace = 4; 42 | } 43 | 44 | message DeleteRouteRequest { 45 | uint32 table = 1; 46 | IpVersion version = 2; 47 | string destination = 3; 48 | } 49 | 50 | message AddMultiPathRouteRequest { 51 | uint32 table = 1; 52 | IpVersion version = 2; 53 | Route route = 3; 54 | } 55 | 56 | message DeleteMultiPathRouteRequest { 57 | uint32 table = 1; 58 | IpVersion version = 2; 59 | string destination = 3; 60 | repeated string gateways = 4; 61 | } 62 | 63 | message Route { 64 | uint32 table = 1; 65 | IpVersion version = 2; 66 | string destination = 3; 67 | Protocol protocol = 4; 68 | Scope scope = 5; 69 | Type type = 6; 70 | repeated NextHop next_hops = 7; 71 | string source = 8; 72 | AdministrativeDistance ad = 9; 73 | uint32 priority = 10; 74 | bool ibgp = 11; 75 | } 76 | 77 | message NextHop { 78 | string gateway = 1; 79 | uint32 weight = 2; 80 | enum NextHopFlags { 81 | EMPTY = 0; DEAD = 1; PERVASIVE = 2; ONLINK = 3; OFFLOAD = 4; LINKDOWN = 16; UNRESOLVED = 32; 82 | } 83 | NextHopFlags flags = 3; 84 | uint32 interface = 4; 85 | } 86 | 87 | // message 88 | enum IpVersion { 89 | Unkown = 0; 90 | V4 = 2; 91 | V6 = 10; 92 | } 93 | 94 | enum AdministrativeDistance { 95 | ADConnected = 0; 96 | ADStatic = 1; 97 | ADEBGP = 20; 98 | ADOSPF = 110; 99 | ADRIP = 120; 100 | ADIBGP = 200; 101 | } 102 | 103 | enum Protocol { 104 | Unspec = 0; 105 | Redirect = 1; 106 | Kernel = 2; 107 | Boot = 3; 108 | Static = 4; 109 | Bgp = 186; 110 | IsIs = 187; 111 | Ospf = 188; 112 | Rip = 189; 113 | } 114 | 115 | enum Type { 116 | UnspecType = 0; 117 | Unicast = 1; 118 | Local = 2; 119 | Broadcast = 3; 120 | Anycast = 4; 121 | Multicast = 5; 122 | Blackhole = 6; 123 | Unreachable = 7; 124 | Prohibit = 8; 125 | Throw = 9; 126 | Nat = 10; 127 | } 128 | 129 | enum Scope { 130 | Universe = 0; 131 | Site = 200; 132 | Link = 253; 133 | Host = 254; 134 | Nowhere = 255; 135 | } 136 | -------------------------------------------------------------------------------- /proto/fib_manager.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package sart.v1; 4 | 5 | option go_package = "."; 6 | 7 | import "fib.proto"; 8 | 9 | service FibManagerApi { 10 | rpc GetChannel(GetChannelRequest) returns (GetChannelResponse); 11 | rpc ListChannel(ListChannelRequest) returns (ListChannelResponse); 12 | rpc GetRoutes(GetRoutesRequest) returns (GetRoutesResponse); 13 | } 14 | 15 | message GetChannelRequest { 16 | string name = 1; 17 | } 18 | 19 | message GetChannelResponse { 20 | Channel channel = 1; 21 | } 22 | 23 | message ListChannelRequest {} 24 | 25 | message ListChannelResponse { 26 | repeated Channel channels = 1; 27 | } 28 | 29 | message Channel { 30 | string name = 1; 31 | repeated ChProtocol subscribers = 2; 32 | repeated ChProtocol publishers = 3; 33 | } 34 | 35 | message ChProtocol { 36 | string type = 1; 37 | string endpoint = 2; // for bgp type 38 | repeated int32 tables = 3; // for kernel type 39 | } 40 | 41 | message GetRoutesRequest { 42 | string channel = 1; 43 | } 44 | 45 | message GetRoutesResponse { 46 | repeated Route routes = 1; 47 | } 48 | -------------------------------------------------------------------------------- /sart/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "sart" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | [target.x86_64-unknown-linux-gnu] 7 | linker = "clang" 8 | rustflags = ["-C", "link-arg=-fuse-ld=mold"] 9 | 10 | [target.aarch64-unknown-linux-gnu] 11 | linker = "clang" 12 | rustflags = ["-C", "link-arg=-fuse-ld=mold"] 13 | 14 | [dependencies] 15 | thiserror = "1.0.38" 16 | clap = { version = "4.1.6", features = ["derive"] } 17 | tonic = "0.11.0" 18 | prost = "0.12.3" 19 | prost-types = "0.12.3" 20 | tokio = { version = "=1.37.0", features = ["full"] } 21 | serde = { version = "1.0.93", features = ["derive"] } 22 | serde_json = "1.0.93" 23 | tabled = "0.15.0" 24 | assert-json-diff = "2.0.2" 25 | 26 | [build-dependencies] 27 | tonic-build = { version = "0.11.0", features = ["prost"] } 28 | prost-build = { version = "0.12.3" } 29 | -------------------------------------------------------------------------------- /sart/build.rs: -------------------------------------------------------------------------------- 1 | use std::env; 2 | use std::path::PathBuf; 3 | 4 | fn main() -> Result<(), Box> { 5 | let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); 6 | 7 | tonic_build::configure() 8 | .build_client(true) 9 | .file_descriptor_set_path(out_dir.join("sart.bin")) // Add this 10 | // .type_attribute( 11 | // ".", 12 | // "#[derive(serde::Serialize)]", 13 | // ) 14 | .out_dir("./src/proto") 15 | .compile( 16 | &[ 17 | "../proto/bgp.proto", 18 | "../proto/fib.proto", 19 | "../proto/fib_manager.proto", 20 | ], 21 | &["../proto"], 22 | ) 23 | .unwrap_or_else(|e| panic!("protobuf compile error: {}", e)); 24 | Ok(()) 25 | } 26 | -------------------------------------------------------------------------------- /sart/src/bgp.rs: -------------------------------------------------------------------------------- 1 | pub(crate) mod cmd; 2 | pub(crate) mod global; 3 | pub(crate) mod neighbor; 4 | pub(crate) mod rib; 5 | -------------------------------------------------------------------------------- /sart/src/bgp/cmd.rs: -------------------------------------------------------------------------------- 1 | use clap::{Parser, Subcommand}; 2 | 3 | use super::{global::GlobalCmd, neighbor::NeighborCmd}; 4 | 5 | #[derive(Debug, Clone, Parser)] 6 | pub(crate) struct BgpCmd { 7 | #[structopt(subcommand)] 8 | pub scope: Scope, 9 | 10 | #[arg( 11 | short = 'e', 12 | long, 13 | global = true, 14 | required = false, 15 | default_value = "localhost:5000", 16 | help = "Endpoint to BGP API server" 17 | )] 18 | pub endpoint: String, 19 | } 20 | 21 | #[derive(Debug, Clone, Subcommand)] 22 | pub(crate) enum Scope { 23 | Global(GlobalCmd), 24 | Neighbor(NeighborCmd), 25 | } 26 | -------------------------------------------------------------------------------- /sart/src/bgp/global.rs: -------------------------------------------------------------------------------- 1 | use clap::{Parser, Subcommand}; 2 | 3 | use crate::data::bgp::BgpInfo; 4 | use crate::error::Error; 5 | use crate::proto::sart::{HealthRequest, SetAsRequest, SetRouterIdRequest, ConfigureMultiPathRequest}; 6 | use crate::{cmd::Output, proto::sart::GetBgpInfoRequest, rpc::connect_bgp}; 7 | 8 | use super::rib::RibCmd; 9 | 10 | #[derive(Debug, Clone, Parser)] 11 | pub(crate) struct GlobalCmd { 12 | #[structopt(subcommand)] 13 | pub action: Action, 14 | } 15 | 16 | #[derive(Debug, Clone, Subcommand)] 17 | pub(crate) enum Action { 18 | Get, 19 | Set { 20 | #[arg(long, help = "Local AS Number")] 21 | asn: Option, 22 | 23 | #[arg(long, help = "Local Router Id")] 24 | router_id: Option, 25 | 26 | #[arg(long, help = "Multi path")] 27 | multi_path: Option 28 | }, 29 | Rib(RibCmd), 30 | Policy, 31 | Health, 32 | } 33 | 34 | pub(crate) async fn get(endpoint: &str, format: Output) -> Result<(), Error> { 35 | let mut client = connect_bgp(endpoint).await; 36 | let res = client 37 | .get_bgp_info(GetBgpInfoRequest {}) 38 | .await 39 | .map_err(Error::FailedToGetResponse)?; 40 | 41 | match format { 42 | Output::Json => { 43 | // TODO: fix me: I can't serialize prost_types::Any to Json 44 | let info = match &res.get_ref().info { 45 | Some(info) => info, 46 | None => return Err(Error::InvalidRPCResponse), 47 | }; 48 | let info_data = BgpInfo::from(info); 49 | let json_data = serde_json::to_string_pretty(&info_data).map_err(Error::Serialize)?; 50 | println!("{json_data}"); 51 | } 52 | Output::Plain => { 53 | let info = match &res.get_ref().info { 54 | Some(info) => info, 55 | None => return Err(Error::InvalidRPCResponse), 56 | }; 57 | println!("Sartd BGP server is running at {}.", info.port,); 58 | println!(" ASN: {}, Router Id: {}", info.asn, info.router_id); 59 | println!(" Multi Path enabled: {}", info.multi_path); 60 | } 61 | } 62 | Ok(()) 63 | } 64 | 65 | pub(crate) async fn set( 66 | endpoint: &str, 67 | asn: Option, 68 | router_id: Option, 69 | multi_path: Option, 70 | ) -> Result<(), Error> { 71 | let mut client = connect_bgp(endpoint).await; 72 | if asn.is_none() && router_id.is_none() && multi_path.is_none() { 73 | return Err(Error::MissingArgument { 74 | msg: "--asn or --router-id or --multi-path".to_string(), 75 | }); 76 | } 77 | if let Some(asn) = asn { 78 | let _res = client 79 | .set_as(SetAsRequest { asn }) 80 | .await 81 | .map_err(Error::FailedToGetResponse)?; 82 | } 83 | if let Some(router_id) = router_id { 84 | let _res = client 85 | .set_router_id(SetRouterIdRequest { router_id }) 86 | .await 87 | .map_err(Error::FailedToGetResponse)?; 88 | } 89 | if let Some(multi_path) = multi_path { 90 | let _res = client 91 | .configure_multi_path(ConfigureMultiPathRequest{ enable: multi_path }) 92 | .await 93 | .map_err(Error::FailedToGetResponse)?; 94 | } 95 | Ok(()) 96 | } 97 | 98 | pub(crate) async fn health(endpoint: &str) -> Result<(), Error> { 99 | let mut client = connect_bgp(endpoint).await; 100 | client 101 | .health(HealthRequest {}) 102 | .await 103 | .map_err(Error::FailedToGetResponse)?; 104 | Ok(()) 105 | } 106 | -------------------------------------------------------------------------------- /sart/src/cmd.rs: -------------------------------------------------------------------------------- 1 | use clap::{Parser, Subcommand, ValueEnum}; 2 | 3 | use crate::{bgp::cmd::BgpCmd, fib::cmd::FibCmd}; 4 | 5 | #[derive(Parser, Debug)] 6 | #[command(author, version, about, long_about = None)] 7 | pub(crate) struct Cmd { 8 | #[arg( 9 | value_enum, 10 | short = 'o', 11 | long, 12 | global = true, 13 | required = false, 14 | default_value = "plain", 15 | help = "Output format" 16 | )] 17 | pub output: Output, 18 | #[clap(subcommand)] 19 | pub sub: SubCmd, 20 | } 21 | 22 | #[derive(Debug, Clone, Subcommand)] 23 | pub(crate) enum SubCmd { 24 | Bgp(BgpCmd), 25 | Fib(FibCmd), 26 | } 27 | 28 | #[derive(Debug, Clone, Parser, ValueEnum)] 29 | pub(crate) enum Output { 30 | Plain, 31 | Json, 32 | } 33 | -------------------------------------------------------------------------------- /sart/src/data.rs: -------------------------------------------------------------------------------- 1 | pub mod bgp; 2 | -------------------------------------------------------------------------------- /sart/src/error.rs: -------------------------------------------------------------------------------- 1 | use thiserror::Error; 2 | use tonic::Status; 3 | 4 | #[derive(Debug, Error)] 5 | pub enum Error { 6 | #[error("missing argument: {}", msg)] 7 | MissingArgument { msg: String }, 8 | #[error("invalid RPC response")] 9 | InvalidRPCResponse, 10 | #[error("failed to get Response")] 11 | FailedToGetResponse(#[from] Status), 12 | #[error("unacceptable attribute")] 13 | UnacceptableAttribute, 14 | #[error("invalid origin value: acceptable")] 15 | InvalidOriginValue, 16 | #[error("invalid channel type")] 17 | InvalidChannelType, 18 | #[error("Failed to serialize")] 19 | Serialize(#[source] serde_json::Error), 20 | #[error("Invalid BGP state {0}")] 21 | InvalidBGPState(u32), 22 | } 23 | -------------------------------------------------------------------------------- /sart/src/fib.rs: -------------------------------------------------------------------------------- 1 | pub(crate) mod cmd; 2 | pub(crate) mod channel; 3 | pub(crate) mod route; 4 | -------------------------------------------------------------------------------- /sart/src/fib/cmd.rs: -------------------------------------------------------------------------------- 1 | use clap::{Parser, Subcommand}; 2 | 3 | use super::channel::ChannelCmd; 4 | 5 | #[derive(Debug, Clone, Parser)] 6 | pub(crate) struct FibCmd { 7 | #[structopt(subcommand)] 8 | pub scope: Scope, 9 | 10 | 11 | #[arg( 12 | short = 'e', 13 | long, 14 | global = true, 15 | required = false, 16 | default_value = "localhost:5001", 17 | help = "Endpoint to FIB API server" 18 | )] 19 | pub endpoint: String, 20 | } 21 | 22 | #[derive(Debug, Clone, Subcommand)] 23 | pub(crate) enum Scope { 24 | Channel(ChannelCmd), 25 | } 26 | -------------------------------------------------------------------------------- /sart/src/fib/route.rs: -------------------------------------------------------------------------------- 1 | use clap::Parser; 2 | use tabled::{Table, Tabled}; 3 | 4 | use crate::{cmd::Output, error::Error, proto::sart::GetRoutesRequest, rpc::connect_fib}; 5 | 6 | use crate::proto::sart::Route; 7 | 8 | #[derive(Debug, Clone, Parser)] 9 | pub(crate) struct RouteCmd { 10 | pub name: String, 11 | } 12 | 13 | pub(crate) async fn get(endpoint: &str, format: Output, name: String) -> Result<(), Error> { 14 | let mut client = connect_fib(endpoint).await; 15 | 16 | let res = client 17 | .get_routes(GetRoutesRequest { channel: name }) 18 | .await 19 | .map_err(Error::FailedToGetResponse)?; 20 | 21 | match format { 22 | Output::Json => {} 23 | Output::Plain => { 24 | let display: Vec = res 25 | .get_ref() 26 | .routes 27 | .iter() 28 | .map(DisplayedRoute::from) 29 | .collect(); 30 | let data = Table::new(display).to_string(); 31 | println!("{}", data); 32 | } 33 | } 34 | Ok(()) 35 | } 36 | 37 | #[derive(Tabled)] 38 | pub(crate) struct DisplayedRoute { 39 | destinaiton: String, 40 | next_hops: String, 41 | protocol: String, 42 | scope: String, 43 | kind: String, 44 | ad: i32, 45 | priority: u32, 46 | } 47 | 48 | impl From<&Route> for DisplayedRoute { 49 | fn from(route: &Route) -> Self { 50 | let mut next_hops_str = String::new(); 51 | for nh in route.next_hops.iter() { 52 | next_hops_str += &format!("{} ", nh.gateway); 53 | } 54 | 55 | let protocol_str = match route.protocol { 56 | 0 => "Unspec".to_string(), 57 | 1 => "Redirect".to_string(), 58 | 2 => "Kernel".to_string(), 59 | 3 => "Boot".to_string(), 60 | 4 => "Static".to_string(), 61 | 186 => "Bgp".to_string(), 62 | 187 => "IsIs".to_string(), 63 | 188 => "Ospf".to_string(), 64 | 189 => "Rib".to_string(), 65 | _ => format!("Unknown({})", route.protocol), 66 | }; 67 | 68 | let scope_str = match route.scope { 69 | 0 => "Universe".to_string(), 70 | 200 => "Site".to_string(), 71 | 253 => "Link".to_string(), 72 | 254 => "Host".to_string(), 73 | 255 => "Nowhere".to_string(), 74 | _ => format!("Unknown({})", route.scope), 75 | }; 76 | 77 | let kind_str = match route.r#type { 78 | 0 => "UnspecType".to_string(), 79 | 1 => "Unicast".to_string(), 80 | 2 => "Local".to_string(), 81 | 3 => "Broadcast".to_string(), 82 | 4 => "Anycast".to_string(), 83 | 5 => "Multicast".to_string(), 84 | 6 => "Blackhole".to_string(), 85 | 7 => "Unreachable".to_string(), 86 | 8 => "Prohibit".to_string(), 87 | 9 => "Throw".to_string(), 88 | 10 => "Nat".to_string(), 89 | _ => format!("Unknown({})", route.r#type), 90 | }; 91 | 92 | DisplayedRoute { 93 | destinaiton: route.destination.clone(), 94 | next_hops: next_hops_str, 95 | protocol: protocol_str.to_string(), 96 | scope: scope_str.to_string(), 97 | kind: kind_str.to_string(), 98 | ad: route.ad, 99 | priority: route.priority, 100 | } 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /sart/src/main.rs: -------------------------------------------------------------------------------- 1 | pub(crate) mod bgp; 2 | pub(crate) mod cmd; 3 | pub(crate) mod data; 4 | pub(crate) mod error; 5 | pub(crate) mod fib; 6 | pub(crate) mod proto; 7 | pub(crate) mod rpc; 8 | pub(crate) mod util; 9 | 10 | use std::net::Ipv4Addr; 11 | 12 | use bgp::cmd::Scope; 13 | use clap::Parser; 14 | use cmd::{Cmd, SubCmd}; 15 | 16 | #[tokio::main] 17 | async fn main() { 18 | let command = Cmd::parse(); 19 | 20 | let output = command.output; 21 | 22 | match command.sub { 23 | SubCmd::Bgp(sub) => match sub.scope { 24 | Scope::Global(g) => match g.action { 25 | bgp::global::Action::Get => bgp::global::get(&sub.endpoint, output).await.unwrap(), 26 | bgp::global::Action::Set { asn, router_id, multi_path } => { 27 | if let Some(router_id) = router_id.as_ref() { 28 | let _: Ipv4Addr = router_id.parse().unwrap(); 29 | } 30 | bgp::global::set(&sub.endpoint, asn, router_id, multi_path) 31 | .await 32 | .unwrap(); 33 | } 34 | bgp::global::Action::Rib(r) => match r.action { 35 | bgp::rib::Action::Add { 36 | prefixes, 37 | attributes, 38 | } => bgp::rib::add(&sub.endpoint, r.afi, r.safi, prefixes, attributes) 39 | .await 40 | .unwrap(), 41 | bgp::rib::Action::Get => bgp::rib::get(&sub.endpoint, output, r.afi, r.safi) 42 | .await 43 | .unwrap(), 44 | bgp::rib::Action::Del { prefixes } => { 45 | bgp::rib::del(&sub.endpoint, r.afi, r.safi, prefixes) 46 | .await 47 | .unwrap() 48 | } 49 | }, 50 | bgp::global::Action::Health => bgp::global::health(&sub.endpoint).await.unwrap(), 51 | _ => unimplemented!(), 52 | }, 53 | Scope::Neighbor(n) => match n.action { 54 | bgp::neighbor::Action::Add { 55 | name, 56 | addr, 57 | r#as: asn, 58 | passive, 59 | } => bgp::neighbor::add(&sub.endpoint, name, &addr, asn, passive) 60 | .await 61 | .unwrap(), 62 | bgp::neighbor::Action::Get { addr } => { 63 | bgp::neighbor::get(&sub.endpoint, output, &addr) 64 | .await 65 | .unwrap() 66 | } 67 | bgp::neighbor::Action::List => { 68 | bgp::neighbor::list(&sub.endpoint, output).await.unwrap() 69 | } 70 | bgp::neighbor::Action::Del { addr } => { 71 | bgp::neighbor::delete(&sub.endpoint, &addr).await.unwrap() 72 | } 73 | bgp::neighbor::Action::Rib { 74 | addr, 75 | kind, 76 | afi, 77 | safi, 78 | } => bgp::neighbor::rib(&sub.endpoint, output, addr, kind, afi, safi) 79 | .await 80 | .unwrap(), 81 | _ => unimplemented!(), 82 | }, 83 | }, 84 | SubCmd::Fib(sub) => match sub.scope { 85 | fib::cmd::Scope::Channel(ch) => match ch.action { 86 | fib::channel::Action::Get { name } => { 87 | fib::channel::get(&sub.endpoint, output, name) 88 | .await 89 | .unwrap() 90 | } 91 | fib::channel::Action::List => { 92 | fib::channel::list(&sub.endpoint, output).await.unwrap() 93 | } 94 | fib::channel::Action::Route(r) => fib::route::get(&sub.endpoint, output, r.name) 95 | .await 96 | .unwrap(), 97 | }, 98 | }, 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /sart/src/proto/google.protobuf.rs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terassyi/sart/82d8bbeb0224e43cc80a03e6198a98226d2ede47/sart/src/proto/google.protobuf.rs -------------------------------------------------------------------------------- /sart/src/proto/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod sart { 2 | include!("sart.v1.rs"); 3 | } 4 | pub mod google { 5 | pub mod protobuf { 6 | include!("google.protobuf.rs"); 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /sart/src/rpc.rs: -------------------------------------------------------------------------------- 1 | use crate::proto; 2 | 3 | pub(crate) async fn connect_bgp( 4 | endpoint: &str, 5 | ) -> proto::sart::bgp_api_client::BgpApiClient { 6 | let endpoint_url = format!("http://{}", endpoint); 7 | proto::sart::bgp_api_client::BgpApiClient::connect(endpoint_url) 8 | .await 9 | .unwrap() 10 | } 11 | 12 | pub(crate) async fn connect_fib(endpoint: &str) -> proto::sart::fib_manager_api_client::FibManagerApiClient { 13 | let endpoint_url = format!("http://{}", endpoint); 14 | proto::sart::fib_manager_api_client::FibManagerApiClient::connect(endpoint_url) 15 | .await 16 | .unwrap() 17 | } 18 | -------------------------------------------------------------------------------- /sart/src/util.rs: -------------------------------------------------------------------------------- 1 | const TYPE_URL_PREFACE: &str = "type.googleapis.com/sart.v1."; 2 | 3 | pub(crate) fn to_any(m: T, name: &str) -> prost_types::Any { 4 | let mut v = Vec::new(); 5 | m.encode(&mut v).unwrap(); 6 | prost_types::Any { 7 | type_url: format!("{}{}", TYPE_URL_PREFACE, name), 8 | value: v, 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /sartcni/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "sart-cni" 3 | version = "0.6.0" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | assert-json-diff = "2.0.2" 10 | bytes = "1.5.0" 11 | ipnet = "2.9.0" 12 | rstest = "0.19.0" 13 | serde = { version = "1.0.195", features = ["derive"] } 14 | serde_json = "1.0.111" 15 | thiserror = "1.0.56" 16 | tracing = "0.1.40" 17 | tonic-build = "0.11.0" 18 | tonic = "0.11.0" 19 | prost = "0.12.3" 20 | prost-types = "0.12.3" 21 | tokio = { version = "1.35.1", features = ["time", "rt-multi-thread", "macros"] } 22 | rscni = { version = "0.0.4", features = ["async"] } 23 | tokio-stream = { version = "0.1.14", features = ["net"] } 24 | 25 | [build-dependencies] 26 | built = "0.7.1" 27 | tonic-build = { version = "0.11.0", features = ["prost"] } 28 | prost-build = { version = "0.12.3" } 29 | -------------------------------------------------------------------------------- /sartcni/build.rs: -------------------------------------------------------------------------------- 1 | use std::env; 2 | use std::path::PathBuf; 3 | 4 | fn main() { 5 | 6 | let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); 7 | let dir = std::fs::read_dir("../proto").unwrap(); 8 | let mut files: Vec = Vec::new(); 9 | for p in dir.into_iter() { 10 | files.push(p.unwrap().path()); 11 | } 12 | println!("{:?}", files); 13 | 14 | tonic_build::configure() 15 | .build_client(true) 16 | .file_descriptor_set_path(out_dir.join("sart.bin")) // Add this 17 | .out_dir("./src/proto") 18 | .compile(&["../proto/cni.proto"], &["../proto"]) 19 | .unwrap_or_else(|e| panic!("protobuf compile error: {}", e)); 20 | } 21 | -------------------------------------------------------------------------------- /sartcni/config/.cargo: -------------------------------------------------------------------------------- 1 | [target.x86_64-unknown-linux-gnu] 2 | runner = 'sudo -E' 3 | -------------------------------------------------------------------------------- /sartcni/src/cmd.rs: -------------------------------------------------------------------------------- 1 | pub mod add; 2 | pub mod check; 3 | pub mod del; 4 | -------------------------------------------------------------------------------- /sartcni/src/config.rs: -------------------------------------------------------------------------------- 1 | use rscni::{error::Error, types::Args}; 2 | use serde::{Deserialize, Serialize}; 3 | 4 | const SART_CONFIG_ENDPOINT: &str = "endpoint"; 5 | const SART_CONFIG_TIMEOUT: &str = "timeout"; 6 | 7 | #[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] 8 | pub(crate) struct Config { 9 | pub(crate) endpoint: String, 10 | pub(crate) timeout: u64, 11 | } 12 | 13 | impl Config { 14 | pub fn parse(conf: &Args) -> Result { 15 | match &conf.config { 16 | Some(config) => { 17 | let endpoint = config 18 | .custom 19 | .get(SART_CONFIG_ENDPOINT) 20 | .ok_or(Error::InvalidNetworkConfig( 21 | "endpoint must be given.".to_string(), 22 | ))? 23 | .as_str() 24 | .ok_or(Error::InvalidNetworkConfig( 25 | "endpoint parameter must be given".to_string(), 26 | ))? 27 | .to_string(); 28 | let timeout = config 29 | .custom 30 | .get(SART_CONFIG_TIMEOUT) 31 | .ok_or(Error::InvalidNetworkConfig( 32 | "timeout parameter must be given".to_string(), 33 | ))? 34 | .as_u64() 35 | .ok_or(Error::InvalidNetworkConfig( 36 | "timeout parameter must be number".to_string(), 37 | ))?; 38 | 39 | Ok(Config { endpoint, timeout }) 40 | } 41 | None => Err(Error::InvalidNetworkConfig( 42 | "configuration must be given from stdin".to_string(), 43 | )), 44 | } 45 | } 46 | } 47 | 48 | #[cfg(test)] 49 | mod tests { 50 | use std::collections::HashMap; 51 | 52 | use rscni::types::NetConf; 53 | use serde_json::json; 54 | 55 | use super::*; 56 | 57 | #[test] 58 | fn parse_config_from_net_conf() { 59 | let args = Args { 60 | config: Some(NetConf { 61 | custom: HashMap::from([ 62 | ("endpoint".to_string(), json!("test")), 63 | ("timeout".to_string(), json!(60)), 64 | ]), 65 | ..Default::default() 66 | }), 67 | ..Default::default() 68 | }; 69 | 70 | let conf = Config::parse(&args).unwrap(); 71 | assert_eq!( 72 | Config { 73 | endpoint: "test".to_string(), 74 | timeout: 60, 75 | }, 76 | conf 77 | ); 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /sartcni/src/error.rs: -------------------------------------------------------------------------------- 1 | pub(crate) const ERROR_CODE_GRPC: u32 = 100; 2 | pub(crate) const ERROR_CODE_TIMEOUT: u32 = 110; 3 | 4 | pub(crate) const ERROR_MSG_GRPC: &str = "Failed to connect gPRC server"; 5 | pub(crate) const ERROR_MSG_TIMEOUT: &str = "Timeout for request"; 6 | -------------------------------------------------------------------------------- /sartcni/src/main.rs: -------------------------------------------------------------------------------- 1 | use cmd::{add::add, check::check, del::del}; 2 | use rscni::{async_skel::Plugin, version::PluginInfo}; 3 | use version::{CNI_VERSION, SUPPORTED_VERSIONS}; 4 | 5 | mod cmd; 6 | mod config; 7 | mod error; 8 | mod mock; 9 | mod proto; 10 | mod version; 11 | 12 | #[tokio::main] 13 | async fn main() { 14 | let version_info = PluginInfo::new( 15 | CNI_VERSION, 16 | SUPPORTED_VERSIONS 17 | .to_vec() 18 | .iter() 19 | .map(|v| v.to_string()) 20 | .collect::>(), 21 | ); 22 | let mut plugin = Plugin::new(add, del, check, version_info, &get_about_info()); 23 | 24 | match plugin.run().await { 25 | Ok(_) => std::process::exit(0), 26 | Err(e) => std::process::exit(1), 27 | } 28 | } 29 | 30 | fn get_about_info() -> String { 31 | format!( 32 | "Sart CNI plugin", 33 | ) 34 | } 35 | -------------------------------------------------------------------------------- /sartcni/src/proto/google.protobuf.rs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terassyi/sart/82d8bbeb0224e43cc80a03e6198a98226d2ede47/sartcni/src/proto/google.protobuf.rs -------------------------------------------------------------------------------- /sartcni/src/version.rs: -------------------------------------------------------------------------------- 1 | pub const CNI_VERSION: &str = "1.0.0"; 2 | 3 | pub const SUPPORTED_VERSIONS: [&str; 6] = ["0.1.0", "0.2.0", "0.3.0", "0.3.1", "0.4.0", "1.0.0"]; 4 | -------------------------------------------------------------------------------- /sartd/.cargo/config: -------------------------------------------------------------------------------- 1 | [target.x86_64-unknown-linux-gnu] 2 | runner = 'sudo -E' 3 | -------------------------------------------------------------------------------- /sartd/.rustfmt.toml: -------------------------------------------------------------------------------- 1 | format_code_in_doc_comments = true 2 | group_imports = "StdExternalCrate" 3 | imports_granularity = "Crate" 4 | imports_layout = "HorizontalVertical" 5 | unstable_features = true 6 | use_field_init_shorthand = true 7 | enum_discrim_align_threshold = 20 8 | -------------------------------------------------------------------------------- /sartd/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "sartd" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | [worksapce] 7 | members = [ 8 | "sartd/bgp", 9 | "sartd/cert", 10 | "sartd/cmd", 11 | "sartd/fib", 12 | "sartd/ipam", 13 | "sartd/kubernetes", 14 | "sartd/mock", 15 | "sartd/proto", 16 | "sartd/trace", 17 | "sartd/util", 18 | ] 19 | 20 | [target.x86_64-unknown-linux-gnu] 21 | linker = "clang" 22 | rustflags = ["-C", "link-arg=-fuse-ld=mold"] 23 | 24 | [target.aarch64-unknown-linux-gnu] 25 | linker = "clang" 26 | rustflags = ["-C", "link-arg=-fuse-ld=mold"] 27 | 28 | [[bin]] 29 | doc = false 30 | name = "sartd" 31 | path = "src/bin/sartd.rs" 32 | 33 | [[bin]] 34 | doc = false 35 | name = "crdgen" 36 | path = "src/bin/crdgen.rs" 37 | 38 | [[bin]] 39 | doc = false 40 | name = "certgen" 41 | path = "src/bin/certgen.rs" 42 | 43 | [[bin]] 44 | doc = false 45 | name = "cni-installer" 46 | path = "src/bin/cni-installer.rs" 47 | 48 | 49 | [dependencies] 50 | anyhow = "1.0.75" 51 | clap = { version = "4.4.8", features = ["derive"] } 52 | kube = { version = "0.87.1" } 53 | k8s-openapi = { version = "0.20.0", features = ["v1_28"] } 54 | # rcgen = "0.11.3" 55 | rcgen = "0.12.0" 56 | serde_yaml = "0.9.27" 57 | time = "0.3.30" 58 | sartd-cmd = { path = "src/cmd" } 59 | sartd-kubernetes = { path = "src/kubernetes" } 60 | rscni = { git = "https://github.com/terassyi/rscni", branch = "main" } 61 | 62 | [patch.crates-io] 63 | # netlink-packet-route = { git = "https://github.com/terassyi/netlink-packet-route", branch = "next-hop-default", features = [ 64 | # "rich_nlas", 65 | # ] } 66 | netlink-packet-route = { git = "https://github.com/terassyi/netlink-packet-route", branch = "next-hop-default" } 67 | -------------------------------------------------------------------------------- /sartd/src/bgp/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "sartd-bgp" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | [dependencies] 7 | byteorder = "1.5.0" 8 | bytes = "1.5.0" 9 | futures = "0.3.29" 10 | ipnet = "2.9.0" 11 | prost = "0.12.3" 12 | prost-types = "0.12.3" 13 | serde = { version = "1.0.193", features = ["derive"] } 14 | serde_yaml = "0.9.29" 15 | socket2 = "0.5.5" 16 | thiserror = "1.0.53" 17 | tokio = { version = "1.35.1", features = [ 18 | "time", 19 | "sync", 20 | "net", 21 | "macros", 22 | "test-util", 23 | ] } 24 | tokio-stream = { version = "0.1.14", features = ["net"] } 25 | tokio-util = { version = "0.7.10", features = ["codec"] } 26 | tonic = "0.10.2" 27 | tracing = "0.1.40" 28 | tracing-futures = "0.2.5" 29 | sartd-proto = { path = "../proto" } 30 | sartd-trace = { path = "../trace" } 31 | sartd-util = { path = "../util" } 32 | rstest = "0.18.2" 33 | -------------------------------------------------------------------------------- /sartd/src/bgp/src/lib.rs: -------------------------------------------------------------------------------- 1 | mod api_server; 2 | mod capability; 3 | pub mod config; 4 | mod error; 5 | mod event; 6 | mod family; 7 | mod packet; 8 | mod path; 9 | mod peer; 10 | mod rib; 11 | pub mod server; 12 | -------------------------------------------------------------------------------- /sartd/src/bgp/src/packet.rs: -------------------------------------------------------------------------------- 1 | pub mod attribute; 2 | pub mod capability; 3 | pub mod codec; 4 | pub mod message; 5 | pub mod mock; 6 | pub mod prefix; 7 | -------------------------------------------------------------------------------- /sartd/src/bgp/src/packet/mock.rs: -------------------------------------------------------------------------------- 1 | use core::task::{Context, Poll}; 2 | use std::io::{Read, Write}; 3 | use tokio::io::{AsyncRead, AsyncWrite}; 4 | 5 | use std::io; 6 | use std::pin::Pin; 7 | 8 | pub struct MockTcpStream { 9 | data: Vec, 10 | } 11 | 12 | impl MockTcpStream { 13 | pub fn new(data: Vec) -> Self { 14 | Self { data } 15 | } 16 | } 17 | 18 | impl Read for MockTcpStream { 19 | fn read(&mut self, _buf: &mut [u8]) -> std::io::Result { 20 | todo!() 21 | } 22 | } 23 | 24 | impl AsyncRead for MockTcpStream { 25 | fn poll_read( 26 | self: Pin<&mut Self>, 27 | _: &mut Context, 28 | buf: &mut tokio::io::ReadBuf<'_>, 29 | ) -> Poll> { 30 | buf.put_slice(&self.data); 31 | Poll::Ready(Ok(())) 32 | } 33 | } 34 | 35 | impl Write for MockTcpStream { 36 | fn write(&mut self, buf: &[u8]) -> io::Result { 37 | self.data.copy_from_slice(buf); 38 | Ok(buf.len()) 39 | } 40 | 41 | fn flush(&mut self) -> std::io::Result<()> { 42 | todo!() 43 | } 44 | } 45 | 46 | impl AsyncWrite for MockTcpStream { 47 | fn poll_write( 48 | self: Pin<&mut Self>, 49 | _: &mut Context<'_>, 50 | _buf: &[u8], 51 | ) -> Poll> { 52 | Poll::Ready(Ok(self.data.len())) 53 | } 54 | 55 | fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { 56 | todo!() 57 | } 58 | 59 | fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { 60 | todo!() 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /sartd/src/bgp/src/peer.rs: -------------------------------------------------------------------------------- 1 | pub mod fsm; 2 | pub mod neighbor; 3 | pub mod peer; 4 | -------------------------------------------------------------------------------- /sartd/src/bgp/src/peer/neighbor.rs: -------------------------------------------------------------------------------- 1 | use std::net::{IpAddr, Ipv4Addr}; 2 | 3 | use crate::{capability::Capability, config::NeighborConfig}; 4 | 5 | #[derive(Debug, Clone, Hash, PartialEq, Eq)] 6 | pub struct NeighborPair { 7 | pub addr: IpAddr, 8 | pub asn: u32, 9 | // pub id: Ipv4Addr, 10 | } 11 | 12 | impl NeighborPair { 13 | pub fn new(addr: IpAddr, asn: u32) -> Self { 14 | Self { addr, asn } 15 | } 16 | } 17 | 18 | impl From<&NeighborConfig> for NeighborPair { 19 | fn from(c: &NeighborConfig) -> Self { 20 | Self { 21 | addr: c.address, 22 | asn: c.asn, 23 | } 24 | } 25 | } 26 | 27 | impl From<&Neighbor> for NeighborPair { 28 | fn from(n: &Neighbor) -> Self { 29 | Self { 30 | addr: n.get_addr(), 31 | asn: n.get_asn(), 32 | } 33 | } 34 | } 35 | 36 | #[derive(Debug, Clone)] 37 | pub struct Neighbor { 38 | asn: u32, 39 | router_id: Ipv4Addr, 40 | addr: IpAddr, 41 | acceptable_capabilities: Vec, 42 | hold_time: u16, 43 | } 44 | 45 | impl Neighbor { 46 | pub fn new(asn: u32) -> Self { 47 | Self { 48 | asn, 49 | router_id: Ipv4Addr::new(0, 0, 0, 0), 50 | addr: IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), 51 | acceptable_capabilities: Vec::new(), 52 | hold_time: 0, 53 | } 54 | } 55 | 56 | pub fn get_asn(&self) -> u32 { 57 | self.asn 58 | } 59 | 60 | pub fn get_router_id(&self) -> Ipv4Addr { 61 | self.router_id 62 | } 63 | 64 | pub fn router_id(&mut self, id: Ipv4Addr) -> &mut Self { 65 | self.router_id = id; 66 | self 67 | } 68 | 69 | pub fn get_addr(&self) -> IpAddr { 70 | self.addr 71 | } 72 | 73 | pub fn addr(&mut self, a: IpAddr) -> &mut Self { 74 | self.addr = a; 75 | self 76 | } 77 | 78 | pub fn get_hold_time(&self) -> u16 { 79 | self.hold_time 80 | } 81 | 82 | pub fn hold_time(&mut self, t: u16) -> &mut Self { 83 | self.hold_time = t; 84 | self 85 | } 86 | } 87 | 88 | impl From for Neighbor { 89 | fn from(conf: NeighborConfig) -> Self { 90 | Self { 91 | asn: conf.asn, 92 | router_id: conf.router_id, 93 | addr: conf.address, 94 | acceptable_capabilities: Vec::new(), 95 | hold_time: 180, // default 96 | } 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /sartd/src/bin/certgen.rs: -------------------------------------------------------------------------------- 1 | use std::{fs::File, io::Write, ops::Add}; 2 | 3 | use clap::Parser; 4 | use rcgen::{Certificate, CertificateParams, ExtendedKeyUsagePurpose, KeyUsagePurpose}; 5 | use time::{Duration, OffsetDateTime}; 6 | 7 | const DEFAULT_SERVICE_URL: &str = "sart-webhook-service.kube-system.svc"; 8 | const DEFAULT_OUTPUT_DIR: &str = "."; 9 | 10 | #[derive(Parser)] 11 | struct Args { 12 | /// TLS hostname 13 | #[arg(long)] 14 | #[clap(default_value = DEFAULT_SERVICE_URL)] 15 | host: String, 16 | 17 | #[arg(long)] 18 | #[clap(default_value = DEFAULT_OUTPUT_DIR)] 19 | out_dir: String, 20 | } 21 | 22 | fn main() -> anyhow::Result<()> { 23 | println!("Generate certificate and key files"); 24 | 25 | let arg = Args::parse(); 26 | 27 | let mut params = CertificateParams::new(vec![arg.host]); 28 | params.not_before = OffsetDateTime::now_utc(); 29 | params.not_after = OffsetDateTime::now_utc().add(Duration::hours(36500 * 24)); 30 | params.key_usages = vec![ 31 | KeyUsagePurpose::DigitalSignature, 32 | KeyUsagePurpose::KeyEncipherment, 33 | ]; 34 | params.extended_key_usages = vec![ExtendedKeyUsagePurpose::ServerAuth]; 35 | 36 | let cert = Certificate::from_params(params)?; 37 | 38 | let certificate_pem = cert.serialize_pem()?; 39 | let private_key_pem = cert.serialize_private_key_pem(); 40 | 41 | let mut certificate_pem_file = File::create(format!("{}/tls.cert", arg.out_dir))?; 42 | let c_metadata = certificate_pem_file.metadata()?; 43 | let mut c_permissions = c_metadata.permissions(); 44 | c_permissions.set_readonly(true); 45 | 46 | let mut private_key_pem_file = File::create(format!("{}/tls.key", arg.out_dir))?; 47 | let p_metadata = private_key_pem_file.metadata()?; 48 | let mut p_permissions = p_metadata.permissions(); 49 | p_permissions.set_readonly(true); 50 | 51 | certificate_pem_file.write_all(certificate_pem.as_bytes())?; 52 | private_key_pem_file.write_all(private_key_pem.as_bytes())?; 53 | 54 | Ok(()) 55 | } 56 | -------------------------------------------------------------------------------- /sartd/src/bin/cni-installer.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | use clap::Parser; 3 | use std::{io::Write, path::Path}; 4 | 5 | #[derive(Debug, Parser)] 6 | struct Args { 7 | #[arg(long = "bin-dir", default_value = DEFAULT_BIN_DIR)] 8 | bin_dir: String, 9 | 10 | #[arg(long = "conf-dir", default_value = DEFAULT_CONF_DIR)] 11 | conf_dir: String, 12 | 13 | #[arg(long = "src-bin-dir", default_value = DEFAULT_SRC_BIN_DIR)] 14 | src_bin_dir: String, 15 | 16 | #[arg(long = "src-conf-dir", default_value = DEFAULT_SRC_CONF_DIR)] 17 | src_conf_dir: String, 18 | 19 | #[arg(long)] 20 | binaries: Option>, 21 | } 22 | 23 | const BIN_NAME: &str = "sart-cni"; 24 | 25 | const DEFAULT_SRC_BIN_DIR: &str = "/host/opt/cni/bin"; 26 | const DEFAULT_SRC_CONF_DIR: &str = "/host/etc/cni/net.d"; 27 | 28 | const DEFAULT_BIN_DIR: &str = "/opt/cni/bin"; 29 | const DEFAULT_CONF_DIR: &str = "/etc/cni/net.d"; 30 | const CNI_CONF_ENV_KEY: &str = "CNI_NETCONF"; 31 | const CNI_CONF_NAME: &str = "10-netconf.conflist"; 32 | 33 | fn main() -> Result<()> { 34 | println!("Install CNI binary and configuration file"); 35 | let arg = Args::parse(); 36 | 37 | let binaries = match arg.binaries { 38 | Some(b) => b.clone(), 39 | None => vec![BIN_NAME.to_string()], 40 | }; 41 | 42 | install_cni_binaries(&binaries, &arg.src_bin_dir, &arg.bin_dir)?; 43 | if std::env::var(CNI_CONF_ENV_KEY).is_ok() { 44 | install_cni_conf_from_env(CNI_CONF_ENV_KEY, &arg.conf_dir)?; 45 | } else { 46 | install_cni_conf(&arg.src_conf_dir, &arg.conf_dir)?; 47 | } 48 | 49 | Ok(()) 50 | } 51 | 52 | fn install_cni_binaries(binaries: &[String], src_dir: &str, dst_dir: &str) -> Result<()> { 53 | std::fs::create_dir_all(dst_dir)?; 54 | 55 | let src = Path::new(src_dir); 56 | let dst = Path::new(dst_dir); 57 | for binary in binaries.iter() { 58 | let src_path = src.join(binary); 59 | let dst_path = dst.join(binary); 60 | std::fs::copy(src_path, dst_path)?; 61 | } 62 | Ok(()) 63 | } 64 | 65 | fn install_cni_conf(src: &str, dst: &str) -> Result<()> { 66 | std::fs::create_dir_all(dst)?; 67 | 68 | // clean up existing conf 69 | let files = std::fs::read_dir(dst)?; 70 | for file in files.into_iter() { 71 | let file = file?; 72 | std::fs::remove_file(file.path())?; 73 | } 74 | 75 | let dst_dir = Path::new(dst); 76 | let new_files = std::fs::read_dir(src)?; 77 | for file in new_files.into_iter() { 78 | let file = file?; 79 | std::fs::copy(file.path(), dst_dir.join(file.file_name()))?; 80 | } 81 | 82 | Ok(()) 83 | } 84 | 85 | fn install_cni_conf_from_env(key: &str, dst: &str) -> Result<()> { 86 | std::fs::create_dir_all(dst)?; 87 | // clean up existing conf 88 | let files = std::fs::read_dir(dst)?; 89 | for file in files.into_iter() { 90 | let file = file?; 91 | std::fs::remove_file(file.path())?; 92 | } 93 | 94 | let dst_dir = Path::new(dst); 95 | let conf = std::env::var(key)?; 96 | 97 | let mut file = std::fs::File::create(dst_dir.join(CNI_CONF_NAME))?; 98 | file.write_all(conf.as_bytes())?; 99 | 100 | Ok(()) 101 | } 102 | -------------------------------------------------------------------------------- /sartd/src/bin/crdgen.rs: -------------------------------------------------------------------------------- 1 | use kube::CustomResourceExt; 2 | use sartd_kubernetes::crd; 3 | 4 | fn main() { 5 | print!( 6 | "{}", 7 | serde_yaml::to_string(&crd::cluster_bgp::ClusterBGP::crd()).unwrap() 8 | ); 9 | println!("---"); 10 | print!( 11 | "{}", 12 | serde_yaml::to_string(&crd::node_bgp::NodeBGP::crd()).unwrap() 13 | ); 14 | println!("---"); 15 | print!( 16 | "{}", 17 | serde_yaml::to_string(&crd::bgp_peer_template::BGPPeerTemplate::crd()).unwrap() 18 | ); 19 | println!("---"); 20 | print!( 21 | "{}", 22 | serde_yaml::to_string(&crd::bgp_peer::BGPPeer::crd()).unwrap() 23 | ); 24 | println!("---"); 25 | print!( 26 | "{}", 27 | serde_yaml::to_string(&crd::bgp_advertisement::BGPAdvertisement::crd()).unwrap() 28 | ); 29 | println!("---"); 30 | print!( 31 | "{}", 32 | serde_yaml::to_string(&crd::address_pool::AddressPool::crd()).unwrap() 33 | ); 34 | println!("---"); 35 | print!( 36 | "{}", 37 | serde_yaml::to_string(&crd::address_block::AddressBlock::crd()).unwrap() 38 | ); 39 | println!("---"); 40 | print!( 41 | "{}", 42 | serde_yaml::to_string(&crd::block_request::BlockRequest::crd()).unwrap() 43 | ); 44 | } 45 | -------------------------------------------------------------------------------- /sartd/src/bin/sartd.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | sartd_cmd::cmd::run() 3 | } 4 | -------------------------------------------------------------------------------- /sartd/src/cert/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "sartd-cert" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | [dependencies] 7 | anyhow = "1.0.75" 8 | clap = { version = "4.4.8", features = ["derive"] } 9 | rcgen = "0.12.0" 10 | rustls = "0.21.9" 11 | rustls-pemfile = "1.0.4" 12 | time = "0.3.30" 13 | -------------------------------------------------------------------------------- /sartd/src/cert/src/constants.rs: -------------------------------------------------------------------------------- 1 | pub const DEFAULT_TLS_CERT: &str = "/etc/sartd/cert/tls.crt"; 2 | pub const DEFAULT_TLS_KEY: &str = "/etc/sartd/cert/tls.key"; 3 | -------------------------------------------------------------------------------- /sartd/src/cert/src/lib.rs: -------------------------------------------------------------------------------- 1 | pub mod constants; 2 | pub mod util; 3 | -------------------------------------------------------------------------------- /sartd/src/cert/src/util.rs: -------------------------------------------------------------------------------- 1 | use std::{fs::File, io::BufReader}; 2 | 3 | use rustls::{Certificate, PrivateKey}; 4 | 5 | pub fn load_certificates_from_pem(path: &str) -> std::io::Result> { 6 | let file = File::open(path)?; 7 | let mut reader = BufReader::new(file); 8 | 9 | let certs = rustls_pemfile::certs(&mut reader)?; 10 | 11 | Ok(certs.into_iter().map(Certificate).collect()) 12 | } 13 | 14 | pub fn load_private_key_from_file(path: &str) -> Result> { 15 | let file = File::open(path)?; 16 | let mut reader = BufReader::new(file); 17 | 18 | let mut keys = rustls_pemfile::pkcs8_private_keys(&mut reader)?; 19 | 20 | match keys.len() { 21 | 0 => Err(format!("No PKC8-encoded private key found in {path}").into()), 22 | 1 => Ok(PrivateKey(keys.remove(0))), 23 | _ => Err(format!("More than one PKC8-encoded private key found in {path}").into()), 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /sartd/src/cmd/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "sartd-cmd" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | [dependencies] 7 | clap = { version = "4.1.12", features = ["derive"] } 8 | sartd-bgp = { path = "../bgp" } 9 | sartd-fib = { path = "../fib" } 10 | sartd-kubernetes = { path = "../kubernetes" } 11 | sartd-trace = { path = "../trace" } 12 | -------------------------------------------------------------------------------- /sartd/src/cmd/src/agent.rs: -------------------------------------------------------------------------------- 1 | use clap::Parser; 2 | use sartd_kubernetes::{agent::{cni::server::CNI_SERVER_ENDPOINT, config::{DEFAULT_HTTPS_PORT, DEFAULT_HTTP_PORT}}, config::Mode}; 3 | 4 | #[derive(Debug, Clone, Parser)] 5 | pub struct AgentCmd { 6 | #[arg( 7 | short, 8 | long, 9 | default_value = "127.0.0.5002", 10 | help = "Endpoint URL running Kubernetes agent" 11 | )] 12 | pub endpoint: String, 13 | 14 | #[arg(short = 'f', long, help = "Config file path for Agent daemon")] 15 | pub file: Option, 16 | 17 | #[arg(long = "tls-cert", help = "path to TLS Certificate for agent")] 18 | pub tls_cert: Option, 19 | 20 | #[arg(long = "tls-key", help = "path to TLS Key for agent")] 21 | pub tls_key: Option, 22 | 23 | #[arg(long = "http-port", default_value_t = DEFAULT_HTTP_PORT, help = "HTTP server serving port")] 24 | pub http_port: u32, 25 | 26 | #[arg(long = "https-port", default_value_t = DEFAULT_HTTPS_PORT, help = "HTTPS server serving port")] 27 | pub https_port: u32, 28 | 29 | #[arg( 30 | long = "peer-state-watcher", 31 | help = "Endpoint URL for BGP peer state watcher" 32 | )] 33 | pub peer_state_watcher: Option, 34 | 35 | #[arg( 36 | long = "mode", 37 | help = "Running mode(Default is Dual)", 38 | default_value_t = Mode::Dual, 39 | )] 40 | pub mode: Mode, 41 | 42 | #[arg( 43 | long = "cni-endpoint", 44 | default_value = CNI_SERVER_ENDPOINT, 45 | help = "Endpoint path running CNI server" 46 | )] 47 | pub cni_endpoint: String, 48 | } 49 | -------------------------------------------------------------------------------- /sartd/src/cmd/src/bgp.rs: -------------------------------------------------------------------------------- 1 | use clap::Parser; 2 | 3 | #[derive(Debug, Clone, Parser)] 4 | pub struct BgpCmd { 5 | #[arg(short = 'f', long, help = "Config file path for BGP daemon")] 6 | pub file: Option, 7 | 8 | #[arg(short, long, help = "Local AS Number")] 9 | pub r#as: Option, 10 | 11 | #[arg(short, long, help = "Local router id(must be ipv4 format)")] 12 | pub router_id: Option, 13 | 14 | #[arg(long = "fib", help = "Fib endpoint url(gRPC) exp) localhost:5001")] 15 | pub fib_endpoint: Option, 16 | 17 | #[arg(long = "table-id", help = "Target fib table id(default is main(254))")] 18 | pub fib_table_id: Option, 19 | 20 | #[arg(long = "exporter", help = "Exporter endpoint url")] 21 | pub exporter: Option, 22 | 23 | #[arg( 24 | short, 25 | long, 26 | default_value = "info", 27 | help = "Log level(trace, debug, info, warn, error)" 28 | )] 29 | pub level: String, 30 | } 31 | -------------------------------------------------------------------------------- /sartd/src/cmd/src/controller.rs: -------------------------------------------------------------------------------- 1 | use clap::Parser; 2 | use sartd_kubernetes::{ 3 | config::Mode, 4 | controller::config::{DEFAULT_HTTPS_PORT, DEFAULT_HTTP_PORT}, 5 | }; 6 | 7 | #[derive(Debug, Clone, Parser)] 8 | pub struct ControllerCmd { 9 | #[arg( 10 | short, 11 | long, 12 | default_value = "127.0.0.5003", 13 | help = "Endpoint URL running Kubernetes agent" 14 | )] 15 | pub endpoint: String, 16 | 17 | #[arg(long = "http-port", default_value_t = DEFAULT_HTTP_PORT, help = "HTTP server serving port")] 18 | pub http_port: u32, 19 | 20 | #[arg(long = "https-port", default_value_t = DEFAULT_HTTPS_PORT, help = "HTTPS server serving port")] 21 | pub https_port: u32, 22 | 23 | #[arg(short = 'f', long, help = "Config file path for Kubernetes controller")] 24 | pub file: Option, 25 | 26 | #[arg(long = "tls-cert", help = "path to TLS Certificate for controller")] 27 | pub tls_cert: Option, 28 | 29 | #[arg(long = "tls-key", help = "path to TLS Key for controller")] 30 | pub tls_key: Option, 31 | 32 | #[arg( 33 | long = "mode", 34 | help = "Running mode(Default is Dual)", 35 | default_value_t = Mode::Dual, 36 | )] 37 | pub mode: Mode, 38 | } 39 | -------------------------------------------------------------------------------- /sartd/src/cmd/src/fib.rs: -------------------------------------------------------------------------------- 1 | use clap::Parser; 2 | 3 | #[derive(Debug, Clone, Parser)] 4 | pub struct FibCmd { 5 | #[arg( 6 | short, 7 | long, 8 | default_value = "127.0.0.1:5001", 9 | help = "Fib manager running endpoint url" 10 | )] 11 | pub endpoint: String, 12 | 13 | #[arg( 14 | short = 'f', 15 | long, 16 | required = true, 17 | help = "Config file path for Fib daemon" 18 | )] 19 | pub file: Option, 20 | } 21 | -------------------------------------------------------------------------------- /sartd/src/cmd/src/lib.rs: -------------------------------------------------------------------------------- 1 | mod agent; 2 | mod bgp; 3 | pub mod cmd; 4 | mod controller; 5 | mod fib; 6 | -------------------------------------------------------------------------------- /sartd/src/fib/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "sartd-fib" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | [dependencies] 7 | futures = "0.3.29" 8 | ipnet = "2.9.0" 9 | netlink-packet-core = "0.7.0" 10 | netlink-packet-route = { version = "0.17.1", features = [ 11 | "rich_nlas", 12 | ] } # Ignore 0.18.x 13 | netlink-sys = "0.8.5" 14 | rtnetlink = "0.13.1" # Ignore 0.14.x 15 | serde = { version = "1.0.193", features = ["derive"] } 16 | thiserror = "1.0.53" 17 | tokio = { version = "1.35.1", features = ["sync", "time"] } 18 | tokio-stream = { version = "0.1.14", features = ["net"] } 19 | tonic = "0.10.2" 20 | tracing = "0.1.40" 21 | sartd-proto = { path = "../proto" } 22 | sartd-trace = { path = "../trace" } 23 | serde_yaml = "0.9.29" 24 | 25 | [patch.crates-io] 26 | netlink-packet-route = { git = "https://github.com/terassyi/netlink-packet-route", branch = "next-hop-default", features = [ 27 | "rich_nlas", 28 | ] } 29 | -------------------------------------------------------------------------------- /sartd/src/fib/src/config.rs: -------------------------------------------------------------------------------- 1 | use std::fs; 2 | 3 | use super::{channel::Channel, error::*}; 4 | use serde::{Deserialize, Serialize}; 5 | 6 | #[derive(Debug, Deserialize, Serialize)] 7 | pub struct Config { 8 | pub endpoint: String, 9 | pub channels: Vec, 10 | } 11 | 12 | impl Config { 13 | pub fn load(file: &str) -> Result { 14 | let contents = fs::read_to_string(file).map_err(Error::StdIoErr)?; 15 | serde_yaml::from_str(&contents).map_err(|_| Error::Config(ConfigError::FailedToLoad)) 16 | } 17 | } 18 | 19 | #[cfg(test)] 20 | mod tests { 21 | use super::Config; 22 | #[test] 23 | fn works_serd_yalm_from_str() { 24 | let yaml_str = r"endpoint: localhost:5001 25 | channels: 26 | - name: kernel_tables 27 | ip_version: ipv4 28 | subscribers: 29 | - protocol: kernel 30 | tables: 31 | - 254 32 | - 8 33 | publishers: 34 | - protocol: bgp 35 | endpoint: localhost:5100 36 | - name: bgp_rib 37 | ip_version: ipv4 38 | subscribers: 39 | - protocol: bgp 40 | endpoint: localhost:5000 41 | publishers: 42 | - protocol: kernel 43 | tables: 44 | - 254 45 | "; 46 | let _conf: Config = serde_yaml::from_str(yaml_str).unwrap(); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /sartd/src/fib/src/error.rs: -------------------------------------------------------------------------------- 1 | use thiserror::Error; 2 | 3 | #[derive(Debug, Error)] 4 | pub enum Error { 5 | #[error("std::io::Error")] 6 | StdIoErr(#[from] std::io::Error), 7 | #[error("failed to communicate with rtnetlink: {}", e)] 8 | FailedToCommunicateWithNetlink { 9 | #[from] 10 | e: rtnetlink::Error, 11 | }, 12 | #[error("failed to communicate with gRPC server/client")] 13 | FailedToCommunicateWithgRPC(#[from] tonic::transport::Error), 14 | #[error("timeout")] 15 | Timeout, 16 | #[error("got error {} from gRPC", e)] 17 | GotgPRCError { 18 | #[from] 19 | e: tonic::Status, 20 | }, 21 | #[error("failed to recv/send via channel")] 22 | FailedToRecvSendViaChannel, 23 | #[error("already exists")] 24 | AlreadyExists, 25 | #[error("failed to get prefix")] 26 | FailedToGetPrefix, 27 | #[error("gateway not found")] 28 | GatewayNotFound, 29 | #[error("invalid ad value")] 30 | InvalidADValue, 31 | #[error("invalid protocol")] 32 | InvalidProtocol, 33 | #[error("invalid scope")] 34 | InvalidScope, 35 | #[error("invalid type")] 36 | InvalidType, 37 | #[error("invalid ip version")] 38 | InvalidIpVersion, 39 | #[error("invalid next hop flag")] 40 | InvalidNextHopFlag, 41 | #[error("failed to parse address")] 42 | FailedToParseAddress, 43 | #[error("destination not found")] 44 | DestinationNotFound, 45 | #[error("config error")] 46 | Config(#[from] ConfigError), 47 | #[error("failed to insert")] 48 | FailedToInsert, 49 | #[error("failed to remove")] 50 | FailedToRemove, 51 | #[error("failed to register")] 52 | FailedToRegister, 53 | #[error("multi path is not equal")] 54 | MultipathIsNotEqual, 55 | } 56 | 57 | #[derive(Debug, Error)] 58 | pub enum ConfigError { 59 | #[error("already configured")] 60 | AlreadyConfigured, 61 | #[error("failed to load")] 62 | FailedToLoad, 63 | #[error("invalid argument")] 64 | InvalidArgument, 65 | #[error("invalid data")] 66 | InvalidData, 67 | } 68 | -------------------------------------------------------------------------------- /sartd/src/fib/src/lib.rs: -------------------------------------------------------------------------------- 1 | mod bgp; 2 | mod channel; 3 | pub mod config; 4 | mod error; 5 | mod kernel; 6 | mod rib; 7 | mod route; 8 | pub mod server; 9 | -------------------------------------------------------------------------------- /sartd/src/ipam/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "sartd-ipam" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | [dependencies] 7 | ipnet = "2.9.0" 8 | thiserror = "1.0.53" 9 | rstest = "0.18.2" 10 | -------------------------------------------------------------------------------- /sartd/src/ipam/src/block_allocator.rs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terassyi/sart/82d8bbeb0224e43cc80a03e6198a98226d2ede47/sartd/src/ipam/src/block_allocator.rs -------------------------------------------------------------------------------- /sartd/src/ipam/src/error.rs: -------------------------------------------------------------------------------- 1 | use thiserror::Error; 2 | 3 | use super::bitset::BitSetError; 4 | 5 | #[derive(Debug, Error, PartialEq, Eq)] 6 | #[allow(dead_code)] 7 | pub enum Error { 8 | #[error("BitSet error: {0}")] 9 | BitSet(#[source] BitSetError), 10 | 11 | #[error("Protocol mismatch")] 12 | ProtocolMistmatch, 13 | 14 | #[error("Not contains")] 15 | NotContains, 16 | 17 | #[error("No releasable address")] 18 | NoReleasableAddress, 19 | 20 | #[error("CIDR too large: {0}")] 21 | CIDRTooLarge(u8), 22 | 23 | #[error("Full")] 24 | Full, 25 | 26 | #[error("AddressBlock not found")] 27 | BlockNotFound, 28 | 29 | #[error("Auto assignable block already exists")] 30 | AutoAssignableBlockAlreadyExists, 31 | } 32 | -------------------------------------------------------------------------------- /sartd/src/ipam/src/lib.rs: -------------------------------------------------------------------------------- 1 | mod allocator; 2 | mod bitset; 3 | pub mod error; 4 | pub mod manager; 5 | -------------------------------------------------------------------------------- /sartd/src/kubernetes/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "sartd-kubernetes" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | [dependencies] 7 | chrono = "0.4.31" 8 | ipnet = "2.9.0" 9 | k8s-openapi = { version = "0.20.0", features = ["schemars", "v1_28"] } 10 | kube = { version = "0.87.2", features = [ 11 | "runtime", 12 | "derive", 13 | "admission", 14 | "rustls-tls", 15 | "client", 16 | ] } 17 | prometheus = "0.13.3" 18 | schemars = "0.8.16" 19 | serde = { version = "1.0.193", features = ["derive"] } 20 | serde_json = "1.0.108" 21 | thiserror = "1.0.53" 22 | tokio = { version = "1.35.1", features = ["rt-multi-thread", "macros", "net"] } 23 | tracing = "0.1.40" 24 | serde_yaml = "0.9.29" 25 | tonic = "0.10.2" 26 | 27 | sartd-cert = { path = "../cert" } 28 | sartd-proto = { path = "../proto" } 29 | sartd-ipam = { path = "../ipam" } 30 | sartd-mock = { path = "../mock" } 31 | sartd-trace = { path = "../trace" } 32 | futures = "0.3.30" 33 | rtnetlink = "0.14.1" 34 | actix-web = { version = "4.4.1", features = ["rustls-0_21"] } 35 | rustls = "0.21.9" 36 | json-patch = "1.2.0" 37 | 38 | # [dev-dependencies] 39 | http = "0.2.10" # Ignore http v1.x 40 | rstest = "0.18.2" 41 | tower-test = "0.4.0" 42 | bytes = "1.5.0" 43 | http-body = "1.0.0" 44 | http-body-util = "0.1.0" 45 | hyper = "0.14.27" # Ignore hyper v1.x 46 | assert-json-diff = "2.0.2" 47 | tokio-stream = { version = "0.1.14", features = ["net"] } 48 | rscni = "0.0.4" 49 | nix = { version = "0.27.1", features = ["sched", "process"] } 50 | netlink-packet-route = "0.19.0" 51 | rand = "0.8.5" 52 | -------------------------------------------------------------------------------- /sartd/src/kubernetes/config/.cargo: -------------------------------------------------------------------------------- 1 | [target.x86_64-unknown-linux-gnu] 2 | runner = 'sudo -E' 3 | -------------------------------------------------------------------------------- /sartd/src/kubernetes/src/agent.rs: -------------------------------------------------------------------------------- 1 | mod bgp; 2 | pub mod cni; 3 | pub mod config; 4 | pub mod context; 5 | pub mod error; 6 | pub mod metrics; 7 | pub mod reconciler; 8 | pub mod server; 9 | -------------------------------------------------------------------------------- /sartd/src/kubernetes/src/agent/bgp.rs: -------------------------------------------------------------------------------- 1 | pub mod rpc; 2 | pub mod speaker; 3 | -------------------------------------------------------------------------------- /sartd/src/kubernetes/src/agent/bgp/rpc.rs: -------------------------------------------------------------------------------- 1 | use std::time::{Duration, Instant}; 2 | 3 | use crate::agent::error::Error; 4 | 5 | async fn connect_bgp( 6 | endpoint: &str, 7 | ) -> Result, Error> { 8 | let endpoint_url = format!("http://{}", endpoint); 9 | sartd_proto::sart::bgp_api_client::BgpApiClient::connect(endpoint_url) 10 | .await 11 | .map_err(Error::FailedToCommunicateWithgRPC) 12 | } 13 | 14 | #[tracing::instrument] 15 | async fn connect_bgp_with_retry( 16 | endpoint: &str, 17 | timeout: Duration, 18 | ) -> Result, Error> { 19 | let deadline = Instant::now() + timeout; 20 | loop { 21 | if Instant::now() > deadline { 22 | return Err(Error::Timeout); 23 | } 24 | match connect_bgp(endpoint).await { 25 | Ok(conn) => return Ok(conn), 26 | Err(e) => { 27 | tracing::error!(error=?e,"failed to connect bgp"); 28 | tokio::time::sleep(Duration::from_millis(500)).await; 29 | } 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /sartd/src/kubernetes/src/agent/bgp/speaker.rs: -------------------------------------------------------------------------------- 1 | use std::time::{Duration, Instant}; 2 | 3 | use crate::agent::error::Error; 4 | 5 | pub async fn connect_bgp( 6 | endpoint: &str, 7 | ) -> Result, Error> { 8 | let endpoint_url = format!("http://{}", endpoint); 9 | sartd_proto::sart::bgp_api_client::BgpApiClient::connect(endpoint_url) 10 | .await 11 | .map_err(Error::FailedToCommunicateWithgRPC) 12 | } 13 | 14 | #[tracing::instrument] 15 | pub async fn connect_bgp_with_retry( 16 | endpoint: &str, 17 | timeout: Duration, 18 | ) -> Result, Error> { 19 | let deadline = Instant::now() + timeout; 20 | loop { 21 | if Instant::now() > deadline { 22 | return Err(Error::Timeout); 23 | } 24 | match connect_bgp(endpoint).await { 25 | Ok(conn) => return Ok(conn), 26 | Err(e) => { 27 | tracing::error!(error=?e,"failed to connect bgp"); 28 | tokio::time::sleep(Duration::from_millis(500)).await; 29 | } 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /sartd/src/kubernetes/src/agent/cni.rs: -------------------------------------------------------------------------------- 1 | pub mod error; 2 | pub mod gc; 3 | mod netlink; 4 | pub mod netns; 5 | pub mod pod; 6 | pub mod server; 7 | -------------------------------------------------------------------------------- /sartd/src/kubernetes/src/agent/cni/error.rs: -------------------------------------------------------------------------------- 1 | use thiserror::Error; 2 | 3 | use super::{netlink, netns}; 4 | 5 | #[derive(Debug, Error)] 6 | pub enum Error { 7 | #[error("NetNS: {0}")] 8 | NetNS(#[source] netns::Error), 9 | 10 | #[error("Netlink: {0}")] 11 | Netlink(#[source] netlink::Error), 12 | 13 | #[error("Kubernetes: {0}")] 14 | Kube(#[source] kube::Error), 15 | 16 | #[error("Missing fields: {0}")] 17 | MissingField(String), 18 | 19 | #[error("Invalid address: {0}")] 20 | InvalidAddress(String), 21 | 22 | #[error("Failed to get lock")] 23 | Lock, 24 | 25 | #[error("Pod already configured")] 26 | AlreadyConfigured(String), 27 | 28 | #[error("Default pool not found")] 29 | DefaultPoolNotFound, 30 | 31 | #[error("Block not found")] 32 | BlockNotFound(String), 33 | 34 | #[error("Failed to receive notification")] 35 | ReceiveNotify, 36 | 37 | #[error("Ipam: {0}")] 38 | Ipam(#[source] sartd_ipam::error::Error), 39 | 40 | #[error("Pod address is not found")] 41 | PodAddressIsNotFound, 42 | 43 | #[error("Allocation not found")] 44 | AllocationNotFound, 45 | 46 | #[error("Addresses don't match")] 47 | AddressNotMatched, 48 | } 49 | -------------------------------------------------------------------------------- /sartd/src/kubernetes/src/agent/config.rs: -------------------------------------------------------------------------------- 1 | use std::fs; 2 | 3 | use serde::{Deserialize, Serialize}; 4 | 5 | use sartd_cert::constants::*; 6 | 7 | use crate::config::{Mode, Tls}; 8 | 9 | use super::{ 10 | cni::server::CNI_SERVER_ENDPOINT, 11 | error::{ConfigError, Error}, 12 | }; 13 | 14 | pub const DEFAULT_HTTP_PORT: u32 = 8000; 15 | pub const DEFAULT_HTTPS_PORT: u32 = 9443; 16 | pub const DEFAULT_ENDPOINT: &str = "0.0.0.0:5002"; 17 | pub const DEFAULT_PEER_STATE_WATCHER_ENDPOINT: &str = "0.0.0.0:5003"; 18 | pub const DEFAULT_REQUEUE_INTERVAL: u64 = 30 * 60; 19 | 20 | #[derive(Debug, Deserialize, Serialize)] 21 | pub struct Config { 22 | pub http_port: u32, 23 | pub https_port: u32, 24 | pub endpoint: String, 25 | pub tls: Tls, 26 | pub requeue_interval: u64, 27 | pub peer_state_watcher: String, 28 | pub mode: Mode, 29 | pub cni_endpoint: Option, 30 | } 31 | 32 | impl Config { 33 | pub fn load(file: &str) -> Result { 34 | let contents = fs::read_to_string(file).map_err(Error::StdIo)?; 35 | serde_yaml::from_str(&contents).map_err(|_| Error::Config(ConfigError::FailedToLoad)) 36 | } 37 | } 38 | 39 | impl Default for Config { 40 | fn default() -> Self { 41 | Self { 42 | http_port: DEFAULT_HTTP_PORT, 43 | https_port: DEFAULT_HTTPS_PORT, 44 | endpoint: DEFAULT_ENDPOINT.to_string(), 45 | tls: Tls { 46 | cert: DEFAULT_TLS_CERT.to_string(), 47 | key: DEFAULT_TLS_KEY.to_string(), 48 | }, 49 | requeue_interval: DEFAULT_REQUEUE_INTERVAL, 50 | peer_state_watcher: DEFAULT_PEER_STATE_WATCHER_ENDPOINT.to_string(), 51 | mode: Mode::default(), 52 | cni_endpoint: Some(CNI_SERVER_ENDPOINT.to_string()), 53 | } 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /sartd/src/kubernetes/src/agent/error.rs: -------------------------------------------------------------------------------- 1 | use sartd_trace::error::TraceableError; 2 | use thiserror::Error; 3 | 4 | #[derive(Debug, Error)] 5 | pub enum Error { 6 | #[error("std::io::Error")] 7 | StdIo(#[from] std::io::Error), 8 | 9 | #[error("Failed to get lock")] 10 | FailedToGetLock, 11 | 12 | #[error("Var Error: {0}")] 13 | Var(#[source] std::env::VarError), 14 | 15 | #[error("Kube Error: {0}")] 16 | Kube(#[source] kube::Error), 17 | 18 | #[error("config error")] 19 | Config(#[from] ConfigError), 20 | 21 | #[error("failed to communicate with rtnetlink: {}", e)] 22 | FailedToCommunicateWithNetlink { 23 | #[from] 24 | e: rtnetlink::Error, 25 | }, 26 | #[error("failed to communicate with gRPC server/client")] 27 | FailedToCommunicateWithgRPC(#[from] tonic::transport::Error), 28 | #[error("timeout")] 29 | Timeout, 30 | #[error("got error {0} from gRPC")] 31 | GotgPRC(#[from] tonic::Status), 32 | 33 | #[error("Local BGP speaker is not configured")] 34 | LocalSpeakerIsNotConfigured, 35 | 36 | #[error("Finalizer Error: {0}")] 37 | // NB: awkward type because finalizer::Error embeds the reconciler error (which is this) 38 | // so boxing this error to break cycles 39 | Finalizer(#[source] Box>), 40 | 41 | #[error("FailedToGetData: {0}")] 42 | FailedToGetData(String), 43 | 44 | #[error("SerializationError: {0}")] 45 | Serialization(#[source] serde_json::Error), 46 | 47 | #[error("CRD Error: {0}")] 48 | CRD(#[source] crate::crd::error::Error), 49 | 50 | #[error("Kubernetes Library Error: {0}")] 51 | KubeLibrary(#[source] crate::error::Error), 52 | 53 | #[error("Ipam Error: {0}")] 54 | Ipam(#[source] sartd_ipam::error::Error), 55 | 56 | #[error("Peer exists")] 57 | PeerExists, 58 | 59 | #[error("Invalid CIDR")] 60 | InvalidCIDR, 61 | 62 | #[error("Auto assignable pool already exists")] 63 | AutoAssignAlreadyExists, 64 | 65 | #[error("Cannot delete")] 66 | CannotDelete, 67 | 68 | #[error("Not empty")] 69 | NotEmpty, 70 | 71 | #[error("Failed to notify")] 72 | FailedToNotify, 73 | 74 | #[error("Missing fields: {0}")] 75 | MissingFields(String), 76 | } 77 | 78 | #[derive(Debug, Error, Clone)] 79 | pub enum ConfigError { 80 | #[error("already configured")] 81 | AlreadyConfigured, 82 | #[error("failed to load")] 83 | FailedToLoad, 84 | #[error("invalid argument")] 85 | InvalidArgument, 86 | #[error("invalid data")] 87 | InvalidData, 88 | } 89 | 90 | impl TraceableError for &Error { 91 | fn metric_label(&self) -> String { 92 | format!("{self:?}").to_lowercase() 93 | } 94 | } 95 | 96 | impl TraceableError for Error { 97 | fn metric_label(&self) -> String { 98 | format!("{self:?}").to_lowercase() 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /sartd/src/kubernetes/src/agent/reconciler.rs: -------------------------------------------------------------------------------- 1 | pub mod address_block; 2 | pub mod bgp_advertisement; 3 | pub mod bgp_peer; 4 | pub mod bgp_peer_watcher; 5 | pub mod node_bgp; 6 | -------------------------------------------------------------------------------- /sartd/src/kubernetes/src/config.rs: -------------------------------------------------------------------------------- 1 | use std::str::FromStr; 2 | 3 | use serde::{Deserialize, Serialize}; 4 | use thiserror::Error; 5 | 6 | #[derive(Debug, Deserialize, Serialize)] 7 | pub struct Tls { 8 | pub cert: String, 9 | pub key: String, 10 | } 11 | 12 | #[derive(Debug, Default, Clone, Copy, Deserialize, Serialize, PartialEq, Eq)] 13 | pub enum Mode { 14 | LB, 15 | CNI, 16 | #[default] 17 | Dual, 18 | } 19 | 20 | impl std::fmt::Display for Mode { 21 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 22 | match self { 23 | Self::CNI => write!(f, "cni"), 24 | Self::LB => write!(f, "lb"), 25 | Self::Dual => write!(f, "dual"), 26 | } 27 | } 28 | } 29 | 30 | #[derive(Debug, Clone, Copy, Error)] 31 | pub struct ParseModeError; 32 | 33 | impl std::fmt::Display for ParseModeError { 34 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 35 | "provided string was not `true` or `false`".fmt(f) 36 | } 37 | } 38 | 39 | impl FromStr for Mode { 40 | type Err = ParseModeError; 41 | fn from_str(s: &str) -> Result { 42 | match s { 43 | "lb" => Ok(Mode::LB), 44 | "cni" => Ok(Mode::CNI), 45 | "dual" => Ok(Mode::Dual), 46 | _ => Err(ParseModeError), 47 | } 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /sartd/src/kubernetes/src/controller.rs: -------------------------------------------------------------------------------- 1 | pub mod config; 2 | pub mod context; 3 | pub mod error; 4 | pub mod metrics; 5 | pub mod reconciler; 6 | pub mod server; 7 | pub mod webhook; 8 | -------------------------------------------------------------------------------- /sartd/src/kubernetes/src/controller/config.rs: -------------------------------------------------------------------------------- 1 | use std::fs; 2 | 3 | use serde::{Deserialize, Serialize}; 4 | 5 | use crate::config::{Mode, Tls}; 6 | 7 | use super::error::{ConfigError, Error}; 8 | 9 | use sartd_cert::constants::{DEFAULT_TLS_CERT, DEFAULT_TLS_KEY}; 10 | 11 | pub const DEFAULT_HTTP_PORT: u32 = 8080; 12 | pub const DEFAULT_HTTPS_PORT: u32 = 8443; 13 | pub const DEFAULT_ENDPOINT: &str = "0.0.0.0:5002"; 14 | pub const DEFAULT_REQUEUE_INTERVAL: u64 = 30 * 60; 15 | 16 | #[derive(Debug, Deserialize, Serialize)] 17 | pub struct Config { 18 | pub http_port: u32, 19 | pub https_port: u32, 20 | pub endpoint: String, 21 | pub tls: Tls, 22 | pub requeue_interval: u64, 23 | pub mode: Mode, 24 | } 25 | 26 | impl Config { 27 | pub fn load(file: &str) -> Result { 28 | let contents = fs::read_to_string(file).map_err(Error::StdIo)?; 29 | serde_yaml::from_str(&contents).map_err(|_| Error::Config(ConfigError::FailedToLoad)) 30 | } 31 | } 32 | 33 | impl Default for Config { 34 | fn default() -> Self { 35 | Self { 36 | http_port: DEFAULT_HTTP_PORT, 37 | https_port: DEFAULT_HTTPS_PORT, 38 | endpoint: DEFAULT_ENDPOINT.to_string(), 39 | tls: Tls { 40 | cert: DEFAULT_TLS_CERT.to_string(), 41 | key: DEFAULT_TLS_KEY.to_string(), 42 | }, 43 | requeue_interval: DEFAULT_REQUEUE_INTERVAL, 44 | mode: Mode::default(), 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /sartd/src/kubernetes/src/controller/reconciler.rs: -------------------------------------------------------------------------------- 1 | pub mod address_block; 2 | pub mod address_pool; 3 | pub mod bgp_advertisement; 4 | pub mod block_request; 5 | pub mod cluster_bgp; 6 | pub mod endpointslice_watcher; 7 | pub mod node_watcher; 8 | pub mod service_watcher; 9 | -------------------------------------------------------------------------------- /sartd/src/kubernetes/src/controller/webhook.rs: -------------------------------------------------------------------------------- 1 | pub mod address_block; 2 | pub mod address_pool; 3 | pub mod bgp_advertisement; 4 | pub mod bgp_peer; 5 | pub mod service; 6 | -------------------------------------------------------------------------------- /sartd/src/kubernetes/src/controller/webhook/address_block.rs: -------------------------------------------------------------------------------- 1 | use actix_web::{web, HttpRequest, HttpResponse, Responder}; 2 | use kube::{ 3 | core::admission::{AdmissionRequest, AdmissionResponse, AdmissionReview}, 4 | ResourceExt, 5 | }; 6 | 7 | use crate::{ 8 | controller::error::Error, 9 | crd::{ 10 | address_block::{AddressBlock, ADDRESS_BLOCK_NODE_LABEL}, 11 | address_pool::AddressType, 12 | }, 13 | util::escape_slash, 14 | }; 15 | 16 | #[tracing::instrument(skip_all)] 17 | pub async fn handle_mutation( 18 | req: HttpRequest, 19 | body: web::Json>, 20 | ) -> impl Responder { 21 | 22 | if let Some(content_type) = req.head().headers.get("content-type") { 23 | if content_type != "application/json" { 24 | let msg = format!("invalid content-type: {:?}", content_type); 25 | 26 | return HttpResponse::BadRequest().json(msg); 27 | } 28 | } 29 | 30 | let admission_req: AdmissionRequest = match body.into_inner().try_into() { 31 | Ok(req) => req, 32 | Err(e) => { 33 | tracing::error!("invalid request: {}", e); 34 | return HttpResponse::InternalServerError() 35 | .json(&AdmissionResponse::invalid(e.to_string()).into_review()); 36 | } 37 | }; 38 | let mut resp = AdmissionResponse::from(&admission_req); 39 | 40 | if let Some(ab) = admission_req.object { 41 | if ab.spec.r#type.ne(&AddressType::Pod) { 42 | return HttpResponse::Ok().json(resp.into_review()); 43 | } 44 | let name = ab.name_any(); 45 | resp = match mutate_address_block(&resp, &ab) { 46 | Ok(res) => { 47 | tracing::info!( 48 | op=?admission_req.operation, 49 | name=name, 50 | "Accepted by mutating webhook", 51 | ); 52 | res 53 | } 54 | Err(e) => { 55 | tracing::warn!( 56 | op=?admission_req.operation, 57 | name=name, 58 | "Denied by mutating webhook", 59 | ); 60 | resp.deny(e.to_string()) 61 | } 62 | }; 63 | } 64 | 65 | HttpResponse::Ok().json(resp.into_review()) 66 | } 67 | 68 | fn mutate_address_block( 69 | res: &AdmissionResponse, 70 | ab: &AddressBlock, 71 | ) -> Result { 72 | match ab.spec.node_ref.as_ref() { 73 | Some(node) => { 74 | let patch = if ab.labels().get(ADDRESS_BLOCK_NODE_LABEL).is_some() { 75 | json_patch::PatchOperation::Replace(json_patch::ReplaceOperation { 76 | path: format!( 77 | "/metadata/labels/{}", 78 | escape_slash(ADDRESS_BLOCK_NODE_LABEL) 79 | ), 80 | value: serde_json::Value::String(node.to_string()), 81 | }) 82 | } else { 83 | json_patch::PatchOperation::Add(json_patch::AddOperation { 84 | path: format!( 85 | "/metadata/labels/{}", 86 | escape_slash(ADDRESS_BLOCK_NODE_LABEL) 87 | ), 88 | value: serde_json::Value::String(node.to_string()), 89 | }) 90 | }; 91 | Ok(res 92 | .clone() 93 | .with_patch(json_patch::Patch(vec![patch])) 94 | .map_err(Error::SerializePatch)?) 95 | } 96 | None => Err(Error::InvalidParameter( 97 | "spec.nodeRef is required for AddressBlock".to_string(), 98 | )), 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /sartd/src/kubernetes/src/controller/webhook/bgp_advertisement.rs: -------------------------------------------------------------------------------- 1 | use actix_web::{web, HttpRequest, HttpResponse, Responder}; 2 | use kube::core::{ 3 | admission::{AdmissionRequest, AdmissionResponse, AdmissionReview}, 4 | response::StatusSummary, 5 | Status, 6 | }; 7 | 8 | use crate::crd::bgp_advertisement::BGPAdvertisement; 9 | 10 | #[tracing::instrument(skip_all)] 11 | pub async fn handle_validation( 12 | req: HttpRequest, 13 | body: web::Json>, 14 | ) -> impl Responder { 15 | 16 | if let Some(content_type) = req.head().headers.get("content-type") { 17 | if content_type != "application/json" { 18 | let msg = format!("invalid content-type: {:?}", content_type); 19 | 20 | return HttpResponse::BadRequest().json(msg); 21 | } 22 | } 23 | let admission_req: AdmissionRequest = match body.into_inner().try_into() { 24 | Ok(req) => req, 25 | Err(e) => { 26 | tracing::error!("invalid request: {}", e); 27 | return HttpResponse::InternalServerError() 28 | .json(&AdmissionResponse::invalid(e.to_string()).into_review()); 29 | } 30 | }; 31 | 32 | let mut resp = AdmissionResponse::from(&admission_req); 33 | 34 | resp.allowed = true; 35 | resp.result = Status { 36 | status: Some(StatusSummary::Success), 37 | message: "nop".to_string(), 38 | code: 200, 39 | reason: "nop".to_string(), 40 | details: None, 41 | }; 42 | HttpResponse::Ok().json(resp.into_review()) 43 | } 44 | -------------------------------------------------------------------------------- /sartd/src/kubernetes/src/controller/webhook/service.rs: -------------------------------------------------------------------------------- 1 | use actix_web::{web, HttpRequest, HttpResponse, Responder}; 2 | use k8s_openapi::api::core::v1::Service; 3 | use kube::{ 4 | core::admission::{AdmissionRequest, AdmissionResponse, AdmissionReview}, 5 | ResourceExt, 6 | }; 7 | 8 | use crate::{ 9 | controller::reconciler::service_watcher::{ 10 | get_allocated_lb_addrs, is_loadbalancer, RELEASE_ANNOTATION, 11 | }, 12 | util::escape_slash, 13 | }; 14 | 15 | #[tracing::instrument(skip_all)] 16 | pub async fn handle_mutation( 17 | req: HttpRequest, 18 | body: web::Json>, 19 | ) -> impl Responder { 20 | 21 | if let Some(content_type) = req.head().headers.get("content-type") { 22 | if content_type != "application/json" { 23 | let msg = format!("invalid content-type: {:?}", content_type); 24 | 25 | return HttpResponse::BadRequest().json(msg); 26 | } 27 | } 28 | 29 | let admission_req: AdmissionRequest = match body.into_inner().try_into() { 30 | Ok(req) => req, 31 | Err(e) => { 32 | tracing::error!("invalid request: {}", e); 33 | return HttpResponse::InternalServerError() 34 | .json(&AdmissionResponse::invalid(e.to_string()).into_review()); 35 | } 36 | }; 37 | 38 | let resp = AdmissionResponse::from(&admission_req); 39 | 40 | // object field must be set 41 | let new_svc = admission_req.object.unwrap(); 42 | 43 | if is_loadbalancer(&new_svc) && new_svc.annotations().get(RELEASE_ANNOTATION).is_some() { 44 | let patches = vec![json_patch::PatchOperation::Remove(json_patch::RemoveOperation { 45 | path: format!("/metadata/annotations/{}", escape_slash(RELEASE_ANNOTATION)), 46 | })]; 47 | let resp = match resp.with_patch(json_patch::Patch(patches)) { 48 | Ok(resp) => resp, 49 | Err(e) => { 50 | tracing::error!(error=?e,name=new_svc.name_any(), namespace=new_svc.namespace().unwrap(), "failed to handle request"); 51 | return HttpResponse::InternalServerError() 52 | .json("failed to handle a webhook request"); 53 | } 54 | }; 55 | return HttpResponse::Ok().json(resp.into_review()); 56 | } 57 | 58 | if let Some(old_svc) = admission_req.old_object { 59 | if is_loadbalancer(&old_svc) && !is_loadbalancer(&new_svc) { 60 | tracing::info!( 61 | name = old_svc.name_any(), 62 | namespace = old_svc.namespace().unwrap(), 63 | "Request to change from LoadBalancer to other Service type", 64 | ); 65 | match get_allocated_lb_addrs(&old_svc) { 66 | Some(allocated) => { 67 | tracing::warn!(old=?old_svc); 68 | let release = allocated 69 | .iter() 70 | .map(|a| a.to_string()) 71 | .collect::>() 72 | .join(","); 73 | tracing::info!(release=?release,"releasable"); 74 | let patches = vec![json_patch::PatchOperation::Add(json_patch::AddOperation { 75 | path: format!("/metadata/annotations/{}", escape_slash(RELEASE_ANNOTATION)), 76 | value: serde_json::Value::String(release), 77 | })]; 78 | let resp = match resp.with_patch(json_patch::Patch(patches)) { 79 | Ok(resp) => resp, 80 | Err(e) => { 81 | tracing::error!(error=?e,name=old_svc.name_any(), namespace=old_svc.namespace().unwrap(), "failed to handle request"); 82 | return HttpResponse::InternalServerError() 83 | .json("failed to handle a webhook request"); 84 | } 85 | }; 86 | return HttpResponse::Ok().json(resp.into_review()); 87 | } 88 | None => { 89 | // nothing to patch 90 | return HttpResponse::Ok().json(resp.into_review()); 91 | } 92 | } 93 | } 94 | } 95 | 96 | HttpResponse::Ok().json(resp.into_review()) 97 | } 98 | -------------------------------------------------------------------------------- /sartd/src/kubernetes/src/crd.rs: -------------------------------------------------------------------------------- 1 | pub mod address_block; 2 | pub mod address_pool; 3 | pub mod bgp_advertisement; 4 | pub mod bgp_peer; 5 | pub mod bgp_peer_template; 6 | pub mod block_request; 7 | pub mod cluster_bgp; 8 | pub mod error; 9 | pub mod node_bgp; 10 | -------------------------------------------------------------------------------- /sartd/src/kubernetes/src/crd/address_block.rs: -------------------------------------------------------------------------------- 1 | use super::address_pool::AddressType; 2 | pub use kube::CustomResource; 3 | use schemars::JsonSchema; 4 | use serde::{Deserialize, Serialize}; 5 | 6 | pub const ADDRESS_BLOCK_FINALIZER_CONTROLLER: &str = "controller.addressblock.sart.terassyi.net/finalizer"; 7 | pub const ADDRESS_BLOCK_FINALIZER_AGENT: &str = "agent.addressblock.sart.terassyi.net/finalizer"; 8 | pub const ADDRESS_BLOCK_NODE_LABEL: &str = "addressblock.sart.terassyi.net/node"; 9 | 10 | #[derive(CustomResource, Debug, Serialize, Deserialize, Default, Clone, JsonSchema)] 11 | #[kube( 12 | group = "sart.terassyi.net", 13 | version = "v1alpha2", 14 | kind = "AddressBlock" 15 | )] 16 | #[kube(status = "AddressBlockStatus")] 17 | #[kube( 18 | printcolumn = r#"{"name":"CIDR", "type":"string", "description":"CIDR of Address pool", "jsonPath":".spec.cidr"}"#, 19 | printcolumn = r#"{"name":"TYPE", "type":"string", "description":"Type of Address pool", "jsonPath":".spec.type"}"#, 20 | printcolumn = r#"{"name":"POOLREF", "type":"string", "description":"pool name", "jsonPath":".spec.poolRef"}"#, 21 | printcolumn = r#"{"name":"NODEREF", "type":"string", "description":"node name", "jsonPath":".spec.nodeRef"}"#, 22 | printcolumn = r#"{"name":"AGE", "type":"date", "description":"Date from created", "jsonPath":".metadata.creationTimestamp"}"# 23 | )] 24 | #[serde(rename_all = "camelCase")] 25 | pub struct AddressBlockSpec { 26 | pub cidr: String, 27 | pub r#type: AddressType, 28 | pub pool_ref: String, 29 | pub node_ref: Option, 30 | pub auto_assign: bool, // default false 31 | } 32 | 33 | #[derive(Deserialize, Serialize, Clone, Default, Debug, JsonSchema)] 34 | pub struct AddressBlockStatus { 35 | pub index: u32, 36 | } 37 | -------------------------------------------------------------------------------- /sartd/src/kubernetes/src/crd/address_pool.rs: -------------------------------------------------------------------------------- 1 | 2 | use kube::CustomResource; 3 | use schemars::JsonSchema; 4 | use serde::{Deserialize, Serialize}; 5 | 6 | pub const ADDRESS_POOL_FINALIZER: &str = "addresspool.sart.terassyi.net/finalizer"; 7 | pub const ADDRESS_POOL_ANNOTATION: &str = "sart.terassyi.net/addresspool"; 8 | pub const LOADBALANCER_ADDRESS_ANNOTATION: &str = "sart.terassyi.net/loadBalancerIPs"; 9 | pub const MAX_BLOCK_SIZE: u32 = 32; 10 | 11 | #[derive(CustomResource, Debug, Serialize, Deserialize, Default, Clone, JsonSchema)] 12 | #[kube( 13 | group = "sart.terassyi.net", 14 | version = "v1alpha2", 15 | kind = "AddressPool" 16 | )] 17 | #[kube(status = "AddressPoolStatus")] 18 | #[kube( 19 | printcolumn = r#"{"name":"CIDR", "type":"string", "description":"CIDR of Address pool", "jsonPath":".spec.cidr"}"#, 20 | printcolumn = r#"{"name":"TYPE", "type":"string", "description":"Type of Address pool", "jsonPath":".spec.type"}"#, 21 | printcolumn = r#"{"name":"BLOCKSIZE", "type":"integer", "description":"block size of CIDR", "jsonPath":".spec.blockSize"}"#, 22 | printcolumn = r#"{"name":"AUTO", "type":"boolean", "description":"auto assign", "jsonPath":".spec.autoAssign"}"#, 23 | printcolumn = r#"{"name":"AGE", "type":"date", "description":"Date from created", "jsonPath":".metadata.creationTimestamp"}"# 24 | )] 25 | #[serde(rename_all = "camelCase")] 26 | pub struct AddressPoolSpec { 27 | pub cidr: String, 28 | pub r#type: AddressType, 29 | pub alloc_type: Option, 30 | pub block_size: Option, 31 | pub auto_assign: Option, // default false 32 | } 33 | 34 | #[derive(Deserialize, Serialize, Clone, Default, Debug, JsonSchema)] 35 | pub struct AddressPoolStatus {} 36 | 37 | #[derive(Deserialize, Serialize, Clone, Copy, Default, Debug, JsonSchema, PartialEq, Eq)] 38 | #[serde(rename_all = "camelCase")] 39 | pub enum AddressType { 40 | #[default] 41 | #[serde(rename = "service")] 42 | Service, 43 | #[serde(rename = "pod")] 44 | Pod, 45 | } 46 | 47 | impl std::fmt::Display for AddressType { 48 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 49 | match self { 50 | Self::Service => write!(f, "service"), 51 | Self::Pod => write!(f, "pod"), 52 | } 53 | } 54 | } 55 | 56 | #[derive(Deserialize, Serialize, Clone, Copy, Default, Debug, JsonSchema)] 57 | #[serde(rename_all = "camelCase")] 58 | pub enum AllocationType { 59 | #[default] 60 | #[serde(rename = "bit")] 61 | Bit, 62 | } 63 | -------------------------------------------------------------------------------- /sartd/src/kubernetes/src/crd/bgp_advertisement.rs: -------------------------------------------------------------------------------- 1 | use std::collections::BTreeMap; 2 | 3 | use ipnet::IpNet; 4 | pub use kube::CustomResource; 5 | use schemars::JsonSchema; 6 | use serde::{Deserialize, Serialize}; 7 | 8 | use super::address_pool::AddressType; 9 | 10 | pub const BGP_ADVERTISEMENT_FINALIZER: &str = "bgpadvertisement.sart.terassyi.net/finalizer"; 11 | 12 | #[derive(CustomResource, Debug, Serialize, Deserialize, Default, Clone, JsonSchema)] 13 | // #[cfg_attr(test, derive(Default))] 14 | #[kube( 15 | group = "sart.terassyi.net", 16 | version = "v1alpha2", 17 | kind = "BGPAdvertisement" 18 | )] 19 | #[kube(status = "BGPAdvertisementStatus")] 20 | #[kube(namespaced)] 21 | #[kube( 22 | printcolumn = r#"{"name":"CIDR", "type":"string", "description":"Advertised CIDR", "jsonPath":".spec.cidr"}"#, 23 | printcolumn = r#"{"name":"TYPE", "type":"string", "description":"Type of advertised CIDR", "jsonPath":".spec.type"}"#, 24 | printcolumn = r#"{"name":"PROTOCOL", "type":"string", "description":"Type of advertised CIDR", "jsonPath":".spec.protocol"}"#, 25 | printcolumn = r#"{"name":"AGE", "type":"date", "description":"Date from created", "jsonPath":".metadata.creationTimestamp"}"# 26 | )] 27 | #[serde(rename_all = "camelCase")] 28 | pub struct BGPAdvertisementSpec { 29 | pub cidr: String, 30 | pub r#type: AddressType, 31 | pub protocol: Protocol, 32 | pub attrs: Option>, // TODO: implement attributes 33 | } 34 | 35 | #[derive(Deserialize, Serialize, Clone, Default, Debug, JsonSchema)] 36 | pub struct BGPAdvertisementStatus { 37 | pub peers: Option>, 38 | } 39 | 40 | #[derive(Deserialize, Serialize, Clone, Default, Debug, JsonSchema, PartialEq, Eq)] 41 | pub enum AdvertiseStatus { 42 | #[default] 43 | NotAdvertised, 44 | Advertised, 45 | Withdraw, 46 | } 47 | 48 | impl std::fmt::Display for AdvertiseStatus { 49 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 50 | match self { 51 | AdvertiseStatus::Advertised => write!(f, "advertised"), 52 | AdvertiseStatus::NotAdvertised => write!(f, "notadvertised"), 53 | AdvertiseStatus::Withdraw => write!(f, "withdraw"), 54 | } 55 | } 56 | } 57 | 58 | #[derive(Deserialize, Serialize, Clone, Default, Debug, JsonSchema)] 59 | #[serde(rename_all = "camelCase")] 60 | pub enum Protocol { 61 | #[default] 62 | #[serde(rename = "ipv4")] 63 | IPv4, 64 | #[serde(rename = "ipv6")] 65 | IPv6, 66 | } 67 | 68 | impl From<&IpNet> for Protocol { 69 | fn from(value: &IpNet) -> Self { 70 | match value { 71 | IpNet::V4(_) => Protocol::IPv4, 72 | IpNet::V6(_) => Protocol::IPv6, 73 | } 74 | } 75 | } 76 | 77 | impl std::fmt::Display for Protocol { 78 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 79 | match self { 80 | Self::IPv4 => write!(f, "ipv4"), 81 | Self::IPv6 => write!(f, "ipv6"), 82 | } 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /sartd/src/kubernetes/src/crd/bgp_peer_template.rs: -------------------------------------------------------------------------------- 1 | use kube::CustomResource; 2 | use schemars::JsonSchema; 3 | use serde::{Deserialize, Serialize}; 4 | 5 | #[derive(CustomResource, Debug, Serialize, Deserialize, Default, Clone, JsonSchema)] 6 | // #[cfg_attr(test, derive(Default))] 7 | #[kube( 8 | group = "sart.terassyi.net", 9 | version = "v1alpha2", 10 | kind = "BGPPeerTemplate" 11 | )] 12 | #[kube(status = "BGPPeerTemplateStatus")] 13 | #[kube( 14 | printcolumn = r#"{"name":"AGE", "type":"date", "description":"Date from created", "jsonPath":".metadata.creationTimestamp"}"# 15 | )] 16 | #[serde(rename_all = "camelCase")] 17 | pub struct BGPPeerTemplateSpec { 18 | pub asn: Option, 19 | pub addr: Option, 20 | pub capabilities: Option>, 21 | pub hold_time: Option, 22 | pub keepalive_time: Option, 23 | pub groups: Option>, 24 | } 25 | 26 | #[derive(Deserialize, Serialize, Clone, Default, Debug, JsonSchema)] 27 | pub struct BGPPeerTemplateStatus {} 28 | -------------------------------------------------------------------------------- /sartd/src/kubernetes/src/crd/block_request.rs: -------------------------------------------------------------------------------- 1 | use kube::CustomResource; 2 | use schemars::JsonSchema; 3 | use serde::{Deserialize, Serialize}; 4 | 5 | pub const BLOCK_REQUEST_FINALIZER: &str = "blockrequest.sart.terassyi.net/finalizer"; 6 | 7 | #[derive(CustomResource, Debug, Serialize, Deserialize, Default, Clone, JsonSchema)] 8 | #[kube( 9 | group = "sart.terassyi.net", 10 | version = "v1alpha2", 11 | kind = "BlockRequest" 12 | )] 13 | #[kube(status = "BlockRequestStatus")] 14 | #[kube( 15 | printcolumn = r#"{"name":"POOL", "type":"string", "description":"Address pool name", "jsonPath":".spec.pool"}"#, 16 | printcolumn = r#"{"name":"NODE", "type":"string", "description":"Node name", "jsonPath":".spec.node"}"# 17 | )] 18 | #[serde(rename_all = "camelCase")] 19 | pub struct BlockRequestSpec { 20 | pub pool: String, 21 | pub node: String, 22 | } 23 | 24 | #[derive(Deserialize, Serialize, Clone, Debug, JsonSchema)] 25 | #[serde(rename_all = "camelCase")] 26 | pub enum BlockRequestStatus {} 27 | -------------------------------------------------------------------------------- /sartd/src/kubernetes/src/crd/cluster_bgp.rs: -------------------------------------------------------------------------------- 1 | use std::collections::BTreeMap; 2 | 3 | use kube::CustomResource; 4 | use schemars::JsonSchema; 5 | use serde::{Deserialize, Serialize}; 6 | 7 | use super::bgp_peer::PeerConfig; 8 | 9 | pub const CLUSTER_BGP_FINALIZER: &str = "clusterbgp.sart.terassyi.net/finalizer"; 10 | pub const ASN_LABEL: &str = "sart.terassyi.net/asn"; 11 | pub const ROUTER_ID_LABEL: &str = "sart.terassyi.net/router-id"; 12 | 13 | #[derive(CustomResource, Debug, Serialize, Deserialize, Default, Clone, JsonSchema)] 14 | #[kube(group = "sart.terassyi.net", version = "v1alpha2", kind = "ClusterBGP")] 15 | #[kube(status = "ClusterBGPStatus")] 16 | #[kube( 17 | printcolumn = r#"{"name":"AGE", "type":"date", "description":"Date from created", "jsonPath":".metadata.creationTimestamp"}"# 18 | )] 19 | #[serde(rename_all = "camelCase")] 20 | pub struct ClusterBGPSpec { 21 | pub node_selector: Option>, 22 | pub asn_selector: AsnSelector, 23 | pub router_id_selector: RouterIdSelector, 24 | pub speaker: SpeakerConfig, 25 | pub peers: Option>, 26 | } 27 | 28 | #[derive(Deserialize, Serialize, Clone, Default, Debug, JsonSchema)] 29 | #[serde(rename_all = "camelCase")] 30 | pub struct ClusterBGPStatus { 31 | pub desired_nodes: Option>, 32 | pub nodes: Option>, 33 | } 34 | 35 | #[derive(Deserialize, Serialize, Clone, Default, Debug, JsonSchema)] 36 | #[serde(rename_all = "camelCase")] 37 | pub struct NodeBGPConfig { 38 | pub asn_selector: AsnSelector, 39 | pub router_id_from: String, 40 | } 41 | 42 | #[derive(Deserialize, Serialize, Clone, Default, Debug, JsonSchema)] 43 | #[serde(rename_all = "camelCase")] 44 | pub struct AsnSelector { 45 | pub from: AsnSelectionType, 46 | pub asn: Option, 47 | } 48 | 49 | #[derive(Debug, Serialize, Deserialize, Default, Clone, JsonSchema, PartialEq, Eq)] 50 | #[serde(rename_all = "camelCase")] 51 | pub enum AsnSelectionType { 52 | #[default] 53 | #[serde(rename = "asn")] 54 | Asn, 55 | #[serde(rename = "label")] 56 | Label, 57 | } 58 | 59 | #[derive(Deserialize, Serialize, Clone, Default, Debug, JsonSchema)] 60 | #[serde(rename_all = "camelCase")] 61 | pub struct RouterIdSelector { 62 | pub from: RouterIdSelectionType, 63 | pub router_id: Option, 64 | } 65 | 66 | #[derive(Debug, Serialize, Deserialize, Default, Clone, JsonSchema, PartialEq, Eq)] 67 | #[serde(rename_all = "camelCase")] 68 | pub enum RouterIdSelectionType { 69 | #[default] 70 | #[serde(rename = "internalAddress")] 71 | InternalAddress, 72 | #[serde(rename = "label")] 73 | Label, 74 | #[serde(rename = "routerId")] 75 | RouterId, 76 | } 77 | 78 | #[derive(Debug, Serialize, Deserialize, Default, Clone, JsonSchema, PartialEq, Eq)] 79 | #[serde(rename_all = "camelCase")] 80 | pub struct SpeakerConfig { 81 | pub path: String, 82 | pub timeout: Option, 83 | pub multipath: Option, 84 | } 85 | -------------------------------------------------------------------------------- /sartd/src/kubernetes/src/crd/error.rs: -------------------------------------------------------------------------------- 1 | use thiserror::Error; 2 | 3 | #[derive(Debug, Error)] 4 | pub enum Error { 5 | #[error("std::io::Error")] 6 | StdIo(#[from] std::io::Error), 7 | 8 | #[error("Kube Error: {0}")] 9 | Kube(#[source] kube::Error), 10 | 11 | #[error("Validation Error: {0}")] 12 | Validation(String), 13 | 14 | #[error("Invalid Peer State: {0}")] 15 | InvalidPeerState(i32), 16 | } 17 | -------------------------------------------------------------------------------- /sartd/src/kubernetes/src/crd/node_bgp.rs: -------------------------------------------------------------------------------- 1 | use core::fmt; 2 | 3 | use kube::CustomResource; 4 | use schemars::JsonSchema; 5 | use serde::{Deserialize, Serialize}; 6 | 7 | use super::bgp_peer::BGPPeerSlim; 8 | 9 | use super::cluster_bgp::SpeakerConfig; 10 | 11 | pub const NODE_BGP_FINALIZER: &str = "nodebgp.sart.terassyi.net/finalizer"; 12 | 13 | #[derive(CustomResource, Debug, Serialize, Deserialize, Default, Clone, JsonSchema)] 14 | // #[cfg_attr(test, derive(Default))] 15 | #[kube(group = "sart.terassyi.net", version = "v1alpha2", kind = "NodeBGP")] 16 | #[kube(status = "NodeBGPStatus")] 17 | #[kube( 18 | printcolumn = r#"{"name":"ASN", "type":"integer", "description":"ASN of the local BGP speaker", "jsonPath":".spec.asn"}"#, 19 | printcolumn = r#"{"name":"ROUTERID", "type":"string", "description":"Router ID of the local BGP speaker", "jsonPath":".spec.routerId"}"#, 20 | printcolumn = r#"{"name":"BACKOFF", "type":"integer", "description":"Back off counter", "jsonPath":".status.backoff"}"#, 21 | printcolumn = r#"{"name":"STATUS", "type":"string", "description":"Status of a local speaker", "jsonPath":".status.conditions[-1:].status"}"#, 22 | printcolumn = r#"{"name":"AGE", "type":"date", "description":"Date from created", "jsonPath":".metadata.creationTimestamp"}"# 23 | )] 24 | #[serde(rename_all = "camelCase")] 25 | pub struct NodeBGPSpec { 26 | pub asn: u32, 27 | pub router_id: String, 28 | pub speaker: SpeakerConfig, 29 | pub peers: Option>, 30 | } 31 | 32 | #[derive(Deserialize, Serialize, Clone, Default, Debug, JsonSchema)] 33 | #[serde(rename_all = "camelCase")] 34 | pub struct NodeBGPStatus { 35 | pub backoff: u32, 36 | pub cluster_bgp_refs: Option>, 37 | pub conditions: Option>, 38 | } 39 | 40 | #[derive(Deserialize, Serialize, Clone, Default, Debug, JsonSchema, PartialEq, Eq)] 41 | pub struct NodeBGPCondition { 42 | pub status: NodeBGPConditionStatus, 43 | pub reason: NodeBGPConditionReason, 44 | } 45 | 46 | #[derive(Deserialize, Serialize, Clone, Default, Debug, JsonSchema, PartialEq, Eq)] 47 | pub enum NodeBGPConditionStatus { 48 | Available, 49 | #[default] 50 | Unavailable, 51 | } 52 | 53 | impl std::fmt::Display for NodeBGPConditionStatus { 54 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 55 | match self { 56 | Self::Available => write!(f, "available"), 57 | Self::Unavailable => write!(f, "unavailable"), 58 | } 59 | } 60 | } 61 | 62 | #[derive(Deserialize, Serialize, Clone, Default, Debug, JsonSchema, PartialEq, Eq)] 63 | pub enum NodeBGPConditionReason { 64 | #[default] 65 | NotConfigured, 66 | Configured, 67 | InvalidConfiguration, 68 | } 69 | -------------------------------------------------------------------------------- /sartd/src/kubernetes/src/error.rs: -------------------------------------------------------------------------------- 1 | use thiserror::Error; 2 | 3 | #[derive(Debug, Error)] 4 | pub enum Error { 5 | #[error("Get Namespace Error")] 6 | GetNamespace, 7 | } 8 | -------------------------------------------------------------------------------- /sartd/src/kubernetes/src/lib.rs: -------------------------------------------------------------------------------- 1 | pub mod agent; 2 | pub mod config; 3 | pub mod context; 4 | pub mod controller; 5 | pub mod crd; 6 | pub mod error; 7 | pub mod fixture; 8 | pub mod metrics; 9 | pub mod util; 10 | -------------------------------------------------------------------------------- /sartd/src/kubernetes/src/metrics.rs: -------------------------------------------------------------------------------- 1 | use kube::Resource; 2 | use prometheus::Registry; 3 | use prometheus::{histogram_opts, opts, HistogramVec, IntCounterVec}; 4 | use tokio::time::Instant; 5 | 6 | #[derive(Debug, Clone)] 7 | pub struct Metrics { 8 | pub reconciliations: IntCounterVec, 9 | pub failures: IntCounterVec, 10 | pub reconcile_duration: HistogramVec, 11 | } 12 | 13 | impl Default for Metrics { 14 | fn default() -> Self { 15 | let reconcile_duration = HistogramVec::new( 16 | histogram_opts!( 17 | "sart_controller_reconcile_duration_seconds", 18 | "The duration of reconcile to complete in seconds" 19 | ) 20 | .buckets(vec![0.01, 0.1, 0.25, 0.5, 1., 5., 15., 60.]), 21 | &[], 22 | ) 23 | .unwrap(); 24 | let failures = IntCounterVec::new( 25 | opts!( 26 | "sart_controller_reconciliation_errors_total", 27 | "reconciliation errors", 28 | ), 29 | &["resource", "instance", "error"], 30 | ) 31 | .unwrap(); 32 | let reconciliations = IntCounterVec::new( 33 | opts!( 34 | "sart_controller_reconciliation_total", 35 | "Total count of reconciliations", 36 | ), 37 | &["resource", "instance"], 38 | ) 39 | .unwrap(); 40 | Metrics { 41 | reconciliations, 42 | failures, 43 | reconcile_duration, 44 | } 45 | } 46 | } 47 | 48 | impl Metrics { 49 | pub fn register(self, registry: &Registry) -> Result { 50 | registry.register(Box::new(self.reconciliations.clone()))?; 51 | registry.register(Box::new(self.failures.clone()))?; 52 | registry.register(Box::new(self.reconcile_duration.clone()))?; 53 | Ok(self) 54 | } 55 | 56 | pub fn reconcile_failure>(&self, resource: &T) { 57 | self.failures 58 | .with_label_values(&[ 59 | &resource.object_ref(&()).kind.unwrap(), 60 | &resource.object_ref(&()).name.unwrap(), 61 | ]) 62 | .inc() 63 | } 64 | 65 | pub fn reconciliation>(&self, resource: &T) { 66 | self.reconciliations 67 | .with_label_values(&[ 68 | &resource.object_ref(&()).kind.unwrap(), 69 | &resource.object_ref(&()).name.unwrap(), 70 | ]) 71 | .inc() 72 | } 73 | } 74 | 75 | /// Smart function duration measurer 76 | /// 77 | /// Relies on Drop to calculate duration and register the observation in the histogram 78 | pub struct ReconcileMeasurer { 79 | start: Instant, 80 | metric: HistogramVec, 81 | } 82 | 83 | impl Drop for ReconcileMeasurer { 84 | fn drop(&mut self) { 85 | #[allow(clippy::cast_precision_loss)] 86 | let duration = self.start.elapsed().as_millis() as f64 / 1000.0; 87 | self.metric.with_label_values(&[]).observe(duration); 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /sartd/src/kubernetes/src/util.rs: -------------------------------------------------------------------------------- 1 | use k8s_openapi::apimachinery::pkg::apis::meta::v1::OwnerReference; 2 | use kube::{core::ApiResource, Resource, ResourceExt}; 3 | 4 | use super::error::Error; 5 | 6 | pub fn create_owner_reference>(owner: &T) -> OwnerReference { 7 | let res = ApiResource::erase::(&()); 8 | OwnerReference { 9 | name: owner.name_any(), 10 | api_version: res.api_version, 11 | kind: res.kind, 12 | uid: match &owner.meta().uid { 13 | Some(uid) => uid.clone(), 14 | None => "".to_string(), 15 | }, 16 | block_owner_deletion: Some(true), 17 | controller: Some(true), 18 | } 19 | } 20 | 21 | pub fn get_namespace>(resource: &T) -> Result { 22 | resource.namespace().ok_or(Error::GetNamespace) 23 | } 24 | 25 | pub fn get_namespaced_name>(resource: &T) -> String { 26 | match resource.namespace() { 27 | Some(ns) => format!("{ns}/{}", resource.name_any()), 28 | None => resource.name_any(), 29 | } 30 | } 31 | 32 | pub fn escape_slash(s: &str) -> String { 33 | s.replace('/', "~1") 34 | } 35 | 36 | // pub fn get_diff(prev: &[String], now: &[String]) -> (Vec, Vec, Vec) { 37 | // let removed = prev 38 | // .iter() 39 | // .filter(|p| !now.contains(p)) 40 | // .cloned() 41 | // .collect::>(); 42 | // let added = now 43 | // .iter() 44 | // .filter(|n| !prev.contains(n) && !removed.contains(n)) 45 | // .cloned() 46 | // .collect::>(); 47 | // let shared = prev 48 | // .iter() 49 | // .filter(|p| now.contains(p)) 50 | // .cloned() 51 | // .collect::>(); 52 | // (added, shared, removed) 53 | // } 54 | 55 | pub fn diff(prev: &[T], now: &[T]) -> (Vec, Vec, Vec) { 56 | let removed = prev 57 | .iter() 58 | .filter(|p| !now.contains(p)) 59 | .cloned() 60 | .collect::>(); 61 | let added = now 62 | .iter() 63 | .filter(|n| !prev.contains(n) && !removed.contains(n)) 64 | .cloned() 65 | .collect::>(); 66 | let shared = prev 67 | .iter() 68 | .filter(|p| now.contains(p)) 69 | .cloned() 70 | .collect::>(); 71 | (added, shared, removed) 72 | 73 | } 74 | -------------------------------------------------------------------------------- /sartd/src/kubernetes/tests/agent_node_bgp_test.rs: -------------------------------------------------------------------------------- 1 | use std::sync::{Arc, Mutex}; 2 | 3 | use kube::{ 4 | api::{DeleteParams, Patch, PatchParams}, 5 | Api, Client, ResourceExt, 6 | }; 7 | use sartd_kubernetes::{ 8 | agent::{self, context::State, metrics::Metrics}, 9 | crd::{ 10 | bgp_peer::BGPPeer, 11 | node_bgp::{NodeBGP, NodeBGPCondition, NodeBGPConditionReason, NodeBGPConditionStatus}, 12 | }, 13 | fixture::test_trace, 14 | }; 15 | 16 | use crate::common::{cleanup_kind, setup_kind, test_node_bgp}; 17 | 18 | mod common; 19 | 20 | #[tokio::test] 21 | #[ignore = "use kind cluster"] 22 | async fn integration_test_agent_node_bgp() { 23 | tracing::info!("Creating a kind cluster"); 24 | setup_kind(); 25 | 26 | test_trace().await; 27 | 28 | tracing::info!("Starting the mock bgp server api server"); 29 | tokio::spawn(async move { 30 | sartd_mock::bgp::run(5000).await; 31 | }); 32 | 33 | tracing::info!("Getting kube client"); 34 | let client = Client::try_default().await.unwrap(); 35 | let ctx = State::default().to_context(client.clone(), 30, Arc::new(Mutex::new(Metrics::default()))); 36 | 37 | let nb = test_node_bgp(); 38 | let nb_api = Api::::all(ctx.client.clone()); 39 | let ssapply = PatchParams::apply("ctrltest"); 40 | 41 | let nb_patch = Patch::Apply(nb.clone()); 42 | 43 | tracing::info!("Creating the NodeBGP resource"); 44 | nb_api 45 | .patch(&nb.name_any(), &ssapply, &nb_patch) 46 | .await 47 | .unwrap(); 48 | 49 | let applied_nb = nb_api.get(&nb.name_any()).await.unwrap(); 50 | 51 | tracing::info!("Reconciling the resource"); 52 | agent::reconciler::node_bgp::reconciler(Arc::new(applied_nb.clone()), ctx.clone()) 53 | .await 54 | .unwrap(); 55 | 56 | tracing::info!("Checking NodeBGP's status"); 57 | let applied_nb = nb_api.get(&nb.name_any()).await.unwrap(); 58 | let binding = applied_nb 59 | .status 60 | .as_ref() 61 | .unwrap() 62 | .conditions 63 | .clone() 64 | .unwrap(); 65 | let last_cond = binding.last().unwrap(); 66 | assert_eq!( 67 | &NodeBGPCondition { 68 | status: NodeBGPConditionStatus::Available, 69 | reason: NodeBGPConditionReason::Configured, 70 | }, 71 | last_cond 72 | ); 73 | 74 | tracing::info!("Reconciling the resource again because of requeue"); 75 | agent::reconciler::node_bgp::reconciler(Arc::new(applied_nb.clone()), ctx.clone()) 76 | .await 77 | .unwrap(); 78 | 79 | tracing::info!("Checking NodeBGP's status"); 80 | let applied_nb = nb_api.get(&nb.name_any()).await.unwrap(); 81 | let binding = applied_nb 82 | .status 83 | .as_ref() 84 | .unwrap() 85 | .conditions 86 | .clone() 87 | .unwrap(); 88 | let last_cond = binding.last().unwrap(); 89 | assert_eq!( 90 | &NodeBGPCondition { 91 | status: NodeBGPConditionStatus::Available, 92 | reason: NodeBGPConditionReason::Configured, 93 | }, 94 | last_cond 95 | ); 96 | 97 | tracing::info!("Reconciling the resource again because of requeue"); 98 | let applied_nb = nb_api.get(&nb.name_any()).await.unwrap(); 99 | agent::reconciler::node_bgp::reconciler(Arc::new(applied_nb.clone()), ctx.clone()) 100 | .await 101 | .unwrap(); 102 | 103 | let bp_api = Api::::all(ctx.client.clone()); 104 | let _created_bp = bp_api 105 | .get(&nb.spec.peers.as_ref().unwrap()[0].name) 106 | .await 107 | .unwrap(); 108 | 109 | tracing::info!("Cleaning up NodeBGP"); 110 | nb_api 111 | .delete(&nb.name_any(), &DeleteParams::default()) 112 | .await 113 | .unwrap(); 114 | 115 | let deleted_nb = nb_api.get(&nb.name_any()).await.unwrap(); 116 | agent::reconciler::node_bgp::reconciler(Arc::new(deleted_nb.clone()), ctx.clone()) 117 | .await 118 | .unwrap(); 119 | 120 | tracing::info!("Cleaning up a kind cluster"); 121 | cleanup_kind(); 122 | } 123 | -------------------------------------------------------------------------------- /sartd/src/kubernetes/tests/config/.cargo: -------------------------------------------------------------------------------- 1 | [target.x86_64-unknown-linux-gnu] 2 | runner = 'sudo -E' 3 | -------------------------------------------------------------------------------- /sartd/src/kubernetes/tests/config/config.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kind.x-k8s.io/v1alpha4 2 | kind: Cluster 3 | networking: 4 | disableDefaultCNI: true 5 | nodes: 6 | - role: control-plane 7 | -------------------------------------------------------------------------------- /sartd/src/kubernetes/tests/controller_address_pool_pod_test.rs: -------------------------------------------------------------------------------- 1 | use std::sync::{Arc, Mutex}; 2 | 3 | use common::{cleanup_kind, setup_kind}; 4 | 5 | use kube::{ 6 | api::{ListParams, Patch, PatchParams}, 7 | Api, Client, ResourceExt, 8 | }; 9 | use sartd_ipam::manager::BlockAllocator; 10 | use sartd_kubernetes::{ 11 | controller::{ 12 | self, 13 | context::{Ctx, State}, 14 | metrics::Metrics, 15 | }, 16 | crd::{ 17 | address_block::AddressBlock, 18 | address_pool::{AddressPool, ADDRESS_POOL_ANNOTATION}, 19 | block_request::BlockRequest, 20 | }, 21 | fixture::{ 22 | reconciler::{test_address_pool_pod, test_block_request}, 23 | test_trace, 24 | }, 25 | }; 26 | 27 | mod common; 28 | 29 | #[tokio::test] 30 | #[ignore = "use kind cluster"] 31 | async fn test_address_pool_pod_handling_request() { 32 | tracing::info!("Creating a kind cluster"); 33 | setup_kind(); 34 | 35 | test_trace().await; 36 | 37 | tracing::info!("Getting kube client"); 38 | let client = Client::try_default().await.unwrap(); 39 | let block_allocator = Arc::new(BlockAllocator::default()); 40 | let ctx = State::default().to_context_with( 41 | client.clone(), 42 | 30, 43 | block_allocator, 44 | Arc::new(Mutex::new(Metrics::default())), 45 | ); 46 | 47 | let ap = test_address_pool_pod(); 48 | 49 | tracing::info!("Creating an AddressPool"); 50 | let ap_api = Api::::all(ctx.client().clone()); 51 | let ssapply = PatchParams::apply("ctrltest"); 52 | let ap_patch = Patch::Apply(ap.clone()); 53 | ap_api 54 | .patch(&ap.name_any(), &ssapply, &ap_patch) 55 | .await 56 | .unwrap(); 57 | 58 | let applied_ap = ap_api.get(&ap.name_any()).await.unwrap(); 59 | 60 | tracing::info!("Reconciling AddressPool"); 61 | controller::reconciler::address_pool::reconciler(Arc::new(applied_ap.clone()), ctx.clone()) 62 | .await 63 | .unwrap(); 64 | 65 | tracing::info!("Getting the applied pool"); 66 | let mut applied_ap = ap_api.get(&ap.name_any()).await.unwrap(); 67 | 68 | assert!(applied_ap.status.is_none()); 69 | 70 | tracing::info!("Creating BlockRequest"); 71 | let br = test_block_request(); 72 | let br_api = Api::::all(ctx.client().clone()); 73 | let br_patch = Patch::Apply(br.clone()); 74 | br_api 75 | .patch(&br.name_any(), &ssapply, &br_patch) 76 | .await 77 | .unwrap(); 78 | 79 | let ab_api = Api::::all(ctx.client().clone()); 80 | 81 | tracing::info!("Changing auto assign to false"); 82 | applied_ap.spec.auto_assign = Some(false); 83 | 84 | tracing::info!("Reconciling AddressPool"); 85 | controller::reconciler::address_pool::reconciler(Arc::new(applied_ap.clone()), ctx.clone()) 86 | .await 87 | .unwrap(); 88 | 89 | tracing::info!("Checking blocks are changed to auto_assign=false"); 90 | let list_params = ListParams::default().labels(&format!( 91 | "{}={}", 92 | ADDRESS_POOL_ANNOTATION, 93 | applied_ap.name_any() 94 | )); 95 | let ab_list = ab_api.list(&list_params).await.unwrap(); 96 | for ab in ab_list.iter() { 97 | assert!(!ab.spec.auto_assign); 98 | } 99 | 100 | tracing::info!("Cleaning up a kind cluster"); 101 | cleanup_kind(); 102 | } 103 | -------------------------------------------------------------------------------- /sartd/src/kubernetes/tests/controller_address_pool_service_test.rs: -------------------------------------------------------------------------------- 1 | use std::sync::{Arc, Mutex}; 2 | 3 | use common::{cleanup_kind, setup_kind}; 4 | 5 | use kube::{ 6 | api::{Patch, PatchParams}, 7 | Api, Client, ResourceExt, 8 | }; 9 | use sartd_ipam::manager::BlockAllocator; 10 | use sartd_kubernetes::{ 11 | controller::{ 12 | self, 13 | context::{Ctx, State}, 14 | metrics::Metrics, 15 | }, 16 | crd::{address_block::AddressBlock, address_pool::AddressPool}, 17 | fixture::{reconciler::test_address_pool_lb, test_trace}, 18 | }; 19 | 20 | mod common; 21 | 22 | #[tokio::test] 23 | #[ignore = "use kind cluster"] 24 | async fn integration_test_address_pool() { 25 | tracing::info!("Creating a kind cluster"); 26 | setup_kind(); 27 | 28 | test_trace().await; 29 | 30 | tracing::info!("Getting kube client"); 31 | let client = Client::try_default().await.unwrap(); 32 | let block_allocator = Arc::new(BlockAllocator::default()); 33 | let ctx = State::default().to_context_with( 34 | client.clone(), 35 | 30, 36 | block_allocator, 37 | Arc::new(Mutex::new(Metrics::default())), 38 | ); 39 | 40 | let ap = test_address_pool_lb(); 41 | 42 | tracing::info!("Creating an AddressPool resource"); 43 | let ap_api = Api::::all(ctx.client().clone()); 44 | let ssapply = PatchParams::apply("ctrltest"); 45 | let ap_patch = Patch::Apply(ap.clone()); 46 | ap_api 47 | .patch(&ap.name_any(), &ssapply, &ap_patch) 48 | .await 49 | .unwrap(); 50 | 51 | let applied_ap = ap_api.get(&ap.name_any()).await.unwrap(); 52 | 53 | // do reconcile 54 | tracing::info!("Reconciling AddressPool"); 55 | controller::reconciler::address_pool::reconciler(Arc::new(applied_ap.clone()), ctx.clone()) 56 | .await 57 | .unwrap(); 58 | 59 | tracing::info!("Getting a AddressBlock resource created by AddressPool"); 60 | let ab_api = Api::::all(ctx.client().clone()); 61 | let ab = ab_api.get(&applied_ap.name_any()).await.unwrap(); 62 | 63 | tracing::info!("Checking created block"); 64 | assert_eq!(applied_ap.spec.cidr, ab.spec.cidr); 65 | assert_eq!( 66 | applied_ap.spec.auto_assign.unwrap_or_default(), 67 | ab.spec.auto_assign 68 | ); 69 | 70 | tracing::info!("Cleaning up a kind cluster"); 71 | cleanup_kind(); 72 | } 73 | -------------------------------------------------------------------------------- /sartd/src/kubernetes/tests/controller_block_request_test.rs: -------------------------------------------------------------------------------- 1 | use std::sync::{Arc, Mutex}; 2 | 3 | use kube::{ 4 | api::{Patch, PatchParams}, 5 | Api, Client, ResourceExt, 6 | }; 7 | use sartd_ipam::manager::BlockAllocator; 8 | use sartd_kubernetes::{ 9 | controller::{ 10 | self, 11 | context::{Ctx, State}, 12 | metrics::Metrics, 13 | }, 14 | crd::{address_pool::AddressPool, block_request::BlockRequest}, 15 | fixture::{ 16 | reconciler::{test_address_pool_pod, test_block_request}, 17 | test_trace, 18 | }, 19 | }; 20 | 21 | use crate::common::{cleanup_kind, setup_kind}; 22 | 23 | mod common; 24 | 25 | #[tokio::test] 26 | #[ignore = "use kind cluster"] 27 | async fn integration_test_block_request() { 28 | tracing::info!("Creating a kind cluster"); 29 | setup_kind(); 30 | 31 | test_trace().await; 32 | 33 | tracing::info!("Getting kube client"); 34 | let client = Client::try_default().await.unwrap(); 35 | 36 | let block_allocator = Arc::new(BlockAllocator::default()); 37 | 38 | let ctx = State::default().to_context_with( 39 | client.clone(), 40 | 30, 41 | block_allocator, 42 | Arc::new(Mutex::new(Metrics::default())), 43 | ); 44 | 45 | tracing::info!("Creating AddressPool"); 46 | let ap = test_address_pool_pod(); 47 | let ap_api = Api::::all(ctx.client().clone()); 48 | let ssapply = PatchParams::apply("ctrltest"); 49 | let ap_patch = Patch::Apply(ap.clone()); 50 | ap_api 51 | .patch(&ap.name_any(), &ssapply, &ap_patch) 52 | .await 53 | .unwrap(); 54 | let applied_ap = ap_api.get(&ap.name_any()).await.unwrap(); 55 | controller::reconciler::address_pool::reconciler(Arc::new(applied_ap), ctx.clone()) 56 | .await 57 | .unwrap(); 58 | 59 | tracing::info!("Reconciling AddressPool"); 60 | 61 | tracing::info!("Creating BlockRequest"); 62 | let br = test_block_request(); 63 | let br_api = Api::::all(ctx.client().clone()); 64 | let br_patch = Patch::Apply(br.clone()); 65 | br_api 66 | .patch(&br.name_any(), &ssapply, &br_patch) 67 | .await 68 | .unwrap(); 69 | 70 | let applied_br = br_api.get(&br.name_any()).await.unwrap(); 71 | 72 | tracing::info!("Reconciling BlockRequest"); 73 | controller::reconciler::block_request::reconciler(Arc::new(applied_br.clone()), ctx.clone()) 74 | .await 75 | .unwrap(); 76 | 77 | tracing::info!("Cleaning up a kind cluster"); 78 | cleanup_kind(); 79 | } 80 | -------------------------------------------------------------------------------- /sartd/src/mock/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "sartd-mock" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | tonic = "0.10.2" 10 | sartd-proto = { path = "../proto" } 11 | -------------------------------------------------------------------------------- /sartd/src/mock/src/lib.rs: -------------------------------------------------------------------------------- 1 | pub mod bgp; 2 | -------------------------------------------------------------------------------- /sartd/src/proto/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "sartd-proto" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | [dependencies] 7 | tonic-build = "0.10.2" 8 | tonic = "0.10.2" 9 | prost = "0.12.3" 10 | prost-types = "0.12.3" 11 | 12 | [build-dependencies] 13 | tonic-build = { version = "0.10.2", features = ["prost"] } 14 | -------------------------------------------------------------------------------- /sartd/src/proto/build.rs: -------------------------------------------------------------------------------- 1 | use std::env; 2 | use std::path::PathBuf; 3 | 4 | fn main() -> Result<(), Box> { 5 | let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); 6 | 7 | tonic_build::configure() 8 | .build_server(true) 9 | .file_descriptor_set_path(out_dir.join("sartd.bin")) // Add this 10 | .out_dir("./src") 11 | .compile( 12 | &[ 13 | "../../../proto/bgp.proto", 14 | "../../../proto/cni.proto", 15 | "../../../proto/fib.proto", 16 | "../../../proto/fib_manager.proto", 17 | ], 18 | &["../../../proto"], 19 | ) 20 | .unwrap_or_else(|e| panic!("protobuf compile error: {}", e)); 21 | Ok(()) 22 | } 23 | -------------------------------------------------------------------------------- /sartd/src/proto/src/google.protobuf.rs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terassyi/sart/82d8bbeb0224e43cc80a03e6198a98226d2ede47/sartd/src/proto/src/google.protobuf.rs -------------------------------------------------------------------------------- /sartd/src/proto/src/lib.rs: -------------------------------------------------------------------------------- 1 | pub mod sart { 2 | include!("sart.v1.rs"); 3 | } 4 | pub mod google { 5 | pub mod protobuf { 6 | include!("google.protobuf.rs"); 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /sartd/src/trace/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "sartd-trace" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | [dependencies] 7 | k8s-openapi = { version = "0.20.0", features = ["v1_28"] } 8 | kube = { version = "0.87.2", features = [] } 9 | opentelemetry = { version = "0.21.0", features = ["trace"] } 10 | prometheus = "0.13.3" 11 | rand = "0.8.5" 12 | tokio = { version = "1.35.1", features = ["time"] } 13 | tracing = "0.1.40" 14 | tracing-opentelemetry = "0.22.0" 15 | tracing-subscriber = { version = "0.3.18", features = ["json", "env-filter"] } 16 | -------------------------------------------------------------------------------- /sartd/src/trace/src/error.rs: -------------------------------------------------------------------------------- 1 | pub trait TraceableError: std::error::Error { 2 | fn metric_label(&self) -> String; 3 | } 4 | -------------------------------------------------------------------------------- /sartd/src/trace/src/init.rs: -------------------------------------------------------------------------------- 1 | use std::str::FromStr; 2 | 3 | use tracing_subscriber::{prelude::*, Registry}; 4 | 5 | #[derive(Debug)] 6 | pub struct TraceConfig { 7 | pub level: String, 8 | pub format: String, 9 | pub file: Option, 10 | pub _metrics_endpoint: Option, 11 | } 12 | 13 | pub async fn prepare_tracing(conf: TraceConfig) { 14 | #[cfg(feature = "telemetry")] 15 | let span = SpanBuilder::default().with_kind(opentelemetry::trace::SpanKind::Internal); 16 | // Configure otel exporter. 17 | 18 | #[cfg(not(feature = "telemetry"))] 19 | if conf.format == "json" { 20 | if let Some(path) = conf.file { 21 | let file = std::fs::File::create(path).unwrap(); 22 | Registry::default() 23 | .with(tracing_subscriber::fmt::Layer::new().with_writer(file)) 24 | .with(tracing_subscriber::fmt::Layer::new().with_ansi(true).json()) 25 | .with(tracing_subscriber::filter::LevelFilter::from_str(&conf.level).unwrap()) 26 | .init(); 27 | } else { 28 | Registry::default() 29 | .with(tracing_subscriber::fmt::Layer::new().with_ansi(true).json()) 30 | .with(tracing_subscriber::filter::LevelFilter::from_str(&conf.level).unwrap()) 31 | .init(); 32 | } 33 | } else if let Some(path) = conf.file { 34 | let file = std::fs::File::create(path).unwrap(); 35 | Registry::default() 36 | .with(tracing_subscriber::fmt::Layer::new().with_writer(file)) 37 | .with(tracing_subscriber::fmt::Layer::new().with_ansi(true)) 38 | .with(tracing_subscriber::filter::LevelFilter::from_str(&conf.level).unwrap()) 39 | .init(); 40 | } else { 41 | Registry::default() 42 | .with(tracing_subscriber::fmt::Layer::new().with_ansi(true)) 43 | .with(tracing_subscriber::filter::LevelFilter::from_str(&conf.level).unwrap()) 44 | .init(); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /sartd/src/trace/src/lib.rs: -------------------------------------------------------------------------------- 1 | pub mod error; 2 | pub mod init; 3 | pub mod metrics; 4 | pub mod telemetry; 5 | -------------------------------------------------------------------------------- /sartd/src/trace/src/metrics.rs: -------------------------------------------------------------------------------- 1 | use kube::Resource; 2 | use prometheus::Registry; 3 | use prometheus::{histogram_opts, opts, HistogramVec, IntCounter, IntCounterVec}; 4 | use tokio::time::Instant; 5 | 6 | use super::error::TraceableError; 7 | 8 | #[derive(Debug, Clone)] 9 | pub struct Metrics { 10 | pub reconciliations: IntCounter, 11 | pub failures: IntCounterVec, 12 | pub reconcile_duration: HistogramVec, 13 | } 14 | 15 | impl Default for Metrics { 16 | fn default() -> Self { 17 | let reconcile_duration = HistogramVec::new( 18 | histogram_opts!( 19 | "doc_controller_reconcile_duration_seconds", 20 | "The duration of reconcile to complete in seconds" 21 | ) 22 | .buckets(vec![0.01, 0.1, 0.25, 0.5, 1., 5., 15., 60.]), 23 | &[], 24 | ) 25 | .unwrap(); 26 | let failures = IntCounterVec::new( 27 | opts!( 28 | "doc_controller_reconciliation_errors_total", 29 | "reconciliation errors", 30 | ), 31 | &["instance", "error"], 32 | ) 33 | .unwrap(); 34 | let reconciliations = 35 | IntCounter::new("doc_controller_reconciliations_total", "reconciliations").unwrap(); 36 | Metrics { 37 | reconciliations, 38 | failures, 39 | reconcile_duration, 40 | } 41 | } 42 | } 43 | 44 | impl Metrics { 45 | pub fn register(self, registry: &Registry) -> Result { 46 | Ok(self) 47 | } 48 | 49 | pub fn reconcile_failure, E: TraceableError>( 50 | &self, 51 | resource: &T, 52 | e: &E, 53 | ) { 54 | self.failures 55 | .with_label_values(&[ 56 | &resource.object_ref(&()).name.unwrap(), 57 | e.metric_label().as_ref(), 58 | ]) 59 | .inc() 60 | } 61 | 62 | pub fn count_and_measure(&self) -> ReconcileMeasurer { 63 | self.reconciliations.inc(); 64 | ReconcileMeasurer { 65 | start: Instant::now(), 66 | metric: self.reconcile_duration.clone(), 67 | } 68 | } 69 | } 70 | 71 | /// Smart function duration measurer 72 | /// 73 | /// Relies on Drop to calculate duration and register the observation in the histogram 74 | pub struct ReconcileMeasurer { 75 | start: Instant, 76 | metric: HistogramVec, 77 | } 78 | 79 | impl Drop for ReconcileMeasurer { 80 | fn drop(&mut self) { 81 | #[allow(clippy::cast_precision_loss)] 82 | let duration = self.start.elapsed().as_millis() as f64 / 1000.0; 83 | self.metric.with_label_values(&[]).observe(duration); 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /sartd/src/trace/src/telemetry.rs: -------------------------------------------------------------------------------- 1 | use opentelemetry::trace::TraceId; 2 | use rand::Rng; 3 | use tracing_subscriber::{prelude::*, Registry}; 4 | 5 | /// Fetch an opentelemetry::trace::TraceId as hex through the full tracing stack 6 | pub fn get_trace_id() -> TraceId { 7 | let mut rng = rand::thread_rng(); 8 | let val: u128 = rng.gen(); 9 | TraceId::from(val) 10 | } 11 | 12 | pub async fn init(level: tracing::Level) { 13 | // Setup tracing layers 14 | #[cfg(feature = "telemetry")] 15 | let telemetry = tracing_opentelemetry::layer().with_tracer(init_tracer().await); 16 | let logger = tracing_subscriber::fmt::layer().compact(); 17 | 18 | // Decide on layers 19 | #[cfg(feature = "telemetry")] 20 | let collector = Registry::default() 21 | .with(telemetry) 22 | .with(logger) 23 | .with(tracing_subscriber::filter::LevelFilter::from_level(level)); 24 | #[cfg(not(feature = "telemetry"))] 25 | let collector = Registry::default() 26 | .with(logger) 27 | .with(tracing_subscriber::filter::LevelFilter::from_level(level)); 28 | 29 | // Initialize tracing 30 | tracing::subscriber::set_global_default(collector).unwrap(); 31 | } 32 | -------------------------------------------------------------------------------- /sartd/src/util/Cargo.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Cargo. 2 | # It is not intended for manual editing. 3 | version = 3 4 | 5 | [[package]] 6 | name = "anyhow" 7 | version = "1.0.82" 8 | source = "registry+https://github.com/rust-lang/crates.io-index" 9 | checksum = "f538837af36e6f6a9be0faa67f9a314f8119e4e4b5867c6ab40ed60360142519" 10 | 11 | [[package]] 12 | name = "bytes" 13 | version = "1.6.0" 14 | source = "registry+https://github.com/rust-lang/crates.io-index" 15 | checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" 16 | 17 | [[package]] 18 | name = "either" 19 | version = "1.10.0" 20 | source = "registry+https://github.com/rust-lang/crates.io-index" 21 | checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a" 22 | 23 | [[package]] 24 | name = "itertools" 25 | version = "0.12.1" 26 | source = "registry+https://github.com/rust-lang/crates.io-index" 27 | checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" 28 | dependencies = [ 29 | "either", 30 | ] 31 | 32 | [[package]] 33 | name = "proc-macro2" 34 | version = "1.0.79" 35 | source = "registry+https://github.com/rust-lang/crates.io-index" 36 | checksum = "e835ff2298f5721608eb1a980ecaee1aef2c132bf95ecc026a11b7bf3c01c02e" 37 | dependencies = [ 38 | "unicode-ident", 39 | ] 40 | 41 | [[package]] 42 | name = "prost" 43 | version = "0.12.4" 44 | source = "registry+https://github.com/rust-lang/crates.io-index" 45 | checksum = "d0f5d036824e4761737860779c906171497f6d55681139d8312388f8fe398922" 46 | dependencies = [ 47 | "bytes", 48 | "prost-derive", 49 | ] 50 | 51 | [[package]] 52 | name = "prost-derive" 53 | version = "0.12.4" 54 | source = "registry+https://github.com/rust-lang/crates.io-index" 55 | checksum = "19de2de2a00075bf566bee3bd4db014b11587e84184d3f7a791bc17f1a8e9e48" 56 | dependencies = [ 57 | "anyhow", 58 | "itertools", 59 | "proc-macro2", 60 | "quote", 61 | "syn", 62 | ] 63 | 64 | [[package]] 65 | name = "prost-types" 66 | version = "0.12.4" 67 | source = "registry+https://github.com/rust-lang/crates.io-index" 68 | checksum = "3235c33eb02c1f1e212abdbe34c78b264b038fb58ca612664343271e36e55ffe" 69 | dependencies = [ 70 | "prost", 71 | ] 72 | 73 | [[package]] 74 | name = "quote" 75 | version = "1.0.36" 76 | source = "registry+https://github.com/rust-lang/crates.io-index" 77 | checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" 78 | dependencies = [ 79 | "proc-macro2", 80 | ] 81 | 82 | [[package]] 83 | name = "sartd-util" 84 | version = "0.1.0" 85 | dependencies = [ 86 | "prost", 87 | "prost-types", 88 | ] 89 | 90 | [[package]] 91 | name = "syn" 92 | version = "2.0.58" 93 | source = "registry+https://github.com/rust-lang/crates.io-index" 94 | checksum = "44cfb93f38070beee36b3fef7d4f5a16f27751d94b187b666a5cc5e9b0d30687" 95 | dependencies = [ 96 | "proc-macro2", 97 | "quote", 98 | "unicode-ident", 99 | ] 100 | 101 | [[package]] 102 | name = "unicode-ident" 103 | version = "1.0.12" 104 | source = "registry+https://github.com/rust-lang/crates.io-index" 105 | checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" 106 | -------------------------------------------------------------------------------- /sartd/src/util/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "sartd-util" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | [dependencies] 7 | prost = "0.12.3" 8 | prost-types = "0.12.3" 9 | -------------------------------------------------------------------------------- /sartd/src/util/src/lib.rs: -------------------------------------------------------------------------------- 1 | const TYPE_URL_PREFACE: &str = "type.googleapis.com/sart.v1."; 2 | 3 | pub fn to_any(m: T, name: &str) -> prost_types::Any { 4 | let mut v = Vec::new(); 5 | m.encode(&mut v).unwrap(); 6 | prost_types::Any { 7 | type_url: format!("{}{}", TYPE_URL_PREFACE, name), 8 | value: v, 9 | } 10 | } 11 | 12 | pub fn type_url(t: &str) -> String { 13 | format!("{}{}", TYPE_URL_PREFACE, t) 14 | } 15 | -------------------------------------------------------------------------------- /sartd/testdata/config.yaml: -------------------------------------------------------------------------------- 1 | port: 179 2 | asn: 6550 3 | router_id: 1.1.1.1 4 | neighbors: 5 | - asn: 100 6 | router_id: 2.2.2.2 7 | address: 127.0.0.1 8 | - asn: 200 9 | router_id: 127.0.0.1 10 | address: '::1' 11 | -------------------------------------------------------------------------------- /sartd/testdata/fib_config.yaml: -------------------------------------------------------------------------------- 1 | endpoint: 127.0.0.1:5001 2 | channels: 3 | - name: kernel_tables 4 | ip_version: ipv4 5 | subscribers: 6 | - protocol: kernel 7 | tables: 8 | - 254 9 | - 8 10 | publishers: 11 | - protocol: bgp 12 | endpoint: 127.0.0.1:5100 13 | - name: bgp_rib 14 | ip_version: ipv4 15 | subscribers: 16 | - protocol: bgp 17 | endpoint: 127.0.0.1:5000 18 | publishers: 19 | - protocol: kernel 20 | tables: 21 | - 254 22 | -------------------------------------------------------------------------------- /sartd/testdata/messages/frr-ibgp-fail: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terassyi/sart/82d8bbeb0224e43cc80a03e6198a98226d2ede47/sartd/testdata/messages/frr-ibgp-fail -------------------------------------------------------------------------------- /sartd/testdata/messages/keepalive: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terassyi/sart/82d8bbeb0224e43cc80a03e6198a98226d2ede47/sartd/testdata/messages/keepalive -------------------------------------------------------------------------------- /sartd/testdata/messages/notification-bad-as-peer: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terassyi/sart/82d8bbeb0224e43cc80a03e6198a98226d2ede47/sartd/testdata/messages/notification-bad-as-peer -------------------------------------------------------------------------------- /sartd/testdata/messages/open-2bytes-asn: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terassyi/sart/82d8bbeb0224e43cc80a03e6198a98226d2ede47/sartd/testdata/messages/open-2bytes-asn -------------------------------------------------------------------------------- /sartd/testdata/messages/open-4bytes-asn: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terassyi/sart/82d8bbeb0224e43cc80a03e6198a98226d2ede47/sartd/testdata/messages/open-4bytes-asn -------------------------------------------------------------------------------- /sartd/testdata/messages/open-bad-message-length: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terassyi/sart/82d8bbeb0224e43cc80a03e6198a98226d2ede47/sartd/testdata/messages/open-bad-message-length -------------------------------------------------------------------------------- /sartd/testdata/messages/open-graceful-restart: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terassyi/sart/82d8bbeb0224e43cc80a03e6198a98226d2ede47/sartd/testdata/messages/open-graceful-restart -------------------------------------------------------------------------------- /sartd/testdata/messages/open-ipv6: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terassyi/sart/82d8bbeb0224e43cc80a03e6198a98226d2ede47/sartd/testdata/messages/open-ipv6 -------------------------------------------------------------------------------- /sartd/testdata/messages/open-optional-parameters: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terassyi/sart/82d8bbeb0224e43cc80a03e6198a98226d2ede47/sartd/testdata/messages/open-optional-parameters -------------------------------------------------------------------------------- /sartd/testdata/messages/route-refresh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terassyi/sart/82d8bbeb0224e43cc80a03e6198a98226d2ede47/sartd/testdata/messages/route-refresh -------------------------------------------------------------------------------- /sartd/testdata/messages/update-as-set: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terassyi/sart/82d8bbeb0224e43cc80a03e6198a98226d2ede47/sartd/testdata/messages/update-as-set -------------------------------------------------------------------------------- /sartd/testdata/messages/update-as4-path: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terassyi/sart/82d8bbeb0224e43cc80a03e6198a98226d2ede47/sartd/testdata/messages/update-as4-path -------------------------------------------------------------------------------- /sartd/testdata/messages/update-as4-path-aggregator: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terassyi/sart/82d8bbeb0224e43cc80a03e6198a98226d2ede47/sartd/testdata/messages/update-as4-path-aggregator -------------------------------------------------------------------------------- /sartd/testdata/messages/update-ipv6-mp-reach-nlri: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terassyi/sart/82d8bbeb0224e43cc80a03e6198a98226d2ede47/sartd/testdata/messages/update-ipv6-mp-reach-nlri -------------------------------------------------------------------------------- /sartd/testdata/messages/update-mp-reach-nlri: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terassyi/sart/82d8bbeb0224e43cc80a03e6198a98226d2ede47/sartd/testdata/messages/update-mp-reach-nlri -------------------------------------------------------------------------------- /sartd/testdata/messages/update-nlri: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terassyi/sart/82d8bbeb0224e43cc80a03e6198a98226d2ede47/sartd/testdata/messages/update-nlri --------------------------------------------------------------------------------