├── pkg ├── cache │ ├── cache_suite_test.go │ ├── fs_ops.go │ ├── cache.go │ ├── cache_test.go │ └── mocks │ │ └── StateCache.go ├── rdma │ ├── rdma_suite_test.go │ ├── rdma_ops.go │ ├── rdma.go │ ├── rdma_test.go │ └── mocks │ │ ├── RdmaManager.go │ │ └── RdmaBasicOps.go ├── types │ └── types.go └── utils │ └── utils.go ├── cmd └── rdma │ ├── main_suite_test.go │ ├── main.go │ └── main_test.go ├── .gitignore ├── .mockery.yml ├── examples ├── sriov_dp_rdma_resource.yaml ├── rdma_test_pod.yaml └── rdma_net_crd.yaml ├── Dockerfile ├── .github ├── workflows │ ├── static-scan.yaml │ ├── codeql.yaml │ ├── image-push-master.yaml │ ├── image-push-release.yaml │ └── buildtest.yaml └── dependabot.yml ├── images ├── README.md └── entrypoint.sh ├── deployment └── rdma-cni-daemonset.yaml ├── go.mod ├── .golangci.yml ├── Makefile ├── README.md ├── go.sum └── LICENSE /pkg/cache/cache_suite_test.go: -------------------------------------------------------------------------------- 1 | package cache_test 2 | 3 | import ( 4 | "testing" 5 | 6 | . "github.com/onsi/ginkgo/v2" 7 | . "github.com/onsi/gomega" 8 | ) 9 | 10 | func TestCache(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "Cache Suite") 13 | } 14 | -------------------------------------------------------------------------------- /pkg/rdma/rdma_suite_test.go: -------------------------------------------------------------------------------- 1 | package rdma_test 2 | 3 | import ( 4 | "testing" 5 | 6 | . "github.com/onsi/ginkgo/v2" 7 | . "github.com/onsi/gomega" 8 | ) 9 | 10 | func TestRdma(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "RdmaManager Suite") 13 | } 14 | -------------------------------------------------------------------------------- /cmd/rdma/main_suite_test.go: -------------------------------------------------------------------------------- 1 | package main_test 2 | 3 | import ( 4 | "testing" 5 | 6 | . "github.com/onsi/ginkgo/v2" 7 | . "github.com/onsi/gomega" 8 | ) 9 | 10 | func TestRdma(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "Rdma Command Suite") 13 | } 14 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, built with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | *.cover 14 | 15 | # Folders 16 | .gopath/ 17 | .idea/ 18 | /bin/ 19 | /build/ 20 | -------------------------------------------------------------------------------- /.mockery.yml: -------------------------------------------------------------------------------- 1 | all: false 2 | dir: "{{.InterfaceDir}}/mocks" 3 | pkgname: mocks 4 | packages: 5 | github.com/k8snetworkplumbingwg/rdma-cni/pkg/cache: 6 | interfaces: 7 | StateCache: 8 | config: 9 | filename: "StateCache.go" 10 | github.com/k8snetworkplumbingwg/rdma-cni/pkg/rdma: 11 | interfaces: 12 | Manager: 13 | config: 14 | filename: "RdmaManager.go" 15 | BasicOps: 16 | config: 17 | filename: "RdmaBasicOps.go" 18 | -------------------------------------------------------------------------------- /examples/sriov_dp_rdma_resource.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: sriovdp-config 5 | namespace: kube-system 6 | data: 7 | config.json: | 8 | { 9 | "resourceList": [ 10 | { 11 | "resourcePrefix": "mellanox.com", 12 | "resourceName": "sriov_rdma", 13 | "selectors": { 14 | "isRdma": true, 15 | "vendors": ["15b3"], 16 | "pfNames": ["enp4s0f0"] 17 | } 18 | } 19 | ] 20 | } 21 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM --platform=$BUILDPLATFORM golang:alpine AS builder 2 | 3 | ARG TARGETPLATFORM 4 | ARG TARGETARCH 5 | ARG TARGETOS 6 | 7 | COPY . /usr/src/rdma-cni 8 | WORKDIR /usr/src/rdma-cni 9 | 10 | ENV HTTP_PROXY=$http_proxy 11 | ENV HTTPS_PROXY=$https_proxy 12 | 13 | RUN apk add --no-cache build-base=~0.5 \ 14 | && make clean && make build TARGET_OS=$TARGETOS TARGET_ARCH=$TARGETARCH 15 | 16 | FROM alpine:3 17 | COPY --from=builder /usr/src/rdma-cni/build/rdma /usr/bin/ 18 | COPY ./images/entrypoint.sh / 19 | 20 | WORKDIR / 21 | LABEL io.k8s.display-name="RDMA CNI" 22 | ENTRYPOINT ["/entrypoint.sh"] 23 | -------------------------------------------------------------------------------- /examples/rdma_test_pod.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | generateName: rdma-test-pod 5 | annotations: 6 | k8s.v1.cni.cncf.io/networks: sriov-rdma-net 7 | spec: 8 | # Uncomment for all in one K8s deployment 9 | # tolerations: 10 | # - key: node-role.kubernetes.io/master 11 | # operator: Exists 12 | # effect: NoSchedule 13 | containers: 14 | - name: rdma-app 15 | image: centos/tools 16 | imagePullPolicy: IfNotPresent 17 | command: [ "/bin/bash", "-c", "--" ] 18 | args: [ "while true; do sleep 300000; done;" ] 19 | resources: 20 | requests: 21 | mellanox.com/sriov_rdma: '1' 22 | limits: 23 | mellanox.com/sriov_rdma: '1' 24 | -------------------------------------------------------------------------------- /.github/workflows/static-scan.yaml: -------------------------------------------------------------------------------- 1 | name: go-static-analysis 2 | on: [push, pull_request] 3 | jobs: 4 | golangci: 5 | name: Lint 6 | runs-on: ubuntu-24.04 7 | steps: 8 | - name: set up Go 9 | uses: actions/setup-go@v6 10 | with: 11 | go-version: 1.24.x 12 | - name: checkout PR 13 | uses: actions/checkout@v6 14 | - name: run make lint 15 | run: make lint 16 | shellcheck: 17 | name: shellcheck 18 | runs-on: ubuntu-24.04 19 | steps: 20 | - name: checkout PR 21 | uses: actions/checkout@v6 22 | - name: run ShellCheck 23 | uses: ludeeus/action-shellcheck@master 24 | hadolint: 25 | runs-on: ubuntu-24.04 26 | name: Hadolint 27 | steps: 28 | - name: checkout PR 29 | uses: actions/checkout@v6 30 | - name: run Hadolint 31 | uses: hadolint/hadolint-action@v3.3.0 32 | with: 33 | dockerfile: Dockerfile 34 | -------------------------------------------------------------------------------- /examples/rdma_net_crd.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: "k8s.cni.cncf.io/v1" 2 | kind: NetworkAttachmentDefinition 3 | metadata: 4 | name: sriov-rdma-net 5 | annotations: 6 | k8s.v1.cni.cncf.io/resourceName: mellanox.com/sriov_rdma 7 | spec: 8 | config: '{ 9 | "cniVersion": "0.3.1", 10 | "name": "sriov-rdma-net", 11 | "plugins": [{ 12 | "type": "sriov", 13 | "ipam": { 14 | "type": "host-local", 15 | "subnet": "10.56.217.0/24", 16 | "routes": [{ 17 | "dst": "0.0.0.0/0" 18 | }], 19 | "gateway": "10.56.217.1" 20 | } 21 | }, 22 | { 23 | "type": "rdma" 24 | }] 25 | }' 26 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | # Docker images 4 | - package-ecosystem: "docker" 5 | directory: "/" 6 | schedule: 7 | interval: "daily" 8 | commit-message: 9 | prefix: "chore:" 10 | 11 | # GitHub Actions 12 | - package-ecosystem: "github-actions" 13 | directory: "/" 14 | schedule: 15 | interval: "weekly" 16 | day: "monday" 17 | 18 | - package-ecosystem: "gomod" 19 | directory: "/" 20 | schedule: 21 | interval: "weekly" 22 | day: "tuesday" 23 | groups: 24 | kubernetes: 25 | patterns: [ "k8s.io/*" ] 26 | ignore: 27 | # Ignore controller-runtime, and Kubernetes major and minor updates. These should be done manually. 28 | - dependency-name: "sigs.k8s.io/controller-runtime" 29 | update-types: [ "version-update:semver-major", "version-update:semver-minor" ] 30 | - dependency-name: "k8s.io/*" 31 | update-types: [ "version-update:semver-major", "version-update:semver-minor" ] 32 | -------------------------------------------------------------------------------- /.github/workflows/codeql.yaml: -------------------------------------------------------------------------------- 1 | name: "CodeQL" 2 | 3 | on: 4 | push: 5 | branches: [ "master" ] 6 | pull_request: 7 | branches: [ "master" ] 8 | schedule: 9 | - cron: "37 4 * * 0" 10 | 11 | jobs: 12 | analyze: 13 | name: Analyze 14 | runs-on: ubuntu-24.04 15 | permissions: 16 | actions: read 17 | contents: read 18 | security-events: write 19 | 20 | strategy: 21 | fail-fast: false 22 | matrix: 23 | language: [ go ] 24 | 25 | steps: 26 | - name: Checkout 27 | uses: actions/checkout@v6 28 | 29 | - name: Initialize CodeQL 30 | uses: github/codeql-action/init@v4 31 | with: 32 | languages: ${{ matrix.language }} 33 | queries: +security-and-quality 34 | 35 | - name: Autobuild 36 | uses: github/codeql-action/autobuild@v4 37 | 38 | - name: Perform CodeQL Analysis 39 | uses: github/codeql-action/analyze@v4 40 | with: 41 | category: "/language:${{ matrix.language }}" 42 | -------------------------------------------------------------------------------- /images/README.md: -------------------------------------------------------------------------------- 1 | ## Dockerfile build 2 | 3 | This is used for distribution of RDMA CNI binary in a Docker image. 4 | 5 | Typically you'd build this from the root of your RDMA CNI clone, and you'd set the `DOCKERFILE` to specify the Dockerfile during build time, `TAG` to specify the image's tag: 6 | 7 | ``` 8 | $ DOCKERFILE=Dockerfile TAG=ghcr.io/k8snetworkplumbingwg/rdma-cni make image 9 | ``` 10 | 11 | --- 12 | 13 | ## Daemonset deployment 14 | 15 | You may wish to deploy RDMA CNI as a daemonset, you can do so by starting with the example Daemonset shown here: 16 | 17 | ``` 18 | $ kubectl create -f ../deployment/rdma-cni-daemonset.yaml 19 | ``` 20 | 21 | > __*Note:*__ The likely best practice here is to build your own image given the Dockerfile, and then push it to your preferred registry, and change the `image` fields in the Daemonset YAML to reference that image. 22 | 23 | --- 24 | 25 | ### Development notes 26 | 27 | Example docker run command: 28 | 29 | ``` 30 | $ docker run -it -v /opt/cni/bin/:/host/opt/cni/bin/ --entrypoint=/bin/sh ghcr.io/k8snetworkplumbingwg/rdma-cni 31 | ``` 32 | 33 | > __*Note:*__ `/opt/cni/bin` is assumed to be the CNI directory where CNI compliant executables are located. 34 | -------------------------------------------------------------------------------- /deployment/rdma-cni-daemonset.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: DaemonSet 4 | metadata: 5 | name: kube-rdma-cni-ds 6 | namespace: kube-system 7 | labels: 8 | tier: node 9 | app: rdma-cni 10 | name: rdma-cni 11 | spec: 12 | selector: 13 | matchLabels: 14 | name: rdma-cni 15 | updateStrategy: 16 | type: RollingUpdate 17 | template: 18 | metadata: 19 | labels: 20 | tier: node 21 | app: rdma-cni 22 | name: rdma-cni 23 | spec: 24 | hostNetwork: true 25 | tolerations: 26 | - operator: Exists 27 | effect: NoSchedule 28 | containers: 29 | - name: rdma-cni 30 | image: ghcr.io/k8snetworkplumbingwg/rdma-cni 31 | imagePullPolicy: IfNotPresent 32 | securityContext: 33 | privileged: true 34 | resources: 35 | requests: 36 | cpu: "100m" 37 | memory: "50Mi" 38 | limits: 39 | cpu: "100m" 40 | memory: "50Mi" 41 | volumeMounts: 42 | - name: cnibin 43 | mountPath: /host/opt/cni/bin 44 | volumes: 45 | - name: cnibin 46 | hostPath: 47 | path: /opt/cni/bin 48 | -------------------------------------------------------------------------------- /pkg/types/types.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | import ( 4 | "github.com/containernetworking/cni/pkg/types" 5 | ) 6 | 7 | type RdmaNetConf struct { 8 | types.NetConf 9 | DeviceID string `json:"deviceID"` // PCI address of a VF in valid sysfs format 10 | Args CNIArgs `json:"args"` // optional arguments passed to CNI as defined in CNI spec 0.2.0 11 | } 12 | 13 | type CNIArgs struct { 14 | CNI RdmaCNIArgs `json:"cni"` 15 | } 16 | 17 | type RdmaCNIArgs struct { 18 | types.CommonArgs 19 | Debug bool `json:"debug"` // Run CNI in debug mode 20 | } 21 | 22 | // RDMA Network state struct version 23 | // minor should be bumped when new fields are added 24 | // major should be bumped when non backward compatible changes are introduced 25 | const RdmaNetStateVersion = "1.0" 26 | 27 | func NewRdmaNetState() RdmaNetState { 28 | return RdmaNetState{Version: RdmaNetStateVersion} 29 | } 30 | 31 | type RdmaNetState struct { 32 | // RDMA network state struct version 33 | Version string `json:"version"` 34 | // PCI device ID associated with the RDMA device 35 | DeviceID string `json:"deviceID"` 36 | // RDMA device name as originally appeared in sandbox 37 | SandboxRdmaDevName string `json:"sandboxRdmaDevName"` 38 | // RDMA device name in container 39 | ContainerRdmaDevName string `json:"containerRdmaDevName"` 40 | } 41 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/k8snetworkplumbingwg/rdma-cni 2 | 3 | go 1.24.2 4 | 5 | require ( 6 | github.com/Mellanox/rdmamap v1.1.0 7 | github.com/containernetworking/cni v1.3.0 8 | github.com/containernetworking/plugins v1.9.0 9 | github.com/onsi/ginkgo/v2 v2.27.2 10 | github.com/onsi/gomega v1.38.3 11 | github.com/rs/zerolog v1.34.0 12 | github.com/spf13/afero v1.15.0 13 | github.com/stretchr/testify v1.11.1 14 | github.com/vishvananda/netlink v1.3.1 15 | ) 16 | 17 | require ( 18 | github.com/Masterminds/semver/v3 v3.4.0 // indirect 19 | github.com/davecgh/go-spew v1.1.1 // indirect 20 | github.com/go-logr/logr v1.4.3 // indirect 21 | github.com/go-task/slim-sprig/v3 v3.0.0 // indirect 22 | github.com/google/go-cmp v0.7.0 // indirect 23 | github.com/google/pprof v0.0.0-20250820193118-f64d9cf942d6 // indirect 24 | github.com/mattn/go-colorable v0.1.13 // indirect 25 | github.com/mattn/go-isatty v0.0.19 // indirect 26 | github.com/pmezard/go-difflib v1.0.0 // indirect 27 | github.com/stretchr/objx v0.5.2 // indirect 28 | github.com/vishvananda/netns v0.0.5 // indirect 29 | go.yaml.in/yaml/v3 v3.0.4 // indirect 30 | golang.org/x/mod v0.27.0 // indirect 31 | golang.org/x/net v0.43.0 // indirect 32 | golang.org/x/sync v0.16.0 // indirect 33 | golang.org/x/sys v0.35.0 // indirect 34 | golang.org/x/text v0.28.0 // indirect 35 | golang.org/x/tools v0.36.0 // indirect 36 | gopkg.in/yaml.v3 v3.0.1 // indirect 37 | ) 38 | -------------------------------------------------------------------------------- /.github/workflows/image-push-master.yaml: -------------------------------------------------------------------------------- 1 | name: "Push RDMA CNI images on merge to master" 2 | 3 | env: 4 | IMAGE_NAME: ghcr.io/${{ github.repository }} 5 | BUILD_PLATFORMS: linux/amd64,linux/arm64,linux/ppc64le,linux/s390x 6 | 7 | on: 8 | push: 9 | branches: 10 | - master 11 | 12 | jobs: 13 | build-and-push-rdma-cni: 14 | name: Image push 15 | runs-on: ubuntu-24.04 16 | 17 | steps: 18 | - name: check out the repo 19 | uses: actions/checkout@v6 20 | 21 | - name: set up QEMU 22 | uses: docker/setup-qemu-action@v3 23 | 24 | - name: set up Docker Buildx 25 | uses: docker/setup-buildx-action@v3 26 | 27 | - name: login to Docker 28 | uses: docker/login-action@v3 29 | with: 30 | registry: ghcr.io 31 | username: ${{ github.repository_owner }} 32 | password: ${{ secrets.GITHUB_TOKEN }} 33 | 34 | - name: Docker metadata 35 | id: docker_meta 36 | uses: docker/metadata-action@v5 37 | with: 38 | images: ${{ env.IMAGE_NAME }} 39 | 40 | - name: Build and push RDMA CNI multi-arch image 41 | uses: docker/build-push-action@v6 42 | with: 43 | context: . 44 | push: true 45 | platforms: ${{ env.BUILD_PLATFORMS }} 46 | tags: | 47 | ${{ env.IMAGE_NAME }}:latest 48 | ${{ env.IMAGE_NAME }}:${{ github.sha }} 49 | labels: ${{ steps.docker_meta.outputs.labels }} 50 | file: ./Dockerfile 51 | 52 | -------------------------------------------------------------------------------- /.github/workflows/image-push-release.yaml: -------------------------------------------------------------------------------- 1 | name: "Push RDMA CNI images on release" 2 | 3 | env: 4 | IMAGE_NAME: ghcr.io/${{ github.repository }} 5 | BUILD_PLATFORMS: linux/amd64,linux/arm64,linux/ppc64le,linux/s390x 6 | 7 | on: 8 | push: 9 | tags: 10 | - v* 11 | 12 | jobs: 13 | build-and-push-rdma-cni: 14 | name: Image push (release) 15 | runs-on: ubuntu-24.04 16 | 17 | steps: 18 | - name: check out the repo 19 | uses: actions/checkout@v6 20 | 21 | - name: Set up QEMU 22 | uses: docker/setup-qemu-action@v3 23 | 24 | - name: set up Docker Buildx 25 | uses: docker/setup-buildx-action@v3 26 | 27 | - name: login to Docker 28 | uses: docker/login-action@v3 29 | with: 30 | registry: ghcr.io 31 | username: ${{ github.repository_owner }} 32 | password: ${{ secrets.GITHUB_TOKEN }} 33 | 34 | - name: Docker metadata 35 | id: docker_meta 36 | uses: docker/metadata-action@v5 37 | with: 38 | images: ${{ env.IMAGE_NAME }} 39 | flavor: | 40 | latest=false 41 | tags: | 42 | type=ref,event=tag 43 | 44 | - name: Build and push RDMA CNI multi-arch image 45 | uses: docker/build-push-action@v6 46 | with: 47 | context: . 48 | push: true 49 | platforms: ${{ env.BUILD_PLATFORMS }} 50 | tags: ${{ steps.docker_meta.outputs.tags }} 51 | labels: ${{ steps.docker_meta.outputs.labels }} 52 | file: ./Dockerfile 53 | -------------------------------------------------------------------------------- /pkg/utils/utils.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "fmt" 5 | "path" 6 | "path/filepath" 7 | "regexp" 8 | 9 | "github.com/vishvananda/netlink" 10 | ) 11 | 12 | // Get VF PCI device associated with the given MAC. 13 | // this method compares with administrative MAC for SRIOV configured net devices 14 | // TODO: move this method to github: Mellanox/sriovnet 15 | func GetVfPciDevFromMAC(mac string) (string, error) { 16 | var err error 17 | var links []netlink.Link 18 | var vfPath string 19 | links, err = netlink.LinkList() 20 | if err != nil { 21 | return "", err 22 | } 23 | matchDevs := []string{} 24 | for _, link := range links { 25 | if len(link.Attrs().Vfs) > 0 { 26 | for i := range link.Attrs().Vfs { 27 | if link.Attrs().Vfs[i].Mac.String() == mac { 28 | vfPath, err = filepath.EvalSymlinks( 29 | fmt.Sprintf("/sys/class/net/%s/device/virtfn%d", link.Attrs().Name, link.Attrs().Vfs[i].ID)) 30 | if err == nil { 31 | matchDevs = append(matchDevs, path.Base(vfPath)) 32 | } 33 | } 34 | } 35 | } 36 | } 37 | 38 | var dev string 39 | switch len(matchDevs) { 40 | case 1: 41 | dev = matchDevs[0] 42 | err = nil 43 | case 0: 44 | err = fmt.Errorf("could not find VF PCI device according to administrative mac address set on PF") 45 | default: 46 | err = fmt.Errorf("found more than one VF PCI device matching provided administrative mac address") 47 | } 48 | return dev, err 49 | } 50 | 51 | // IsPCIAddress returns whether the input is a valid PCI address. 52 | func IsPCIAddress(pciAddress string) bool { 53 | re := regexp.MustCompile(`^[0-9a-fA-F]{4}:[0-9a-fA-F]{2}:[0-9a-fA-F]{2}\.[0-7]$`) 54 | return re.MatchString(pciAddress) 55 | } 56 | -------------------------------------------------------------------------------- /.github/workflows/buildtest.yaml: -------------------------------------------------------------------------------- 1 | name: go-build-and-test-amd64 2 | on: 3 | push: 4 | pull_request: 5 | schedule: 6 | - cron: "0 8 * * 0" # every sunday 7 | jobs: 8 | build: 9 | name: build 10 | strategy: 11 | matrix: 12 | go-version: [1.24.x] 13 | os: [ubuntu-22.04] 14 | goos: [linux] 15 | goarch: [amd64] 16 | runs-on: ${{ matrix.os }} 17 | steps: 18 | - name: set up Go 19 | uses: actions/setup-go@v6 20 | with: 21 | go-version: ${{ matrix.go-version }} 22 | - name: check out code into the Go module directory 23 | uses: actions/checkout@v6 24 | - name: build test for ${{ matrix.goarch }} 25 | env: 26 | GOARCH: ${{ matrix.goarch }} 27 | GOOS: ${{ matrix.goos }} 28 | run: make build 29 | 30 | test: 31 | name: test 32 | runs-on: ubuntu-24.04 33 | needs: build 34 | steps: 35 | - name: set up Go 36 | uses: actions/setup-go@v6 37 | with: 38 | go-version: 1.24.x 39 | - name: check out code into the Go module directory 40 | uses: actions/checkout@v6 41 | - name: run unit-test 42 | run: make test 43 | 44 | coverage: 45 | runs-on: ubuntu-24.04 46 | needs: build 47 | name: coverage 48 | steps: 49 | - name: Set up Go 50 | uses: actions/setup-go@v6 51 | with: 52 | go-version: 1.24.x 53 | - name: Check out code into the Go module directory 54 | uses: actions/checkout@v6 55 | - name: Go test with coverage 56 | run: make test-coverage # sudo needed for netns change in test 57 | - name: Coveralls 58 | uses: coverallsapp/github-action@v2 59 | with: 60 | github-token: ${{ secrets.GITHUB_TOKEN }} 61 | file: rdma-cni.cover -------------------------------------------------------------------------------- /images/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Always exit on errors. 4 | set -e 5 | 6 | # Set known directories. 7 | CNI_BIN_DIR="/host/opt/cni/bin" 8 | RDMA_CNI_BIN_FILE="/usr/bin/rdma" 9 | NO_SLEEP=0 10 | 11 | # Give help text for parameters. 12 | usage() 13 | { 14 | printf "This is an entrypoint script for RDMA CNI to overlay its\n" 15 | printf "binary into location in a filesystem. The binary file will\n" 16 | printf "be copied to the corresponding directory.\n" 17 | printf "\n" 18 | printf "./entrypoint.sh\n" 19 | printf "\t-h --help\n" 20 | printf "\t--cni-bin-dir=%s\n" "$CNI_BIN_DIR" 21 | printf "\t--rdma-cni-bin-file=%s\n" "$RDMA_CNI_BIN_FILE" 22 | printf "\t--no-sleep\n" 23 | } 24 | 25 | # Parse parameters given as arguments to this script. 26 | while [ "$1" != "" ]; do 27 | PARAM=$(echo "$1" | awk -F= '{print $1}') 28 | VALUE=$(echo "$1" | awk -F= '{print $2}') 29 | case $PARAM in 30 | -h | --help) 31 | usage 32 | exit 33 | ;; 34 | --cni-bin-dir) 35 | CNI_BIN_DIR=$VALUE 36 | ;; 37 | --rdma-cni-bin-file) 38 | RDMA_CNI_BIN_FILE=$VALUE 39 | ;; 40 | --no-sleep) 41 | NO_SLEEP=1 42 | ;; 43 | *) 44 | /bin/echo "ERROR: unknown parameter \"$PARAM\"" 45 | usage 46 | exit 1 47 | ;; 48 | esac 49 | shift 50 | done 51 | 52 | 53 | # Loop through and verify each location each. 54 | for i in $CNI_BIN_DIR $RDMA_CNI_BIN_FILE 55 | do 56 | if [ ! -e "$i" ]; then 57 | /bin/echo "Location $i does not exist" 58 | exit 1; 59 | fi 60 | done 61 | 62 | # Copy file into proper place. 63 | cp -f "$RDMA_CNI_BIN_FILE" "$CNI_BIN_DIR" 64 | 65 | if [ $NO_SLEEP -eq 1 ]; then 66 | exit 0 67 | fi 68 | 69 | echo "Entering sleep... (success)" 70 | trap : TERM INT 71 | 72 | # Sleep forever. 73 | # sleep infinity is not available in alpine; instead lets go sleep for ~68 years. Hopefully that's enough sleep 74 | sleep 2147483647 & wait 75 | -------------------------------------------------------------------------------- /pkg/rdma/rdma_ops.go: -------------------------------------------------------------------------------- 1 | package rdma 2 | 3 | import ( 4 | "github.com/Mellanox/rdmamap" 5 | "github.com/vishvananda/netlink" 6 | ) 7 | 8 | // Interface to be used by RDMA manager for basic operations 9 | type BasicOps interface { 10 | // Equivalent to netlink.RdmaLinkByName(...) 11 | RdmaLinkByName(name string) (*netlink.RdmaLink, error) 12 | // Equivalent to netlink.RdmaLinkSetNsFd(...) 13 | RdmaLinkSetNsFd(link *netlink.RdmaLink, fd uint32) error 14 | // Equivalent to netlink.RdmaSystemGetNetnsMode(...) 15 | RdmaSystemGetNetnsMode() (string, error) 16 | // Equivalent to netlink.RdmaSystemSetNetnsMode(...) 17 | RdmaSystemSetNetnsMode(newMode string) error 18 | // Equivalent to rdmamap.GetRdmaDevicesForPcidev(...) 19 | GetRdmaDevicesForPcidev(pcidevName string) []string 20 | // Equivalent to rdmamap.GetRdmaDevicesForAuxdev(...) 21 | GetRdmaDevicesForAuxdev(auxDev string) []string 22 | } 23 | 24 | func newRdmaBasicOps() BasicOps { 25 | return &rdmaBasicOpsImpl{} 26 | } 27 | 28 | type rdmaBasicOpsImpl struct { 29 | } 30 | 31 | // Equivalent to netlink.RdmaLinkByName(...) 32 | func (rdma *rdmaBasicOpsImpl) RdmaLinkByName(name string) (*netlink.RdmaLink, error) { 33 | return netlink.RdmaLinkByName(name) 34 | } 35 | 36 | // Equivalent to netlink.RdmaLinkSetNsFd(...) 37 | func (rdma *rdmaBasicOpsImpl) RdmaLinkSetNsFd(link *netlink.RdmaLink, fd uint32) error { 38 | return netlink.RdmaLinkSetNsFd(link, fd) 39 | } 40 | 41 | // Equivalent to netlink.RdmaSystemGetNetnsMode(...) 42 | func (rdma *rdmaBasicOpsImpl) RdmaSystemGetNetnsMode() (string, error) { 43 | return netlink.RdmaSystemGetNetnsMode() 44 | } 45 | 46 | // Equivalent to netlink.RdmaSystemSetNetnsMode(...) 47 | func (rdma *rdmaBasicOpsImpl) RdmaSystemSetNetnsMode(newMode string) error { 48 | return netlink.RdmaSystemSetNetnsMode(newMode) 49 | } 50 | 51 | // Equivalent to rdmamap.GetRdmaDevicesForPcidev(...) 52 | func (rdma *rdmaBasicOpsImpl) GetRdmaDevicesForPcidev(pcidevName string) []string { 53 | return rdmamap.GetRdmaDevicesForPcidev(pcidevName) 54 | } 55 | 56 | // Equivalent to rdmamap.GetRdmaDevicesForAuxdev(...) 57 | func (rdma *rdmaBasicOpsImpl) GetRdmaDevicesForAuxdev(auxDev string) []string { 58 | return rdmamap.GetRdmaDevicesForAuxdev(auxDev) 59 | } 60 | -------------------------------------------------------------------------------- /pkg/cache/fs_ops.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "os" 5 | 6 | "github.com/spf13/afero" 7 | ) 8 | 9 | func newFsOps() FileSystemOps { 10 | return &stdFileSystemOps{} 11 | } 12 | 13 | // interface consolidating all file system operations 14 | type FileSystemOps interface { 15 | // Eqivalent to os.ReadFile(...) 16 | ReadFile(filename string) ([]byte, error) 17 | // Eqivalent to os.WriteFile(...) 18 | WriteFile(filename string, data []byte, perm os.FileMode) error 19 | // Eqivalent to os.MkdirAll(...) 20 | MkdirAll(path string, perm os.FileMode) error 21 | // Equivalent to os.Remove(...) 22 | Remove(name string) error 23 | // Equvalent to os.Stat(...) 24 | Stat(name string) (os.FileInfo, error) 25 | } 26 | 27 | type stdFileSystemOps struct{} 28 | 29 | func (sfs *stdFileSystemOps) ReadFile(filename string) ([]byte, error) { 30 | return os.ReadFile(filename) 31 | } 32 | 33 | func (sfs *stdFileSystemOps) WriteFile(filename string, data []byte, perm os.FileMode) error { 34 | return os.WriteFile(filename, data, perm) 35 | } 36 | 37 | func (sfs *stdFileSystemOps) MkdirAll(path string, perm os.FileMode) error { 38 | return os.MkdirAll(path, perm) 39 | } 40 | 41 | func (sfs *stdFileSystemOps) Remove(name string) error { 42 | return os.Remove(name) 43 | } 44 | 45 | func (sfs *stdFileSystemOps) Stat(name string) (os.FileInfo, error) { 46 | return os.Stat(name) 47 | } 48 | 49 | // Fake fileSystemOps used for Unit testing 50 | func newFakeFileSystemOps() FileSystemOps { 51 | return &fakeFileSystemOps{fakefs: afero.Afero{Fs: afero.NewMemMapFs()}} 52 | } 53 | 54 | type fakeFileSystemOps struct { 55 | fakefs afero.Afero 56 | } 57 | 58 | func (ffs *fakeFileSystemOps) ReadFile(filename string) ([]byte, error) { 59 | return ffs.fakefs.ReadFile(filename) 60 | } 61 | 62 | func (ffs *fakeFileSystemOps) WriteFile(filename string, data []byte, perm os.FileMode) error { 63 | return ffs.fakefs.WriteFile(filename, data, perm) 64 | } 65 | 66 | func (ffs *fakeFileSystemOps) MkdirAll(path string, perm os.FileMode) error { 67 | return ffs.fakefs.MkdirAll(path, perm) 68 | } 69 | 70 | func (ffs *fakeFileSystemOps) Remove(name string) error { 71 | return ffs.fakefs.Remove(name) 72 | } 73 | 74 | func (ffs *fakeFileSystemOps) Stat(name string) (os.FileInfo, error) { 75 | return ffs.fakefs.Stat(name) 76 | } 77 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | linters-settings: 2 | revive: 3 | rules: 4 | - name: dot-imports 5 | arguments: 6 | - allowedPackages: 7 | - "github.com/onsi/ginkgo/v2" 8 | - "github.com/onsi/gomega" 9 | dupl: 10 | threshold: 150 11 | funlen: 12 | lines: 100 13 | statements: 50 14 | goconst: 15 | min-len: 2 16 | min-occurrences: 2 17 | gocognit: 18 | min-complexity: 30 19 | goimports: 20 | local-prefixes: github.com/k8snetworkplumbingwg/rdma-cni 21 | mnd: 22 | # don't include the "operation" and "assign" 23 | checks: [argument,case,condition,return] 24 | govet: 25 | settings: 26 | printf: 27 | funcs: 28 | - (github.com/rs/zerolog/zerolog.Event).Msgf 29 | lll: 30 | line-length: 120 31 | misspell: 32 | locale: US 33 | prealloc: 34 | # Report preallocation suggestions only on simple loops that have no returns/breaks/continues/gotos in them. 35 | # True by default. 36 | simple: true 37 | range-loops: true # Report preallocation suggestions on range loops, true by default 38 | for-loops: false # Report preallocation suggestions on for loops, false by default 39 | 40 | linters: 41 | # please, do not use `enable-all`: it's deprecated and will be removed soon. 42 | # inverted configuration with `enable-all` and `disable` is not scalable during updates of golangci-lint 43 | disable-all: true 44 | enable: 45 | - bodyclose 46 | - dogsled 47 | - dupl 48 | - errcheck 49 | - funlen 50 | - gochecknoinits 51 | - goconst 52 | - gocritic 53 | - gocognit 54 | - gofmt 55 | - goimports 56 | - mnd 57 | - goprintffuncname 58 | - gosec 59 | - gosimple 60 | - govet 61 | - ineffassign 62 | - lll 63 | - misspell 64 | - nakedret 65 | - prealloc 66 | - revive 67 | - rowserrcheck 68 | - staticcheck 69 | - stylecheck 70 | - typecheck 71 | - unconvert 72 | - unparam 73 | - unused 74 | - whitespace 75 | 76 | issues: 77 | # Excluding configuration per-path, per-linter, per-text and per-source 78 | exclude-rules: 79 | - path: _test\.go 80 | linters: 81 | - mnd 82 | - goconst 83 | - dupl 84 | - text: "Magic number: 1" 85 | linters: 86 | - mnd 87 | -------------------------------------------------------------------------------- /pkg/cache/cache.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "path/filepath" 7 | "strings" 8 | ) 9 | 10 | const ( 11 | dirPerms = 0o700 12 | filePerms = 0o600 13 | ) 14 | 15 | var ( 16 | // CacheDir is used By default for caching CNI network state 17 | CacheDir = "/var/lib/cni/rdma" 18 | ) 19 | 20 | type StateRef string 21 | 22 | type StateCache interface { 23 | // Get State reference identifier for 24 | GetStateRef(network string, cid string, ifname string) StateRef 25 | // Save state to cache 26 | Save(ref StateRef, state interface{}) error 27 | // Load state from cache 28 | Load(ref StateRef, state interface{}) error 29 | // Delete state from cache 30 | Delete(ref StateRef) error 31 | } 32 | 33 | // Create a new RDMA state Cache that will Save/Load state 34 | func NewStateCache() StateCache { 35 | return &FsStateCache{basePath: CacheDir, fsOps: newFsOps()} 36 | } 37 | 38 | type FsStateCache struct { 39 | basePath string 40 | fsOps FileSystemOps 41 | } 42 | 43 | func (sc *FsStateCache) GetStateRef(network, cid, ifname string) StateRef { 44 | return StateRef(strings.Join([]string{network, cid, ifname}, "-")) 45 | } 46 | 47 | func (sc *FsStateCache) Save(ref StateRef, state interface{}) error { 48 | sRef := string(ref) 49 | bytes, err := json.Marshal(state) 50 | if err != nil { 51 | return err 52 | } 53 | 54 | //nolint:gomnd 55 | if err = sc.fsOps.MkdirAll(sc.basePath, dirPerms); err != nil { 56 | return fmt.Errorf("failed to create data cache directory(%q): %v", sc.basePath, err) 57 | } 58 | 59 | path := filepath.Join(sc.basePath, sRef) 60 | 61 | //nolint:gomnd 62 | err = sc.fsOps.WriteFile(path, bytes, filePerms) 63 | if err != nil { 64 | return fmt.Errorf("failed to write cache data in the path(%q): %v", path, err) 65 | } 66 | 67 | return err 68 | } 69 | 70 | func (sc *FsStateCache) Load(ref StateRef, state interface{}) error { 71 | sRef := string(ref) 72 | path := filepath.Join(sc.basePath, sRef) 73 | bytes, err := sc.fsOps.ReadFile(path) 74 | if err != nil { 75 | return fmt.Errorf("failed to read cache data in the path(%q): %v", path, err) 76 | } 77 | return json.Unmarshal(bytes, state) 78 | } 79 | 80 | func (sc *FsStateCache) Delete(ref StateRef) error { 81 | sRef := string(ref) 82 | path := filepath.Join(sc.basePath, sRef) 83 | if err := sc.fsOps.Remove(path); err != nil { 84 | return fmt.Errorf("error removing cache file %q: %v", path, err) 85 | } 86 | return nil 87 | } 88 | -------------------------------------------------------------------------------- /pkg/rdma/rdma.go: -------------------------------------------------------------------------------- 1 | package rdma 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/containernetworking/plugins/pkg/ns" 7 | ) 8 | 9 | const ( 10 | RdmaSysModeExclusive = "exclusive" 11 | RdmaSysModeShared = "shared" 12 | ) 13 | 14 | func NewRdmaManager() Manager { 15 | return &rdmaManagerNetlink{rdmaOps: newRdmaBasicOps()} 16 | } 17 | 18 | type Manager interface { 19 | // Move RDMA device from current network namespace to network namespace 20 | MoveRdmaDevToNs(rdmaDev string, netNs ns.NetNS) error 21 | // Get RDMA devices associated with the given PCI device in D:B:D.f format e.g 0000:04:00.0 22 | GetRdmaDevsForPciDev(pciDev string) []string 23 | // Get RDMA devices associated with the given auxiliary device. For example, for input mlx5_core.sf.4, returns 24 | // [mlx5_0,mlx5_10,..] 25 | GetRdmaDevsForAuxDev(auxDev string) []string 26 | // Get RDMA subsystem namespace awareness mode ["exclusive" | "shared"] 27 | GetSystemRdmaMode() (string, error) 28 | // Set RDMA subsystem namespace awareness mode ["exclusive" | "shared"] 29 | SetSystemRdmaMode(mode string) error 30 | } 31 | 32 | type rdmaManagerNetlink struct { 33 | rdmaOps BasicOps 34 | } 35 | 36 | // Move RDMA device to network namespace 37 | func (rmn *rdmaManagerNetlink) MoveRdmaDevToNs(rdmaDev string, netNs ns.NetNS) error { 38 | rdmaLink, err := rmn.rdmaOps.RdmaLinkByName(rdmaDev) 39 | if err != nil { 40 | return fmt.Errorf("cannot find RDMA link from name: %s", rdmaDev) 41 | } 42 | err = rmn.rdmaOps.RdmaLinkSetNsFd(rdmaLink, uint32(netNs.Fd())) 43 | if err != nil { 44 | return fmt.Errorf("failed to move RDMA dev %s to namespace. %v", rdmaDev, err) 45 | } 46 | return nil 47 | } 48 | 49 | // Get RDMA device associated with the given PCI device in D:B:D.f format e.g 0000:04:00.1 50 | func (rmn *rdmaManagerNetlink) GetRdmaDevsForPciDev(pciDev string) []string { 51 | return rmn.rdmaOps.GetRdmaDevicesForPcidev(pciDev) 52 | } 53 | 54 | // Get RDMA devices associated with the given auxiliary device. For example, for input mlx5_core.sf.4, returns 55 | // [mlx5_0,mlx5_10,..] 56 | func (rmn *rdmaManagerNetlink) GetRdmaDevsForAuxDev(auxDev string) []string { 57 | return rmn.rdmaOps.GetRdmaDevicesForAuxdev(auxDev) 58 | } 59 | 60 | // Get RDMA subsystem namespace awareness mode ["exclusive" | "shared"] 61 | func (rmn *rdmaManagerNetlink) GetSystemRdmaMode() (string, error) { 62 | return rmn.rdmaOps.RdmaSystemGetNetnsMode() 63 | } 64 | 65 | // Set RDMA subsystem namespace awareness mode ["exclusive" | "shared"] 66 | func (rmn *rdmaManagerNetlink) SetSystemRdmaMode(mode string) error { 67 | return rmn.rdmaOps.RdmaSystemSetNetnsMode(mode) 68 | } 69 | -------------------------------------------------------------------------------- /pkg/cache/cache_test.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "path" 5 | 6 | . "github.com/onsi/ginkgo/v2" 7 | . "github.com/onsi/gomega" 8 | ) 9 | 10 | type myTestState struct { 11 | FirstState string `json:"firstState"` 12 | SecondState int `json:"secondState"` 13 | } 14 | 15 | var _ = Describe("Cache - Simple marshall-able state-object cache", func() { 16 | var stateCache StateCache 17 | var fs FileSystemOps 18 | JustBeforeEach(func() { 19 | fs = newFakeFileSystemOps() 20 | stateCache = &FsStateCache{basePath: CacheDir, fsOps: fs} 21 | }) 22 | 23 | Describe("Get State reference", func() { 24 | Context("Basic call", func() { 25 | It("Should return --", func() { 26 | Expect(stateCache.GetStateRef("myNet", "containerUniqueIdentifier", "net1")).To( 27 | BeEquivalentTo("myNet-containerUniqueIdentifier-net1")) 28 | }) 29 | }) 30 | }) 31 | 32 | Describe("Save and Load State", func() { 33 | var sRef StateRef 34 | JustBeforeEach(func() { 35 | sRef = stateCache.GetStateRef("mynet", "cid", "net1") 36 | }) 37 | 38 | Context("Save and Load with marshallable object", func() { 39 | It("Should save/load the state", func() { 40 | savedState := myTestState{FirstState: "first", SecondState: 42} 41 | var loadedState myTestState 42 | Expect(stateCache.Save(sRef, &savedState)).Should(Succeed()) 43 | _, err := fs.Stat(path.Join(CacheDir, string(sRef))) 44 | Expect(err).ToNot(HaveOccurred()) 45 | Expect(stateCache.Load(sRef, &loadedState)).Should(Succeed()) 46 | Expect(loadedState).Should(Equal(savedState)) 47 | }) 48 | }) 49 | Context("Load non-existent state", func() { 50 | It("Should fail", func() { 51 | var loadedState myTestState 52 | Expect(stateCache.Load(sRef, &loadedState)).ShouldNot(Succeed()) 53 | }) 54 | }) 55 | }) 56 | 57 | Describe("Delete State", func() { 58 | var sRef StateRef 59 | JustBeforeEach(func() { 60 | sRef = stateCache.GetStateRef("mynet", "cid", "net1") 61 | }) 62 | 63 | Context("Delete a saved state", func() { 64 | It("Should not exist after delete", func() { 65 | savedState := myTestState{FirstState: "first", SecondState: 42} 66 | Expect(stateCache.Save(sRef, &savedState)).Should(Succeed()) 67 | _, err := fs.Stat(path.Join(CacheDir, string(sRef))) 68 | Expect(err).ToNot(HaveOccurred()) 69 | Expect(stateCache.Delete(sRef)).Should(Succeed()) 70 | _, err = fs.Stat(path.Join(CacheDir, string(sRef))) 71 | Expect(err).To(HaveOccurred()) 72 | }) 73 | }) 74 | Context("Delete a non existent state", func() { 75 | It("Should Fail", func() { 76 | altRef := stateCache.GetStateRef("alt-mynet", "cid", "net1") 77 | Expect(stateCache.Delete(altRef)).To(HaveOccurred()) 78 | _, err := fs.Stat(path.Join(CacheDir, string(altRef))) 79 | Expect(err).To(HaveOccurred()) 80 | }) 81 | }) 82 | }) 83 | }) 84 | -------------------------------------------------------------------------------- /pkg/rdma/rdma_test.go: -------------------------------------------------------------------------------- 1 | package rdma 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/containernetworking/plugins/pkg/ns" 7 | . "github.com/onsi/ginkgo/v2" 8 | . "github.com/onsi/gomega" 9 | "github.com/stretchr/testify/mock" 10 | "github.com/vishvananda/netlink" 11 | 12 | "github.com/k8snetworkplumbingwg/rdma-cni/pkg/rdma/mocks" 13 | ) 14 | 15 | type dummyNetNs struct { 16 | ns.NetNS 17 | fd uintptr 18 | } 19 | 20 | func (dns *dummyNetNs) Fd() uintptr { 21 | return dns.fd 22 | } 23 | 24 | var _ = Describe("Rdma Manager", func() { 25 | var ( 26 | rdmaManager Manager 27 | rdmaOpsMock mocks.MockBasicOps 28 | t GinkgoTInterface 29 | ) 30 | 31 | JustBeforeEach(func() { 32 | rdmaOpsMock = mocks.MockBasicOps{} 33 | rdmaManager = &rdmaManagerNetlink{rdmaOps: &rdmaOpsMock} 34 | t = GinkgoT() 35 | }) 36 | 37 | Describe("Test GetRdmaDevsForPciDev()", func() { 38 | Context("Basic Call, no failure from RdmaBasicOps", func() { 39 | It("Should pass and return value as provided by rdmaBasicOps", func() { 40 | retVal := []string{"mlx5_3"} 41 | rdmaOpsMock.On("GetRdmaDevicesForPcidev", mock.AnythingOfType("string")).Return(retVal, nil) 42 | ret := rdmaManager.GetRdmaDevsForPciDev("04:00.1") 43 | rdmaOpsMock.AssertExpectations(t) 44 | Expect(ret).To(Equal(retVal)) 45 | }) 46 | }) 47 | Context("no RDMA devices for PCI dev returned from RdmaBasicOps", func() { 48 | It("Should return the same", func() { 49 | retVal := []string{} 50 | rdmaOpsMock.On("GetRdmaDevicesForPcidev", mock.AnythingOfType("string")).Return(retVal, nil) 51 | ret := rdmaManager.GetRdmaDevsForPciDev("04:00.1") 52 | rdmaOpsMock.AssertExpectations(t) 53 | Expect(ret).To(Equal(retVal)) 54 | }) 55 | }) 56 | }) 57 | 58 | Describe("Test GetSystemRdmaMode()", func() { 59 | Context("Basic Call - no error", func() { 60 | It("Is a Proxy for RdmaBasicOps.GetSystemRdmaMode", func() { 61 | retVal := RdmaSysModeExclusive 62 | rdmaOpsMock.On("RdmaSystemGetNetnsMode").Return(retVal, nil) 63 | ret, err := rdmaManager.GetSystemRdmaMode() 64 | rdmaOpsMock.AssertExpectations(t) 65 | Expect(err).ToNot(HaveOccurred()) 66 | Expect(ret).To(Equal(retVal)) 67 | }) 68 | }) 69 | Context("Basic Call - with error", func() { 70 | It("Is a Proxy for RdmaBasicOps.GetSystemRdmaMode", func() { 71 | retErr := fmt.Errorf("some Error") 72 | retVal := "dummy" 73 | rdmaOpsMock.On("RdmaSystemGetNetnsMode").Return(retVal, retErr) 74 | ret, err := rdmaManager.GetSystemRdmaMode() 75 | rdmaOpsMock.AssertExpectations(t) 76 | Expect(err).To(Equal(retErr)) 77 | Expect(ret).To(Equal(retVal)) 78 | }) 79 | }) 80 | }) 81 | 82 | Describe("Test SetSystemRdmaMode()", func() { 83 | Context("Basic Call - no error", func() { 84 | It("Is a Proxy for RdmaBasicOps.SetSystemRdmaMode", func() { 85 | mode := RdmaSysModeExclusive 86 | rdmaOpsMock.On("RdmaSystemSetNetnsMode", mode).Return(nil) 87 | err := rdmaManager.SetSystemRdmaMode(mode) 88 | rdmaOpsMock.AssertExpectations(t) 89 | Expect(err).ToNot(HaveOccurred()) 90 | }) 91 | }) 92 | Context("Basic Call - with error", func() { 93 | It("Is a Proxy for RdmaBasicOps.SetSystemRdmaMode", func() { 94 | retErr := fmt.Errorf("some Error") 95 | mode := "dummy" 96 | rdmaOpsMock.On("RdmaSystemSetNetnsMode", mode).Return(retErr) 97 | err := rdmaManager.SetSystemRdmaMode(mode) 98 | rdmaOpsMock.AssertExpectations(t) 99 | Expect(err).To(Equal(retErr)) 100 | }) 101 | }) 102 | }) 103 | 104 | Describe("Test MoveRdmaDevToNs()", func() { 105 | Context("Basic Call - no error", func() { 106 | It("Calls rdmaOps.RdmaLinkSetNsFd with the correct netNS file desc and the rdma Link index", func() { 107 | link := &netlink.RdmaLink{} 108 | netNs := &dummyNetNs{fd: 17} 109 | rdmaOpsMock.On("RdmaLinkByName", mock.AnythingOfType("string")).Return(link, nil) 110 | rdmaOpsMock.On("RdmaLinkSetNsFd", link, uint32(netNs.Fd())).Return(nil) 111 | err := rdmaManager.MoveRdmaDevToNs("mlx5_9", netNs) 112 | rdmaOpsMock.AssertExpectations(t) 113 | Expect(err).ToNot(HaveOccurred()) 114 | }) 115 | }) 116 | Context("Basic Call - with error", func() { 117 | It("returns error in case rdma link cannot be retrieved", func() { 118 | netNs := &dummyNetNs{fd: 17} 119 | rdmaOpsMock.On("RdmaLinkByName", mock.AnythingOfType("string")).Return(nil, fmt.Errorf("error")) 120 | err := rdmaManager.MoveRdmaDevToNs("mlx5_9", netNs) 121 | rdmaOpsMock.AssertExpectations(t) 122 | Expect(err).To(HaveOccurred()) 123 | }) 124 | It("returns error in case rdma link fails to move to namespace", func() { 125 | link := &netlink.RdmaLink{} 126 | netNs := &dummyNetNs{fd: 17} 127 | rdmaOpsMock.On("RdmaLinkByName", mock.AnythingOfType("string")).Return(link, nil) 128 | rdmaOpsMock.On("RdmaLinkSetNsFd", link, uint32(netNs.Fd())).Return(fmt.Errorf("error")) 129 | err := rdmaManager.MoveRdmaDevToNs("mlx5_9", netNs) 130 | rdmaOpsMock.AssertExpectations(t) 131 | Expect(err).To(HaveOccurred()) 132 | }) 133 | }) 134 | }) 135 | }) 136 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Package related 2 | BINARY_NAME=rdma 3 | PACKAGE=rdma-cni 4 | ORG_PATH=github.com/k8snetworkplumbingwg 5 | REPO_PATH=$(ORG_PATH)/$(PACKAGE) 6 | PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST)))) 7 | BINDIR =$(PROJECT_DIR)/bin 8 | BUILDDIR=$(PROJECT_DIR)/build 9 | GOFILES=$(shell find . -name *.go | grep -vE "(\/vendor\/)|(_test.go)") 10 | PKGS=$(or $(PKG),$(shell cd $(PROJECT_DIR) && go list ./... | grep -v "^$(PACKAGE)/vendor/" | grep -v ".*/mocks")) 11 | TESTPKGS = $(shell go list -f '{{ if or .TestGoFiles .XTestGoFiles }}{{ .ImportPath }}{{ end }}' $(PKGS)) 12 | 13 | # Version 14 | VERSION?=master 15 | DATE=`date -Iseconds` 16 | COMMIT?=`git rev-parse --verify HEAD` 17 | LDFLAGS="-X main.version=$(VERSION) -X main.commit=$(COMMIT) -X main.date=$(DATE)" 18 | 19 | # Docker 20 | IMAGE_BUILDER?=@docker 21 | IMAGEDIR=$(PROJECT_DIR)/images 22 | DOCKERFILE?=$(PROJECT_DIR)/Dockerfile 23 | TAG?=ghcr.io/k8snetworkplumbingwg/rdma-cni 24 | IMAGE_BUILD_OPTS?= 25 | # Accept proxy settings for docker 26 | # To pass proxy for Docker invoke it as 'make image HTTP_POXY=http://192.168.0.1:8080' 27 | DOCKERARGS= 28 | ifdef HTTP_PROXY 29 | DOCKERARGS += --build-arg http_proxy=$(HTTP_PROXY) 30 | endif 31 | ifdef HTTPS_PROXY 32 | DOCKERARGS += --build-arg https_proxy=$(HTTPS_PROXY) 33 | endif 34 | IMAGE_BUILD_OPTS += $(DOCKERARGS) 35 | 36 | # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) 37 | ifeq (,$(shell go env GOBIN)) 38 | GOBIN := $(shell go env GOPATH)/bin 39 | else 40 | GOBIN := $(shell go env GOBIN) 41 | endif 42 | 43 | TARGET_OS ?= $(shell go env GOOS) 44 | TARGET_ARCH ?= $(shell go env GOARCH) 45 | 46 | # Options for go build command 47 | GO_BUILD_OPTS ?= CGO_ENABLED=0 GOOS=$(TARGET_OS) GOARCH=$(TARGET_ARCH) 48 | GO_TAGS ?=-tags no_openssl 49 | 50 | # Go tools 51 | GOLANGCI_LINT = $(BINDIR)/golangci-lint 52 | # golangci-lint version should be updated periodically 53 | # we keep it fixed to avoid it from unexpectedly failing on the project 54 | # in case of a version bump 55 | GOLANGCI_LINT_VER = v1.64.7 56 | MOCKERY_VERSION ?= v3.5.4 57 | TIMEOUT = 30 58 | Q = $(if $(filter 1,$V),,@) 59 | 60 | .PHONY: all 61 | all: generate lint build 62 | 63 | $(BUILDDIR): ; $(info Creating build directory...) 64 | @cd $(PROJECT_DIR) && mkdir -p $@ 65 | 66 | $(BINDIR): ; $(info Creating bin directory...) 67 | @cd $(PROJECT_DIR) && mkdir -p $@ 68 | 69 | build: $(BUILDDIR)/$(BINARY_NAME) ; $(info Building $(BINARY_NAME)...) @ ## Build executable file 70 | $(info Done!) 71 | 72 | $(BUILDDIR)/$(BINARY_NAME): $(GOFILES) | $(BUILDDIR) 73 | @$(GO_BUILD_OPTS) go build -o $(BUILDDIR)/$(BINARY_NAME) $(GO_TAGS) -ldflags $(LDFLAGS) -v cmd/rdma/main.go 74 | 75 | # Tools 76 | $(GOLANGCI_LINT): ; $(info building golangci-lint...) 77 | $Q curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(BINDIR) $(GOLANGCI_LINT_VER) 78 | 79 | MOCKERY= $(BINDIR)/mockery 80 | $(MOCKERY): ; $(info building mockery...) 81 | $Q GOBIN=$(BINDIR) go install github.com/vektra/mockery/v3@$(MOCKERY_VERSION) 82 | 83 | GOVERALLS = $(BINDIR)/goveralls 84 | $(BINDIR)/goveralls: ; $(info building goveralls...) 85 | $Q GOBIN=$(BINDIR) go install github.com/mattn/goveralls@v0.0.12 86 | 87 | # Tests 88 | .PHONY: lint 89 | lint: | $(GOLANGCI_LINT) ; $(info running golangci-lint...) @ ## Run golangci-lint 90 | $Q $(GOLANGCI_LINT) run ./... 91 | 92 | TEST_TARGETS := test-default test-bench test-short test-verbose test-race 93 | .PHONY: $(TEST_TARGETS) test-xml check test tests 94 | test-bench: ARGS=-run=__absolutelynothing__ -bench=. ## Run benchmarks 95 | test-short: ARGS=-short ## Run only short tests 96 | test-verbose: ARGS=-v ## Run tests in verbose mode with coverage reporting 97 | test-race: ARGS=-race ## Run tests with race detector 98 | $(TEST_TARGETS): NAME=$(MAKECMDGOALS:test-%=%) 99 | $(TEST_TARGETS): test 100 | check test tests: ; $(info running $(NAME:%=% )tests...) @ ## Run tests 101 | $Q go test -timeout $(TIMEOUT)s $(ARGS) $(TESTPKGS) 102 | 103 | test-xml: | $(GO2XUNIT) ; $(info running $(NAME:%=% )tests...) @ ## Run tests with xUnit output 104 | $Q 2>&1 go test -timeout $(TIMEOUT)s -v $(TESTPKGS) | tee test/tests.output 105 | $(GO2XUNIT) -fail -input test/tests.output -output test/tests.xml 106 | 107 | COVERAGE_MODE = set 108 | .PHONY: test-coverage test-coverage-tools 109 | test-coverage-tools: | $(GOVERALLS) 110 | test-coverage: COVERAGE_DIR := $(PROJECT_DIR)/test 111 | test-coverage: test-coverage-tools ; $(info running coverage tests...) @ ## Run coverage tests 112 | $Q go test -covermode=$(COVERAGE_MODE) -coverprofile=rdma-cni.cover $(PKGS) 113 | 114 | # Container image 115 | .PHONY: image 116 | image: ; $(info Building Docker image...) @ ## Build conatiner image 117 | $(IMAGE_BUILDER) build -t $(TAG) -f $(DOCKERFILE) $(PROJECT_DIR) $(IMAGE_BUILD_OPTS) 118 | 119 | # Misc 120 | .PHONY: clean 121 | clean: ; $(info Cleaning...) @ ## Cleanup everything 122 | @rm -rf $(BUILDDIR) 123 | @rm -rf $(BINDIR) 124 | @rm -rf test 125 | 126 | .PHONY: generate 127 | generate: generate-mocks ## Run all generate-* targets 128 | 129 | generate-mocks: $(MOCKERY) ## Generate mocks 130 | $(MOCKERY) 131 | 132 | .PHONY: help 133 | help: ## Show this message 134 | @grep -E '^[ a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | \ 135 | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' 136 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](http://www.apache.org/licenses/LICENSE-2.0) 2 | [![Go Report Card](https://goreportcard.com/badge/github.com/k8snetworkplumbingwg/rdma-cni)](https://goreportcard.com/report/github.com/k8snetworkplumbingwg/rdma-cni) 3 | [![Build&Tests](https://github.com/k8snetworkplumbingwg/rdma-cni/actions/workflows/buildtest.yaml/badge.svg?branch=master)](https://github.com/k8snetworkplumbingwg/rdma-cni/actions/workflows/buildtest.yaml) 4 | [![Coverage Status](https://coveralls.io/repos/github/k8snetworkplumbingwg/rdma-cni/badge.svg)](https://coveralls.io/github/k8snetworkplumbingwg/rdma-cni) 5 | 6 | # RDMA CNI plugin 7 | CNI compliant plugin for network namespace aware RDMA interfaces. 8 | 9 | RDMA CNI plugin allows network namespace isolation for RDMA workloads in a containerized environment. 10 | 11 | # Overview 12 | RDMA CNI plugin is intended to be run as a chained CNI plugin (introduced in [CNI Specifications `v0.3.0`](https://github.com/containernetworking/cni/blob/v0.3.0/SPEC.md#network-configuration)). 13 | It ensures isolation of RDMA traffic from other workloads in the system by moving the associated RDMA interfaces of the 14 | provided network interface to the container's network namespace path. 15 | 16 | The main use-case (for now...) is for containerized SR-IOV workloads orchestrated by [Kubernetes](https://kubernetes.io/) 17 | that perform [RDMA](https://community.mellanox.com/s/article/what-is-rdma-x) and wish to leverage network namespace 18 | isolation of RDMA devices introduced in linux kernel `5.3.0`. 19 | 20 | # Requirements 21 | 22 | ## Hardware 23 | SR-IOV capable NIC which supports RDMA. 24 | 25 | ### Supported Hardware 26 | 27 | #### Mellanox Network adapters 28 | ConnectX®-4 and above 29 | 30 | ## Operating System 31 | Linux distribution 32 | 33 | ### Kernel 34 | Kernel based on `5.3.0` or newer, RDMA modules loaded in the system. 35 | [`rdma-core`](https://github.com/linux-rdma/rdma-core) package provides means to automatically load relevant modules 36 | on system start. 37 | 38 | > __*Note:*__ For deployments that use Mellanox out-of-tree driver (Mellanox OFED), Mellanox OFED version `4.7` or newer 39 | >is required. In this case it is not required to use a Kernel based on `5.3.0` or newer. 40 | 41 | ### Pacakges 42 | [`iproute2`](https://mirrors.edge.kernel.org/pub/linux/utils/net/iproute2/) package based on kernel `5.3.0` or newer 43 | installed on the system. 44 | 45 | > __*Note:*__ It is recommended that the required packages are installed by your system's package manager. 46 | 47 | > __*Note:*__ For deployments using Mellanox OFED, `iproute2` package is bundled with the driver under `/opt/mellanox/iproute2/` 48 | 49 | ## Deployment requirements (Kubernetes) 50 | Please refer to the relevant link on how to deploy each component. 51 | For a Kubernetes deployment, each SR-IOV capable worker node should have: 52 | 53 | - [SR-IOV network device plugin](https://github.com/intel/sriov-network-device-plugin) deployed and configured with an [RDMA enabled resource](https://github.com/intel/sriov-network-device-plugin/tree/master/docs/rdma) 54 | - [Multus CNI](https://github.com/intel/multus-cni) `v3.4.1` or newer deployed and configured 55 | - Per fabric SR-IOV supporting CNI deployed 56 | - __Ethernet__: [SR-IOV CNI](https://github.com/intel/sriov-cni) 57 | 58 | > __*Note:*__: Kubernetes version 1.16 or newer is required for deploying as daemonset 59 | 60 | # RDMA CNI configurations 61 | ```json 62 | { 63 | "cniVersion": "0.3.1", 64 | "type": "rdma", 65 | "args": { 66 | "cni": { 67 | "debug": true 68 | } 69 | } 70 | } 71 | ``` 72 | > __*Note:*__ "args" keyword is optional. 73 | 74 | # Deployment 75 | 76 | ## System configuration 77 | It is recommended to set RDMA subsystem namespace awareness mode to `exclusive` on OS boot. 78 | 79 | Set RDMA subsystem namespace awareness mode to `exclusive` via `ib_core` module parameter: 80 | ```console 81 | ~$ echo "options ib_core netns_mode=0" >> /etc/modprobe.d/ib_core.conf 82 | ``` 83 | 84 | Set RDMA subsystem namespace awareness mode to `exclusive` via rdma tool: 85 | ```console 86 | ~$ rdma system set netns exclusive 87 | ``` 88 | 89 | > __*Note:*__ When changing RDMA subsystem netns mode, kernel requires that no network namespaces to exist in the system. 90 | 91 | ## Deploy RDMA CNI 92 | ```bash 93 | $ kubectl apply -f https://raw.githubusercontent.com/k8snetworkplumbingwg/rdma-cni/refs/tags/v1.5.0/deployment/rdma-cni-daemonset.yaml 94 | ``` 95 | 96 | ## Deploy workload 97 | Pod definition can be found in the example below. 98 | ```bash 99 | $ kubectl apply -f https://raw.githubusercontent.com/k8snetworkplumbingwg/rdma-cni/refs/tags/v1.5.0/examples/rdma_test_pod.yaml 100 | ``` 101 | 102 | ## Example resource 103 | 104 | - [Pod](./examples/my_rdma_test_pod.yaml): test workload 105 | - [SR-IOV Network Device Plugin ConfigMap](./examples/sriov_dp_rdma_resource.yaml): defines an RDMA enabled SR-IOV resource pool named: mellanox.com/sriov_rdma 106 | - [Network CRD](./examples/rdma_net_crd.yaml): defines a network, `sriov-network`, associated with an rdma enabled resurce, `mellanox.com/sriov_rdma`. The CNI plugins that will be executed in a chain are for Pods that request this network are: _sriov_, _rdma_ CNIs 107 | 108 | # Development 109 | It is recommended to use the same go version as defined in `.travis.yml` 110 | to avoid potential build related issues during development (newer version will most likely work as well). 111 | 112 | ### Build from source 113 | ```bash 114 | $ git clone https://github.com/k8snetworkplumbingwg/rdma-cni.git 115 | $ cd rdma-cni 116 | $ make 117 | ``` 118 | Upon a successful build, `rdma` binary can be found under `./build`. 119 | For small deployments (e.g a kubernetes test cluster/AIO K8s deployment) you can: 120 | 1. Copy `rdma` binary to the CNI dir in each worker node. 121 | 2. Build container image, push to your own image repo then modify the deployment template and deploy. 122 | 123 | #### Run tests: 124 | ```bash 125 | $ make tests 126 | ``` 127 | 128 | #### Build image: 129 | ```bash 130 | $ make image 131 | ``` 132 | 133 | # Limitations 134 | ## Ethernet 135 | ### RDMA workloads utilizing RDMA Connection Manager (CM) 136 | For Mellanox Hardware, due to kernel limitation, it is required to pre-allocate MACs for all VFs in the deployment 137 | if an RDMA workload wishes to utilize RMDA CM to establish connection. 138 | 139 | This is done in the following manner: 140 | 141 | __Set VF administrative MAC address :__ 142 | 143 | ``` 144 | $ ip link set vf mac 145 | ``` 146 | 147 | __Unbind/Bind VF driver :__ 148 | 149 | ``` 150 | $ echo > /sys/bus/pci/drivers/mlx5_core/unbind 151 | $ echo > /sys/bus/pci/drivers/mlx5_core/bind 152 | ``` 153 | 154 | __Example:__ 155 | ``` 156 | $ ip link set enp4s0f0 vf 3 mac 02:03:00:00:48:56 157 | $ echo 0000:03:00.5 > /sys/bus/pci/drivers/mlx5_core/unbind 158 | $ echo 0000:03:00.5 > /sys/bus/pci/drivers/mlx5_core/bind 159 | ``` 160 | Doing so will populate the VF's node and pord GUID required for RDMA CM to establish connection. 161 | -------------------------------------------------------------------------------- /pkg/cache/mocks/StateCache.go: -------------------------------------------------------------------------------- 1 | // Code generated by mockery; DO NOT EDIT. 2 | // github.com/vektra/mockery 3 | // template: testify 4 | 5 | package mocks 6 | 7 | import ( 8 | "github.com/k8snetworkplumbingwg/rdma-cni/pkg/cache" 9 | mock "github.com/stretchr/testify/mock" 10 | ) 11 | 12 | // NewMockStateCache creates a new instance of MockStateCache. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. 13 | // The first argument is typically a *testing.T value. 14 | func NewMockStateCache(t interface { 15 | mock.TestingT 16 | Cleanup(func()) 17 | }) *MockStateCache { 18 | mock := &MockStateCache{} 19 | mock.Mock.Test(t) 20 | 21 | t.Cleanup(func() { mock.AssertExpectations(t) }) 22 | 23 | return mock 24 | } 25 | 26 | // MockStateCache is an autogenerated mock type for the StateCache type 27 | type MockStateCache struct { 28 | mock.Mock 29 | } 30 | 31 | type MockStateCache_Expecter struct { 32 | mock *mock.Mock 33 | } 34 | 35 | func (_m *MockStateCache) EXPECT() *MockStateCache_Expecter { 36 | return &MockStateCache_Expecter{mock: &_m.Mock} 37 | } 38 | 39 | // Delete provides a mock function for the type MockStateCache 40 | func (_mock *MockStateCache) Delete(ref cache.StateRef) error { 41 | ret := _mock.Called(ref) 42 | 43 | if len(ret) == 0 { 44 | panic("no return value specified for Delete") 45 | } 46 | 47 | var r0 error 48 | if returnFunc, ok := ret.Get(0).(func(cache.StateRef) error); ok { 49 | r0 = returnFunc(ref) 50 | } else { 51 | r0 = ret.Error(0) 52 | } 53 | return r0 54 | } 55 | 56 | // MockStateCache_Delete_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Delete' 57 | type MockStateCache_Delete_Call struct { 58 | *mock.Call 59 | } 60 | 61 | // Delete is a helper method to define mock.On call 62 | // - ref cache.StateRef 63 | func (_e *MockStateCache_Expecter) Delete(ref interface{}) *MockStateCache_Delete_Call { 64 | return &MockStateCache_Delete_Call{Call: _e.mock.On("Delete", ref)} 65 | } 66 | 67 | func (_c *MockStateCache_Delete_Call) Run(run func(ref cache.StateRef)) *MockStateCache_Delete_Call { 68 | _c.Call.Run(func(args mock.Arguments) { 69 | var arg0 cache.StateRef 70 | if args[0] != nil { 71 | arg0 = args[0].(cache.StateRef) 72 | } 73 | run( 74 | arg0, 75 | ) 76 | }) 77 | return _c 78 | } 79 | 80 | func (_c *MockStateCache_Delete_Call) Return(err error) *MockStateCache_Delete_Call { 81 | _c.Call.Return(err) 82 | return _c 83 | } 84 | 85 | func (_c *MockStateCache_Delete_Call) RunAndReturn(run func(ref cache.StateRef) error) *MockStateCache_Delete_Call { 86 | _c.Call.Return(run) 87 | return _c 88 | } 89 | 90 | // GetStateRef provides a mock function for the type MockStateCache 91 | func (_mock *MockStateCache) GetStateRef(network string, cid string, ifname string) cache.StateRef { 92 | ret := _mock.Called(network, cid, ifname) 93 | 94 | if len(ret) == 0 { 95 | panic("no return value specified for GetStateRef") 96 | } 97 | 98 | var r0 cache.StateRef 99 | if returnFunc, ok := ret.Get(0).(func(string, string, string) cache.StateRef); ok { 100 | r0 = returnFunc(network, cid, ifname) 101 | } else { 102 | r0 = ret.Get(0).(cache.StateRef) 103 | } 104 | return r0 105 | } 106 | 107 | // MockStateCache_GetStateRef_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetStateRef' 108 | type MockStateCache_GetStateRef_Call struct { 109 | *mock.Call 110 | } 111 | 112 | // GetStateRef is a helper method to define mock.On call 113 | // - network string 114 | // - cid string 115 | // - ifname string 116 | func (_e *MockStateCache_Expecter) GetStateRef(network interface{}, cid interface{}, ifname interface{}) *MockStateCache_GetStateRef_Call { 117 | return &MockStateCache_GetStateRef_Call{Call: _e.mock.On("GetStateRef", network, cid, ifname)} 118 | } 119 | 120 | func (_c *MockStateCache_GetStateRef_Call) Run(run func(network string, cid string, ifname string)) *MockStateCache_GetStateRef_Call { 121 | _c.Call.Run(func(args mock.Arguments) { 122 | var arg0 string 123 | if args[0] != nil { 124 | arg0 = args[0].(string) 125 | } 126 | var arg1 string 127 | if args[1] != nil { 128 | arg1 = args[1].(string) 129 | } 130 | var arg2 string 131 | if args[2] != nil { 132 | arg2 = args[2].(string) 133 | } 134 | run( 135 | arg0, 136 | arg1, 137 | arg2, 138 | ) 139 | }) 140 | return _c 141 | } 142 | 143 | func (_c *MockStateCache_GetStateRef_Call) Return(stateRef cache.StateRef) *MockStateCache_GetStateRef_Call { 144 | _c.Call.Return(stateRef) 145 | return _c 146 | } 147 | 148 | func (_c *MockStateCache_GetStateRef_Call) RunAndReturn(run func(network string, cid string, ifname string) cache.StateRef) *MockStateCache_GetStateRef_Call { 149 | _c.Call.Return(run) 150 | return _c 151 | } 152 | 153 | // Load provides a mock function for the type MockStateCache 154 | func (_mock *MockStateCache) Load(ref cache.StateRef, state interface{}) error { 155 | ret := _mock.Called(ref, state) 156 | 157 | if len(ret) == 0 { 158 | panic("no return value specified for Load") 159 | } 160 | 161 | var r0 error 162 | if returnFunc, ok := ret.Get(0).(func(cache.StateRef, interface{}) error); ok { 163 | r0 = returnFunc(ref, state) 164 | } else { 165 | r0 = ret.Error(0) 166 | } 167 | return r0 168 | } 169 | 170 | // MockStateCache_Load_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Load' 171 | type MockStateCache_Load_Call struct { 172 | *mock.Call 173 | } 174 | 175 | // Load is a helper method to define mock.On call 176 | // - ref cache.StateRef 177 | // - state interface{} 178 | func (_e *MockStateCache_Expecter) Load(ref interface{}, state interface{}) *MockStateCache_Load_Call { 179 | return &MockStateCache_Load_Call{Call: _e.mock.On("Load", ref, state)} 180 | } 181 | 182 | func (_c *MockStateCache_Load_Call) Run(run func(ref cache.StateRef, state interface{})) *MockStateCache_Load_Call { 183 | _c.Call.Run(func(args mock.Arguments) { 184 | var arg0 cache.StateRef 185 | if args[0] != nil { 186 | arg0 = args[0].(cache.StateRef) 187 | } 188 | var arg1 interface{} 189 | if args[1] != nil { 190 | arg1 = args[1].(interface{}) 191 | } 192 | run( 193 | arg0, 194 | arg1, 195 | ) 196 | }) 197 | return _c 198 | } 199 | 200 | func (_c *MockStateCache_Load_Call) Return(err error) *MockStateCache_Load_Call { 201 | _c.Call.Return(err) 202 | return _c 203 | } 204 | 205 | func (_c *MockStateCache_Load_Call) RunAndReturn(run func(ref cache.StateRef, state interface{}) error) *MockStateCache_Load_Call { 206 | _c.Call.Return(run) 207 | return _c 208 | } 209 | 210 | // Save provides a mock function for the type MockStateCache 211 | func (_mock *MockStateCache) Save(ref cache.StateRef, state interface{}) error { 212 | ret := _mock.Called(ref, state) 213 | 214 | if len(ret) == 0 { 215 | panic("no return value specified for Save") 216 | } 217 | 218 | var r0 error 219 | if returnFunc, ok := ret.Get(0).(func(cache.StateRef, interface{}) error); ok { 220 | r0 = returnFunc(ref, state) 221 | } else { 222 | r0 = ret.Error(0) 223 | } 224 | return r0 225 | } 226 | 227 | // MockStateCache_Save_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Save' 228 | type MockStateCache_Save_Call struct { 229 | *mock.Call 230 | } 231 | 232 | // Save is a helper method to define mock.On call 233 | // - ref cache.StateRef 234 | // - state interface{} 235 | func (_e *MockStateCache_Expecter) Save(ref interface{}, state interface{}) *MockStateCache_Save_Call { 236 | return &MockStateCache_Save_Call{Call: _e.mock.On("Save", ref, state)} 237 | } 238 | 239 | func (_c *MockStateCache_Save_Call) Run(run func(ref cache.StateRef, state interface{})) *MockStateCache_Save_Call { 240 | _c.Call.Run(func(args mock.Arguments) { 241 | var arg0 cache.StateRef 242 | if args[0] != nil { 243 | arg0 = args[0].(cache.StateRef) 244 | } 245 | var arg1 interface{} 246 | if args[1] != nil { 247 | arg1 = args[1].(interface{}) 248 | } 249 | run( 250 | arg0, 251 | arg1, 252 | ) 253 | }) 254 | return _c 255 | } 256 | 257 | func (_c *MockStateCache_Save_Call) Return(err error) *MockStateCache_Save_Call { 258 | _c.Call.Return(err) 259 | return _c 260 | } 261 | 262 | func (_c *MockStateCache_Save_Call) RunAndReturn(run func(ref cache.StateRef, state interface{}) error) *MockStateCache_Save_Call { 263 | _c.Call.Return(run) 264 | return _c 265 | } 266 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= 2 | github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= 3 | github.com/Mellanox/rdmamap v1.1.0 h1:A/W1wAXw+6vm58f3VklrIylgV+eDJlPVIMaIKuxgUT4= 4 | github.com/Mellanox/rdmamap v1.1.0/go.mod h1:fN+/V9lf10ABnDCwTaXRjeeWijLt2iVLETnK+sx/LY8= 5 | github.com/containernetworking/cni v1.3.0 h1:v6EpN8RznAZj9765HhXQrtXgX+ECGebEYEmnuFjskwo= 6 | github.com/containernetworking/cni v1.3.0/go.mod h1:Bs8glZjjFfGPHMw6hQu82RUgEPNGEaBb9KS5KtNMnJ4= 7 | github.com/containernetworking/plugins v1.9.0 h1:Mg3SXBdRGkdXyFC4lcwr6u2ZB2SDeL6LC3U+QrEANuQ= 8 | github.com/containernetworking/plugins v1.9.0/go.mod h1:JG3BxoJifxxHBhG3hFyxyhid7JgRVBu/wtooGEvWf1c= 9 | github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= 10 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 11 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 12 | github.com/gkampitakis/ciinfo v0.3.2 h1:JcuOPk8ZU7nZQjdUhctuhQofk7BGHuIy0c9Ez8BNhXs= 13 | github.com/gkampitakis/ciinfo v0.3.2/go.mod h1:1NIwaOcFChN4fa/B0hEBdAb6npDlFL8Bwx4dfRLRqAo= 14 | github.com/gkampitakis/go-diff v1.3.2 h1:Qyn0J9XJSDTgnsgHRdz9Zp24RaJeKMUHg2+PDZZdC4M= 15 | github.com/gkampitakis/go-diff v1.3.2/go.mod h1:LLgOrpqleQe26cte8s36HTWcTmMEur6OPYerdAAS9tk= 16 | github.com/gkampitakis/go-snaps v0.5.15 h1:amyJrvM1D33cPHwVrjo9jQxX8g/7E2wYdZ+01KS3zGE= 17 | github.com/gkampitakis/go-snaps v0.5.15/go.mod h1:HNpx/9GoKisdhw9AFOBT1N7DBs9DiHo/hGheFGBZ+mc= 18 | github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= 19 | github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= 20 | github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= 21 | github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= 22 | github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw= 23 | github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= 24 | github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= 25 | github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= 26 | github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= 27 | github.com/google/pprof v0.0.0-20250820193118-f64d9cf942d6 h1:EEHtgt9IwisQ2AZ4pIsMjahcegHh6rmhqxzIRQIyepY= 28 | github.com/google/pprof v0.0.0-20250820193118-f64d9cf942d6/go.mod h1:I6V7YzU0XDpsHqbsyrghnFZLO1gwK6NPTNvmetQIk9U= 29 | github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE= 30 | github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung= 31 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 32 | github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= 33 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 34 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 35 | github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= 36 | github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= 37 | github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= 38 | github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= 39 | github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= 40 | github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= 41 | github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= 42 | github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE= 43 | github.com/mfridman/tparse v0.18.0/go.mod h1:gEvqZTuCgEhPbYk/2lS3Kcxg1GmTxxU7kTC8DvP0i/A= 44 | github.com/onsi/ginkgo/v2 v2.27.2 h1:LzwLj0b89qtIy6SSASkzlNvX6WktqurSHwkk2ipF/Ns= 45 | github.com/onsi/ginkgo/v2 v2.27.2/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo= 46 | github.com/onsi/gomega v1.38.3 h1:eTX+W6dobAYfFeGC2PV6RwXRu/MyT+cQguijutvkpSM= 47 | github.com/onsi/gomega v1.38.3/go.mod h1:ZCU1pkQcXDO5Sl9/VVEGlDyp+zm0m1cmeG5TOzLgdh4= 48 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 49 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 50 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 51 | github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= 52 | github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= 53 | github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= 54 | github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY= 55 | github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ= 56 | github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= 57 | github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= 58 | github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= 59 | github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= 60 | github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= 61 | github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= 62 | github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= 63 | github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= 64 | github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= 65 | github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= 66 | github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= 67 | github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= 68 | github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= 69 | github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= 70 | github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= 71 | github.com/vishvananda/netlink v1.3.1 h1:3AEMt62VKqz90r0tmNhog0r/PpWKmrEShJU0wJW6bV0= 72 | github.com/vishvananda/netlink v1.3.1/go.mod h1:ARtKouGSTGchR8aMwmkzC0qiNPrrWO5JS/XMVl45+b4= 73 | github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= 74 | github.com/vishvananda/netns v0.0.5 h1:DfiHV+j8bA32MFM7bfEunvT8IAqQ/NzSJHtcmW5zdEY= 75 | github.com/vishvananda/netns v0.0.5/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= 76 | go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= 77 | go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= 78 | golang.org/x/mod v0.27.0 h1:kb+q2PyFnEADO2IEF935ehFUXlWiNjJWtRNgBLSfbxQ= 79 | golang.org/x/mod v0.27.0/go.mod h1:rWI627Fq0DEoudcK+MBkNkCe0EetEaDSwJJkCcjpazc= 80 | golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= 81 | golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= 82 | golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= 83 | golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= 84 | golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 85 | golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 86 | golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 87 | golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 88 | golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 89 | golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 90 | golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= 91 | golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= 92 | golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= 93 | golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= 94 | golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg= 95 | golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s= 96 | google.golang.org/protobuf v1.36.7 h1:IgrO7UwFQGJdRNXH/sQux4R1Dj1WAKcLElzeeRaXV2A= 97 | google.golang.org/protobuf v1.36.7/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= 98 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 99 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= 100 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 101 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 102 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 103 | -------------------------------------------------------------------------------- /pkg/rdma/mocks/RdmaManager.go: -------------------------------------------------------------------------------- 1 | // Code generated by mockery; DO NOT EDIT. 2 | // github.com/vektra/mockery 3 | // template: testify 4 | 5 | package mocks 6 | 7 | import ( 8 | "github.com/containernetworking/plugins/pkg/ns" 9 | mock "github.com/stretchr/testify/mock" 10 | ) 11 | 12 | // NewMockManager creates a new instance of MockManager. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. 13 | // The first argument is typically a *testing.T value. 14 | func NewMockManager(t interface { 15 | mock.TestingT 16 | Cleanup(func()) 17 | }) *MockManager { 18 | mock := &MockManager{} 19 | mock.Mock.Test(t) 20 | 21 | t.Cleanup(func() { mock.AssertExpectations(t) }) 22 | 23 | return mock 24 | } 25 | 26 | // MockManager is an autogenerated mock type for the Manager type 27 | type MockManager struct { 28 | mock.Mock 29 | } 30 | 31 | type MockManager_Expecter struct { 32 | mock *mock.Mock 33 | } 34 | 35 | func (_m *MockManager) EXPECT() *MockManager_Expecter { 36 | return &MockManager_Expecter{mock: &_m.Mock} 37 | } 38 | 39 | // GetRdmaDevsForAuxDev provides a mock function for the type MockManager 40 | func (_mock *MockManager) GetRdmaDevsForAuxDev(auxDev string) []string { 41 | ret := _mock.Called(auxDev) 42 | 43 | if len(ret) == 0 { 44 | panic("no return value specified for GetRdmaDevsForAuxDev") 45 | } 46 | 47 | var r0 []string 48 | if returnFunc, ok := ret.Get(0).(func(string) []string); ok { 49 | r0 = returnFunc(auxDev) 50 | } else { 51 | if ret.Get(0) != nil { 52 | r0 = ret.Get(0).([]string) 53 | } 54 | } 55 | return r0 56 | } 57 | 58 | // MockManager_GetRdmaDevsForAuxDev_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetRdmaDevsForAuxDev' 59 | type MockManager_GetRdmaDevsForAuxDev_Call struct { 60 | *mock.Call 61 | } 62 | 63 | // GetRdmaDevsForAuxDev is a helper method to define mock.On call 64 | // - auxDev string 65 | func (_e *MockManager_Expecter) GetRdmaDevsForAuxDev(auxDev interface{}) *MockManager_GetRdmaDevsForAuxDev_Call { 66 | return &MockManager_GetRdmaDevsForAuxDev_Call{Call: _e.mock.On("GetRdmaDevsForAuxDev", auxDev)} 67 | } 68 | 69 | func (_c *MockManager_GetRdmaDevsForAuxDev_Call) Run(run func(auxDev string)) *MockManager_GetRdmaDevsForAuxDev_Call { 70 | _c.Call.Run(func(args mock.Arguments) { 71 | var arg0 string 72 | if args[0] != nil { 73 | arg0 = args[0].(string) 74 | } 75 | run( 76 | arg0, 77 | ) 78 | }) 79 | return _c 80 | } 81 | 82 | func (_c *MockManager_GetRdmaDevsForAuxDev_Call) Return(strings []string) *MockManager_GetRdmaDevsForAuxDev_Call { 83 | _c.Call.Return(strings) 84 | return _c 85 | } 86 | 87 | func (_c *MockManager_GetRdmaDevsForAuxDev_Call) RunAndReturn(run func(auxDev string) []string) *MockManager_GetRdmaDevsForAuxDev_Call { 88 | _c.Call.Return(run) 89 | return _c 90 | } 91 | 92 | // GetRdmaDevsForPciDev provides a mock function for the type MockManager 93 | func (_mock *MockManager) GetRdmaDevsForPciDev(pciDev string) []string { 94 | ret := _mock.Called(pciDev) 95 | 96 | if len(ret) == 0 { 97 | panic("no return value specified for GetRdmaDevsForPciDev") 98 | } 99 | 100 | var r0 []string 101 | if returnFunc, ok := ret.Get(0).(func(string) []string); ok { 102 | r0 = returnFunc(pciDev) 103 | } else { 104 | if ret.Get(0) != nil { 105 | r0 = ret.Get(0).([]string) 106 | } 107 | } 108 | return r0 109 | } 110 | 111 | // MockManager_GetRdmaDevsForPciDev_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetRdmaDevsForPciDev' 112 | type MockManager_GetRdmaDevsForPciDev_Call struct { 113 | *mock.Call 114 | } 115 | 116 | // GetRdmaDevsForPciDev is a helper method to define mock.On call 117 | // - pciDev string 118 | func (_e *MockManager_Expecter) GetRdmaDevsForPciDev(pciDev interface{}) *MockManager_GetRdmaDevsForPciDev_Call { 119 | return &MockManager_GetRdmaDevsForPciDev_Call{Call: _e.mock.On("GetRdmaDevsForPciDev", pciDev)} 120 | } 121 | 122 | func (_c *MockManager_GetRdmaDevsForPciDev_Call) Run(run func(pciDev string)) *MockManager_GetRdmaDevsForPciDev_Call { 123 | _c.Call.Run(func(args mock.Arguments) { 124 | var arg0 string 125 | if args[0] != nil { 126 | arg0 = args[0].(string) 127 | } 128 | run( 129 | arg0, 130 | ) 131 | }) 132 | return _c 133 | } 134 | 135 | func (_c *MockManager_GetRdmaDevsForPciDev_Call) Return(strings []string) *MockManager_GetRdmaDevsForPciDev_Call { 136 | _c.Call.Return(strings) 137 | return _c 138 | } 139 | 140 | func (_c *MockManager_GetRdmaDevsForPciDev_Call) RunAndReturn(run func(pciDev string) []string) *MockManager_GetRdmaDevsForPciDev_Call { 141 | _c.Call.Return(run) 142 | return _c 143 | } 144 | 145 | // GetSystemRdmaMode provides a mock function for the type MockManager 146 | func (_mock *MockManager) GetSystemRdmaMode() (string, error) { 147 | ret := _mock.Called() 148 | 149 | if len(ret) == 0 { 150 | panic("no return value specified for GetSystemRdmaMode") 151 | } 152 | 153 | var r0 string 154 | var r1 error 155 | if returnFunc, ok := ret.Get(0).(func() (string, error)); ok { 156 | return returnFunc() 157 | } 158 | if returnFunc, ok := ret.Get(0).(func() string); ok { 159 | r0 = returnFunc() 160 | } else { 161 | r0 = ret.Get(0).(string) 162 | } 163 | if returnFunc, ok := ret.Get(1).(func() error); ok { 164 | r1 = returnFunc() 165 | } else { 166 | r1 = ret.Error(1) 167 | } 168 | return r0, r1 169 | } 170 | 171 | // MockManager_GetSystemRdmaMode_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetSystemRdmaMode' 172 | type MockManager_GetSystemRdmaMode_Call struct { 173 | *mock.Call 174 | } 175 | 176 | // GetSystemRdmaMode is a helper method to define mock.On call 177 | func (_e *MockManager_Expecter) GetSystemRdmaMode() *MockManager_GetSystemRdmaMode_Call { 178 | return &MockManager_GetSystemRdmaMode_Call{Call: _e.mock.On("GetSystemRdmaMode")} 179 | } 180 | 181 | func (_c *MockManager_GetSystemRdmaMode_Call) Run(run func()) *MockManager_GetSystemRdmaMode_Call { 182 | _c.Call.Run(func(args mock.Arguments) { 183 | run() 184 | }) 185 | return _c 186 | } 187 | 188 | func (_c *MockManager_GetSystemRdmaMode_Call) Return(s string, err error) *MockManager_GetSystemRdmaMode_Call { 189 | _c.Call.Return(s, err) 190 | return _c 191 | } 192 | 193 | func (_c *MockManager_GetSystemRdmaMode_Call) RunAndReturn(run func() (string, error)) *MockManager_GetSystemRdmaMode_Call { 194 | _c.Call.Return(run) 195 | return _c 196 | } 197 | 198 | // MoveRdmaDevToNs provides a mock function for the type MockManager 199 | func (_mock *MockManager) MoveRdmaDevToNs(rdmaDev string, netNs ns.NetNS) error { 200 | ret := _mock.Called(rdmaDev, netNs) 201 | 202 | if len(ret) == 0 { 203 | panic("no return value specified for MoveRdmaDevToNs") 204 | } 205 | 206 | var r0 error 207 | if returnFunc, ok := ret.Get(0).(func(string, ns.NetNS) error); ok { 208 | r0 = returnFunc(rdmaDev, netNs) 209 | } else { 210 | r0 = ret.Error(0) 211 | } 212 | return r0 213 | } 214 | 215 | // MockManager_MoveRdmaDevToNs_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'MoveRdmaDevToNs' 216 | type MockManager_MoveRdmaDevToNs_Call struct { 217 | *mock.Call 218 | } 219 | 220 | // MoveRdmaDevToNs is a helper method to define mock.On call 221 | // - rdmaDev string 222 | // - netNs ns.NetNS 223 | func (_e *MockManager_Expecter) MoveRdmaDevToNs(rdmaDev interface{}, netNs interface{}) *MockManager_MoveRdmaDevToNs_Call { 224 | return &MockManager_MoveRdmaDevToNs_Call{Call: _e.mock.On("MoveRdmaDevToNs", rdmaDev, netNs)} 225 | } 226 | 227 | func (_c *MockManager_MoveRdmaDevToNs_Call) Run(run func(rdmaDev string, netNs ns.NetNS)) *MockManager_MoveRdmaDevToNs_Call { 228 | _c.Call.Run(func(args mock.Arguments) { 229 | var arg0 string 230 | if args[0] != nil { 231 | arg0 = args[0].(string) 232 | } 233 | var arg1 ns.NetNS 234 | if args[1] != nil { 235 | arg1 = args[1].(ns.NetNS) 236 | } 237 | run( 238 | arg0, 239 | arg1, 240 | ) 241 | }) 242 | return _c 243 | } 244 | 245 | func (_c *MockManager_MoveRdmaDevToNs_Call) Return(err error) *MockManager_MoveRdmaDevToNs_Call { 246 | _c.Call.Return(err) 247 | return _c 248 | } 249 | 250 | func (_c *MockManager_MoveRdmaDevToNs_Call) RunAndReturn(run func(rdmaDev string, netNs ns.NetNS) error) *MockManager_MoveRdmaDevToNs_Call { 251 | _c.Call.Return(run) 252 | return _c 253 | } 254 | 255 | // SetSystemRdmaMode provides a mock function for the type MockManager 256 | func (_mock *MockManager) SetSystemRdmaMode(mode string) error { 257 | ret := _mock.Called(mode) 258 | 259 | if len(ret) == 0 { 260 | panic("no return value specified for SetSystemRdmaMode") 261 | } 262 | 263 | var r0 error 264 | if returnFunc, ok := ret.Get(0).(func(string) error); ok { 265 | r0 = returnFunc(mode) 266 | } else { 267 | r0 = ret.Error(0) 268 | } 269 | return r0 270 | } 271 | 272 | // MockManager_SetSystemRdmaMode_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SetSystemRdmaMode' 273 | type MockManager_SetSystemRdmaMode_Call struct { 274 | *mock.Call 275 | } 276 | 277 | // SetSystemRdmaMode is a helper method to define mock.On call 278 | // - mode string 279 | func (_e *MockManager_Expecter) SetSystemRdmaMode(mode interface{}) *MockManager_SetSystemRdmaMode_Call { 280 | return &MockManager_SetSystemRdmaMode_Call{Call: _e.mock.On("SetSystemRdmaMode", mode)} 281 | } 282 | 283 | func (_c *MockManager_SetSystemRdmaMode_Call) Run(run func(mode string)) *MockManager_SetSystemRdmaMode_Call { 284 | _c.Call.Run(func(args mock.Arguments) { 285 | var arg0 string 286 | if args[0] != nil { 287 | arg0 = args[0].(string) 288 | } 289 | run( 290 | arg0, 291 | ) 292 | }) 293 | return _c 294 | } 295 | 296 | func (_c *MockManager_SetSystemRdmaMode_Call) Return(err error) *MockManager_SetSystemRdmaMode_Call { 297 | _c.Call.Return(err) 298 | return _c 299 | } 300 | 301 | func (_c *MockManager_SetSystemRdmaMode_Call) RunAndReturn(run func(mode string) error) *MockManager_SetSystemRdmaMode_Call { 302 | _c.Call.Return(run) 303 | return _c 304 | } 305 | -------------------------------------------------------------------------------- /cmd/rdma/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "flag" 7 | "fmt" 8 | "os" 9 | "runtime" 10 | 11 | "github.com/containernetworking/cni/pkg/skel" 12 | "github.com/containernetworking/cni/pkg/types" 13 | current "github.com/containernetworking/cni/pkg/types/100" 14 | cniversion "github.com/containernetworking/cni/pkg/version" 15 | "github.com/containernetworking/plugins/pkg/ns" 16 | "github.com/rs/zerolog" 17 | "github.com/rs/zerolog/log" 18 | 19 | "github.com/k8snetworkplumbingwg/rdma-cni/pkg/cache" 20 | "github.com/k8snetworkplumbingwg/rdma-cni/pkg/rdma" 21 | rdmatypes "github.com/k8snetworkplumbingwg/rdma-cni/pkg/types" 22 | "github.com/k8snetworkplumbingwg/rdma-cni/pkg/utils" 23 | ) 24 | 25 | //nolint:gochecknoinits 26 | func init() { 27 | // this ensures that main runs only on main thread (thread group leader). 28 | // since namespace ops (unshare, setns) are done for a single thread, we 29 | // must ensure that the goroutine does not jump from OS thread to thread 30 | runtime.LockOSThread() 31 | } 32 | 33 | // Sets the initial log level configurations 34 | // this is overridden by the "debug" CNI arg 35 | var ( 36 | logLevel = zerolog.InfoLevel 37 | ) 38 | 39 | var ( 40 | version = "master@git" 41 | commit = "unknown commit" 42 | date = "unknown date" 43 | ) 44 | 45 | type NsManager interface { 46 | GetNS(string) (ns.NetNS, error) 47 | GetCurrentNS() (ns.NetNS, error) 48 | } 49 | 50 | type nsManagerImpl struct { 51 | } 52 | 53 | func (nsm *nsManagerImpl) GetNS(nspath string) (ns.NetNS, error) { 54 | return ns.GetNS(nspath) 55 | } 56 | 57 | func (nsm *nsManagerImpl) GetCurrentNS() (ns.NetNS, error) { 58 | return ns.GetCurrentNS() 59 | } 60 | 61 | func newNsManager() NsManager { 62 | return &nsManagerImpl{} 63 | } 64 | 65 | type rdmaCniPlugin struct { 66 | rdmaManager rdma.Manager 67 | nsManager NsManager 68 | stateCache cache.StateCache 69 | } 70 | 71 | // Ensure RDMA subsystem mode is set to exclusive. 72 | func (plugin *rdmaCniPlugin) ensureRdmaSystemMode() error { 73 | mode, err := plugin.rdmaManager.GetSystemRdmaMode() 74 | if err != nil { 75 | return fmt.Errorf("failed to get RDMA subsystem namespace awareness mode. %v", err) 76 | } 77 | log.Debug().Msgf("RDMA subsystem mode: %s", mode) 78 | if mode != rdma.RdmaSysModeExclusive { 79 | return fmt.Errorf("RDMA subsystem namespace awareness mode is set to %s, "+ 80 | "expecting it to be set to %s, invalid system configurations", mode, rdma.RdmaSysModeExclusive) 81 | } 82 | return nil 83 | } 84 | 85 | func (plugin *rdmaCniPlugin) deriveDeviceIDFromResult(result *current.Result) (string, error) { 86 | log.Warn().Msgf("DeviceID attribute in network configuration is empty, " + 87 | "this may indicated that the delegate plugin is out of date.") 88 | 89 | var deviceID string 90 | var err error 91 | if len(result.Interfaces) == 1 { 92 | log.Debug().Msgf("Attempting to derive DeviceID from MAC.") 93 | deviceID, err = utils.GetVfPciDevFromMAC(result.Interfaces[0].Mac) 94 | if err != nil { 95 | return deviceID, fmt.Errorf("failed to derive PCI device ID from mac %q. %v", result.Interfaces[0].Mac, err) 96 | } 97 | } else { 98 | return deviceID, fmt.Errorf("\"DeviceID\" network configuration attribute is required for rdma CNI") 99 | } 100 | return deviceID, nil 101 | } 102 | 103 | // Parse network configurations 104 | func (plugin *rdmaCniPlugin) parseConf(data []byte, envArgs string) (*rdmatypes.RdmaNetConf, error) { 105 | conf := rdmatypes.RdmaNetConf{} 106 | // Parse CNI args passed as env variables (not used ATM) 107 | if envArgs != "" { 108 | commonCniArgs := &conf.Args.CNI 109 | err := types.LoadArgs(envArgs, commonCniArgs) 110 | if err != nil { 111 | return nil, err 112 | } 113 | log.Debug().Msgf("ENV CNI_ARGS: %+v", commonCniArgs) 114 | } 115 | 116 | if err := json.Unmarshal(data, &conf); err != nil { 117 | return nil, fmt.Errorf("failed to load netconf: %+v", err) 118 | } 119 | log.Debug().Msgf("Network Configuration: %+v", conf) 120 | return &conf, nil 121 | } 122 | 123 | // Move RDMA device to namespace 124 | func (plugin *rdmaCniPlugin) moveRdmaDevToNs(rdmaDev, nsPath string) error { 125 | log.Debug().Msgf("moving RDMA device %s to namespace %s", rdmaDev, nsPath) 126 | 127 | targetNs, err := plugin.nsManager.GetNS(nsPath) 128 | if err != nil { 129 | return fmt.Errorf("failed to open network namespace %s: %v", nsPath, err) 130 | } 131 | defer targetNs.Close() 132 | 133 | err = plugin.rdmaManager.MoveRdmaDevToNs(rdmaDev, targetNs) 134 | if err != nil { 135 | return fmt.Errorf("failed to move RDMA device %s to namespace. %v", rdmaDev, err) 136 | } 137 | return nil 138 | } 139 | 140 | // Move RDMA device from namespace to current (default) namespace 141 | func (plugin *rdmaCniPlugin) moveRdmaDevFromNs(rdmaDev, nsPath string) error { 142 | log.Debug().Msgf("INFO: moving RDMA device %s from namespace %s to default namespace", rdmaDev, nsPath) 143 | 144 | sourceNs, err := plugin.nsManager.GetNS(nsPath) 145 | if err != nil { 146 | return fmt.Errorf("failed to open network namespace %s: %v", nsPath, err) 147 | } 148 | defer sourceNs.Close() 149 | 150 | targetNs, err := plugin.nsManager.GetCurrentNS() 151 | if err != nil { 152 | return fmt.Errorf("failed to open current network namespace: %v", err) 153 | } 154 | defer targetNs.Close() 155 | 156 | err = sourceNs.Do(func(_ ns.NetNS) error { 157 | // Move RDMA device to default namespace 158 | return plugin.rdmaManager.MoveRdmaDevToNs(rdmaDev, targetNs) 159 | }) 160 | if err != nil { 161 | return fmt.Errorf("failed to move RDMA device %s to default namespace. %v", rdmaDev, err) 162 | } 163 | return err 164 | } 165 | 166 | func (plugin *rdmaCniPlugin) CmdAdd(args *skel.CmdArgs) error { 167 | log.Info().Msgf("RDMA-CNI: cmdAdd") 168 | var err error 169 | var conf *rdmatypes.RdmaNetConf 170 | conf, err = plugin.parseConf(args.StdinData, args.Args) 171 | if err != nil { 172 | return err 173 | } 174 | if conf.Args.CNI.Debug { 175 | setDebugMode() 176 | } 177 | 178 | log.Debug().Msgf("cmdAdd: args: %+v ", args) 179 | 180 | // Ensure RDMA-CNI was called as part of a chain, and parse PrevResult 181 | if conf.RawPrevResult == nil { 182 | return fmt.Errorf("RDMA-CNI is expected to be called as part of a plugin chain") 183 | } 184 | err = cniversion.ParsePrevResult(&conf.NetConf) 185 | if err != nil { 186 | return err 187 | } 188 | result, err := current.NewResultFromResult(conf.PrevResult) 189 | if err != nil { 190 | return err 191 | } 192 | log.Debug().Msgf("prev results: %+v", result) 193 | 194 | // Ensure RDMA subsystem mode 195 | err = plugin.ensureRdmaSystemMode() 196 | if err != nil { 197 | return err 198 | } 199 | 200 | // Delegate plugin may not add Device ID to the network configuration, if so, 201 | // attempt to derive it from PrevResult Mac address with some sysfs voodoo 202 | if conf.DeviceID == "" { 203 | if conf.DeviceID, err = plugin.deriveDeviceIDFromResult(result); err != nil { 204 | return err 205 | } 206 | } 207 | 208 | // Move RDMA device to container namespace 209 | rdmaDev, err := plugin.getRDMADevice(conf.DeviceID) 210 | if err != nil { 211 | return fmt.Errorf("failed to get RDMA device for device ID %s: %w", conf.DeviceID, err) 212 | } 213 | 214 | err = plugin.moveRdmaDevToNs(rdmaDev, args.Netns) 215 | if err != nil { 216 | return fmt.Errorf("failed to move RDMA device %s to namespace. %v", rdmaDev, err) 217 | } 218 | 219 | // Save RDMA state 220 | state := rdmatypes.NewRdmaNetState() 221 | state.DeviceID = conf.DeviceID 222 | state.SandboxRdmaDevName = rdmaDev 223 | state.ContainerRdmaDevName = rdmaDev 224 | pRef := plugin.stateCache.GetStateRef(conf.Name, args.ContainerID, args.IfName) 225 | err = plugin.stateCache.Save(pRef, &state) 226 | if err != nil { 227 | // Move RDMA dev back to current namespace 228 | restoreErr := plugin.moveRdmaDevFromNs(state.ContainerRdmaDevName, args.Netns) 229 | if restoreErr != nil { 230 | return fmt.Errorf( 231 | "save to cache failed %v, failed while restoring namespace for RDMA device %s. %v", 232 | err, rdmaDev, restoreErr) 233 | } 234 | return err 235 | } 236 | return types.PrintResult(result, conf.CNIVersion) 237 | } 238 | 239 | func (plugin *rdmaCniPlugin) CmdCheck(args *skel.CmdArgs) error { 240 | log.Info().Msgf("cmdCheck() not Implemented. args: %v ", args) 241 | return nil 242 | } 243 | 244 | func (plugin *rdmaCniPlugin) CmdDel(args *skel.CmdArgs) error { 245 | log.Info().Msgf("RDMA-CNI: cmdDel") 246 | conf, err := plugin.parseConf(args.StdinData, args.Args) 247 | if err != nil { 248 | return err 249 | } 250 | if conf.Args.CNI.Debug { 251 | setDebugMode() 252 | } 253 | log.Debug().Msgf("CmdDel() args: %v ", args) 254 | 255 | // Container already exited, so no Namespace. if no Namespace, we got nothing to clean. 256 | // this may happen in Infra containers as described in https://github.com/kubernetes/kubernetes/pull/35240 257 | if args.Netns == "" { 258 | return nil 259 | } 260 | 261 | // Load RDMA device state from cache 262 | rdmaState := rdmatypes.RdmaNetState{} 263 | pRef := plugin.stateCache.GetStateRef(conf.Name, args.ContainerID, args.IfName) 264 | err = plugin.stateCache.Load(pRef, &rdmaState) 265 | if err != nil { 266 | log.Warn().Msgf("failed to load cache entry(%q). it may have been deleted by a previous CMD_DEL call. %v", pRef, err) 267 | return nil 268 | } 269 | 270 | // Move RDMA device to default namespace 271 | err = plugin.moveRdmaDevFromNs(rdmaState.ContainerRdmaDevName, args.Netns) 272 | if err != nil { 273 | return fmt.Errorf( 274 | "failed to restore RDMA device %s to default namespace. %v", rdmaState.ContainerRdmaDevName, err) 275 | } 276 | 277 | err = plugin.stateCache.Delete(pRef) 278 | if err != nil { 279 | log.Warn().Msgf("failed to delete cache entry(%q). %v", pRef, err) 280 | } 281 | return nil 282 | } 283 | 284 | // getRDMADevice returns the first RDMA device found for the given deviceID. 285 | func (plugin *rdmaCniPlugin) getRDMADevice(deviceID string) (string, error) { 286 | var rdmaDevs []string 287 | if utils.IsPCIAddress(deviceID) { 288 | rdmaDevs = plugin.rdmaManager.GetRdmaDevsForPciDev(deviceID) 289 | if len(rdmaDevs) == 0 { 290 | return "", errors.New("no RDMA devices found") 291 | } 292 | } else { 293 | rdmaDevs = plugin.rdmaManager.GetRdmaDevsForAuxDev(deviceID) 294 | if len(rdmaDevs) == 0 { 295 | return "", errors.New("no RDMA devices found") 296 | } 297 | } 298 | 299 | if len(rdmaDevs) != 1 { 300 | // Expecting exactly one RDMA device 301 | return "", fmt.Errorf( 302 | "discovered more than one RDMA device %v. Unsupported state", rdmaDevs) 303 | } 304 | 305 | return rdmaDevs[0], nil 306 | } 307 | 308 | func setupLogging() { 309 | zerolog.SetGlobalLevel(logLevel) 310 | log.Logger = log.Output(zerolog.ConsoleWriter{ 311 | Out: os.Stderr, 312 | TimeFormat: zerolog.TimeFieldFormat, 313 | NoColor: true}) 314 | } 315 | 316 | func setDebugMode() { 317 | zerolog.SetGlobalLevel(zerolog.DebugLevel) 318 | } 319 | 320 | func printVersionString() string { 321 | return fmt.Sprintf("rdma-cni cni version:%s, commit:%s, date:%s", version, commit, date) 322 | } 323 | 324 | func main() { 325 | // Init command line flags to clear vendor packages' flags, especially in init() 326 | flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError) 327 | 328 | // add version flag 329 | versionOpt := false 330 | flag.BoolVar(&versionOpt, "version", false, "Show application version") 331 | flag.BoolVar(&versionOpt, "v", false, "Show application version") 332 | flag.Parse() 333 | if versionOpt { 334 | fmt.Printf("%s\n", printVersionString()) 335 | return 336 | } 337 | 338 | setupLogging() 339 | plugin := rdmaCniPlugin{ 340 | rdmaManager: rdma.NewRdmaManager(), 341 | nsManager: newNsManager(), 342 | stateCache: cache.NewStateCache(), 343 | } 344 | skel.PluginMainFuncs( 345 | skel.CNIFuncs{ 346 | Add: plugin.CmdAdd, 347 | Check: plugin.CmdCheck, 348 | Del: plugin.CmdDel, 349 | }, 350 | cniversion.All, "") 351 | } 352 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /cmd/rdma/main_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | 7 | "github.com/containernetworking/cni/pkg/skel" 8 | "github.com/containernetworking/cni/pkg/types" 9 | current "github.com/containernetworking/cni/pkg/types/100" 10 | "github.com/containernetworking/plugins/pkg/ns" 11 | . "github.com/onsi/ginkgo/v2" 12 | . "github.com/onsi/gomega" 13 | "github.com/stretchr/testify/mock" 14 | 15 | "github.com/k8snetworkplumbingwg/rdma-cni/pkg/cache" 16 | cacheMocks "github.com/k8snetworkplumbingwg/rdma-cni/pkg/cache/mocks" 17 | "github.com/k8snetworkplumbingwg/rdma-cni/pkg/rdma" 18 | rdmaMocks "github.com/k8snetworkplumbingwg/rdma-cni/pkg/rdma/mocks" 19 | rdmaTypes "github.com/k8snetworkplumbingwg/rdma-cni/pkg/types" 20 | ) 21 | 22 | func generateNetConfCmdDel(netName string) rdmaTypes.RdmaNetConf { 23 | return rdmaTypes.RdmaNetConf{ 24 | NetConf: types.NetConf{ 25 | CNIVersion: "0.4.0", 26 | Name: netName, 27 | Type: "rdma", 28 | Capabilities: nil, 29 | IPAM: types.IPAM{}, 30 | DNS: types.DNS{}, 31 | RawPrevResult: nil, 32 | PrevResult: nil, 33 | }, 34 | DeviceID: "", 35 | } 36 | } 37 | 38 | func generateNetConfCmdAdd(netName, cIfname, deviceID string) rdmaTypes.RdmaNetConf { 39 | prevResult := current.Result{ 40 | CNIVersion: "0.4.0", 41 | Interfaces: []*current.Interface{{ 42 | Name: cIfname, 43 | Mac: "42:86:24:84:4f:b1", 44 | Sandbox: "/proc/1/ns/net", 45 | }}, 46 | IPs: nil, 47 | Routes: nil, 48 | DNS: types.DNS{}, 49 | } 50 | bytes, _ := json.Marshal(prevResult) 51 | var raw map[string]interface{} 52 | _ = json.Unmarshal(bytes, &raw) 53 | 54 | return rdmaTypes.RdmaNetConf{ 55 | NetConf: types.NetConf{ 56 | CNIVersion: "0.4.0", 57 | Name: netName, 58 | Type: "rdma", 59 | Capabilities: nil, 60 | IPAM: types.IPAM{}, 61 | DNS: types.DNS{}, 62 | RawPrevResult: raw, 63 | PrevResult: nil, 64 | }, 65 | DeviceID: deviceID, 66 | } 67 | } 68 | 69 | func generateArgs(nsPath, cid, cIfname string, netconf *rdmaTypes.RdmaNetConf) skel.CmdArgs { 70 | bytes, _ := json.Marshal(*netconf) 71 | return skel.CmdArgs{ 72 | ContainerID: cid, 73 | Netns: nsPath, 74 | IfName: cIfname, 75 | Args: "", 76 | Path: "", 77 | StdinData: bytes, 78 | } 79 | } 80 | 81 | func generateRdmaNetState(deviceID, sanboxRdmaDev, containerRdmaDev string) rdmaTypes.RdmaNetState { 82 | state := rdmaTypes.NewRdmaNetState() 83 | state.DeviceID = deviceID 84 | state.SandboxRdmaDevName = sanboxRdmaDev 85 | state.ContainerRdmaDevName = containerRdmaDev 86 | return state 87 | } 88 | 89 | type dummyNetNs struct { 90 | fd uintptr 91 | path string 92 | } 93 | 94 | func (dns *dummyNetNs) Fd() uintptr { 95 | return dns.fd 96 | } 97 | 98 | func (dns *dummyNetNs) Do(toRun func(ns.NetNS) error) error { 99 | return toRun(&dummyNetNs{fd: 19, path: "dummy/path"}) 100 | } 101 | 102 | func (dns *dummyNetNs) Set() error { 103 | return nil 104 | } 105 | 106 | func (dns *dummyNetNs) Path() string { 107 | return dns.path 108 | } 109 | 110 | func (dns *dummyNetNs) Close() error { 111 | return nil 112 | } 113 | 114 | type dummyNsMananger struct { 115 | } 116 | 117 | func (nsm *dummyNsMananger) GetNS(nspath string) (ns.NetNS, error) { 118 | return &dummyNetNs{path: nspath, fd: 17}, nil 119 | } 120 | 121 | func (nsm *dummyNsMananger) GetCurrentNS() (ns.NetNS, error) { 122 | return &dummyNetNs{path: "/proc/2/ns/net", fd: 17}, nil 123 | } 124 | 125 | var _ = Describe("Main", func() { 126 | var ( 127 | plugin rdmaCniPlugin 128 | dummyNsMgr dummyNsMananger 129 | rdmaMgrMock rdmaMocks.MockManager 130 | stateCacheMock cacheMocks.MockStateCache 131 | t GinkgoTInterface 132 | ) 133 | 134 | JustBeforeEach(func() { 135 | rdmaMgrMock = rdmaMocks.MockManager{} 136 | dummyNsMgr = dummyNsMananger{} 137 | stateCacheMock = cacheMocks.MockStateCache{} 138 | t = GinkgoT() 139 | plugin = rdmaCniPlugin{ 140 | rdmaManager: &rdmaMgrMock, 141 | stateCache: &stateCacheMock, 142 | nsManager: &dummyNsMgr, 143 | } 144 | }) 145 | 146 | Describe("Test ensureRdmaSystemMode()", func() { 147 | 148 | Context("Bad flows", func() { 149 | It("Should error out if rdma system namespace mode is not exclusive", func() { 150 | rdmaMgrMock.On("GetSystemRdmaMode").Return(rdma.RdmaSysModeShared, nil) 151 | err := plugin.ensureRdmaSystemMode() 152 | Expect(err).To(HaveOccurred()) 153 | rdmaMgrMock.AssertExpectations(t) 154 | }) 155 | It("Should error out on failure to get rdma system namespace mode", func() { 156 | retErr := fmt.Errorf("error") 157 | rdmaMgrMock.On("GetSystemRdmaMode").Return("", retErr) 158 | err := plugin.ensureRdmaSystemMode() 159 | Expect(err).To(HaveOccurred()) 160 | rdmaMgrMock.AssertExpectations(t) 161 | }) 162 | }) 163 | Context("Good flow", func() { 164 | It("Should succeed if rdma system namespace mode is exclusive", func() { 165 | rdmaMgrMock.On("GetSystemRdmaMode").Return(rdma.RdmaSysModeExclusive, nil) 166 | Expect(plugin.ensureRdmaSystemMode()).To(Succeed()) 167 | rdmaMgrMock.AssertExpectations(t) 168 | }) 169 | }) 170 | }) 171 | 172 | Describe("Test moveRdmaDevToNs()", func() { 173 | 174 | Context("Good flow", func() { 175 | It("Should succeed and move RDMA device to given namespace", func() { 176 | rdmaDev := "mlx5_5" 177 | nsPath := "/proc/666/ns/net" 178 | containerNs, _ := dummyNsMgr.GetNS(nsPath) 179 | rdmaMgrMock.On("MoveRdmaDevToNs", rdmaDev, containerNs).Return(nil) 180 | Expect(plugin.moveRdmaDevToNs(rdmaDev, nsPath)).To(Succeed()) 181 | rdmaMgrMock.AssertExpectations(t) 182 | }) 183 | }) 184 | Context("Bad flow", func() { 185 | It("Should fail", func() { 186 | retErr := fmt.Errorf("error occurred") 187 | rdmaMgrMock.On( 188 | "MoveRdmaDevToNs", 189 | mock.AnythingOfType("string"), 190 | mock.AnythingOfType("*main.dummyNetNs")).Return(retErr) 191 | err := plugin.moveRdmaDevToNs("mlx5_5", "/proc/666/ns/net") 192 | Expect(err).To(HaveOccurred()) 193 | rdmaMgrMock.AssertExpectations(t) 194 | }) 195 | }) 196 | }) 197 | 198 | Describe("Test moveRdmaDevFromNs()", func() { 199 | 200 | Context("Good flow", func() { 201 | It("Should succeed and move RDMA device to current namespace", func() { 202 | rdmaDev := "mlx5_5" 203 | nsPath := "/proc/666/ns/net" 204 | currNs, _ := dummyNsMgr.GetCurrentNS() 205 | rdmaMgrMock.On("MoveRdmaDevToNs", rdmaDev, currNs).Return(nil) 206 | Expect(plugin.moveRdmaDevFromNs(rdmaDev, nsPath)).To(Succeed()) 207 | rdmaMgrMock.AssertExpectations(t) 208 | }) 209 | }) 210 | Context("Bad flow", func() { 211 | It("Should fail", func() { 212 | retErr := fmt.Errorf("error occurred") 213 | rdmaMgrMock.On("MoveRdmaDevToNs", 214 | mock.AnythingOfType("string"), 215 | mock.AnythingOfType("*main.dummyNetNs")).Return(retErr) 216 | err := plugin.moveRdmaDevFromNs("mlx5_5", "/proc/666/ns/net") 217 | Expect(err).To(HaveOccurred()) 218 | rdmaMgrMock.AssertExpectations(t) 219 | }) 220 | }) 221 | }) 222 | 223 | Describe("Test CmdAdd()", func() { 224 | Context("Valid configuration provided", func() { 225 | It("Should succeed and move Rdma device associated with provided PCI DeviceID to Namespace", func() { 226 | pciDev := "0000:04:00.5" 227 | netName := "rdma-net" 228 | rdmaDev := "mlx5_4" 229 | cIfname := "net1" 230 | cid := "a1b2c3d4e5f6" 231 | cnsPath := "/proc/12444/ns/net" 232 | cns, _ := dummyNsMgr.GetNS(cnsPath) 233 | netconf := generateNetConfCmdAdd(netName, cIfname, pciDev) 234 | args := generateArgs(cnsPath, cid, cIfname, &netconf) 235 | rdmaMgrMock.On("GetSystemRdmaMode").Return(rdma.RdmaSysModeExclusive, nil) 236 | rdmaMgrMock.On("GetRdmaDevsForPciDev", pciDev).Return([]string{rdmaDev}, nil) 237 | rdmaMgrMock.On("MoveRdmaDevToNs", rdmaDev, cns).Return(nil) 238 | stateCacheMock.On("GetStateRef", netName, cid, cIfname).Return(cache.StateRef("some-ref")) 239 | expectedState := generateRdmaNetState(pciDev, rdmaDev, rdmaDev) 240 | stateCacheMock.On("Save", mock.AnythingOfType("cache.StateRef"), &expectedState).Return(nil) 241 | err := plugin.CmdAdd(&args) 242 | Expect(err).ToNot(HaveOccurred()) 243 | rdmaMgrMock.AssertExpectations(t) 244 | stateCacheMock.AssertExpectations(t) 245 | }) 246 | It("Should succeed and move Rdma device associated with auxiliary device DeviceID to Namespace", func() { 247 | auxDev := "mlx5_core.sf.6" 248 | netName := "rdma-net" 249 | rdmaDev := "mlx5_6" 250 | cIfname := "net2" 251 | cid := "a6b5c4d3e2f1" 252 | cnsPath := "/proc/11142/ns/net" 253 | cns, _ := dummyNsMgr.GetNS(cnsPath) 254 | netconf := generateNetConfCmdAdd(netName, cIfname, auxDev) 255 | args := generateArgs(cnsPath, cid, cIfname, &netconf) 256 | rdmaMgrMock.On("GetSystemRdmaMode").Return(rdma.RdmaSysModeExclusive, nil) 257 | rdmaMgrMock.On("GetRdmaDevsForAuxDev", auxDev).Return([]string{rdmaDev}, nil) 258 | rdmaMgrMock.On("MoveRdmaDevToNs", rdmaDev, cns).Return(nil) 259 | stateCacheMock.On("GetStateRef", netName, cid, cIfname).Return(cache.StateRef("some-ref")) 260 | expectedState := generateRdmaNetState(auxDev, rdmaDev, rdmaDev) 261 | stateCacheMock.On("Save", mock.AnythingOfType("cache.StateRef"), &expectedState).Return(nil) 262 | err := plugin.CmdAdd(&args) 263 | Expect(err).ToNot(HaveOccurred()) 264 | rdmaMgrMock.AssertExpectations(t) 265 | stateCacheMock.AssertExpectations(t) 266 | }) 267 | }) 268 | // TODO(adrian): Add additional tests to cover bad flows / differen network configurations 269 | }) 270 | 271 | Describe("Test CmdDel()", func() { 272 | Context("Valid configuration provided", func() { 273 | It("Should succeed and move Rdma device associated with PCI net device back to sandbox namespace", func() { 274 | pciDev := "0000:04:00.5" 275 | netName := "rdma-net" 276 | rdmaDev := "mlx5_4" 277 | cIfname := "net1" 278 | cid := "a1b2c3d4e5f6" 279 | cnsPath := "/proc/12444/ns/net" 280 | cns, _ := dummyNsMgr.GetCurrentNS() 281 | rdmaState := generateRdmaNetState(pciDev, rdmaDev, rdmaDev) 282 | netconf := generateNetConfCmdDel(netName) 283 | args := generateArgs(cnsPath, cid, cIfname, &netconf) 284 | stateCacheMock.On("GetStateRef", netName, cid, cIfname).Return(cache.StateRef("some-ref")) 285 | stateCacheMock.On("Load", mock.AnythingOfType("cache.StateRef"), 286 | mock.AnythingOfType("*types.RdmaNetState")).Return(nil).Run(func(args mock.Arguments) { 287 | arg := args.Get(1).(*rdmaTypes.RdmaNetState) 288 | *arg = rdmaState 289 | }) 290 | rdmaMgrMock.On("MoveRdmaDevToNs", rdmaDev, cns).Return(nil) 291 | stateCacheMock.On("Delete", mock.AnythingOfType("cache.StateRef")).Return(nil) 292 | err := plugin.CmdDel(&args) 293 | Expect(err).ToNot(HaveOccurred()) 294 | rdmaMgrMock.AssertExpectations(t) 295 | stateCacheMock.AssertExpectations(t) 296 | }) 297 | It("Should succeed and move Rdma device associated with auxiliary device back to sandbox namespace", func() { 298 | auxDev := "mlx5_core.sf.6" 299 | netName := "rdma-net" 300 | rdmaDev := "mlx5_6" 301 | cIfname := "net2" 302 | cid := "a1b2c3d4e5f6" 303 | cnsPath := "/proc/12444/ns/net" 304 | cns, _ := dummyNsMgr.GetCurrentNS() 305 | rdmaState := generateRdmaNetState(auxDev, rdmaDev, rdmaDev) 306 | netconf := generateNetConfCmdDel(netName) 307 | args := generateArgs(cnsPath, cid, cIfname, &netconf) 308 | stateCacheMock.On("GetStateRef", netName, cid, cIfname).Return(cache.StateRef("some-ref")) 309 | stateCacheMock.On("Load", mock.AnythingOfType("cache.StateRef"), 310 | mock.AnythingOfType("*types.RdmaNetState")).Return(nil).Run(func(args mock.Arguments) { 311 | arg := args.Get(1).(*rdmaTypes.RdmaNetState) 312 | *arg = rdmaState 313 | }) 314 | rdmaMgrMock.On("MoveRdmaDevToNs", rdmaDev, cns).Return(nil) 315 | stateCacheMock.On("Delete", mock.AnythingOfType("cache.StateRef")).Return(nil) 316 | err := plugin.CmdDel(&args) 317 | Expect(err).ToNot(HaveOccurred()) 318 | rdmaMgrMock.AssertExpectations(t) 319 | stateCacheMock.AssertExpectations(t) 320 | }) 321 | }) 322 | // TODO(adrian): Add additional tests to cover bad flows / different network configurations 323 | }) 324 | 325 | Describe("Test CmdCheck()", func() { 326 | It("Should basically do nothing", func() { 327 | Expect(plugin.CmdCheck(nil)).To(Succeed()) 328 | }) 329 | }) 330 | }) 331 | -------------------------------------------------------------------------------- /pkg/rdma/mocks/RdmaBasicOps.go: -------------------------------------------------------------------------------- 1 | // Code generated by mockery; DO NOT EDIT. 2 | // github.com/vektra/mockery 3 | // template: testify 4 | 5 | package mocks 6 | 7 | import ( 8 | mock "github.com/stretchr/testify/mock" 9 | "github.com/vishvananda/netlink" 10 | ) 11 | 12 | // NewMockBasicOps creates a new instance of MockBasicOps. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. 13 | // The first argument is typically a *testing.T value. 14 | func NewMockBasicOps(t interface { 15 | mock.TestingT 16 | Cleanup(func()) 17 | }) *MockBasicOps { 18 | mock := &MockBasicOps{} 19 | mock.Mock.Test(t) 20 | 21 | t.Cleanup(func() { mock.AssertExpectations(t) }) 22 | 23 | return mock 24 | } 25 | 26 | // MockBasicOps is an autogenerated mock type for the BasicOps type 27 | type MockBasicOps struct { 28 | mock.Mock 29 | } 30 | 31 | type MockBasicOps_Expecter struct { 32 | mock *mock.Mock 33 | } 34 | 35 | func (_m *MockBasicOps) EXPECT() *MockBasicOps_Expecter { 36 | return &MockBasicOps_Expecter{mock: &_m.Mock} 37 | } 38 | 39 | // GetRdmaDevicesForAuxdev provides a mock function for the type MockBasicOps 40 | func (_mock *MockBasicOps) GetRdmaDevicesForAuxdev(auxDev string) []string { 41 | ret := _mock.Called(auxDev) 42 | 43 | if len(ret) == 0 { 44 | panic("no return value specified for GetRdmaDevicesForAuxdev") 45 | } 46 | 47 | var r0 []string 48 | if returnFunc, ok := ret.Get(0).(func(string) []string); ok { 49 | r0 = returnFunc(auxDev) 50 | } else { 51 | if ret.Get(0) != nil { 52 | r0 = ret.Get(0).([]string) 53 | } 54 | } 55 | return r0 56 | } 57 | 58 | // MockBasicOps_GetRdmaDevicesForAuxdev_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetRdmaDevicesForAuxdev' 59 | type MockBasicOps_GetRdmaDevicesForAuxdev_Call struct { 60 | *mock.Call 61 | } 62 | 63 | // GetRdmaDevicesForAuxdev is a helper method to define mock.On call 64 | // - auxDev string 65 | func (_e *MockBasicOps_Expecter) GetRdmaDevicesForAuxdev(auxDev interface{}) *MockBasicOps_GetRdmaDevicesForAuxdev_Call { 66 | return &MockBasicOps_GetRdmaDevicesForAuxdev_Call{Call: _e.mock.On("GetRdmaDevicesForAuxdev", auxDev)} 67 | } 68 | 69 | func (_c *MockBasicOps_GetRdmaDevicesForAuxdev_Call) Run(run func(auxDev string)) *MockBasicOps_GetRdmaDevicesForAuxdev_Call { 70 | _c.Call.Run(func(args mock.Arguments) { 71 | var arg0 string 72 | if args[0] != nil { 73 | arg0 = args[0].(string) 74 | } 75 | run( 76 | arg0, 77 | ) 78 | }) 79 | return _c 80 | } 81 | 82 | func (_c *MockBasicOps_GetRdmaDevicesForAuxdev_Call) Return(strings []string) *MockBasicOps_GetRdmaDevicesForAuxdev_Call { 83 | _c.Call.Return(strings) 84 | return _c 85 | } 86 | 87 | func (_c *MockBasicOps_GetRdmaDevicesForAuxdev_Call) RunAndReturn(run func(auxDev string) []string) *MockBasicOps_GetRdmaDevicesForAuxdev_Call { 88 | _c.Call.Return(run) 89 | return _c 90 | } 91 | 92 | // GetRdmaDevicesForPcidev provides a mock function for the type MockBasicOps 93 | func (_mock *MockBasicOps) GetRdmaDevicesForPcidev(pcidevName string) []string { 94 | ret := _mock.Called(pcidevName) 95 | 96 | if len(ret) == 0 { 97 | panic("no return value specified for GetRdmaDevicesForPcidev") 98 | } 99 | 100 | var r0 []string 101 | if returnFunc, ok := ret.Get(0).(func(string) []string); ok { 102 | r0 = returnFunc(pcidevName) 103 | } else { 104 | if ret.Get(0) != nil { 105 | r0 = ret.Get(0).([]string) 106 | } 107 | } 108 | return r0 109 | } 110 | 111 | // MockBasicOps_GetRdmaDevicesForPcidev_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetRdmaDevicesForPcidev' 112 | type MockBasicOps_GetRdmaDevicesForPcidev_Call struct { 113 | *mock.Call 114 | } 115 | 116 | // GetRdmaDevicesForPcidev is a helper method to define mock.On call 117 | // - pcidevName string 118 | func (_e *MockBasicOps_Expecter) GetRdmaDevicesForPcidev(pcidevName interface{}) *MockBasicOps_GetRdmaDevicesForPcidev_Call { 119 | return &MockBasicOps_GetRdmaDevicesForPcidev_Call{Call: _e.mock.On("GetRdmaDevicesForPcidev", pcidevName)} 120 | } 121 | 122 | func (_c *MockBasicOps_GetRdmaDevicesForPcidev_Call) Run(run func(pcidevName string)) *MockBasicOps_GetRdmaDevicesForPcidev_Call { 123 | _c.Call.Run(func(args mock.Arguments) { 124 | var arg0 string 125 | if args[0] != nil { 126 | arg0 = args[0].(string) 127 | } 128 | run( 129 | arg0, 130 | ) 131 | }) 132 | return _c 133 | } 134 | 135 | func (_c *MockBasicOps_GetRdmaDevicesForPcidev_Call) Return(strings []string) *MockBasicOps_GetRdmaDevicesForPcidev_Call { 136 | _c.Call.Return(strings) 137 | return _c 138 | } 139 | 140 | func (_c *MockBasicOps_GetRdmaDevicesForPcidev_Call) RunAndReturn(run func(pcidevName string) []string) *MockBasicOps_GetRdmaDevicesForPcidev_Call { 141 | _c.Call.Return(run) 142 | return _c 143 | } 144 | 145 | // RdmaLinkByName provides a mock function for the type MockBasicOps 146 | func (_mock *MockBasicOps) RdmaLinkByName(name string) (*netlink.RdmaLink, error) { 147 | ret := _mock.Called(name) 148 | 149 | if len(ret) == 0 { 150 | panic("no return value specified for RdmaLinkByName") 151 | } 152 | 153 | var r0 *netlink.RdmaLink 154 | var r1 error 155 | if returnFunc, ok := ret.Get(0).(func(string) (*netlink.RdmaLink, error)); ok { 156 | return returnFunc(name) 157 | } 158 | if returnFunc, ok := ret.Get(0).(func(string) *netlink.RdmaLink); ok { 159 | r0 = returnFunc(name) 160 | } else { 161 | if ret.Get(0) != nil { 162 | r0 = ret.Get(0).(*netlink.RdmaLink) 163 | } 164 | } 165 | if returnFunc, ok := ret.Get(1).(func(string) error); ok { 166 | r1 = returnFunc(name) 167 | } else { 168 | r1 = ret.Error(1) 169 | } 170 | return r0, r1 171 | } 172 | 173 | // MockBasicOps_RdmaLinkByName_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'RdmaLinkByName' 174 | type MockBasicOps_RdmaLinkByName_Call struct { 175 | *mock.Call 176 | } 177 | 178 | // RdmaLinkByName is a helper method to define mock.On call 179 | // - name string 180 | func (_e *MockBasicOps_Expecter) RdmaLinkByName(name interface{}) *MockBasicOps_RdmaLinkByName_Call { 181 | return &MockBasicOps_RdmaLinkByName_Call{Call: _e.mock.On("RdmaLinkByName", name)} 182 | } 183 | 184 | func (_c *MockBasicOps_RdmaLinkByName_Call) Run(run func(name string)) *MockBasicOps_RdmaLinkByName_Call { 185 | _c.Call.Run(func(args mock.Arguments) { 186 | var arg0 string 187 | if args[0] != nil { 188 | arg0 = args[0].(string) 189 | } 190 | run( 191 | arg0, 192 | ) 193 | }) 194 | return _c 195 | } 196 | 197 | func (_c *MockBasicOps_RdmaLinkByName_Call) Return(rdmaLink *netlink.RdmaLink, err error) *MockBasicOps_RdmaLinkByName_Call { 198 | _c.Call.Return(rdmaLink, err) 199 | return _c 200 | } 201 | 202 | func (_c *MockBasicOps_RdmaLinkByName_Call) RunAndReturn(run func(name string) (*netlink.RdmaLink, error)) *MockBasicOps_RdmaLinkByName_Call { 203 | _c.Call.Return(run) 204 | return _c 205 | } 206 | 207 | // RdmaLinkSetNsFd provides a mock function for the type MockBasicOps 208 | func (_mock *MockBasicOps) RdmaLinkSetNsFd(link *netlink.RdmaLink, fd uint32) error { 209 | ret := _mock.Called(link, fd) 210 | 211 | if len(ret) == 0 { 212 | panic("no return value specified for RdmaLinkSetNsFd") 213 | } 214 | 215 | var r0 error 216 | if returnFunc, ok := ret.Get(0).(func(*netlink.RdmaLink, uint32) error); ok { 217 | r0 = returnFunc(link, fd) 218 | } else { 219 | r0 = ret.Error(0) 220 | } 221 | return r0 222 | } 223 | 224 | // MockBasicOps_RdmaLinkSetNsFd_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'RdmaLinkSetNsFd' 225 | type MockBasicOps_RdmaLinkSetNsFd_Call struct { 226 | *mock.Call 227 | } 228 | 229 | // RdmaLinkSetNsFd is a helper method to define mock.On call 230 | // - link *netlink.RdmaLink 231 | // - fd uint32 232 | func (_e *MockBasicOps_Expecter) RdmaLinkSetNsFd(link interface{}, fd interface{}) *MockBasicOps_RdmaLinkSetNsFd_Call { 233 | return &MockBasicOps_RdmaLinkSetNsFd_Call{Call: _e.mock.On("RdmaLinkSetNsFd", link, fd)} 234 | } 235 | 236 | func (_c *MockBasicOps_RdmaLinkSetNsFd_Call) Run(run func(link *netlink.RdmaLink, fd uint32)) *MockBasicOps_RdmaLinkSetNsFd_Call { 237 | _c.Call.Run(func(args mock.Arguments) { 238 | var arg0 *netlink.RdmaLink 239 | if args[0] != nil { 240 | arg0 = args[0].(*netlink.RdmaLink) 241 | } 242 | var arg1 uint32 243 | if args[1] != nil { 244 | arg1 = args[1].(uint32) 245 | } 246 | run( 247 | arg0, 248 | arg1, 249 | ) 250 | }) 251 | return _c 252 | } 253 | 254 | func (_c *MockBasicOps_RdmaLinkSetNsFd_Call) Return(err error) *MockBasicOps_RdmaLinkSetNsFd_Call { 255 | _c.Call.Return(err) 256 | return _c 257 | } 258 | 259 | func (_c *MockBasicOps_RdmaLinkSetNsFd_Call) RunAndReturn(run func(link *netlink.RdmaLink, fd uint32) error) *MockBasicOps_RdmaLinkSetNsFd_Call { 260 | _c.Call.Return(run) 261 | return _c 262 | } 263 | 264 | // RdmaSystemGetNetnsMode provides a mock function for the type MockBasicOps 265 | func (_mock *MockBasicOps) RdmaSystemGetNetnsMode() (string, error) { 266 | ret := _mock.Called() 267 | 268 | if len(ret) == 0 { 269 | panic("no return value specified for RdmaSystemGetNetnsMode") 270 | } 271 | 272 | var r0 string 273 | var r1 error 274 | if returnFunc, ok := ret.Get(0).(func() (string, error)); ok { 275 | return returnFunc() 276 | } 277 | if returnFunc, ok := ret.Get(0).(func() string); ok { 278 | r0 = returnFunc() 279 | } else { 280 | r0 = ret.Get(0).(string) 281 | } 282 | if returnFunc, ok := ret.Get(1).(func() error); ok { 283 | r1 = returnFunc() 284 | } else { 285 | r1 = ret.Error(1) 286 | } 287 | return r0, r1 288 | } 289 | 290 | // MockBasicOps_RdmaSystemGetNetnsMode_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'RdmaSystemGetNetnsMode' 291 | type MockBasicOps_RdmaSystemGetNetnsMode_Call struct { 292 | *mock.Call 293 | } 294 | 295 | // RdmaSystemGetNetnsMode is a helper method to define mock.On call 296 | func (_e *MockBasicOps_Expecter) RdmaSystemGetNetnsMode() *MockBasicOps_RdmaSystemGetNetnsMode_Call { 297 | return &MockBasicOps_RdmaSystemGetNetnsMode_Call{Call: _e.mock.On("RdmaSystemGetNetnsMode")} 298 | } 299 | 300 | func (_c *MockBasicOps_RdmaSystemGetNetnsMode_Call) Run(run func()) *MockBasicOps_RdmaSystemGetNetnsMode_Call { 301 | _c.Call.Run(func(args mock.Arguments) { 302 | run() 303 | }) 304 | return _c 305 | } 306 | 307 | func (_c *MockBasicOps_RdmaSystemGetNetnsMode_Call) Return(s string, err error) *MockBasicOps_RdmaSystemGetNetnsMode_Call { 308 | _c.Call.Return(s, err) 309 | return _c 310 | } 311 | 312 | func (_c *MockBasicOps_RdmaSystemGetNetnsMode_Call) RunAndReturn(run func() (string, error)) *MockBasicOps_RdmaSystemGetNetnsMode_Call { 313 | _c.Call.Return(run) 314 | return _c 315 | } 316 | 317 | // RdmaSystemSetNetnsMode provides a mock function for the type MockBasicOps 318 | func (_mock *MockBasicOps) RdmaSystemSetNetnsMode(newMode string) error { 319 | ret := _mock.Called(newMode) 320 | 321 | if len(ret) == 0 { 322 | panic("no return value specified for RdmaSystemSetNetnsMode") 323 | } 324 | 325 | var r0 error 326 | if returnFunc, ok := ret.Get(0).(func(string) error); ok { 327 | r0 = returnFunc(newMode) 328 | } else { 329 | r0 = ret.Error(0) 330 | } 331 | return r0 332 | } 333 | 334 | // MockBasicOps_RdmaSystemSetNetnsMode_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'RdmaSystemSetNetnsMode' 335 | type MockBasicOps_RdmaSystemSetNetnsMode_Call struct { 336 | *mock.Call 337 | } 338 | 339 | // RdmaSystemSetNetnsMode is a helper method to define mock.On call 340 | // - newMode string 341 | func (_e *MockBasicOps_Expecter) RdmaSystemSetNetnsMode(newMode interface{}) *MockBasicOps_RdmaSystemSetNetnsMode_Call { 342 | return &MockBasicOps_RdmaSystemSetNetnsMode_Call{Call: _e.mock.On("RdmaSystemSetNetnsMode", newMode)} 343 | } 344 | 345 | func (_c *MockBasicOps_RdmaSystemSetNetnsMode_Call) Run(run func(newMode string)) *MockBasicOps_RdmaSystemSetNetnsMode_Call { 346 | _c.Call.Run(func(args mock.Arguments) { 347 | var arg0 string 348 | if args[0] != nil { 349 | arg0 = args[0].(string) 350 | } 351 | run( 352 | arg0, 353 | ) 354 | }) 355 | return _c 356 | } 357 | 358 | func (_c *MockBasicOps_RdmaSystemSetNetnsMode_Call) Return(err error) *MockBasicOps_RdmaSystemSetNetnsMode_Call { 359 | _c.Call.Return(err) 360 | return _c 361 | } 362 | 363 | func (_c *MockBasicOps_RdmaSystemSetNetnsMode_Call) RunAndReturn(run func(newMode string) error) *MockBasicOps_RdmaSystemSetNetnsMode_Call { 364 | _c.Call.Return(run) 365 | return _c 366 | } 367 | --------------------------------------------------------------------------------