├── .gitignore ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── cmd └── main.go ├── deploy ├── daemonset.yaml └── test-pod.yaml ├── go.mod ├── go.sum └── pkg ├── common └── constant.go ├── device_plugin ├── api.go ├── device_monitor.go ├── register.go └── server.go └── utils └── fswatcher.go /.gitignore: -------------------------------------------------------------------------------- 1 | # If you prefer the allow list template instead of the deny list, see community template: 2 | # https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore 3 | # 4 | # Binaries for programs and plugins 5 | *.exe 6 | *.exe~ 7 | *.dll 8 | *.so 9 | *.dylib 10 | *.idea 11 | # Test binary, built with `go test -c` 12 | *.test 13 | 14 | # Output of the go coverage tool, specifically when used with LiteIDE 15 | *.out 16 | 17 | # Dependency directories (remove the comment below to include it) 18 | # vendor/ 19 | 20 | # Go workspace file 21 | go.work 22 | go.work.sum 23 | dist 24 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.22.5 AS builder 2 | 3 | WORKDIR /app 4 | 5 | # Copy go.mod and go.sum files and download dependencies 6 | COPY go.mod go.sum ./ 7 | ENV GOPROXY=https://goproxy.cn,direct 8 | RUN go mod download 9 | 10 | # Copy the entire project 11 | COPY . . 12 | 13 | # Build the project 14 | RUN CGO_ENABLED=0 GOOS=linux go build -o bin/i-device-plugin cmd/main.go 15 | 16 | FROM alpine:latest 17 | 18 | WORKDIR /root/ 19 | 20 | # Copy the binary from the builder stage 21 | COPY --from=builder /app/bin/i-device-plugin . 22 | 23 | ENTRYPOINT ["./i-device-plugin"] 24 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 意琦行 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | IMG ?= docker.io/lixd96/i-device-plugin:latest 2 | 3 | .PHONY: build 4 | build: 5 | CGO_ENABLED=0 GOOS=linux go build -o bin/i-device-plugin cmd/main.go 6 | 7 | .PHONY:build-image 8 | build-image: 9 | docker build -t ${IMG} . -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # i-device-plugin 2 | k8s device-plugin demo. 3 | 4 | * i-device-plugin 将会新增资源 `lixueduan.com/gopher`。 5 | * 将会扫描获取 `/etc/gophers` 目录下的文件作为对应的设备。 6 | * 将设备分配给 Pod 后，会在 Pod 中新增环境变量`Gopher=$deviceId` 7 | 8 | 9 | 不熟悉 k8s device plugin 的同学可以看下这篇原理分析文章 --> [Kubernetes教程(二一)---自定义资源支持：K8s Device Plugin 从原理到实现](https://www.lixueduan.com/posts/kubernetes/21-device-plugin/) 10 | 11 | ### 微信公众号：探索云原生 12 | 13 | 一个云原生打工人的探索之路，专注云原生，Go，坚持分享最佳实践、经验干货。 14 | 15 | 扫描下面二维码，关注我即时获取更新~ 16 | 17 | ![](https://img.lixueduan.com/about/wechat/qrcode_search.png) 18 | 19 | 20 | ## 构建镜像 21 | ```bash 22 | make build-image 23 | ``` 24 | 25 | ## 部署 26 | 使用 DaemonSet 来部署 `i-device-plugin`，以便其能运行到集群中的所有节点上。 27 | 28 | ```bash 29 | kubectl apply -f deploy/daemonset.yaml 30 | ``` 31 | 检测 Pod 运行情况 32 | 33 | ```bash 34 | [root@test ~]# kubectl -n kube-system get po 35 | i-device-plugin-vnw6z 1/1 Running 0 17s 36 | ``` 37 | 38 | ## 测试 39 | 40 | 新增设备,在该 Demo 中，把 /etc/gophers 目录下的文件作为设备，因此我们只需要到 /etc/gophers 目录下创建文件，模拟有新的设备接入即可。 41 | ```bash 42 | mkdir /etc/gophers 43 | 44 | touch /etc/gophers/g1 45 | ``` 46 | 查看 device plugin pod 日志,可以正常感知到设备 47 | ```bash 48 | I0719 14:01:00.308599 1 device_monitor.go:70] fsnotify device event: /etc/gophers/g1 CREATE 49 | I0719 14:01:00.308986 1 device_monitor.go:79] find new device [g1] 50 | I0719 14:01:00.309017 1 device_monitor.go:70] fsnotify device event: /etc/gophers/g1 CHMOD 51 | I0719 14:01:00.309141 1 api.go:32] device update,new device list [g1] 52 | ``` 53 | 查看 node capacity 信息，能够看到新增的资源 54 | ```bash 55 | [root@test ~]# kubectl get node argo-1 -oyaml|grep capacity -A 7 56 | capacity: 57 | cpu: "4" 58 | ephemeral-storage: 20960236Ki 59 | hugepages-1Gi: "0" 60 | hugepages-2Mi: "0" 61 | lixueduan.com/gopher: "1" 62 | memory: 8154984Ki 63 | pods: "110" 64 | ``` 65 | 66 | 创建 Pod 申请该资源 67 | ```bash 68 | kubectl apply -f deploy/test-pod.yaml 69 | ``` 70 | Pod 启动成功 71 | 72 | ```bash 73 | [root@test ~]# kubectl get po 74 | NAME READY STATUS RESTARTS AGE 75 | gopher-pod 1/1 Running 0 27s 76 | ``` 77 | 78 | 之前分配设备是添加 Gopher=xxx 这个环境变量，现在看下是否正常分配 79 | 80 | ```bash 81 | [root@test ~]# kubectl exec -it gopher-pod -- env|grep Gopher 82 | Gopher=g1 83 | ``` 84 | 85 | ok,环境变量存在，可以看到分配给该 Pod 的设备是 g1。 -------------------------------------------------------------------------------- /cmd/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/lixd/i-device-plugin/pkg/device_plugin" 5 | "github.com/lixd/i-device-plugin/pkg/utils" 6 | "k8s.io/klog/v2" 7 | ) 8 | 9 | func main() { 10 | klog.Infof("device plugin starting") 11 | dp := device_plugin.NewGopherDevicePlugin() 12 | go dp.Run() 13 | 14 | // register when device plugin start 15 | if err := dp.Register(); err != nil { 16 | klog.Fatalf("register to kubelet failed: %v", err) 17 | } 18 | 19 | // watch kubelet.sock,when kubelet restart,exit device plugin,then will restart by DaemonSet 20 | stop := make(chan struct{}) 21 | err := utils.WatchKubelet(stop) 22 | if err != nil { 23 | klog.Fatalf("start to kubelet failed: %v", err) 24 | } 25 | 26 | <-stop 27 | klog.Infof("kubelet restart,exiting") 28 | } 29 | -------------------------------------------------------------------------------- /deploy/daemonset.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: DaemonSet 3 | metadata: 4 | name: i-device-plugin 5 | namespace: kube-system 6 | labels: 7 | app: i-device-plugin 8 | spec: 9 | selector: 10 | matchLabels: 11 | app: i-device-plugin 12 | template: 13 | metadata: 14 | labels: 15 | app: i-device-plugin 16 | spec: 17 | containers: 18 | - name: i-device-plugin 19 | image: docker.io/lixd96/i-device-plugin:latest 20 | imagePullPolicy: IfNotPresent 21 | resources: 22 | limits: 23 | cpu: "1" 24 | memory: "512Mi" 25 | requests: 26 | cpu: "0.1" 27 | memory: "128Mi" 28 | volumeMounts: 29 | - name: device-plugin 30 | mountPath: /var/lib/kubelet/device-plugins 31 | - name: gophers 32 | mountPath: /etc/gophers 33 | volumes: 34 | - name: device-plugin 35 | hostPath: 36 | path: /var/lib/kubelet/device-plugins 37 | - name: gophers 38 | hostPath: 39 | path: /etc/gophers 40 | -------------------------------------------------------------------------------- /deploy/test-pod.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: gopher-pod 5 | spec: 6 | containers: 7 | - name: gopher-container 8 | image: busybox 9 | command: ["sh", "-c", "echo Hello, Kubernetes! && sleep 3600"] 10 | resources: 11 | requests: 12 | lixueduan.com/gopher: "1" 13 | limits: 14 | lixueduan.com/gopher: "1" -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/lixd/i-device-plugin 2 | 3 | go 1.22.5 4 | 5 | require ( 6 | github.com/fsnotify/fsnotify v1.7.0 7 | github.com/pkg/errors v0.9.1 8 | google.golang.org/grpc v1.58.3 9 | k8s.io/klog/v2 v2.130.1 10 | k8s.io/kubelet v0.30.3 11 | ) 12 | 13 | require ( 14 | github.com/go-logr/logr v1.4.1 // indirect 15 | github.com/gogo/protobuf v1.3.2 // indirect 16 | github.com/golang/protobuf v1.5.4 // indirect 17 | golang.org/x/net v0.23.0 // indirect 18 | golang.org/x/sys v0.18.0 // indirect 19 | golang.org/x/text v0.14.0 // indirect 20 | google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect 21 | google.golang.org/protobuf v1.33.0 // indirect 22 | ) 23 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= 2 | github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= 3 | github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= 4 | github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= 5 | github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= 6 | github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= 7 | github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= 8 | github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= 9 | github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= 10 | github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 11 | github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= 12 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 13 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 14 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 15 | github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 16 | github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 17 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 18 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 19 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 20 | golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 21 | golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 22 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 23 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 24 | golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 25 | golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 26 | golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= 27 | golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= 28 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 29 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 30 | golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 31 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 32 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 33 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 34 | golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= 35 | golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 36 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 37 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 38 | golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= 39 | golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= 40 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 41 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 42 | golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= 43 | golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= 44 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 45 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 46 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 47 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 48 | google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d h1:uvYuEyMHKNt+lT4K3bN6fGswmK8qSvcreM3BwjDh+y4= 49 | google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= 50 | google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ= 51 | google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= 52 | google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= 53 | google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= 54 | k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= 55 | k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= 56 | k8s.io/kubelet v0.30.3 h1:KvGWDdhzD0vEyDyGTCjsDc8D+0+lwRMw3fJbfQgF7ys= 57 | k8s.io/kubelet v0.30.3/go.mod h1:D9or45Vkzcqg55CEiqZ8dVbwP3Ksj7DruEVRS9oq3Ys= 58 | -------------------------------------------------------------------------------- /pkg/common/constant.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import "time" 4 | 5 | const ( 6 | ResourceName string = "lixueduan.com/gopher" 7 | DevicePath string = "/etc/gophers" 8 | DeviceSocket string = "gopher.sock" 9 | ConnectTimeout = time.Second * 5 10 | ) 11 | -------------------------------------------------------------------------------- /pkg/device_plugin/api.go: -------------------------------------------------------------------------------- 1 | package device_plugin 2 | 3 | import ( 4 | "context" 5 | "github.com/pkg/errors" 6 | "k8s.io/klog/v2" 7 | pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1" 8 | "strings" 9 | ) 10 | 11 | // GetDevicePluginOptions returns options to be communicated with Device 12 | // Manager 13 | func (c *GopherDevicePlugin) GetDevicePluginOptions(_ context.Context, _ *pluginapi.Empty) (*pluginapi.DevicePluginOptions, error) { 14 | return &pluginapi.DevicePluginOptions{ 15 | PreStartRequired: true, 16 | // 如果需要使用 GetPreferredAllocation，需要指定开启 17 | GetPreferredAllocationAvailable: true, 18 | }, nil 19 | } 20 | 21 | // ListAndWatch returns a stream of List of Devices 22 | // Whenever a Device state change or a Device disappears, ListAndWatch 23 | // returns the new list 24 | func (c *GopherDevicePlugin) ListAndWatch(_ *pluginapi.Empty, srv pluginapi.DevicePlugin_ListAndWatchServer) error { 25 | devs := c.dm.Devices() 26 | klog.Infof("find devices [%s]", String(devs)) 27 | 28 | err := srv.Send(&pluginapi.ListAndWatchResponse{Devices: devs}) 29 | if err != nil { 30 | return errors.WithMessage(err, "send device failed") 31 | } 32 | 33 | klog.Infoln("waiting for device update") 34 | for range c.dm.notify { 35 | devs = c.dm.Devices() 36 | klog.Infof("device update,new device list [%s]", String(devs)) 37 | _ = srv.Send(&pluginapi.ListAndWatchResponse{Devices: devs}) 38 | } 39 | return nil 40 | } 41 | 42 | // GetPreferredAllocation returns a preferred set of devices to allocate 43 | // from a list of available ones. The resulting preferred allocation is not 44 | // guaranteed to be the allocation ultimately performed by the 45 | // devicemanager. It is only designed to help the devicemanager make a more 46 | // informed allocation decision when possible. 47 | func (c *GopherDevicePlugin) GetPreferredAllocation(_ context.Context, _ *pluginapi.PreferredAllocationRequest) (*pluginapi.PreferredAllocationResponse, error) { 48 | klog.Infoln("[GetPreferredAllocation] running") 49 | return &pluginapi.PreferredAllocationResponse{}, nil 50 | } 51 | 52 | // Allocate is called during container creation so that the Device 53 | // Plugin can run device specific operations and instruct Kubelet 54 | // of the steps to make the Device available in the container 55 | func (c *GopherDevicePlugin) Allocate(_ context.Context, reqs *pluginapi.AllocateRequest) (*pluginapi.AllocateResponse, error) { 56 | ret := &pluginapi.AllocateResponse{} 57 | for _, req := range reqs.ContainerRequests { 58 | klog.Infof("[Allocate] received request: %v", strings.Join(req.DevicesIDs, ",")) 59 | resp := pluginapi.ContainerAllocateResponse{ 60 | Envs: map[string]string{ 61 | "Gopher": strings.Join(req.DevicesIDs, ","), 62 | }, 63 | } 64 | ret.ContainerResponses = append(ret.ContainerResponses, &resp) 65 | } 66 | return ret, nil 67 | } 68 | 69 | // PreStartContainer is called, if indicated by Device Plugin during registeration phase, 70 | // before each container start. Device plugin can run device specific operations 71 | // such as reseting the device before making devices available to the container 72 | func (c *GopherDevicePlugin) PreStartContainer(_ context.Context, _ *pluginapi.PreStartContainerRequest) (*pluginapi.PreStartContainerResponse, error) { 73 | return &pluginapi.PreStartContainerResponse{}, nil 74 | } 75 | -------------------------------------------------------------------------------- /pkg/device_plugin/device_monitor.go: -------------------------------------------------------------------------------- 1 | package device_plugin 2 | 3 | import ( 4 | "fmt" 5 | "github.com/fsnotify/fsnotify" 6 | "github.com/pkg/errors" 7 | "io/fs" 8 | "k8s.io/klog/v2" 9 | pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1" 10 | "path" 11 | "path/filepath" 12 | "strings" 13 | ) 14 | 15 | type DeviceMonitor struct { 16 | path string 17 | devices map[string]*pluginapi.Device 18 | notify chan struct{} // notify when device update 19 | } 20 | 21 | func NewDeviceMonitor(path string) *DeviceMonitor { 22 | return &DeviceMonitor{ 23 | path: path, 24 | devices: make(map[string]*pluginapi.Device), 25 | notify: make(chan struct{}), 26 | } 27 | } 28 | 29 | // List all device 30 | func (d *DeviceMonitor) List() error { 31 | err := filepath.Walk(d.path, func(path string, info fs.FileInfo, err error) error { 32 | if info.IsDir() { 33 | klog.Infof("%s is dir,skip", path) 34 | return nil 35 | } 36 | 37 | d.devices[info.Name()] = &pluginapi.Device{ 38 | ID: info.Name(), 39 | Health: pluginapi.Healthy, 40 | } 41 | return nil 42 | }) 43 | 44 | return errors.WithMessagef(err, "walk [%s] failed", d.path) 45 | } 46 | 47 | // Watch device change 48 | func (d *DeviceMonitor) Watch() error { 49 | klog.Infoln("watching devices") 50 | 51 | w, err := fsnotify.NewWatcher() 52 | if err != nil { 53 | return errors.WithMessage(err, "new watcher failed") 54 | } 55 | defer w.Close() 56 | 57 | errChan := make(chan error) 58 | go func() { 59 | defer func() { 60 | if r := recover(); r != nil { 61 | errChan <- fmt.Errorf("device watcher panic:%v", r) 62 | } 63 | }() 64 | for { 65 | select { 66 | case event, ok := <-w.Events: 67 | if !ok { 68 | continue 69 | } 70 | klog.Infof("fsnotify device event: %s %s", event.Name, event.Op.String()) 71 | 72 | if event.Op == fsnotify.Create { 73 | dev := path.Base(event.Name) 74 | d.devices[dev] = &pluginapi.Device{ 75 | ID: dev, 76 | Health: pluginapi.Healthy, 77 | } 78 | d.notify <- struct{}{} 79 | klog.Infof("find new device [%s]", dev) 80 | } else if event.Op&fsnotify.Remove == fsnotify.Remove { 81 | dev := path.Base(event.Name) 82 | delete(d.devices, dev) 83 | d.notify <- struct{}{} 84 | klog.Infof("device [%s] removed", dev) 85 | } 86 | 87 | case err, ok := <-w.Errors: 88 | if !ok { 89 | continue 90 | } 91 | klog.Errorf("fsnotify watch device failed:%v", err) 92 | } 93 | } 94 | }() 95 | 96 | err = w.Add(d.path) 97 | if err != nil { 98 | return fmt.Errorf("watch device error:%v", err) 99 | } 100 | 101 | return <-errChan 102 | } 103 | 104 | // Devices transformer map to slice 105 | func (d *DeviceMonitor) Devices() []*pluginapi.Device { 106 | devices := make([]*pluginapi.Device, 0, len(d.devices)) 107 | for _, device := range d.devices { 108 | devices = append(devices, device) 109 | } 110 | return devices 111 | } 112 | 113 | func String(devs []*pluginapi.Device) string { 114 | ids := make([]string, 0, len(devs)) 115 | for _, device := range devs { 116 | ids = append(ids, device.ID) 117 | } 118 | return strings.Join(ids, ",") 119 | } 120 | -------------------------------------------------------------------------------- /pkg/device_plugin/register.go: -------------------------------------------------------------------------------- 1 | package device_plugin 2 | 3 | import ( 4 | "context" 5 | "github.com/lixd/i-device-plugin/pkg/common" 6 | "github.com/pkg/errors" 7 | pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1" 8 | "path" 9 | ) 10 | 11 | // Register registers the device plugin for the given resourceName with Kubelet. 12 | func (c *GopherDevicePlugin) Register() error { 13 | conn, err := connect(pluginapi.KubeletSocket, common.ConnectTimeout) 14 | if err != nil { 15 | return errors.WithMessagef(err, "connect to %s failed", pluginapi.KubeletSocket) 16 | } 17 | defer conn.Close() 18 | 19 | client := pluginapi.NewRegistrationClient(conn) 20 | reqt := &pluginapi.RegisterRequest{ 21 | Version: pluginapi.Version, 22 | Endpoint: path.Base(common.DeviceSocket), 23 | ResourceName: common.ResourceName, 24 | } 25 | 26 | _, err = client.Register(context.Background(), reqt) 27 | if err != nil { 28 | return errors.WithMessage(err, "register to kubelet failed") 29 | } 30 | return nil 31 | } 32 | -------------------------------------------------------------------------------- /pkg/device_plugin/server.go: -------------------------------------------------------------------------------- 1 | package device_plugin 2 | 3 | import ( 4 | "context" 5 | "github.com/lixd/i-device-plugin/pkg/common" 6 | "github.com/pkg/errors" 7 | "google.golang.org/grpc" 8 | "google.golang.org/grpc/credentials/insecure" 9 | pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1" 10 | "log" 11 | "net" 12 | "os" 13 | "path" 14 | "syscall" 15 | "time" 16 | ) 17 | 18 | type GopherDevicePlugin struct { 19 | server *grpc.Server 20 | stop chan struct{} // this channel signals to stop the device plugin 21 | dm *DeviceMonitor 22 | } 23 | 24 | func NewGopherDevicePlugin() *GopherDevicePlugin { 25 | return &GopherDevicePlugin{ 26 | server: grpc.NewServer(grpc.EmptyServerOption{}), 27 | stop: make(chan struct{}), 28 | dm: NewDeviceMonitor(common.DevicePath), 29 | } 30 | } 31 | 32 | // Run start gRPC server and watcher 33 | func (c *GopherDevicePlugin) Run() error { 34 | err := c.dm.List() 35 | if err != nil { 36 | log.Fatalf("list device error: %v", err) 37 | } 38 | 39 | go func() { 40 | if err = c.dm.Watch(); err != nil { 41 | log.Println("watch devices error") 42 | } 43 | }() 44 | 45 | pluginapi.RegisterDevicePluginServer(c.server, c) 46 | // delete old unix socket before start 47 | socket := path.Join(pluginapi.DevicePluginPath, common.DeviceSocket) 48 | err = syscall.Unlink(socket) 49 | if err != nil && !os.IsNotExist(err) { 50 | return errors.WithMessagef(err, "delete socket %s failed", socket) 51 | } 52 | 53 | sock, err := net.Listen("unix", socket) 54 | if err != nil { 55 | return errors.WithMessagef(err, "listen unix %s failed", socket) 56 | } 57 | 58 | go c.server.Serve(sock) 59 | 60 | // Wait for server to start by launching a blocking connection 61 | conn, err := connect(common.DeviceSocket, 5*time.Second) 62 | if err != nil { 63 | return err 64 | } 65 | conn.Close() 66 | 67 | return nil 68 | } 69 | 70 | // dial establishes the gRPC communication with the registered device plugin. 71 | func connect(socketPath string, timeout time.Duration) (*grpc.ClientConn, error) { 72 | ctx, cancel := context.WithTimeout(context.Background(), timeout) 73 | defer cancel() 74 | c, err := grpc.DialContext(ctx, socketPath, 75 | grpc.WithTransportCredentials(insecure.NewCredentials()), 76 | grpc.WithBlock(), 77 | grpc.WithContextDialer(func(ctx context.Context, addr string) (net.Conn, error) { 78 | if deadline, ok := ctx.Deadline(); ok { 79 | return net.DialTimeout("unix", addr, time.Until(deadline)) 80 | } 81 | return net.DialTimeout("unix", addr, common.ConnectTimeout) 82 | }), 83 | ) 84 | if err != nil { 85 | return nil, err 86 | } 87 | 88 | return c, nil 89 | } 90 | -------------------------------------------------------------------------------- /pkg/utils/fswatcher.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "github.com/fsnotify/fsnotify" 5 | "github.com/pkg/errors" 6 | "k8s.io/klog/v2" 7 | pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1" 8 | ) 9 | 10 | // WatchKubelet restart device plugin when kubelet restarted 11 | func WatchKubelet(stop chan<- struct{}) error { 12 | watcher, err := fsnotify.NewWatcher() 13 | if err != nil { 14 | return errors.WithMessage(err, "Unable to create fsnotify watcher") 15 | } 16 | defer watcher.Close() 17 | 18 | go func() { 19 | // Start listening for events. 20 | for { 21 | select { 22 | case event, ok := <-watcher.Events: 23 | if !ok { 24 | continue 25 | } 26 | klog.Infof("fsnotify events: %s %v", event.Name, event.Op.String()) 27 | if event.Name == pluginapi.KubeletSocket && event.Op == fsnotify.Create { 28 | klog.Warning("inotify: kubelet.sock created, restarting.") 29 | stop <- struct{}{} 30 | } 31 | case err, ok := <-watcher.Errors: 32 | if !ok { 33 | continue 34 | } 35 | klog.Errorf("fsnotify failed restarting,detail:%v", err) 36 | } 37 | } 38 | }() 39 | 40 | // watch kubelet.sock 41 | err = watcher.Add(pluginapi.KubeletSocket) 42 | if err != nil { 43 | return errors.WithMessagef(err, "Unable to add path %s to watcher", pluginapi.KubeletSocket) 44 | } 45 | return nil 46 | } 47 | --------------------------------------------------------------------------------